bulltrackers-module 1.0.474 → 1.0.475

package/functions/task-engine/helpers/social_helpers.js

@@ -0,0 +1,282 @@
+/**
+ * @fileoverview Social task handling for task engine
+ * Handles fetching social posts for instruments, PIs, and signed-in users
+ */
+
+const { FieldValue, FieldPath } = require('@google-cloud/firestore');
+const crypto = require('crypto');
+const { getGcidForUser } = require('../../social-task-handler/helpers/handler_helpers');
+
+const MAX_POSTS_TO_STORE = 30;
+
+/**
+ * Handles social fetch task for instruments, PIs, or signed-in users
+ * @param {object} taskData - Contains { type, id, username, since }
+ * @param {object} config - Task engine configuration
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager
+ * @param {object} batchCounterRef - Optional Firestore reference for batch counter
+ * @returns {Promise<void>}
+ */
+async function handleSocialFetch(taskData, config, dependencies, batchCounterRef = null) {
+    const { db, logger, headerManager, proxyManager } = dependencies;
+    const { type, id, username, since } = taskData;
+    const cid = id;
+    
+    let fetchUrlBase = '';
+    let targetCollectionPath = '';
+    
+    // 1. Target URL Construction (using GCID for Users)
+    try {
+        if (type === 'INSTRUMENT') {
+            const socialApiBaseUrl = config.social?.socialApiBaseUrl || 'https://www.etoro.com/api/edm-streams/v1/feed/instrument/';
+            fetchUrlBase = `${socialApiBaseUrl}${cid}`;
+            const socialInsightsCollection = config.social?.socialInsightsCollectionName || 'social_insights';
+            targetCollectionPath = `${socialInsightsCollection}/${new Date().toISOString().slice(0, 10)}/posts`;
+        } else if (type === 'POPULAR_INVESTOR' || type === 'SIGNED_IN_USER') {
+            // REVERSE LOOKUP: Convert CID to GCID
+            const gcid = await getGcidForUser(dependencies, config.social || {}, cid, username);
+            
+            const userFeedApiUrl = config.social?.userFeedApiUrl || 'https://www.etoro.com/api/edm-streams/v1/feed/user/top/';
+            fetchUrlBase = `${userFeedApiUrl}${gcid}`;
+            targetCollectionPath = (type === 'SIGNED_IN_USER') 
+                ? `${config.social?.signedInUserSocialCollection || 'signed_in_users_social'}/${cid}/posts`
+                : `${config.social?.piSocialCollectionName || 'pi_social_posts'}/${cid}/posts`;
+        }
+    } catch (err) {
+        logger.log('ERROR', `[SocialFetch] Initialization failed for ${type} ${cid}.`, err);
+        return;
+    }
+
+    const taskId = `social-${cid}-${Date.now()}`;
+    logger.log('INFO', `[SocialFetch/${taskId}] Starting fetch for ${type} ${cid}. Limit: ${MAX_POSTS_TO_STORE} posts.`);
+
+    let offset = 0;
+    const take = 10;
+    let totalSaved = 0;
+    let keepFetching = true;
+    const processedPostsCollection = config.social?.processedPostsCollectionName || 'social_processed_registry';
+    const processedRef = db.collection(processedPostsCollection);
+
+    try {
+        while (keepFetching && totalSaved < MAX_POSTS_TO_STORE) {
+            const clientRequestId = crypto.randomUUID();
+            const url = `${fetchUrlBase}?take=${take}&offset=${offset}&reactionsPageSize=20&badgesExperimentIsEnabled=false&client_request_id=${clientRequestId}`;
+            const selectedHeader = await headerManager.selectHeader();
+            
+            const requestHeaders = {
+                'Accept': 'application/json',
+                'Referer': 'https://www.etoro.com/',
+                ...selectedHeader.header
+            };
+
+            logger.log('INFO', `[SocialFetch/${taskId}] Requesting URL: ${url}`);
+
+            let response;
+            try {
+                response = await proxyManager.fetch(url, { headers: requestHeaders });
+                if (!response.ok) throw new Error(`Status ${response.status}`);
+                headerManager.updatePerformance(selectedHeader.id, true);
+            } catch (err) {
+                logger.log('WARN', `[SocialFetch/${taskId}] Fetch failed for ${url}: ${err.message}. Retrying direct.`);
+                try {
+                    const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
+                    response = await directFetch(url, { headers: requestHeaders });
+                    if (!response.ok) throw new Error(`Direct status ${response.status}`);
+                    headerManager.updatePerformance(selectedHeader.id, true);
+                } catch (directErr) {
+                    logger.log('ERROR', `[SocialFetch/${taskId}] All fetch methods failed.`, { error: directErr.message });
+                    headerManager.updatePerformance(selectedHeader.id, false);
+                    break;
+                }
+            }
+
+            const page = await response.json();
+            const discussions = page?.discussions || [];
+            logger.log('INFO', `[SocialFetch/${taskId}] API returned ${discussions.length} items for ${cid} at offset ${offset}.`);
+
+            if (discussions.length === 0) {
+                keepFetching = false;
+                continue;
+            }
+
+            const postIds = discussions.map(d => d.post?.id).filter(Boolean);
+            const existingIds = new Set();
+            if (postIds.length > 0) {
+                const existingDocs = await processedRef.where(FieldPath.documentId(), 'in', postIds).get();
+                existingDocs.forEach(d => existingIds.add(d.id));
+            }
+
+            // Filter and prepare posts: only new posts in English
+            const newPostsWithDiscussion = discussions
+                .filter(d => {
+                    const post = d.post;
+                    if (!post || !post.id || !post.message?.text || existingIds.has(post.id)) return false;
+                    const lang = (post.message.languageCode || 'unknown').toLowerCase();
+                    return lang === 'en' || lang === 'english';
+                })
+                .slice(0, MAX_POSTS_TO_STORE - totalSaved);
+
+            if (newPostsWithDiscussion.length === 0) {
+                keepFetching = false;
+                continue;
+            }
+
+            // Process posts in batches
+            const batch = db.batch();
+            let pageBatchCount = 0;
+
+            // Process first post with AI analysis (if available)
+            if (newPostsWithDiscussion.length > 0 && dependencies.geminiModel) {
+                const discussion = newPostsWithDiscussion[0];
+                const post = discussion.post;
+                const snippet = post.message.text.substring(0, 500);
+                
+                try {
+                    const { getAdvancedAnalysisFromGemini } = require('../../social-task-handler/helpers/handler_helpers');
+                    const aiResult = await getAdvancedAnalysisFromGemini(dependencies, snippet);
+                    
+                    const docData = {
+                        postId: post.id,
+                        text: post.message.text,
+                        ownerId: post.owner?.id,
+                        username: post.owner?.username,
+                        createdAt: post.created,
+                        fetchedAt: FieldValue.serverTimestamp(),
+                        snippet: snippet,
+                        stats: {
+                            likes: discussion.emotionsData?.like?.paging?.totalCount || 0,
+                            comments: discussion.summary?.totalCommentsAndReplies || 0
+                        },
+                        aiAnalysis: aiResult,
+                        tags: post.tags?.map(t => t.market?.symbolName).filter(Boolean) || []
+                    };
+
+                    batch.set(db.collection(targetCollectionPath).doc(post.id), docData);
+                    batch.set(processedRef.doc(post.id), { processedAt: FieldValue.serverTimestamp() });
+                    pageBatchCount++;
+                } catch (aiError) {
+                    logger.log('WARN', `[SocialFetch/${taskId}] AI analysis failed, using default`, aiError);
+                    // Fall through to default processing
+                }
+            }
+
+            // Process remaining posts without AI (or if AI failed)
+            const postsToProcess = newPostsWithDiscussion.slice(pageBatchCount > 0 ? 1 : 0);
+            for (const discussion of postsToProcess) {
+                if (totalSaved + pageBatchCount >= MAX_POSTS_TO_STORE) {
+                    keepFetching = false;
+                    break;
+                }
+
+                const post = discussion.post;
+                const docData = {
+                    postId: post.id,
+                    text: post.message.text,
+                    ownerId: post.owner?.id,
+                    username: post.owner?.username,
+                    createdAt: post.created,
+                    fetchedAt: FieldValue.serverTimestamp(),
+                    snippet: post.message.text.substring(0, 500),
+                    stats: {
+                        likes: discussion.emotionsData?.like?.paging?.totalCount || 0,
+                        comments: discussion.summary?.totalCommentsAndReplies || 0
+                    },
+                    aiAnalysis: {
+                        overallSentiment: "Neutral",
+                        topics: [],
+                        isSpam: false,
+                        qualityScore: 0.5
+                    },
+                    tags: post.tags?.map(t => t.market?.symbolName).filter(Boolean) || []
+                };
+
+                batch.set(db.collection(targetCollectionPath).doc(post.id), docData);
+                batch.set(processedRef.doc(post.id), { processedAt: FieldValue.serverTimestamp() });
+                pageBatchCount++;
+            }
+
+            if (pageBatchCount > 0) {
+                await batch.commit();
+                totalSaved += pageBatchCount;
+            }
+            offset += take;
+        }
+
+        // Write date tracking document after successful fetch
+        if (totalSaved > 0 && (type === 'SIGNED_IN_USER' || type === 'POPULAR_INVESTOR')) {
+            const today = new Date().toISOString().split('T')[0];
+            let trackingCollectionPath;
+            
+            if (type === 'SIGNED_IN_USER') {
+                trackingCollectionPath = config.social?.signedInUserSocialCollection || 'signed_in_users_social';
+            } else {
+                trackingCollectionPath = config.social?.piSocialCollectionName || 'pi_social_posts';
+            }
+            
+            // Write to a single tracking document at the root of the collection
+            const trackingDocRef = db.collection(trackingCollectionPath).doc('_dates');
+            const trackingData = {
+                [`fetchedDates.${today}`]: true,
+                lastUpdated: FieldValue.serverTimestamp()
+            };
+            
+            await trackingDocRef.set(trackingData, { merge: true });
+            logger.log('INFO', `[SocialFetch/${taskId}] Updated date tracking for ${type}: ${today}`);
+        }
+
+        // Decrement batch counter if it exists
+        if (batchCounterRef && (type === 'SIGNED_IN_USER' || type === 'POPULAR_INVESTOR')) {
+            try {
+                await batchCounterRef.update({
+                    remainingTasks: FieldValue.increment(-1),
+                    lastUpdated: FieldValue.serverTimestamp()
+                });
+                
+                // Check if this was the last task
+                const counterDoc = await batchCounterRef.get();
+                if (counterDoc.exists) {
+                    const counterData = counterDoc.data();
+                    if (counterData.remainingTasks <= 0 && !counterData.rootDataIndexed) {
+                        // All tasks complete - trigger root data indexer
+                        const today = new Date().toISOString().split('T')[0];
+                        logger.log('INFO', `[SocialFetch] All social batch tasks complete. Triggering root data indexer for ${today}...`);
+                        
+                        try {
+                            const { runRootDataIndexer } = require('../../root-data-indexer/index');
+                            const rootDataIndexerConfig = config.rootDataIndexer || {};
+                            const indexerConfig = {
+                                ...rootDataIndexerConfig,
+                                targetDate: today
+                            };
+                            
+                            await runRootDataIndexer(indexerConfig, dependencies);
+                            
+                            // Mark as indexed
+                            await batchCounterRef.update({
+                                rootDataIndexed: true,
+                                rootDataIndexedAt: FieldValue.serverTimestamp()
+                            });
+                            
+                            logger.log('SUCCESS', `[SocialFetch] Root data indexer completed for ${today}`);
+                        } catch (indexerError) {
+                            logger.log('ERROR', `[SocialFetch] Failed to run root data indexer for ${today}`, indexerError);
+                        }
+                    }
+                }
+            } catch (counterError) {
+                logger.log('WARN', `[SocialFetch] Failed to update batch counter`, counterError);
+            }
+        }
+
+        logger.log('SUCCESS', `[SocialFetch/${taskId}] Process complete for ${cid}. Total stored: ${totalSaved}/${MAX_POSTS_TO_STORE}`);
+
+    } catch (error) {
+        logger.log('ERROR', `[SocialFetch/${taskId}] Fatal error in processing.`, error);
+        throw error;
+    } finally {
+        await headerManager.flushPerformanceUpdates();
+    }
+}
+
+module.exports = { handleSocialFetch };
+

package/functions/task-engine/utils/task_engine_utils.js

@@ -14,6 +14,7 @@ const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
 const { handleUpdate } = require('../helpers/update_helpers'); 
 const { handlePopularInvestorUpdate, handleOnDemandUserUpdate } = require('../helpers/popular_investor_helpers');
+const { handleSocialFetch } = require('../helpers/social_helpers');
 const pLimit = require('p-limit'); 
 
 /**
@@ -29,33 +30,37 @@ function parseTaskPayload(message, logger) {
 }
 
 /**
- * Sorts tasks into update and other (discover/verify/PI/OnDemand).
+ * Sorts tasks into update, other (discover/verify/PI/OnDemand), and social.
  */
 async function prepareTaskBatches(tasks, batchManager, logger) {
-    const tasksToRun = [], otherTasks = [];
+    const tasksToRun = [], otherTasks = [], socialTasks = [];
     
     for (const task of tasks) {
         if (task.type === 'update') { 
             // Standard portfolio updates (Normal/Speculator ONLY)
             // NOTE: Popular Investors use type 'POPULAR_INVESTOR_UPDATE', not 'update'
             tasksToRun.push(task); 
+        } else if (task.type && task.type.startsWith('SOCIAL_')) {
+            // Social fetch tasks
+            socialTasks.push(task);
         } else {
             // Discover, Verify, Popular Investor (POPULAR_INVESTOR_UPDATE), Signed-In User (ON_DEMAND_USER_UPDATE)
             otherTasks.push(task); 
         }
     }
     
-    logger.log('INFO', `[TaskEngine] Task sorting: ${tasksToRun.length} UPDATE (normal/speculator), ${otherTasks.length} other (PI/signed-in/discover/verify)`);
+    logger.log('INFO', `[TaskEngine] Task sorting: ${tasksToRun.length} UPDATE (normal/speculator), ${otherTasks.length} other (PI/signed-in/discover/verify), ${socialTasks.length} social`);
     
-    return { tasksToRun, cidsToLookup: new Map(), otherTasks };
+    return { tasksToRun, cidsToLookup: new Map(), otherTasks, socialTasks };
 }
 
 /**
  * Executes all tasks.
  * (FIXED: Concurrency limit increased to 5)
+ * (NEW: Added batch counter support for root data indexing)
  */
-async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
-    const { logger, batchManager } = dependencies; 
+async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId, batchCounterRef = null, targetDate = null, socialTasks = [], socialCounterRef = null) {
+    const { logger, batchManager, db } = dependencies; 
     
     // [CRITICAL FIX] Increased from 1 to 5.
     // A limit of 1 was causing timeouts on batches of 500 tasks (500s > 60s/540s timeout).
@@ -130,28 +135,103 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
         allTaskPromises.push(
             limit(() => 
                 handleUpdate(task, subTaskId, dependencies, config) 
-                    .then(() => taskCounters.update++)
+                    .then(async () => {
+                        taskCounters.update++;
+                        
+                        // Decrement batch counter if it exists
+                        if (batchCounterRef) {
+                            try {
+                                await batchCounterRef.update({
+                                    remainingTasks: require('@google-cloud/firestore').FieldValue.increment(-1),
+                                    lastUpdated: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                                });
+                                
+                                // Check if this was the last task
+                                const counterDoc = await batchCounterRef.get();
+                                if (counterDoc.exists) {
+                                    const counterData = counterDoc.data();
+                                    if (counterData.remainingTasks <= 0 && !counterData.rootDataIndexed) {
+                                        // All tasks complete - trigger root data indexer
+                                        logger.log('INFO', `[TaskEngine/${taskId}] All batch tasks complete. Triggering root data indexer for ${targetDate}...`);
+                                        
+                                        try {
+                                            const { runRootDataIndexer } = require('../../root-data-indexer/index');
+                                            const rootDataIndexerConfig = config.rootDataIndexer || {};
+                                            const indexerConfig = {
+                                                ...rootDataIndexerConfig,
+                                                targetDate: targetDate
+                                            };
+                                            
+                                            await runRootDataIndexer(indexerConfig, dependencies);
+                                            
+                                            // Mark as indexed
+                                            await batchCounterRef.update({
+                                                rootDataIndexed: true,
+                                                rootDataIndexedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                                            });
+                                            
+                                            logger.log('SUCCESS', `[TaskEngine/${taskId}] Root data indexer completed for ${targetDate}`);
+                                        } catch (indexerError) {
+                                            logger.log('ERROR', `[TaskEngine/${taskId}] Failed to run root data indexer for ${targetDate}`, indexerError);
+                                        }
+                                    }
+                                }
+                            } catch (counterError) {
+                                logger.log('WARN', `[TaskEngine/${taskId}] Failed to update batch counter`, counterError);
+                            }
+                        }
+                    })
                     .catch(err => {
                         logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message });
                         taskCounters.failed++;
+                        
+                        // Still decrement counter on failure to avoid blocking
+                        if (batchCounterRef) {
+                            batchCounterRef.update({
+                                remainingTasks: require('@google-cloud/firestore').FieldValue.increment(-1)
+                            }).catch(() => {});
+                        }
+                    })
+            )
+        );
+    }
+
+    // 3. Queue social tasks
+    for (const task of socialTasks) {
+        const taskData = task.data || task;
+        const socialType = task.type === 'SOCIAL_INSTRUMENT_FETCH' ? 'INSTRUMENT' :
+                          task.type === 'SOCIAL_PI_FETCH' ? 'POPULAR_INVESTOR' : 'SIGNED_IN_USER';
+        
+        allTaskPromises.push(
+            limit(() => 
+                handleSocialFetch({
+                    type: socialType,
+                    id: taskData.id,
+                    username: taskData.username,
+                    since: taskData.since
+                }, config, dependencies, socialCounterRef)
+                    .then(() => taskCounters.social = (taskCounters.social || 0) + 1)
+                    .catch(err => {
+                        logger.log('ERROR', `[TaskEngine/${taskId}] Error in social fetch for ${taskData.id}`, { errorMessage: err.message });
+                        taskCounters.failed++;
                     })
             )
         );
     }
 
-    // 3. Wait for ALL tasks to complete
+    // 4. Wait for ALL tasks to complete
     await Promise.all(allTaskPromises);
 
-    // 4. Flush any remaining data in the buffer
+    // 5. Flush any remaining data in the buffer
     if (batchManager) {
         logger.log('INFO', `[TaskEngine/${taskId}] Triggering final batch flush...`);
         await batchManager.flushBatches();
     }
 
-    // 5. Log final summary
+    // 6. Log final summary
     logger.log(
         taskCounters.failed > 0 ? 'WARN' : 'SUCCESS', 
-        `[TaskEngine/${taskId}] Processed all tasks. Updates: ${taskCounters.update}, Discovers: ${taskCounters.discover}, Verifies: ${taskCounters.verify}, PI: ${taskCounters.popular_investor}, OnDemand: ${taskCounters.on_demand}, Unknown: ${taskCounters.unknown}, Failed: ${taskCounters.failed}.`
+        `[TaskEngine/${taskId}] Processed all tasks. Updates: ${taskCounters.update}, Discovers: ${taskCounters.discover}, Verifies: ${taskCounters.verify}, PI: ${taskCounters.popular_investor}, OnDemand: ${taskCounters.on_demand}, Social: ${taskCounters.social || 0}, Unknown: ${taskCounters.unknown}, Failed: ${taskCounters.failed}.`
     );
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.474",
+  "version": "1.0.475",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [