bulltrackers-module 1.0.474 → 1.0.476

package/functions/task-engine/handler_creator.js

@@ -7,14 +7,21 @@ const { handleDiscover } = require('./helpers/discover_helpers');
 const { handleVerify } = require('./helpers/verify_helpers');
 const { handleUpdate } = require('./helpers/update_helpers');
 const { handlePopularInvestorUpdate, handleOnDemandUserUpdate } = require('./helpers/popular_investor_helpers');
+const { handleSocialFetch } = require('./helpers/social_helpers');
 
 // IMPORT THE UTILS TO HANDLE BATCHES
 const { executeTasks, prepareTaskBatches } = require('./utils/task_engine_utils'); 
 
 async function handleRequest(message, context, configObj, dependencies) {
-    // Support both old format (single config) and new format (object with taskEngine and rootDataIndexer)
+    // Support both old format (single config) and new format (object with taskEngine, rootDataIndexer, and social)
     const config = configObj.taskEngine || configObj; // Backward compatibility
     const rootDataIndexerConfig = configObj.rootDataIndexer;
+    const socialConfig = configObj.social;
+    
+    // Merge social config into main config for easy access
+    if (socialConfig) {
+        config.social = socialConfig;
+    }
     const { logger, batchManager, db } = dependencies;
     
     // [CRITICAL FIX] Max Age increased to 25m to match the larger dedup window.
@@ -118,10 +125,55 @@ async function handleRequest(message, context, configObj, dependencies) {
         });
         
         const taskId = context.eventId || 'batch-' + Date.now();
+        const today = new Date().toISOString().split('T')[0];
+        const { db } = dependencies;
+        
+        // Initialize counter for batch processing (only for update tasks that need root data indexing)
+        const updateTasksCount = payload.tasks.filter(t => t.type === 'update').length;
+        let batchCounterRef = null;
+        
+        if (updateTasksCount > 0) {
+            try {
+                batchCounterRef = db.collection('task_engine_batch_counters').doc(`${today}-${taskId}`);
+                await batchCounterRef.set({
+                    totalTasks: updateTasksCount,
+                    remainingTasks: updateTasksCount,
+                    startedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    taskId: taskId
+                });
+                logger.log('INFO', `[TaskEngine] Initialized batch counter: ${updateTasksCount} update tasks for ${today}`);
+            } catch (counterError) {
+                logger.log('WARN', `[TaskEngine] Failed to initialize batch counter`, counterError);
+            }
+        }
         
         try {
-            const { tasksToRun, otherTasks } = await prepareTaskBatches(payload.tasks, null, logger);
-            await executeTasks(tasksToRun, otherTasks, dependencies, configObj, taskId);
+            const { tasksToRun, otherTasks, socialTasks } = await prepareTaskBatches(payload.tasks, null, logger);
+            
+            // Initialize social batch counter if we have social tasks
+            let socialCounterRef = null;
+            const userSocialTasksCount = socialTasks.filter(t => 
+                t.type === 'SOCIAL_PI_FETCH' || t.type === 'SOCIAL_SIGNED_IN_USER_FETCH'
+            ).length;
+            
+            if (userSocialTasksCount > 0) {
+                try {
+                    const socialCounterId = `social-${today}-${taskId}`;
+                    socialCounterRef = db.collection('social_batch_counters').doc(socialCounterId);
+                    await socialCounterRef.set({
+                        totalTasks: userSocialTasksCount,
+                        remainingTasks: userSocialTasksCount,
+                        startedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                        taskId: taskId,
+                        date: today
+                    });
+                    logger.log('INFO', `[TaskEngine] Initialized social batch counter: ${userSocialTasksCount} user social tasks for ${today}`);
+                } catch (counterError) {
+                    logger.log('WARN', `[TaskEngine] Failed to initialize social batch counter`, counterError);
+                }
+            }
+            
+            await executeTasks(tasksToRun, otherTasks, dependencies, configObj, taskId, batchCounterRef, today, socialTasks, socialCounterRef);
         } catch (batchError) {
             logger.log('ERROR', `[TaskEngine] Error processing batch. Message will be acknowledged to prevent retry loop.`, {
                 error: batchError.message,
@@ -186,6 +238,24 @@ async function handleRequest(message, context, configObj, dependencies) {
                 }
                 await handleOnDemandUserUpdate(onDemandData, configObj, dependencies);
                 break;
+            case 'SOCIAL_INSTRUMENT_FETCH':
+            case 'SOCIAL_PI_FETCH':
+            case 'SOCIAL_SIGNED_IN_USER_FETCH':
+                const socialData = data || payload;
+                if (!socialData.id) {
+                    logger.log('ERROR', `[TaskEngine] Social fetch task missing required field 'id'`, { data: socialData });
+                    return;
+                }
+                // Map task type to social handler type
+                const socialType = type === 'SOCIAL_INSTRUMENT_FETCH' ? 'INSTRUMENT' :
+                                  type === 'SOCIAL_PI_FETCH' ? 'POPULAR_INVESTOR' : 'SIGNED_IN_USER';
+                await handleSocialFetch({
+                    type: socialType,
+                    id: socialData.id,
+                    username: socialData.username,
+                    since: socialData.since
+                }, config, dependencies);
+                break;
             default:
                 logger.log('WARN', `[TaskEngine] Unknown task type: ${type}`);
         }

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -27,7 +27,9 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
         throw new Error('taskData is required for POPULAR_INVESTOR_UPDATE');
     }
     
-    const { cid, username, requestId, source } = taskData;
+    const { cid, username, requestId, source, metadata } = taskData;
+    // Extract targetCid from metadata if present (for optimization)
+    const targetCid = metadata?.targetCid || cid;
     
     // Validate required fields
     if (!cid && !username) {
@@ -388,39 +390,46 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
                     });
                 }
                 
-                // Trigger computation for PopularInvestorProfileMetrics
+                // Trigger computations with dependency chain resolution
                 const { pubsub } = dependencies;
                 if (pubsub) {
-                    const computationTopic = config.computationSystem?.computationTopicStandard || 'computation-tasks';
-                    const topic = pubsub.topic(computationTopic);
-                    const crypto = require('crypto');
+                    const { triggerComputationWithDependencies } = require('../../computation-system/helpers/on_demand_helpers');
                     
-                    const computationMessage = {
-                        action: 'RUN_COMPUTATION_DATE',
-                        computation: 'PopularInvestorProfileMetrics',
-                        date: today,
-                        pass: '1', // Standard pass
-                        dispatchId: crypto.randomUUID(),
-                        triggerReason: 'on_demand_pi_fetch',
-                        resources: 'standard',
-                        metadata: {
-                            onDemand: true,
-                            requestId: requestId,
-                            piCid: cid,
-                            piUsername: username
-                        },
-                        traceContext: {
-                            traceId: crypto.randomBytes(16).toString('hex'),
-                            spanId: crypto.randomBytes(8).toString('hex'),
-                            sampled: true
-                        }
-                    };
+                    // For Popular Investors, we need:
+                    // 1. PopularInvestorProfileMetrics (main profile)
+                    // 2. SignedInUserPIPersonalizedMetrics (if PI is also a signed-in user)
+                    const computationsToTrigger = ['PopularInvestorProfileMetrics'];
                     
-                    await topic.publishMessage({
-                        data: Buffer.from(JSON.stringify(computationMessage))
-                    });
+                    // Check if this PI is also a signed-in user
+                    const signedInUserRef = db.collection('signed_in_users').doc(String(cid));
+                    const signedInUserDoc = await signedInUserRef.get();
+                    if (signedInUserDoc.exists) {
+                        computationsToTrigger.push('SignedInUserPIPersonalizedMetrics');
+                    }
+                    
+                    const triggeredMessages = [];
+                    for (const computation of computationsToTrigger) {
+                        try {
+                            const messages = await triggerComputationWithDependencies(
+                                computation,
+                                today,
+                                dependencies,
+                                config.computationSystem || {},
+                                {
+                                    triggerReason: 'on_demand_pi_fetch',
+                                    requestId: requestId,
+                                    piCid: cid,
+                                    piUsername: username,
+                                    targetCid: targetCid // Pass targetCid for optimization
+                                }
+                            );
+                            triggeredMessages.push(...messages);
+                        } catch (compError) {
+                            logger.log('ERROR', `[PI Update] Failed to trigger ${computation}`, compError);
+                        }
+                    }
                     
-                    logger.log('INFO', `[PI Update] Triggered computation PopularInvestorProfileMetrics for PI ${cid} (${username}) for date ${today}`);
+                    logger.log('INFO', `[PI Update] Triggered ${triggeredMessages.length} computation messages for PI ${cid} (${username}) for date ${today}`);
                 } else {
                     logger.log('WARN', `[PI Update] PubSub not available, cannot trigger computation`);
                 }
@@ -478,7 +487,15 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
  * Handles the On-Demand update for a Signed-In User.
  */
 async function handleOnDemandUserUpdate(taskData, config, dependencies) {
-    const { cid, username, requestId, source } = taskData;
+    const { cid, username, requestId, source, metadata } = taskData;
+    const data = taskData.data || {}; // Extract data object
+    const includeSocial = data.includeSocial === true; // Check if social should be fetched
+    const since = data.since || new Date(Date.now() - (7 * 24 * 60 * 60 * 1000)).toISOString(); // Default to 7 days
+    const portfolioOnly = data.portfolioOnly === false ? false : true; // Default to true (fetch portfolio)
+    
+    // Extract targetCid from metadata if present (for optimization)
+    const targetCid = metadata?.targetCid || cid;
+    const isNewUser = metadata?.isNewUser === true; // Flag for new user signup
     
     // [FIX] Destructure dependencies first
     const { logger, proxyManager, batchManager, headerManager, db } = dependencies;
@@ -539,63 +556,71 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
     };
 
     let fetchSuccess = false;
+    let portfolioFetched = false;
+    let historyFetched = false;
 
     try {
-        // Portfolio Fetch
-        const portfolioUrl = `${ETORO_API_PORTFOLIO_URL}?cid=${cid}&client_request_id=${uuid}`;
-        logger.log('INFO', `[On-Demand DEBUG] Fetching Portfolio URL: ${portfolioUrl}`);
+        // Portfolio Fetch (only if portfolioOnly is true)
+        if (portfolioOnly) {
+            const portfolioUrl = `${ETORO_API_PORTFOLIO_URL}?cid=${cid}&client_request_id=${uuid}`;
+            logger.log('INFO', `[On-Demand DEBUG] Fetching Portfolio URL: ${portfolioUrl}`);
 
-        let portfolioRes = null;
-        let portfolioSuccess = false;
-        
-        // [CIRCUIT BREAKER] Try proxy only if circuit is not open
-        if (shouldTryProxy()) {
-            try {
-                portfolioRes = await proxyManager.fetch(portfolioUrl, requestOptions);
-                if (portfolioRes.ok) {
-                    portfolioSuccess = true;
-                    recordProxyOutcome(true);
-                } else {
-                    throw new Error(`Proxy returned non-ok status: ${portfolioRes.status}`);
+            let portfolioRes = null;
+            let portfolioSuccess = false;
+            
+            // [CIRCUIT BREAKER] Try proxy only if circuit is not open
+            if (shouldTryProxy()) {
+                try {
+                    portfolioRes = await proxyManager.fetch(portfolioUrl, requestOptions);
+                    if (portfolioRes.ok) {
+                        portfolioSuccess = true;
+                        recordProxyOutcome(true);
+                    } else {
+                        throw new Error(`Proxy returned non-ok status: ${portfolioRes.status}`);
+                    }
+                } catch (proxyError) {
+                    recordProxyOutcome(false);
+                    logger.log('WARN', `[On-Demand] Proxy failed (${getFailureCount()}/${getMaxFailures()}). Error: ${proxyError.message}. Trying direct fetch...`);
                 }
-            } catch (proxyError) {
-                recordProxyOutcome(false);
-                logger.log('WARN', `[On-Demand] Proxy failed (${getFailureCount()}/${getMaxFailures()}). Error: ${proxyError.message}. Trying direct fetch...`);
+            } else {
+                logger.log('INFO', `[On-Demand] Circuit breaker open (${getFailureCount()}/${getMaxFailures()} failures). Skipping proxy, using direct fetch.`);
             }
-        } else {
-            logger.log('INFO', `[On-Demand] Circuit breaker open (${getFailureCount()}/${getMaxFailures()} failures). Skipping proxy, using direct fetch.`);
-        }
-        
-        // Fallback to direct fetch if proxy failed or circuit is open
-        if (!portfolioSuccess) {
-            try {
-                const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
-                portfolioRes = await directFetch(portfolioUrl, requestOptions);
-                if (!portfolioRes.ok) {
-                    throw new Error(`Failed to fetch portfolio for ${cid}: ${portfolioRes.status}`);
+            
+            // Fallback to direct fetch if proxy failed or circuit is open
+            if (!portfolioSuccess) {
+                try {
+                    const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
+                    portfolioRes = await directFetch(portfolioUrl, requestOptions);
+                    if (!portfolioRes.ok) {
+                        throw new Error(`Failed to fetch portfolio for ${cid}: ${portfolioRes.status}`);
+                    }
+                    portfolioSuccess = true;
+                } catch (fetchErr) {
+                    logger.log('ERROR', `[On-Demand] Direct fetch also failed for ${username}`, fetchErr);
+                    throw new Error(`Failed to fetch portfolio for ${cid}. Direct: ${fetchErr.message}`);
                 }
-                portfolioSuccess = true;
-            } catch (fetchErr) {
-                logger.log('ERROR', `[On-Demand] Direct fetch also failed for ${username}`, fetchErr);
-                throw new Error(`Failed to fetch portfolio for ${cid}. Direct: ${fetchErr.message}`);
             }
-        }
 
-        const portfolioData = await portfolioRes.json();
-        portfolioData.lastUpdated = new Date();
-        portfolioData.username = username;
-        
-        fetchSuccess = true;
+            const portfolioData = await portfolioRes.json();
+            portfolioData.lastUpdated = new Date();
+            portfolioData.username = username;
+            
+            fetchSuccess = true;
+            portfolioFetched = true;
 
-        await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'signed_in_user');
+            await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'signed_in_user');
+        } else {
+            logger.log('INFO', `[On-Demand Update] Skipping portfolio fetch (portfolioOnly=false) for ${username}`);
+        }
 
-        // History Fetch - Single Page with Large ItemsPerPage
-        const oneYearAgo = new Date();
-        oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
-        const historyBaseUrl = ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
-        const historyUrl = `${historyBaseUrl}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
+        // History Fetch - Single Page with Large ItemsPerPage (only if portfolioOnly is true)
+        if (portfolioOnly) {
+            const oneYearAgo = new Date();
+            oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
+            const historyBaseUrl = ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
+            const historyUrl = `${historyBaseUrl}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
 
-        logger.log('INFO', `[On-Demand DEBUG] Fetching History URL: ${historyUrl}`);
+            logger.log('INFO', `[On-Demand DEBUG] Fetching History URL: ${historyUrl}`);
 
         let historyRes = null;
         let historySuccess = false;
@@ -644,15 +669,103 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                     p => VALID_REASONS.includes(p.CloseReason)
                 );
             }
-            await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyData, 'signed_in_user');
+                await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyData, 'signed_in_user');
+                historyFetched = true;
+            } else {
+                logger.log('WARN', `[On-Demand Update] History fetch failed for ${username} (${historyRes.status})`);
+            }
         } else {
-            logger.log('WARN', `[On-Demand Update] History fetch failed for ${username} (${historyRes.status})`);
+            logger.log('INFO', `[On-Demand Update] Skipping history fetch (portfolioOnly=false) for ${username}`);
         }
 
-        logger.log('SUCCESS', `[On-Demand Update] Complete for ${username}`);
+        // Fetch social data if requested (for user signup or explicit request)
+        let socialFetched = false;
+        if (includeSocial && username) {
+            try {
+                logger.log('INFO', `[On-Demand Update] Fetching social data for ${username} (${cid})...`);
+                
+                // Import social helper functions
+                const { getGcidForUser } = require('../../social-task-handler/helpers/handler_helpers');
+                const { FieldValue, FieldPath } = require('@google-cloud/firestore');
+                
+                // Get GCID for user
+                const gcid = await getGcidForUser(dependencies, config.social || {}, cid, username);
+                
+                // Fetch social posts
+                const userFeedApiUrl = config.social?.userFeedApiUrl || 'https://www.etoro.com/api/edm-streams/v1/feed/user/top/';
+                const fetchUrl = `${userFeedApiUrl}${gcid}?take=10&offset=0&reactionsPageSize=20&badgesExperimentIsEnabled=false&client_request_id=${uuid}`;
+                
+                const socialResponse = await proxyManager.fetch(fetchUrl, requestOptions);
+                if (socialResponse.ok) {
+                    const socialData = await socialResponse.json();
+                    const discussions = socialData?.discussions || [];
+                    
+                    // Store social posts (simplified version - full logic would be in merged handler)
+                    const signedInUserSocialCollection = config.social?.signedInUserSocialCollection || 'signed_in_users_social';
+                    const processedPostsCollection = config.social?.processedPostsCollectionName || 'social_processed_registry';
+                    const processedRef = db.collection(processedPostsCollection);
+                    
+                    const MAX_POSTS = 30;
+                    let saved = 0;
+                    
+                    for (const discussion of discussions.slice(0, MAX_POSTS)) {
+                        const post = discussion.post;
+                        if (!post || !post.id) continue;
+                        
+                        // Check if already processed
+                        const existing = await processedRef.doc(post.id).get();
+                        if (existing.exists) continue;
+                        
+                        // Store post
+                        const postData = {
+                            postId: post.id,
+                            text: post.message?.text || '',
+                            ownerId: post.owner?.id,
+                            username: post.owner?.username,
+                            createdAt: post.created,
+                            fetchedAt: FieldValue.serverTimestamp(),
+                            snippet: (post.message?.text || '').substring(0, 500),
+                            stats: {
+                                likes: discussion.emotionsData?.like?.paging?.totalCount || 0,
+                                comments: discussion.summary?.totalCommentsAndReplies || 0
+                            },
+                            aiAnalysis: {
+                                overallSentiment: "Neutral",
+                                topics: [],
+                                isSpam: false,
+                                qualityScore: 0.5
+                            },
+                            tags: post.tags?.map(t => t.market?.symbolName).filter(Boolean) || []
+                        };
+                        
+                        await db.collection(signedInUserSocialCollection).doc(String(cid)).collection('posts').doc(post.id).set(postData);
+                        await processedRef.doc(post.id).set({ processedAt: FieldValue.serverTimestamp() });
+                        saved++;
+                    }
+                    
+                    // Update date tracking document
+                    const trackingDocRef = db.collection(signedInUserSocialCollection).doc('_dates');
+                    await trackingDocRef.set({
+                        [`fetchedDates.${today}`]: true,
+                        lastUpdated: FieldValue.serverTimestamp()
+                    }, { merge: true });
+                    
+                    socialFetched = true;
+                    logger.log('INFO', `[On-Demand Update] Fetched and stored ${saved} social posts for ${username}`);
+                }
+            } catch (socialError) {
+                logger.log('WARN', `[On-Demand Update] Social fetch failed for ${username}`, socialError);
+                // Continue - social failure shouldn't block portfolio/history processing
+            }
+        }
         
-        // Update request status and trigger computation if this is a sync request
-        if (requestId && source === 'on_demand_sync' && db) {
+        logger.log('SUCCESS', `[On-Demand Update] Complete for ${username} (Portfolio: ${portfolioFetched ? '✓' : '✗'}, History: ${historyFetched ? '✓' : '✗'}, Social: ${socialFetched ? '✓' : '✗'})`);
+        
+        // Update root data indexer after all data is fetched
+        // Then trigger computations for both sync requests AND user signups
+        const shouldTriggerComputations = (requestId && (source === 'on_demand_sync' || source === 'user_signup')) || isNewUser;
+        
+        if (shouldTriggerComputations && db) {
             try {
                 const requestRef = db.collection('user_sync_requests')
                     .doc(String(cid))
@@ -693,39 +806,44 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                     });
                 }
                 
-                // Trigger computation for SignedInUserProfileMetrics
+                // Trigger computations with dependency chain resolution
                 const { pubsub } = dependencies;
                 if (pubsub) {
-                    const computationTopic = config.computationSystem?.computationTopicStandard || 'computation-tasks';
-                    const topic = pubsub.topic(computationTopic);
-                    const crypto = require('crypto');
+                    const { triggerComputationWithDependencies } = require('../../computation-system/helpers/on_demand_helpers');
                     
-                    const computationMessage = {
-                        action: 'RUN_COMPUTATION_DATE',
-                        computation: 'SignedInUserProfileMetrics',
-                        date: today,
-                        pass: '1', // Standard pass
-                        dispatchId: crypto.randomUUID(),
-                        triggerReason: 'on_demand_sync',
-                        resources: 'standard',
-                        metadata: {
-                            onDemand: true,
-                            requestId: requestId,
-                            userCid: cid,
-                            username: username
-                        },
-                        traceContext: {
-                            traceId: crypto.randomBytes(16).toString('hex'),
-                            spanId: crypto.randomBytes(8).toString('hex'),
-                            sampled: true
-                        }
-                    };
+                    // For signed-in users, we need to trigger multiple computations:
+                    // 1. SignedInUserProfileMetrics (main profile)
+                    // 2. SignedInUserCopiedList, SignedInUserCopiedPIs, SignedInUserPastCopies (copied data)
+                    const computationsToTrigger = [
+                        'SignedInUserProfileMetrics',
+                        'SignedInUserCopiedList',
+                        'SignedInUserCopiedPIs',
+                        'SignedInUserPastCopies'
+                    ];
                     
-                    await topic.publishMessage({
-                        data: Buffer.from(JSON.stringify(computationMessage))
-                    });
+                    const triggeredMessages = [];
+                    for (const computation of computationsToTrigger) {
+                        try {
+                            const messages = await triggerComputationWithDependencies(
+                                computation,
+                                today,
+                                dependencies,
+                                config.computationSystem || {},
+                                {
+                                    triggerReason: source === 'user_signup' ? 'user_signup' : 'on_demand_sync',
+                                    requestId: requestId,
+                                    userCid: cid,
+                                    username: username,
+                                    targetCid: targetCid // Pass targetCid for optimization
+                                }
+                            );
+                            triggeredMessages.push(...messages);
+                        } catch (compError) {
+                            logger.log('ERROR', `[On-Demand Update] Failed to trigger ${computation}`, compError);
+                        }
+                    }
                     
-                    logger.log('INFO', `[On-Demand Update] Triggered computation SignedInUserProfileMetrics for user ${cid} (${username}) for date ${today}`);
+                    logger.log('INFO', `[On-Demand Update] Triggered ${triggeredMessages.length} computation messages for user ${cid} (${username}) for date ${today}`);
                     
                     // Don't mark as completed yet - wait for computation to finish
                     // The status will remain 'computing' until computation completes