bulltrackers-module 1.0.152 → 1.0.153

package/functions/social-task-handler/helpers/handler_helpers.js

@@ -15,75 +15,21 @@ const { FieldValue, FieldPath } = require('@google-cloud/firestore');
  */
 async function getSentimentFromGemini(dependencies, snippet) {
     const { logger, geminiModel } = dependencies;
-
-    if (!geminiModel) {
-        logger.log('WARN', '[getSentimentFromGemini] Gemini model not found in dependencies.');
-        return { overallSentiment: "Neutral", topics: [] }; // Return default object
-    }
-
-    // --- NEW PROMPT ---
-    const prompt = `You are a financial analyst. Analyze the following social media post.
-Your task is to provide:
-1.  The overall sentiment (Bullish, Bearish, or Neutral) toward the main asset. Return only one word.
-2.  A list of key topics or events mentioned (e.g., "FOMC", "CPI", "Earnings", "Inflation", "War", "Acquisition"). If no specific event is mentioned, return an empty array.
-
-Return ONLY a valid JSON object in this exact format:
-{
-  "overallSentiment": "...",
-  "topics": ["...", "..."]
-}
-
-Post: "${snippet}"`
-    // --- END NEW PROMPT ---
-
-    try {
-        const request = {
-            contents: [
-                {
-                    role: "user",
-                    parts: [ { text: prompt } ]
-                }
-            ],
-            generationConfig: {
-                temperature: 0.1,
-                topP: 0.1,
-                maxOutputTokens: 256 // Increased tokens for JSON
-            }
-        };
-
+    if (!geminiModel) { logger.log('WARN', '[getSentimentFromGemini] Gemini model not found in dependencies.'); return { overallSentiment: "Neutral", topics: [] };  }
+    const prompt = `You are a financial analyst. Analyze the following social media post. Your task is to provide: 1.  The overall sentiment (Bullish, Bearish, or Neutral) toward the main asset. Return only one word. 2.  A list of key topics or events mentioned (e.g., "FOMC", "CPI", "Earnings", "Inflation", "War", "Acquisition"). If no specific event is mentioned, return an empty array. Return ONLY a valid JSON object in this exact format: { "overallSentiment": "...", "topics": ["...", "..."] } Post: "${snippet}"`
+    try { const request = { contents: [ { role: "user", parts: [ { text: prompt } ] } ], generationConfig: { temperature: 0.1, topP: 0.1, maxOutputTokens: 256  } };
         const result = await geminiModel.generateContent(request);
         const response = result.response;
-        
         const text = response?.candidates?.[0]?.content?.parts?.[0]?.text?.trim() || '';
-
-        // --- NEW: Parse JSON response ---
         try {
-            // Find the JSON block
             const jsonMatch = text.match(/\{[\s\S]*\}/);
             if (jsonMatch && jsonMatch[0]) {
                 const parsedJson = JSON.parse(jsonMatch[0]);
-                // Validate structure
-                if (parsedJson && parsedJson.overallSentiment && Array.isArray(parsedJson.topics)) {
-                    logger.log('INFO', `[getSentimentFromGemini] Classified sentiment: ${parsedJson.overallSentiment}, Topics: ${parsedJson.topics.join(', ')}`);
-                    return parsedJson; // Return the full object
-                }
-            }
+                if (parsedJson && parsedJson.overallSentiment && Array.isArray(parsedJson.topics)) { logger.log('INFO', `[getSentimentFromGemini] Classified sentiment: ${parsedJson.overallSentiment}, Topics: ${parsedJson.topics.join(', ')}`); return parsedJson; } }
             logger.log('WARN', `[getSentimentFromGemini] Unexpected JSON structure from AI: "${text}". Defaulting.`);
             return { overallSentiment: "Neutral", topics: [] };
-        } catch (parseError) {
-            logger.log('WARN', `[getSentimentFromGemini] Failed to parse JSON response from AI: "${text}". Defaulting.`, { error: parseError.message });
-            return { overallSentiment: "Neutral", topics: [] };
-        }
-        // --- END NEW JSON PARSE ---
-
-    } catch (error) {
-        logger.log('ERROR', '[getSentimentFromGemini] Error calling Gemini API.', { 
-            errorMessage: error.message,
-            errorStack: error.stack
-        });
-        // Default to 'Neutral' on API error to avoid halting the pipeline
-        return { overallSentiment: "Neutral", topics: [] }; // Return default object
-    }
+        } catch (parseError) { logger.log('WARN', `[getSentimentFromGemini] Failed to parse JSON response from AI: "${text}". Defaulting.`, { error: parseError.message }); return { overallSentiment: "Neutral", topics: [] }; }
+    } catch (error) { logger.log('ERROR', '[getSentimentFromGemini] Error calling Gemini API.', {  errorMessage: error.message, errorStack: error.stack }); return { overallSentiment: "Neutral", topics: [] };  }
 }
 
 
@@ -97,166 +43,73 @@ Post: "${snippet}"`
  */
 exports.handleSocialTask = async (message, context, config, dependencies) => {
     const { db, logger, headerManager, proxyManager } = dependencies;
-
     let task;
-    try {
-        task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
-    } catch (e) {
-        logger.log('ERROR', '[SocialTask] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
-        return; // Acknowledge the message
-    }
-
+    try { task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8')); } catch (e) { logger.log('ERROR', '[SocialTask] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data }); return;  }
     const { tickerId, since } = task;
     const sinceDate = new Date(since);
     const taskId = `social-${tickerId}-${context.eventId || Date.now()}`;
     logger.log('INFO', `[SocialTask/${taskId}] Processing ticker ${tickerId} for posts since ${since}.`);
-
-    // --- Config validation ---
-    if (!config.socialApiBaseUrl || !config.socialInsightsCollectionName || !config.processedPostsCollectionName) {
-        logger.log('ERROR', `[SocialTask/${taskId}] Missing required configuration.`);
-        throw new Error('Missing required configuration for Social Task.');
-    }
-
+    if (!config.socialApiBaseUrl || !config.socialInsightsCollectionName || !config.processedPostsCollectionName) { logger.log('ERROR', `[SocialTask/${taskId}] Missing required configuration.`); throw new Error('Missing required configuration for Social Task.'); }
     const processedPostsRef = db.collection(config.processedPostsCollectionName);
     const today = new Date().toISOString().slice(0, 10);
     const insightsCollectionRef = db.collection(config.socialInsightsCollectionName).doc(today).collection('posts');
-
     let offset = 0;
-    const take = 10; // hardcode to 10
+    const take = 10; 
     let keepFetching = true;
     let postsProcessed = 0;
     let postsSaved = 0;
-    const processedInThisRun = new Set(); // Local dedupe
-
+    const processedInThisRun = new Set(); 
     try {
         while (keepFetching) {
             const url = `${config.socialApiBaseUrl}${tickerId}?take=${take}&offset=${offset}&reactionsPageSize=20`;
             logger.log('TRACE', `[SocialTask/${taskId}] Fetching: ${url}`);
-            
             const selectedHeader = await headerManager.selectHeader();
             let wasSuccess = false;
             let response;
-            
-            try {
-                response = await proxyManager.fetch(url, { headers: selectedHeader.header });
-                if (!response.ok) {
-                    throw new Error(`API error ${response.status}`);
-                }
-                wasSuccess = true;
-            } catch (fetchError) {
-                logger.log('WARN', `[SocialTask/${taskId}] Fetch failed for offset ${offset}.`, { err: fetchError.message });
-                keepFetching = false; // Stop on fetch error
-                if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, false);
-                continue;
-            } finally {
-                if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
-            }
-
+            try { response = await proxyManager.fetch(url, { headers: selectedHeader.header }); if (!response.ok) { throw new Error(`API error ${response.status}`); } wasSuccess = true;
+            } catch (fetchError) { logger.log('WARN', `[SocialTask/${taskId}] Fetch failed for offset ${offset}.`, { err: fetchError.message }); keepFetching = false;  if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, false); continue;
+            } finally { if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess); }
             const page = await response.json();
             const discussions = page?.discussions;
-
-            if (!Array.isArray(discussions) || discussions.length === 0) {
-                logger.log('INFO', `[SocialTask/${taskId}] No more posts found at offset ${offset}. Stopping.`);
-                keepFetching = false;
-                continue;
-            }
-
+            if (!Array.isArray(discussions) || discussions.length === 0) { logger.log('INFO', `[SocialTask/${taskId}] No more posts found at offset ${offset}. Stopping.`); keepFetching = false; continue; }
             const postIds = discussions.map(d => d.post.id).filter(Boolean);
-            if (postIds.length === 0) {
-                offset += take;
-                continue;
-            }
-
-            // --- Deduplication Check ---
+            if (postIds.length === 0) { offset += take; continue; }
             const existingDocs = await processedPostsRef.where(FieldPath.documentId(), 'in', postIds).get();
             const existingIds = new Set(existingDocs.docs.map(d => d.id));
-
             const batch = db.batch();
             let newPostsInBatch = 0;
-
             for (const discussion of discussions) {
                 const post = discussion?.post;
                 if (!post || !post.id || !post.message?.text) continue;
-
-                // Stop pagination if we've reached posts older than our window
                 const postCreatedDate = new Date(post.created);
-                if (postCreatedDate < sinceDate) {
-                    keepFetching = false;
-                    continue; 
-                }
-
-                // Skip if already processed
-                if (existingIds.has(post.id) || processedInThisRun.has(post.id)) {
-                    continue;
-                }
-                
-                // Filter language (accept 'en' and 'en-gb')
+                if (postCreatedDate < sinceDate) { keepFetching = false; continue;  }
+                if (existingIds.has(post.id) || processedInThisRun.has(post.id)) { continue; }
                 const lang = post.message.languageCode || 'unknown';
-                if (lang !== 'en' && lang !== 'en-gb') {
-                    continue; // Skip non-English posts
-                }
-
-                // --- Process the new post ---
+                if (lang !== 'en' && lang !== 'en-gb') { continue; }
                 postsProcessed++;
                 processedInThisRun.add(post.id);
                 const text = post.message.text;
+                const likeCount    = discussion.emotionsData?.like?.paging?.totalCount || 0;
+                const commentCount = discussion.summary?.totalCommentsAndReplies       || 0;
+                let pollData       = null;
 
-                // --- START: Enhanced Data Extraction ---
-                const likeCount = discussion.emotionsData?.like?.paging?.totalCount || 0;
-                const commentCount = discussion.summary?.totalCommentsAndReplies || 0;
-                let pollData = null;
-
-                // Check for poll data and extract it
                 if (post.type === 'Poll' && post.metadata?.poll?.options) {
-                    pollData = {
-                        id: post.metadata.poll.id,
-                        options: post.metadata.poll.options.map(opt => ({
-                            id: opt.id,
-                            text: opt.text,
-                            votesCount: opt.votesCount || 0
-                        }))
-                    };
-                }
-                // --- END: Enhanced Data Extraction ---
+                    pollData = { id: post.metadata.poll.id, options: post.metadata.poll.options.map(opt => ({ id: opt.id, text: opt.text, votesCount: opt.votesCount || 0 })) }; }
 
 
-                // 1. Truncate for AI
-                const MAX_CHARS = 500; // ~125 tokens, very cheap
+                const MAX_CHARS = 500;
                 let snippet = text;
-                if (text.length > (MAX_CHARS * 2)) {
-                  // Create a "summary" snippet
-                  snippet = text.substring(0, MAX_CHARS) + " ... " + text.substring(text.length - MAX_CHARS);
-                } else if (text.length > MAX_CHARS) {
-                  // Just truncate
-                  snippet = text.substring(0, MAX_CHARS);
-                }
+                if (text.length > (MAX_CHARS * 2)) { snippet = text.substring(0, MAX_CHARS) + " ... " + text.substring(text.length - MAX_CHARS);
+                } else if (text.length > MAX_CHARS) { snippet = text.substring(0, MAX_CHARS); }
 
-                // 2. AI Sentiment Analysis
-                // Pass full dependencies object
-                // --- MODIFIED: Store the entire result object ---
                 const sentimentResult = await getSentimentFromGemini(dependencies, snippet);
 
-                // 3. Prepare data for storage
-                const postData = {
-                    sentiment: sentimentResult, // <-- Store the full {overallSentiment, topics} object
-                    textSnippet: snippet,
-                    fullText: text, // Store the full original text
-                    language: lang,
-                    tickers: post.tags.map(t => t.market?.symbolName).filter(Boolean),
-                    postOwnerId: post.owner?.id,
-                    likeCount: likeCount, // Store like count
-                    commentCount: commentCount, // Store comment count
-                    pollData: pollData, // Store poll data (will be null if not a poll)
-                    createdAt: post.created,
-                    fetchedAt: FieldValue.serverTimestamp()
-                };
-                // --- END MODIFICATION ---
+
+                const postData = { sentiment: sentimentResult,  textSnippet: snippet, fullText: text, language: lang, tickers: post.tags.map(t => t.market?.symbolName).filter(Boolean), postOwnerId: post.owner?.id, likeCount: likeCount, commentCount: commentCount,  pollData: pollData,  createdAt: post.created, fetchedAt: FieldValue.serverTimestamp() };
                 
-                // 4. Add to batch for `daily_social_insights`
                 const insightDocRef = insightsCollectionRef.doc(post.id);
                 batch.set(insightDocRef, postData);
 
-                // 5. Add to batch for `processed_social_posts` (dedupe collection)
                 const dedupeDocRef = processedPostsRef.doc(post.id);
                 batch.set(dedupeDocRef, { processedAt: FieldValue.serverTimestamp() });
                 
@@ -269,21 +122,11 @@ exports.handleSocialTask = async (message, context, config, dependencies) => {
                 logger.log('INFO', `[SocialTask/${taskId}] Saved ${newPostsInBatch} new posts from offset ${offset}.`);
             }
 
-            // Continue to next page
             offset += take;
         }
 
         logger.log('SUCCESS', `[SocialTask/${taskId}] Run complete. Processed ${postsProcessed} new posts, saved ${postsSaved}.`);
 
-    } catch (error) {
-        logger.log('ERROR', `[SocialTask/${taskId}] Fatal error during task execution.`, { errorMessage: error.message, errorStack: error.stack });
-        throw error;
-    } finally {
-        // Always flush header performance
-        try {
-            await headerManager.flushPerformanceUpdates();
-        } catch (flushError) {
-            logger.log('ERROR', `[SocialTask/${taskId}] Failed to flush header performance.`, { errorMessage: flushError.message });
-        }
-    }
+    } catch (error) { logger.log('ERROR', `[SocialTask/${taskId}] Fatal error during task execution.`, { errorMessage: error.message, errorStack: error.stack }); throw error;
+    } finally { try { await headerManager.flushPerformanceUpdates(); } catch (flushError) { logger.log('ERROR', `[SocialTask/${taskId}] Failed to flush header performance.`, { errorMessage: flushError.message }); } }
 };

package/functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers.js

@@ -14,26 +14,11 @@ const { FieldValue } = require('@google-cloud/firestore');
 exports.runCleanup = async (config, dependencies) => {
     const { logger } = dependencies;
     logger.log('INFO', '[CleanupHelpers] Running cleanup orchestrator...');
-
-    try {
-        // Start cleanup for pending users
-        const { batch: batchAfterPending, count: pendingRemoved } = await cleanupPendingSpeculators(config, dependencies);
-
-        // Continue with the same batch for stale speculators
+    try { const { batch: batchAfterPending, count: pendingRemoved } = await cleanupPendingSpeculators(config, dependencies);
         const { batch: finalBatch, count: staleRemoved } = await cleanupStaleSpeculators(config, dependencies, batchAfterPending);
-
-        if (pendingRemoved > 0 || staleRemoved > 0) {
-            await finalBatch.commit();
-            logger.log('SUCCESS', `[CleanupHelpers] Cleanup commit successful. Removed ${pendingRemoved} pending, ${staleRemoved} stale speculators.`);
-            return { pendingRemoved, staleRemoved };
-        } else {
-            logger.log('SUCCESS', '[CleanupHelpers] No stale users found in pending or blocks.');
-            return { pendingRemoved: 0, staleRemoved: 0 };
-        }
-    } catch (error) {
-        logger.log('ERROR', '[CleanupHelpers] FATAL error during cleanup orchestration', { errorMessage: error.message, errorStack: error.stack });
-        throw error;
-    }
+        if (pendingRemoved > 0 || staleRemoved > 0) { await finalBatch.commit(); logger.log('SUCCESS', `[CleanupHelpers] Cleanup commit successful. Removed ${pendingRemoved} pending, ${staleRemoved} stale speculators.`); return { pendingRemoved, staleRemoved };
+        } else { logger.log('SUCCESS', '[CleanupHelpers] No stale users found in pending or blocks.'); return { pendingRemoved: 0, staleRemoved: 0 }; }
+    } catch (error) { logger.log('ERROR', '[CleanupHelpers] FATAL error during cleanup orchestration', { errorMessage: error.message, errorStack: error.stack }); throw error; }
 };
 
 /**
@@ -43,18 +28,15 @@ async function cleanupPendingSpeculators(config, dependencies) {
     const { db, logger } = dependencies;
     logger.log('INFO', '[CleanupHelpers] Starting pending speculator cleanup...');
     
-    const batch = db.batch(); // Use db from dependencies
+    const batch = db.batch(); 
     let stalePendingUsersRemoved = 0;
-    const pendingCollectionRef = db.collection(config.pendingSpeculatorsCollectionName); // Use db
+    const pendingCollectionRef = db.collection(config.pendingSpeculatorsCollectionName);
     const staleThreshold = new Date();
     staleThreshold.setHours(staleThreshold.getHours() - config.pendingGracePeriodHours);
 
     try {
         const pendingSnapshot = await pendingCollectionRef.get();
-        if (pendingSnapshot.empty) {
-            logger.log('INFO', '[CleanupHelpers] Pending speculators collection is empty.');
-            return { batch, count: 0 };
-        }
+        if (pendingSnapshot.empty) { logger.log('INFO', '[CleanupHelpers] Pending speculators collection is empty.'); return { batch, count: 0 }; }
 
         for (const doc of pendingSnapshot.docs) {
             const pendingData = doc.data().users || {};
@@ -63,23 +45,11 @@ async function cleanupPendingSpeculators(config, dependencies) {
 
             for (const userId in pendingData) {
                 const timestamp = pendingData[userId]?.toDate ? pendingData[userId].toDate() : null;
-                if (timestamp && timestamp < staleThreshold) {
-                    updates[`users.${userId}`] = FieldValue.delete();
-                    stalePendingUsersRemoved++;
-                    updatesInDoc++;
-                }
-            }
+                if (timestamp && timestamp < staleThreshold) { updates[`users.${userId}`] = FieldValue.delete(); stalePendingUsersRemoved++; updatesInDoc++; } }
 
-            if (updatesInDoc > 0) {
-                 logger.log('TRACE', `[CleanupHelpers] Marking ${updatesInDoc} users for removal from pending doc ${doc.id}`);
-                batch.update(doc.ref, updates);
-            }
-        }
+            if (updatesInDoc > 0) { logger.log('TRACE', `[CleanupHelpers] Marking ${updatesInDoc} users for removal from pending doc ${doc.id}`); batch.update(doc.ref, updates); } }
         logger.log('INFO', `[CleanupHelpers] Marked ${stalePendingUsersRemoved} total stale pending users for removal.`);
-    } catch (error) {
-         logger.log('ERROR', '[CleanupHelpers] Error cleaning pending speculators', { errorMessage: error.message });
-         throw error;
-    }
+    } catch (error) { logger.log('ERROR', '[CleanupHelpers] Error cleaning pending speculators', { errorMessage: error.message }); throw error; }
     return { batch, count: stalePendingUsersRemoved };
 }
 
@@ -90,17 +60,14 @@ async function cleanupStaleSpeculators(config, dependencies, batch) {
     const { db, logger } = dependencies;
     logger.log('INFO', '[CleanupHelpers] Starting stale speculator cleanup from blocks...');
     let totalUsersRemoved = 0;
-    const blocksCollectionRef = db.collection(config.speculatorBlocksCollectionName); // Use db
+    const blocksCollectionRef = db.collection(config.speculatorBlocksCollectionName); 
     const gracePeriodDate = new Date();
     gracePeriodDate.setDate(gracePeriodDate.getDate() - config.activityGracePeriodDays);
     const blockCountsUpdate = {}; 
 
     try {
         const blocksSnapshot = await blocksCollectionRef.get();
-        if (blocksSnapshot.empty) {
-             logger.log('INFO', '[CleanupHelpers] Speculator blocks collection is empty.');
-            return { batch, count: 0 };
-        }
+        if (blocksSnapshot.empty) { logger.log('INFO', '[CleanupHelpers] Speculator blocks collection is empty.'); return { batch, count: 0 }; }
 
         for (const doc of blocksSnapshot.docs) {
             const blockId = doc.id;
@@ -116,45 +83,19 @@ async function cleanupStaleSpeculators(config, dependencies, batch) {
                 const userData = users[userKey];
                 const lastHeldTimestamp = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : null;
 
-                if (lastHeldTimestamp && lastHeldTimestamp < gracePeriodDate) {
-                    updates[userKey] = FieldValue.delete();
-                    usersRemovedFromBlock++;
+                if (lastHeldTimestamp && lastHeldTimestamp < gracePeriodDate) { updates[userKey] = FieldValue.delete(); usersRemovedFromBlock++;
 
                     if (userData.instruments && Array.isArray(userData.instruments)) {
-                        userData.instruments.forEach(instrumentId => {
-                            const instrumentBlockKey = `${instrumentId}_${blockId}`;
-                            if (!blockCountsUpdate[instrumentBlockKey]) {
-                                blockCountsUpdate[instrumentBlockKey] = 0;
-                            }
-                            blockCountsUpdate[instrumentBlockKey]--;
-                        });
-                    }
-                }
-            }
-
-            if (usersRemovedFromBlock > 0) {
-                logger.log('TRACE', `[CleanupHelpers] Marking ${usersRemovedFromBlock} users for removal from block ${blockId}.`);
-                batch.update(doc.ref, updates);
-                totalUsersRemoved += usersRemovedFromBlock;
-            }
-        }
+                        userData.instruments.forEach(instrumentId => { const instrumentBlockKey = `${instrumentId}_${blockId}`; if (!blockCountsUpdate[instrumentBlockKey]) { blockCountsUpdate[instrumentBlockKey] = 0; } blockCountsUpdate[instrumentBlockKey]--; }); } } }
+            if (usersRemovedFromBlock > 0) { logger.log('TRACE', `[CleanupHelpers] Marking ${usersRemovedFromBlock} users for removal from block ${blockId}.`); batch.update(doc.ref, updates); totalUsersRemoved += usersRemovedFromBlock; } }
 
         if (totalUsersRemoved > 0 && Object.keys(blockCountsUpdate).length > 0) {
-            const countsRef = db.doc(config.speculatorBlockCountsDocPath); // Use db
+            const countsRef = db.doc(config.speculatorBlockCountsDocPath);
             const finalCountUpdates = {};
-            for (const key in blockCountsUpdate) {
-                finalCountUpdates[`counts.${key}`] = FieldValue.increment(blockCountsUpdate[key]);
-            }
+            for (const key in blockCountsUpdate) { finalCountUpdates[`counts.${key}`] = FieldValue.increment(blockCountsUpdate[key]); }
              logger.log('TRACE', '[CleanupHelpers] Staging block count decrements.', { updates: finalCountUpdates });
-            batch.set(countsRef, finalCountUpdates, { merge: true });
-        }
-
+            batch.set(countsRef, finalCountUpdates, { merge: true }); }
         logger.log('INFO', `[CleanupHelpers] Marked ${totalUsersRemoved} total stale speculators for removal from blocks.`);
-
-    } catch (error) {
-        logger.log('ERROR', '[CleanupHelpers] Error cleaning stale speculators', { errorMessage: error.message });
-        throw error;
-    }
-
+    } catch (error) { logger.log('ERROR', '[CleanupHelpers] Error cleaning stale speculators', { errorMessage: error.message }); throw error; }
     return { batch, count: totalUsersRemoved };
 }

package/functions/task-engine/handler_creator.js

@@ -5,12 +5,7 @@
  */
 
 // Import the new utility functions that contain the logic
-const {
-    parseTaskPayload,
-    prepareTaskBatches,
-    runUsernameLookups,
-    executeTasks
-} = require('./utils/task_engine_utils'); // <-- Import from our new utils file
+const { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks } = require('./utils/task_engine_utils'); 
 
 /**
  * Main pipe: pipe.taskEngine.handleRequest
@@ -31,8 +26,7 @@ async function handleRequest(message, context, config, dependencies) {
         await runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config, batchManager, logger);
         await executeTasks(tasksToRun, otherTasks, dependencies, config, taskId);
     } catch (error) {logger.log('ERROR', `[TaskEngine/${taskId}] Failed during batch processing.`, { errorMessage: error.message, errorStack: error.stack });
-    } finally {
-        try {logger.log('INFO', `[TaskEngine/${taskId}] Flushing all accumulated batches...`);
+    } finally { try {logger.log('INFO', `[TaskEngine/${taskId}] Flushing all accumulated batches...`);
         await batchManager.flushBatches();
         logger.log('INFO', `[TaskEngine/${taskId}] Final batch and header flush complete.`);} catch (flushError) {logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });}}
 }

package/functions/task-engine/helpers/update_helpers.js

@@ -23,13 +23,11 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, {
         if (!header) { logger.log('ERROR', '[lookupUsernames] Could not select a header.'); continue; }
         let success = false;
         try {
-            const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, {
-                method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch)
-            });
+            const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) });
             if (!res.ok) throw new Error(`API status ${res.status}`);
             const data = await res.json();
             if (Array.isArray(data)) allUsers.push(...data);
-            success = true;
+            success = true; logger.log('DEBUG', 'Looked up usernames', { batch })
         } catch (err) {
             logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message });
         } finally { headerManager.updatePerformance(header.id, success); }
@@ -38,110 +36,46 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, {
     return allUsers;
 }
 
-// --- START MODIFICATION: Added historyFetchedForUser argument ---
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username, historyFetchedForUser) {
-    const { userId, instruments, instrumentId, userType } = task; // ⚠️ Now supports both
-    
-    // ⚠️ Support both old (instrumentId) and new (instruments array) format
-    const instrumentsToProcess = userType === 'speculator' 
-        ? (instruments || [instrumentId]) // New format or fallback to old
-        : [undefined]; // Normal users don't have instruments
-    
+    const { userId, instruments, instrumentId, userType } = task;
+    const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
-    
     let portfolioHeader = await headerManager.selectHeader();
     let historyHeader = null;
     if (!portfolioHeader) throw new Error("Could not select portfolio header.");
-    
     let wasHistorySuccess = false, isPrivate = false;
-    
-    // --- FIX: 'fetchHistory' MUST be declared here, outside the 'try' block ---
     let fetchHistory = false;
-    
     try {
-        // Fetch history ONCE per user
         const promisesToRun = [];
-        // --- 'fetchHistory' is no longer declared here ---
-        
         if (!historyFetchedForUser.has(userId)) {
             historyHeader = await headerManager.selectHeader();
-            if (historyHeader) {
-                fetchHistory = true; // This now sets the outer variable
-                historyFetchedForUser.add(userId);
-                const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
-                promisesToRun.push(proxyManager.fetch(historyUrl, { headers: historyHeader.header }));
-            }
-        }
-        
-        // Process history result (if fetched)
+            if (historyHeader) { fetchHistory = true;  historyFetchedForUser.add(userId); const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`; promisesToRun.push(proxyManager.fetch(historyUrl, { headers: historyHeader.header })); } }
         if (fetchHistory) {
             const results = await Promise.allSettled(promisesToRun);
             const historyRes = results[0];
-            if (historyRes.status === 'fulfilled' && historyRes.value.ok) {
-                const data = await historyRes.value.json();
-                wasHistorySuccess = true; 
-                await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
-            }
-        }
-        
-        // Now fetch portfolio for EACH instrument (speculators) or once (normal)
+            if (historyRes.status === 'fulfilled' && historyRes.value.ok) { const data = await historyRes.value.json(); wasHistorySuccess = true; await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType); } }
         for (const instrumentId of instrumentsToProcess) {
-            const portfolioUrl = userType === 'speculator'
-                ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}`
-                : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
-            
+            const portfolioUrl = userType === 'speculator' ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
             let wasPortfolioSuccess = false;
-            
             const portfolioRes = await proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header });
-            
             if (portfolioRes.ok) {
                 const body = await portfolioRes.text();
-                if (body.includes("user is PRIVATE")) {
-                    isPrivate = true;
-                    break; // Stop processing this user
-                } else {
-                    wasPortfolioSuccess = true;
-                    await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, instrumentId);
-                }
-            }
-            
+                if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `User ${userId} is private. Removing from updates.`) ; break; 
+                } else { wasPortfolioSuccess = true; await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, instrumentId); } }
+                logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl })
             headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
-            
-            // Re-select header for next instrument
             if (instrumentsToProcess.length > 1 && instrumentId !== instrumentsToProcess[instrumentsToProcess.length - 1]) {
-                portfolioHeader = await headerManager.selectHeader();
-            }
-        }
-        
-        // Handle private user
+                portfolioHeader = await headerManager.selectHeader(); } }
         if (isPrivate) {
             logger.log('WARN', `User ${userId} is private. Removing from updates.`);
-            // Delete for ALL instruments
-            for (const instrumentId of instrumentsToProcess) {
-                await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
-            }
+            for (const instrumentId of instrumentsToProcess) { await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId); }
             const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
-            for (const instrumentId of instrumentsToProcess) {
-                const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
-                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
-            }
-            return;
-        }
-        
-        // Update timestamps
-        for (const instrumentId of instrumentsToProcess) {
-            await batchManager.updateUserTimestamp(userId, userType, instrumentId);
-        }
-        if (userType === 'speculator') {
-            await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
-        }
-        
-    } finally {
-        if (historyHeader && fetchHistory) {
-            headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
-        }
-    }
+            for (const instrumentId of instrumentsToProcess) { const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`; await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true }); }
+            return; }
+        for (const instrumentId of instrumentsToProcess) { await batchManager.updateUserTimestamp(userId, userType, instrumentId); }
+        if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
+    } finally { if (historyHeader && fetchHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); }}
 }
 
 module.exports = { handleUpdate, lookupUsernames };