bulltrackers-module 1.0.360 → 1.0.362

package/functions/core/utils/firestore_utils.js

@@ -257,6 +257,10 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
             username: item.UserName
         }));
 
+        if (targets.length > 0) {
+            logger.log('INFO', `[Core Utils Debug] First PI Target Mapped: ${JSON.stringify(targets[0])}`);
+        }
+
         logger.log('INFO', `[Core Utils] Found ${targets.length} Popular Investors from ${today} ranking.`);
         return targets;
 

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -2,13 +2,16 @@
  * @fileoverview Logic for handling Popular Investor and On-Demand User updates.
  * REFACTORED: Uses FirestoreBatchManager for sharded storage.
  * UPDATED: On-Demand User Update now fetches Trade History (1 Year).
+ * FIXED: Added client_request_id to all PI fetches to prevent 500 errors.
+ * FIXED: Changed Deep Dive limit to Top 10 assets.
+ * DEBUG: Added verbose URL logging.
  */
 
 const crypto = require('crypto');
 
 /**
  * Handles the update task for a Popular Investor.
- * Fetches: Overall Portfolio, Top 20 Deep Positions, Trade History.
+ * Fetches: Overall Portfolio, Top 10 Deep Positions, Trade History.
  * Stores them via BatchManager to ensure sharding.
  * @param {object} taskData - Contains { cid, username }.
  * @param {object} config - Task Engine configuration.
@@ -29,13 +32,20 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
     const today = new Date().toISOString().split('T')[0];
     const blockId = '19M'; 
 
+    // Generate Request UUID (Critical for preventing 500s on PI endpoints)
+    const uuid = crypto.randomUUID ? crypto.randomUUID() : `req-${Date.now()}-${Math.floor(Math.random()*10000)}`;
+
     try {
-        // 1. Fetch Overall Portfolio
-        const portfolioUrl = `${ETORO_API_PORTFOLIO_URL}?cid=${cid}`;
+        // --- 1. Fetch Overall Portfolio ---
+        const portfolioUrl = `${ETORO_API_PORTFOLIO_URL}?cid=${cid}&client_request_id=${uuid}`;
+        
+        // [DEBUG] Log the EXACT URL being used
+        logger.log('INFO', `[PI DEBUG] Fetching Portfolio for ${username} (${cid}) URL: ${portfolioUrl}`);
+
         const portfolioRes = await proxyManager.fetch(portfolioUrl, { method: 'GET' });
         
         if (!portfolioRes.ok) {
-            throw new Error(`Failed to fetch portfolio for ${cid}: ${portfolioRes.status}`);
+            throw new Error(`Failed to fetch portfolio for ${cid}. Status: ${portfolioRes.status} URL: ${portfolioUrl}`);
         }
         const portfolioData = await portfolioRes.json();
         
@@ -46,22 +56,28 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
         // Batch Write (Overall)
         await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'popular_investor');
 
-        // 2. Identify Top Assets for Deep Dive
+        // --- 2. Identify Top Assets for Deep Dive ---
         const aggregatedPositions = portfolioData.AggregatedPositions || [];
         const topPositions = aggregatedPositions
             .sort((a, b) => b.Invested - a.Invested)
-            .slice(0, 20);
+            .slice(0, 10); // Limit to Top 10
 
         logger.log('INFO', `[PI Update] Fetching deep positions for top ${topPositions.length} assets for ${username}`);
 
         const deepPositions = [];
         for (const pos of topPositions) {
-            const posUrl = `${ETORO_API_POSITIONS_URL}?cid=${cid}&InstrumentID=${pos.InstrumentID}`;
+            const posUrl = `${ETORO_API_POSITIONS_URL}?cid=${cid}&InstrumentID=${pos.InstrumentID}&client_request_id=${uuid}`;
+            
+            // [DEBUG] Log the EXACT Deep Dive URL
+            logger.log('INFO', `[PI DEBUG] Fetching Position for ${username} InstID: ${pos.InstrumentID} URL: ${posUrl}`);
+
             try {
                 const deepRes = await proxyManager.fetch(posUrl, { method: 'GET' });
                 if (deepRes.ok) {
                     const deepData = await deepRes.json();
                     deepPositions.push({ instrumentId: pos.InstrumentID, ...deepData });
+                } else {
+                    logger.log('WARN', `[PI DEBUG] Failed deep fetch. Status: ${deepRes.status} URL: ${posUrl}`);
                 }
             } catch (err) {
                 logger.log('WARN', `[PI Update] Failed deep fetch for inst ${pos.InstrumentID} user ${cid}`, err);
@@ -73,21 +89,23 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
             await batchManager.addToPortfolioBatch(String(cid), blockId, today, deepPayload, 'popular_investor_deep');
         }
 
-        // 3. Fetch Trade History (Last 1 Year) with Pagination
-        // [FIX] Implemented Pagination
+        // --- 3. Fetch Trade History (Last 1 Year) with Pagination ---
         const oneYearAgo = new Date();
         oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
-        const uuid = crypto.randomUUID ? crypto.randomUUID() : '0205aca7-bd37-4884-8455-f28ce1add2de';
+        
         const historyBaseUrl = ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
         
         let allHistoryTrades = [];
         let page = 1;
         let hasMore = true;
-        const MAX_PAGES = 10; // Safety limit (10k trades)
+        const MAX_PAGES = 10; 
 
         while (hasMore && page <= MAX_PAGES) {
              const historyUrl = `${historyBaseUrl}?StartTime=${oneYearAgo.toISOString()}&PageNumber=${page}&ItemsPerPage=1000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
              
+             // [DEBUG] Log the EXACT History URL
+             logger.log('INFO', `[PI DEBUG] Fetching History Page ${page} for ${username} URL: ${historyUrl}`);
+
              try {
                 const historyRes = await proxyManager.fetch(historyUrl, { method: 'GET' });
                 if (historyRes.ok) {
@@ -97,12 +115,12 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
                     if (trades.length > 0) {
                         allHistoryTrades = allHistoryTrades.concat(trades);
                         page++;
-                        hasMore = (trades.length === 1000); // If full page, assumes more
+                        hasMore = (trades.length === 1000); 
                     } else {
                         hasMore = false;
                     }
                 } else {
-                    logger.log('WARN', `[PI Update] History fetch failed on page ${page} for ${username}`);
+                    logger.log('WARN', `[PI Update] History fetch failed on page ${page} for ${username}. Status: ${historyRes.status}`);
                     hasMore = false;
                 }
              } catch (e) {
@@ -130,8 +148,6 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
 
 /**
  * Handles the On-Demand update for a Signed-In User.
- * Fetches: Overall Portfolio AND Trade History.
- * Triggered by User Login/Verification.
  */
 async function handleOnDemandUserUpdate(taskData, config, dependencies) {
     const { cid, username } = taskData;
@@ -141,11 +157,15 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
     logger.log('INFO', `[On-Demand Update] Fetching Portfolio & History for Signed-In User: ${username} (${cid})`);
     
     const today = new Date().toISOString().split('T')[0];
-    const blockId = '19M'; // Standard block for consistent indexing
+    const blockId = '19M'; 
+    const uuid = crypto.randomUUID ? crypto.randomUUID() : `req-${Date.now()}-${Math.floor(Math.random()*10000)}`;
 
     try {
-        // --- 1. Fetch Portfolio ---
-        const portfolioUrl = `${ETORO_API_PORTFOLIO_URL}?cid=${cid}`;
+        const portfolioUrl = `${ETORO_API_PORTFOLIO_URL}?cid=${cid}&client_request_id=${uuid}`;
+        
+        // [DEBUG] Log URL for On-Demand
+        logger.log('INFO', `[On-Demand DEBUG] Fetching Portfolio URL: ${portfolioUrl}`);
+
         const portfolioRes = await proxyManager.fetch(portfolioUrl, { method: 'GET' });
 
         if (!portfolioRes.ok) {
@@ -156,43 +176,34 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         portfolioData.lastUpdated = new Date();
         portfolioData.username = username;
 
-        // Batch Write (Signed-In Portfolio)
         await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'signed_in_user');
 
-        // --- 2. Fetch Trade History (Last 1 Year) ---
-        // This is critical for the new DAG requirements
+        // History Fetch
         const oneYearAgo = new Date();
         oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
-        
-        const uuid = crypto.randomUUID ? crypto.randomUUID() : '0205aca7-bd37-4884-8455-f28ce1add2de';
         const historyBaseUrl = ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
         const historyUrl = `${historyBaseUrl}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=1000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
 
+        logger.log('INFO', `[On-Demand DEBUG] Fetching History URL: ${historyUrl}`);
+
         const historyRes = await proxyManager.fetch(historyUrl, { method: 'GET' });
         
         if (historyRes.ok) {
             const historyData = await historyRes.json();
             historyData.fetchedAt = new Date();
             
-            // [FIX] Apply History Filtering (remove CopyTrading noise)
-            const VALID_REASONS = [0, 1, 5]; // 1=Close, 5=SL/TP etc. Remove 17/18 (Copy)
+            const VALID_REASONS = [0, 1, 5]; 
             if (historyData.PublicHistoryPositions) {
-                const originalCount = historyData.PublicHistoryPositions.length;
                 historyData.PublicHistoryPositions = historyData.PublicHistoryPositions.filter(
                     p => VALID_REASONS.includes(p.CloseReason)
                 );
-                logger.log('INFO', `[On-Demand Update] Filtered history for ${username}: ${originalCount} -> ${historyData.PublicHistoryPositions.length}`);
             }
-
-            // Batch Write (Signed-In History)
             await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyData, 'signed_in_user');
-            
         } else {
-            logger.log('WARN', `[On-Demand Update] History fetch failed for ${username} (${historyRes.status}), proceeding with Portfolio only.`);
+            logger.log('WARN', `[On-Demand Update] History fetch failed for ${username} (${historyRes.status})`);
         }
 
         logger.log('SUCCESS', `[On-Demand Update] Complete for ${username}`);
-
     } catch (error) {
         logger.log('ERROR', `[On-Demand Update] Failed for ${username}`, error);
         throw error;

package/functions/task-engine/utils/task_engine_utils.js

@@ -11,7 +11,9 @@
 
 const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
-const { handleUpdate } = require('../helpers/update_helpers'); // Removed lookupUsernames import
+const { handleUpdate } = require('../helpers/update_helpers'); 
+// --- FIX: Import the handlers for the new task types ---
+const { handlePopularInvestorUpdate, handleOnDemandUserUpdate } = require('../helpers/popular_investor_helpers');
 const pLimit = require('p-limit'); 
 
 /**
@@ -27,24 +29,21 @@ function parseTaskPayload(message, logger) {
 }
 
 /**
- * Sorts tasks into update and other (discover/verify).
- * REFACTORED: Simplified. No username lookup logic needed.
+ * Sorts tasks into update and other (discover/verify/PI/OnDemand).
  */
 async function prepareTaskBatches(tasks, batchManager, logger) {
     const tasksToRun = [], otherTasks = [];
     
     for (const task of tasks) {
         if (task.type === 'update') { 
-            // New API uses CID (userId), so we push directly to run.
+            // Standard portfolio updates (Normal/Speculator)
             tasksToRun.push(task); 
         } else {
+            // Discover, Verify, Popular Investor, Signed-In User
             otherTasks.push(task); 
         }
     }
     
-    // We explicitly return empty structures for compatibility if handler_creator expects them,
-    // though ideally handler_creator should also be simplified.
-    // For now, we return compatible object structure.
     return { tasksToRun, cidsToLookup: new Map(), otherTasks };
 }
 
@@ -53,26 +52,53 @@ async function prepareTaskBatches(tasks, batchManager, logger) {
  * (REFACTORED: Concurrency limit set to 1)
  */
 async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
-    const { logger, batchManager } = dependencies; 
+    const { logger } = dependencies; 
     
-    // --- REFACTOR 1: REMOVE CONCURRENCY ---
-    // Set limit to 1 to serialize all tasks and prevent AppScript throttling.
     const limit = pLimit(1); 
-    // --- END REFACTOR 1 ---
     
     const allTaskPromises = [];
-    let taskCounters = { update: 0, discover: 0, verify: 0, unknown: 0, failed: 0 };
+    let taskCounters = { update: 0, discover: 0, verify: 0, popular_investor: 0, on_demand: 0, unknown: 0, failed: 0 };
 
-    // 1. Queue 'other' tasks (discover, verify)
+    // 1. Queue 'other' tasks
     for (const task of otherTasks) {
-        const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
-        const handler = { discover: handleDiscover, verify: handleVerify }[task.type];
+        const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || task.cid || 'sub'}`;
+        
+        // --- FIX START: Explicitly handle new task types ---
+        if (task.type === 'POPULAR_INVESTOR_UPDATE') {
+            allTaskPromises.push(limit(() => 
+                handlePopularInvestorUpdate(task, config, dependencies)
+                    .then(() => taskCounters.popular_investor++)
+                    .catch(err => {
+                        logger.log('ERROR', `[TaskEngine/${taskId}] Error in POPULAR_INVESTOR_UPDATE for ${task.cid || task.username}`, { errorMessage: err.message });
+                        taskCounters.failed++;
+                    })
+            ));
+            continue;
+        }
+
+        if (task.type === 'ON_DEMAND_USER_UPDATE') {
+             allTaskPromises.push(limit(() => 
+                handleOnDemandUserUpdate(task, config, dependencies)
+                    .then(() => taskCounters.on_demand++)
+                    .catch(err => {
+                        logger.log('ERROR', `[TaskEngine/${taskId}] Error in ON_DEMAND_USER_UPDATE for ${task.cid || task.username}`, { errorMessage: err.message });
+                        taskCounters.failed++;
+                    })
+            ));
+            continue;
+        }
+        // --- FIX END ---
+
+        // Handle legacy types (discover/verify)
+        // Normalize type to lowercase to handle potential mismatch if Orchestrator sends uppercase
+        const normalizedType = task.type.toLowerCase();
+        const handler = { discover: handleDiscover, verify: handleVerify }[normalizedType];
 
         if (handler) {
             allTaskPromises.push(
                 limit(() => 
                     handler(task, subTaskId, dependencies, config)
-                        .then(() => taskCounters[task.type]++)
+                        .then(() => taskCounters[normalizedType]++)
                         .catch(err => {
                             logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${subTaskId}`, { errorMessage: err.message });
                             taskCounters.failed++;
@@ -85,13 +111,8 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
         }
     }
 
-    // 2. Queue 'update' tasks
+    // 2. Queue 'update' tasks (Standard Normal/Speculator)
     for (const task of tasksToRun) {
-        // We unpack 'task' directly now, no wrapping object {task, username}
-        // However, we must ensure backward compatibility if the array was {task, username} before.
-        // In prepareTaskBatches above, we pushed raw 'task'. 
-        // So we use 'task' directly.
-        
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
         allTaskPromises.push(
             limit(() => 
@@ -105,15 +126,14 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
         );
     }
 
-    // 3. Wait for ALL tasks to complete (now sequentially)
+    // 3. Wait for ALL tasks to complete
     await Promise.all(allTaskPromises);
 
     // 4. Log final summary
     logger.log(
         taskCounters.failed > 0 ? 'WARN' : 'SUCCESS', 
-        `[TaskEngine/${taskId}] Processed all tasks. Updates: ${taskCounters.update}, Discovers: ${taskCounters.discover}, Verifies: ${taskCounters.verify}, Unknown: ${taskCounters.unknown}, Failed: ${taskCounters.failed}.`
+        `[TaskEngine/${taskId}] Processed all tasks. Updates: ${taskCounters.update}, Discovers: ${taskCounters.discover}, Verifies: ${taskCounters.verify}, PI: ${taskCounters.popular_investor}, OnDemand: ${taskCounters.on_demand}, Unknown: ${taskCounters.unknown}, Failed: ${taskCounters.failed}.`
     );
 }
 
-// Note: runUsernameLookups removed from exports
 module.exports = { parseTaskPayload, prepareTaskBatches, executeTasks };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.360",
+  "version": "1.0.362",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [