bulltrackers-module 1.0.135 → 1.0.137

package/functions/core/utils/firestore_utils.js

@@ -260,25 +260,48 @@ async function getSpeculatorsToUpdate(dependencies, config) {
     const { dateThreshold, gracePeriodThreshold, speculatorBlocksCollectionName } = config;
     logger.log('INFO','[Core Utils] Getting speculators to update...');
     const updates = [];
+    
+    // ⚠️ NEW: Collect per user first
+    const userMap = new Map(); // userId -> { instruments: Set }
+    
     try {
         const blocksRef = db.collection(speculatorBlocksCollectionName);
         const snapshot = await blocksRef.get();
-        if (snapshot.empty) { logger.log('INFO','[Core Utils] No speculator blocks found.'); return []; }
+        if (snapshot.empty) { 
+            logger.log('INFO','[Core Utils] No speculator blocks found.'); 
+            return []; 
+        }
+        
         snapshot.forEach(doc => {
             const blockData = doc.data();
             for (const key in blockData) {
-                 if (!key.startsWith('users.')) continue; 
-                 const userId = key.split('.')[1];
-                 if (!userId) continue; 
-                 const userData = blockData[key]; 
-                 const lastVerified = userData.lastVerified?.toDate ? userData.lastVerified.toDate() : new Date(0);
-                 const lastHeld = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : new Date(0);
-                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) { if (userData.instruments && Array.isArray(userData.instruments)) { userData.instruments.forEach(instrumentId => { updates.push({ userId, instrumentId }); });
-                     }
-                 }
+                if (!key.startsWith('users.')) continue; 
+                const userId = key.split('.')[1];
+                if (!userId) continue; 
+                const userData = blockData[key]; 
+                const lastVerified = userData.lastVerified?.toDate ? userData.lastVerified.toDate() : new Date(0);
+                const lastHeld = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : new Date(0);
+                
+                if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) {
+                    if (!userMap.has(userId)) {
+                        userMap.set(userId, new Set());
+                    }
+                    if (userData.instruments && Array.isArray(userData.instruments)) {
+                        userData.instruments.forEach(id => userMap.get(userId).add(id));
+                    }
+                }
             }
         });
-        logger.log('INFO',`[Core Utils] Found ${updates.length} speculator user/instrument pairs to update.`);
+        
+        // ⚠️ NEW: Return one task per user with ALL instruments
+        for (const [userId, instrumentSet] of userMap) {
+            updates.push({ 
+                userId, 
+                instruments: Array.from(instrumentSet) // ⚠️ Array of all instruments
+            });
+        }
+        
+        logger.log('INFO',`[Core Utils] Found ${updates.length} speculator users to update (covering ${[...userMap.values()].reduce((sum, set) => sum + set.size, 0)} total instruments).`);
         return updates;
     } catch (error) {
         logger.log('ERROR','[Core Utils] Error getting speculators to update', { errorMessage: error.message });

package/functions/task-engine/helpers/update_helpers.js

@@ -40,100 +40,104 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, {
 
 // --- START MODIFICATION: Added historyFetchedForUser argument ---
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username, historyFetchedForUser) {
-// --- END MODIFICATION ---
-
-    const { userId, instrumentId, userType } = task;
-    const portfolioUrl = userType === 'speculator'
-        ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}`
-        : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+    const { userId, instruments, instrumentId, userType } = task; // ⚠️ Now supports both
+    
+    // ⚠️ Support both old (instrumentId) and new (instruments array) format
+    const instrumentsToProcess = userType === 'speculator' 
+        ? (instruments || [instrumentId]) // New format or fallback to old
+        : [undefined]; // Normal users don't have instruments
     
-    // --- MODIFICATION: Moved historyUrl definition inside conditional ---
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
-
-    // --- MODIFICATION: Select history header only if needed ---
+    
     let portfolioHeader = await headerManager.selectHeader();
-    let historyHeader = null; // Will be selected if needed
+    let historyHeader = null;
     if (!portfolioHeader) throw new Error("Could not select portfolio header.");
-
-    let wasPortfolioSuccess = false, wasHistorySuccess = false, isPrivate = false, fetchHistory = false; // <-- FIX: 'fetchHistory' declared here
-
+    
+    let wasHistorySuccess = false, isPrivate = false;
+    
     try {
-        // --- START MODIFICATION: Build promises conditionally ---
+        // Fetch history ONCE per user
         const promisesToRun = [];
-
-        // 1. Always fetch portfolio
-        promisesToRun.push(proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header }));
-
-        // 2. Conditionally fetch history
-        // let fetchHistory = false; // <-- FIX: This line is removed
+        let fetchHistory = false;
+        
         if (!historyFetchedForUser.has(userId)) {
-            historyHeader = await headerManager.selectHeader(); // Select header just-in-time
-            if (!historyHeader) {
-                logger.log('WARN', `[UPDATE] Could not select history header for ${userId}, skipping history fetch for this batch.`);
-                historyFetchedForUser.add(userId); // Add to set to prevent retries in this batch
-            } else {
-                fetchHistory = true; // <-- FIX: Assigns to outer scope variable
-                historyFetchedForUser.add(userId); // Mark as fetched for this batch
+            historyHeader = await headerManager.selectHeader();
+            if (historyHeader) {
+                fetchHistory = true;
+                historyFetchedForUser.add(userId);
                 const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
                 promisesToRun.push(proxyManager.fetch(historyUrl, { headers: historyHeader.header }));
             }
         }
-        // --- END MODIFICATION ---
-
-        // --- Run Promises ---
-        const results = await Promise.allSettled(promisesToRun);
-        const portfolioRes = results[0];
-        const historyRes = fetchHistory ? results[1] : null; // History is only index 1 if we fetched it
-
-        // --- Process Portfolio (results[0]) ---
-        if (portfolioRes.status === 'fulfilled' && portfolioRes.value.ok) {
-            const body = await portfolioRes.value.text();
-            if (body.includes("user is PRIVATE")) isPrivate = true;
-            else { wasPortfolioSuccess = true; await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, instrumentId); }
-        } else {
-            let errMsg = portfolioRes.status === 'rejected' ? portfolioRes.reason.message : `API status ${portfolioRes.value.status}`;
-            let rawText = portfolioRes.value?.text ? await portfolioRes.value.text() : 'N/A';
-            logger.log('WARN', `[UPDATE] Portfolio fetch failed for ${userId}`, { error: errMsg, proxyResponse: rawText });
-        }
-
-        // --- Process History (results[1], if it exists) ---
-        if (fetchHistory && historyRes) { // Check if we ran this promise
+        
+        // Process history result (if fetched)
+        if (fetchHistory) {
+            const results = await Promise.allSettled(promisesToRun);
+            const historyRes = results[0];
             if (historyRes.status === 'fulfilled' && historyRes.value.ok) {
                 const data = await historyRes.value.json();
                 wasHistorySuccess = true; 
                 await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
-            } else {
-                // History fetch failed
-                let errMsg = historyRes.status === 'rejected' ? historyRes.reason.message : `API status ${historyRes.value.status}`;
-                let rawText = historyRes.value?.text ? await historyRes.value.text() : 'N/A';
-                logger.log('WARN', `[UPDATE] History fetch failed for ${userId} (${username})`, { error: errMsg, proxyResponse: rawText });
             }
         }
-        // --- END MODIFICATION ---
-
-
-        // --- Private user handling ---
+        
+        // Now fetch portfolio for EACH instrument (speculators) or once (normal)
+        for (const instrumentId of instrumentsToProcess) {
+            const portfolioUrl = userType === 'speculator'
+                ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}`
+                : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+            
+            let wasPortfolioSuccess = false;
+            
+            const portfolioRes = await proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header });
+            
+            if (portfolioRes.ok) {
+                const body = await portfolioRes.text();
+                if (body.includes("user is PRIVATE")) {
+                    isPrivate = true;
+                    break; // Stop processing this user
+                } else {
+                    wasPortfolioSuccess = true;
+                    await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, instrumentId);
+                }
+            }
+            
+            headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
+            
+            // Re-select header for next instrument
+            if (instrumentsToProcess.length > 1 && instrumentId !== instrumentsToProcess[instrumentsToProcess.length - 1]) {
+                portfolioHeader = await headerManager.selectHeader();
+            }
+        }
+        
+        // Handle private user
         if (isPrivate) {
             logger.log('WARN', `User ${userId} is private. Removing from updates.`);
-            await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
-
-            const blockCountsRef = db.doc(userType === 'speculator' ? config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS : config.FIRESTORE_DOC_BLOCK_COUNTS);
-            const incrementField = userType === 'speculator' ? `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}` : `counts.${Math.floor(userId/1e6)*1e6}`;
-            await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
+            // Delete for ALL instruments
+            for (const instrumentId of instrumentsToProcess) {
+                await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
+            }
+            const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
+            for (const instrumentId of instrumentsToProcess) {
+                const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
+                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
+            }
             return;
         }
-
-        if (wasPortfolioSuccess || wasHistorySuccess) {
+        
+        // Update timestamps
+        for (const instrumentId of instrumentsToProcess) {
             await batchManager.updateUserTimestamp(userId, userType, instrumentId);
-            if (userType === 'speculator') await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
         }
-    } finally {
-        if (portfolioHeader) headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
+        if (userType === 'speculator') {
+            await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
+        }
         
-        // --- MODIFICATION: Only update history header if we used it ---
-        if (fetchHistory && historyHeader) headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); // <-- FIX: This line now works
-        // --- END MODIFICATION ---
+    } finally {
+        if (historyHeader && fetchHistory) {
+            headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
+        }
     }
 }
 

package/functions/task-engine/helpers/verify_helpers.js

@@ -59,14 +59,30 @@ async function handleVerify(task, taskId, { db, logger, ...dependencies }, confi
         }
     }
 
+    // --- START FIX: MODIFIED FOR SHARDING ---
     if (Object.keys(usernameMap).length) {
-        const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc('cid_map_shard_1');
-        batch.set(mapRef, usernameMap, { merge: true });
-        logger.log('INFO', `[VERIFY] Staging ${Object.keys(usernameMap).length} username updates.`);
+        // Group updates by shard
+        const shardedUpdates = {};
+        for (const cid in usernameMap) {
+            // Re-implement the sharding logic here
+            const shardId = `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
+            if (!shardedUpdates[shardId]) {
+                shardedUpdates[shardId] = {};
+            }
+            shardedUpdates[shardId][cid] = usernameMap[cid];
+        }
+
+        // Write each shard to its own document
+        for (const shardId in shardedUpdates) {
+            const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc(shardId);
+            batch.set(mapRef, shardedUpdates[shardId], { merge: true });
+        }
+        logger.log('INFO', `[VERIFY] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`);
     }
+    // --- END FIX ---
 
     await batch.commit();
     if (validUserCount) logger.log('INFO', `[VERIFY] Verified and stored ${validUserCount} new ${userType} users.`);
 }
 
-module.exports = { handleVerify };
+module.exports = { handleVerify };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -31,6 +31,13 @@ class FirestoreBatchManager {
         logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
+    // --- START FIX: ADDED SHARDING HELPER ---
+    _getUsernameShardId(cid) {
+        // Shard across 10 documents (supports ~200k users assuming 50 bytes/user)
+        return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
+    }
+    // --- END FIX ---
+
     _scheduleFlush() {
         if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS);
     }
@@ -40,6 +47,7 @@ class FirestoreBatchManager {
         this.usernameMap.clear();
         this.logger.log('INFO', '[BATCH] Refreshing username map from Firestore...');
         try {
+            // This correctly gets ALL documents (shards) in the collection
             const snapshot = await this.db.collection(this.usernameMapCollectionName).get();
             snapshot.forEach(doc => {
                 const data = doc.data();
@@ -54,14 +62,23 @@ class FirestoreBatchManager {
 
     getUsername(cid) { return this.usernameMap.get(String(cid)); }
 
+    // --- START FIX: MODIFIED FOR SHARDING ---
     addUsernameMapUpdate(cid, username) {
         if (!username) return;
         const cidStr = String(cid);
         this.usernameMap.set(cidStr, username);
-        this.usernameMapUpdates[cidStr] = { username };
-        this.logger.log('TRACE', `[BATCH] Queued username update for ${cidStr}.`);
+
+        // Organize updates by shard ID
+        const shardId = this._getUsernameShardId(cidStr);
+        if (!this.usernameMapUpdates[shardId]) {
+            this.usernameMapUpdates[shardId] = {};
+        }
+        this.usernameMapUpdates[shardId][cidStr] = { username };
+
+        this.logger.log('TRACE', `[BATCH] Queued username update for ${cidStr} in ${shardId}.`);
         this._scheduleFlush();
     }
+    // --- END FIX ---
 
     async addToTradingHistoryBatch(userId, blockId, date, historyData, userType) {
         const collection = userType === 'speculator' ? this.speculatorHistoryCollectionName : this.normalHistoryCollectionName;
@@ -151,12 +168,22 @@ class FirestoreBatchManager {
             delete this.speculatorTimestampFixBatch[docPath];
         }
 
-        if (Object.keys(this.usernameMapUpdates).length) {
-            firestoreBatch.set(this.db.collection(this.usernameMapCollectionName).doc('cid_map_shard_1'), this.usernameMapUpdates, { merge: true });
-            batchOps++;
-            this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(this.usernameMapUpdates).length} username map updates.`);
-            this.usernameMapUpdates = {};
+        // --- START FIX: MODIFIED FOR SHARDING ---
+        // Loop over each shardId in the updates object
+        for (const shardId in this.usernameMapUpdates) {
+            const updates = this.usernameMapUpdates[shardId];
+            if (updates && Object.keys(updates).length > 0) {
+                firestoreBatch.set(
+                    this.db.collection(this.usernameMapCollectionName).doc(shardId),
+                    updates,
+                    { merge: true }
+                );
+                batchOps++;
+                this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(updates).length} username updates to ${shardId}.`);
+            }
         }
+        this.usernameMapUpdates = {}; // Clear updates after staging them
+        // --- END FIX ---
 
         if (this.processedSpeculatorCids.size) {
             const cids = Array.from(this.processedSpeculatorCids);
@@ -180,4 +207,4 @@ class FirestoreBatchManager {
     }
 }
 
-module.exports = { FirestoreBatchManager };
+module.exports = { FirestoreBatchManager };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.135",
+  "version": "1.0.137",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [