bulltrackers-module 1.0.126 → 1.0.128

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -4,373 +4,180 @@
  * OPTIMIZED: Added logic to handle speculator timestamp fixes within the batch.
  * --- MODIFIED: Added username map caching and trading history batching. ---
  */
-
 const { FieldValue } = require('@google-cloud/firestore');
-// No longer requires logger from sharedsetup, it's passed in.
 
 class FirestoreBatchManager {
-    /**
-     * @param {Firestore} db - A Firestore instance.
-     * @param {IntelligentHeaderManager} headerManager - An IntelligentHeaderManager instance.
-     * @param {object} logger - A logger instance.
-     * @param {object} config - Configuration object.
-     */
     constructor(db, headerManager, logger, config) {
-        this.db = db; // Renamed from firestore
+        this.db = db;
         this.headerManager = headerManager;
-        this.logger = logger; // Added logger
+        this.logger = logger;
         this.config = config;
 
         this.portfolioBatch = {};
         this.timestampBatch = {};
-        this.processedSpeculatorCids = new Set();
+        this.tradingHistoryBatch = {};
         this.speculatorTimestampFixBatch = {};
-        
-        // --- NEW STATE ---
-        this.tradingHistoryBatch = {}; // For the new trade history data
-        this.usernameMap = new Map(); // In-memory cache: Map<cid, username>
-        this.usernameMapUpdates = {}; // Batched updates for Firestore: { [cid]: { username } }
+        this.usernameMap = new Map();
+        this.usernameMapUpdates = {};
         this.usernameMapLastLoaded = 0;
+        this.processedSpeculatorCids = new Set();
+
         this.usernameMapCollectionName = config.FIRESTORE_COLLECTION_USERNAME_MAP;
         this.normalHistoryCollectionName = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
         this.speculatorHistoryCollectionName = config.FIRESTORE_COLLECTION_SPECULATOR_HISTORY;
-        // --- END NEW STATE ---
 
         this.batchTimeout = null;
 
-        this.logger.log('INFO', 'FirestoreBatchManager initialized.');
+        logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
-    /**
-     * Schedules a flush if one isn't already scheduled.
-     */
     _scheduleFlush() {
-        if (!this.batchTimeout) {
-            this.batchTimeout = setTimeout(
-                () => this.flushBatches(),
-                this.config.TASK_ENGINE_FLUSH_INTERVAL_MS
-            );
-        }
+        if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS);
     }
 
-    // --- NEW: Load the username map into memory ---
     async loadUsernameMap() {
-        // Cache map for 1 hour to reduce reads
-        if (Date.now() - this.usernameMapLastLoaded < 3600000) {
-            return;
-        }
-
-        this.logger.log('INFO', '[BATCH] Refreshing username map from Firestore...');
+        if (Date.now() - this.usernameMapLastLoaded < 3600000) return;
         this.usernameMap.clear();
-        
+        this.logger.log('INFO', '[BATCH] Refreshing username map from Firestore...');
         try {
             const snapshot = await this.db.collection(this.usernameMapCollectionName).get();
-            if (snapshot.empty) {
-                this.logger.log('WARN', '[BATCH] Username map collection is empty.');
-            } else {
-                snapshot.forEach(doc => {
-                    const data = doc.data();
-                    // Assumes doc.id is CID, or data is { [cid]: { username } }
-                    for (const cid in data) {
-                        if (data[cid] && data[cid].username) {
-                            this.usernameMap.set(String(cid), data[cid].username);
-                        }
-                    }
-                });
-            }
+            snapshot.forEach(doc => {
+                const data = doc.data();
+                for (const cid in data) if (data[cid]?.username) this.usernameMap.set(String(cid), data[cid].username);
+            });
             this.usernameMapLastLoaded = Date.now();
-            this.logger.log('INFO', `[BATCH] Loaded ${this.usernameMap.size} usernames into memory cache.`);
-        } catch (error) {
-            this.logger.log('ERROR', '[BATCH] Failed to load username map.', { errorMessage: error.message });
+            this.logger.log('INFO', `[BATCH] Loaded ${this.usernameMap.size} usernames.`);
+        } catch (e) {
+            this.logger.log('ERROR', '[BATCH] Failed to load username map.', { errorMessage: e.message });
         }
     }
 
-    // --- NEW: Get username from in-memory cache ---
-    getUsername(cid) {
-        return this.usernameMap.get(String(cid));
-    }
+    getUsername(cid) { return this.usernameMap.get(String(cid)); }
 
-    // --- NEW: Add a username to the cache and the write batch ---
     addUsernameMapUpdate(cid, username) {
-        const cidStr = String(cid);
         if (!username) return;
-
-        // Update in-memory cache immediately
+        const cidStr = String(cid);
         this.usernameMap.set(cidStr, username);
-        
-        // Add to Firestore write batch
-        // We'll write this to a single-field doc for simple merging
-        this.usernameMapUpdates[cidStr] = { username: username };
+        this.usernameMapUpdates[cidStr] = { username };
         this.logger.log('TRACE', `[BATCH] Queued username update for ${cidStr}.`);
         this._scheduleFlush();
     }
-    
-    // --- NEW: Add trading history to its own batch ---
-    async addToTradingHistoryBatch(userId, blockId, date, historyData, userType) {
-        const collection = userType === 'speculator'
-            ? this.speculatorHistoryCollectionName
-            : this.normalHistoryCollectionName;
-        
-        const basePath = `${collection}/${blockId}/snapshots/${date}`;
 
-        if (!this.tradingHistoryBatch[basePath]) {
-            this.tradingHistoryBatch[basePath] = {};
-        }
-
-        this.tradingHistoryBatch[basePath][userId] = historyData;
-
-        // Trigger flush based on portfolio batch size, assuming they'll be roughly equal
-       const totalUsersInBatch = Object.values(this.tradingHistoryBatch).reduce((sum, users) => sum + Object.keys(users).length, 0);
-
-        if (totalUsersInBatch >= this.config.TASK_ENGINE_MAX_BATCH_SIZE) {
-            // DO NOT await a flush. Just schedule one.
-            this._scheduleFlush();
-        } else {
-            this._scheduleFlush();
-        }
+    async addToTradingHistoryBatch(userId, blockId, date, historyData, userType) {
+        const collection = userType === 'speculator' ? this.speculatorHistoryCollectionName : this.normalHistoryCollectionName;
+        const path = `${collection}/${blockId}/snapshots/${date}`;
+        this.tradingHistoryBatch[path] ??= {};
+        this.tradingHistoryBatch[path][userId] = historyData;
+        this._scheduleFlush();
     }
 
-    /**
-     * Adds a portfolio to the batch.
-     * @param {string} userId 
-     * @param {string} blockId 
-     * @param {string} date
-     * @param {object} portfolioData 
-     * @param {string} userType 
-     * @param {string|null} instrumentId 
-     */
-    async addToPortfolioBatch(userId, blockId, date, portfolioData, userType, instrumentId = null) {
-        const collection = userType === 'speculator'
-            ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS
-            : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
-        const basePath = `${collection}/${blockId}/snapshots/${date}`;
-
-        if (!this.portfolioBatch[basePath]) {
-            this.portfolioBatch[basePath] = {};
-        }
-
-        this.portfolioBatch[basePath][userId] = portfolioData;
-
-        const totalUsersInBatch = Object.values(this.portfolioBatch).reduce((sum, users) => sum + Object.keys(users).length, 0);
-
-        if (totalUsersInBatch >= this.config.TASK_ENGINE_MAX_BATCH_SIZE) {
-            // DO NOT await a flush. Just schedule one.
-            this._scheduleFlush();
-        } else {
-            this._scheduleFlush();
-        }
+    async addToPortfolioBatch(userId, blockId, date, portfolioData, userType) {
+        const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
+        const path = `${collection}/${blockId}/snapshots/${date}`;
+        this.portfolioBatch[path] ??= {};
+        this.portfolioBatch[path][userId] = portfolioData;
+        this._scheduleFlush();
     }
 
-    /**
-     * Adds a user timestamp update to the batch.
-     * @param {string} userId 
-     * @param {string} userType 
-     * @param {string|null} instrumentId 
-     */
     async updateUserTimestamp(userId, userType, instrumentId = null) {
-        const collection = userType === 'speculator'
-            ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS
-            : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
-        const docId = userType === 'speculator' ? 'speculators' : 'normal';
-        const docPath = `${collection}/${docId}`;
-
-        if (!this.timestampBatch[docPath]) {
-            this.timestampBatch[docPath] = {};
-        }
-
-        const timestampKey = userType === 'speculator' ? `${userId}_${instrumentId}` : userId;
-        this.timestampBatch[docPath][timestampKey] = new Date();
-
-        if (Object.keys(this.timestampBatch[docPath]).length >= this.config.TASK_ENGINE_MAX_BATCH_SIZE) {
-        // DO NOT await a flush. Just schedule one.
-            this._scheduleFlush();
-        } else {
-            this._scheduleFlush();
-        }
+        const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
+        const docPath = `${collection}/${userType === 'speculator' ? 'speculators' : 'normal'}`;
+        this.timestampBatch[docPath] ??= {};
+        const key = userType === 'speculator' ? `${userId}_${instrumentId}` : userId;
+        this.timestampBatch[docPath][key] = new Date();
+        this._scheduleFlush();
     }
 
-    /**
-     * Removes a user timestamp from the batch (e.g., if user is private).
-     * @param {string} userId 
-     * @param {string} userType 
-     * @param {string|null} instrumentId 
-     */
     deleteFromTimestampBatch(userId, userType, instrumentId) {
-        const collection = userType === 'speculator'
-            ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS
-            : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
-        const docId = userType === 'speculator' ? 'speculators' : 'normal';
-        const docPath = `${collection}/${docId}`;
-
+        const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
+        const docPath = `${collection}/${userType === 'speculator' ? 'speculators' : 'normal'}`;
         if (this.timestampBatch[docPath]) {
-            const timestampKey = userType === 'speculator' ? `${userId}_${instrumentId}` : userId;
-            delete this.timestampBatch[docPath][timestampKey];
+            const key = userType === 'speculator' ? `${userId}_${instrumentId}` : userId;
+            delete this.timestampBatch[docPath][key];
         }
     }
 
-    /**
-     * Adds discovered speculator CIDs to the in-memory set for later deletion.
-     * @param {Array<string>} cids 
-     */
-    addProcessedSpeculatorCids(cids) {
-        cids.forEach(cid => this.processedSpeculatorCids.add(cid));
-    }
+    addProcessedSpeculatorCids(cids) { cids.forEach(cid => this.processedSpeculatorCids.add(cid)); }
 
-    /**
-     * --- OPTIMIZATION: ADD THIS NEW METHOD ---
-     * Adds a speculator timestamp fix to the batch.
-     * @param {string} userId
-     * @param {string} orchestratorBlockId (e.g., "1000000")
-     */
     async addSpeculatorTimestampFix(userId, orchestratorBlockId) {
         const docPath = `${this.config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS}/${orchestratorBlockId}`;
-
-        if (!this.speculatorTimestampFixBatch[docPath]) {
-            this.speculatorTimestampFixBatch[docPath] = {};
-        }
-
-        // These are the two fields the orchestrator reads
+        this.speculatorTimestampFixBatch[docPath] ??= {};
         this.speculatorTimestampFixBatch[docPath][`users.${userId}.lastVerified`] = new Date();
         this.speculatorTimestampFixBatch[docPath][`users.${userId}.lastHeldSpeculatorAsset`] = new Date();
-        
-        this.logger.log('TRACE', `[BATCH] Queued speculator timestamp fix for user ${userId} in block ${orchestratorBlockId}`);
-        this._scheduleFlush(); // Schedule a flush
+        this.logger.log('TRACE', `[BATCH] Queued speculator timestamp fix for ${userId} in block ${orchestratorBlockId}`);
+        this._scheduleFlush();
     }
-    // --- END NEW METHOD ---
 
-    /**
-     * Helper to flush a specific batch type (portfolio or history).
-     * @param {object} batchData - The batch object (e.g., this.portfolioBatch)
-     * @param {Firestore.Batch} firestoreBatch - The main Firestore batch object
-     * @param {string} logName - A name for logging (e.g., "Portfolio")
-     * @returns {number} The number of batch operations added.
-     */
     _flushDataBatch(batchData, firestoreBatch, logName) {
-        let operationCount = 0;
+        let count = 0;
         for (const basePath in batchData) {
-            const userPortfolios = batchData[basePath];
-            const userIds = Object.keys(userPortfolios);
-
-            if (userIds.length > 0) {
-                for (let i = 0; i < userIds.length; i += this.config.TASK_ENGINE_MAX_USERS_PER_SHARD) {
-                    const chunkUserIds = userIds.slice(i, i + this.config.TASK_ENGINE_MAX_USERS_PER_SHARD);
-                    
-                    const chunkData = {};
-                    chunkUserIds.forEach(userId => {
-                        chunkData[userId] = userPortfolios[userId];
-                    });
-
-                    const docRef = this.db.collection(`${basePath}/parts`).doc(); // Use random ID
-                    firestoreBatch.set(docRef, chunkData);
-                    operationCount++;
-                }
-                this.logger.log('INFO', `[BATCH] Staged ${userIds.length} users' ${logName} data into ${Math.ceil(userIds.length / this.config.TASK_ENGINE_MAX_USERS_PER_SHARD)} shards for ${basePath}.`);
+            const users = batchData[basePath];
+            const userIds = Object.keys(users);
+            if (!userIds.length) continue;
+            for (let i = 0; i < userIds.length; i += this.config.TASK_ENGINE_MAX_USERS_PER_SHARD) {
+                const chunkData = Object.fromEntries(userIds.slice(i, i + this.config.TASK_ENGINE_MAX_USERS_PER_SHARD).map(id => [id, users[id]]));
+                firestoreBatch.set(this.db.collection(`${basePath}/parts`).doc(), chunkData);
+                count++;
             }
+            this.logger.log('INFO', `[BATCH] Staged ${userIds.length} ${logName} users in ${Math.ceil(userIds.length / this.config.TASK_ENGINE_MAX_USERS_PER_SHARD)} shards for ${basePath}.`);
             delete batchData[basePath];
         }
-        return operationCount;
+        return count;
     }
 
-
-    /**
-     * Flushes all pending writes to Firestore and updates header performance.
-     */
     async flushBatches() {
-        if (this.batchTimeout) {
-            clearTimeout(this.batchTimeout);
-            this.batchTimeout = null;
-        }
+        if (this.batchTimeout) { clearTimeout(this.batchTimeout); this.batchTimeout = null; }
 
-        const promises = [];
-        const firestoreBatch = this.db.batch(); // Use this.db
-        let batchOperationCount = 0;
+        const firestoreBatch = this.db.batch();
+        let batchOps = 0;
 
-        // --- REFACTOR: Use helper for both batches ---
-        batchOperationCount += this._flushDataBatch(this.portfolioBatch, firestoreBatch, "Portfolio");
-        batchOperationCount += this._flushDataBatch(this.tradingHistoryBatch, firestoreBatch, "Trade History");
-        // --- END REFACTOR ---
+        batchOps += this._flushDataBatch(this.portfolioBatch, firestoreBatch, 'Portfolio');
+        batchOps += this._flushDataBatch(this.tradingHistoryBatch, firestoreBatch, 'Trade History');
 
         for (const docPath in this.timestampBatch) {
-            if (Object.keys(this.timestampBatch[docPath]).length > 0) {
-                const [collection] = docPath.split('/');
-                const docRef = this.db.collection(collection).doc('timestamps').collection('users').doc('normal'); // Use this.db
-
-                if (batchOperationCount < 450) { 
-                    firestoreBatch.set(docRef, { users: this.timestampBatch[docPath] }, { merge: true });
-                    batchOperationCount++;
-                } else {
-                    promises.push(firestoreBatch.commit());
-                    const newBatch = this.db.batch(); // Use this.db
-                    newBatch.set(docRef, { users: this.timestampBatch[docPath] }, { merge: true });
-                    promises.push(newBatch.commit());
-                    batchOperationCount = 1;
-                }
-                delete this.timestampBatch[docPath];
-            }
-        }
-
-        if (this.processedSpeculatorCids.size > 0) {
-            this.logger.log('INFO', `[BATCH] Flushing ${this.processedSpeculatorCids.size} processed speculator CIDs from pending list documents.`);
-            const cidsToDelete = Array.from(this.processedSpeculatorCids);
-            this.processedSpeculatorCids.clear(); 
-
-            const pendingDocsSnapshot = await this.db.collection(this.config.PENDING_SPECULATORS_COLLECTION).get(); // Use this.db
-
-            if (!pendingDocsSnapshot.empty) {
-                const deletePromises = [];
-                pendingDocsSnapshot.forEach(doc => {
-                    const docData = doc.data().users || {};
-                    const cidsInThisDoc = cidsToDelete.filter(cid => docData.hasOwnProperty(cid));
-
-                    if (cidsInThisDoc.length > 0) {
-                        const deleteBatch = this.db.batch(); // Use this.db
-                        const updates = {};
-                        cidsInThisDoc.forEach(cid => {
-                            updates[`users.${cid}`] = FieldValue.delete();
-                        });
-                        deleteBatch.update(doc.ref, updates);
-                        deletePromises.push(deleteBatch.commit());
-                        this.logger.log('INFO', `[BATCH] Staged deletion of ${cidsInThisDoc.length} CIDs from document: ${doc.id}`);
-                    }
-                });
-                promises.push(...deletePromises);
-            }
+            const timestamps = this.timestampBatch[docPath];
+            if (!Object.keys(timestamps).length) continue;
+            const docRef = this.db.collection(docPath.split('/')[0]).doc('timestamps').collection('users').doc('normal');
+            firestoreBatch.set(docRef, { users: timestamps }, { merge: true });
+            batchOps++;
+            delete this.timestampBatch[docPath];
         }
 
-        // --- OPTIMIZATION: ADD THIS LOOP ---
         for (const docPath in this.speculatorTimestampFixBatch) {
             const updates = this.speculatorTimestampFixBatch[docPath];
-            if (Object.keys(updates).length > 0) {
-                const docRef = this.db.doc(docPath);
-                
-                // Add to the existing batch
-                firestoreBatch.set(docRef, updates, { merge: true });
-                batchOperationCount++;
-            }
+            if (!Object.keys(updates).length) continue;
+            firestoreBatch.set(this.db.doc(docPath), updates, { merge: true });
+            batchOps++;
             delete this.speculatorTimestampFixBatch[docPath];
         }
-        
-        // --- NEW: Flush Username Map Updates ---
-        if (Object.keys(this.usernameMapUpdates).length > 0) {
-            this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(this.usernameMapUpdates).length} username map updates.`);
-            // Simple sharding: just merge into one doc for now.
-            // A more robust solution would shard based on CID.
-            const mapDocRef = this.db.collection(this.usernameMapCollectionName).doc('cid_map_shard_1');
-            firestoreBatch.set(mapDocRef, this.usernameMapUpdates, { merge: true });
-            batchOperationCount++;
-            this.usernameMapUpdates = {}; // Clear the batch
-        }
-        // --- END NEW ---
 
-        if (batchOperationCount > 0) {
-            promises.push(firestoreBatch.commit());
-        }
-
-        promises.push(this.headerManager.flushPerformanceUpdates());
-
-        await Promise.all(promises);
+        if (Object.keys(this.usernameMapUpdates).length) {
+            firestoreBatch.set(this.db.collection(this.usernameMapCollectionName).doc('cid_map_shard_1'), this.usernameMapUpdates, { merge: true });
+            batchOps++;
+            this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(this.usernameMapUpdates).length} username map updates.`);
+            this.usernameMapUpdates = {};
+        }
+
+        if (this.processedSpeculatorCids.size) {
+            const cids = Array.from(this.processedSpeculatorCids);
+            this.processedSpeculatorCids.clear();
+            const snapshot = await this.db.collection(this.config.PENDING_SPECULATORS_COLLECTION).get();
+            snapshot.forEach(doc => {
+                const docData = doc.data().users || {};
+                const cidsInDoc = cids.filter(cid => docData[cid]);
+                if (!cidsInDoc.length) return;
+                const delBatch = this.db.batch();
+                const updates = Object.fromEntries(cidsInDoc.map(cid => [`users.${cid}`, FieldValue.delete()]));
+                delBatch.update(doc.ref, updates);
+                delBatch.commit();
+                this.logger.log('INFO', `[BATCH] Deleted ${cidsInDoc.length} CIDs from ${doc.id}`);
+            });
+        }
+
+        if (batchOps) await firestoreBatch.commit();
+        await this.headerManager.flushPerformanceUpdates();
         this.logger.log('INFO', '[BATCH] All batches flushed successfully.');
     }
 }
 
-module.exports = { FirestoreBatchManager };
+module.exports = { FirestoreBatchManager };

package/functions/task-engine/utils/task_engine_utils.js

@@ -0,0 +1,99 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
+ * (MODIFIED: To pass down a Set to track history fetches)
+ */
+
+/**
+ * @fileoverview Utility functions for the Task Engine handler.
+ * This file contains the "how" - the logic for parsing, sorting,
+ * and orchestrating task execution.
+ */
+
+const { handleDiscover } = require('../helpers/discover_helpers');
+const { handleVerify } = require('../helpers/verify_helpers');
+const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
+
+/**
+ * Parses Pub/Sub message into task array.
+ */
+function parseTaskPayload(message, logger) {
+    if (!message?.data) return logger.log('ERROR', '[TaskEngine] Invalid message structure.', { message }) || null;
+    let payload;
+    try { payload = JSON.parse(Buffer.from(message.data, 'base64').toString()); }
+    catch (e) { logger.log('ERROR', '[TaskEngine] Failed to parse message', { error: e.message }); return null; }
+    if (!Array.isArray(payload.tasks) || payload.tasks.length === 0) {
+        logger.log(payload.tasks?.length ? 'WARN' : 'ERROR', '[TaskEngine] Invalid or empty tasks array.', { payload });
+        return null;
+    }
+    return payload.tasks;
+}
+
+/**
+ * Sorts tasks into update (with username), lookup (missing username), and other (discover/verify).
+ */
+async function prepareTaskBatches(tasks, batchManager, logger) {
+    const tasksToRun = [], cidsToLookup = new Map(), otherTasks = [];
+    await batchManager.loadUsernameMap();
+    for (const task of tasks) {
+        if (task.type === 'update') {
+            const username = batchManager.getUsername(task.userId);
+            username ? tasksToRun.push({ task, username }) : cidsToLookup.set(String(task.userId), task);
+        } else otherTasks.push(task);
+    }
+    logger.log('INFO', `[TaskEngine] Sorting complete. Known: ${tasksToRun.length}, Lookup: ${cidsToLookup.size}, Other: ${otherTasks.length}`);
+    return { tasksToRun, cidsToLookup, otherTasks };
+}
+
+/**
+ * Runs username lookups for missing CIDs and adds to tasksToRun.
+ */
+async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config, batchManager, logger) {
+    if (!cidsToLookup.size) return;
+    logger.log('INFO', `[TaskEngine] Looking up ${cidsToLookup.size} usernames...`);
+    const foundUsers = await lookupUsernames([...cidsToLookup.keys()], dependencies, config);
+    for (const u of foundUsers) {
+        const cid = String(u.CID), username = u.Value.UserName;
+        batchManager.addUsernameMapUpdate(cid, username);
+        const task = cidsToLookup.get(cid);
+        if (task) { tasksToRun.push({ task, username }); cidsToLookup.delete(cid); }
+    }
+    if (cidsToLookup.size) logger.log('WARN', `[TaskEngine] Could not find ${cidsToLookup.size} usernames (likely private).`, { skippedCids: [...cidsToLookup.keys()] });
+}
+
+/**
+ * Executes all tasks.
+ */
+async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
+    const { logger } = dependencies;
+    
+    // --- START MODIFICATION ---
+    // This Set will track history fetches *only for this batch*.
+    const historyFetchedForUser = new Set();
+    // --- END MODIFICATION ---
+
+
+    // Process discover/verify tasks
+    for (const task of otherTasks) {
+        const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
+        const handler = { discover: handleDiscover, verify: handleVerify }[task.type];
+        if (handler) try { await handler(task, subTaskId, dependencies, config); } 
+        catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${subTaskId}`, { errorMessage: err.message }); }
+        else logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type: ${task.type}`);
+    }
+
+    // Process update tasks (all have usernames)
+    for (const { task, username } of tasksToRun) {
+        const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
+        try {
+            // --- START MODIFICATION ---
+            // Pass the Set to the update handler
+            await handleUpdate(task, subTaskId, dependencies, config, username, historyFetchedForUser);
+            // --- END MODIFICATION ---
+        }
+        catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message }); }
+    }
+
+    logger.log('SUCCESS', `[TaskEngine/${taskId}] Processed all tasks.`);
+}
+
+module.exports = { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.126",
+  "version": "1.0.128",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [