bulltrackers-module 1.0.105 → 1.0.106

package/functions/core/utils/firestore_utils.js

@@ -1,433 +1,433 @@
-/**
- * @fileoverview Core Firestore utility functions.
- * REFACTORED: All functions are now stateless and receive dependencies.
- * 'db' (Firestore instance) and 'logger' are passed via a 'dependencies' object.
- */
-const { FieldValue, FieldPath } = require('@google-cloud/firestore');
-
-/**
- * Fetches and merges the most recent portfolio data for all normal users.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.normalUserCollectionName - e.g., 'NormalUserPortfolios'.
- * @param {string} config.snapshotsSubCollectionName - e.g., 'snapshots'.
- * @param {string} config.partsSubCollectionName - e.g., 'parts'.
- * @returns {Promise<object>} A single object containing all user portfolios.
- */
-async function getLatestNormalUserPortfolios(dependencies, config) {
-    const { db, logger } = dependencies;
-    const { normalUserCollectionName, snapshotsSubCollectionName, partsSubCollectionName } = config;
-
-    logger.log('INFO', `[Core Utils] Fetching latest portfolios from ${normalUserCollectionName}...`);
-    const allPortfolios = {};
-
-    const yesterday = new Date();
-    yesterday.setDate(yesterday.getDate() - 1);
-    const dateString = yesterday.toISOString().slice(0, 10);
-
-    const blockDocs = await db.collection(normalUserCollectionName).listDocuments();
-
-    for (const blockDoc of blockDocs) {
-        const snapshotDocRef = blockDoc.collection(snapshotsSubCollectionName).doc(dateString);
-        const partsCollectionRef = snapshotDocRef.collection(partsSubCollectionName);
-        const partsSnapshot = await partsCollectionRef.get();
-
-        if (!partsSnapshot.empty) {
-            partsSnapshot.forEach(partDoc => {
-                Object.assign(allPortfolios, partDoc.data());
-            });
-        }
-    }
-
-    logger.log('INFO', `[Core Utils] Found ${Object.keys(allPortfolios).length} user portfolios from ${dateString}'s snapshot.`);
-    return allPortfolios;
-}
-
-/**
- * Resets the proxy locks map in the performance document.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.proxyPerformanceDocPath - e.g., 'system_state/proxy_performance'.
- * @returns {Promise<void>}
- */
-async function resetProxyLocks(dependencies, config) {
-    const { db, logger } = dependencies;
-    const { proxyPerformanceDocPath } = config;
-
-    logger.log('INFO','[Core Utils] Resetting proxy locks...');
-    try {
-        if (!proxyPerformanceDocPath) {
-            logger.log('ERROR', '[Core Utils] Missing proxyPerformanceDocPath. Cannot reset locks.');
-            return;
-        }
-        
-        const perfDocRef = db.doc(proxyPerformanceDocPath);
-
-        await perfDocRef.update({
-            locks: FieldValue.delete() 
-        });
-
-        logger.log('INFO',`[Core Utils] Proxy locks map reset in ${proxyPerformanceDocPath}.`);
-    } catch (error) {
-        if (error.code === 5) { // 5 = NOT_FOUND
-             logger.log('WARN',`[Core Utils] Proxy performance doc or 'locks' field not found at ${proxyPerformanceDocPath}. No locks to reset.`);
-        } else {
-            logger.log('ERROR','[Core Utils] Error resetting proxy locks', { errorMessage: error.message, path: proxyPerformanceDocPath });
-        }
-    }
-}
-
-/**
- * Fetches and aggregates block capacities.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} userType - 'normal' or 'speculator'.
- * @param {string} config.speculatorBlockCountsDocPath - Path to speculator counts doc.
- * @param {string} config.normalBlockCountsDocPath - Path to normal counts doc.
- * @returns {Promise<object>} Object containing counts.
- */
-async function getBlockCapacities(dependencies, config, userType) {
-    const { db, logger } = dependencies;
-    const { speculatorBlockCountsDocPath, normalBlockCountsDocPath } = config;
-
-    logger.log('INFO',`[Core Utils] Getting block capacities for ${userType}...`);
-    try {
-        const docPath = userType === 'speculator'
-            ? speculatorBlockCountsDocPath
-            : normalBlockCountsDocPath;
-            
-        if (!docPath) {
-             logger.log('ERROR', `[Core Utils] Missing block counts document path for ${userType}.`);
-             return {};
-        }
-
-        const countsRef = db.doc(docPath);
-        const countsDoc = await countsRef.get();
-        if (!countsDoc.exists) {
-            logger.log('WARN',`[Core Utils] Block counts document not found for ${userType} at ${docPath}. Returning empty.`);
-            return {};
-        }
-        return countsDoc.data().counts || {};
-    } catch (error) {
-        logger.log('ERROR',`[Core Utils] Error getting block capacities for ${userType}`, { errorMessage: error.message });
-        throw error;
-    }
-}
-
-/**
- * Fetches user IDs from multiple sources for exclusion lists.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} userType - 'normal' or 'speculator'.
- * @param {string} config.specBlocksCollection - e.g., 'SpeculatorBlocks'.
- * @param {string} config.pendingSpecCollection - e.g., 'PendingSpeculators'.
- * @param {string} config.invalidSpecCollection - e.g., 'InvalidSpeculators'.
- * @param {Set<string>} config.existingNormalUserIds - PRE-FETCHED normal user IDs.
- * @returns {Promise<Set<string>>} A Set containing unique user IDs.
- */
-async function getExclusionIds(dependencies, config, userType) {
-    const { db, logger } = dependencies;
-    const { 
-        specBlocksCollection, 
-        pendingSpecCollection, 
-        invalidSpecCollection,
-        existingNormalUserIds // Get the pre-fetched IDs
-    } = config;
-    
-    logger.log('INFO',`[Core Utils] Getting exclusion IDs for ${userType} discovery...`);
-    
-    const exclusionIds = new Set(existingNormalUserIds);
-    logger.log('TRACE', `[Core Utils] Loaded ${exclusionIds.size} existing normal user IDs for exclusion.`);
-    
-    const promises = [];
-
-    try {
-        // 1. Existing Speculators
-        const specBlocksRef = db.collection(specBlocksCollection);
-        promises.push(specBlocksRef.get().then(snapshot => {
-            snapshot.forEach(doc => {
-                 const users = doc.data().users || {};
-                 Object.keys(users).forEach(key => exclusionIds.add(key.split('.')[1]));
-            });
-            logger.log('TRACE','[Core Utils] Fetched existing speculator IDs for exclusion.');
-        }));
-
-        // 2. Pending Speculators
-        if (userType === 'speculator') {
-            const pendingRef = db.collection(pendingSpecCollection);
-             promises.push(pendingRef.get().then(snapshot => {
-                snapshot.forEach(doc => {
-                    Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));
-                });
-                logger.log('TRACE','[Core Utils] Fetched pending speculator IDs for exclusion.');
-             }));
-        }
-
-        // 3. Invalid Speculators
-        const invalidRef = db.collection(invalidSpecCollection);
-         promises.push(invalidRef.get().then(snapshot => {
-             snapshot.forEach(doc => {
-                 const data = doc.data();
-                 if (data) {
-                    Object.keys(data.users || {}).forEach(cid => exclusionIds.add(cid));
-                 }
-             });
-              logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');
-         }));
-
-        await Promise.all(promises);
-        logger.log('INFO',`[Core Utils] Total unique exclusion IDs found: ${exclusionIds.size}`);
-        return exclusionIds;
-
-    } catch (error)
- {
-        logger.log('ERROR','[Core Utils] Error getting exclusion IDs', { errorMessage: error.message });
-        throw error;
-    }
-}
-
-/**
- * Scans normal user portfolios for potential speculator candidates.
- * @param {object} dependencies - Contains db, logger.
- * @param {Set<string>} exclusionIds - IDs to exclude.
- * @param {Set<number>} speculatorInstrumentSet - Set of instrument IDs.
- * @param {object} latestNormalPortfolios - PRE-FETCHED portfolio object.
- * @returns {Promise<string[]>} Array of prioritized speculator CIDs.
- */
-async function getPrioritizedSpeculators(dependencies, exclusionIds, speculatorInstrumentSet, latestNormalPortfolios) {
-    const { logger } = dependencies;
-    logger.log('INFO','[Core Utils] Scanning normal users for prioritized speculators...');
-    const candidates = new Set();
-
-    try {
-        for (const userId in latestNormalPortfolios) {
-            if (exclusionIds.has(userId)) continue;
-
-            const portfolio = latestNormalPortfolios[userId];
-            const holdsSpeculatorAsset = portfolio?.AggregatedPositions?.some(p =>
-                speculatorInstrumentSet.has(p.InstrumentID)
-            );
-
-            if (holdsSpeculatorAsset) {
-                candidates.add(userId);
-            }
-        }
-        logger.log('INFO',`[Core Utils] Found ${candidates.size} potential prioritized speculators.`);
-        return Array.from(candidates);
-    } catch (error) {
-        logger.log('ERROR','[Core Utils] Error getting prioritized speculators', { errorMessage: error.message });
-        throw error;
-    }
-}
-
-/**
- * Deletes all documents within a specified collection path.
- * @param {object} dependencies - Contains db, logger.
- * @param {string} collectionPath - The path to the collection to clear.
- * @param {number} maxBatchSize - Firestore batch size limit.
- * @returns {Promise<void>}
- */
-async function clearCollection(dependencies, collectionPath, maxBatchSize = 400) {
-    const { db, logger } = dependencies;
-    logger.log('WARN', `[Core Utils] Starting SCORCHED EARTH delete for collection: ${collectionPath}...`);
-    try {
-        const collectionRef = db.collection(collectionPath);
-        let query = collectionRef.limit(maxBatchSize);
-        let snapshot;
-        let deleteCount = 0;
-
-        while (true) {
-            snapshot = await query.get();
-            if (snapshot.size === 0) {
-                break; 
-            }
-
-            const batch = db.batch();
-            snapshot.docs.forEach(doc => batch.delete(doc.ref));
-            await batch.commit();
-            deleteCount += snapshot.size;
-
-            if (snapshot.size < maxBatchSize) {
-                break; 
-            }
-             query = collectionRef.limit(maxBatchSize); 
-        }
-        logger.log('SUCCESS', `[Core Utils] Scorched earth complete. Deleted ${deleteCount} documents from ${collectionPath}.`);
-    } catch (error) {
-        logger.log('ERROR', `[Core Utils] Error clearing collection ${collectionPath}`, { errorMessage: error.message });
-        throw error;
-    }
-}
-
-/**
- * Writes an array of user IDs into sharded Firestore documents.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.collectionPath - Base path for sharded documents (e.g., 'PendingSpeculators').
- * @param {string[]} config.items - The user IDs to write.
- * @param {Date} config.timestamp - Timestamp to associate with each user ID.
- * @param {number} config.maxFieldsPerDoc - Max user IDs per sharded document.
- * @param {number} config.maxWritesPerBatch - Max documents to write per batch commit.
- * @returns {Promise<void>}
- */
-async function batchWriteShardedIds(dependencies, config) {
-    const { db, logger } = dependencies;
-    const {
-        collectionPath,
-        items,
-        timestamp,
-        maxFieldsPerDoc,
-        maxWritesPerBatch
-    } = config;
-
-    logger.log('INFO', `[Core Utils] Batch writing ${items.length} IDs to sharded path: ${collectionPath} (max ${maxFieldsPerDoc}/doc, ${maxWritesPerBatch} docs/batch)...`);
-    if (items.length === 0) return;
-
-    try {
-        const collectionRef = db.collection(collectionPath);
-        let batch = db.batch();
-        let currentDocFields = {};
-        let currentFieldCount = 0;
-        let batchWriteCount = 0;
-        let docCounter = 0;
-
-        for (let i = 0; i < items.length; i++) {
-            const userId = items[i];
-            const key = `users.${userId}`; 
-            currentDocFields[key] = timestamp;
-            currentFieldCount++;
-
-            if (currentFieldCount >= maxFieldsPerDoc || i === items.length - 1) {
-                const docRef = collectionRef.doc(`pending_${docCounter}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`);
-                batch.set(docRef, currentDocFields); 
-                batchWriteCount++;
-
-                currentDocFields = {};
-                currentFieldCount = 0;
-                docCounter++;
-
-                if (batchWriteCount >= maxWritesPerBatch || i === items.length - 1) {
-                    await batch.commit();
-                    batch = db.batch(); 
-                    batchWriteCount = 0;
-                }
-            }
-        }
-         logger.log('SUCCESS', `[Core Utils] Sharded write complete for ${collectionPath}. Created ${docCounter} documents.`);
-    } catch (error) {
-        logger.log('ERROR', `[Core Utils] Error during sharded write to ${collectionPath}`, { errorMessage: error.message });
-        throw error;
-    }
-}
-
-/**
- * Gets normal users whose timestamp indicates they need updating.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {Date} config.dateThreshold - Users processed before this date will be returned.
- * @param {string} config.normalUserCollectionName - e.g., 'NormalUserPortfolios'.
- * @param {string} config.normalUserTimestampsDocId - e.g., 'normal'.
- * @returns {Promise<string[]>} Array of user IDs to update.
- */
-async function getNormalUsersToUpdate(dependencies, config) {
-    const { db, logger } = dependencies;
-    const { dateThreshold, normalUserCollectionName, normalUserTimestampsDocId } = config;
-
-    logger.log('INFO','[Core Utils] Getting normal users to update...');
-    const usersToUpdate = [];
-    try {
-        const timestampDocRef = db.collection(normalUserCollectionName)
-                                 .doc('timestamps')
-                                 .collection('users')
-                                 .doc(normalUserTimestampsDocId); 
-        const timestampDoc = await timestampDocRef.get();
-
-        if (!timestampDoc.exists) {
-            logger.log('WARN',`[Core Utils] Normal user timestamp document not found at ${timestampDocRef.path}.`);
-            return [];
-        }
-
-        const timestamps = timestampDoc.data().users || {};
-        for (const userId in timestamps) {
-            const lastProcessed = timestamps[userId]?.toDate ? timestamps[userId].toDate() : new Date(0);
-            if (lastProcessed < dateThreshold) {
-                usersToUpdate.push(userId);
-            }
-        }
-        logger.log('INFO',`[Core Utils] Found ${usersToUpdate.length} normal users to update.`);
-        return usersToUpdate;
-    } catch (error) {
-        logger.log('ERROR','[Core Utils] Error getting normal users to update', { errorMessage: error.message });
-        throw error;
-    }
-}
-
-/**
- * Gets speculator users/instruments whose data needs updating.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {Date} config.dateThreshold - Verification date threshold.
- * @param {Date} config.gracePeriodThreshold - Last held asset date threshold.
- * @param {string} config.speculatorBlocksCollectionName - e.g., 'SpeculatorBlocks'.
- * @returns {Promise<object[]>} Array of objects like { userId: string, instrumentId: number }.
- */
-async function getSpeculatorsToUpdate(dependencies, config) {
-    const { db, logger } = dependencies;
-    const { dateThreshold, gracePeriodThreshold, speculatorBlocksCollectionName } = config;
-
-    logger.log('INFO','[Core Utils] Getting speculators to update...');
-    const updates = [];
-    try {
-        const blocksRef = db.collection(speculatorBlocksCollectionName);
-        const snapshot = await blocksRef.get();
-
-        if (snapshot.empty) {
-            logger.log('INFO','[Core Utils] No speculator blocks found.');
-            return [];
-        }
-
-        snapshot.forEach(doc => {
-            const blockData = doc.data();
-
-            // Iterate over the document's top-level keys
-            for (const key in blockData) {
-                 // Filter for keys that match the 'users.CID' format
-                 if (!key.startsWith('users.')) continue; 
-
-                 const userId = key.split('.')[1];
-                 if (!userId) continue; // Safety check
-
-                 const userData = blockData[key]; // Get the user's map
-
-                 const lastVerified = userData.lastVerified?.toDate ? userData.lastVerified.toDate() : new Date(0);
-                 const lastHeld = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : new Date(0);
-
-                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) {
-                     if (userData.instruments && Array.isArray(userData.instruments)) {
-                         userData.instruments.forEach(instrumentId => {
-                             updates.push({ userId, instrumentId });
-                         });
-                     }
-                 }
-            }
-        });
-
-        logger.log('INFO',`[Core Utils] Found ${updates.length} speculator user/instrument pairs to update.`);
-        return updates;
-    } catch (error) {
-        logger.log('ERROR','[Core Utils] Error getting speculators to update', { errorMessage: error.message });
-        throw error;
-    }
-}
-
-module.exports = {
-  getLatestNormalUserPortfolios,
-  resetProxyLocks,
-  getBlockCapacities,
-  getExclusionIds,
-  getPrioritizedSpeculators,
-  clearCollection,
-  batchWriteShardedIds,
-  getNormalUsersToUpdate,
-  getSpeculatorsToUpdate,
-};
+/**
+ * @fileoverview Core Firestore utility functions.
+ * REFACTORED: All functions are now stateless and receive dependencies.
+ * 'db' (Firestore instance) and 'logger' are passed via a 'dependencies' object.
+ */
+const { FieldValue, FieldPath } = require('@google-cloud/firestore');
+
+/**
+ * Fetches and merges the most recent portfolio data for all normal users.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {object} config - Configuration object.
+ * @param {string} config.normalUserCollectionName - e.g., 'NormalUserPortfolios'.
+ * @param {string} config.snapshotsSubCollectionName - e.g., 'snapshots'.
+ * @param {string} config.partsSubCollectionName - e.g., 'parts'.
+ * @returns {Promise<object>} A single object containing all user portfolios.
+ */
+async function getLatestNormalUserPortfolios(dependencies, config) {
+    const { db, logger } = dependencies;
+    const { normalUserCollectionName, snapshotsSubCollectionName, partsSubCollectionName } = config;
+
+    logger.log('INFO', `[Core Utils] Fetching latest portfolios from ${normalUserCollectionName}...`);
+    const allPortfolios = {};
+
+    const yesterday = new Date();
+    yesterday.setDate(yesterday.getDate() - 1);
+    const dateString = yesterday.toISOString().slice(0, 10);
+
+    const blockDocs = await db.collection(normalUserCollectionName).listDocuments();
+
+    for (const blockDoc of blockDocs) {
+        const snapshotDocRef = blockDoc.collection(snapshotsSubCollectionName).doc(dateString);
+        const partsCollectionRef = snapshotDocRef.collection(partsSubCollectionName);
+        const partsSnapshot = await partsCollectionRef.get();
+
+        if (!partsSnapshot.empty) {
+            partsSnapshot.forEach(partDoc => {
+                Object.assign(allPortfolios, partDoc.data());
+            });
+        }
+    }
+
+    logger.log('INFO', `[Core Utils] Found ${Object.keys(allPortfolios).length} user portfolios from ${dateString}'s snapshot.`);
+    return allPortfolios;
+}
+
+/**
+ * Resets the proxy locks map in the performance document.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {object} config - Configuration object.
+ * @param {string} config.proxyPerformanceDocPath - e.g., 'system_state/proxy_performance'.
+ * @returns {Promise<void>}
+ */
+async function resetProxyLocks(dependencies, config) {
+    const { db, logger } = dependencies;
+    const { proxyPerformanceDocPath } = config;
+
+    logger.log('INFO','[Core Utils] Resetting proxy locks...');
+    try {
+        if (!proxyPerformanceDocPath) {
+            logger.log('ERROR', '[Core Utils] Missing proxyPerformanceDocPath. Cannot reset locks.');
+            return;
+        }
+        
+        const perfDocRef = db.doc(proxyPerformanceDocPath);
+
+        await perfDocRef.update({
+            locks: FieldValue.delete() 
+        });
+
+        logger.log('INFO',`[Core Utils] Proxy locks map reset in ${proxyPerformanceDocPath}.`);
+    } catch (error) {
+        if (error.code === 5) { // 5 = NOT_FOUND
+             logger.log('WARN',`[Core Utils] Proxy performance doc or 'locks' field not found at ${proxyPerformanceDocPath}. No locks to reset.`);
+        } else {
+            logger.log('ERROR','[Core Utils] Error resetting proxy locks', { errorMessage: error.message, path: proxyPerformanceDocPath });
+        }
+    }
+}
+
+/**
+ * Fetches and aggregates block capacities.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {object} config - Configuration object.
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {string} config.speculatorBlockCountsDocPath - Path to speculator counts doc.
+ * @param {string} config.normalBlockCountsDocPath - Path to normal counts doc.
+ * @returns {Promise<object>} Object containing counts.
+ */
+async function getBlockCapacities(dependencies, config, userType) {
+    const { db, logger } = dependencies;
+    const { speculatorBlockCountsDocPath, normalBlockCountsDocPath } = config;
+
+    logger.log('INFO',`[Core Utils] Getting block capacities for ${userType}...`);
+    try {
+        const docPath = userType === 'speculator'
+            ? speculatorBlockCountsDocPath
+            : normalBlockCountsDocPath;
+            
+        if (!docPath) {
+             logger.log('ERROR', `[Core Utils] Missing block counts document path for ${userType}.`);
+             return {};
+        }
+
+        const countsRef = db.doc(docPath);
+        const countsDoc = await countsRef.get();
+        if (!countsDoc.exists) {
+            logger.log('WARN',`[Core Utils] Block counts document not found for ${userType} at ${docPath}. Returning empty.`);
+            return {};
+        }
+        return countsDoc.data().counts || {};
+    } catch (error) {
+        logger.log('ERROR',`[Core Utils] Error getting block capacities for ${userType}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Fetches user IDs from multiple sources for exclusion lists.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {object} config - Configuration object.
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {string} config.specBlocksCollection - e.g., 'SpeculatorBlocks'.
+ * @param {string} config.pendingSpecCollection - e.g., 'PendingSpeculators'.
+ * @param {string} config.invalidSpecCollection - e.g., 'InvalidSpeculators'.
+ * @param {Set<string>} config.existingNormalUserIds - PRE-FETCHED normal user IDs.
+ * @returns {Promise<Set<string>>} A Set containing unique user IDs.
+ */
+async function getExclusionIds(dependencies, config, userType) {
+    const { db, logger } = dependencies;
+    const { 
+        specBlocksCollection, 
+        pendingSpecCollection, 
+        invalidSpecCollection,
+        existingNormalUserIds // Get the pre-fetched IDs
+    } = config;
+    
+    logger.log('INFO',`[Core Utils] Getting exclusion IDs for ${userType} discovery...`);
+    
+    const exclusionIds = new Set(existingNormalUserIds);
+    logger.log('TRACE', `[Core Utils] Loaded ${exclusionIds.size} existing normal user IDs for exclusion.`);
+    
+    const promises = [];
+
+    try {
+        // 1. Existing Speculators
+        const specBlocksRef = db.collection(specBlocksCollection);
+        promises.push(specBlocksRef.get().then(snapshot => {
+            snapshot.forEach(doc => {
+                 const users = doc.data().users || {};
+                 Object.keys(users).forEach(key => exclusionIds.add(key.split('.')[1]));
+            });
+            logger.log('TRACE','[Core Utils] Fetched existing speculator IDs for exclusion.');
+        }));
+
+        // 2. Pending Speculators
+        if (userType === 'speculator') {
+            const pendingRef = db.collection(pendingSpecCollection);
+             promises.push(pendingRef.get().then(snapshot => {
+                snapshot.forEach(doc => {
+                    Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));
+                });
+                logger.log('TRACE','[Core Utils] Fetched pending speculator IDs for exclusion.');
+             }));
+        }
+
+        // 3. Invalid Speculators
+        const invalidRef = db.collection(invalidSpecCollection);
+         promises.push(invalidRef.get().then(snapshot => {
+             snapshot.forEach(doc => {
+                 const data = doc.data();
+                 if (data) {
+                    Object.keys(data.users || {}).forEach(cid => exclusionIds.add(cid));
+                 }
+             });
+              logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');
+         }));
+
+        await Promise.all(promises);
+        logger.log('INFO',`[Core Utils] Total unique exclusion IDs found: ${exclusionIds.size}`);
+        return exclusionIds;
+
+    } catch (error)
+ {
+        logger.log('ERROR','[Core Utils] Error getting exclusion IDs', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Scans normal user portfolios for potential speculator candidates.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {Set<string>} exclusionIds - IDs to exclude.
+ * @param {Set<number>} speculatorInstrumentSet - Set of instrument IDs.
+ * @param {object} latestNormalPortfolios - PRE-FETCHED portfolio object.
+ * @returns {Promise<string[]>} Array of prioritized speculator CIDs.
+ */
+async function getPrioritizedSpeculators(dependencies, exclusionIds, speculatorInstrumentSet, latestNormalPortfolios) {
+    const { logger } = dependencies;
+    logger.log('INFO','[Core Utils] Scanning normal users for prioritized speculators...');
+    const candidates = new Set();
+
+    try {
+        for (const userId in latestNormalPortfolios) {
+            if (exclusionIds.has(userId)) continue;
+
+            const portfolio = latestNormalPortfolios[userId];
+            const holdsSpeculatorAsset = portfolio?.AggregatedPositions?.some(p =>
+                speculatorInstrumentSet.has(p.InstrumentID)
+            );
+
+            if (holdsSpeculatorAsset) {
+                candidates.add(userId);
+            }
+        }
+        logger.log('INFO',`[Core Utils] Found ${candidates.size} potential prioritized speculators.`);
+        return Array.from(candidates);
+    } catch (error) {
+        logger.log('ERROR','[Core Utils] Error getting prioritized speculators', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Deletes all documents within a specified collection path.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {string} collectionPath - The path to the collection to clear.
+ * @param {number} maxBatchSize - Firestore batch size limit.
+ * @returns {Promise<void>}
+ */
+async function clearCollection(dependencies, collectionPath, maxBatchSize = 400) {
+    const { db, logger } = dependencies;
+    logger.log('WARN', `[Core Utils] Starting SCORCHED EARTH delete for collection: ${collectionPath}...`);
+    try {
+        const collectionRef = db.collection(collectionPath);
+        let query = collectionRef.limit(maxBatchSize);
+        let snapshot;
+        let deleteCount = 0;
+
+        while (true) {
+            snapshot = await query.get();
+            if (snapshot.size === 0) {
+                break; 
+            }
+
+            const batch = db.batch();
+            snapshot.docs.forEach(doc => batch.delete(doc.ref));
+            await batch.commit();
+            deleteCount += snapshot.size;
+
+            if (snapshot.size < maxBatchSize) {
+                break; 
+            }
+             query = collectionRef.limit(maxBatchSize); 
+        }
+        logger.log('SUCCESS', `[Core Utils] Scorched earth complete. Deleted ${deleteCount} documents from ${collectionPath}.`);
+    } catch (error) {
+        logger.log('ERROR', `[Core Utils] Error clearing collection ${collectionPath}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Writes an array of user IDs into sharded Firestore documents.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {object} config - Configuration object.
+ * @param {string} config.collectionPath - Base path for sharded documents (e.g., 'PendingSpeculators').
+ * @param {string[]} config.items - The user IDs to write.
+ * @param {Date} config.timestamp - Timestamp to associate with each user ID.
+ * @param {number} config.maxFieldsPerDoc - Max user IDs per sharded document.
+ * @param {number} config.maxWritesPerBatch - Max documents to write per batch commit.
+ * @returns {Promise<void>}
+ */
+async function batchWriteShardedIds(dependencies, config) {
+    const { db, logger } = dependencies;
+    const {
+        collectionPath,
+        items,
+        timestamp,
+        maxFieldsPerDoc,
+        maxWritesPerBatch
+    } = config;
+
+    logger.log('INFO', `[Core Utils] Batch writing ${items.length} IDs to sharded path: ${collectionPath} (max ${maxFieldsPerDoc}/doc, ${maxWritesPerBatch} docs/batch)...`);
+    if (items.length === 0) return;
+
+    try {
+        const collectionRef = db.collection(collectionPath);
+        let batch = db.batch();
+        let currentDocFields = {};
+        let currentFieldCount = 0;
+        let batchWriteCount = 0;
+        let docCounter = 0;
+
+        for (let i = 0; i < items.length; i++) {
+            const userId = items[i];
+            const key = `users.${userId}`; 
+            currentDocFields[key] = timestamp;
+            currentFieldCount++;
+
+            if (currentFieldCount >= maxFieldsPerDoc || i === items.length - 1) {
+                const docRef = collectionRef.doc(`pending_${docCounter}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`);
+                batch.set(docRef, currentDocFields); 
+                batchWriteCount++;
+
+                currentDocFields = {};
+                currentFieldCount = 0;
+                docCounter++;
+
+                if (batchWriteCount >= maxWritesPerBatch || i === items.length - 1) {
+                    await batch.commit();
+                    batch = db.batch(); 
+                    batchWriteCount = 0;
+                }
+            }
+        }
+         logger.log('SUCCESS', `[Core Utils] Sharded write complete for ${collectionPath}. Created ${docCounter} documents.`);
+    } catch (error) {
+        logger.log('ERROR', `[Core Utils] Error during sharded write to ${collectionPath}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Gets normal users whose timestamp indicates they need updating.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {object} config - Configuration object.
+ * @param {Date} config.dateThreshold - Users processed before this date will be returned.
+ * @param {string} config.normalUserCollectionName - e.g., 'NormalUserPortfolios'.
+ * @param {string} config.normalUserTimestampsDocId - e.g., 'normal'.
+ * @returns {Promise<string[]>} Array of user IDs to update.
+ */
+async function getNormalUsersToUpdate(dependencies, config) {
+    const { db, logger } = dependencies;
+    const { dateThreshold, normalUserCollectionName, normalUserTimestampsDocId } = config;
+
+    logger.log('INFO','[Core Utils] Getting normal users to update...');
+    const usersToUpdate = [];
+    try {
+        const timestampDocRef = db.collection(normalUserCollectionName)
+                                 .doc('timestamps')
+                                 .collection('users')
+                                 .doc(normalUserTimestampsDocId); 
+        const timestampDoc = await timestampDocRef.get();
+
+        if (!timestampDoc.exists) {
+            logger.log('WARN',`[Core Utils] Normal user timestamp document not found at ${timestampDocRef.path}.`);
+            return [];
+        }
+
+        const timestamps = timestampDoc.data().users || {};
+        for (const userId in timestamps) {
+            const lastProcessed = timestamps[userId]?.toDate ? timestamps[userId].toDate() : new Date(0);
+            if (lastProcessed < dateThreshold) {
+                usersToUpdate.push(userId);
+            }
+        }
+        logger.log('INFO',`[Core Utils] Found ${usersToUpdate.length} normal users to update.`);
+        return usersToUpdate;
+    } catch (error) {
+        logger.log('ERROR','[Core Utils] Error getting normal users to update', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Gets speculator users/instruments whose data needs updating.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {object} config - Configuration object.
+ * @param {Date} config.dateThreshold - Verification date threshold.
+ * @param {Date} config.gracePeriodThreshold - Last held asset date threshold.
+ * @param {string} config.speculatorBlocksCollectionName - e.g., 'SpeculatorBlocks'.
+ * @returns {Promise<object[]>} Array of objects like { userId: string, instrumentId: number }.
+ */
+async function getSpeculatorsToUpdate(dependencies, config) {
+    const { db, logger } = dependencies;
+    const { dateThreshold, gracePeriodThreshold, speculatorBlocksCollectionName } = config;
+
+    logger.log('INFO','[Core Utils] Getting speculators to update...');
+    const updates = [];
+    try {
+        const blocksRef = db.collection(speculatorBlocksCollectionName);
+        const snapshot = await blocksRef.get();
+
+        if (snapshot.empty) {
+            logger.log('INFO','[Core Utils] No speculator blocks found.');
+            return [];
+        }
+
+        snapshot.forEach(doc => {
+            const blockData = doc.data();
+
+            // Iterate over the document's top-level keys
+            for (const key in blockData) {
+                 // Filter for keys that match the 'users.CID' format
+                 if (!key.startsWith('users.')) continue; 
+
+                 const userId = key.split('.')[1];
+                 if (!userId) continue; // Safety check
+
+                 const userData = blockData[key]; // Get the user's map
+
+                 const lastVerified = userData.lastVerified?.toDate ? userData.lastVerified.toDate() : new Date(0);
+                 const lastHeld = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : new Date(0);
+
+                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) {
+                     if (userData.instruments && Array.isArray(userData.instruments)) {
+                         userData.instruments.forEach(instrumentId => {
+                             updates.push({ userId, instrumentId });
+                         });
+                     }
+                 }
+            }
+        });
+
+        logger.log('INFO',`[Core Utils] Found ${updates.length} speculator user/instrument pairs to update.`);
+        return updates;
+    } catch (error) {
+        logger.log('ERROR','[Core Utils] Error getting speculators to update', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+module.exports = {
+  getLatestNormalUserPortfolios,
+  resetProxyLocks,
+  getBlockCapacities,
+  getExclusionIds,
+  getPrioritizedSpeculators,
+  clearCollection,
+  batchWriteShardedIds,
+  getNormalUsersToUpdate,
+  getSpeculatorsToUpdate,
+};