bulltrackers-module 1.0.0 → 1.0.2

package/functions/core/utils/firestore_utils.js

@@ -47,8 +47,6 @@ async function getLatestNormalUserPortfolios(normalUserCollectionName, snapshots
 
 /**
  * Resets the isLocked status for all proxies.
- * Note: This implementation differs from the original orchestrator's FIRESTORE_DOC_PROXY_PERFORMANCE logic.
- * This logic matches the original file provided for the module.
  * @async
  * @param {string} proxiesCollectionName - e.g., 'Proxies'.
  * @returns {Promise<void>}
@@ -117,8 +115,8 @@ async function getBlockCapacities(userType, speculatorBlockCountsDocPath, normal
  * @param {object} config - Configuration object.
  * @param {string} config.specBlocksCollection - e.g., 'SpeculatorBlocks'.
  * @param {string} config.pendingSpecCollection - e.g., 'PendingSpeculators'.
- * @param {string} config.normalUserCollection - e.g., 'NormalUserPortfolios'.
  * @param {string} config.invalidSpecCollection - e.g., 'InvalidSpeculators'.
+ * @param {Set<string>} config.existingNormalUserIds - PRE-FETCHED normal user IDs.
  * @returns {Promise<Set<string>>} A Set containing unique user IDs.
  */
 async function getExclusionIds(userType, config) {
@@ -126,11 +124,14 @@ async function getExclusionIds(userType, config) {
     const { 
         specBlocksCollection, 
         pendingSpecCollection, 
-        normalUserCollection, 
-        invalidSpecCollection 
+        invalidSpecCollection,
+        existingNormalUserIds // Get the pre-fetched IDs
     } = config;
     
-    const exclusionIds = new Set();
+    // Initialize with the pre-fetched normal users
+    const exclusionIds = new Set(existingNormalUserIds);
+    logger.log('TRACE', `[Core Utils] Loaded ${exclusionIds.size} existing normal user IDs for exclusion.`);
+    
     const promises = [];
 
     try {
@@ -155,31 +156,18 @@ async function getExclusionIds(userType, config) {
              }));
         }
 
-        // 3. Existing Normal Users
-        const normalPortfoliosRef = db.collection(normalUserCollection);
-        promises.push(normalPortfoliosRef.listDocuments().then(async blockRefs => {
-            for(const blockRef of blockRefs) {
-                // Assuming 'snapshots' and 'parts' subcollections as per original logic
-                const snapshotRef = blockRef.collection('snapshots'); 
-                const latestSnapshotQuery = snapshotRef.orderBy(FieldPath.documentId(), 'desc').limit(1);
-                const latestSnapshot = await latestSnapshotQuery.get();
-                if (!latestSnapshot.empty) {
-                    const partsRef = latestSnapshot.docs[0].ref.collection('parts');
-                    const partsSnapshot = await partsRef.get();
-                    partsSnapshot.forEach(partDoc => {
-                        Object.keys(partDoc.data()).forEach(uid => exclusionIds.add(uid));
-                    });
-                }
-            }
-             logger.log('TRACE','[Core Utils] Fetched existing normal user IDs for exclusion.');
-        }));
-
+        // 3. Existing Normal Users (REMOVED - Now passed in)
+        logger.log('TRACE','[Core Utils] Skipping normal user fetch, already provided.');
 
         // 4. Invalid Speculators
         const invalidRef = db.collection(invalidSpecCollection);
          promises.push(invalidRef.get().then(snapshot => {
              snapshot.forEach(doc => {
-                 Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));
+                 const data = doc.data();
+                 if (data) {
+                    // Use .users map as per original orchestrator
+                    Object.keys(data.users || {}).forEach(cid => exclusionIds.add(cid));
+                 }
              });
               logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');
          }));
@@ -200,21 +188,16 @@ async function getExclusionIds(userType, config) {
  * @async
  * @param {Set<string>} exclusionIds - IDs to exclude.
  * @param {Set<number>} speculatorInstrumentSet - Set of instrument IDs.
- * @param {string} normalUserCollectionName - e.g., 'NormalUserPortfolios'.
- * @param {string} snapshotsSubCollectionName - e.g., 'snapshots'.
- * @param {string} partsSubCollectionName - e.g., 'parts'.
+ * @param {object} latestNormalPortfolios - PRE-FETCHED portfolio object.
  * @returns {Promise<string[]>} Array of prioritized speculator CIDs.
  */
-async function getPrioritizedSpeculators(exclusionIds, speculatorInstrumentSet, normalUserCollectionName, snapshotsSubCollectionName, partsSubCollectionName) {
+async function getPrioritizedSpeculators(exclusionIds, speculatorInstrumentSet, latestNormalPortfolios) {
     logger.log('INFO','[Core Utils] Scanning normal users for prioritized speculators...');
     const candidates = new Set();
 
     try {
-        const latestNormalPortfolios = await getLatestNormalUserPortfolios(
-            normalUserCollectionName, 
-            snapshotsSubCollectionName, 
-            partsSubCollectionName
-        );
+        // No longer need to fetch, just use the provided object
+        // const latestNormalPortfolios = await getLatestNormalUserPortfolios(...);
 
         for (const userId in latestNormalPortfolios) {
             if (exclusionIds.has(userId)) continue;

package/functions/orchestrator/helpers/discovery_helpers.js

@@ -96,21 +96,31 @@ async function getDiscoveryCandidates(userType, blocksToFill, config) {
     const isSpeculator = userType === 'speculator';
     const dispatchedCids = new Set();
 
+    // --- OPTIMIZATION: Fetch normal user data ONCE ---
+    const latestNormalPortfolios = await firestoreUtils.getLatestNormalUserPortfolios(
+        normalUserCollection,
+        snapshotsSubCollectionName,
+        partsSubCollectionName
+    );
+    const existingNormalUserIds = new Set(Object.keys(latestNormalPortfolios));
+    // --- End Optimization ---
+
+    // Pass the fetched normal user IDs directly into getExclusionIds
     const exclusionIds = await firestoreUtils.getExclusionIds(userType, {
         specBlocksCollection, 
         pendingSpecCollection, 
-        normalUserCollection, 
-        invalidSpecCollection
+        normalUserCollection, // Pass name for consistency, though fetch is skipped
+        invalidSpecCollection,
+        existingNormalUserIds: existingNormalUserIds // <-- Pass pre-fetched IDs
     });
 
     // --- Prioritization for Speculators ---
     if (isSpeculator) {
+        // Pass the fetched portfolio data directly to avoid a second fetch
         const prioritizedCandidates = await firestoreUtils.getPrioritizedSpeculators(
             exclusionIds, 
             speculatorInstrumentSet,
-            normalUserCollection,
-            snapshotsSubCollectionName,
-            partsSubCollectionName
+            latestNormalPortfolios // <-- Pass pre-fetched data
         );
         logger.log('INFO', `Found ${prioritizedCandidates.length} potential new speculators from existing user pool.`);
 
@@ -178,7 +188,7 @@ async function dispatchDiscovery(userType, candidates, config) {
         topicName,
         dispatchBatchSize,
         pubsubBatchSize,
-        pendingSpeculatorsCollection,
+        pendingSpecCollection, // <-- Change to singular
         pendingMaxFieldsPerDoc,
         pendingMaxWritesPerBatch
     } = config;
@@ -193,9 +203,9 @@ async function dispatchDiscovery(userType, candidates, config) {
     const cidsArray = Array.from(candidates);
 
     if (isSpeculator) {
-        await firestoreUtils.clearCollection(pendingSpeculatorsCollection);
+        await firestoreUtils.clearCollection(pendingSpecCollection); // <-- Change to singular
         await firestoreUtils.batchWriteShardedIds(
-            pendingSpeculatorsCollection, 
+            pendingSpecCollection, // <-- And this line
             cidsArray, 
             new Date(),
             pendingMaxFieldsPerDoc,
@@ -218,7 +228,7 @@ async function dispatchDiscovery(userType, candidates, config) {
         }
     }
 
-    await pubsubUtils.batchPublishTasks(
+    await pubsubUtils.batchWriteShardedTasks(
         topicName, 
         tasks, 
         `${userType} discovery`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [