bulltrackers-module 1.0.207 → 1.0.208

package/functions/orchestrator/helpers/discovery_helpers.js

@@ -12,18 +12,30 @@
  */
 async function checkDiscoveryNeed(userType, config, dependencies) {
     const { logger, firestoreUtils } = dependencies;
+    const { targetUsersPerBlock, normalBlockCountsDocPath, allHighValueBlocks } = config;
+    
+    // --- REFACTOR: ORGANIC SPECULATOR DISCOVERY ---
+    // We no longer "hunt" for speculators using block targets. 
+    // Speculators are now discovered organically via the Normal User Update Loop (Feedback Loop).
+    if (userType === 'speculator') {
+        logger.log('INFO', '[Orchestrator Helpers] Speculator discovery is Organic/Feedback-Driven. Skipping active discovery checks.');
+        return { needsDiscovery: false, blocksToFill: [] };
+    }
+
     logger.log('INFO', `[Orchestrator Helpers] Checking discovery need for ${userType}...`);
-    const { targetUsersPerBlock, speculatorBlockCountsDocPath, normalBlockCountsDocPath,speculatorInstruments,allHighValueBlocks} = config;
-    const isSpeculator = userType === 'speculator';
-    const blockCounts = await firestoreUtils.getBlockCapacities(dependencies, { speculatorBlockCountsDocPath, normalBlockCountsDocPath }, userType);
-    let underPopulatedBlocks = [];
-    if (isSpeculator) {for (const instrument of speculatorInstruments) { for (const block of allHighValueBlocks) 
-        {const instrumentBlockKey = `${instrument}_${block.startId}`;
-        if ((blockCounts[instrumentBlockKey] || 0) < targetUsersPerBlock) {underPopulatedBlocks.push({ ...block, instrument });}}}} 
-        else {underPopulatedBlocks = allHighValueBlocks.filter(block => (blockCounts[block.startId] || 0) < targetUsersPerBlock);}
-        const needsDiscovery = underPopulatedBlocks.length > 0;
-        if (!needsDiscovery) {logger.log('SUCCESS', `✅ All blocks are at target capacity for ${userType} users.`);} 
-        else {logger.log('INFO', `Found ${underPopulatedBlocks.length} underpopulated blocks/instruments for ${userType}.`);}
+    
+    // Logic for Normal Users remains (fill blocks to target)
+    const blockCounts = await firestoreUtils.getBlockCapacities(dependencies, { normalBlockCountsDocPath }, userType);
+    
+    const underPopulatedBlocks = allHighValueBlocks.filter(block => (blockCounts[block.startId] || 0) < targetUsersPerBlock);
+    const needsDiscovery = underPopulatedBlocks.length > 0;
+
+    if (!needsDiscovery) {
+        logger.log('SUCCESS', `✅ All blocks are at target capacity for ${userType} users.`);
+    } else {
+        logger.log('INFO', `Found ${underPopulatedBlocks.length} underpopulated blocks for ${userType}.`);
+    }
+
     return { needsDiscovery, blocksToFill: underPopulatedBlocks };
 }
 
@@ -38,29 +50,70 @@ async function checkDiscoveryNeed(userType, config, dependencies) {
 async function getDiscoveryCandidates(userType, blocksToFill, config, dependencies) {
     const { logger, firestoreUtils } = dependencies;
     logger.log('INFO', `[Orchestrator Helpers] Getting discovery candidates for ${userType}...`);
-    const {maxTasksPerRun,discoveryBatchSize,maxRandomCidsToDiscover,specBlocksCollection,pendingSpecCollection,normalUserCollection,invalidSpecCollection,speculatorInstrumentSet,snapshotsSubCollectionName,partsSubCollectionName} = config;
-    const isSpeculator = userType === 'speculator';
+    
+    const {
+        maxTasksPerRun,
+        discoveryBatchSize,
+        maxRandomCidsToDiscover,
+        specBlocksCollection,
+        pendingSpecCollection,
+        normalUserCollection,
+        invalidSpecCollection,
+        snapshotsSubCollectionName,
+        partsSubCollectionName
+    } = config;
+
     const dispatchedCids = new Set();
-    const latestNormalPortfolios = await firestoreUtils.getLatestNormalUserPortfolios(dependencies,{ normalUserCollectionName: normalUserCollection, snapshotsSubCollectionName, partsSubCollectionName });
+
+    // Fetch exclusion list to avoid re-discovering existing users
+    // Note: For Normal users, we check against existing Normal users.
+    const latestNormalPortfolios = await firestoreUtils.getLatestNormalUserPortfolios(dependencies, { 
+        normalUserCollectionName: normalUserCollection, 
+        snapshotsSubCollectionName, 
+        partsSubCollectionName 
+    });
+    
     const existingNormalUserIds = new Set(Object.keys(latestNormalPortfolios));
-    const exclusionIds = await firestoreUtils.getExclusionIds(dependencies, {specBlocksCollection, pendingSpecCollection, normalUserCollection,invalidSpecCollection,existingNormalUserIds}, userType);
-    if (isSpeculator) {const prioritizedCandidates = await firestoreUtils.getPrioritizedSpeculators(dependencies,exclusionIds, speculatorInstrumentSet,latestNormalPortfolios);
-        logger.log('INFO', `Found ${prioritizedCandidates.length} potential new speculators from existing user pool.`);
-        prioritizedCandidates.forEach(cidStr => {if (dispatchedCids.size < maxTasksPerRun) {dispatchedCids.add(cidStr);}});
-        logger.log('INFO', `Added ${dispatchedCids.size} prioritized speculators.`);}
+    const exclusionIds = await firestoreUtils.getExclusionIds(dependencies, {
+        specBlocksCollection, 
+        pendingSpecCollection, 
+        normalUserCollection,
+        invalidSpecCollection,
+        existingNormalUserIds
+    }, userType);
+
+    // --- RANDOM DISCOVERY LOOP (For Normal Users) ---
     let dispatchedRandomCidCount = 0;
-    const initialDispatchedCount = dispatchedCids.size;
-    while ((dispatchedRandomCidCount + initialDispatchedCount) < maxRandomCidsToDiscover &&dispatchedCids.size < maxTasksPerRun && blocksToFill.length > 0){
+    
+    while ((dispatchedRandomCidCount + dispatchedCids.size) < maxRandomCidsToDiscover && 
+           dispatchedCids.size < maxTasksPerRun && 
+           blocksToFill.length > 0) {
+        
         const blockIndex = dispatchedCids.size % blocksToFill.length;
         const block = blocksToFill[blockIndex];
         if (!block) break; 
+
         for (let j = 0; j < discoveryBatchSize; j++) {
-        if ((dispatchedRandomCidCount + initialDispatchedCount) >= maxRandomCidsToDiscover || dispatchedCids.size >= maxTasksPerRun) break;
-        let randomId;
-        let retryCount = 0;
-        const MAX_RETRIES = 50;
-        do {if (++retryCount > MAX_RETRIES) break;randomId = String(Math.floor(Math.random() * 1000000) + block.startId);} while (exclusionIds.has(randomId) ||dispatchedCids.has(randomId));
-        if (retryCount <= MAX_RETRIES) {dispatchedCids.add(randomId);dispatchedRandomCidCount++;}}}
+            if ((dispatchedRandomCidCount + dispatchedCids.size) >= maxRandomCidsToDiscover || 
+                dispatchedCids.size >= maxTasksPerRun) break;
+
+            let randomId;
+            let retryCount = 0;
+            const MAX_RETRIES = 50;
+            
+            do {
+                if (++retryCount > MAX_RETRIES) break;
+                // Generate random CID within the block range
+                randomId = String(Math.floor(Math.random() * 1000000) + block.startId);
+            } while (exclusionIds.has(randomId) || dispatchedCids.has(randomId));
+
+            if (retryCount <= MAX_RETRIES) {
+                dispatchedCids.add(randomId);
+                dispatchedRandomCidCount++;
+            }
+        }
+    }
+
     logger.log('INFO', `Generated ${dispatchedRandomCidCount} random CIDs for ${userType} discovery.`);
     logger.log('INFO', `Total candidates for dispatch: ${dispatchedCids.size}`);
     return dispatchedCids; 
@@ -75,34 +128,54 @@ async function getDiscoveryCandidates(userType, blocksToFill, config, dependenci
  * @returns {Promise<void>}
  */
 async function dispatchDiscovery(userType, candidates, config, dependencies) {
-    const { logger, firestoreUtils, pubsubUtils, pubsub } = dependencies; 
-    const {topicName,dispatchBatchSize,pubsubBatchSize, pendingSpecCollection, pendingMaxFieldsPerDoc,pendingMaxWritesPerBatch} = config;
-    if (candidates.size === 0) {logger.log('INFO', `[Orchestrator Helpers] No ${userType} candidates to dispatch.`);return;}
+    const { logger, pubsub } = dependencies; 
+    const { topicName, dispatchBatchSize } = config;
+
+    if (candidates.size === 0) {
+        logger.log('INFO', `[Orchestrator Helpers] No ${userType} candidates to dispatch.`);
+        return;
+    }
+
     logger.log('INFO', `[Orchestrator Helpers] Dispatching ${candidates.size} discovery tasks for ${userType}...`);
-    const isSpeculator = userType === 'speculator';
+
     const cidsArray = Array.from(candidates);
-    if (isSpeculator) 
-        {await firestoreUtils.clearCollection(dependencies, pendingSpecCollection); 
-        await firestoreUtils.batchWriteShardedIds(dependencies, {collectionPath: pendingSpecCollection,items: cidsArray, timestamp: new Date(),maxFieldsPerDoc: pendingMaxFieldsPerDoc,maxWritesPerBatch: pendingMaxWritesPerBatch});}
     const tasks = [];
-    for (let i = 0; i < cidsArray.length; i += dispatchBatchSize) {const batchCids = cidsArray.slice(i, i + dispatchBatchSize).map(cid => parseInt(cid)); 
-    if (batchCids.length > 0) {
-    const blockId = Math.floor(batchCids[0] / 1000000) * 1000000;
-    tasks.push({type: 'discover',cids: batchCids,blockId, userType});}}
+
+    for (let i = 0; i < cidsArray.length; i += dispatchBatchSize) {
+        const batchCids = cidsArray.slice(i, i + dispatchBatchSize).map(cid => parseInt(cid)); 
+        if (batchCids.length > 0) {
+            const blockId = Math.floor(batchCids[0] / 1000000) * 1000000;
+            tasks.push({
+                type: 'discover',
+                cids: batchCids,
+                blockId, 
+                userType
+            });
+        }
+    }
+
     const topic = pubsub.topic(topicName);
     let totalCidsPublished = 0;
     let messagesPublished = 0;
+
     for (let i = 0; i < tasks.length; i += dispatchBatchSize) {
         const batchOfTasks = tasks.slice(i, i + dispatchBatchSize);
         const messagePayload = { tasks: batchOfTasks };
-    try {await topic.publishMessage({ json: messagePayload });
-        const cidsInThisMessage = batchOfTasks.reduce((acc, task) => acc + task.cids.length, 0);
-        totalCidsPublished += cidsInThisMessage;
-        messagesPublished++;
-        logger.log('INFO', `[Orchestrator Helpers] Dispatched batch ${messagesPublished} with ${batchOfTasks.length} discover tasks (${cidsInThisMessage} CIDs) as 1 Pub/Sub message.`);} 
-    catch (publishError) {
-        logger.log('ERROR', `[Orchestrator Helpers] Failed to publish discover batch ${messagesPublished + 1}.`, { error: publishError.message });}}
-        logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${totalCidsPublished} CIDs in ${tasks.length} tasks, grouped into ${messagesPublished} Pub/Sub messages for ${userType} discovery.`);
+
+        try {
+            await topic.publishMessage({ json: messagePayload });
+            
+            const cidsInThisMessage = batchOfTasks.reduce((acc, task) => acc + task.cids.length, 0);
+            totalCidsPublished += cidsInThisMessage;
+            messagesPublished++;
+            
+            logger.log('INFO', `[Orchestrator Helpers] Dispatched batch ${messagesPublished} with ${batchOfTasks.length} discover tasks (${cidsInThisMessage} CIDs) as 1 Pub/Sub message.`);
+        } catch (publishError) {
+            logger.log('ERROR', `[Orchestrator Helpers] Failed to publish discover batch ${messagesPublished + 1}.`, { error: publishError.message });
+        }
+    }
+    
+    logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${totalCidsPublished} CIDs in ${tasks.length} tasks for ${userType} discovery.`);
 }
 
 module.exports = { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.207",
+  "version": "1.0.208",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [