bulltrackers-module 1.0.206 → 1.0.208

package/functions/orchestrator/helpers/discovery_helpers.js

@@ -12,18 +12,30 @@
  */
 async function checkDiscoveryNeed(userType, config, dependencies) {
     const { logger, firestoreUtils } = dependencies;
+    const { targetUsersPerBlock, normalBlockCountsDocPath, allHighValueBlocks } = config;
+    
+    // --- REFACTOR: ORGANIC SPECULATOR DISCOVERY ---
+    // We no longer "hunt" for speculators using block targets. 
+    // Speculators are now discovered organically via the Normal User Update Loop (Feedback Loop).
+    if (userType === 'speculator') {
+        logger.log('INFO', '[Orchestrator Helpers] Speculator discovery is Organic/Feedback-Driven. Skipping active discovery checks.');
+        return { needsDiscovery: false, blocksToFill: [] };
+    }
+
     logger.log('INFO', `[Orchestrator Helpers] Checking discovery need for ${userType}...`);
-    const { targetUsersPerBlock, speculatorBlockCountsDocPath, normalBlockCountsDocPath,speculatorInstruments,allHighValueBlocks} = config;
-    const isSpeculator = userType === 'speculator';
-    const blockCounts = await firestoreUtils.getBlockCapacities(dependencies, { speculatorBlockCountsDocPath, normalBlockCountsDocPath }, userType);
-    let underPopulatedBlocks = [];
-    if (isSpeculator) {for (const instrument of speculatorInstruments) { for (const block of allHighValueBlocks) 
-        {const instrumentBlockKey = `${instrument}_${block.startId}`;
-        if ((blockCounts[instrumentBlockKey] || 0) < targetUsersPerBlock) {underPopulatedBlocks.push({ ...block, instrument });}}}} 
-        else {underPopulatedBlocks = allHighValueBlocks.filter(block => (blockCounts[block.startId] || 0) < targetUsersPerBlock);}
-        const needsDiscovery = underPopulatedBlocks.length > 0;
-        if (!needsDiscovery) {logger.log('SUCCESS', `✅ All blocks are at target capacity for ${userType} users.`);} 
-        else {logger.log('INFO', `Found ${underPopulatedBlocks.length} underpopulated blocks/instruments for ${userType}.`);}
+    
+    // Logic for Normal Users remains (fill blocks to target)
+    const blockCounts = await firestoreUtils.getBlockCapacities(dependencies, { normalBlockCountsDocPath }, userType);
+    
+    const underPopulatedBlocks = allHighValueBlocks.filter(block => (blockCounts[block.startId] || 0) < targetUsersPerBlock);
+    const needsDiscovery = underPopulatedBlocks.length > 0;
+
+    if (!needsDiscovery) {
+        logger.log('SUCCESS', `✅ All blocks are at target capacity for ${userType} users.`);
+    } else {
+        logger.log('INFO', `Found ${underPopulatedBlocks.length} underpopulated blocks for ${userType}.`);
+    }
+
     return { needsDiscovery, blocksToFill: underPopulatedBlocks };
 }
 
@@ -38,29 +50,70 @@ async function checkDiscoveryNeed(userType, config, dependencies) {
 async function getDiscoveryCandidates(userType, blocksToFill, config, dependencies) {
     const { logger, firestoreUtils } = dependencies;
     logger.log('INFO', `[Orchestrator Helpers] Getting discovery candidates for ${userType}...`);
-    const {maxTasksPerRun,discoveryBatchSize,maxRandomCidsToDiscover,specBlocksCollection,pendingSpecCollection,normalUserCollection,invalidSpecCollection,speculatorInstrumentSet,snapshotsSubCollectionName,partsSubCollectionName} = config;
-    const isSpeculator = userType === 'speculator';
+    
+    const {
+        maxTasksPerRun,
+        discoveryBatchSize,
+        maxRandomCidsToDiscover,
+        specBlocksCollection,
+        pendingSpecCollection,
+        normalUserCollection,
+        invalidSpecCollection,
+        snapshotsSubCollectionName,
+        partsSubCollectionName
+    } = config;
+
     const dispatchedCids = new Set();
-    const latestNormalPortfolios = await firestoreUtils.getLatestNormalUserPortfolios(dependencies,{ normalUserCollectionName: normalUserCollection, snapshotsSubCollectionName, partsSubCollectionName });
+
+    // Fetch exclusion list to avoid re-discovering existing users
+    // Note: For Normal users, we check against existing Normal users.
+    const latestNormalPortfolios = await firestoreUtils.getLatestNormalUserPortfolios(dependencies, { 
+        normalUserCollectionName: normalUserCollection, 
+        snapshotsSubCollectionName, 
+        partsSubCollectionName 
+    });
+    
     const existingNormalUserIds = new Set(Object.keys(latestNormalPortfolios));
-    const exclusionIds = await firestoreUtils.getExclusionIds(dependencies, {specBlocksCollection, pendingSpecCollection, normalUserCollection,invalidSpecCollection,existingNormalUserIds}, userType);
-    if (isSpeculator) {const prioritizedCandidates = await firestoreUtils.getPrioritizedSpeculators(dependencies,exclusionIds, speculatorInstrumentSet,latestNormalPortfolios);
-        logger.log('INFO', `Found ${prioritizedCandidates.length} potential new speculators from existing user pool.`);
-        prioritizedCandidates.forEach(cidStr => {if (dispatchedCids.size < maxTasksPerRun) {dispatchedCids.add(cidStr);}});
-        logger.log('INFO', `Added ${dispatchedCids.size} prioritized speculators.`);}
+    const exclusionIds = await firestoreUtils.getExclusionIds(dependencies, {
+        specBlocksCollection, 
+        pendingSpecCollection, 
+        normalUserCollection,
+        invalidSpecCollection,
+        existingNormalUserIds
+    }, userType);
+
+    // --- RANDOM DISCOVERY LOOP (For Normal Users) ---
     let dispatchedRandomCidCount = 0;
-    const initialDispatchedCount = dispatchedCids.size;
-    while ((dispatchedRandomCidCount + initialDispatchedCount) < maxRandomCidsToDiscover &&dispatchedCids.size < maxTasksPerRun && blocksToFill.length > 0){
+    
+    while ((dispatchedRandomCidCount + dispatchedCids.size) < maxRandomCidsToDiscover && 
+           dispatchedCids.size < maxTasksPerRun && 
+           blocksToFill.length > 0) {
+        
         const blockIndex = dispatchedCids.size % blocksToFill.length;
         const block = blocksToFill[blockIndex];
         if (!block) break; 
+
         for (let j = 0; j < discoveryBatchSize; j++) {
-        if ((dispatchedRandomCidCount + initialDispatchedCount) >= maxRandomCidsToDiscover || dispatchedCids.size >= maxTasksPerRun) break;
-        let randomId;
-        let retryCount = 0;
-        const MAX_RETRIES = 50;
-        do {if (++retryCount > MAX_RETRIES) break;randomId = String(Math.floor(Math.random() * 1000000) + block.startId);} while (exclusionIds.has(randomId) ||dispatchedCids.has(randomId));
-        if (retryCount <= MAX_RETRIES) {dispatchedCids.add(randomId);dispatchedRandomCidCount++;}}}
+            if ((dispatchedRandomCidCount + dispatchedCids.size) >= maxRandomCidsToDiscover || 
+                dispatchedCids.size >= maxTasksPerRun) break;
+
+            let randomId;
+            let retryCount = 0;
+            const MAX_RETRIES = 50;
+            
+            do {
+                if (++retryCount > MAX_RETRIES) break;
+                // Generate random CID within the block range
+                randomId = String(Math.floor(Math.random() * 1000000) + block.startId);
+            } while (exclusionIds.has(randomId) || dispatchedCids.has(randomId));
+
+            if (retryCount <= MAX_RETRIES) {
+                dispatchedCids.add(randomId);
+                dispatchedRandomCidCount++;
+            }
+        }
+    }
+
     logger.log('INFO', `Generated ${dispatchedRandomCidCount} random CIDs for ${userType} discovery.`);
     logger.log('INFO', `Total candidates for dispatch: ${dispatchedCids.size}`);
     return dispatchedCids; 
@@ -75,34 +128,54 @@ async function getDiscoveryCandidates(userType, blocksToFill, config, dependenci
  * @returns {Promise<void>}
  */
 async function dispatchDiscovery(userType, candidates, config, dependencies) {
-    const { logger, firestoreUtils, pubsubUtils, pubsub } = dependencies; 
-    const {topicName,dispatchBatchSize,pubsubBatchSize, pendingSpecCollection, pendingMaxFieldsPerDoc,pendingMaxWritesPerBatch} = config;
-    if (candidates.size === 0) {logger.log('INFO', `[Orchestrator Helpers] No ${userType} candidates to dispatch.`);return;}
+    const { logger, pubsub } = dependencies; 
+    const { topicName, dispatchBatchSize } = config;
+
+    if (candidates.size === 0) {
+        logger.log('INFO', `[Orchestrator Helpers] No ${userType} candidates to dispatch.`);
+        return;
+    }
+
     logger.log('INFO', `[Orchestrator Helpers] Dispatching ${candidates.size} discovery tasks for ${userType}...`);
-    const isSpeculator = userType === 'speculator';
+
     const cidsArray = Array.from(candidates);
-    if (isSpeculator) 
-        {await firestoreUtils.clearCollection(dependencies, pendingSpecCollection); 
-        await firestoreUtils.batchWriteShardedIds(dependencies, {collectionPath: pendingSpecCollection,items: cidsArray, timestamp: new Date(),maxFieldsPerDoc: pendingMaxFieldsPerDoc,maxWritesPerBatch: pendingMaxWritesPerBatch});}
     const tasks = [];
-    for (let i = 0; i < cidsArray.length; i += dispatchBatchSize) {const batchCids = cidsArray.slice(i, i + dispatchBatchSize).map(cid => parseInt(cid)); 
-    if (batchCids.length > 0) {
-    const blockId = Math.floor(batchCids[0] / 1000000) * 1000000;
-    tasks.push({type: 'discover',cids: batchCids,blockId, userType});}}
+
+    for (let i = 0; i < cidsArray.length; i += dispatchBatchSize) {
+        const batchCids = cidsArray.slice(i, i + dispatchBatchSize).map(cid => parseInt(cid)); 
+        if (batchCids.length > 0) {
+            const blockId = Math.floor(batchCids[0] / 1000000) * 1000000;
+            tasks.push({
+                type: 'discover',
+                cids: batchCids,
+                blockId, 
+                userType
+            });
+        }
+    }
+
     const topic = pubsub.topic(topicName);
     let totalCidsPublished = 0;
     let messagesPublished = 0;
+
     for (let i = 0; i < tasks.length; i += dispatchBatchSize) {
         const batchOfTasks = tasks.slice(i, i + dispatchBatchSize);
         const messagePayload = { tasks: batchOfTasks };
-    try {await topic.publishMessage({ json: messagePayload });
-        const cidsInThisMessage = batchOfTasks.reduce((acc, task) => acc + task.cids.length, 0);
-        totalCidsPublished += cidsInThisMessage;
-        messagesPublished++;
-        logger.log('INFO', `[Orchestrator Helpers] Dispatched batch ${messagesPublished} with ${batchOfTasks.length} discover tasks (${cidsInThisMessage} CIDs) as 1 Pub/Sub message.`);} 
-    catch (publishError) {
-        logger.log('ERROR', `[Orchestrator Helpers] Failed to publish discover batch ${messagesPublished + 1}.`, { error: publishError.message });}}
-        logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${totalCidsPublished} CIDs in ${tasks.length} tasks, grouped into ${messagesPublished} Pub/Sub messages for ${userType} discovery.`);
+
+        try {
+            await topic.publishMessage({ json: messagePayload });
+            
+            const cidsInThisMessage = batchOfTasks.reduce((acc, task) => acc + task.cids.length, 0);
+            totalCidsPublished += cidsInThisMessage;
+            messagesPublished++;
+            
+            logger.log('INFO', `[Orchestrator Helpers] Dispatched batch ${messagesPublished} with ${batchOfTasks.length} discover tasks (${cidsInThisMessage} CIDs) as 1 Pub/Sub message.`);
+        } catch (publishError) {
+            logger.log('ERROR', `[Orchestrator Helpers] Failed to publish discover batch ${messagesPublished + 1}.`, { error: publishError.message });
+        }
+    }
+    
+    logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${totalCidsPublished} CIDs in ${tasks.length} tasks for ${userType} discovery.`);
 }
 
 module.exports = { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery };

package/functions/task-engine/helpers/update_helpers.js

@@ -1,5 +1,6 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
+ * (OPTIMIZED V4: Auto-Speculator Detection via History/Portfolio Intersection)
  * (OPTIMIZED V3: Removed obsolete username lookup logic)
  * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
  * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
@@ -26,27 +27,54 @@ function shouldTryProxy() {
  */
 function recordProxyOutcome(success) {
     if (success) {
-        if (_consecutiveProxyFailures > 0) {
-            // Optional: Only log recovery to reduce noise
-        }
         _consecutiveProxyFailures = 0;
     } else {
         _consecutiveProxyFailures++;
     }
 }
 
+/**
+ * --- NEW HELPER: Speculator Detector ---
+ * intersections: (History: Leverage > 1) AND (Portfolio: Currently Owned)
+ */
+function detectSpeculatorTargets(historyData, portfolioData) {
+    if (!historyData?.PublicHistoryPositions || !portfolioData?.AggregatedPositions) return [];
+
+    // 1. Identify assets that have EVER been traded with leverage > 1
+    const leveragedAssets = new Set();
+    for (const pos of historyData.PublicHistoryPositions) {
+        if (pos.Leverage > 1 && pos.InstrumentID) {
+            leveragedAssets.add(pos.InstrumentID);
+        }
+    }
+
+    if (leveragedAssets.size === 0) return [];
+
+    // 2. Check if the user CURRENTLY owns any of these assets
+    const targets = [];
+    for (const pos of portfolioData.AggregatedPositions) {
+        if (leveragedAssets.has(pos.InstrumentID)) {
+            targets.push(pos.InstrumentID);
+        }
+    }
+    
+    return targets;
+}
+
 /**
  * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
  */
-async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config) {
-    // Note: 'username' param removed from signature as it is no longer needed.
+async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager, pubsub }, config) {
     const { userId, instruments, instrumentId, userType } = task;
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
     let isPrivate = false;
     
-    // DOWNGRADED TO TRACE
+    // Captured data for detection logic
+    let capturedHistory = null;
+    let capturedPortfolio = null;
+
     logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
@@ -63,11 +91,10 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
             } else {
                 
                 // --- REFACTOR: New Granular API Logic ---
-                // No username required. Uses CID (userId) directly.
                 const d = new Date();
                 d.setFullYear(d.getFullYear() - 1);
                 const oneYearAgoStr = d.toISOString();
-                const uuid = crypto.randomUUID ? crypto.randomUUID() : '0205aca7-bd37-4884-8455-f28ce1add2de'; // Fallback for older nodes
+                const uuid = crypto.randomUUID ? crypto.randomUUID() : '0205aca7-bd37-4884-8455-f28ce1add2de'; 
 
                 const historyUrl = `https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat?StartTime=${oneYearAgoStr}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${userId}&client_request_id=${uuid}`;
                 const options = { headers: historyHeader.header };
@@ -79,13 +106,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                         logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
                         response = await proxyManager.fetch(historyUrl, options);
                         if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-                        
                         wasHistorySuccess = true; 
                         proxyUsedForHistory = true;
-                        recordProxyOutcome(true); // Reset
+                        recordProxyOutcome(true); 
 
                     } catch (proxyError) {
-                        recordProxyOutcome(false); // Count failure
+                        recordProxyOutcome(false); 
                         logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
                     }
                 }
@@ -105,6 +131,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
                 if (wasHistorySuccess) { 
                     const data = await response.json();
+                    capturedHistory = data; // Capture for later
                     await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  
                 }
             }
@@ -118,7 +145,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     } 
     
     // --- 2. Process Portfolio Fetches (Sequentially) ---
-    logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); // DOWNGRADED TO TRACE
+    logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); 
     
     for (const instId of instrumentsToProcess) { 
         if (isPrivate) { 
@@ -141,13 +168,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
                 response = await proxyManager.fetch(portfolioUrl, options);
                 if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); 
-                
                 wasPortfolioSuccess = true; 
                 proxyUsedForPortfolio = true;
-                recordProxyOutcome(true); // Reset
+                recordProxyOutcome(true);
 
             } catch (proxyError) { 
-                recordProxyOutcome(false); // Count failure
+                recordProxyOutcome(false);
                 logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via Proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
             }
         }
@@ -172,8 +198,9 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
             try {
                 const portfolioJson = JSON.parse(body);
+                capturedPortfolio = portfolioJson; // Capture for detection
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
-                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio processed successfully.`); // DOWNGRADED TO TRACE
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio processed successfully.`); 
 
             } catch (parseError) { 
                 wasPortfolioSuccess = false; 
@@ -186,7 +213,34 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
     } 
 
-    // --- 5. Handle Private Users & Timestamps ---
+    // --- 5. SPECULATOR DETECTION & QUEUEING (NEW) ---
+    // Only run detection if:
+    // 1. We are processing a Normal User (userType !== 'speculator')
+    // 2. We successfully fetched both history and portfolio
+    // 3. We have PubSub available to queue new tasks
+    if (userType !== 'speculator' && capturedHistory && capturedPortfolio && pubsub && config.PUBSUB_TOPIC_TASK_ENGINE) {
+        try {
+            const speculatorAssets = detectSpeculatorTargets(capturedHistory, capturedPortfolio);
+            if (speculatorAssets.length > 0) {
+                logger.log('INFO', `[handleUpdate/${userId}] DETECTED SPECULATOR BEHAVIOR. Queuing ${speculatorAssets.length} targeted updates.`);
+                
+                const newTasks = speculatorAssets.map(assetId => ({
+                    type: 'update',
+                    userType: 'speculator',
+                    userId: userId,
+                    instrumentId: assetId
+                }));
+
+                // Publish to Task Engine (Tasks are wrapped in a 'tasks' array payload)
+                const dataBuffer = Buffer.from(JSON.stringify({ tasks: newTasks }));
+                await pubsub.topic(config.PUBSUB_TOPIC_TASK_ENGINE).publishMessage({ data: dataBuffer });
+            }
+        } catch (detectionError) {
+            logger.log('ERROR', `[handleUpdate/${userId}] Error during Speculator Detection.`, { error: detectionError.message });
+        }
+    }
+
+    // --- 6. Handle Private Users & Timestamps ---
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
         for (const instrumentId of instrumentsToProcess) { 
@@ -207,7 +261,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
-    logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); // DOWNGRADED TO TRACE
+    logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); 
 }
 
 module.exports = { handleUpdate };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.206",
+  "version": "1.0.208",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [