bulltrackers-module 1.0.123 → 1.0.124

package/functions/orchestrator/helpers/discovery_helpers.js

@@ -1,3 +1,8 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/orchestrator/helpers/discovery_helpers.js
+ * (FIXED: Publishes 'discover' tasks in the correct batch format { tasks: [...] })
+ */
+
 /**
  * @fileoverview Orchestrator's discovery sub-pipes.
  * REFACTORED: All functions are now stateless and receive dependencies.
@@ -164,11 +169,11 @@ async function getDiscoveryCandidates(userType, blocksToFill, config, dependenci
  * @returns {Promise<void>}
  */
 async function dispatchDiscovery(userType, candidates, config, dependencies) {
-    const { logger, firestoreUtils, pubsubUtils } = dependencies;
+    const { logger, firestoreUtils, pubsubUtils, pubsub } = dependencies; // <-- Add pubsub
     const {
         topicName,
         dispatchBatchSize,
-        pubsubBatchSize,
+        pubsubBatchSize, // This is no longer used by this function
         pendingSpecCollection, 
         pendingMaxFieldsPerDoc,
         pendingMaxWritesPerBatch
@@ -208,14 +213,43 @@ async function dispatchDiscovery(userType, candidates, config, dependencies) {
         }
     }
 
-    // Pass dependencies and config to core util
-    await pubsubUtils.batchPublishTasks(dependencies, {
-        topicName, 
-        tasks, 
-        taskType: `${userType} discovery`,
-        maxPubsubBatchSize: pubsubBatchSize
-    });
-    logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${tasks.length} task messages (${candidates.size} CIDs) for ${userType} discovery.`);
+    // --- START FIX: RE-IMPLEMENT BATCHING LOGIC ---
+    const topic = pubsub.topic(topicName);
+    let totalCidsPublished = 0;
+    let messagesPublished = 0;
+
+    // 'tasks' is an array of 'discover' task objects. We group them into
+    // batches and publish each batch as a single message.
+    // We use `dispatchBatchSize` here to determine how many 'discover' tasks go into one message.
+    // This value should probably be small, like 1-10.
+    
+    // Let's re-use `dispatchBatchSize` as the number of *tasks* per message.
+    // If `tasks.length` is 200 and `dispatchBatchSize` is 50, this will create 4 messages.
+    // This seems correct.
+
+    for (let i = 0; i < tasks.length; i += dispatchBatchSize) {
+        // This batch contains multiple 'discover' tasks
+        const batchOfTasks = tasks.slice(i, i + dispatchBatchSize);
+        
+        // Wrap this batch in the new payload format
+        const messagePayload = { tasks: batchOfTasks };
+
+        try {
+            await topic.publishMessage({ json: messagePayload });
+            
+            // Log the number of CIDs in this message
+            const cidsInThisMessage = batchOfTasks.reduce((acc, task) => acc + task.cids.length, 0);
+            totalCidsPublished += cidsInThisMessage;
+            messagesPublished++;
+            
+            logger.log('INFO', `[Orchestrator Helpers] Dispatched batch ${messagesPublished} with ${batchOfTasks.length} discover tasks (${cidsInThisMessage} CIDs) as 1 Pub/Sub message.`);
+        } catch (publishError) {
+            logger.log('ERROR', `[Orchestrator Helpers] Failed to publish discover batch ${messagesPublished + 1}.`, { error: publishError.message });
+        }
+    }
+    
+    logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${totalCidsPublished} CIDs in ${tasks.length} tasks, grouped into ${messagesPublished} Pub/Sub messages for ${userType} discovery.`);
+    // --- END FIX ---
 }
 
 
@@ -223,4 +257,4 @@ module.exports = {
   checkDiscoveryNeed,
   getDiscoveryCandidates,
   dispatchDiscovery,
-};
+};

package/functions/task-engine/helpers/discover_helpers.js

@@ -1,3 +1,8 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/discover_helpers.js
+ * (FIXED: Publishes 'verify' task in the correct batch format { tasks: [...] })
+ */
+
 /**
  * @fileoverview Sub-pipe: pipe.taskEngine.handleDiscover
  * REFACTORED: Now stateless and receives dependencies.
@@ -119,10 +124,12 @@ async function handleDiscover(task, taskId, dependencies, config) {
             };
             // --- END MODIFICATION ---
 
+            // --- FIX: WRAP IN BATCH FORMAT ---
             // Use pubsub from dependencies
             await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH)
-                               .publishMessage({ json: verificationTask });
-            logger.log('INFO', `[DISCOVER] Verification message published was : ${JSON.stringify(verificationTask)} `);
+                               .publishMessage({ json: { tasks: [verificationTask] } });
+            logger.log('INFO', `[DISCOVER] Verification message published was : ${JSON.stringify({ tasks: [verificationTask] })} `);
+            // --- END FIX ---
             logger.log('INFO', `[DISCOVER] Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
         }
         

package/functions/task-engine/helpers/update_helpers.js

@@ -1,3 +1,8 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
+ * (FIXED: Enhanced error logging for portfolio and history fetches)
+ */
+
 /**
  * @fileoverview Sub-pipe: pipe.taskEngine.handleUpdate
  * REFACTORED: Now stateless and receives dependencies.
@@ -122,12 +127,30 @@ async function handleUpdate(task, taskId, dependencies, config, username) {
                 wasPortfolioSuccess = true;
                 const portfolioData = JSON.parse(responseBody);
                 // Add to portfolio batch
+                console.log('DEBUG', `[UPDATE] SUCCESS Adding portfolio data for user ${userId} to batch.`);
                 await batchManager.addToPortfolioBatch(userId, portfolioStorageBlockId, today, portfolioData, userType, instrumentId);
+                
             }
         } else {
-            // Log failure
-            const errorMsg = portfolioResult.status === 'rejected' ? portfolioResult.reason.message : `API status ${portfolioResult.value.status}`;
-            logger.log('WARN', `[UPDATE] Failed to fetch PORTFOLIO for user ${userId}.`, { error: errorMsg });
+            // --- FIX: ENHANCED LOGGING ---
+            let errorMsg = 'Unknown fetch error';
+            let rawErrorText = 'N/A';
+
+            if (portfolioResult.status === 'rejected') {
+                errorMsg = portfolioResult.reason.message;
+            } else if (portfolioResult.value) {
+                errorMsg = portfolioResult.value.error?.message || `API status ${portfolioResult.value.status}`;
+                // Get the raw text from the proxy manager's response
+                if (typeof portfolioResult.value.text === 'function') {
+                    rawErrorText = await portfolioResult.value.text();
+                }
+            }
+            
+            logger.log('WARN', `[UPDATE] Failed to fetch PORTFOLIO for user ${userId}.`, { 
+                error: errorMsg,
+                proxyResponse: rawErrorText // This will contain "urlfetch" or other quota messages
+            });
+            // --- END FIX ---
         }
 
         // --- Process History Result ---
@@ -135,11 +158,28 @@ async function handleUpdate(task, taskId, dependencies, config, username) {
             const historyData = await historyResult.value.json();
             wasHistorySuccess = true;
             // Add to history batch
+            console.log('DEBUG', `[UPDATE] SUCCESS Adding history data for user ${userId} to batch.`);
             await batchManager.addToTradingHistoryBatch(userId, portfolioStorageBlockId, today, historyData, userType);
         } else {
-            // Log failure
-            const errorMsg = historyResult.status === 'rejected' ? historyResult.reason.message : `API status ${historyResult.value.status}`;
-            logger.log('WARN', `[UPDATE] Failed to fetch HISTORY for user ${userId} (${username}).`, { error: errorMsg });
+            // --- FIX: ENHANCED LOGGING ---
+            let errorMsg = 'Unknown fetch error';
+            let rawErrorText = 'N/A';
+
+            if (historyResult.status === 'rejected') {
+                errorMsg = historyResult.reason.message;
+            } else if (historyResult.value) {
+                errorMsg = historyResult.value.error?.message || `API status ${historyResult.value.status}`;
+                // Get the raw text from the proxy manager's response
+                if (typeof historyResult.value.text === 'function') {
+                    rawErrorText = await historyResult.value.text();
+                }
+            }
+            
+            logger.log('WARN', `[UPDATE] Failed to fetch HISTORY for user ${userId} (${username}).`, { 
+                error: errorMsg,
+                proxyResponse: rawErrorText // This will contain "urlfetch" or other quota messages
+            });
+            // --- END FIX ---
         }
 
         // --- Handle Private User (if detected) ---

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.123",
+  "version": "1.0.124",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [