bulltrackers-module 1.0.358 → 1.0.360

package/functions/core/utils/firestore_utils.js

@@ -2,17 +2,12 @@
  * @fileoverview Core Firestore utility functions.
  * REFACTORED: All functions are now stateless and receive dependencies.
  * 'db' (Firestore instance) and 'logger' are passed via a 'dependencies' object.
+ * UPDATED: Added support for Popular Investor (Rankings) and Signed-In User fetching.
  */
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 
 /**
  * Fetches and merges the most recent portfolio data for all normal users.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.normalUserCollectionName - e.g., 'NormalUserPortfolios'.
- * @param {string} config.snapshotsSubCollectionName - e.g., 'snapshots'.
- * @param {string} config.partsSubCollectionName - e.g., 'parts'.
- * @returns {Promise<object>} A single object containing all user portfolios.
  */
 async function getLatestNormalUserPortfolios(dependencies, config) {
     const { db, logger } = dependencies;
@@ -34,10 +29,6 @@ async function getLatestNormalUserPortfolios(dependencies, config) {
 
 /**
  * Resets the proxy locks map in the performance document.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.proxyPerformanceDocPath - e.g., 'system_state/proxy_performance'.
- * @returns {Promise<void>}
  */
 async function resetProxyLocks(dependencies, config) {
     const { db, logger } = dependencies;
@@ -55,12 +46,6 @@ async function resetProxyLocks(dependencies, config) {
 
 /**
  * Fetches and aggregates block capacities.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} userType - 'normal' or 'speculator'.
- * @param {string} config.speculatorBlockCountsDocPath - Path to speculator counts doc.
- * @param {string} config.normalBlockCountsDocPath - Path to normal counts doc.
- * @returns {Promise<object>} Object containing counts.
  */
 async function getBlockCapacities(dependencies, config, userType) {
     const { db, logger } = dependencies;
@@ -78,14 +63,6 @@ async function getBlockCapacities(dependencies, config, userType) {
 
 /**
  * Fetches user IDs from multiple sources for exclusion lists.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} userType - 'normal' or 'speculator'.
- * @param {string} config.specBlocksCollection - e.g., 'SpeculatorBlocks'.
- * @param {string} config.pendingSpecCollection - e.g., 'PendingSpeculators'.
- * @param {string} config.invalidSpecCollection - e.g., 'InvalidSpeculators'.
- * @param {Set<string>} config.existingNormalUserIds - PRE-FETCHED normal user IDs.
- * @returns {Promise<Set<string>>} A Set containing unique user IDs.
  */
 async function getExclusionIds(dependencies, config, userType) {
     const { db, logger } = dependencies;
@@ -95,15 +72,12 @@ async function getExclusionIds(dependencies, config, userType) {
     logger.log('TRACE', `[Core Utils] Loaded ${exclusionIds.size} existing normal user IDs for exclusion.`);
     const promises = [];
     try {
-        // 1. Existing Speculators
         const specBlocksRef = db.collection(specBlocksCollection);
         promises.push(specBlocksRef.get().then(snapshot => {snapshot.forEach(doc => {const users = doc.data().users || {}; Object.keys(users).forEach(key => exclusionIds.add(key.split('.')[1])); });
         logger.log('TRACE','[Core Utils] Fetched existing speculator IDs for exclusion.');}));
-        // 2. Pending Speculators
         if (userType === 'speculator') {const pendingRef = db.collection(pendingSpecCollection);
         promises.push(pendingRef.get().then(snapshot => {snapshot.forEach(doc => {Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));});
         logger.log('TRACE','[Core Utils] Fetched pending speculator IDs for exclusion.');})); }
-        // 3. Invalid Speculators
         const invalidRef = db.collection(invalidSpecCollection);
         promises.push(invalidRef.get().then(snapshot => { snapshot.forEach(doc => {const data = doc.data();if (data) {Object.keys(data.users || {}).forEach(cid => exclusionIds.add(cid));}});
         logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');}));
@@ -117,11 +91,6 @@ async function getExclusionIds(dependencies, config, userType) {
 
 /**
  * Scans normal user portfolios for potential speculator candidates.
- * @param {object} dependencies - Contains db, logger.
- * @param {Set<string>} exclusionIds - IDs to exclude.
- * @param {Set<number>} speculatorInstrumentSet - Set of instrument IDs.
- * @param {object} latestNormalPortfolios - PRE-FETCHED portfolio object.
- * @returns {Promise<string[]>} Array of prioritized speculator CIDs.
  */
 async function getPrioritizedSpeculators(dependencies, exclusionIds, speculatorInstrumentSet, latestNormalPortfolios) {
     const { logger } = dependencies;
@@ -143,10 +112,6 @@ async function getPrioritizedSpeculators(dependencies, exclusionIds, speculatorI
 
 /**
  * Deletes all documents within a specified collection path.
- * @param {object} dependencies - Contains db, logger.
- * @param {string} collectionPath - The path to the collection to clear.
- * @param {number} maxBatchSize - Firestore batch size limit.
- * @returns {Promise<void>}
  */
 async function clearCollection(dependencies, collectionPath, maxBatchSize = 400) {
     const { db, logger } = dependencies;
@@ -174,14 +139,6 @@ async function clearCollection(dependencies, collectionPath, maxBatchSize = 400)
 
 /**
  * Writes an array of user IDs into sharded Firestore documents.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.collectionPath - Base path for sharded documents (e.g., 'PendingSpeculators').
- * @param {string[]} config.items - The user IDs to write.
- * @param {Date} config.timestamp - Timestamp to associate with each user ID.
- * @param {number} config.maxFieldsPerDoc - Max user IDs per sharded document.
- * @param {number} config.maxWritesPerBatch - Max documents to write per batch commit.
- * @returns {Promise<void>}
  */
 async function batchWriteShardedIds(dependencies, config) {
     const { db, logger } = dependencies;
@@ -219,12 +176,6 @@ async function batchWriteShardedIds(dependencies, config) {
 
 /**
  * Gets normal users whose timestamp indicates they need updating.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {Date} config.dateThreshold - Users processed before this date will be returned.
- * @param {string} config.normalUserCollectionName - e.g., 'NormalUserPortfolios'.
- * @param {string} config.normalUserTimestampsDocId - e.g., 'normal'.
- * @returns {Promise<string[]>} Array of user IDs to update.
  */
 async function getNormalUsersToUpdate(dependencies, config) {
     const { db, logger } = dependencies;
@@ -248,12 +199,6 @@ async function getNormalUsersToUpdate(dependencies, config) {
 
 /**
  * Gets speculator users/instruments whose data needs updating.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {Date} config.dateThreshold - Verification date threshold.
- * @param {Date} config.gracePeriodThreshold - Last held asset date threshold.
- * @param {string} config.speculatorBlocksCollectionName - e.g., 'SpeculatorBlocks'.
- * @returns {Promise<object[]>} Array of objects like { userId: string, instrumentId: number }.
  */
 async function getSpeculatorsToUpdate(dependencies, config) {
     const { db, logger } = dependencies;
@@ -281,4 +226,93 @@ async function getSpeculatorsToUpdate(dependencies, config) {
     } catch (error) { logger.log('ERROR','[Core Utils] Error getting speculators to update', { errorMessage: error.message }); throw error; }
 }
 
-module.exports = { getLatestNormalUserPortfolios, resetProxyLocks, getBlockCapacities, getExclusionIds, getPrioritizedSpeculators, clearCollection, batchWriteShardedIds, getNormalUsersToUpdate, getSpeculatorsToUpdate, };
+/**
+ * [NEW] Fetches Popular Investors from the daily rankings document.
+ */
+async function getPopularInvestorsToUpdate(dependencies, config) {
+    const { db, logger } = dependencies;
+    const collectionName = config.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings';
+
+    logger.log('INFO', `[Core Utils] Getting Popular Investors to update from ${collectionName}...`);
+    
+    // Construct today's document ID (YYYY-MM-DD)
+    const today = new Date().toISOString().split('T')[0];
+    const docPath = `${collectionName}/${today}`;
+
+    try {
+        const docRef = db.doc(docPath);
+        const docSnap = await docRef.get();
+
+        if (!docSnap.exists) {
+            logger.log('WARN', `[Core Utils] No Popular Investor rankings found for date: ${today} at ${docPath}`);
+            return [];
+        }
+
+        const data = docSnap.data();
+        const items = data.Items || [];
+
+        // Map to { cid, username }
+        const targets = items.map(item => ({
+            cid: String(item.CustomerId),
+            username: item.UserName
+        }));
+
+        logger.log('INFO', `[Core Utils] Found ${targets.length} Popular Investors from ${today} ranking.`);
+        return targets;
+
+    } catch (error) {
+        logger.log('ERROR', '[Core Utils] Error getting Popular Investors', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * [NEW] Fetches all Signed-In Users for daily update.
+ */
+async function getSignedInUsersToUpdate(dependencies, config) {
+    const { db, logger } = dependencies;
+    const collectionName = config.signedInUsersCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_PORTFOLIOS || 'signed_in_users';
+
+    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${collectionName}...`);
+
+    try {
+        const snapshot = await db.collection(collectionName).get();
+        
+        if (snapshot.empty) {
+            logger.log('INFO', '[Core Utils] No Signed-In Users found.');
+            return [];
+        }
+
+        const targets = [];
+        snapshot.forEach(doc => {
+            const data = doc.data();
+            // Assuming the doc ID is the CID, or it's a field 'cid'
+            const cid = data.cid || doc.id; 
+            const username = data.username || 'Unknown';
+            // Only push if we have a valid CID
+            if(cid) targets.push({ cid: String(cid), username });
+        });
+
+        logger.log('INFO', `[Core Utils] Found ${targets.length} Signed-In Users.`);
+        return targets;
+
+    } catch (error) {
+        logger.log('ERROR', '[Core Utils] Error getting Signed-In Users', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+module.exports = { 
+    getLatestNormalUserPortfolios, 
+    resetProxyLocks, 
+    getBlockCapacities, 
+    getExclusionIds, 
+    getPrioritizedSpeculators, 
+    clearCollection, 
+    batchWriteShardedIds, 
+    getNormalUsersToUpdate, 
+    getSpeculatorsToUpdate,
+    // New Exports
+    getPopularInvestorsToUpdate,
+    getSignedInUsersToUpdate
+};

package/functions/orchestrator/helpers/update_helpers.js

@@ -5,7 +5,7 @@
 
 /**
  * Sub-pipe: pipe.orchestrator.getUpdateTargets
- * @param {string} userType - 'normal' or 'speculator'.
+ * @param {string} userType - 'normal', 'speculator', 'popular_investor', or 'signed_in_user'.
  * @param {object} thresholds - Contains date thresholds.
  * @param {object} config - The *global* updateConfig object.
  * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils.
@@ -16,24 +16,39 @@ async function getUpdateTargets(userType, thresholds, config, dependencies) {
     logger.log('INFO', `[Orchestrator Helpers] Getting update targets for ${userType}...`);
     let targets = [];
     try {
-        if (userType === 'normal') { targets = await firestoreUtils.getNormalUsersToUpdate(dependencies,{
-        dateThreshold: thresholds.dateThreshold,
-        normalUserCollectionName: config.normalUserCollectionName,
-        normalUserTimestampsDocId: config.normalUserTimestampsDocId
-    }); } 
-    else if (userType === 'speculator') { targets = await firestoreUtils.getSpeculatorsToUpdate(dependencies,{
-        dateThreshold: thresholds.dateThreshold,
-        gracePeriodThreshold: thresholds.gracePeriodThreshold,
-        speculatorBlocksCollectionName: config.speculatorBlocksCollectionName}); }
-    logger.log('SUCCESS', `[Orchestrator Helpers] Found ${targets.length} targets for ${userType} update.`);
-    return targets;} 
-    catch (error) { logger.log('ERROR', `[Orchestrator Helpers] Error getting update targets for ${userType}`, { errorMessage: error.message }); throw error; }
+        if (userType === 'normal') { 
+            targets = await firestoreUtils.getNormalUsersToUpdate(dependencies,{
+                dateThreshold: thresholds.dateThreshold,
+                normalUserCollectionName: config.normalUserCollectionName,
+                normalUserTimestampsDocId: config.normalUserTimestampsDocId
+            }); 
+        } 
+        else if (userType === 'speculator') { 
+            targets = await firestoreUtils.getSpeculatorsToUpdate(dependencies,{
+                dateThreshold: thresholds.dateThreshold,
+                gracePeriodThreshold: thresholds.gracePeriodThreshold,
+                speculatorBlocksCollectionName: config.speculatorBlocksCollectionName
+            }); 
+        }
+        else if (userType === 'popular_investor') {
+            targets = await firestoreUtils.getPopularInvestorsToUpdate(dependencies, config);
+        }
+        else if (userType === 'signed_in_user') {
+            targets = await firestoreUtils.getSignedInUsersToUpdate(dependencies, config);
+        }
+        
+        logger.log('SUCCESS', `[Orchestrator Helpers] Found ${targets.length} targets for ${userType} update.`);
+        return targets;
+    } catch (error) { 
+        logger.log('ERROR', `[Orchestrator Helpers] Error getting update targets for ${userType}`, { errorMessage: error.message }); 
+        throw error; 
+    }
 }
 
 /**
  * Sub-pipe: pipe.orchestrator.dispatchUpdates
  * @param {Array<any>} targets - Array of targets from getUpdateTargets.
- * @param {string} userType - 'normal' or 'speculator'.
+ * @param {string} userType - 'normal', 'speculator', 'popular_investor', or 'signed_in_user'.
  * @param {object} config - The *global* updateConfig object.
  * @param {object} dependencies - Contains db, pubsub, logger, pubsubUtils.
  * @returns {Promise<void>}
@@ -41,11 +56,43 @@ async function getUpdateTargets(userType, thresholds, config, dependencies) {
 async function dispatchUpdates(targets, userType, config, dependencies) {
     const { logger, pubsubUtils } = dependencies;
     const { dispatcherTopicName, taskBatchSize, pubsubBatchSize } = config;
-    if (targets.length === 0) { logger.log('INFO', `[Orchestrator Helpers] No ${userType} update targets to dispatch.`); return; }
-        logger.log('INFO', `[Orchestrator Helpers] Dispatching ${targets.length} update tasks for ${userType} to ${dispatcherTopicName}...`);
-        const individualTasks = targets.map(target => ({ type: 'update', userId: userType === 'normal' ? target : target.userId, instruments: userType === 'speculator' ? target.instruments : undefined, userType }));        const dispatcherMessages = [];
-        for (let i = 0; i < individualTasks.length; i += taskBatchSize) { dispatcherMessages.push({ tasks: individualTasks.slice(i, i + taskBatchSize) }); }
-        try { await pubsubUtils.batchPublishTasks(dependencies,{ topicName: dispatcherTopicName, tasks: dispatcherMessages, taskType: `${userType} update batch`, maxPubsubBatchSize: pubsubBatchSize }); logger.log('SUCCESS', `[Orchestrator Helpers] Published ${dispatcherMessages.length} messages (${individualTasks.length} tasks) for ${userType} updates to the dispatcher.`); }
-        catch (error) { logger.log('ERROR', `[Orchestrator Helpers] Error dispatching update tasks for ${userType}`, { errorMessage: error.message }); throw error; }}
+    
+    if (targets.length === 0) { 
+        logger.log('INFO', `[Orchestrator Helpers] No ${userType} update targets to dispatch.`); 
+        return; 
+    }
+    
+    logger.log('INFO', `[Orchestrator Helpers] Dispatching ${targets.length} update tasks for ${userType} to ${dispatcherTopicName}...`);
+    
+    const individualTasks = targets.map(target => {
+        if (userType === 'normal') {
+            return { type: 'update', userId: target, userType };
+        } else if (userType === 'speculator') {
+            return { type: 'update', userId: target.userId, instruments: target.instruments, userType };
+        } else if (userType === 'popular_investor') {
+            return { type: 'POPULAR_INVESTOR_UPDATE', cid: target.cid, username: target.username };
+        } else if (userType === 'signed_in_user') {
+            return { type: 'ON_DEMAND_USER_UPDATE', cid: target.cid, username: target.username };
+        }
+    });
 
-module.exports = { getUpdateTargets, dispatchUpdates };
+    const dispatcherMessages = [];
+    for (let i = 0; i < individualTasks.length; i += taskBatchSize) { 
+        dispatcherMessages.push({ tasks: individualTasks.slice(i, i + taskBatchSize) }); 
+    }
+    
+    try { 
+        await pubsubUtils.batchPublishTasks(dependencies,{ 
+            topicName: dispatcherTopicName, 
+            tasks: dispatcherMessages, 
+            taskType: `${userType} update batch`, 
+            maxPubsubBatchSize: pubsubBatchSize 
+        }); 
+        logger.log('SUCCESS', `[Orchestrator Helpers] Published ${dispatcherMessages.length} messages (${individualTasks.length} tasks) for ${userType} updates to the dispatcher.`); 
+    } catch (error) { 
+        logger.log('ERROR', `[Orchestrator Helpers] Error dispatching update tasks for ${userType}`, { errorMessage: error.message }); 
+        throw error; 
+    }
+}
+
+module.exports = { getUpdateTargets, dispatchUpdates };

package/functions/orchestrator/index.js

@@ -21,8 +21,34 @@ async function runUpdateOrchestrator(config, deps) {
     const { logger, firestoreUtils } = deps;
     logger.log('INFO', '🚀 Update Orchestrator triggered...');
     await firestoreUtils.resetProxyLocks(deps, config);
+    
+    // 1. Normal Users
     await runUpdates('normal', config.updateConfig, config, deps);
+    
+    // 2. Speculators
     await runUpdates('speculator', config.updateConfig, config, deps);
+
+    // 3. Popular Investors
+    try {
+        const piConfig = { 
+            ...config.updateConfig, 
+            popularInvestorRankingsCollection: config.updateConfig.popularInvestorRankingsCollection || process.env.FIRESTORE_COLLECTION_PI_RANKINGS || 'popular_investor_rankings'
+        };
+        await runUpdates('popular_investor', piConfig, config, deps);
+    } catch (e) {
+        logger.log('ERROR', '[Orchestrator] Failed to run Popular Investor updates.', e);
+    }
+
+    // 4. Signed-In Users
+    try {
+        const signedInConfig = { 
+            ...config.updateConfig, 
+            signedInUsersCollection: config.updateConfig.signedInUsersCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_PORTFOLIOS || 'signed_in_users'
+        };
+        await runUpdates('signed_in_user', signedInConfig, config, deps);
+    } catch (e) {
+        logger.log('ERROR', '[Orchestrator] Failed to run Signed-In User updates.', e);
+    }
 }
 
 /** Stage 3: Run discovery for a single user type */
@@ -61,4 +87,4 @@ async function runUpdates(userType, updateConfig, globalConfig, deps) {
     logger.log('SUCCESS', `[Orchestrator] Dispatched update tasks for ${userType}.`);
 }
 
-module.exports = { runDiscoveryOrchestrator, runUpdateOrchestrator, runDiscovery, runUpdates };
+module.exports = { runDiscoveryOrchestrator, runUpdateOrchestrator, runDiscovery, runUpdates };

package/functions/task-engine/handler_creator.js

@@ -1,19 +1,14 @@
 /**
  * @fileoverview Main entry point for the Task Engine Cloud Function.
- * Routes incoming Pub/Sub messages to the appropriate helper functions.
  */
 const { handleDiscover } = require('./helpers/discover_helpers');
 const { handleVerify } = require('./helpers/verify_helpers');
-const { handleUpdate } = require('./helpers/update_helpers'); // Existing logic for "Speculators"
+const { handleUpdate } = require('./helpers/update_helpers');
 const { handlePopularInvestorUpdate, handleOnDemandUserUpdate } = require('./helpers/popular_investor_helpers');
 
-/**
- * Handles the incoming request from Pub/Sub or Direct HTTP.
- * @param {object} message - The Pub/Sub message object (or req.body).
- * @param {object} context - The event context.
- * @param {object} config - The Task Engine configuration.
- * @param {object} dependencies - Injected dependencies (db, logger, proxyManager, etc.).
- */
+// IMPORT THE UTILS TO HANDLE BATCHES
+const { executeTasks, prepareTaskBatches } = require('./utils/task_engine_utils'); 
+
 async function handleRequest(message, context, config, dependencies) {
     const { logger } = dependencies;
     
@@ -27,12 +22,27 @@ async function handleRequest(message, context, config, dependencies) {
         return;
     }
 
+    // --- FIX START: Handle Batch vs Single ---
+    
+    // CASE A: Payload is a Batch (from Dispatcher)
+    if (payload.tasks && Array.isArray(payload.tasks)) {
+        logger.log('INFO', `[TaskEngine] Received BATCH of ${payload.tasks.length} tasks.`);
+        const taskId = context.eventId || 'batch-' + Date.now();
+        
+        // Use existing utils to execute the batch
+        const { tasksToRun, otherTasks } = await prepareTaskBatches(payload.tasks, null, logger);
+        await executeTasks(tasksToRun, otherTasks, dependencies, config, taskId);
+        return;
+    }
+
+    // CASE B: Payload is a Single Task (from Cron/On-Demand)
     const { type, data } = payload;
     
     if (!type) {
         logger.log('WARN', '[TaskEngine] Received message with no type.', payload);
         return;
     }
+    // --- FIX END ---
 
     logger.log('INFO', `[TaskEngine] Processing Task: ${type}`, { dataSummary: JSON.stringify(data).substring(0, 100) });
 
@@ -40,23 +50,18 @@ async function handleRequest(message, context, config, dependencies) {
     try {
         switch (type) {
             case 'DISCOVER':
-                await handleDiscover(data, config, dependencies);
+                await handleDiscover(data, 'single-discover', dependencies, config); // Added taskId arg
                 break;
             case 'VERIFY':
-                await handleVerify(data, config, dependencies);
+                await handleVerify(data, 'single-verify', dependencies, config); // Added taskId arg
                 break;
             case 'UPDATE': 
-                // Existing logic for "Speculator" updates (high volume, low fidelity)
-                await handleUpdate(data, config, dependencies);
+                await handleUpdate(data, 'single-update', dependencies, config); // Added taskId arg
                 break;
             case 'POPULAR_INVESTOR_UPDATE':
-                // New logic for Popular Investors (daily cron, high fidelity)
-                // data should contain { cid, username }
                 await handlePopularInvestorUpdate(data, config, dependencies);
                 break;
             case 'ON_DEMAND_USER_UPDATE':
-                // New logic for Signed-In Users (immediate trigger, cost optimized)
-                // data should contain { cid, username }
                 await handleOnDemandUserUpdate(data, config, dependencies);
                 break;
             default:
@@ -64,8 +69,6 @@ async function handleRequest(message, context, config, dependencies) {
         }
     } catch (err) {
         logger.log('ERROR', `[TaskEngine] Error processing task ${type}`, err);
-        // We might want to throw here to trigger Pub/Sub retry, depending on the error nature
-        // For now, logging error is sufficient as the sub-handlers perform their own try/catch blocks
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.358",
+  "version": "1.0.360",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [