bulltrackers-module 1.0.0

package/functions/core/utils/firestore_utils.js

@@ -0,0 +1,427 @@
+/**
+ * @fileoverview Core Firestore utility functions for the Bulltrackers Module.
+ */
+const { Firestore, FieldValue, FieldPath } = require('@google-cloud/firestore');
+const { logger } = require("sharedsetup")(__filename);
+
+const db = new Firestore();
+
+const MAX_FIRESTORE_BATCH_SIZE = 500; // Firestore batch limit
+
+/**
+ * Fetches and merges the most recent portfolio data for all normal users.
+ * This function is required by getPrioritizedSpeculators.
+ * @async
+ * @param {string} normalUserCollectionName - e.g., 'NormalUserPortfolios'.
+ * @param {string} snapshotsSubCollectionName - e.g., 'snapshots'.
+ * @param {string} partsSubCollectionName - e.g., 'parts'.
+ * @returns {Promise<object>} A single object containing all user portfolios.
+ */
+async function getLatestNormalUserPortfolios(normalUserCollectionName, snapshotsSubCollectionName, partsSubCollectionName) {
+    logger.log('INFO', `[Core Utils] Fetching latest portfolios from ${normalUserCollectionName}...`);
+    const allPortfolios = {};
+
+    // Get yesterday's date string
+    const yesterday = new Date();
+    yesterday.setDate(yesterday.getDate() - 1);
+    const dateString = yesterday.toISOString().slice(0, 10);
+
+    const blockDocs = await db.collection(normalUserCollectionName).listDocuments();
+
+    for (const blockDoc of blockDocs) {
+        const snapshotDocRef = blockDoc.collection(snapshotsSubCollectionName).doc(dateString);
+        const partsCollectionRef = snapshotDocRef.collection(partsSubCollectionName);
+        const partsSnapshot = await partsCollectionRef.get();
+
+        if (!partsSnapshot.empty) {
+            partsSnapshot.forEach(partDoc => {
+                Object.assign(allPortfolios, partDoc.data());
+            });
+        }
+    }
+
+    logger.log('INFO', `[Core Utils] Found ${Object.keys(allPortfolios).length} user portfolios from ${dateString}'s snapshot.`);
+    return allPortfolios;
+}
+
+
+/**
+ * Resets the isLocked status for all proxies.
+ * Note: This implementation differs from the original orchestrator's FIRESTORE_DOC_PROXY_PERFORMANCE logic.
+ * This logic matches the original file provided for the module.
+ * @async
+ * @param {string} proxiesCollectionName - e.g., 'Proxies'.
+ * @returns {Promise<void>}
+ */
+async function resetProxyLocks(proxiesCollectionName) {
+    logger.log('INFO','[Core Utils] Resetting proxy locks...');
+    try {
+        const proxiesRef = db.collection(proxiesCollectionName);
+        const snapshot = await proxiesRef.where('status', '==', 1).get(); // Only fetch locked ones
+
+        if (snapshot.empty) {
+            logger.log('INFO','[Core Utils] No proxies found in a locked state.');
+            return;
+        }
+
+        const batch = db.batch();
+        snapshot.docs.forEach(doc => {
+            batch.update(doc.ref, { status: 0 }); // Use 0 for unlocked
+        });
+
+        await batch.commit();
+        logger.log('INFO',`[Core Utils] Proxy locks reset for ${snapshot.size} proxies.`);
+    } catch (error) {
+        logger.log('ERROR','[Core Utils] Error resetting proxy locks', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Fetches and aggregates block capacities.
+ * @async
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {string} speculatorBlockCountsDocPath - Path to speculator counts doc.
+ * @param {string} normalBlockCountsDocPath - Path to normal counts doc.
+ * @returns {Promise<object>} Object containing counts.
+ */
+async function getBlockCapacities(userType, speculatorBlockCountsDocPath, normalBlockCountsDocPath) {
+    logger.log('INFO',`[Core Utils] Getting block capacities for ${userType}...`);
+    try {
+        const docPath = userType === 'speculator'
+            ? speculatorBlockCountsDocPath
+            : normalBlockCountsDocPath;
+            
+        if (!docPath) {
+             logger.log('ERROR', `[Core Utils] Missing block counts document path for ${userType}.`);
+             return {};
+        }
+
+        const countsRef = db.doc(docPath);
+        const countsDoc = await countsRef.get();
+        if (!countsDoc.exists) {
+            logger.log('WARN',`[Core Utils] Block counts document not found for ${userType} at ${docPath}. Returning empty.`);
+            return {};
+        }
+        return countsDoc.data().counts || {};
+    } catch (error) {
+        logger.log('ERROR',`[Core Utils] Error getting block capacities for ${userType}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Fetches user IDs from multiple sources for exclusion lists.
+ * @async
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {object} config - Configuration object.
+ * @param {string} config.specBlocksCollection - e.g., 'SpeculatorBlocks'.
+ * @param {string} config.pendingSpecCollection - e.g., 'PendingSpeculators'.
+ * @param {string} config.normalUserCollection - e.g., 'NormalUserPortfolios'.
+ * @param {string} config.invalidSpecCollection - e.g., 'InvalidSpeculators'.
+ * @returns {Promise<Set<string>>} A Set containing unique user IDs.
+ */
+async function getExclusionIds(userType, config) {
+    logger.log('INFO',`[Core Utils] Getting exclusion IDs for ${userType} discovery...`);
+    const { 
+        specBlocksCollection, 
+        pendingSpecCollection, 
+        normalUserCollection, 
+        invalidSpecCollection 
+    } = config;
+    
+    const exclusionIds = new Set();
+    const promises = [];
+
+    try {
+        // 1. Existing Speculators
+        const specBlocksRef = db.collection(specBlocksCollection);
+        promises.push(specBlocksRef.get().then(snapshot => {
+            snapshot.forEach(doc => {
+                 const users = doc.data().users || {};
+                 Object.keys(users).forEach(key => exclusionIds.add(key.split('.')[1]));
+            });
+            logger.log('TRACE','[Core Utils] Fetched existing speculator IDs for exclusion.');
+        }));
+
+        // 2. Pending Speculators
+        if (userType === 'speculator') {
+            const pendingRef = db.collection(pendingSpecCollection);
+             promises.push(pendingRef.get().then(snapshot => {
+                snapshot.forEach(doc => {
+                    Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));
+                });
+                logger.log('TRACE','[Core Utils] Fetched pending speculator IDs for exclusion.');
+             }));
+        }
+
+        // 3. Existing Normal Users
+        const normalPortfoliosRef = db.collection(normalUserCollection);
+        promises.push(normalPortfoliosRef.listDocuments().then(async blockRefs => {
+            for(const blockRef of blockRefs) {
+                // Assuming 'snapshots' and 'parts' subcollections as per original logic
+                const snapshotRef = blockRef.collection('snapshots'); 
+                const latestSnapshotQuery = snapshotRef.orderBy(FieldPath.documentId(), 'desc').limit(1);
+                const latestSnapshot = await latestSnapshotQuery.get();
+                if (!latestSnapshot.empty) {
+                    const partsRef = latestSnapshot.docs[0].ref.collection('parts');
+                    const partsSnapshot = await partsRef.get();
+                    partsSnapshot.forEach(partDoc => {
+                        Object.keys(partDoc.data()).forEach(uid => exclusionIds.add(uid));
+                    });
+                }
+            }
+             logger.log('TRACE','[Core Utils] Fetched existing normal user IDs for exclusion.');
+        }));
+
+
+        // 4. Invalid Speculators
+        const invalidRef = db.collection(invalidSpecCollection);
+         promises.push(invalidRef.get().then(snapshot => {
+             snapshot.forEach(doc => {
+                 Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));
+             });
+              logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');
+         }));
+
+
+        await Promise.all(promises);
+        logger.log('INFO',`[Core Utils] Total unique exclusion IDs found: ${exclusionIds.size}`);
+        return exclusionIds;
+
+    } catch (error) {
+        logger.log('ERROR','[Core Utils] Error getting exclusion IDs', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Scans normal user portfolios for potential speculator candidates.
+ * @async
+ * @param {Set<string>} exclusionIds - IDs to exclude.
+ * @param {Set<number>} speculatorInstrumentSet - Set of instrument IDs.
+ * @param {string} normalUserCollectionName - e.g., 'NormalUserPortfolios'.
+ * @param {string} snapshotsSubCollectionName - e.g., 'snapshots'.
+ * @param {string} partsSubCollectionName - e.g., 'parts'.
+ * @returns {Promise<string[]>} Array of prioritized speculator CIDs.
+ */
+async function getPrioritizedSpeculators(exclusionIds, speculatorInstrumentSet, normalUserCollectionName, snapshotsSubCollectionName, partsSubCollectionName) {
+    logger.log('INFO','[Core Utils] Scanning normal users for prioritized speculators...');
+    const candidates = new Set();
+
+    try {
+        const latestNormalPortfolios = await getLatestNormalUserPortfolios(
+            normalUserCollectionName, 
+            snapshotsSubCollectionName, 
+            partsSubCollectionName
+        );
+
+        for (const userId in latestNormalPortfolios) {
+            if (exclusionIds.has(userId)) continue;
+
+            const portfolio = latestNormalPortfolios[userId];
+            const holdsSpeculatorAsset = portfolio?.AggregatedPositions?.some(p =>
+                speculatorInstrumentSet.has(p.InstrumentID)
+            );
+
+            if (holdsSpeculatorAsset) {
+                candidates.add(userId);
+            }
+        }
+        logger.log('INFO',`[Core Utils] Found ${candidates.size} potential prioritized speculators.`);
+        return Array.from(candidates);
+    } catch (error) {
+        logger.log('ERROR','[Core Utils] Error getting prioritized speculators', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Deletes all documents within a specified collection path (scorched earth).
+ * @async
+ * @param {string} collectionPath - The path to the collection to clear.
+ * @returns {Promise<void>}
+ */
+async function clearCollection(collectionPath) {
+    logger.log('WARN', `[Core Utils] Starting SCORCHED EARTH delete for collection: ${collectionPath}...`);
+    try {
+        const collectionRef = db.collection(collectionPath);
+        const batchSize = MAX_FIRESTORE_BATCH_SIZE;
+        let query = collectionRef.limit(batchSize);
+        let snapshot;
+        let deleteCount = 0;
+
+        while (true) {
+            snapshot = await query.get();
+            if (snapshot.size === 0) {
+                break; 
+            }
+
+            const batch = db.batch();
+            snapshot.docs.forEach(doc => batch.delete(doc.ref));
+            await batch.commit();
+            deleteCount += snapshot.size;
+
+            if (snapshot.size < batchSize) {
+                break; 
+            }
+             query = collectionRef.limit(batchSize); 
+        }
+        logger.log('SUCCESS', `[Core Utils] Scorched earth complete. Deleted ${deleteCount} documents from ${collectionPath}.`);
+    } catch (error) {
+        logger.log('ERROR', `[Core Utils] Error clearing collection ${collectionPath}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Writes an array of user IDs into sharded Firestore documents, matching the original
+ * orchestrator's "ultra aggressive" small batch logic.
+ * @async
+ * @param {string} collectionPath - Base path for sharded documents (e.g., 'PendingSpeculators').
+ * @param {string[]} items - The user IDs to write.
+ * @param {Date} timestamp - Timestamp to associate with each user ID.
+ * @param {number} maxFieldsPerDoc - Max user IDs per sharded document.
+ * @param {number} maxWritesPerBatch - Max documents to write per batch commit.
+ * @returns {Promise<void>}
+ */
+async function batchWriteShardedIds(collectionPath, items, timestamp, maxFieldsPerDoc, maxWritesPerBatch) {
+    logger.log('INFO', `[Core Utils] Batch writing ${items.length} IDs to sharded path: ${collectionPath} (max ${maxFieldsPerDoc}/doc, ${maxWritesPerBatch} docs/batch)...`);
+    if (items.length === 0) return;
+
+    try {
+        const collectionRef = db.collection(collectionPath);
+        let batch = db.batch();
+        let currentDocFields = {};
+        let currentFieldCount = 0;
+        let batchWriteCount = 0;
+        let docCounter = 0;
+
+        for (let i = 0; i < items.length; i++) {
+            const userId = items[i];
+            const key = `users.${userId}`; 
+            currentDocFields[key] = timestamp;
+            currentFieldCount++;
+
+            if (currentFieldCount >= maxFieldsPerDoc || i === items.length - 1) {
+                const docRef = collectionRef.doc(`pending_${docCounter}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`);
+                batch.set(docRef, currentDocFields); 
+                batchWriteCount++;
+
+                currentDocFields = {};
+                currentFieldCount = 0;
+                docCounter++;
+
+                if (batchWriteCount >= maxWritesPerBatch || i === items.length - 1) {
+                    await batch.commit();
+                    batch = db.batch(); 
+                    batchWriteCount = 0;
+                }
+            }
+        }
+         logger.log('SUCCESS', `[Core Utils] Sharded write complete for ${collectionPath}. Created ${docCounter} documents.`);
+    } catch (error) {
+        logger.log('ERROR', `[Core Utils] Error during sharded write to ${collectionPath}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+
+/**
+ * Gets normal users whose timestamp indicates they need updating.
+ * @async
+ * @param {Date} dateThreshold - Users processed before this date will be returned.
+ * @param {string} normalUserCollectionName - e.g., 'NormalUserPortfolios'.
+ * @param {string} normalUserTimestampsDocId - e.g., 'normal'.
+ * @returns {Promise<string[]>} Array of user IDs to update.
+ */
+async function getNormalUsersToUpdate(dateThreshold, normalUserCollectionName, normalUserTimestampsDocId) {
+    logger.log('INFO','[Core Utils] Getting normal users to update...');
+    const usersToUpdate = [];
+    try {
+        const timestampDocRef = db.collection(normalUserCollectionName)
+                                 .doc('timestamps')
+                                 .collection('users')
+                                 .doc(normalUserTimestampsDocId); 
+        const timestampDoc = await timestampDocRef.get();
+
+        if (!timestampDoc.exists) {
+            logger.log('WARN',`[Core Utils] Normal user timestamp document not found at ${timestampDocRef.path}.`);
+            return [];
+        }
+
+        const timestamps = timestampDoc.data().users || {};
+        for (const userId in timestamps) {
+            const lastProcessed = timestamps[userId]?.toDate ? timestamps[userId].toDate() : new Date(0);
+            if (lastProcessed < dateThreshold) {
+                usersToUpdate.push(userId);
+            }
+        }
+        logger.log('INFO',`[Core Utils] Found ${usersToUpdate.length} normal users to update.`);
+        return usersToUpdate;
+    } catch (error) {
+        logger.log('ERROR','[Core Utils] Error getting normal users to update', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+/**
+ * Gets speculator users/instruments whose data needs updating.
+ * @async
+ * @param {Date} dateThreshold - Verification date threshold.
+ * @param {Date} gracePeriodThreshold - Last held asset date threshold.
+ * @param {string} speculatorBlocksCollectionName - e.g., 'SpeculatorBlocks'.
+ * @returns {Promise<object[]>} Array of objects like { userId: string, instrumentId: number }.
+ */
+async function getSpeculatorsToUpdate(dateThreshold, gracePeriodThreshold, speculatorBlocksCollectionName) {
+    logger.log('INFO','[Core Utils] Getting speculators to update...');
+    const updates = [];
+    try {
+        const blocksRef = db.collection(speculatorBlocksCollectionName);
+        const snapshot = await blocksRef.get();
+
+        if (snapshot.empty) {
+            logger.log('INFO','[Core Utils] No speculator blocks found.');
+            return [];
+        }
+
+        snapshot.forEach(doc => {
+            const blockData = doc.data();
+            const users = blockData.users || {};
+
+            for (const key in users) {
+                 const userId = key.split('.')[1];
+                 const userData = users[key];
+
+                 const lastVerified = userData.lastVerified?.toDate ? userData.lastVerified.toDate() : new Date(0);
+                 const lastHeld = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : new Date(0);
+
+                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) {
+                     if (userData.instruments && Array.isArray(userData.instruments)) {
+                         userData.instruments.forEach(instrumentId => {
+                             updates.push({ userId, instrumentId });
+                         });
+                     }
+                 }
+            }
+        });
+
+        logger.log('INFO',`[Core Utils] Found ${updates.length} speculator user/instrument pairs to update.`);
+        return updates;
+    } catch (error) {
+        logger.log('ERROR','[Core Utils] Error getting speculators to update', { errorMessage: error.message });
+        throw error;
+    }
+}
+
+module.exports = {
+  getLatestNormalUserPortfolios, // Export new function
+  resetProxyLocks,
+  getBlockCapacities,
+  getExclusionIds,
+  getPrioritizedSpeculators,
+  clearCollection,
+  batchWriteShardedIds,
+  getNormalUsersToUpdate,
+  getSpeculatorsToUpdate,
+};

package/functions/core/utils/index.js

@@ -0,0 +1,13 @@
+/**
+ * @fileoverview Exports core utility modules.
+ */
+
+const firestoreUtils = require('./firestore_utils');
+const pubsubUtils = require('./pubsub_utils');
+const loggingWrapper = require('./logging_wrapper');
+
+module.exports = {
+  firestore: firestoreUtils,
+  pubsub: pubsubUtils,
+  logging: loggingWrapper,
+};

package/functions/core/utils/logging_wrapper.js

@@ -0,0 +1,56 @@
+/**
+ * @fileoverview Wrappers for Cloud Functions and logic stages for consistent logging and error handling.
+ */
+
+// Placeholder for a potential shared logger instance if needed
+// const { logger } = require('shared-logger-package'); // Example
+
+/**
+ * Wraps an entire HTTP Cloud Function for standardized logging and response handling.
+ * @async
+ * @param {string} functionName - The name of the Cloud Function.
+ * @param {Function} logicCallback - The async function containing the core logic.
+ * @returns {Function} An Express middleware function.
+ */
+function handleCloudFunction(functionName, logicCallback) {
+  return async (req, res) => {
+    console.log(`[${functionName}] Function triggered.`); // Replace with actual logger
+    try {
+      await logicCallback(req, res); // Pass req and res if needed by the logic
+      if (!res.headersSent) {
+          console.log(`[${functionName}] Function completed successfully.`);
+          res.status(200).send(`${functionName} completed.`);
+      }
+    } catch (error) {
+      console.error(`[${functionName}] FATAL Error: ${error.message}`, error.stack); // Replace with actual logger
+       if (!res.headersSent) {
+           res.status(500).send(`Internal Server Error in ${functionName}.`);
+       }
+    }
+  };
+}
+
+/**
+ * Wraps a specific stage within a Cloud Function's logic for logging start/end and errors.
+ * @async
+ * @param {string} stageName - A descriptive name for the stage.
+ * @param {Function} logicCallback - The async function containing the stage's logic.
+ * @returns {Promise<any>} The result of the logicCallback.
+ * @throws Will re-throw any errors caught during the stage execution.
+ */
+async function handleLogicStage(stageName, logicCallback) {
+  console.log(`[Stage: ${stageName}] Starting...`); // Replace with actual logger
+  try {
+    const result = await logicCallback();
+    console.log(`[Stage: ${stageName}] Completed successfully.`); // Replace with actual logger
+    return result;
+  } catch (error) {
+    console.error(`[Stage: ${stageName}] Error: ${error.message}`, error.stack); // Replace with actual logger
+    throw error; // Re-throw to allow the main handler to catch it
+  }
+}
+
+module.exports = {
+  handleCloudFunction,
+  handleLogicStage,
+};

package/functions/core/utils/pubsub_utils.js

@@ -0,0 +1,51 @@
+/**
+ * @fileoverview Core Pub/Sub utility functions for the Bulltrackers Module.
+ */
+const { PubSub } = require('@google-cloud/pubsub');
+const { logger } = require("sharedsetup")(__filename);
+
+const pubsub = new PubSub();
+// const MAX_PUBSUB_BATCH_SIZE = 500; // Removed from here
+
+/**
+ * Publishes an array of tasks to a specified Pub/Sub topic in batches.
+ * @async
+ * @param {string} topicName - The name of the Pub/Sub topic.
+ * @param {Array<object>} tasks - The tasks to publish (each object will be JSON serialized).
+ * @param {string} taskType - A descriptor for the task type (for logging).
+ * @param {number} [maxPubsubBatchSize=500] - Max messages to publish in one client batch.
+ * @returns {Promise<void>}
+ */
+async function batchPublishTasks(topicName, tasks, taskType, maxPubsubBatchSize = 500) {
+    if (!tasks || tasks.length === 0) {
+        logger.log('INFO',`[Core Utils] No ${taskType} tasks to publish to ${topicName}.`);
+        return;
+    }
+    logger.log('INFO',`[Core Utils] Publishing ${tasks.length} ${taskType} tasks to ${topicName}...`);
+    const topic = pubsub.topic(topicName);
+    let messagesPublished = 0;
+
+    try {
+        for (let i = 0; i < tasks.length; i += maxPubsubBatchSize) {
+            const batchTasks = tasks.slice(i, i + maxPubsubBatchSize);
+            const batchPromises = batchTasks.map(task => {
+                const dataBuffer = Buffer.from(JSON.stringify(task));
+                return topic.publishMessage({ data: dataBuffer })
+                    .catch(err => logger.log('ERROR', `[Core Utils] Failed to publish single message for ${taskType}`, { error: err.message, task: task }));
+            });
+            await Promise.all(batchPromises);
+            messagesPublished += batchTasks.length;
+            logger.log('TRACE', `[Core Utils] Published batch ${Math.ceil((i + 1) / maxPubsubBatchSize)} for ${taskType} (${batchTasks.length} messages)`);
+        }
+
+        logger.log('SUCCESS', `[Core Utils] Finished publishing ${messagesPublished} ${taskType} tasks to ${topicName}.`);
+
+    } catch (error) {
+        logger.log('ERROR', `[Core Utils] Error during batch publishing of ${taskType} tasks to ${topicName}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+module.exports = {
+  batchPublishTasks,
+};

package/functions/orchestrator/helpers/discovery_helpers.js

@@ -0,0 +1,235 @@
+/**
+ * @fileoverview Helper functions specific to the Orchestrator's discovery logic.
+ */
+const { logger } = require("sharedsetup")(__filename);
+const coreUtils = require('../../core/utils');
+const { firestore: firestoreUtils, pubsub: pubsubUtils } = coreUtils;
+
+// --- All Constants Removed ---
+
+/**
+ * Checks if discovery is needed for a given user type based on block capacities.
+ * @async
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {number} targetUsersPerBlock - Target number of users per block.
+ * @param {object} config - Configuration object.
+ * @param {string} config.speculatorBlockCountsDocPath - Path to speculator counts doc.
+ * @param {string} config.normalBlockCountsDocPath - Path to normal counts doc.
+ * @param {Array<number>} config.speculatorInstruments - Array of instrument IDs.
+ * @param {Array<object>} config.allHighValueBlocks - Array of block objects (e.g., [{ startId: 19000000 }]).
+ * @returns {Promise<{needsDiscovery: boolean, blocksToFill: Array<object>}>}
+ */
+async function checkDiscoveryNeed(userType, targetUsersPerBlock, config) {
+    logger.log('INFO', `[Orchestrator Helpers] Checking discovery need for ${userType}...`);
+    const { 
+        speculatorBlockCountsDocPath, 
+        normalBlockCountsDocPath,
+        speculatorInstruments,
+        allHighValueBlocks
+    } = config;
+
+    const isSpeculator = userType === 'speculator';
+    const blockCounts = await firestoreUtils.getBlockCapacities(
+        userType, 
+        speculatorBlockCountsDocPath, 
+        normalBlockCountsDocPath
+    );
+
+    let underPopulatedBlocks = [];
+    if (isSpeculator) {
+        for (const instrument of speculatorInstruments) {
+            for (const block of allHighValueBlocks) {
+                const instrumentBlockKey = `${instrument}_${block.startId}`;
+                if ((blockCounts[instrumentBlockKey] || 0) < targetUsersPerBlock) {
+                    underPopulatedBlocks.push({ ...block, instrument });
+                }
+            }
+        }
+    } else {
+        underPopulatedBlocks = allHighValueBlocks.filter(
+            block => (blockCounts[block.startId] || 0) < targetUsersPerBlock
+        );
+    }
+
+    const needsDiscovery = underPopulatedBlocks.length > 0;
+    if (!needsDiscovery) {
+        logger.log('SUCCESS', `✅ All blocks are at target capacity for ${userType} users.`);
+    } else {
+        logger.log('INFO', `Found ${underPopulatedBlocks.length} underpopulated blocks/instruments for ${userType}.`);
+    }
+    return { needsDiscovery, blocksToFill: underPopulatedBlocks };
+}
+
+/**
+ * Gathers candidate user IDs for discovery.
+ * @async
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {Array<object>} blocksToFill - From checkDiscoveryNeed.
+ * @param {object} config - Configuration object.
+ * @param {number} config.maxTasksPerRun - Max total candidates to generate.
+ * @param {number} config.discoveryBatchSize - Size of each random batch to generate.
+ * @param {number} config.maxRandomCidsToDiscover - Max random CIDs to add.
+ * @param {string} config.specBlocksCollection - e.g., 'SpeculatorBlocks'.
+ * @param {string} config.pendingSpecCollection - e.g., 'PendingSpeculators'.
+ * @param {string} config.normalUserCollection - e.g., 'NormalUserPortfolios'.
+ * @param {string} config.invalidSpecCollection - e.g., 'InvalidSpeculators'.
+ * @param {Set<number>} config.speculatorInstrumentSet - Set of instrument IDs.
+ * @param {string} config.snapshotsSubCollectionName - e.g., 'snapshots'.
+ * @param {string} config.partsSubCollectionName - e.g., 'parts'.
+ * @returns {Promise<Set<string>>} Set of candidate CIDs.
+ */
+async function getDiscoveryCandidates(userType, blocksToFill, config) {
+    logger.log('INFO', `[Orchestrator Helpers] Getting discovery candidates for ${userType}...`);
+    const {
+        maxTasksPerRun,
+        discoveryBatchSize,
+        maxRandomCidsToDiscover,
+        specBlocksCollection,
+        pendingSpecCollection,
+        normalUserCollection,
+        invalidSpecCollection,
+        speculatorInstrumentSet,
+        snapshotsSubCollectionName,
+        partsSubCollectionName
+    } = config;
+    
+    const isSpeculator = userType === 'speculator';
+    const dispatchedCids = new Set();
+
+    const exclusionIds = await firestoreUtils.getExclusionIds(userType, {
+        specBlocksCollection, 
+        pendingSpecCollection, 
+        normalUserCollection, 
+        invalidSpecCollection
+    });
+
+    // --- Prioritization for Speculators ---
+    if (isSpeculator) {
+        const prioritizedCandidates = await firestoreUtils.getPrioritizedSpeculators(
+            exclusionIds, 
+            speculatorInstrumentSet,
+            normalUserCollection,
+            snapshotsSubCollectionName,
+            partsSubCollectionName
+        );
+        logger.log('INFO', `Found ${prioritizedCandidates.length} potential new speculators from existing user pool.`);
+
+        prioritizedCandidates.forEach(cidStr => {
+            if (dispatchedCids.size < maxTasksPerRun) {
+                dispatchedCids.add(cidStr);
+            }
+        });
+        logger.log('INFO', `Added ${dispatchedCids.size} prioritized speculators.`);
+    }
+
+    // --- Random Generation ---
+    let dispatchedRandomCidCount = 0;
+    const initialDispatchedCount = dispatchedCids.size;
+
+    while ((dispatchedRandomCidCount + initialDispatchedCount) < maxRandomCidsToDiscover &&
+        dispatchedCids.size < maxTasksPerRun && blocksToFill.length > 0)
+    {
+        const blockIndex = dispatchedCids.size % blocksToFill.length;
+        const block = blocksToFill[blockIndex];
+        if (!block) break; 
+
+        for (let j = 0; j < discoveryBatchSize; j++) {
+            if ((dispatchedRandomCidCount + initialDispatchedCount) >= maxRandomCidsToDiscover || dispatchedCids.size >= maxTasksPerRun) break;
+
+            let randomId;
+            let retryCount = 0;
+            const MAX_RETRIES = 50; // This can remain as it's algorithm-specific
+            do {
+                if (++retryCount > MAX_RETRIES) break;
+                randomId = String(Math.floor(Math.random() * 1000000) + block.startId);
+            } while (
+                exclusionIds.has(randomId) ||
+                dispatchedCids.has(randomId)
+            );
+
+            if (retryCount <= MAX_RETRIES) {
+                dispatchedCids.add(randomId);
+                dispatchedRandomCidCount++;
+            }
+        }
+    }
+
+    logger.log('INFO', `Generated ${dispatchedRandomCidCount} random CIDs for ${userType} discovery.`);
+    logger.log('INFO', `Total candidates for dispatch: ${dispatchedCids.size}`);
+    return dispatchedCids; 
+}
+
+/**
+ * Manages pending lists and dispatches discovery tasks.
+ * @async
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {Set<string>} candidates - Set of candidate CIDs.
+ * @param {object} config - Configuration object.
+ * @param {string} config.topicName - The Pub/Sub topic to publish tasks to.
+ * @param {number} config.dispatchBatchSize - How many CIDs per Pub/Sub message (task).
+ * @param {number} config.pubsubBatchSize - Max messages per Pub/Sub publish call.
+ * @param {string} config.pendingSpeculatorsCollection - e.g., 'PendingSpeculators'.
+ * @param {number} config.pendingMaxFieldsPerDoc - Max fields for sharded doc.
+ * @param {number} config.pendingMaxWritesPerBatch - Max docs for sharded write batch.
+ * @returns {Promise<void>}
+ */
+async function dispatchDiscovery(userType, candidates, config) {
+    const {
+        topicName,
+        dispatchBatchSize,
+        pubsubBatchSize,
+        pendingSpeculatorsCollection,
+        pendingMaxFieldsPerDoc,
+        pendingMaxWritesPerBatch
+    } = config;
+
+    if (candidates.size === 0) {
+        logger.log('INFO', `[Orchestrator Helpers] No ${userType} candidates to dispatch.`);
+        return;
+    }
+    logger.log('INFO', `[Orchestrator Helpers] Dispatching ${candidates.size} discovery tasks for ${userType}...`);
+    
+    const isSpeculator = userType === 'speculator';
+    const cidsArray = Array.from(candidates);
+
+    if (isSpeculator) {
+        await firestoreUtils.clearCollection(pendingSpeculatorsCollection);
+        await firestoreUtils.batchWriteShardedIds(
+            pendingSpeculatorsCollection, 
+            cidsArray, 
+            new Date(),
+            pendingMaxFieldsPerDoc,
+            pendingMaxWritesPerBatch
+        );
+    }
+
+    // Format candidates into tasks IN BATCHES
+    const tasks = [];
+    for (let i = 0; i < cidsArray.length; i += dispatchBatchSize) {
+        const batchCids = cidsArray.slice(i, i + dispatchBatchSize).map(cid => parseInt(cid)); 
+        if (batchCids.length > 0) {
+            const blockId = Math.floor(batchCids[0] / 1000000) * 1000000;
+            tasks.push({
+                type: 'discover',
+                cids: batchCids,
+                blockId, 
+                userType
+            });
+        }
+    }
+
+    await pubsubUtils.batchPublishTasks(
+        topicName, 
+        tasks, 
+        `${userType} discovery`,
+        pubsubBatchSize
+    );
+    logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${tasks.length} task messages (${candidates.size} CIDs) for ${userType} discovery.`);
+}
+
+
+module.exports = {
+  checkDiscoveryNeed,
+  getDiscoveryCandidates,
+  dispatchDiscovery,
+};

package/functions/orchestrator/helpers/index.js

@@ -0,0 +1,11 @@
+/**
+ * @fileoverview Exports Orchestrator helper modules.
+ */
+
+const discoveryHelpers = require('./discovery_helpers');
+const updateHelpers = require('./update_helpers');
+
+module.exports = {
+  discovery: discoveryHelpers,
+  updates: updateHelpers,
+};

package/functions/orchestrator/helpers/update_helpers.js

@@ -0,0 +1,96 @@
+/**
+ * @fileoverview Helper functions specific to the Orchestrator's update logic.
+ */
+const { logger } = require("sharedsetup")(__filename);
+const coreUtils = require('../../core/utils');
+const { firestore: firestoreUtils, pubsub: pubsubUtils } = coreUtils;
+
+// --- Constants Removed ---
+
+/**
+ * Gets the target users/instruments that need updating based on thresholds.
+ * @async
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {object} thresholds - Contains date thresholds.
+ * @param {Date} thresholds.dateThreshold - The primary date threshold.
+ * @param {Date} [thresholds.gracePeriodThreshold] - Speculator inactivity grace period.
+ * @param {object} config - Configuration object.
+ * @param {string} config.normalUserCollectionName - e.g., 'NormalUserPortfolios'.
+ * @param {string} config.normalUserTimestampsDocId - e.g., 'normal'.
+ * @param {string} config.speculatorBlocksCollectionName - e.g., 'SpeculatorBlocks'.
+ * @returns {Promise<Array<any>>} Array of targets (user IDs or user/instrument objects).
+ */
+async function getUpdateTargets(userType, thresholds, config) {
+    logger.log('INFO', `[Orchestrator Helpers] Getting update targets for ${userType}...`);
+    let targets = [];
+
+    try {
+        if (userType === 'normal') {
+            targets = await firestoreUtils.getNormalUsersToUpdate(
+                thresholds.dateThreshold,
+                config.normalUserCollectionName,
+                config.normalUserTimestampsDocId
+            );
+        } else if (userType === 'speculator') {
+            targets = await firestoreUtils.getSpeculatorsToUpdate(
+                thresholds.dateThreshold, 
+                thresholds.gracePeriodThreshold,
+                config.speculatorBlocksCollectionName
+            );
+        }
+        logger.log('SUCCESS', `[Orchestrator Helpers] Found ${targets.length} targets for ${userType} update.`);
+        return targets;
+    } catch (error) {
+         logger.log('ERROR', `[Orchestrator Helpers] Error getting update targets for ${userType}`, { errorMessage: error.message });
+         throw error;
+    }
+}
+
+/**
+ * Formats targets into tasks and dispatches them to the specified topic.
+ * @async
+ * @param {Array<any>} targets - Array of targets from getUpdateTargets.
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {string} topicName - The Pub/Sub topic to publish tasks to (Dispatcher topic).
+ * @param {number} taskBatchSize - Max tasks per dispatcher message (e.g., 500).
+ * @param {number} pubsubBatchSize - Max messages per Pub/Sub publish call.
+ * @returns {Promise<void>}
+ */
+async function dispatchUpdates(targets, userType, topicName, taskBatchSize, pubsubBatchSize) {
+    if (targets.length === 0) {
+        logger.log('INFO', `[Orchestrator Helpers] No ${userType} update targets to dispatch.`);
+        return;
+    }
+    logger.log('INFO', `[Orchestrator Helpers] Dispatching ${targets.length} update tasks for ${userType} to ${topicName}...`);
+
+    const individualTasks = targets.map(target => ({
+        type: 'update',
+        userId: userType === 'normal' ? target : target.userId,
+        instrumentId: userType === 'speculator' ? target.instrumentId : undefined,
+        userType,
+    }));
+
+    const dispatcherMessages = [];
+    for (let i = 0; i < individualTasks.length; i += taskBatchSize) {
+        const batch = individualTasks.slice(i, i + taskBatchSize);
+        dispatcherMessages.push({ tasks: batch }); // Wrap the batch
+    }
+
+    try {
+        await pubsubUtils.batchPublishTasks(
+            topicName, 
+            dispatcherMessages, 
+            `${userType} update batch`,
+            pubsubBatchSize // Pass this down to the core util
+        );
+        logger.log('SUCCESS', `[Orchestrator Helpers] Published ${dispatcherMessages.length} messages (${individualTasks.length} tasks) for ${userType} updates to the dispatcher.`);
+    } catch (error) {
+        logger.log('ERROR', `[Orchestrator Helpers] Error dispatching update tasks for ${userType}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+module.exports = {
+  getUpdateTargets,
+  dispatchUpdates,
+};

package/functions/orchestrator/index.js

@@ -0,0 +1,11 @@
+/**
+ * @fileoverview Exports modules related to the Orchestrator function.
+ */
+
+const helpers = require('./helpers');
+// const utils = require('./utils'); // If function-specific utils are needed
+
+module.exports = {
+  helpers,
+  // utils,
+};

package/functions/orchestrator/utils/index.js

@@ -0,0 +1,10 @@
+/**
+ * @fileoverview Exports Orchestrator-specific utility functions.
+ * (Currently empty as per design, but structure is ready).
+ */
+
+// Add Orchestrator-specific utils here if needed.
+
+module.exports = {
+  // export utils
+};

package/index.js

@@ -0,0 +1,16 @@
+/**
+ * @fileoverview Main entry point for the BulltrackersModule package.
+ * Exports core utilities and function-specific modules.
+ */
+
+const coreUtils = require('./functions/core/utils');
+const orchestrator = require('./functions/orchestrator'); // Adjust path as needed
+
+module.exports = {
+  core: {
+    utils: coreUtils,
+    logging: coreUtils.logging // Expose logging wrapper directly under core
+  },
+  Orchestrator: orchestrator,
+  // Add other function modules here as they are refactored (e.g., TaskEngine, ComputationSystem)
+};

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "bulltrackers-module",
+  "version": "1.0.0",
+  "description": "Helper Functions for Bulltrackers.",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "functions/",
+    "utils/"
+  ],
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "bulltrackers",
+    "etoro",
+    "precompute",
+    "calculations",
+    "finance"
+  ],
+  "dependencies": {
+    "@google-cloud/firestore": "^7.11.3",
+    "sharedsetup": "latest",
+    "require-all": "^3.0.0"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}