bulltrackers-module 1.0.104 → 1.0.106

package/functions/core/utils/pubsub_utils.js

@@ -1,53 +1,53 @@
-/**
- * @fileoverview Core Pub/Sub utility functions.
- * REFACTORED: All functions are now stateless and receive dependencies.
- * 'pubsub' (PubSub instance) and 'logger' are passed via a 'dependencies' object.
- */
-
-/**
- * Publishes an array of tasks to a specified Pub/Sub topic in batches.
- * @async
- * @param {object} dependencies - Contains pubsub, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.topicName - The name of the Pub/Sub topic.
- * @param {Array<object>} config.tasks - The tasks to publish.
- * @param {string} config.taskType - A descriptor for the task type (for logging).
- * @param {number} [config.maxPubsubBatchSize=500] - Max messages to publish in one client batch.
- * @returns {Promise<void>}
- */
-async function batchPublishTasks(dependencies, config) {
-    const { pubsub, logger } = dependencies;
-    const { topicName, tasks, taskType, maxPubsubBatchSize = 500 } = config;
-
-    if (!tasks || tasks.length === 0) {
-        logger.log('INFO',`[Core Utils] No ${taskType} tasks to publish to ${topicName}.`);
-        return;
-    }
-    logger.log('INFO',`[Core Utils] Publishing ${tasks.length} ${taskType} tasks to ${topicName}...`);
-    const topic = pubsub.topic(topicName);
-    let messagesPublished = 0;
-
-    try {
-        for (let i = 0; i < tasks.length; i += maxPubsubBatchSize) {
-            const batchTasks = tasks.slice(i, i + maxPubsubBatchSize);
-            const batchPromises = batchTasks.map(task => {
-                const dataBuffer = Buffer.from(JSON.stringify(task));
-                return topic.publishMessage({ data: dataBuffer })
-                    .catch(err => logger.log('ERROR', `[Core Utils] Failed to publish single message for ${taskType}`, { error: err.message, task: task }));
-            });
-            await Promise.all(batchPromises);
-            messagesPublished += batchTasks.length;
-            logger.log('TRACE', `[Core Utils] Published batch ${Math.ceil((i + 1) / maxPubsubBatchSize)} for ${taskType} (${batchTasks.length} messages)`);
-        }
-
-        logger.log('SUCCESS', `[Core Utils] Finished publishing ${messagesPublished} ${taskType} tasks to ${topicName}.`);
-
-    } catch (error) {
-        logger.log('ERROR', `[Core Utils] Error during batch publishing of ${taskType} tasks to ${topicName}`, { errorMessage: error.message });
-        throw error;
-    }
-}
-
-module.exports = {
-  batchPublishTasks,
-};
+/**
+ * @fileoverview Core Pub/Sub utility functions.
+ * REFACTORED: All functions are now stateless and receive dependencies.
+ * 'pubsub' (PubSub instance) and 'logger' are passed via a 'dependencies' object.
+ */
+
+/**
+ * Publishes an array of tasks to a specified Pub/Sub topic in batches.
+ * @async
+ * @param {object} dependencies - Contains pubsub, logger.
+ * @param {object} config - Configuration object.
+ * @param {string} config.topicName - The name of the Pub/Sub topic.
+ * @param {Array<object>} config.tasks - The tasks to publish.
+ * @param {string} config.taskType - A descriptor for the task type (for logging).
+ * @param {number} [config.maxPubsubBatchSize=500] - Max messages to publish in one client batch.
+ * @returns {Promise<void>}
+ */
+async function batchPublishTasks(dependencies, config) {
+    const { pubsub, logger } = dependencies;
+    const { topicName, tasks, taskType, maxPubsubBatchSize = 500 } = config;
+
+    if (!tasks || tasks.length === 0) {
+        logger.log('INFO',`[Core Utils] No ${taskType} tasks to publish to ${topicName}.`);
+        return;
+    }
+    logger.log('INFO',`[Core Utils] Publishing ${tasks.length} ${taskType} tasks to ${topicName}...`);
+    const topic = pubsub.topic(topicName);
+    let messagesPublished = 0;
+
+    try {
+        for (let i = 0; i < tasks.length; i += maxPubsubBatchSize) {
+            const batchTasks = tasks.slice(i, i + maxPubsubBatchSize);
+            const batchPromises = batchTasks.map(task => {
+                const dataBuffer = Buffer.from(JSON.stringify(task));
+                return topic.publishMessage({ data: dataBuffer })
+                    .catch(err => logger.log('ERROR', `[Core Utils] Failed to publish single message for ${taskType}`, { error: err.message, task: task }));
+            });
+            await Promise.all(batchPromises);
+            messagesPublished += batchTasks.length;
+            logger.log('TRACE', `[Core Utils] Published batch ${Math.ceil((i + 1) / maxPubsubBatchSize)} for ${taskType} (${batchTasks.length} messages)`);
+        }
+
+        logger.log('SUCCESS', `[Core Utils] Finished publishing ${messagesPublished} ${taskType} tasks to ${topicName}.`);
+
+    } catch (error) {
+        logger.log('ERROR', `[Core Utils] Error during batch publishing of ${taskType} tasks to ${topicName}`, { errorMessage: error.message });
+        throw error;
+    }
+}
+
+module.exports = {
+  batchPublishTasks,
+};

package/functions/dispatcher/helpers/dispatch_helpers.js

@@ -1,47 +1,47 @@
-/**
- * @fileoverview Sub-pipe for dispatching tasks.
- * REFACTORED: Now stateless and receives dependencies.
- */
-
-const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
-
-/**
- * Sub-pipe: pipe.dispatcher.dispatchTasksInBatches
- * @param {Array} tasks - Array of tasks to publish.
- * @param {object} dependencies - Contains pubsub, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.topicName - Target Pub/Sub topic name.
- * @param {number} config.batchSize - Number of tasks per batch.
- * @param {number} config.batchDelayMs - Delay between batches in milliseconds.
- */
-async function dispatchTasksInBatches(tasks, dependencies, config) {
-    const { pubsub, logger } = dependencies;
-    const { topicName, batchSize, batchDelayMs } = config;
-    const topic = pubsub.topic(topicName);
-    let totalTasksQueued = 0;
-
-    logger.log('INFO', `[Module Dispatcher] Received ${tasks.length} tasks. Creating batches...`);
-
-    for (let i = 0; i < tasks.length; i += batchSize) {
-        const batch = tasks.slice(i, i + batchSize);
-
-        try {
-            await Promise.all(batch.map(task => topic.publishMessage({ json: task })));
-            totalTasksQueued += batch.length;
-            logger.log('INFO', `[Module Dispatcher] Dispatched batch ${Math.ceil((i + 1) / batchSize)} with ${batch.length} tasks.`);
-
-            if (i + batchSize < tasks.length) {
-                await sleep(batchDelayMs);
-            }
-        } catch (publishError) {
-            logger.log('ERROR', `[Module Dispatcher] Failed to publish batch ${Math.ceil((i + 1) / batchSize)}. Error: ${publishError.message}`, { errorStack: publishError.stack });
-        }
-    }
-
-    logger.log('SUCCESS', `[Module Dispatcher] Successfully dispatched ${totalTasksQueued} tasks in ${Math.ceil(tasks.length / batchSize)} batches.`);
-    return totalTasksQueued;
-}
-
-module.exports = {
-    dispatchTasksInBatches
-};
+/**
+ * @fileoverview Sub-pipe for dispatching tasks.
+ * REFACTORED: Now stateless and receives dependencies.
+ */
+
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+/**
+ * Sub-pipe: pipe.dispatcher.dispatchTasksInBatches
+ * @param {Array} tasks - Array of tasks to publish.
+ * @param {object} dependencies - Contains pubsub, logger.
+ * @param {object} config - Configuration object.
+ * @param {string} config.topicName - Target Pub/Sub topic name.
+ * @param {number} config.batchSize - Number of tasks per batch.
+ * @param {number} config.batchDelayMs - Delay between batches in milliseconds.
+ */
+async function dispatchTasksInBatches(tasks, dependencies, config) {
+    const { pubsub, logger } = dependencies;
+    const { topicName, batchSize, batchDelayMs } = config;
+    const topic = pubsub.topic(topicName);
+    let totalTasksQueued = 0;
+
+    logger.log('INFO', `[Module Dispatcher] Received ${tasks.length} tasks. Creating batches...`);
+
+    for (let i = 0; i < tasks.length; i += batchSize) {
+        const batch = tasks.slice(i, i + batchSize);
+
+        try {
+            await Promise.all(batch.map(task => topic.publishMessage({ json: task })));
+            totalTasksQueued += batch.length;
+            logger.log('INFO', `[Module Dispatcher] Dispatched batch ${Math.ceil((i + 1) / batchSize)} with ${batch.length} tasks.`);
+
+            if (i + batchSize < tasks.length) {
+                await sleep(batchDelayMs);
+            }
+        } catch (publishError) {
+            logger.log('ERROR', `[Module Dispatcher] Failed to publish batch ${Math.ceil((i + 1) / batchSize)}. Error: ${publishError.message}`, { errorStack: publishError.stack });
+        }
+    }
+
+    logger.log('SUCCESS', `[Module Dispatcher] Successfully dispatched ${totalTasksQueued} tasks in ${Math.ceil(tasks.length / batchSize)} batches.`);
+    return totalTasksQueued;
+}
+
+module.exports = {
+    dispatchTasksInBatches
+};

package/functions/dispatcher/index.js

@@ -1,53 +1,53 @@
-/**
- * @fileoverview Main entry point for the Dispatcher function.
- * REFACTORED: This file now contains the main pipe function 'handleRequest'.
- */
-
-// --- 1. REMOVE the circular require ---
-// const { pipe } = require('../../index'); // <<< REMOVE THIS LINE
-
-// --- 2. ADD direct require for the specific sub-pipe needed ---
-const { dispatchTasksInBatches } = require('./helpers/dispatch_helpers');
-
-
-/**
- * Main pipe: pipe.dispatcher.handleRequest
- * This is the Pub/Sub triggered handler.
- * @param {object} message - The Pub/Sub message.
- * @param {object} context - The message context.
- * @param {object} config - The dispatcher-specific config.
- * @param {object} dependencies - Contains pubsub, logger.
- */
-async function handleRequest(message, context, config, dependencies) {
-    const { logger } = dependencies;
-    try {
-        if (!message.data) {
-            logger.log('WARN', '[Module Dispatcher] Received message without data.');
-            return;
-        }
-        const decodedMessage = JSON.parse(Buffer.from(message.data, 'base64').toString());
-        const { tasks } = decodedMessage;
-
-        if (!tasks || !Array.isArray(tasks) || tasks.length === 0) {
-            logger.log('WARN', '[Module Dispatcher] Received message with no valid tasks. Nothing to do.');
-            return;
-        }
-
-        if (!config || !config.topicName || !config.batchSize || !config.batchDelayMs) {
-            logger.log('ERROR', '[Module Dispatcher] Invalid configuration provided.', { config });
-            throw new Error("Dispatcher module received invalid configuration.");
-        }
-
-        // --- 3. Use the directly required function ---
-        // Call the sub-pipe -> becomes dispatchTasksInBatches
-        await dispatchTasksInBatches(tasks, dependencies, config); // <<< USE DIRECTLY
-
-    } catch (error) {
-        logger.log('ERROR', '[Module Dispatcher] FATAL error processing message', { errorMessage: error.message, errorStack: error.stack });
-        throw error;
-    }
-}
-
-module.exports = {
-    handleRequest,
+/**
+ * @fileoverview Main entry point for the Dispatcher function.
+ * REFACTORED: This file now contains the main pipe function 'handleRequest'.
+ */
+
+// --- 1. REMOVE the circular require ---
+// const { pipe } = require('../../index'); // <<< REMOVE THIS LINE
+
+// --- 2. ADD direct require for the specific sub-pipe needed ---
+const { dispatchTasksInBatches } = require('./helpers/dispatch_helpers');
+
+
+/**
+ * Main pipe: pipe.dispatcher.handleRequest
+ * This is the Pub/Sub triggered handler.
+ * @param {object} message - The Pub/Sub message.
+ * @param {object} context - The message context.
+ * @param {object} config - The dispatcher-specific config.
+ * @param {object} dependencies - Contains pubsub, logger.
+ */
+async function handleRequest(message, context, config, dependencies) {
+    const { logger } = dependencies;
+    try {
+        if (!message.data) {
+            logger.log('WARN', '[Module Dispatcher] Received message without data.');
+            return;
+        }
+        const decodedMessage = JSON.parse(Buffer.from(message.data, 'base64').toString());
+        const { tasks } = decodedMessage;
+
+        if (!tasks || !Array.isArray(tasks) || tasks.length === 0) {
+            logger.log('WARN', '[Module Dispatcher] Received message with no valid tasks. Nothing to do.');
+            return;
+        }
+
+        if (!config || !config.topicName || !config.batchSize || !config.batchDelayMs) {
+            logger.log('ERROR', '[Module Dispatcher] Invalid configuration provided.', { config });
+            throw new Error("Dispatcher module received invalid configuration.");
+        }
+
+        // --- 3. Use the directly required function ---
+        // Call the sub-pipe -> becomes dispatchTasksInBatches
+        await dispatchTasksInBatches(tasks, dependencies, config); // <<< USE DIRECTLY
+
+    } catch (error) {
+        logger.log('ERROR', '[Module Dispatcher] FATAL error processing message', { errorMessage: error.message, errorStack: error.stack });
+        throw error;
+    }
+}
+
+module.exports = {
+    handleRequest,
 };

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -1,125 +1,125 @@
-/**
- * @fileoverview Main pipe: pipe.maintenance.runFetchPrices
- * REFACTORED: Now writes to the new sharded `asset_prices` collection.
- * FIXED: Uses docRef.update() to correctly merge nested price maps
- * instead of docRef.set({ merge: true }) which creates flat keys.
- */
-const { FieldValue } = require('@google-cloud/firestore');
-
-// How many tickers to group into one Firestore document
-const SHARD_SIZE = 40;
-
-/**
- * Main pipe: pipe.maintenance.runFetchPrices
- * @param {object} config - Configuration object.
- * @param {object} dependencies - Contains db, logger, headerManager, proxyManager.
- * @returns {Promise<{success: boolean, message: string, instrumentsProcessed?: number}>}
- */
-exports.fetchAndStorePrices = async (config, dependencies) => {
-    const { db, logger, headerManager, proxyManager } = dependencies;
-
-    logger.log('INFO', '[PriceFetcherHelpers] Starting Daily Closing Price Update...');
-    let selectedHeader = null;
-    let wasSuccessful = false;
-
-    // --- NEW: Use the new config key, or fallback to the old one --- TODO Implement the config
-    const priceCollectionName =  'asset_prices';
-
-    try {
-        if (!config.etoroApiUrl) {
-            throw new Error("Missing required configuration: etoroApiUrl.");
-        }
-
-        selectedHeader = await headerManager.selectHeader();
-        if (!selectedHeader || !selectedHeader.header) {
-            throw new Error("Could not select a valid header for the request.");
-        }
-
-        const fetchOptions = {
-             headers: selectedHeader.header,
-             timeout: 60000
-        };
-
-        logger.log('INFO', `[PriceFetcherHelpers] Using header ID: ${selectedHeader.id}`);
-
-        const response = await proxyManager.fetch(config.etoroApiUrl, fetchOptions);
-
-        if (!response || typeof response.text !== 'function') {
-            throw new Error(`Invalid response structure received from proxy.`);
-        }
-
-        if (!response.ok) {
-            const errorBody = await response.text();
-            throw new Error(`API returned status ${response.status}: ${errorBody}`);
-        }
-        wasSuccessful = true;
-
-        const results = await response.json();
-        if (!Array.isArray(results)) {
-            throw new Error('Invalid response format from API. Expected an array.');
-        }
-
-        // --- START MODIFICATION ---
-        
-        logger.log('INFO', `[PriceFetcherHelpers] Received ${results.length} instrument prices. Sharding...`);
-        const shardUpdates = {}; // { "shard_0": { ... }, "shard_1": { ... } }
-
-        for (const instrumentData of results) {
-            const dailyData = instrumentData?.ClosingPrices?.Daily;
-            const instrumentId = instrumentData.InstrumentId;
-
-            if (instrumentId && dailyData?.Price && dailyData?.Date) {
-                const instrumentIdStr = String(instrumentId);
-                const dateKey = dailyData.Date.substring(0, 10);
-                
-                // Determine shard ID
-                const shardId = `shard_${parseInt(instrumentIdStr, 10) % SHARD_SIZE}`;
-                
-                if (!shardUpdates[shardId]) {
-                    shardUpdates[shardId] = {};
-                }
-                
-                // Use dot notation to define the update path
-                const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
-                const updatePath = `${instrumentIdStr}.lastUpdated`;
-
-                shardUpdates[shardId][pricePath] = dailyData.Price;
-                shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp();
-            }
-        }
-
-        // Commit all shard updates in parallel
-        const batchPromises = [];
-        for (const shardId in shardUpdates) {
-            const docRef = db.collection(priceCollectionName).doc(shardId);
-            const payload = shardUpdates[shardId];
-            
-            // --- THIS IS THE FIX ---
-            // Use .update() to correctly merge data into nested maps.
-            // Using .set(payload, { merge: true }) creates the flat, broken keys.
-            batchPromises.push(docRef.update(payload));
-            // --- END FIX ---
-        }
-
-        await Promise.all(batchPromises);
-        
-        // --- END MODIFICATION ---
-
-        const successMessage = `Successfully processed and saved daily prices for ${results.length} instruments to ${batchPromises.length} shards.`;
-        logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`);
-        return { success: true, message: successMessage, instrumentsProcessed: results.length };
-
-    } catch (error) {
-        logger.log('ERROR', '[PriceFetcherHelpers] Fatal error during closing price update', {
-            errorMessage: error.message,
-            errorStack: error.stack,
-            headerId: selectedHeader ? selectedHeader.id : 'not-selected'
-        });
-        throw error;
-    } finally {
-        if (selectedHeader) {
-            await headerManager.updatePerformance(selectedHeader.id, wasSuccessful);
-            await headerManager.flushPerformanceUpdates();
-        }
-    }
+/**
+ * @fileoverview Main pipe: pipe.maintenance.runFetchPrices
+ * REFACTORED: Now writes to the new sharded `asset_prices` collection.
+ * FIXED: Uses docRef.update() to correctly merge nested price maps
+ * instead of docRef.set({ merge: true }) which creates flat keys.
+ */
+const { FieldValue } = require('@google-cloud/firestore');
+
+// How many tickers to group into one Firestore document
+const SHARD_SIZE = 40;
+
+/**
+ * Main pipe: pipe.maintenance.runFetchPrices
+ * @param {object} config - Configuration object.
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager.
+ * @returns {Promise<{success: boolean, message: string, instrumentsProcessed?: number}>}
+ */
+exports.fetchAndStorePrices = async (config, dependencies) => {
+    const { db, logger, headerManager, proxyManager } = dependencies;
+
+    logger.log('INFO', '[PriceFetcherHelpers] Starting Daily Closing Price Update...');
+    let selectedHeader = null;
+    let wasSuccessful = false;
+
+    // --- NEW: Use the new config key, or fallback to the old one --- TODO Implement the config
+    const priceCollectionName =  'asset_prices';
+
+    try {
+        if (!config.etoroApiUrl) {
+            throw new Error("Missing required configuration: etoroApiUrl.");
+        }
+
+        selectedHeader = await headerManager.selectHeader();
+        if (!selectedHeader || !selectedHeader.header) {
+            throw new Error("Could not select a valid header for the request.");
+        }
+
+        const fetchOptions = {
+             headers: selectedHeader.header,
+             timeout: 60000
+        };
+
+        logger.log('INFO', `[PriceFetcherHelpers] Using header ID: ${selectedHeader.id}`);
+
+        const response = await proxyManager.fetch(config.etoroApiUrl, fetchOptions);
+
+        if (!response || typeof response.text !== 'function') {
+            throw new Error(`Invalid response structure received from proxy.`);
+        }
+
+        if (!response.ok) {
+            const errorBody = await response.text();
+            throw new Error(`API returned status ${response.status}: ${errorBody}`);
+        }
+        wasSuccessful = true;
+
+        const results = await response.json();
+        if (!Array.isArray(results)) {
+            throw new Error('Invalid response format from API. Expected an array.');
+        }
+
+        // --- START MODIFICATION ---
+        
+        logger.log('INFO', `[PriceFetcherHelpers] Received ${results.length} instrument prices. Sharding...`);
+        const shardUpdates = {}; // { "shard_0": { ... }, "shard_1": { ... } }
+
+        for (const instrumentData of results) {
+            const dailyData = instrumentData?.ClosingPrices?.Daily;
+            const instrumentId = instrumentData.InstrumentId;
+
+            if (instrumentId && dailyData?.Price && dailyData?.Date) {
+                const instrumentIdStr = String(instrumentId);
+                const dateKey = dailyData.Date.substring(0, 10);
+                
+                // Determine shard ID
+                const shardId = `shard_${parseInt(instrumentIdStr, 10) % SHARD_SIZE}`;
+                
+                if (!shardUpdates[shardId]) {
+                    shardUpdates[shardId] = {};
+                }
+                
+                // Use dot notation to define the update path
+                const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
+                const updatePath = `${instrumentIdStr}.lastUpdated`;
+
+                shardUpdates[shardId][pricePath] = dailyData.Price;
+                shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp();
+            }
+        }
+
+        // Commit all shard updates in parallel
+        const batchPromises = [];
+        for (const shardId in shardUpdates) {
+            const docRef = db.collection(priceCollectionName).doc(shardId);
+            const payload = shardUpdates[shardId];
+            
+            // --- THIS IS THE FIX ---
+            // Use .update() to correctly merge data into nested maps.
+            // Using .set(payload, { merge: true }) creates the flat, broken keys.
+            batchPromises.push(docRef.update(payload));
+            // --- END FIX ---
+        }
+
+        await Promise.all(batchPromises);
+        
+        // --- END MODIFICATION ---
+
+        const successMessage = `Successfully processed and saved daily prices for ${results.length} instruments to ${batchPromises.length} shards.`;
+        logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`);
+        return { success: true, message: successMessage, instrumentsProcessed: results.length };
+
+    } catch (error) {
+        logger.log('ERROR', '[PriceFetcherHelpers] Fatal error during closing price update', {
+            errorMessage: error.message,
+            errorStack: error.stack,
+            headerId: selectedHeader ? selectedHeader.id : 'not-selected'
+        });
+        throw error;
+    } finally {
+        if (selectedHeader) {
+            await headerManager.updatePerformance(selectedHeader.id, wasSuccessful);
+            await headerManager.flushPerformanceUpdates();
+        }
+    }
 };