bulltrackers-module 1.0.104 → 1.0.106

package/functions/orchestrator/index.js

@@ -1,148 +1,148 @@
-/**
- * @fileoverview Main orchestration logic.
- * REFACTORED: This file now contains the main pipe functions
- * that are called by the Cloud Function entry points.
- * They receive all dependencies.
- */
-
-const { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery } = require('./helpers/discovery_helpers');
-const { getUpdateTargets, dispatchUpdates } = require('./helpers/update_helpers');
-
-
-/**
- * Main pipe for running the complete discovery orchestration process.
- * This is the function the Cloud Function entry point will call.
- * @param {object} config - The full orchestratorConfig object.
- * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils.
- * @returns {Promise<void>}
- */
-async function runDiscoveryOrchestrator(config, dependencies) {
-    const { logger, firestoreUtils } = dependencies;
-    logger.log('INFO', '🚀 Discovery Orchestrator triggered via module...');
-
-    // Call sub-pipe from core
-    await firestoreUtils.resetProxyLocks(dependencies, config);
-
-    // Call sub-pipe for 'normal'
-    // Pass the directly required sub-pipe functions implicitly via runDiscovery call context
-    await runDiscovery('normal', config.discoveryConfig.normal, config, dependencies);
-
-    // Call sub-pipe for 'speculator'
-    await runDiscovery('speculator', config.discoveryConfig.speculator, config, dependencies);
-}
-
-/**
- * Main pipe for running the complete update orchestration process.
- * This is the function the Cloud Function entry point will call.
- * @param {object} config - The full orchestratorConfig object.
- * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils, pubsubUtils.
- * @returns {Promise<void>}
- */
-async function runUpdateOrchestrator(config, dependencies) {
-    const { logger, firestoreUtils } = dependencies;
-    logger.log('INFO', '🚀 Update Orchestrator triggered via module...');
-
-    // Call sub-pipe from core
-    await firestoreUtils.resetProxyLocks(dependencies, config);
-
-    // Call sub-pipe for 'normal'
-    await runUpdates('normal', config.updateConfig, config, dependencies);
-
-    // Call sub-pipe for 'speculator'
-    await runUpdates('speculator', config.updateConfig, config, dependencies);
-}
-
-// --- Internal Helpers (which are also exposed as sub-pipes) ---
-
-/**
- * Runs the discovery process for a *single* user type.
- * This is an internal helper, but also maps to pipe.orchestrator.runDiscovery
- * @param {string} userType - 'normal' or 'speculator'.
- * @param {object} userTypeConfig - The specific config slice (e.g., config.discoveryConfig.normal).
- * @param {object} globalConfig - Global config.
- * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils, pubsubUtils.
- */
-async function runDiscovery(userType, userTypeConfig, globalConfig, dependencies) {
-    const { logger } = dependencies;
-    logger.log('INFO', `[Module Orchestrator] Starting discovery for ${userType} users...`);
-
-    // --- 3. Use the directly required functions ---
-    // 1. pipe.orchestrator.checkDiscoveryNeed -> becomes checkDiscoveryNeed
-    const { needsDiscovery, blocksToFill } = await checkDiscoveryNeed(
-        userType,
-        userTypeConfig,
-        dependencies
-    );
-
-    if (!needsDiscovery) {
-        logger.log('INFO', `[Module Orchestrator] No discovery needed for ${userType}.`);
-        return;
-    }
-
-    // 2. pipe.orchestrator.getDiscoveryCandidates -> becomes getDiscoveryCandidates
-    const candidates = await getDiscoveryCandidates(
-        userType,
-        blocksToFill,
-        userTypeConfig,
-        dependencies
-    );
-
-    // 3. pipe.orchestrator.dispatchDiscovery -> becomes dispatchDiscovery
-    await dispatchDiscovery(
-        userType,
-        candidates,
-        userTypeConfig,
-        dependencies
-    );
-
-    logger.log('SUCCESS', `[Module Orchestrator] Dispatched discovery tasks for ${userType}.`);
-}
-
-/**
- * Runs the update process for a *single* user type.
- * This is an internal helper, but also maps to pipe.orchestrator.runUpdates
- * @param {string} userType - 'normal' or 'speculator'.
- * @param {object} updateConfig - The config.updateConfig object.
- * @param {object} globalConfig - Global config.
- * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils, pubsubUtils.
- */
-async function runUpdates(userType, updateConfig, globalConfig, dependencies) {
-    const { logger } = dependencies;
-    logger.log('INFO', `[Module Orchestrator] Collecting users for daily update (${userType})...`);
-
-    const now = new Date();
-    const startOfTodayUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
-    const DaysAgoUTC = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
-
-    const thresholds = {
-        dateThreshold: startOfTodayUTC,
-        gracePeriodThreshold: DaysAgoUTC
-    };
-
-    // --- 4. Use the directly required functions ---
-    // 1. pipe.orchestrator.getUpdateTargets -> becomes getUpdateTargets
-    const targets = await getUpdateTargets(
-        userType,
-        thresholds,
-        updateConfig,
-        dependencies
-    );
-
-    // 2. pipe.orchestrator.dispatchUpdates -> becomes dispatchUpdates
-    await dispatchUpdates(
-        targets,
-        userType,
-        updateConfig,
-        dependencies
-    );
-
-    logger.log('SUCCESS', `[Module Orchestrator] Dispatched update tasks for ${userType}.`);
-}
-
-module.exports = {
-  runDiscoveryOrchestrator,
-  runUpdateOrchestrator,
-  // Exporting these internal helpers so the main index.js can map them
-  runDiscovery,
-  runUpdates
+/**
+ * @fileoverview Main orchestration logic.
+ * REFACTORED: This file now contains the main pipe functions
+ * that are called by the Cloud Function entry points.
+ * They receive all dependencies.
+ */
+
+const { checkDiscoveryNeed, getDiscoveryCandidates, dispatchDiscovery } = require('./helpers/discovery_helpers');
+const { getUpdateTargets, dispatchUpdates } = require('./helpers/update_helpers');
+
+
+/**
+ * Main pipe for running the complete discovery orchestration process.
+ * This is the function the Cloud Function entry point will call.
+ * @param {object} config - The full orchestratorConfig object.
+ * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils.
+ * @returns {Promise<void>}
+ */
+async function runDiscoveryOrchestrator(config, dependencies) {
+    const { logger, firestoreUtils } = dependencies;
+    logger.log('INFO', '🚀 Discovery Orchestrator triggered via module...');
+
+    // Call sub-pipe from core
+    await firestoreUtils.resetProxyLocks(dependencies, config);
+
+    // Call sub-pipe for 'normal'
+    // Pass the directly required sub-pipe functions implicitly via runDiscovery call context
+    await runDiscovery('normal', config.discoveryConfig.normal, config, dependencies);
+
+    // Call sub-pipe for 'speculator'
+    await runDiscovery('speculator', config.discoveryConfig.speculator, config, dependencies);
+}
+
+/**
+ * Main pipe for running the complete update orchestration process.
+ * This is the function the Cloud Function entry point will call.
+ * @param {object} config - The full orchestratorConfig object.
+ * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils, pubsubUtils.
+ * @returns {Promise<void>}
+ */
+async function runUpdateOrchestrator(config, dependencies) {
+    const { logger, firestoreUtils } = dependencies;
+    logger.log('INFO', '🚀 Update Orchestrator triggered via module...');
+
+    // Call sub-pipe from core
+    await firestoreUtils.resetProxyLocks(dependencies, config);
+
+    // Call sub-pipe for 'normal'
+    await runUpdates('normal', config.updateConfig, config, dependencies);
+
+    // Call sub-pipe for 'speculator'
+    await runUpdates('speculator', config.updateConfig, config, dependencies);
+}
+
+// --- Internal Helpers (which are also exposed as sub-pipes) ---
+
+/**
+ * Runs the discovery process for a *single* user type.
+ * This is an internal helper, but also maps to pipe.orchestrator.runDiscovery
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {object} userTypeConfig - The specific config slice (e.g., config.discoveryConfig.normal).
+ * @param {object} globalConfig - Global config.
+ * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils, pubsubUtils.
+ */
+async function runDiscovery(userType, userTypeConfig, globalConfig, dependencies) {
+    const { logger } = dependencies;
+    logger.log('INFO', `[Module Orchestrator] Starting discovery for ${userType} users...`);
+
+    // --- 3. Use the directly required functions ---
+    // 1. pipe.orchestrator.checkDiscoveryNeed -> becomes checkDiscoveryNeed
+    const { needsDiscovery, blocksToFill } = await checkDiscoveryNeed(
+        userType,
+        userTypeConfig,
+        dependencies
+    );
+
+    if (!needsDiscovery) {
+        logger.log('INFO', `[Module Orchestrator] No discovery needed for ${userType}.`);
+        return;
+    }
+
+    // 2. pipe.orchestrator.getDiscoveryCandidates -> becomes getDiscoveryCandidates
+    const candidates = await getDiscoveryCandidates(
+        userType,
+        blocksToFill,
+        userTypeConfig,
+        dependencies
+    );
+
+    // 3. pipe.orchestrator.dispatchDiscovery -> becomes dispatchDiscovery
+    await dispatchDiscovery(
+        userType,
+        candidates,
+        userTypeConfig,
+        dependencies
+    );
+
+    logger.log('SUCCESS', `[Module Orchestrator] Dispatched discovery tasks for ${userType}.`);
+}
+
+/**
+ * Runs the update process for a *single* user type.
+ * This is an internal helper, but also maps to pipe.orchestrator.runUpdates
+ * @param {string} userType - 'normal' or 'speculator'.
+ * @param {object} updateConfig - The config.updateConfig object.
+ * @param {object} globalConfig - Global config.
+ * @param {object} dependencies - Contains db, pubsub, logger, firestoreUtils, pubsubUtils.
+ */
+async function runUpdates(userType, updateConfig, globalConfig, dependencies) {
+    const { logger } = dependencies;
+    logger.log('INFO', `[Module Orchestrator] Collecting users for daily update (${userType})...`);
+
+    const now = new Date();
+    const startOfTodayUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
+    const DaysAgoUTC = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
+
+    const thresholds = {
+        dateThreshold: startOfTodayUTC,
+        gracePeriodThreshold: DaysAgoUTC
+    };
+
+    // --- 4. Use the directly required functions ---
+    // 1. pipe.orchestrator.getUpdateTargets -> becomes getUpdateTargets
+    const targets = await getUpdateTargets(
+        userType,
+        thresholds,
+        updateConfig,
+        dependencies
+    );
+
+    // 2. pipe.orchestrator.dispatchUpdates -> becomes dispatchUpdates
+    await dispatchUpdates(
+        targets,
+        userType,
+        updateConfig,
+        dependencies
+    );
+
+    logger.log('SUCCESS', `[Module Orchestrator] Dispatched update tasks for ${userType}.`);
+}
+
+module.exports = {
+  runDiscoveryOrchestrator,
+  runUpdateOrchestrator,
+  // Exporting these internal helpers so the main index.js can map them
+  runDiscovery,
+  runUpdates
 };

package/functions/price-backfill/helpers/handler_helpers.js

@@ -1,123 +1,116 @@
-/**
- * @fileoverview Main pipe: pipe.maintenance.runBackfillAssetPrices
- * A one-time function to backfill historical price data from eToro's
- * candle API into the new sharded `asset_prices` collection.
- */
-
-const { FieldValue } = require('@google-cloud/firestore'); // Todo inject this
-const pLimit = require('p-limit'); // TODO inject this
-
-// How many tickers to fetch in parallel
-const CONCURRENT_REQUESTS = 10;
-// How many days of history to fetch
-const DAYS_TO_FETCH = 365;
-// How many tickers to group into one Firestore document
-const SHARD_SIZE = 40;
-
-// EXP BACKOFF
-async function fetchWithExponentialBackoff(fn, maxAttempts = 8) {
-    let attempt = 0;
-    while (attempt < maxAttempts) {
-        try {
-            return await fn();
-        } catch (err) {
-            attempt++;
-            if (attempt >= maxAttempts) throw err;
-            const delay = Math.min(1000 * Math.pow(2, attempt), 60000); // capped 60s
-            await new Promise(res => setTimeout(res, delay));
-        }
-    }
-}
-
-/**
- * Main pipe: pipe.maintenance.runBackfillAssetPrices
- * @param {object} config - Configuration object.
- * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, calculationUtils.
- */
-
-exports.runBackfillAssetPrices = async (config, dependencies) => {
-    const { db, logger, headerManager, proxyManager, calculationUtils } = dependencies;
-    const { loadInstrumentMappings } = calculationUtils;
-
-    logger.log('INFO', '[PriceBackfill] Starting historical price backfill...');
-
-    let mappings;
-    try {
-        mappings = await loadInstrumentMappings();
-        if (!mappings || !mappings.instrumentToTicker) {
-            throw new Error("Failed to load instrument mappings.");
-        }
-    } catch (e) {
-        logger.log('ERROR', '[PriceBackfill] Could not load instrument mappings.', { err: e.message });
-        return;
-    }
-
-    const instrumentIds = Object.keys(mappings.instrumentToTicker);
-    logger.log('INFO', `[PriceBackfill] Found ${instrumentIds.length} instruments to backfill.`);
-
-    const limit = pLimit(CONCURRENT_REQUESTS);
-    let successCount = 0;
-    let errorCount = 0;
-
-    const promises = instrumentIds.map(instrumentId => {
-        return limit(async () => {
-            try {
-                const ticker = mappings.instrumentToTicker[instrumentId] || `unknown_${instrumentId}`;
-                const url = `https://candle.etoro.com/candles/asc.json/OneDay/${DAYS_TO_FETCH}/${instrumentId}`; 
-
-                const selectedHeader = await headerManager.selectHeader();
-
-                const response = await fetchWithExponentialBackoff(() =>
-                    proxyManager.fetch(url, {
-                        headers: selectedHeader.header,
-                        timeout: 20000
-                    })
-                );
-
-                if (!response.ok) {
-                    throw new Error(`API error ${response.status} for instrument ${instrumentId}`);
-                }
-
-                headerManager.updatePerformance(selectedHeader.id, true);
-
-                const data = await response.json();
-                const candles = data?.Candles?.[0]?.Candles;
-
-                if (!Array.isArray(candles) || candles.length === 0) {
-                    logger.log('WARN', `[PriceBackfill] No candle data returned for ${ticker} (${instrumentId})`);
-                    return;
-                }
-
-                const prices = {};
-                for (const candle of candles) {
-                    const dateKey = candle.FromDate.substring(0, 10);
-                    prices[dateKey] = candle.Close;
-                }
-
-                const shardId = `shard_${parseInt(instrumentId, 10) % SHARD_SIZE}`;
-                const docRef = db.collection('asset_prices').doc(shardId);
-
-                const payload = {
-                    [instrumentId]: {
-                        ticker: ticker,
-                        prices: prices,
-                        lastUpdated: FieldValue.serverTimestamp()
-                    }
-                };
-
-                await docRef.set(payload, { merge: true });
-                logger.log('TRACE', `[PriceBackfill] Successfully stored data for ${ticker} (${instrumentId}) in ${shardId}`);
-                successCount++;
-
-            } catch (err) {
-                logger.log('ERROR', `[PriceBackfill] Failed to process instrument ${instrumentId}`, { err: err.message });
-                errorCount++;
-            }
-        });
-    });
-
-    await Promise.all(promises);
-    await headerManager.flushPerformanceUpdates();
-
-    logger.log('SUCCESS', `[PriceBackfill] Backfill complete. Success: ${successCount}, Failed: ${errorCount}`);
-};
+/**
+ * @fileoverview Main pipe: pipe.maintenance.runBackfillAssetPrices
+ * A one-time function to backfill historical price data from eToro's
+ * candle API into the new sharded `asset_prices` collection.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore'); // Todo inject this
+
+const pLimit = require('p-limit'); // TODO inject this
+
+// How many tickers to fetch in parallel
+const CONCURRENT_REQUESTS = 10;
+// How many days of history to fetch
+const DAYS_TO_FETCH = 365;
+// How many tickers to group into one Firestore document
+const SHARD_SIZE = 40;
+
+/**
+ * Main pipe: pipe.maintenance.runBackfillAssetPrices
+ * @param {object} config - Configuration object.
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager, calculationUtils.
+ */
+// --- MODIFIED: Removed calculationUtils, as it's in dependencies ---
+exports.runBackfillAssetPrices = async (config, dependencies) => {
+    const { db, logger, headerManager, proxyManager, calculationUtils } = dependencies;
+    const { loadInstrumentMappings } = calculationUtils; // <-- Get function from dependencies
+    // --- END MODIFIED ---
+    
+    logger.log('INFO', '[PriceBackfill] Starting historical price backfill...');
+    
+    let mappings;
+    try {
+        // --- MODIFIED: Use the injected utils ---
+        mappings = await loadInstrumentMappings();
+        // --- END MODIFIED ---
+        if (!mappings || !mappings.instrumentToTicker) {
+            throw new Error("Failed to load instrument mappings.");
+        }
+    } catch (e) {
+        logger.log('ERROR', '[PriceBackfill] Could not load instrument mappings.', { err: e.message });
+        return;
+    }
+
+    const instrumentIds = Object.keys(mappings.instrumentToTicker);
+    logger.log('INFO', `[PriceBackfill] Found ${instrumentIds.length} instruments to backfill.`);
+
+    const limit = pLimit(CONCURRENT_REQUESTS);
+    let successCount = 0;
+    let errorCount = 0;
+
+    const promises = instrumentIds.map(instrumentId => {
+        return limit(async () => {
+            try {
+                const ticker = mappings.instrumentToTicker[instrumentId] || `unknown_${instrumentId}`;
+                const url = `https://candle.etoro.com/candles/asc.json/OneDay/${DAYS_TO_FETCH}/${instrumentId}`; //TODO implement config value
+                
+                const selectedHeader = await headerManager.selectHeader();
+                let wasSuccess = false;
+
+                const response = await proxyManager.fetch(url, {
+                    headers: selectedHeader.header,
+                    timeout: 20000
+                });
+
+                if (!response.ok) {
+                    throw new Error(`API error ${response.status} for instrument ${instrumentId}`);
+                }
+                wasSuccess = true;
+                headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+
+                const data = await response.json();
+                const candles = data?.Candles?.[0]?.Candles;
+
+                if (!Array.isArray(candles) || candles.length === 0) {
+                    logger.log('WARN', `[PriceBackfill] No candle data returned for ${ticker} (${instrumentId})`);
+                    return;
+                }
+
+                // Format data as a map
+                const prices = {};
+                for (const candle of candles) {
+                    const dateKey = candle.FromDate.substring(0, 10);
+                    prices[dateKey] = candle.Close;
+                }
+
+                // Determine shard ID
+                const shardId = `shard_${parseInt(instrumentId, 10) % SHARD_SIZE}`;
+                const docRef = db.collection('asset_prices').doc(shardId); // TODO implement config value
+
+                const payload = {
+                    [instrumentId]: {
+                        ticker: ticker,
+                        prices: prices,
+                        lastUpdated: FieldValue.serverTimestamp()
+                    }
+                };
+                
+                // Write to Firestore
+                await docRef.set(payload, { merge: true });
+                logger.log('TRACE', `[PriceBackfill] Successfully stored data for ${ticker} (${instrumentId}) in ${shardId}`);
+                successCount++;
+
+            } catch (err) {
+                logger.log('ERROR', `[PriceBackfill] Failed to process instrument ${instrumentId}`, { err: err.message });
+                errorCount++;
+            }
+        });
+    });
+
+    await Promise.all(promises);
+
+    // Flush any remaining header updates
+    await headerManager.flushPerformanceUpdates();
+
+    logger.log('SUCCESS', `[PriceBackfill] Backfill complete. Success: ${successCount}, Failed: ${errorCount}`);
+};