bulltrackers-module 1.0.84 → 1.0.86

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -1,9 +1,12 @@
 /**
  * @fileoverview Main pipe: pipe.maintenance.runFetchPrices
- * REFACTORED: Now stateless and receives dependencies.
+ * REFACTORED: Now writes to the new sharded `asset_prices` collection.
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
+// How many tickers to group into one Firestore document
+const SHARD_SIZE = 40;
+
 /**
  * Main pipe: pipe.maintenance.runFetchPrices
  * @param {object} config - Configuration object.
@@ -13,13 +16,16 @@ const { FieldValue } = require('@google-cloud/firestore');
 exports.fetchAndStorePrices = async (config, dependencies) => {
     const { db, logger, headerManager, proxyManager } = dependencies;
 
-    logger.log('INFO', '[PriceFetcherHelpers] Starting Daily Closing Price Update via module...');
+    logger.log('INFO', '[PriceFetcherHelpers] Starting Daily Closing Price Update...');
     let selectedHeader = null;
     let wasSuccessful = false;
 
+    // --- NEW: Use the new config key, or fallback to the old one --- TODO Implement the config
+    const priceCollectionName =  'asset_prices';
+
     try {
-        if (!config.etoroApiUrl || !config.firestoreCollection || !config.batchLimit) {
-            throw new Error("Missing required configuration: etoroApiUrl, firestoreCollection, or batchLimit.");
+        if (!config.etoroApiUrl) {
+            throw new Error("Missing required configuration: etoroApiUrl.");
         }
 
         selectedHeader = await headerManager.selectHeader();
@@ -32,15 +38,11 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
              timeout: 60000
         };
 
-        logger.log('INFO', `[PriceFetcherHelpers] Using header ID: ${selectedHeader.id}`, {
-            userAgent: selectedHeader.header['User-Agent']
-        });
+        logger.log('INFO', `[PriceFetcherHelpers] Using header ID: ${selectedHeader.id}`);
 
-        const  response  = await proxyManager.fetch(config.etoroApiUrl, fetchOptions);
+        const response = await proxyManager.fetch(config.etoroApiUrl, fetchOptions);
 
         if (!response || typeof response.text !== 'function') {
-            const responseString = JSON.stringify(response, null, 2);
-            logger.log('ERROR', `[PriceFetcherHelpers] Invalid or incomplete response received. Response object: ${responseString}`);
             throw new Error(`Invalid response structure received from proxy.`);
         }
 
@@ -51,54 +53,56 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
         wasSuccessful = true;
 
         const results = await response.json();
-        if (!results || !Array.isArray(results)) {
+        if (!Array.isArray(results)) {
             throw new Error('Invalid response format from API. Expected an array.');
         }
 
-         const promises = [];
-         let batch = db.batch(); // Use db
-         let instrumentsProcessed = 0;
-         let batchCount = 0;
-
-         for (const instrumentData of results) {
-             const dailyData = instrumentData?.ClosingPrices?.Daily;
-
-             if (instrumentData.InstrumentId && dailyData?.Price && dailyData?.Date) {
-                 const instrumentIdStr = String(instrumentData.InstrumentId);
-                 const dateKey = dailyData.Date.substring(0, 10);
-
-                 const updatePayload = {
-                     instrumentId: instrumentData.InstrumentId,
-                     lastUpdated: FieldValue.serverTimestamp(),
-                     [`prices.${dateKey}`]: dailyData.Price
-                 };
-
-                 const docRef = db.collection(config.firestoreCollection).doc(instrumentIdStr); // Use db
-                 batch.set(docRef, updatePayload, { merge: true });
-
-                 instrumentsProcessed++;
-                 batchCount++;
-
-                 if (batchCount >= config.batchLimit) {
-                     logger.log('TRACE', `[PriceFetcherHelpers] Committing batch of ${batchCount} instruments...`);
-                     promises.push(batch.commit());
-                     batch = db.batch(); // Use db
-                     batchCount = 0;
-                     await new Promise(resolve => setTimeout(resolve, 50));
-                 }
-             }
-         }
+        // --- START MODIFICATION ---
+        
+        logger.log('INFO', `[PriceFetcherHelpers] Received ${results.length} instrument prices. Sharding...`);
+        const shardUpdates = {}; // { "shard_0": { ... }, "shard_1": { ... } }
+
+        for (const instrumentData of results) {
+            const dailyData = instrumentData?.ClosingPrices?.Daily;
+            const instrumentId = instrumentData.InstrumentId;
+
+            if (instrumentId && dailyData?.Price && dailyData?.Date) {
+                const instrumentIdStr = String(instrumentId);
+                const dateKey = dailyData.Date.substring(0, 10);
+                
+                // Determine shard ID
+                const shardId = `shard_${parseInt(instrumentIdStr, 10) % SHARD_SIZE}`;
+                
+                if (!shardUpdates[shardId]) {
+                    shardUpdates[shardId] = {};
+                }
+                
+                // Use dot notation to update only the specific price for the specific instrument
+                const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
+                const updatePath = `${instrumentIdStr}.lastUpdated`;
+
+                shardUpdates[shardId][pricePath] = dailyData.Price;
+                shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp();
+            }
+        }
 
-         if (batchCount > 0) {
-             logger.log('TRACE', `[PriceFetcherHelpers] Committing final batch of ${batchCount} instruments...`);
-              promises.push(batch.commit());
-         }
+        // Commit all shard updates in parallel
+        const batchPromises = [];
+        for (const shardId in shardUpdates) {
+            const docRef = db.collection(priceCollectionName).doc(shardId);
+            const payload = shardUpdates[shardId];
+            
+            // Use set with merge:true to update all instruments in the shard doc at once
+            batchPromises.push(docRef.set(payload, { merge: true }));
+        }
 
-          await Promise.all(promises);
+        await Promise.all(batchPromises);
+        
+        // --- END MODIFICATION ---
 
-         const successMessage = `Successfully processed and saved daily prices for ${instrumentsProcessed} instruments via module.`;
-         logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`, { instrumentsProcessed });
-         return { success: true, message: successMessage, instrumentsProcessed };
+        const successMessage = `Successfully processed and saved daily prices for ${results.length} instruments to ${batchPromises.length} shards.`;
+        logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`);
+        return { success: true, message: successMessage, instrumentsProcessed: results.length };
 
     } catch (error) {
         logger.log('ERROR', '[PriceFetcherHelpers] Fatal error during closing price update', {
@@ -110,8 +114,7 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
     } finally {
         if (selectedHeader) {
             await headerManager.updatePerformance(selectedHeader.id, wasSuccessful);
-            // Also flush performance, as this is a standalone function
             await headerManager.flushPerformanceUpdates();
         }
     }
-};
+};

package/functions/price-backfill/helpers/handler_helpers.js

@@ -0,0 +1,112 @@
+/**
+ * @fileoverview Main pipe: pipe.maintenance.runBackfillAssetPrices
+ * A one-time function to backfill historical price data from eToro's
+ * candle API into the new sharded `asset_prices` collection.
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+// Import the mapping loader from your shared calculations package
+const { loadInstrumentMappings } = require('aiden-shared-calculations-unified').utils;
+const pLimit = require('p-limit');
+
+// How many tickers to fetch in parallel
+const CONCURRENT_REQUESTS = 10;
+// How many days of history to fetch
+const DAYS_TO_FETCH = 365;
+// How many tickers to group into one Firestore document
+const SHARD_SIZE = 40;
+
+/**
+ * Main pipe: pipe.maintenance.runBackfillAssetPrices
+ * @param {object} config - Configuration object.
+ * @param {object} dependencies - Contains db, logger, headerManager, proxyManager.
+ */
+exports.runBackfillAssetPrices = async (config, dependencies) => {
+    const { db, logger, headerManager, proxyManager } = dependencies;
+    
+    logger.log('INFO', '[PriceBackfill] Starting historical price backfill...');
+    
+    let mappings;
+    try {
+        mappings = await loadInstrumentMappings();
+        if (!mappings || !mappings.instrumentToTicker) {
+            throw new Error("Failed to load instrument mappings.");
+        }
+    } catch (e) {
+        logger.log('ERROR', '[PriceBackfill] Could not load instrument mappings.', { err: e.message });
+        return;
+    }
+
+    const instrumentIds = Object.keys(mappings.instrumentToTicker);
+    logger.log('INFO', `[PriceBackfill] Found ${instrumentIds.length} instruments to backfill.`);
+
+    const limit = pLimit(CONCURRENT_REQUESTS);
+    let successCount = 0;
+    let errorCount = 0;
+
+    const promises = instrumentIds.map(instrumentId => {
+        return limit(async () => {
+            try {
+                const ticker = mappings.instrumentToTicker[instrumentId] || `unknown_${instrumentId}`;
+                const url = `https://candle.etoro.com/candles/asc.json/OneDay/${DAYS_TO_FETCH}/${instrumentId}`;
+                
+                const selectedHeader = await headerManager.selectHeader();
+                let wasSuccess = false;
+
+                const response = await proxyManager.fetch(url, {
+                    headers: selectedHeader.header,
+                    timeout: 20000
+                });
+
+                if (!response.ok) {
+                    throw new Error(`API error ${response.status} for instrument ${instrumentId}`);
+                }
+                wasSuccess = true;
+                headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+
+                const data = await response.json();
+                const candles = data?.Candles?.[0]?.Candles;
+
+                if (!Array.isArray(candles) || candles.length === 0) {
+                    logger.log('WARN', `[PriceBackfill] No candle data returned for ${ticker} (${instrumentId})`);
+                    return;
+                }
+
+                // Format data as a map
+                const prices = {};
+                for (const candle of candles) {
+                    const dateKey = candle.FromDate.substring(0, 10);
+                    prices[dateKey] = candle.Close;
+                }
+
+                // Determine shard ID
+                const shardId = `shard_${parseInt(instrumentId, 10) % SHARD_SIZE}`;
+                const docRef = db.collection('asset_prices').doc(shardId); // TODO implement config value
+
+                const payload = {
+                    [instrumentId]: {
+                        ticker: ticker,
+                        prices: prices,
+                        lastUpdated: FieldValue.serverTimestamp()
+                    }
+                };
+                
+                // Write to Firestore
+                await docRef.set(payload, { merge: true });
+                logger.log('TRACE', `[PriceBackfill] Successfully stored data for ${ticker} (${instrumentId}) in ${shardId}`);
+                successCount++;
+
+            } catch (err) {
+                logger.log('ERROR', `[PriceBackfill] Failed to process instrument ${instrumentId}`, { err: err.message });
+                errorCount++;
+            }
+        });
+    });
+
+    await Promise.all(promises);
+
+    // Flush any remaining header updates
+    await headerManager.flushPerformanceUpdates();
+
+    logger.log('SUCCESS', `[PriceBackfill] Backfill complete. Success: ${successCount}, Failed: ${errorCount}`);
+};

package/index.js

@@ -90,6 +90,7 @@ const maintenance = {
     // --- NEW SOCIAL SENTIMENT FUNCTIONS ---
     runSocialOrchestrator: require('./functions/social-orchestrator/helpers/orchestrator_helpers').runSocialOrchestrator,
     handleSocialTask: require('./functions/social-task-handler/helpers/handler_helpers').handleSocialTask,
+    runBackfillAssetPrices: require('./functions/price-backfill/helpers/handler_helpers').runBackfillAssetPrices,
     // --- END NEW ---
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.84",
+  "version": "1.0.86",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -18,7 +18,8 @@
     "functions/appscript-api/",
     "functions/user-activity-sampler/",
     "functions/social-orchestrator/",
-    "functions/social-task-handler/"
+    "functions/social-task-handler/",
+    "functions/price-backfill/"
   ],
   "keywords": [
     "bulltrackers",
@@ -31,11 +32,11 @@
     "@google-cloud/firestore": "^7.11.3",
     "sharedsetup": "latest",
     "require-all": "^3.0.0",
-    "aiden-shared-calculations-unified": "1.0.9",
+    "aiden-shared-calculations-unified": "1.0.11",
     "@google-cloud/pubsub": "latest",
     "express": "^4.19.2",
     "cors": "^2.8.5",
-    "p-limit": "latest"
+    "p-limit": "^3.1.0"
   },
   "engines": {
     "node": ">=20"