@pioneer-platform/markets 8.11.13 → 8.11.23

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @pioneer-platform/markets
 
+## 8.11.23
+
+### Patch Changes
+
+- cache work
+- Updated dependencies
+  - @pioneer-platform/pro-token@0.8.1
+  - @pioneer-platform/pioneer-discovery@8.11.23
+
 ## 8.11.13
 
 ### Patch Changes

package/lib/index.js

@@ -57,6 +57,7 @@ var TAG = " | market-module | ";
 // @ts-ignore
 var pioneer_discovery_1 = require("@pioneer-platform/pioneer-discovery");
 var caip_1 = require("@shapeshiftoss/caip");
+var Bottleneck = require('bottleneck');
 var axiosLib = require('axios');
 var Axios = axiosLib.default || axiosLib;
 var https = require('https');
@@ -68,15 +69,22 @@ var axios = Axios.create({
 });
 var axiosRetry = require('axios-retry');
 axiosRetry(axios, {
-    retries: 5, // number of retries
+    retries: 3, // Reduced from 5 to 3
     retryDelay: function (retryCount) {
         console.log("retry attempt: ".concat(retryCount));
-        return retryCount * 1000; // time interval between retries
+        return retryCount * 2000; // Increased from 1s to 2s backoff
     },
     retryCondition: function (error) {
-        console.error(error);
-        // if retry condition is not specified, by default idempotent requests are retried
-        return error.response.status === 503;
+        var _a;
+        var status = (_a = error.response) === null || _a === void 0 ? void 0 : _a.status;
+        // CRITICAL FIX: Never retry rate limits (429) - respect the API!
+        // Also never retry 4xx client errors (except 503 service unavailable)
+        if (status === 429 || status === 403) {
+            console.warn("Rate limit or forbidden (".concat(status, "), not retrying"));
+            return false;
+        }
+        // Only retry on actual service unavailability
+        return status === 503 || status === 502 || status === 504;
     },
 });
 var ProToken = require("@pioneer-platform/pro-token");
@@ -102,6 +110,30 @@ if (CMC_PRO_API_KEY) {
 var GLOBAL_RATES_COINCAP;
 var GLOBAL_RATES_COINGECKO;
 var GLOBAL_RATES_CMC;
+// FIX: Request coalescing to prevent thundering herd
+// Tracks in-flight API requests to prevent duplicate calls for same asset
+var pendingPriceRequests = new Map();
+var pendingPriceRequestsWithSource = new Map();
+// FIX: Rate limiters for external APIs
+// CoinGecko free tier: 50 calls/minute = 1.2s between calls (we use 1.5s for safety)
+var coingeckoLimiter = new Bottleneck({
+    minTime: 1500, // 1.5 seconds between calls (40 calls/minute)
+    maxConcurrent: 1
+});
+// CoinMarketCap: Varies by plan, using conservative limits
+// Basic plan: 333 calls/day ≈ 1 call every 4 minutes, but allow bursts
+var coinmarketcapLimiter = new Bottleneck({
+    reservoir: 10, // Allow 10 calls initially
+    reservoirRefreshAmount: 10,
+    reservoirRefreshInterval: 60 * 1000, // Refresh 10 calls every minute
+    minTime: 2000, // 2 seconds between calls
+    maxConcurrent: 1
+});
+// CoinCap API 3.0: Similar to CoinGecko
+var coincapLimiter = new Bottleneck({
+    minTime: 1500, // 1.5 seconds between calls
+    maxConcurrent: 1
+});
 module.exports = {
     init: function (settings) {
         if (settings === null || settings === void 0 ? void 0 : settings.apiKey) {
@@ -113,6 +145,10 @@ module.exports = {
     getAssetPriceByCaip: function (caip, returnSource) {
         return get_asset_price_by_caip(caip, returnSource);
     },
+    // NEW: Batch price lookup (significantly reduces API calls)
+    getBatchPricesByCaip: function (caips, returnSource) {
+        return get_batch_prices_by_caip(caips, returnSource);
+    },
     // Legacy bulk fetch functions
     getAssetsCoinCap: function () {
         return get_assets_coincap();
@@ -684,10 +720,12 @@ var caip_to_identifiers = function (caip) {
 };
 /**
  * Get price from CoinGecko by coin ID
+ * Now with rate limiting to prevent API spam
  */
 var get_price_from_coingecko = function (coingeckoId) {
     return __awaiter(this, void 0, void 0, function () {
-        var tag, url, response, price, error_1, status_3;
+        var tag, error_1, status_3;
+        var _this = this;
         var _a;
         return __generator(this, function (_b) {
             switch (_b.label) {
@@ -696,18 +734,29 @@ var get_price_from_coingecko = function (coingeckoId) {
                     _b.label = 1;
                 case 1:
                     _b.trys.push([1, 3, , 4]);
-                    url = "".concat(URL_COINGECKO, "simple/price?ids=").concat(coingeckoId, "&vs_currencies=usd");
-                    log.debug(tag, "Fetching from CoinGecko: ".concat(url));
-                    return [4 /*yield*/, axios.get(url)];
-                case 2:
-                    response = _b.sent();
-                    if (response.data && response.data[coingeckoId] && response.data[coingeckoId].usd) {
-                        price = parseFloat(response.data[coingeckoId].usd);
-                        log.debug(tag, "\u2705 CoinGecko price for ".concat(coingeckoId, ": $").concat(price));
-                        return [2 /*return*/, price];
-                    }
-                    log.debug(tag, "No price data from CoinGecko for ".concat(coingeckoId));
-                    return [2 /*return*/, 0];
+                    return [4 /*yield*/, coingeckoLimiter.schedule(function () { return __awaiter(_this, void 0, void 0, function () {
+                            var url, response, price;
+                            return __generator(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0:
+                                        url = "".concat(URL_COINGECKO, "simple/price?ids=").concat(coingeckoId, "&vs_currencies=usd");
+                                        log.debug(tag, "Fetching from CoinGecko: ".concat(url));
+                                        return [4 /*yield*/, axios.get(url)];
+                                    case 1:
+                                        response = _a.sent();
+                                        if (response.data && response.data[coingeckoId] && response.data[coingeckoId].usd) {
+                                            price = parseFloat(response.data[coingeckoId].usd);
+                                            log.debug(tag, "\u2705 CoinGecko price for ".concat(coingeckoId, ": $").concat(price));
+                                            return [2 /*return*/, price];
+                                        }
+                                        log.debug(tag, "No price data from CoinGecko for ".concat(coingeckoId));
+                                        return [2 /*return*/, 0];
+                                }
+                            });
+                        }); })];
+                case 2: 
+                // Apply rate limiting
+                return [2 /*return*/, _b.sent()];
                 case 3:
                     error_1 = _b.sent();
                     status_3 = ((_a = error_1.response) === null || _a === void 0 ? void 0 : _a.status) || error_1.status;
@@ -725,49 +774,63 @@ var get_price_from_coingecko = function (coingeckoId) {
 };
 /**
  * Get price from CoinMarketCap by symbol
+ * Now with rate limiting to prevent API spam
  */
 var get_price_from_coinmarketcap = function (symbol) {
     return __awaiter(this, void 0, void 0, function () {
-        var tag, url, response, coinData, priceData, price, error_2, status_4;
-        var _a, _b, _c;
-        return __generator(this, function (_d) {
-            switch (_d.label) {
+        var tag, error_2, status_4;
+        var _this = this;
+        var _a;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
                 case 0:
                     tag = TAG + ' | get_price_from_coinmarketcap | ';
                     if (!CMC_PRO_API_KEY) {
                         log.debug(tag, 'CMC_PRO_API_KEY not set, skipping CoinMarketCap');
                         return [2 /*return*/, 0];
                     }
-                    _d.label = 1;
+                    _b.label = 1;
                 case 1:
-                    _d.trys.push([1, 3, , 4]);
-                    url = "".concat(URL_COINMARKETCAP, "cryptocurrency/quotes/latest?symbol=").concat(symbol, "&convert=USD");
-                    log.debug(tag, "Fetching from CoinMarketCap: ".concat(url));
-                    return [4 /*yield*/, axios.get(url, {
-                            headers: {
-                                'X-CMC_PRO_API_KEY': CMC_PRO_API_KEY,
-                                'Accept': 'application/json'
-                            }
-                        })];
-                case 2:
-                    response = _d.sent();
-                    if (response.data && response.data.data && response.data.data[symbol]) {
-                        coinData = response.data.data[symbol];
-                        priceData = (_b = (_a = coinData.quote) === null || _a === void 0 ? void 0 : _a.USD) === null || _b === void 0 ? void 0 : _b.price;
-                        if (!priceData || isNaN(parseFloat(priceData))) {
-                            log.warn(tag, "\u26A0\uFE0F CoinMarketCap returned invalid price for ".concat(symbol));
-                            log.warn(tag, "Symbol: ".concat(coinData.symbol, ", Slug: ").concat(coinData.slug, ", Quote keys:"), Object.keys(coinData.quote || {}));
-                            return [2 /*return*/, 0];
-                        }
-                        price = parseFloat(priceData);
-                        log.debug(tag, "\u2705 CoinMarketCap price for ".concat(symbol, ": $").concat(price));
-                        return [2 /*return*/, price];
-                    }
-                    log.debug(tag, "No price data from CoinMarketCap for ".concat(symbol));
-                    return [2 /*return*/, 0];
+                    _b.trys.push([1, 3, , 4]);
+                    return [4 /*yield*/, coinmarketcapLimiter.schedule(function () { return __awaiter(_this, void 0, void 0, function () {
+                            var url, response, coinData, priceData, price;
+                            var _a, _b;
+                            return __generator(this, function (_c) {
+                                switch (_c.label) {
+                                    case 0:
+                                        url = "".concat(URL_COINMARKETCAP, "cryptocurrency/quotes/latest?symbol=").concat(symbol, "&convert=USD");
+                                        log.debug(tag, "Fetching from CoinMarketCap: ".concat(url));
+                                        return [4 /*yield*/, axios.get(url, {
+                                                headers: {
+                                                    'X-CMC_PRO_API_KEY': CMC_PRO_API_KEY,
+                                                    'Accept': 'application/json'
+                                                }
+                                            })];
+                                    case 1:
+                                        response = _c.sent();
+                                        if (response.data && response.data.data && response.data.data[symbol]) {
+                                            coinData = response.data.data[symbol];
+                                            priceData = (_b = (_a = coinData.quote) === null || _a === void 0 ? void 0 : _a.USD) === null || _b === void 0 ? void 0 : _b.price;
+                                            if (!priceData || isNaN(parseFloat(priceData))) {
+                                                log.warn(tag, "\u26A0\uFE0F CoinMarketCap returned invalid price for ".concat(symbol));
+                                                log.warn(tag, "Symbol: ".concat(coinData.symbol, ", Slug: ").concat(coinData.slug, ", Quote keys:"), Object.keys(coinData.quote || {}));
+                                                return [2 /*return*/, 0];
+                                            }
+                                            price = parseFloat(priceData);
+                                            log.debug(tag, "\u2705 CoinMarketCap price for ".concat(symbol, ": $").concat(price));
+                                            return [2 /*return*/, price];
+                                        }
+                                        log.debug(tag, "No price data from CoinMarketCap for ".concat(symbol));
+                                        return [2 /*return*/, 0];
+                                }
+                            });
+                        }); })];
+                case 2: 
+                // Apply rate limiting
+                return [2 /*return*/, _b.sent()];
                 case 3:
-                    error_2 = _d.sent();
-                    status_4 = ((_c = error_2.response) === null || _c === void 0 ? void 0 : _c.status) || error_2.status;
+                    error_2 = _b.sent();
+                    status_4 = ((_a = error_2.response) === null || _a === void 0 ? void 0 : _a.status) || error_2.status;
                     if (status_4 === 429 || status_4 === 403) {
                         log.warn(tag, "CoinMarketCap rate limit (".concat(status_4, ") for ").concat(symbol));
                     }
@@ -782,10 +845,12 @@ var get_price_from_coinmarketcap = function (symbol) {
 };
 /**
  * Get price from CoinCap by symbol
+ * Now with rate limiting to prevent API spam
  */
 var get_price_from_coincap = function (symbol) {
     return __awaiter(this, void 0, void 0, function () {
-        var tag, url, response, price, error_3, status_5;
+        var tag, error_3, status_5;
+        var _this = this;
         var _a;
         return __generator(this, function (_b) {
             switch (_b.label) {
@@ -798,23 +863,34 @@ var get_price_from_coincap = function (symbol) {
                     _b.label = 1;
                 case 1:
                     _b.trys.push([1, 3, , 4]);
-                    url = "".concat(URL_COINCAP, "assets/").concat(symbol);
-                    log.debug(tag, "Fetching from CoinCap: ".concat(url));
-                    return [4 /*yield*/, axios.get(url, {
-                            headers: {
-                                'Accept': 'application/json',
-                                'Authorization': "Bearer ".concat(COINCAP_API_KEY)
-                            }
-                        })];
-                case 2:
-                    response = _b.sent();
-                    if (response.data && response.data.data && response.data.data.priceUsd) {
-                        price = parseFloat(response.data.data.priceUsd);
-                        log.debug(tag, "\u2705 CoinCap price for ".concat(symbol, ": $").concat(price));
-                        return [2 /*return*/, price];
-                    }
-                    log.debug(tag, "No price data from CoinCap for ".concat(symbol));
-                    return [2 /*return*/, 0];
+                    return [4 /*yield*/, coincapLimiter.schedule(function () { return __awaiter(_this, void 0, void 0, function () {
+                            var url, response, price;
+                            return __generator(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0:
+                                        url = "".concat(URL_COINCAP, "assets/").concat(symbol);
+                                        log.debug(tag, "Fetching from CoinCap: ".concat(url));
+                                        return [4 /*yield*/, axios.get(url, {
+                                                headers: {
+                                                    'Accept': 'application/json',
+                                                    'Authorization': "Bearer ".concat(COINCAP_API_KEY)
+                                                }
+                                            })];
+                                    case 1:
+                                        response = _a.sent();
+                                        if (response.data && response.data.data && response.data.data.priceUsd) {
+                                            price = parseFloat(response.data.data.priceUsd);
+                                            log.debug(tag, "\u2705 CoinCap price for ".concat(symbol, ": $").concat(price));
+                                            return [2 /*return*/, price];
+                                        }
+                                        log.debug(tag, "No price data from CoinCap for ".concat(symbol));
+                                        return [2 /*return*/, 0];
+                                }
+                            });
+                        }); })];
+                case 2: 
+                // Apply rate limiting
+                return [2 /*return*/, _b.sent()];
                 case 3:
                     error_3 = _b.sent();
                     status_5 = ((_a = error_3.response) === null || _a === void 0 ? void 0 : _a.status) || error_3.status;
@@ -881,14 +957,41 @@ var get_price_from_mayascan = function () {
  * Returns: { price: number, source: string } or backwards compatible number
  */
 var get_asset_price_by_caip = function (caip_2) {
+    return __awaiter(this, arguments, void 0, function (caip, returnSource) {
+        var tag, pendingMap, existingRequest, pricePromise;
+        if (returnSource === void 0) { returnSource = false; }
+        return __generator(this, function (_a) {
+            tag = TAG + ' | get_asset_price_by_caip | ';
+            log.debug(tag, "Looking up price for: ".concat(caip));
+            pendingMap = returnSource ? pendingPriceRequestsWithSource : pendingPriceRequests;
+            existingRequest = pendingMap.get(caip);
+            if (existingRequest) {
+                log.debug(tag, "Coalescing duplicate request for: ".concat(caip));
+                return [2 /*return*/, existingRequest];
+            }
+            pricePromise = get_asset_price_by_caip_internal(caip, returnSource);
+            // Store promise for request coalescing
+            pendingMap.set(caip, pricePromise);
+            // Cleanup after completion (success or failure)
+            pricePromise.finally(function () {
+                pendingMap.delete(caip);
+            });
+            return [2 /*return*/, pricePromise];
+        });
+    });
+};
+/**
+ * Internal implementation (separated for request coalescing)
+ */
+var get_asset_price_by_caip_internal = function (caip_2) {
     return __awaiter(this, arguments, void 0, function (caip, returnSource) {
         var tag, mayaPrice, isCacao, identifiers, price, cacaoPrice;
         if (returnSource === void 0) { returnSource = false; }
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
-                    tag = TAG + ' | get_asset_price_by_caip | ';
-                    log.debug(tag, "Looking up price for: ".concat(caip));
+                    tag = TAG + ' | get_asset_price_by_caip_internal | ';
+                    log.debug(tag, "Fetching price for: ".concat(caip));
                     if (!(caip === 'cosmos:mayachain-mainnet-v1/denom:maya' || caip.toLowerCase().includes('denom:maya'))) return [3 /*break*/, 2];
                     log.debug(tag, 'MAYA token detected, using MayaScan API');
                     return [4 /*yield*/, get_price_from_mayascan('maya')];
@@ -1082,3 +1185,134 @@ var get_price = function (asset) {
         });
     });
 };
+/**
+ * NEW: Batch price lookup by CAIP
+ * Fetches multiple asset prices in a single API call (up to 250 assets)
+ * Reduces API calls by 95%+ for portfolio operations
+ */
+var get_batch_prices_by_caip = function (caips_1) {
+    return __awaiter(this, arguments, void 0, function (caips, returnSource) {
+        var tag, results, caipToIdMap, specialCaips, _i, caips_2, caip, identifiers, coingeckoIds, batchSize, i, batch, idsParam, url, response, _a, batch_1, coingeckoId, mapping, price, error_5, _b, specialCaips_1, caip, price, error_6, missingCaips, _c, missingCaips_1, caip, _d, _e, error_7;
+        var _f;
+        if (returnSource === void 0) { returnSource = false; }
+        return __generator(this, function (_g) {
+            switch (_g.label) {
+                case 0:
+                    tag = TAG + ' | get_batch_prices_by_caip | ';
+                    log.info(tag, "Fetching batch prices for ".concat(caips.length, " assets"));
+                    results = {};
+                    caipToIdMap = {};
+                    specialCaips = [];
+                    for (_i = 0, caips_2 = caips; _i < caips_2.length; _i++) {
+                        caip = caips_2[_i];
+                        // Handle special cases (MAYA, CACAO)
+                        if (caip === 'cosmos:mayachain-mainnet-v1/denom:maya' || caip.toLowerCase().includes('denom:maya')) {
+                            specialCaips.push(caip);
+                            continue;
+                        }
+                        if (caip === 'cosmos:mayachain-mainnet-v1/slip44:931') {
+                            specialCaips.push(caip);
+                            continue;
+                        }
+                        identifiers = caip_to_identifiers(caip);
+                        if (identifiers === null || identifiers === void 0 ? void 0 : identifiers.coingeckoId) {
+                            caipToIdMap[identifiers.coingeckoId] = { caip: caip, coingeckoId: identifiers.coingeckoId };
+                        }
+                    }
+                    coingeckoIds = Object.keys(caipToIdMap);
+                    if (!(coingeckoIds.length > 0)) return [3 /*break*/, 8];
+                    _g.label = 1;
+                case 1:
+                    _g.trys.push([1, 7, , 8]);
+                    batchSize = 250;
+                    i = 0;
+                    _g.label = 2;
+                case 2:
+                    if (!(i < coingeckoIds.length)) return [3 /*break*/, 6];
+                    batch = coingeckoIds.slice(i, i + batchSize);
+                    idsParam = batch.join(',');
+                    url = "".concat(URL_COINGECKO, "simple/price?ids=").concat(idsParam, "&vs_currencies=usd");
+                    log.debug(tag, "Fetching batch ".concat(i / batchSize + 1, ": ").concat(batch.length, " assets"));
+                    return [4 /*yield*/, axios.get(url)];
+                case 3:
+                    response = _g.sent();
+                    // Map results back to CAIPs
+                    for (_a = 0, batch_1 = batch; _a < batch_1.length; _a++) {
+                        coingeckoId = batch_1[_a];
+                        mapping = caipToIdMap[coingeckoId];
+                        if ((_f = response.data[coingeckoId]) === null || _f === void 0 ? void 0 : _f.usd) {
+                            price = parseFloat(response.data[coingeckoId].usd);
+                            results[mapping.caip] = returnSource
+                                ? { price: price, source: 'coingecko' }
+                                : price;
+                        }
+                    }
+                    if (!(i + batchSize < coingeckoIds.length)) return [3 /*break*/, 5];
+                    return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 1500); })];
+                case 4:
+                    _g.sent();
+                    _g.label = 5;
+                case 5:
+                    i += batchSize;
+                    return [3 /*break*/, 2];
+                case 6:
+                    log.info(tag, "\u2705 Batch fetched ".concat(Object.keys(results).length, " prices from CoinGecko"));
+                    return [3 /*break*/, 8];
+                case 7:
+                    error_5 = _g.sent();
+                    log.error(tag, "CoinGecko batch fetch failed:", error_5.message);
+                    return [3 /*break*/, 8];
+                case 8:
+                    _b = 0, specialCaips_1 = specialCaips;
+                    _g.label = 9;
+                case 9:
+                    if (!(_b < specialCaips_1.length)) return [3 /*break*/, 14];
+                    caip = specialCaips_1[_b];
+                    _g.label = 10;
+                case 10:
+                    _g.trys.push([10, 12, , 13]);
+                    return [4 /*yield*/, get_asset_price_by_caip(caip, returnSource)];
+                case 11:
+                    price = _g.sent();
+                    results[caip] = price;
+                    return [3 /*break*/, 13];
+                case 12:
+                    error_6 = _g.sent();
+                    log.warn(tag, "Failed to fetch special case: ".concat(caip));
+                    return [3 /*break*/, 13];
+                case 13:
+                    _b++;
+                    return [3 /*break*/, 9];
+                case 14:
+                    missingCaips = caips.filter(function (caip) { return !results[caip]; });
+                    if (!(missingCaips.length > 0)) return [3 /*break*/, 20];
+                    log.info(tag, "Falling back to individual fetch for ".concat(missingCaips.length, " missing prices"));
+                    _c = 0, missingCaips_1 = missingCaips;
+                    _g.label = 15;
+                case 15:
+                    if (!(_c < missingCaips_1.length)) return [3 /*break*/, 20];
+                    caip = missingCaips_1[_c];
+                    _g.label = 16;
+                case 16:
+                    _g.trys.push([16, 18, , 19]);
+                    _d = results;
+                    _e = caip;
+                    return [4 /*yield*/, get_asset_price_by_caip(caip, returnSource)];
+                case 17:
+                    _d[_e] = _g.sent();
+                    return [3 /*break*/, 19];
+                case 18:
+                    error_7 = _g.sent();
+                    log.warn(tag, "Failed to fetch fallback price for ".concat(caip));
+                    results[caip] = returnSource ? { price: 0, source: 'none' } : 0;
+                    return [3 /*break*/, 19];
+                case 19:
+                    _c++;
+                    return [3 /*break*/, 15];
+                case 20:
+                    log.info(tag, "\u2705 Batch complete: ".concat(Object.keys(results).length, "/").concat(caips.length, " prices fetched"));
+                    return [2 /*return*/, results];
+            }
+        });
+    });
+};

package/package.json

@@ -1,18 +1,19 @@
 {
   "name": "@pioneer-platform/markets",
-  "version": "8.11.13",
+  "version": "8.11.23",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
   "dependencies": {
-    "@pioneer-platform/default-redis": "^8.11.0",
+    "@pioneer-platform/default-redis": "^8.11.7",
     "@pioneer-platform/loggerdog": "^8.11.0",
     "@pioneer-platform/pioneer-coins": "^9.11.0",
-    "@pioneer-platform/pioneer-discovery": "^8.11.11",
+    "@pioneer-platform/pioneer-discovery": "^8.11.23",
     "@pioneer-platform/pioneer-types": "^8.11.0",
-    "@pioneer-platform/pro-token": "^0.8.0",
+    "@pioneer-platform/pro-token": "^0.8.1",
     "@shapeshiftoss/caip": "^9.0.0-alpha.0",
     "axios": "^1.6.0",
     "axios-retry": "^3.2.0",
+    "bottleneck": "^2.19.5",
     "dotenv": "^8.2.0"
   },
   "scripts": {