@defillama/sdk 4.0.69 → 4.0.71

package/build/util/computeTVL.d.ts

@@ -0,0 +1,6 @@
+import { Balances } from "../types";
+export default function computeTVL(balances: Balances, timestamp?: number): Promise<{
+    usdTvl: number;
+    usdTokenBalances: Balances;
+}>;
+export declare function getUnique(addresses: string[]): string[];

package/build/util/computeTVL.js

@@ -0,0 +1,185 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getUnique = void 0;
+var _1 = require(".");
+var generalUtil_1 = require("../generalUtil");
+var node_fetch_1 = __importDefault(require("node-fetch"));
+var currentPriceCache = {};
+var priceCaches = {};
+var lastPriceUpdate = 0;
+var confidenceThreshold = 0.5;
+var priceUpdateTime = 1000 * 60 * 30; // 30 minutes
+function computeTVL(balances, timestamp) {
+    return __awaiter(this, void 0, void 0, function () {
+        var usdTokenBalances, usdTvl, keys, priceCache;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    usdTokenBalances = {};
+                    usdTvl = 0;
+                    keys = Object.keys(balances).filter(function (i) { return +balances[i] > 0; }).map(tokenToKey);
+                    return [4 /*yield*/, updatePriceCache(keys, timestamp)];
+                case 1:
+                    _a.sent();
+                    priceCache = getPriceCache(timestamp);
+                    Object.entries(balances).forEach(function (_a) {
+                        var _b;
+                        var token = _a[0], balance = _a[1];
+                        if (+balance <= 0)
+                            return;
+                        var key = tokenToKey(token).toLowerCase();
+                        var _c = (_b = priceCache[key]) !== null && _b !== void 0 ? _b : {}, price = _c.price, confidence = _c.confidence, _d = _c.decimals, decimals = _d === void 0 ? 0 : _d, _e = _c.symbol, symbol = _e === void 0 ? token : _e;
+                        if (!price || confidence < confidenceThreshold)
+                            return;
+                        var value = +balance * price / (Math.pow(10, decimals));
+                        usdTvl += value;
+                        if (symbol === 'ETH')
+                            symbol = 'WETH';
+                        (0, generalUtil_1.sumSingleBalance)(usdTokenBalances, symbol, value);
+                    });
+                    return [2 /*return*/, { usdTvl: usdTvl, usdTokenBalances: usdTokenBalances }];
+            }
+        });
+    });
+}
+exports.default = computeTVL;
+function tokenToKey(token) {
+    if (token.startsWith("0x"))
+        token = "ethereum:".concat(token);
+    else if (!token.includes(':'))
+        token = "coingecko:".concat(token);
+    return token;
+}
+function updatePriceCache(keys, timestamp) {
+    return __awaiter(this, void 0, void 0, function () {
+        function getPrices(keys) {
+            return __awaiter(this, void 0, void 0, function () {
+                var coins_1, coins;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0:
+                            if (!!timestamp) return [3 /*break*/, 2];
+                            return [4 /*yield*/, (0, node_fetch_1.default)("https://coins.llama.fi/prices/current/".concat(keys.join(','))).then(function (res) { return res.json(); })];
+                        case 1:
+                            coins_1 = (_a.sent()).coins;
+                            return [2 /*return*/, coins_1];
+                        case 2: return [4 /*yield*/, (0, node_fetch_1.default)("https://coins.llama.fi/prices?source=internal", {
+                                method: 'POST',
+                                body: JSON.stringify({ coins: keys, timestamp: timestamp }),
+                                headers: { "Content-Type": "application/json" },
+                            }).then(function (res) { return res.json(); })];
+                        case 3:
+                            coins = (_a.sent()).coins;
+                            return [2 /*return*/, coins];
+                    }
+                });
+            });
+        }
+        var pricesCache, missingKeys, chunks, _i, chunks_1, chunk, coins, _a, _b, _c, token, data;
+        return __generator(this, function (_d) {
+            switch (_d.label) {
+                case 0:
+                    if (isNearlyNow(timestamp))
+                        timestamp = undefined;
+                    keys = getUnique(keys);
+                    pricesCache = getPriceCache(timestamp);
+                    if (!timestamp && lastPriceUpdate && +Date.now() > lastPriceUpdate + priceUpdateTime) {
+                        lastPriceUpdate = +Date.now();
+                        Object.keys(pricesCache).forEach(function (key) { return delete pricesCache[key]; }); // clear cache   
+                    }
+                    // hardcode cache for USDT
+                    pricesCache['coingecko:tether'] = {
+                        "price": 1,
+                        "symbol": "USDT",
+                        "confidence": 1
+                    };
+                    pricesCache['ethereum:0xdac17f958d2ee523a2206206994597c13d831ec7'] = {
+                        "price": 1,
+                        "symbol": "USDT",
+                        "decimals": 6,
+                        "confidence": 1
+                    };
+                    missingKeys = keys.filter(function (key) { return !pricesCache[key.toLowerCase()]; });
+                    chunks = (0, _1.sliceIntoChunks)(missingKeys, 100);
+                    _i = 0, chunks_1 = chunks;
+                    _d.label = 1;
+                case 1:
+                    if (!(_i < chunks_1.length)) return [3 /*break*/, 4];
+                    chunk = chunks_1[_i];
+                    return [4 /*yield*/, getPrices(chunk)];
+                case 2:
+                    coins = _d.sent();
+                    for (_a = 0, _b = Object.entries(coins); _a < _b.length; _a++) {
+                        _c = _b[_a], token = _c[0], data = _c[1];
+                        pricesCache[token.toLowerCase()] = data;
+                    }
+                    chunk.map(function (i) { return i.toLowerCase(); }).filter(function (i) { return !pricesCache[i]; }).forEach(function (i) { return pricesCache[i] = {}; });
+                    _d.label = 3;
+                case 3:
+                    _i++;
+                    return [3 /*break*/, 1];
+                case 4: return [2 /*return*/];
+            }
+        });
+    });
+}
+function getPriceCache(timestamp) {
+    if (timestamp)
+        timestamp = getClosestHourlyTimestamp(timestamp);
+    if (timestamp && !priceCaches[timestamp])
+        priceCaches[timestamp] = {};
+    return timestamp ? priceCaches[timestamp] : currentPriceCache;
+}
+function getUnique(addresses) {
+    var set = {};
+    addresses.forEach(function (i) { return set[i] = true; });
+    return Object.keys(set);
+}
+exports.getUnique = getUnique;
+function isNearlyNow(timestamp) {
+    if (!timestamp)
+        return true;
+    return Math.abs(timestamp - +Date.now() / 1000) < 60 * 60;
+}
+function getClosestHourlyTimestamp(timestamp) {
+    return Math.floor(timestamp / 3600) * 3600;
+}

package/build/util/index.js

@@ -135,78 +135,155 @@ var intialBlocks = {
     terra: 4724001,
     crab: 4969901
 };
+var blockscoutEndpoints = {
+    celo: 'https://explorer.celo.org',
+    kava: 'https://explorer.kava.io',
+    onus: 'https://explorer.onuschain.io',
+    base: 'https://base.blockscout.com',
+    scroll: 'https://blockscout.scroll.io',
+};
+function getBlockscoutBlock(timestamp, chain) {
+    return __awaiter(this, void 0, void 0, function () {
+        var api, res, data;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    api = "".concat(blockscoutEndpoints[chain], "/api?module=block&action=getblocknobytime&timestamp=").concat(timestamp, "&closest=before");
+                    return [4 /*yield*/, (0, node_fetch_1.default)(api)];
+                case 1:
+                    res = _a.sent();
+                    return [4 /*yield*/, res.json()];
+                case 2:
+                    data = _a.sent();
+                    if (data.status !== '1')
+                        throw new Error(data.message);
+                    return [2 /*return*/, {
+                            timestamp: timestamp,
+                            block: +data.result.blockNumber
+                        }];
+            }
+        });
+    });
+}
+var blockTimeCache = {};
 function lookupBlock(timestamp, extraParams) {
     var _a, _b, _c;
     if (extraParams === void 0) { extraParams = {}; }
     return __awaiter(this, void 0, void 0, function () {
-        var chain, api, res, data, low, lowBlock, highBlock, firstBlock, lastBlock, provider, firstBlockNum, block, i, time, allowedTimeRange, acceptableBlockImprecision, blockImprecision, imprecision, getPrecision_1, blockDiff, timeDiff, avgBlockTime, closeBlock, midBlock, blocks, e_1;
+        function updateBlock(blocks) {
+            if (blocks === void 0) { blocks = []; }
+            blocks.forEach(addBlockToCache);
+            var getPrecision = function (block) { return block.timestamp - timestamp > 0 ? block.timestamp - timestamp : timestamp - block.timestamp; };
+            blocks.push(highBlock, lowBlock);
+            blocks.sort(function (a, b) { return getPrecision(a) - getPrecision(b); });
+            block = blocks[0];
+            // find the closest upper and lower bound between 4 points
+            lowBlock = blocks.filter(function (i) { return i.timestamp < timestamp; }).reduce(function (lowestBlock, block) { return (timestamp - lowestBlock.timestamp) < (timestamp - block.timestamp) ? lowestBlock : block; });
+            highBlock = blocks.filter(function (i) { return i.timestamp > timestamp; }).reduce(function (highestBlock, block) { return (highestBlock.timestamp - timestamp) < (block.timestamp - timestamp) ? highestBlock : block; });
+            imprecision = getPrecision(block);
+            blockImprecision = highBlock.number - lowBlock.number;
+            // debugLog(`chain: ${chain} block: ${block.number} #calls: ${i} imprecision: ${Number((imprecision)/60).toFixed(2)} (min) block diff: ${blockImprecision} Time Taken: ${Number((Date.now()-time)/1000).toFixed(2)} (in sec)`)
+        }
+        function getChainBlockTimeCache(chain) {
+            if (!blockTimeCache[chain])
+                blockTimeCache[chain] = {};
+            return blockTimeCache[chain];
+        }
+        function addBlockToCache(block) {
+            getChainBlockTimeCache(chain)[block.number] = block;
+        }
+        function getLowBlock() {
+            return Object.values(getChainBlockTimeCache(chain))
+                .filter(function (i) { return i.timestamp < timestamp; })
+                .sort(function (a, b) { return b.timestamp - a.timestamp; })[0];
+        }
+        function getHighBlock() {
+            return Object.values(getChainBlockTimeCache(chain))
+                .filter(function (i) { return i.timestamp > timestamp; })
+                .sort(function (a, b) { return a.timestamp - b.timestamp; })[0];
+        }
+        var chain, response, e_1, api, res, data, low, lowBlock, highBlock, block, i, time, allowedTimeRange, acceptableBlockImprecision, blockImprecision, imprecision, firstBlock, lastBlock, provider, firstBlockNum, blockDiff, timeDiff, avgBlockTime, closeBlock, midBlock, blocks, e_2;
         var _d;
         return __generator(this, function (_e) {
             switch (_e.label) {
                 case 0:
                     chain = (_a = extraParams === null || extraParams === void 0 ? void 0 : extraParams.chain) !== null && _a !== void 0 ? _a : "ethereum";
-                    if (!(chain === 'celo')) return [3 /*break*/, 3];
-                    api = "https://explorer.celo.org/mainnet/api?module=block&action=getblocknobytime&timestamp=".concat(timestamp, "&closest=before");
-                    return [4 /*yield*/, (0, node_fetch_1.default)(api)];
+                    if (!blockscoutEndpoints[chain]) return [3 /*break*/, 4];
+                    _e.label = 1;
                 case 1:
+                    _e.trys.push([1, 3, , 4]);
+                    return [4 /*yield*/, getBlockscoutBlock(timestamp, chain)];
+                case 2:
+                    response = _e.sent();
+                    return [2 /*return*/, response];
+                case 3:
+                    e_1 = _e.sent();
+                    (0, debugLog_1.debugLog)('error fetching block from blockscout', e_1);
+                    return [3 /*break*/, 4];
+                case 4:
+                    if (!(chain === 'waves')) return [3 /*break*/, 7];
+                    api = "https://nodes.wavesnodes.com/blocks/heightByTimestamp/".concat(timestamp);
+                    return [4 /*yield*/, (0, node_fetch_1.default)(api)];
+                case 5:
                     res = _e.sent();
                     return [4 /*yield*/, res.json()];
-                case 2:
+                case 6:
                     data = _e.sent();
                     return [2 /*return*/, {
                             timestamp: timestamp,
-                            block: +data.result.blockNumber
+                            block: +data.height
                         }];
-                case 3:
+                case 7:
                     low = (_b = intialBlocks[chain]) !== null && _b !== void 0 ? _b : 100;
-                    _e.label = 4;
-                case 4:
-                    _e.trys.push([4, 14, , 15]);
+                    lowBlock = getLowBlock();
+                    highBlock = getHighBlock();
+                    i = 0;
+                    time = Date.now();
+                    allowedTimeRange = 15 * 60 // how much imprecision is allowed (15 minutes now)
+                    ;
+                    acceptableBlockImprecision = chain === 'ethereum' ? 20 : 200;
+                    _e.label = 8;
+                case 8:
+                    _e.trys.push([8, 17, , 18]);
                     firstBlock = void 0, lastBlock = void 0;
                     provider = getExtraProvider(chain);
-                    if (!['evmos'].includes(chain)) return [3 /*break*/, 7];
+                    if (!['evmos'].includes(chain)) return [3 /*break*/, 11];
                     return [4 /*yield*/, getBlock(provider, "latest", chain)];
-                case 5:
+                case 9:
                     lastBlock = _e.sent();
                     firstBlockNum = lastBlock.number;
                     switch (chain) {
                         default: firstBlockNum -= 2 * 1e5; // evmos hold only the last 200k block data
                     }
                     return [4 /*yield*/, getBlock(provider, firstBlockNum, chain)];
-                case 6:
+                case 10:
                     firstBlock = _e.sent();
-                    return [3 /*break*/, 9];
-                case 7: return [4 /*yield*/, Promise.all([
+                    return [3 /*break*/, 13];
+                case 11: return [4 /*yield*/, Promise.all([
                         getBlock(provider, "latest", chain),
-                        getBlock(provider, low, chain),
+                        lowBlock ? lowBlock : getBlock(provider, low, chain),
                     ])];
-                case 8:
+                case 12:
                     _d = _e.sent(), lastBlock = _d[0], firstBlock = _d[1];
-                    _e.label = 9;
-                case 9:
+                    _e.label = 13;
+                case 13:
                     lowBlock = firstBlock;
-                    highBlock = lastBlock;
+                    if (!highBlock)
+                        highBlock = lastBlock;
                     if (lastBlock.timestamp - timestamp < -30 * 60) {
                         throw new Error("Last block of chain \"".concat(chain, "\" is further than 30 minutes into the past. Provider is \"").concat((_c = provider === null || provider === void 0 ? void 0 : provider.connection) === null || _c === void 0 ? void 0 : _c.url, "\""));
                     }
-                    if (Math.abs(lastBlock.timestamp - timestamp) < 60 * 30) {
+                    if (Math.abs(highBlock.timestamp - timestamp) < 60 * 30) {
                         // Short-circuit in case we are trying to get the current block
                         return [2 /*return*/, {
-                                block: lastBlock.number,
-                                timestamp: lastBlock.timestamp
+                                block: highBlock.number,
+                                timestamp: highBlock.timestamp
                             }];
                     }
-                    block = void 0;
-                    i = 0;
-                    time = Date.now();
-                    allowedTimeRange = 15 * 60 // how much imprecision is allowed (15 minutes now)
-                    ;
-                    acceptableBlockImprecision = chain === 'ethereum' ? 20 : 200;
-                    blockImprecision = void 0;
-                    imprecision = void 0;
-                    getPrecision_1 = function (block) { return block.timestamp - timestamp > 0 ? block.timestamp - timestamp : timestamp - block.timestamp; };
-                    _e.label = 10;
-                case 10:
+                    updateBlock();
+                    _e.label = 14;
+                case 14:
+                    if (!(imprecision > allowedTimeRange && blockImprecision > acceptableBlockImprecision)) return [3 /*break*/, 16];
                     ++i;
                     blockDiff = highBlock.number - lowBlock.number;
                     timeDiff = highBlock.timestamp - lowBlock.timestamp;
@@ -219,21 +296,11 @@ function lookupBlock(timestamp, extraParams) {
                             getBlock(provider, closeBlock, chain),
                             getBlock(provider, midBlock, chain),
                         ])];
-                case 11:
+                case 15:
                     blocks = _e.sent();
-                    blocks.push(highBlock, lowBlock);
-                    blocks.sort(function (a, b) { return getPrecision_1(a) - getPrecision_1(b); });
-                    block = blocks[0];
-                    // find the closest upper and lower bound between 4 points
-                    lowBlock = blocks.filter(function (i) { return i.timestamp < timestamp; }).reduce(function (lowestBlock, block) { return (timestamp - lowestBlock.timestamp) < (timestamp - block.timestamp) ? lowestBlock : block; });
-                    highBlock = blocks.filter(function (i) { return i.timestamp > timestamp; }).reduce(function (highestBlock, block) { return (highestBlock.timestamp - timestamp) < (block.timestamp - timestamp) ? highestBlock : block; });
-                    imprecision = getPrecision_1(block);
-                    blockImprecision = highBlock.number - lowBlock.number;
-                    _e.label = 12;
-                case 12:
-                    if (imprecision > allowedTimeRange && blockImprecision > acceptableBlockImprecision) return [3 /*break*/, 10];
-                    _e.label = 13;
-                case 13:
+                    updateBlock(blocks);
+                    return [3 /*break*/, 14];
+                case 16:
                     (0, debugLog_1.debugLog)("chain: ".concat(chain, " block: ").concat(block.number, " #calls: ").concat(i, " imprecision: ").concat(Number((imprecision) / 60).toFixed(2), " (min) Time Taken: ").concat(Number((Date.now() - time) / 1000).toFixed(2), " (in sec)"));
                     if (chain !== "bsc" && // this check is there because bsc halted the chain for few days
                         Math.abs(block.timestamp - timestamp) > 3600) {
@@ -243,11 +310,11 @@ function lookupBlock(timestamp, extraParams) {
                             block: block.number,
                             timestamp: block.timestamp
                         }];
-                case 14:
-                    e_1 = _e.sent();
-                    console.log(e_1);
+                case 17:
+                    e_2 = _e.sent();
+                    (0, debugLog_1.debugLog)(e_2);
                     throw new Error("Couldn't find block height for chain ".concat(chain, ", RPC node rugged"));
-                case 15: return [2 /*return*/];
+                case 18: return [2 /*return*/];
             }
         });
     });
@@ -257,7 +324,7 @@ exports.lookupBlock = lookupBlock;
 function getLogs(params) {
     var _a;
     return __awaiter(this, void 0, void 0, function () {
-        var filter, logs, blockSpread, currentBlock, nextBlock, partLogs, e_2;
+        var filter, logs, blockSpread, currentBlock, nextBlock, partLogs, e_3;
         return __generator(this, function (_b) {
             switch (_b.label) {
                 case 0:
@@ -287,14 +354,14 @@ function getLogs(params) {
                     currentBlock = nextBlock;
                     return [3 /*break*/, 5];
                 case 4:
-                    e_2 = _b.sent();
+                    e_3 = _b.sent();
                     if (blockSpread >= 2e3) {
                         // We got too many results
                         // We could chop it up into 2K block spreads as that is guaranteed to always return but then we'll have to make a lot of queries (easily >1000), so instead we'll keep dividing the block spread by two until we make it
                         blockSpread = Math.floor(blockSpread / 2);
                     }
                     else {
-                        throw e_2;
+                        throw e_3;
                     }
                     return [3 /*break*/, 5];
                 case 5: return [3 /*break*/, 1];

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@defillama/sdk",
-  "version": "4.0.69",
+  "version": "4.0.71",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {
     "start": "npm run build && node build/index.js",
     "build": "tsc",
     "watch": "tsc --watch",
-    "test": "jest",
+    "test": "LLAMA_DEBUG_MODE='' SDK_DEBUG='' jest",
     "update-providers": "ts-node src/util/updateProviderList.ts",
     "format": "prettier --write \"src/**/*.ts\"",
     "prepublishOnly": "npm run build"