@defillama/sdk 5.0.4 → 5.0.6

package/build/util/cache.js

@@ -257,7 +257,10 @@ function getTempLocalCache(_a) {
             removePromisesAndFunctions(fileData);
             _fs.writeFileSync(filePath, JSON.stringify(fileData));
         }
-        catch (error) { }
+        catch (error) {
+            (0, debugLog_1.debugLog)('Error saving cache:', error);
+        }
+        process.exit(0);
     }
     function removePromisesAndFunctions(data) {
         if (typeof data === 'object') {

package/build/util/index.d.ts

@@ -3,6 +3,7 @@ import type { Address } from "../types";
 import { EventLog } from "ethers";
 import runInPromisePoolOrig from "./promisePool";
 export { getLatestBlock, getTimestamp, lookupBlock, } from "./blocks";
+export { convertToBigInt } from "../generalUtil";
 export declare const runInPromisePool: typeof runInPromisePoolOrig;
 export declare function sliceIntoChunks(arr: any[], chunkSize?: number): any[][];
 export declare function getLogs(params: {
@@ -11,7 +12,7 @@ export declare function getLogs(params: {
     keys: string[];
     fromBlock: number;
     toBlock: number;
-    topics?: string[];
+    topics?: (string | null)[];
     chain?: Chain;
 }): Promise<{
     output: EventLog[];

package/build/util/index.js

@@ -50,7 +50,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizeBalances = exports.normalizePrefixes = exports.normalizeAddress = exports.getLogs = exports.sliceIntoChunks = exports.runInPromisePool = exports.lookupBlock = exports.getTimestamp = exports.getLatestBlock = void 0;
+exports.normalizeBalances = exports.normalizePrefixes = exports.normalizeAddress = exports.getLogs = exports.sliceIntoChunks = exports.runInPromisePool = exports.convertToBigInt = exports.lookupBlock = exports.getTimestamp = exports.getLatestBlock = void 0;
 var general_1 = require("../general");
 var ethers_1 = require("ethers");
 var generalUtil_1 = require("../generalUtil");
@@ -60,6 +60,8 @@ var blocks_1 = require("./blocks");
 Object.defineProperty(exports, "getLatestBlock", { enumerable: true, get: function () { return blocks_1.getLatestBlock; } });
 Object.defineProperty(exports, "getTimestamp", { enumerable: true, get: function () { return blocks_1.getTimestamp; } });
 Object.defineProperty(exports, "lookupBlock", { enumerable: true, get: function () { return blocks_1.lookupBlock; } });
+var generalUtil_2 = require("../generalUtil");
+Object.defineProperty(exports, "convertToBigInt", { enumerable: true, get: function () { return generalUtil_2.convertToBigInt; } });
 exports.runInPromisePool = promisePool_1.default;
 function sliceIntoChunks(arr, chunkSize) {
     if (chunkSize === void 0) { chunkSize = 100; }
@@ -75,7 +77,7 @@ exports.sliceIntoChunks = sliceIntoChunks;
 function getLogs(params) {
     var _a;
     return __awaiter(this, void 0, void 0, function () {
-        var filter, logs, blockSpread, currentBlock, nextBlock, logParams, partLogs, e_1, error;
+        var filter, logs, blockSpread, currentBlock, provider, nextBlock, logParams, partLogs, e_1, error;
         return __generator(this, function (_b) {
             switch (_b.label) {
                 case 0:
@@ -91,6 +93,7 @@ function getLogs(params) {
                     logs = [];
                     blockSpread = params.toBlock - params.fromBlock;
                     currentBlock = params.fromBlock;
+                    provider = (0, general_1.getProvider)(params.chain, true);
                     _b.label = 1;
                 case 1:
                     if (!(currentBlock < params.toBlock)) return [3 /*break*/, 6];
@@ -99,7 +102,7 @@ function getLogs(params) {
                     _b.label = 2;
                 case 2:
                     _b.trys.push([2, 4, , 5]);
-                    return [4 /*yield*/, (0, general_1.getProvider)(params.chain, true).getLogs(logParams)];
+                    return [4 /*yield*/, provider.getLogs(logParams)];
                 case 3:
                     partLogs = _b.sent();
                     logs = logs.concat(partLogs);
@@ -107,14 +110,14 @@ function getLogs(params) {
                     return [3 /*break*/, 5];
                 case 4:
                     e_1 = _b.sent();
-                    (0, debugLog_1.debugLog)("Error fetching logs for chain ".concat(params.chain, " blockSpread: ").concat(blockSpread, ". ").concat((0, generalUtil_1.formError)(e_1)));
+                    (0, debugLog_1.debugLog)("Error fetching logs for chain ".concat(params.chain, " blockSpread: ").concat(blockSpread, ". ").concat((0, generalUtil_1.formError)(e_1, { chain: params.chain, target: params.target, provider: provider })));
                     if (blockSpread >= 2e3) {
                         // We got too many results
                         // We could chop it up into 2K block spreads as that is guaranteed to always return but then we'll have to make a lot of queries (easily >1000), so instead we'll keep dividing the block spread by two until we make it
                         blockSpread = Math.floor(blockSpread / 2);
                     }
                     else {
-                        error = (0, generalUtil_1.formError)(e_1);
+                        error = (0, generalUtil_1.formError)(e_1, { chain: params.chain, target: params.target, provider: provider });
                         error.message = "[chain: ".concat(params.chain, "] ").concat(error === null || error === void 0 ? void 0 : error.message, " params: ").concat(JSON.stringify(logParams));
                         throw e_1;
                     }
@@ -129,8 +132,11 @@ function getLogs(params) {
                                 output: logs.map(function (log) { return log.topics; })
                             }];
                     }
-                    // ethers v5 logs had this but not ethers v6, so adding field to keep it compatible
-                    logs.forEach(function (log) { var _a; return log.logIndex = (_a = log.logIndex) !== null && _a !== void 0 ? _a : log.index; });
+                    // ethers v5 logs had 'logIndex' but not ethers v6, so adding field to keep it compatible
+                    logs.forEach(function (log) {
+                        log.logIndex = log.index;
+                        delete log.provider;
+                    });
                     return [2 /*return*/, {
                             output: logs
                         }];

package/build/util/logs.d.ts

@@ -1,12 +1,12 @@
 import { EventLog } from "ethers";
 import { Address } from "../types";
 export type GetLogsOptions = {
-    target: Address;
+    target?: Address;
     topic?: string;
     keys?: string[];
     fromBlock?: number;
     toBlock?: number;
-    topics?: string[];
+    topics?: (string | null)[];
     extraTopics?: string[];
     chain?: string;
     eventAbi?: string | any;
@@ -17,9 +17,10 @@ export type GetLogsOptions = {
     entireLog?: boolean;
     cacheInCloud?: boolean;
     onlyArgs?: boolean;
+    targets?: Address[];
+    flatten?: boolean;
 };
-export declare function getLogs({ target, chain, fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, // can be passed as input as extra filter arguments
-skipCache, skipCacheRead, entireLog, cacheInCloud, onlyArgs, keys, }: GetLogsOptions): Promise<EventLog[] | any[]>;
+export declare function getLogs(options: GetLogsOptions): Promise<EventLog[] | EventLog[][] | any[]>;
 export type logCache = {
     logs: EventLog[];
     metadata?: {

package/build/util/logs.js

@@ -1,4 +1,15 @@
 "use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -35,6 +46,9 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
         if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
     }
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getLogs = void 0;
 var _1 = require(".");
@@ -42,11 +56,9 @@ var ethers_1 = require("ethers");
 var blocks_1 = require("./blocks");
 var cache_1 = require("./cache");
 var debugLog_1 = require("./debugLog");
+var p_limit_1 = __importDefault(require("p-limit"));
 var currentVersion = 'v2';
-function getLogs(_a) {
-    var target = _a.target, _b = _a.chain, chain = _b === void 0 ? 'ethereum' : _b, fromBlock = _a.fromBlock, toBlock = _a.toBlock, fromTimestamp = _a.fromTimestamp, toTimestamp = _a.toTimestamp, eventAbi = _a.eventAbi, topic = _a.topic, topics = _a.topics, extraTopics = _a.extraTopics, // can be passed as input as extra filter arguments
-    _c = _a.skipCache, // can be passed as input as extra filter arguments
-    skipCache = _c === void 0 ? false : _c, _d = _a.skipCacheRead, skipCacheRead = _d === void 0 ? false : _d, _e = _a.entireLog, entireLog = _e === void 0 ? false : _e, _f = _a.cacheInCloud, cacheInCloud = _f === void 0 ? false : _f, _g = _a.onlyArgs, onlyArgs = _g === void 0 ? false : _g, _h = _a.keys, keys = _h === void 0 ? [] : _h;
+function getLogs(options) {
     return __awaiter(this, void 0, void 0, function () {
         function addLogsToCache(fromBlock, toBlock) {
             return __awaiter(this, void 0, void 0, function () {
@@ -58,8 +70,7 @@ function getLogs(_a) {
                             metadata = cacheData.metadata;
                             return [4 /*yield*/, (0, _1.getLogs)({
                                     chain: chain,
-                                    target: target,
-                                    topic: topic,
+                                    target: target, topic: topic,
                                     keys: keys,
                                     topics: topics,
                                     fromBlock: fromBlock,
@@ -123,7 +134,7 @@ function getLogs(_a) {
                 throw new Error('extraKey is required');
             if (keys.length)
                 extraKey += '-' + keys.join('-').toLowerCase();
-            return "event-logs/".concat(chain, "/").concat(target.toLowerCase(), "/").concat(target.toLowerCase());
+            return "event-logs/".concat(chain, "/").concat(target.toLowerCase(), "-").concat(extraKey);
         }
         // we need to form filter topic with indexed keyword, else it messes up generated topic string
         function toFilterTopic(topic) {
@@ -135,36 +146,47 @@ function getLogs(_a) {
             var fragment = topic.fragments[0];
             return (0, ethers_1.id)("".concat(fragment.name, "(").concat(fragment.inputs.map(function (i) { return i.type; }).join(','), ")"));
         }
-        var iface, cacheData, isFirstRun, metadata, logs;
+        var target, _a, chain, fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, _b, skipCache, _c, skipCacheRead, _d, entireLog, _e, cacheInCloud, _f, onlyArgs, _g, keys, targets, _h, flatten, limiter, newOptions_1, res, iface, cacheData, isFirstRun, metadata_1, logs;
         return __generator(this, function (_j) {
             switch (_j.label) {
                 case 0:
-                    if (!target)
-                        throw new Error('target is required');
+                    target = options.target, _a = options.chain, chain = _a === void 0 ? 'ethereum' : _a, fromBlock = options.fromBlock, toBlock = options.toBlock, fromTimestamp = options.fromTimestamp, toTimestamp = options.toTimestamp, eventAbi = options.eventAbi, topic = options.topic, topics = options.topics, extraTopics = options.extraTopics, _b = options.skipCache, skipCache = _b === void 0 ? false : _b, _c = options.skipCacheRead, skipCacheRead = _c === void 0 ? false : _c, _d = options.entireLog, entireLog = _d === void 0 ? false : _d, _e = options.cacheInCloud, cacheInCloud = _e === void 0 ? false : _e, _f = options.onlyArgs, onlyArgs = _f === void 0 ? false : _f, _g = options.keys, keys = _g === void 0 ? [] : _g, targets = options.targets, _h = options.flatten, flatten = _h === void 0 ? true : _h;
+                    if (!target && !(targets === null || targets === void 0 ? void 0 : targets.length))
+                        throw new Error('target|targets is required');
                     if (!fromBlock && !fromTimestamp)
                         throw new Error('fromBlock or fromTimestamp is required');
                     if (!toBlock && !toTimestamp)
                         throw new Error('toBlock or toTimestamp is required');
+                    limiter = getChainLimiter(chain);
                     if (!!fromBlock) return [3 /*break*/, 2];
-                    return [4 /*yield*/, (0, blocks_1.getBlock)(chain, fromTimestamp)];
+                    return [4 /*yield*/, limiter(function () { return (0, blocks_1.getBlock)(chain, fromTimestamp); })];
                 case 1:
                     fromBlock = (_j.sent()).block;
                     _j.label = 2;
                 case 2:
                     if (!!toBlock) return [3 /*break*/, 4];
-                    return [4 /*yield*/, (0, blocks_1.getBlock)(chain, toTimestamp)];
+                    return [4 /*yield*/, limiter(function () { return (0, blocks_1.getBlock)(chain, toTimestamp); })];
                 case 3:
                     toBlock = (_j.sent()).block;
                     _j.label = 4;
                 case 4:
-                    if ((!topics || !topics.length) && !topic) {
-                        if (topic) {
+                    if (!(targets === null || targets === void 0 ? void 0 : targets.length)) return [3 /*break*/, 6];
+                    newOptions_1 = __assign(__assign({}, options), { fromBlock: fromBlock, toBlock: toBlock });
+                    delete newOptions_1.targets;
+                    return [4 /*yield*/, Promise.all(targets.map(function (i) { return getLogs(__assign(__assign({}, newOptions_1), { target: i })); }))];
+                case 5:
+                    res = _j.sent();
+                    if (flatten)
+                        return [2 /*return*/, res.flat()];
+                    return [2 /*return*/, res];
+                case 6:
+                    if (eventAbi)
+                        iface = new ethers_1.Interface([eventAbi]);
+                    if ((!topics || !topics.length)) {
+                        if (topic)
                             topic = toFilterTopic(topic);
-                        }
-                        else if (eventAbi) {
-                            iface = new ethers_1.Interface([eventAbi]);
+                        else if (eventAbi)
                             topic = toFilterTopic(iface);
-                        }
                         else {
                             throw new Error('eventAbi | topic | topics are required');
                         }
@@ -173,30 +195,30 @@ function getLogs(_a) {
                             topics.push.apply(topics, extraTopics);
                     }
                     return [4 /*yield*/, _getCache()];
-                case 5:
+                case 7:
                     cacheData = _j.sent();
                     isFirstRun = !cacheData.metadata;
                     if (isFirstRun)
                         cacheData.metadata = { version: currentVersion };
-                    if (!isFirstRun) return [3 /*break*/, 7];
-                    return [4 /*yield*/, addLogsToCache(fromBlock, toBlock)];
-                case 6:
-                    _j.sent();
-                    return [3 /*break*/, 11];
-                case 7:
-                    metadata = cacheData.metadata;
-                    if (!(fromBlock < metadata.fromBlock)) return [3 /*break*/, 9];
-                    return [4 /*yield*/, addLogsToCache(fromBlock, metadata.fromBlock - 1)];
+                    if (!isFirstRun) return [3 /*break*/, 9];
+                    return [4 /*yield*/, limiter(function () { return addLogsToCache(fromBlock, toBlock); })];
                 case 8:
                     _j.sent();
-                    _j.label = 9;
+                    return [3 /*break*/, 13];
                 case 9:
-                    if (!(toBlock > metadata.toBlock)) return [3 /*break*/, 11];
-                    return [4 /*yield*/, addLogsToCache(metadata.toBlock + 1, toBlock)];
+                    metadata_1 = cacheData.metadata;
+                    if (!(fromBlock < metadata_1.fromBlock)) return [3 /*break*/, 11];
+                    return [4 /*yield*/, limiter(function () { return addLogsToCache(fromBlock, metadata_1.fromBlock); })];
                 case 10:
                     _j.sent();
                     _j.label = 11;
                 case 11:
+                    if (!(toBlock > metadata_1.toBlock)) return [3 /*break*/, 13];
+                    return [4 /*yield*/, limiter(function () { return addLogsToCache(metadata_1.toBlock, toBlock); })];
+                case 12:
+                    _j.sent();
+                    _j.label = 13;
+                case 13:
                     logs = cacheData.logs.filter(function (i) {
                         if (i.blockNumber < fromBlock)
                             return false;
@@ -212,4 +234,16 @@ function getLogs(_a) {
     });
 }
 exports.getLogs = getLogs;
+var chainLimiters = {};
+function getChainLimiter(chain) {
+    if (!chainLimiters[chain])
+        chainLimiters[chain] = createLimiter(chain);
+    return chainLimiters[chain];
+    function createLimiter(chain) {
+        var _a;
+        var defaultLimit = chain === 'fantom' ? 10 : 20;
+        var limit = +((_a = process.env["".concat(chain.toUpperCase(), "_RPC_GET_LOGS_CONCURRENCY_LIMIT")]) !== null && _a !== void 0 ? _a : defaultLimit);
+        return (0, p_limit_1.default)(limit);
+    }
+}
 exports.default = getLogs;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.4",
+  "version": "5.0.6",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {