@defillama/sdk 5.0.136 → 5.0.139

package/build/util/LlamaProvider.js

@@ -9,6 +9,10 @@ const providers_json_1 = __importDefault(require("../providers.json"));
 const env_1 = require("./env");
 const debugLog_1 = require("./debugLog");
 const axios_1 = __importDefault(require("axios"));
+let buildProviders = undefined;
+if (process.env['TEST_MODE']) {
+    buildProviders = require('../../build/providers.json');
+}
 const rpcRequestCounter = {};
 class LlamaProvider extends ethers_1.FallbackProvider {
     constructor({ providers, chainId, chainName, archivalRPCs, }, fallbackConfig) {
@@ -110,6 +114,12 @@ class LlamaProvider extends ethers_1.FallbackProvider {
             runners.splice(1, 0, primaryRunner);
         }
         runners = runners.slice(0, attempts);
+        // print stats every 5 minutes
+        let currentTimeMS = Date.now();
+        if (currentTimeMS - lastPrintTimeMS > 1000 * 60 * 5) {
+            lastPrintTimeMS = currentTimeMS;
+            printRPCCallStats();
+        }
         // if (this.chainName === 'rsk' && method === 'getLogs') {
         //   const hostString = getEnvValue('RSK_ARCHIVAL_RPC')
         //   if (!hostString) 
@@ -175,8 +185,9 @@ function getProvider(chain = "ethereum", _getArchivalNode = false) {
     const rpcKey = chain === 'tron' ? 'tron_evm' : chain;
     if (exports.providers[chain])
         return exports.providers[chain];
+    const pList = buildProviders !== null && buildProviders !== void 0 ? buildProviders : providers_json_1.default;
     // use RPC from env variable if set else use RPC from providers.json
-    let rpcList = (0, env_1.getChainRPCs)(rpcKey, (_a = providers_json_1.default[chain]) === null || _a === void 0 ? void 0 : _a.rpc);
+    let rpcList = (0, env_1.getChainRPCs)(rpcKey, (_a = pList[chain]) === null || _a === void 0 ? void 0 : _a.rpc);
     let archivalRPCList = (0, env_1.getArchivalRPCs)(rpcKey);
     if (!rpcList) {
         // @ts-ignore (throwing error here would alter function behavior and have side effects)
@@ -309,9 +320,11 @@ function getMedianBlockValue(blocks) {
     const mid = Math.floor(blocks.length / 3);
     return blocks.length % 2 !== 0 ? blocks[mid] : (blocks[mid - 1] + blocks[mid]) / 2;
 }
-process.on('exit', () => {
+let lastPrintTimeMS = Date.now();
+function printRPCCallStats() {
     if (Object.keys(rpcRequestCounter).length > 7) {
         (0, debugLog_1.debugLog)('RPC request count per chain');
         (0, debugLog_1.debugTable)(rpcRequestCounter);
     }
-});
+}
+process.on('exit', printRPCCallStats);

package/build/util/env.d.ts

@@ -6,7 +6,7 @@ export declare function getMaxParallelRequests(chain: string): number;
 export declare function getParallelGetLogsLimit(chain: string, providerUrl?: string): number;
 export declare function getParallelGetBlocksLimit(chain: string): number;
 export declare function getEnvRPC(chain: string): string | undefined;
-export declare function getEnvValue(key: string, defaultValue?: string): string | undefined;
+export declare function getEnvValue(key: string, defaultValue?: string): string;
 export declare function getEnvCacheFolder(defaultCacheFolder: string): string;
 export declare function getEnvMulticallAddress(chain: string): string | undefined;
 export declare function getDefaultChunkSize(chain: string): number;

package/build/util/env.js

@@ -1,10 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getChainRPCs = exports.getArchivalRPCs = exports.getBatchMaxCount = exports.getDefaultChunkSize = exports.getEnvMulticallAddress = exports.getEnvCacheFolder = exports.getEnvValue = exports.getEnvRPC = exports.getParallelGetBlocksLimit = exports.getParallelGetLogsLimit = exports.getMaxParallelRequests = exports.getChainId = exports.ENV_CONSTANTS = void 0;
-const _ENV_CONSTANTS = {
-    DEBUG_ENABLED: getEnvValue('DEBUG') === "true" || process.env.LLAMA_DEBUG_MODE ? 'true' : 'false',
-    DEBUG_LEVEL2: process.env.LLAMA_DEBUG_LEVEL2 ? 'true' : 'false',
-};
 const whitelistedEnvConstants = [
     'R2_ENDPOINT', 'R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY', 'TRON_PRO_API_KEY', 'COINS_API_KEY',
     'ELASTICSEARCH_CONFIG', 'GRAPH_API_KEY', 'LLAMA_INDEXER_ENDPOINT', 'LLAMA_INDEXER_API_KEY',
@@ -12,6 +8,7 @@ const whitelistedEnvConstants = [
 const defaultEnvValues = {
     INTERNAL_SDK_CACHE_FILE: './cache.json',
     TRON_RPC_CONCURRENCY_LIMIT: '5',
+    LLAMA_INDEXER_ADDRESS_CHUNK_SIZE: '20',
     TRON_RPC: 'https://api.trongrid.io',
     TRON_EVM_RPC: 'https://rpc.ankr.com/tron_jsonrpc',
     NAKA_RPC: 'https://node.nakachain.xyz',
@@ -23,6 +20,10 @@ const defaultEnvValues = {
     ZETA_RPC: "https://zetachain-evm.blockpi.network/v1/rpc/public,https://zetachain-mainnet-archive.allthatnode.com:8545",
     DEFIVERSE_RPC: "https://rpc.defi-verse.org/",
 };
+const _ENV_CONSTANTS = {
+    DEBUG_ENABLED: getEnvValue('DEBUG') === "true" || process.env.LLAMA_DEBUG_MODE ? 'true' : 'false',
+    DEBUG_LEVEL2: process.env.LLAMA_DEBUG_LEVEL2 ? 'true' : 'false',
+};
 whitelistedEnvConstants.forEach(key => _ENV_CONSTANTS[key] = getEnvValue(key));
 Object.keys(defaultEnvValues).forEach(key => _ENV_CONSTANTS[key] = getEnvValue(key, defaultEnvValues[key]));
 // doing this as I am paranoid that that empty object when first declared might be exported before values are set
@@ -65,6 +66,7 @@ exports.getEnvRPC = getEnvRPC;
 function getEnvValue(key, defaultValue) {
     var _a, _b, _c;
     key = key.toUpperCase();
+    defaultValue = defaultValue !== null && defaultValue !== void 0 ? defaultValue : defaultEnvValues[key];
     return (_c = (_b = (_a = process.env['LLAMA_SDK_' + key]) !== null && _a !== void 0 ? _a : process.env['SDK_' + key]) !== null && _b !== void 0 ? _b : process.env[key]) !== null && _c !== void 0 ? _c : defaultValue;
 }
 exports.getEnvValue = getEnvValue;

package/build/util/index.js

@@ -87,10 +87,14 @@ async function _getLogs(params) {
         }
         catch (e) {
             (0, debugLog_1.debugLog)(`Error fetching logs for chain ${params.chain} blockSpread: ${blockSpread}. ${(0, generalUtil_1.formError)(e, { chain: params.chain, target: params.target, provider })}`);
-            if (blockSpread >= 1500) {
+            if (blockSpread >= 1001) {
                 // We got too many results
-                // We could chop it up into 1500 block spreads as that is guaranteed to always return but then we'll have to make a lot of queries (easily >1000), so instead we'll keep dividing the block spread by two until we make it
+                // We could chop it up into 1001 block spreads as that is guaranteed to always return but then we'll have to make a lot of queries (easily >1000), so instead we'll keep dividing the block spread by two until we make it
                 blockSpread = Math.floor(blockSpread / 2);
+                if (blockSpread > 510 && blockSpread < 2000)
+                    blockSpread = 999;
+                if (blockSpread > 5100 && blockSpread < 10000)
+                    blockSpread = 9999;
             }
             else {
                 const error = (0, generalUtil_1.formError)(e, { chain: params.chain, target: params.target, provider });

package/build/util/indexer.js

@@ -16,6 +16,7 @@ const logs_1 = require("./logs");
 const _1 = require(".");
 const indexerURL = (0, env_1.getEnvValue)('LLAMA_INDEXER_ENDPOINT');
 const LLAMA_INDEXER_API_KEY = (0, env_1.getEnvValue)('LLAMA_INDEXER_API_KEY');
+const addressChunkSize = +(0, env_1.getEnvValue)('LLAMA_INDEXER_ADDRESS_CHUNK_SIZE');
 const axiosIndexer = axios_1.default.create({
     headers: {
         "x-api-key": LLAMA_INDEXER_API_KEY,
@@ -139,8 +140,21 @@ async function getLogs({ chain = 'ethereum', topic, topics, fromBlock, toBlock,
         throw new Error('Chain not supported');
     if (!debugMode)
         debugMode = debugLog_1.DEBUG_LEVEL2;
-    if (((topics === null || topics === void 0 ? void 0 : topics.length) && topics.length > 1) || (extraTopics === null || extraTopics === void 0 ? void 0 : extraTopics.length))
-        throw new Error('TODO: topics and extraTopics part are not yet imeplemented');
+    let topic1;
+    let topic2;
+    let topic3;
+    if (((topics === null || topics === void 0 ? void 0 : topics.length) && topics.length > 1)) {
+        if (topics === null || topics === void 0 ? void 0 : topics.length) {
+            topic1 = topics[1];
+            topic2 = topics[2];
+            topic3 = topics[3];
+        }
+    }
+    else if (extraTopics === null || extraTopics === void 0 ? void 0 : extraTopics.length) {
+        topic1 = extraTopics[0];
+        topic2 = extraTopics[1];
+        topic3 = extraTopics[2];
+    }
     if (topics === null || topics === void 0 ? void 0 : topics.length)
         topic = topics[0];
     if (!eventAbi)
@@ -186,12 +200,15 @@ async function getLogs({ chain = 'ethereum', topic, topics, fromBlock, toBlock,
         console.time(debugTimeKey);
     }
     const addressSet = new Set(address === null || address === void 0 ? void 0 : address.split(','));
-    const addressChunks = (0, _1.sliceIntoChunks)((_c = address === null || address === void 0 ? void 0 : address.split(',')) !== null && _c !== void 0 ? _c : [], 10);
+    const addressChunks = (0, _1.sliceIntoChunks)((_c = address === null || address === void 0 ? void 0 : address.split(',')) !== null && _c !== void 0 ? _c : [], addressChunkSize);
     // to ensure that we have at least one chunk to process if no target is provided
     if (noTarget && addressChunks.length === 0)
         addressChunks.push(undefined);
     for (const chunk of addressChunks) {
-        let _logAgg = [];
+        if (Array.isArray(chunk) && chunk.length === 0) {
+            throw new Error('Address chunk cannot be empty');
+        }
+        let logCount = 0;
         do {
             const params = {
                 addresses: hasAddressFilter ? chunk.join(',') : undefined,
@@ -199,25 +216,25 @@ async function getLogs({ chain = 'ethereum', topic, topics, fromBlock, toBlock,
                 topic0: topic,
                 from_block: fromBlock,
                 to_block: toBlock,
-                limit,
-                offset,
+                topic1, topic2, topic3,
+                limit, offset,
+                noTarget,
             };
             const { data: { logs: _logs, totalCount } } = await axiosIndexer(`/logs`, { params, });
             // getLogs uses 'address' field to return log source, so we add the field here to make it compatible
             _logs.forEach((l) => {
                 l.address = l.source;
+                const iswhitelisted = !addressSet.size || addressSet.has(l.source.toLowerCase());
+                if (iswhitelisted) {
+                    logs.push(l);
+                }
             });
-            _logAgg.push(..._logs.filter((l) => {
-                if (!addressSet.size)
-                    return true;
-                return addressSet.has(l === null || l === void 0 ? void 0 : l.source.toLowerCase());
-            }));
+            logCount += _logs.length;
             offset += limit;
             // If we have all the logs, or we have reached the limit, or there are no logs, we stop
-            if (_logs.length < limit || totalCount <= _logAgg.length || _logs.length === 0)
+            if (_logs.length < limit || totalCount <= logCount || _logs.length === 0)
                 hasMore = false;
         } while (all && hasMore);
-        logs.push(..._logAgg);
     }
     if (debugMode) {
         console.timeEnd(debugTimeKey);
@@ -239,7 +256,7 @@ async function getLogs({ chain = 'ethereum', topic, topics, fromBlock, toBlock,
         log.transactionHash = log.transaction_hash;
         log.blockNumber = log.block_number;
         log.topics = [log.topic0, log.topic1, log.topic2, log.topic3,].filter(t => t !== '').map(i => ethers_1.ethers.zeroPadValue(i, 32));
-        const deleteKeys = ['chain', 'address', 'block_number', 'log_index', 'topic0', 'topic1', 'topic2', 'topic3', 'decodedArgs', 'transaction_hash',];
+        const deleteKeys = ['chain', 'block_number', 'log_index', 'topic0', 'topic1', 'topic2', 'topic3', 'decodedArgs', 'transaction_hash',];
         deleteKeys.forEach(k => delete log[k]);
         const parsedLog = iface === null || iface === void 0 ? void 0 : iface.parseLog({ data: log.data, topics: log.topics, });
         log.args = parsedLog === null || parsedLog === void 0 ? void 0 : parsedLog.args;

package/build/util/updateProviderList.js

@@ -97,7 +97,7 @@ function filterRPCs(rpc) {
         if (i.includes('$'))
             return false; // remove anything where api key is injected
         // reject websocket, http, testnet, devnet, and anything with '='
-        if (/(wss\:|ws\:|http\:|test|devnet|terminet\.io|huobichain\.com|getblock\.io|bitstack\.com|nodereal.io\/v1|pokt\.network|ankr\.com|\=)/.test(i))
+        if (/(wss\:|ws\:|http\:|test|devnet|terminet\.io|huobichain\.com|getblock\.io|bitstack\.com|nodereal.io\/v1|pokt\.network|ankr\.com|histori\.xyz|owlracle\.info\=)/.test(i))
             return false; // remove anything with blacklisted words
         return true;
     }).map((i) => {

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.136",
+  "version": "5.0.139",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {
     "start": "npm run build && node build/index.js",
     "build": "tsc",
     "watch": "tsc --watch",
-    "test": "LLAMA_DEBUG_MODE='' SDK_DEBUG='' jest",
+    "load-providers": "wget https://unpkg.com/@defillama/sdk@latest/build/providers.json -O build/providers.json",
+    "test": "npm run load-providers && TEST_MODE=true LLAMA_DEBUG_MODE='' SDK_DEBUG='' LLAMA_DEBUG_LEVEL2='' jest",
     "update-providers": "ts-node src/util/updateProviderList.ts",
     "remove-rpcs": "ts-node src/util/removeRedundantRPCs.ts",
     "format": "prettier --write \"src/**/*.ts\"",