@defillama/sdk 5.0.14 → 5.0.18

package/build/types.d.ts

@@ -1,4 +1,3 @@
-import { Chain } from "./general";
 export type Address = string;
 export type StringNumber = string;
 export type Block = number | string;
@@ -20,6 +19,7 @@ export type CallOptions = {
     skipCache?: boolean;
     logArray?: LogArray;
 };
+export type Chain = string;
 export type MulticallOptions = {
     abi: string | any;
     calls: CallsParams[] | (string | number)[];

package/build/util/LlamaProvider.d.ts

@@ -0,0 +1,34 @@
+import { AbstractProvider, AddressLike, Block, BlockTag, FallbackProvider, Filter, FilterByBlockHash, Log, TransactionRequest } from "ethers";
+import { Chain } from "../types";
+type ProviderWithURL = {
+    url: string;
+    provider: AbstractProvider;
+};
+export declare class LlamaProvider extends FallbackProvider {
+    isCustomLlamaProvider: boolean;
+    rpcs: ProviderWithURL[];
+    archivalRPCs: ProviderWithURL[];
+    _isReady: Promise<void>;
+    chainName: string;
+    chainId: number;
+    constructor({ providers, chainId, chainName, archivalRPCs, }: {
+        providers: ProviderWithURL[];
+        archivalRPCs: ProviderWithURL[];
+        chainId: number;
+        chainName: string;
+    }, fallbackConfig?: any);
+    getNetwork(): any;
+    _removeProvider(rpc: string): void;
+    _ready(): Promise<void>;
+    call(_tx: TransactionRequest): Promise<string>;
+    getBalance(address: AddressLike, blockTag?: BlockTag): Promise<bigint>;
+    getBlock(block: BlockTag, prefetchTxs?: boolean | undefined): Promise<Block | null>;
+    getLogs(_filter: Filter | FilterByBlockHash): Promise<Log[]>;
+    _performAction(method: string, params: any, attempts?: number): Promise<any>;
+}
+export declare const providers: {
+    [chain: string]: AbstractProvider;
+};
+export declare function setProvider(chain: Chain, provider: AbstractProvider): void;
+export declare function getProvider(chain?: Chain, _getArchivalNode?: boolean): AbstractProvider;
+export {};

package/build/util/LlamaProvider.js

@@ -0,0 +1,283 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getProvider = exports.setProvider = exports.providers = exports.LlamaProvider = void 0;
+const ethers_1 = require("ethers");
+const providers_json_1 = __importDefault(require("../providers.json"));
+const env_1 = require("./env");
+const debugLog_1 = require("./debugLog");
+const axios_1 = __importDefault(require("axios"));
+class LlamaProvider extends ethers_1.FallbackProvider {
+    constructor({ providers, chainId, chainName, archivalRPCs, }, fallbackConfig) {
+        const networkish = { name: chainName, chainId };
+        super(providers.map(i => i.provider), networkish, fallbackConfig);
+        this.isCustomLlamaProvider = true;
+        this.rpcs = providers;
+        this.archivalRPCs = archivalRPCs;
+        this.chainName = chainName;
+        this.chainId = chainId;
+        this._isReady = this._ready();
+    }
+    getNetwork() {
+        return { chainId: this.chainId, name: this.chainName };
+    }
+    _removeProvider(rpc) {
+        this.rpcs = this.rpcs.filter(i => i.url !== rpc);
+        this.archivalRPCs = this.archivalRPCs.filter(i => i.url !== rpc);
+    }
+    async _ready() {
+        const outOfSyncLimit = 1000;
+        const _this = this;
+        const currentBlocks = {};
+        await Promise.all(this.rpcs.map(checkRPCWorks));
+        if (this.archivalRPCs.length) {
+            await Promise.all(this.archivalRPCs.map(checkRPCWorks));
+        }
+        // we are doing this because I noticed that one of the rpcs can go rouge and return very high block number
+        const medianBlockValue = getMedianBlockValue(Object.values(currentBlocks));
+        Object.entries(currentBlocks).forEach(([url, block]) => {
+            if (medianBlockValue - block > outOfSyncLimit) {
+                (0, debugLog_1.debugLog)(`RPC ${url} is out of sync by ${medianBlockValue - block} blocks`);
+                _this.rpcs = _this.rpcs.filter(i => i.url !== url);
+                _this.archivalRPCs = _this.archivalRPCs.filter(i => i.url !== url);
+            }
+        });
+        async function checkRPCWorks({ url, provider }) {
+            try {
+                const block = await httpRPC.getBlockNumber(url);
+                currentBlocks[url] = block;
+            }
+            catch (e) {
+                // debugLog(`${_this.chainName} skipping RPC ${url} is not working, error: ${(e as any).message}`)
+                _this._removeProvider(url);
+            }
+        }
+    }
+    async call(_tx) {
+        var _a;
+        return this._performAction('call', [{ to: _tx.to, data: _tx.data }, (_a = _tx.blockTag) !== null && _a !== void 0 ? _a : 'latest']);
+    }
+    async getBalance(address, blockTag = 'latest') {
+        return this._performAction('getBalance', [address, blockTag]);
+    }
+    async getBlock(block, prefetchTxs) {
+        return this._performAction('getBlock', [block, false]);
+    }
+    async getLogs(_filter) {
+        return this._performAction('getLogs', [_filter], 3);
+    }
+    async _performAction(method, params, attempts = 7) {
+        await this._isReady;
+        let runners = [...this.rpcs];
+        if (method === 'getLogs') {
+            runners = [...this.archivalRPCs];
+            this.rpcs.filter(i => !runners.map(i => i.url).includes(i.url)).forEach(i => runners.push(i)); // ensure that there are no duplicates
+            runners = runners.filter(i => !i.url.includes('llamarpc.com'));
+        }
+        let primaryRunner = runners[0];
+        if (!primaryRunner)
+            throw new Error('No RPCs available for ' + this.chainName);
+        let errors = [];
+        runners = runners.slice(1).sort(() => Math.random() - 0.5); // randomize order of runners
+        const isArchivalRequest = ['call', 'getBalance'].includes(method) && params[1] !== 'latest';
+        let noPlayingAround = primaryRunner.url.includes('llama.fi') || method === 'getLogs' || isArchivalRequest;
+        if (noPlayingAround) {
+            // if primary runner is llama.fi or alchemy, then try it first
+            runners.unshift(primaryRunner);
+        }
+        else {
+            // add it as second runner
+            runners.splice(1, 0, primaryRunner);
+        }
+        runners = runners.slice(0, attempts);
+        for (const runner of runners) {
+            try {
+                const result = await httpRPC[method](runner.url, params);
+                return result;
+            }
+            catch (e) {
+                // console.log('failed', runner.url, (e as any).message)
+                errors.push({ host: runner.url, error: e });
+            }
+        }
+        throw { llamaRPCError: true, errors };
+    }
+}
+exports.LlamaProvider = LlamaProvider;
+function createProvider(name, rpcString, chainId = 400069, archivalRPCList = []) {
+    chainId = (0, env_1.getChainId)(name, chainId);
+    const networkish = { name, chainId };
+    const rpcList = rpcString.split(',');
+    if ((0, env_1.getEnvValue)('HISTORICAL')) {
+        if (chainId === 1) {
+            (0, debugLog_1.debugLog)("RPC providers set to historical, only the first RPC provider will be used");
+        }
+        return getProviderObject(rpcList[0], name);
+    }
+    else {
+        try {
+            const fallbackConfig = { cacheTimeout: 5 * 1000, quorum: 1, eventQuorum: 1, };
+            const getProviderWithURL = (url) => ({ url, provider: getProviderObject(url, name) });
+            const providers = rpcList.map(getProviderWithURL);
+            const archivalRPCs = archivalRPCList.map(getProviderWithURL);
+            return new LlamaProvider({ providers, archivalRPCs, chainId, chainName: name, }, fallbackConfig);
+            // return new FallbackProvider(providers, networkish, fallbackConfig)
+        }
+        catch (e) {
+            // debugLog(`Error creating provider for ${name} with RPCs: ${rpcList.join(', ')}`)
+            // we dont throw errors for chains not present in providers.json, these can be non-evm chains like solana
+            if (providers_json_1.default[name])
+                throw e;
+            return null;
+        }
+    }
+    function getProviderObject(url, chain) {
+        /**
+         *  Options for configuring a [[JsonRpcApiProvider]]. Much of this
+         *  is targetted towards sub-classes, which often will not expose
+         *  any of these options to their consumers.
+         *
+         *  **``polling``** - use the polling strategy is used immediately
+         *  for events; otherwise, attempt to use filters and fall back onto
+         *  polling (default: ``false``)
+         *
+         *  **``staticNetwork``** - do not request chain ID on requests to
+         *  validate the underlying chain has not changed (default: ``null``)
+         *
+         *  This should **ONLY** be used if it is **certain** that the network
+         *  cannot change, such as when using INFURA (since the URL dictates the
+         *  network). If the network is assumed static and it does change, this
+         *  can have tragic consequences. For example, this **CANNOT** be used
+         *  with MetaMask, since the used can select a new network from the
+         *  drop-down at any time.
+         *
+         *  **``batchStallTime``** - how long (ms) to aggregate requests into a
+         *  single batch. ``0`` indicates batching will only encompass the current
+         *  event loop. If ``batchMaxCount = 1``, this is ignored. (default: ``10``)
+         *
+         *  **``batchMaxSize``** - target maximum size (bytes) to allow per batch
+         *  request (default: 1Mb)
+         *
+         *  **``batchMaxCount``** - maximum number of requests to allow in a batch.
+         *  If ``batchMaxCount = 1``, then batching is disabled. (default: ``100``)
+         *
+         *  **``cacheTimeout``** - passed as [[AbstractProviderOptions]].
+          *  **``cacheTimeout``** - how long to cache a low-level ``_perform``
+          *  for, based on input parameters. This reduces the number of calls
+          *  to getChainId and getBlockNumber, but may break test chains which
+          *  can perform operations (internally) synchronously. Use ``-1`` to
+          *  disable, ``0`` will only buffer within the same event loop and
+          *  any other value is in ms. (default: ``250``)
+         */
+        // const batchMaxSize = 10 * (1024 * 1024) // 10Mb
+        // some rpcs throw error if batchMaxCount is set higher than 100
+        const batchMaxCount = (0, env_1.getBatchMaxCount)(chain);
+        const jsonRpcApiProviderOptions = { staticNetwork: true, batchMaxCount: 1, };
+        if (url.startsWith('wss://')) {
+            delete jsonRpcApiProviderOptions.batchMaxCount;
+            return new ethers_1.WebSocketProvider(url, networkish, jsonRpcApiProviderOptions);
+        }
+        return new ethers_1.JsonRpcProvider(url, networkish, jsonRpcApiProviderOptions);
+    }
+}
+exports.providers = {};
+function setProvider(chain, provider) {
+    exports.providers[chain] = provider;
+}
+exports.setProvider = setProvider;
+function getProvider(chain = "ethereum", _getArchivalNode = false) {
+    var _a, _b;
+    if (exports.providers[chain])
+        return exports.providers[chain];
+    // use RPC from env variable if set else use RPC from providers.json
+    let rpcList = (0, env_1.getChainRPCs)(chain, (_a = providers_json_1.default[chain]) === null || _a === void 0 ? void 0 : _a.rpc);
+    let archivalRPCList = (0, env_1.getArchivalRPCs)(chain);
+    if (!rpcList) {
+        // @ts-ignore (throwing error here would alter function behavior and have side effects)
+        return null;
+    }
+    if (!exports.providers[chain])
+        exports.providers[chain] = createProvider(chain, rpcList, (_b = providers_json_1.default[chain]) === null || _b === void 0 ? void 0 : _b.chainId, archivalRPCList);
+    return exports.providers[chain];
+}
+exports.getProvider = getProvider;
+function toHex(n) {
+    if (typeof n === 'string')
+        return n;
+    return '0x' + n.toString(16);
+}
+const httpRPC = {
+    getBlockNumber: async (rpc) => {
+        const { data } = await axios_1.default.post(rpc, {
+            jsonrpc: '2.0', id: 1, params: [],
+            method: 'eth_blockNumber',
+        }, {
+            timeout: 3000
+        });
+        if (data.error)
+            throw data.error;
+        return parseInt(data.result);
+    },
+    getBlock: async (rpc, params) => {
+        params[0] = toHex(params[0]);
+        const { data: { result, error } } = await axios_1.default.post(rpc, {
+            jsonrpc: '2.0', id: 1, params,
+            method: 'eth_getBlockByNumber',
+        }, {
+            timeout: 3000
+        });
+        if (error)
+            throw error;
+        result.number = parseInt(result.number);
+        result.timestamp = parseInt(result.timestamp);
+        return result;
+    },
+    getBalance: async (rpc, params) => {
+        params[1] = toHex(params[1]);
+        const { data: { result, error } } = await axios_1.default.post(rpc, {
+            jsonrpc: '2.0', id: 1, params,
+            method: 'eth_getBalance',
+        }, {
+            timeout: 5000
+        });
+        if (error)
+            throw error;
+        return BigInt(result).toString();
+    },
+    call: async (rpc, params) => {
+        params[1] = toHex(params[1]);
+        const { data } = await axios_1.default.post(rpc, {
+            jsonrpc: '2.0', id: 1, params,
+            method: 'eth_call',
+        }, {
+            timeout: 5000
+        });
+        if (data.error)
+            throw data.error;
+        return data.result;
+    },
+    getLogs: async (rpc, params) => {
+        params[0].fromBlock = toHex(params[0].fromBlock);
+        params[0].toBlock = toHex(params[0].toBlock);
+        const { data: { result, error } } = await axios_1.default.post(rpc, {
+            jsonrpc: '2.0', id: 1, params,
+            method: 'eth_getLogs',
+        });
+        if (error)
+            throw error;
+        result.forEach((i) => {
+            i.blockNumber = parseInt(i.blockNumber);
+            i.transactionIndex = parseInt(i.transactionIndex);
+            i.logIndex = parseInt(i.logIndex);
+            i.index = i.logIndex;
+        });
+        return result;
+    },
+};
+function getMedianBlockValue(blocks) {
+    blocks.sort((a, b) => a - b);
+    const mid = Math.floor(blocks.length / 3);
+    return blocks.length % 2 !== 0 ? blocks[mid] : (blocks[mid - 1] + blocks[mid]) / 2;
+}

package/build/util/env.js

@@ -32,8 +32,10 @@ function getMaxParallelRequests(chain) {
 }
 exports.getMaxParallelRequests = getMaxParallelRequests;
 function getParallelGetLogsLimit(chain, providerUrl) {
-    const defaultLimit = getEnvValue('GET_LOGS_CONCURRENCY_LIMIT', '5');
-    const defaultAlchemyLimit = getEnvValue('ALCHEMY_GET_LOGS_CONCURRENCY_LIMIT', '4');
+    let defaultLimit = getEnvValue('GET_LOGS_CONCURRENCY_LIMIT', '25');
+    if (chain === 'fantom')
+        defaultLimit = '10';
+    const defaultAlchemyLimit = getEnvValue('ALCHEMY_GET_LOGS_CONCURRENCY_LIMIT', '25');
     if (providerUrl && providerUrl.includes('alchemy.com'))
         return +(defaultAlchemyLimit);
     return +(getEnvValue(`${chain}_RPC_GET_LOGS_CONCURRENCY_LIMIT`, defaultLimit));
@@ -76,9 +78,8 @@ function getBatchMaxCount(chain) {
     if (getEnvValue(key))
         return +getEnvValue(key);
     switch (chain) {
-        case 'cronos': return 5;
         case 'zkfair': return 1;
-        default: return +getEnvValue('BATCH_MAX_COUNT', '99');
+        default: return +getEnvValue('BATCH_MAX_COUNT', '5');
     }
 }
 exports.getBatchMaxCount = getBatchMaxCount;
@@ -91,10 +92,13 @@ function getArchivalRPCs(chain) {
 exports.getArchivalRPCs = getArchivalRPCs;
 function getChainRPCs(chain, defaultList = []) {
     const key = chain + '_RPC';
-    if (getEnvValue(key))
-        return getEnvValue(key);
-    if (defaultList.length === 0)
-        return undefined;
-    return defaultList.join(',');
+    const envValue = getEnvValue(key);
+    if (defaultList.length) {
+        const listString = defaultList.join(',');
+        if (envValue)
+            return envValue + ',' + listString;
+        return listString;
+    }
+    return envValue || undefined;
 }
 exports.getChainRPCs = getChainRPCs;

package/build/util/updateProviderList.js

@@ -6,26 +6,74 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const providers_json_1 = __importDefault(require("../providers.json"));
 const fs_1 = __importDefault(require("fs"));
 const axios_1 = __importDefault(require("axios"));
+const debugLog_1 = require("./debugLog");
+const providerList = providers_json_1.default;
 async function main() {
-    let { data: chainData } = await (0, axios_1.default)('https://chainid.network/chains.json');
-    const existingChainIds = new Set(Object.values(providers_json_1.default).map(i => i.chainId));
-    const existingChainNames = new Set(Object.keys(providers_json_1.default).map(i => i.toLowerCase()));
+    let { data: chainData } = await (0, axios_1.default)('https://chainlist.org/rpcs.json');
+    const providerIDMap = {};
+    Object.values(providerList).forEach((i) => providerIDMap[i.chainId] = i.rpc);
     chainData = chainData
         .filter((i) => i.rpc.length)
         .filter((i) => !i.status || i.status === 'active')
-        .filter((i) => i.shortName)
-        .filter((i) => !existingChainIds.has(i.chainId))
-        .filter((i) => !existingChainNames.has(i.shortName.toLowerCase()));
-    chainData.forEach((i) => {
-        i.rpc = i.rpc.filter((j) => !/wss:/.test(j));
+        .filter((i) => i.shortName);
+    for (const i of chainData) {
+        i.rpc = await filterForWorkingRPCs(i.rpc.map((j) => j.url), i.name, i.chainId);
+        if (!i.rpc.length)
+            continue;
+        if (providerIDMap[i.chainId]) {
+            providerIDMap[i.chainId].push(...i.rpc);
+        }
+        else if (providerList[i.shortName.toLowerCase()]) {
+            (0, debugLog_1.debugLog)(`Duplicate short name ${i.chainId} for ${i.shortName}, doing nothing`);
+        }
+        else {
+            providerList[i.shortName.toLowerCase()] = {
+                rpc: i.rpc,
+                chainId: i.chainId
+            };
+        }
+    }
+    Object.values(providerList).forEach((i) => {
+        i.rpc = Array.from(new Set(filterRPCs(i.rpc)));
     });
-    const newList = { ...providers_json_1.default };
-    chainData.forEach((i) => {
-        newList[i.shortName.toLowerCase()] = {
-            rpc: i.rpc,
-            chainId: i.chainId
-        };
+    fs_1.default.writeFileSync(__dirname + '/../providers.json', JSON.stringify(providerList));
+}
+function filterRPCs(rpc) {
+    return rpc.filter((i) => {
+        if (i.endsWith('/demo'))
+            return false; // remove demo rpc
+        if (i.includes('$'))
+            return false; // remove anything where api key is injected
+        if (i.startsWith('wss://') || i.startsWith('ws://'))
+            return false; // remove websocket rpcs
+        return true;
+    }).map((i) => {
+        if (i.endsWith('/'))
+            return i.slice(0, -1);
+        return i;
     });
-    fs_1.default.writeFileSync(__dirname + '/../providers.json', JSON.stringify(newList));
+}
+async function filterForWorkingRPCs(rpc, chain, chainId) {
+    if (rpc.length < 10)
+        return rpc;
+    rpc = filterRPCs(rpc);
+    const promises = await Promise.all(rpc.map(async (i) => {
+        try {
+            const { data } = await axios_1.default.post(i, {
+                jsonrpc: '2.0',
+                method: 'eth_blockNumber',
+                params: [],
+                id: 1
+            }, {
+                timeout: 3000
+            });
+            if (data.result)
+                return i;
+        }
+        catch (e) {
+            // console.log((e as any).message, i, chain)
+        }
+    }));
+    return rpc.filter((_i, index) => promises[index]);
 }
 main();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.14",
+  "version": "5.0.18",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {