@defillama/sdk 5.0.83 → 5.0.85

package/build/util/blockscout.js

@@ -7,17 +7,16 @@ exports.getBlockscoutBlock = exports.blockscoutEndpoints = void 0;
 const axios_1 = __importDefault(require("axios"));
 exports.blockscoutEndpoints = {
     celo: 'https://explorer.celo.org',
-    kava: 'https://explorer.kava.io',
     onus: 'https://explorer.onuschain.io',
     base: 'https://base.blockscout.com',
-    scroll: 'https://blockscout.scroll.io',
+    // scroll: 'https://blockscout.scroll.io',
     canto: 'https://explorer.plexnode.wtf',
 };
 async function getBlockscoutBlock(timestamp, chain) {
     const api = `${exports.blockscoutEndpoints[chain]}/api?module=block&action=getblocknobytime&timestamp=${timestamp}&closest=before`;
     const { data } = await (0, axios_1.default)(api);
     if (data.status !== '1')
-        throw new Error(data.message);
+        throw new Error('Issue fetching block from blockscout: ' + api);
     return {
         timestamp,
         block: +data.result.blockNumber,

package/build/util/debugLog.d.ts

@@ -1,3 +1,4 @@
 export declare const DEBUG_ENABLED: boolean;
+export declare const DEBUG_LEVEL2: boolean;
 export declare function debugLog(...args: any): void;
 export declare function debugTable(...args: any): void;

package/build/util/debugLog.js

@@ -1,16 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.debugTable = exports.debugLog = exports.DEBUG_ENABLED = void 0;
+exports.debugTable = exports.debugLog = exports.DEBUG_LEVEL2 = exports.DEBUG_ENABLED = void 0;
 const env_1 = require("./env");
 exports.DEBUG_ENABLED = env_1.ENV_CONSTANTS.DEBUG_ENABLED === "true";
+exports.DEBUG_LEVEL2 = env_1.ENV_CONSTANTS.DEBUG_LEVEL2 === "true";
 function debugLog(...args) {
-    if (!exports.DEBUG_ENABLED)
+    if (!exports.DEBUG_ENABLED && !exports.DEBUG_LEVEL2)
         return;
     console.log(...args);
 }
 exports.debugLog = debugLog;
 function debugTable(...args) {
-    if (!exports.DEBUG_ENABLED)
+    if (!exports.DEBUG_ENABLED && !exports.DEBUG_LEVEL2)
         return;
     console.table(...args);
 }

package/build/util/env.js

@@ -3,10 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getChainRPCs = exports.getArchivalRPCs = exports.getBatchMaxCount = exports.getDefaultChunkSize = exports.getEnvMulticallAddress = exports.getEnvCacheFolder = exports.getEnvValue = exports.getEnvRPC = exports.getParallelGetBlocksLimit = exports.getParallelGetLogsLimit = exports.getMaxParallelRequests = exports.getChainId = exports.ENV_CONSTANTS = void 0;
 const _ENV_CONSTANTS = {
     DEBUG_ENABLED: getEnvValue('DEBUG') === "true" || process.env.LLAMA_DEBUG_MODE ? 'true' : 'false',
+    DEBUG_LEVEL2: process.env.LLAMA_DEBUG_LEVEL2 ? 'true' : 'false',
 };
 const whitelistedEnvConstants = [
     'R2_ENDPOINT', 'R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY', 'TRON_PRO_API_KEY', 'COINS_API_KEY',
-    'ELASTICSEARCH_CONFIG', 'GRAPH_API_KEY'
+    'ELASTICSEARCH_CONFIG', 'GRAPH_API_KEY', 'LLAMA_INDEXER_ENDPOINT',
 ];
 const defaultEnvValues = {
     INTERNAL_SDK_CACHE_FILE: './cache.json',

package/build/util/graph.js

@@ -34,6 +34,7 @@ async function request(endpoint, query, { cache = false, cacheKey, variables, wi
     }
 }
 exports.request = request;
+let graphKeyWarned = false;
 function modifyEndpoint(endpoint, network = 'arbitrum') {
     // example: https://api.thegraph.com/subgraphs/name/yieldyak/reinvest-tracker
     let graphKey = (0, env_1.getEnvValue)('GRAPH_API_KEY');
@@ -43,7 +44,9 @@ function modifyEndpoint(endpoint, network = 'arbitrum') {
         return endpoint;
     else if (!graphKey) {
         graphKey = "5a1340b49fa9efe00" + "21452daa260564e";
-        console.log("GRAPH_API_KEY env variable is not set, using the default api key");
+        if (!graphKeyWarned)
+            console.log("GRAPH_API_KEY env variable is not set, using the default api key");
+        graphKeyWarned = true;
     }
     endpoint = endpoint.replace('[api-key]', graphKey);
     return endpoint;

package/build/util/indexer.d.ts

@@ -0,0 +1,39 @@
+import { Address } from "../types";
+export declare const supportedChainSet: Set<unknown>;
+declare enum TokenTypes {
+    ERC20 = "erc20",
+    ERC721 = "erc721"
+}
+export declare function getTokens(address: string | string[], { onlyWhitelisted, skipCacheRead, skipCache, chain, tokenType, }?: {
+    onlyWhitelisted?: boolean;
+    skipCacheRead?: boolean;
+    skipCache?: boolean;
+    chain?: string;
+    tokenType?: TokenTypes;
+}): Promise<{
+    [chain: string]: string[];
+}>;
+export type IndexerGetLogsOptions = {
+    target?: Address;
+    topic?: string;
+    fromBlock?: number;
+    toBlock?: number;
+    topics?: (string | null)[];
+    extraTopics?: string[];
+    chain?: string;
+    eventAbi?: string | any;
+    fromTimestamp?: number;
+    toTimestamp?: number;
+    entireLog?: boolean;
+    cacheInCloud?: boolean;
+    onlyArgs?: boolean;
+    targets?: Address[];
+    flatten?: boolean;
+    all?: boolean;
+    limit?: number;
+    offset?: number;
+    debugMode?: boolean;
+};
+export declare function getLogs({ chain, topic, topics, fromBlock, toBlock, all, limit, offset, target, targets, eventAbi, entireLog, flatten, extraTopics, fromTimestamp, toTimestamp, debugMode }: IndexerGetLogsOptions): Promise<any[]>;
+export declare function isIndexerEnabled(chain?: string): boolean;
+export {};

package/build/util/indexer.js

@@ -0,0 +1,230 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isIndexerEnabled = exports.getLogs = exports.getTokens = exports.supportedChainSet = void 0;
+const axios_1 = __importDefault(require("axios"));
+const env_1 = require("./env");
+const cache_1 = require("./cache");
+const debugLog_1 = require("./debugLog");
+const generalUtil_1 = require("../generalUtil");
+const general_1 = require("../general");
+const ethers_1 = require("ethers");
+const logs_1 = require("./logs");
+const blocks_1 = require("./blocks");
+const indexerURL = (0, env_1.getEnvValue)('LLAMA_INDEXER_ENDPOINT');
+const state = { chainIndexStatus: {} };
+const cacheTime = 1 * 60 * 1000; // 1 minutes - we cache sync status for 1 minute
+async function getChainIndexStatus() {
+    if (!indexerURL)
+        throw new Error('Llama Indexer URL not set');
+    if (state.timestamp && (Date.now() - state.timestamp) < cacheTime)
+        return state.chainIndexStatus;
+    state.timestamp = Date.now();
+    state.chainIndexStatus = _getState();
+    return state.chainIndexStatus;
+    async function _getState() {
+        const { data: { syncStatus } } = await axios_1.default.get(`${indexerURL}/syncStatus`);
+        const syncInfo = {};
+        syncStatus.forEach((d) => {
+            const chain = indexerChainIdChainMapping[d.chain];
+            syncInfo[chain] = {
+                block: d.lastIndexedBlock,
+                timestamp: +new Date(d.lastIndexedDate)
+            };
+        });
+        state.chainIndexStatus = syncInfo;
+        return syncInfo;
+    }
+}
+const indexerChainIdChainMapping = {
+    1: 'ethereum',
+    10: 'optimism',
+    56: 'bsc',
+    100: 'xdai',
+    137: 'polygon',
+    204: 'op_bnb',
+    250: 'fantom',
+    324: 'era',
+    1101: 'polygon_zkevm',
+    8453: 'base',
+    34443: 'mode',
+    42170: 'arbitrum_nova',
+    42161: 'arbitrum',
+    43114: 'avalanche',
+    59144: 'linea',
+    81457: 'blast',
+    534352: 'scroll',
+};
+exports.supportedChainSet = new Set(Object.values(indexerChainIdChainMapping));
+const chainToIDMapping = {};
+Object.entries(indexerChainIdChainMapping).forEach(([id, chain]) => {
+    chainToIDMapping[chain] = id;
+});
+var TokenTypes;
+(function (TokenTypes) {
+    TokenTypes["ERC20"] = "erc20";
+    TokenTypes["ERC721"] = "erc721";
+})(TokenTypes || (TokenTypes = {}));
+async function getTokens(address, { onlyWhitelisted = true, skipCacheRead = false, skipCache = false, chain, tokenType, } = {}) {
+    var _a, _b;
+    if (!indexerURL)
+        throw new Error('Llama Indexer URL not set');
+    if (!address)
+        throw new Error('Address is required either as a string or an array of strings');
+    if (Array.isArray(address) && !address.length)
+        throw new Error('Address array cannot be empty');
+    if (Array.isArray(address))
+        address = address.join(',');
+    address = address.toLowerCase();
+    const project = 'llama-indexer-cache';
+    const key = onlyWhitelisted ? address : `${address}/all`;
+    const file = `${project}/${key}`;
+    const timeNow = Math.floor(Date.now() / 1e3);
+    const THREE_DAYS = 3 * 24 * 3600;
+    let cache = {};
+    if (!skipCacheRead && !skipCache) {
+        cache = (_a = (await (0, cache_1.readCache)(file))) !== null && _a !== void 0 ? _a : {};
+        if (cache.timestamp && (timeNow - cache.timestamp) < THREE_DAYS)
+            return cache.tokens;
+    }
+    (0, debugLog_1.debugLog)('[Indexer] Pulling tokens for ' + address);
+    const tokens = (_b = cache.tokens) !== null && _b !== void 0 ? _b : {};
+    const { data: { balances } } = await axios_1.default.get(`${indexerURL}/balances`, {
+        params: {
+            addresses: address,
+            chain: chain,
+            type: tokenType,
+        }
+    });
+    balances.filter((b) => +b.total_amount > 0).forEach((b) => {
+        const chain = indexerChainIdChainMapping[b.chain];
+        if (!chain) {
+            return;
+        }
+        if (!tokens[chain])
+            tokens[chain] = [];
+        tokens[chain].push(b.address);
+    });
+    const tokenCache = { timestamp: timeNow, tokens, };
+    Object.entries(tokens).forEach(([chain, values]) => {
+        values.push(general_1.ETHER_ADDRESS);
+        tokens[chain] = (0, generalUtil_1.getUniqueAddresses)(values);
+    });
+    if (!skipCache)
+        await (0, cache_1.writeCache)(file, tokenCache);
+    return tokens;
+}
+exports.getTokens = getTokens;
+async function getLogs({ chain = 'ethereum', topic, topics, fromBlock, toBlock, all = true, limit = 1000, offset = 0, target, targets, eventAbi, entireLog = false, flatten = true, extraTopics, fromTimestamp, toTimestamp, debugMode = false }) {
+    var _a, _b;
+    if (!indexerURL)
+        throw new Error('Llama Indexer URL not set');
+    const chainId = chainToIDMapping[chain];
+    if (!chainId)
+        throw new Error('Chain not supported');
+    if (!debugMode)
+        debugMode = debugLog_1.DEBUG_LEVEL2;
+    if ((topics === null || topics === void 0 ? void 0 : topics.length) || (extraTopics === null || extraTopics === void 0 ? void 0 : extraTopics.length))
+        throw new Error('TODO: topics and extraTopics part are not yet imeplemented');
+    // if (!target && !targets?.length) throw new Error('target|targets is required')
+    if (!fromBlock && !fromTimestamp)
+        throw new Error('fromBlock or fromTimestamp is required');
+    if (!toBlock && !toTimestamp)
+        throw new Error('toBlock or toTimestamp is required');
+    if (!fromBlock)
+        fromBlock = await (0, blocks_1.getBlockNumber)(chain, fromTimestamp);
+    if (!toBlock)
+        toBlock = await (0, blocks_1.getBlockNumber)(chain, toTimestamp);
+    if (!fromBlock || !toBlock)
+        throw new Error('fromBlock and toBlock must be > 0');
+    let address = target;
+    if (typeof target === 'string')
+        targets = [target];
+    if (Array.isArray(targets))
+        address = targets.join(',');
+    if (address)
+        address = address.toLowerCase();
+    const chainIndexStatus = await getChainIndexStatus();
+    const lastIndexedBlock = (_b = (_a = chainIndexStatus[chain]) === null || _a === void 0 ? void 0 : _a.block) !== null && _b !== void 0 ? _b : 0;
+    if (lastIndexedBlock < toBlock)
+        throw new Error(`Indexer not up to date for ${chain}. Last indexed block: ${lastIndexedBlock}, requested block: ${toBlock}`);
+    // Create an Interface object
+    let iface;
+    if (eventAbi)
+        iface = new ethers_1.Interface([eventAbi]);
+    if (topic)
+        topic = (0, logs_1.toFilterTopic)(topic);
+    else if (eventAbi)
+        topic = (0, logs_1.toFilterTopic)(iface);
+    let hasMore = true;
+    let logs = [];
+    const debugTimeKey = `Indexer-getLogs-${chain}-${topic}-${address}_${Math.random()}`;
+    if (debugMode) {
+        (0, debugLog_1.debugLog)('[Indexer] Pulling logs ' + debugTimeKey);
+        console.time(debugTimeKey);
+    }
+    do {
+        const params = {
+            address,
+            chain: chainId,
+            topic0: topic,
+            from_block: fromBlock,
+            to_block: toBlock,
+            limit,
+            offset,
+        };
+        const { data: { logs: _logs, totalCount } } = await axios_1.default.get(`${indexerURL}/logs`, { params });
+        logs.push(..._logs);
+        offset += limit;
+        // If we have all the logs, or we have reached the limit, or there are no logs, we stop
+        if (_logs.length < limit || totalCount <= logs.length || _logs.length === 0)
+            hasMore = false;
+    } while (all && hasMore);
+    if (debugMode) {
+        console.timeEnd(debugTimeKey);
+        (0, debugLog_1.debugLog)('Logs pulled ' + chain, address, logs.length);
+    }
+    const mappedLogs = [];
+    let addressIndexMap = {};
+    const splitByAddress = (targets === null || targets === void 0 ? void 0 : targets.length) && !flatten;
+    if (splitByAddress) {
+        for (let i = 0; i < targets.length; i++) {
+            addressIndexMap[targets[i].toLowerCase()] = i;
+            mappedLogs.push([]);
+        }
+    }
+    logs = logs.map((log) => {
+        log.logIndex = log.log_index;
+        log.index = log.log_index;
+        log.transactionHash = log.transaction_hash;
+        log.blockNumber = log.block_number;
+        log.topics = [log.topic0, log.topic1, log.topic2, log.topic3,].filter(t => t !== '').map(i => ethers_1.ethers.zeroPadValue(i, 32));
+        const deleteKeys = ['chain', 'address', 'block_number', 'log_index', 'topic0', 'topic1', 'topic2', 'topic3', 'decodedArgs', 'transaction_hash',];
+        deleteKeys.forEach(k => delete log[k]);
+        const parsedLog = iface === null || iface === void 0 ? void 0 : iface.parseLog({
+            data: log.data,
+            topics: log.topics,
+        });
+        log.args = parsedLog === null || parsedLog === void 0 ? void 0 : parsedLog.args;
+        log = !entireLog ? log.args : log;
+        if (splitByAddress) {
+            const index = addressIndexMap[log.source.toLowerCase()];
+            mappedLogs[index].push(log);
+        }
+        return log;
+    });
+    if (splitByAddress)
+        return mappedLogs;
+    return logs;
+}
+exports.getLogs = getLogs;
+function isIndexerEnabled(chain) {
+    if (!indexerURL)
+        return false;
+    if (chain && !exports.supportedChainSet.has(chain))
+        return false;
+    return true;
+}
+exports.isIndexerEnabled = isIndexerEnabled;

package/build/util/logs.d.ts

@@ -1,4 +1,4 @@
-import { EventLog } from "ethers";
+import { EventLog, Interface } from "ethers";
 import { Address } from "../types";
 export type GetLogsOptions = {
     target?: Address;
@@ -19,8 +19,12 @@ export type GetLogsOptions = {
     onlyArgs?: boolean;
     targets?: Address[];
     flatten?: boolean;
+    skipIndexer?: boolean;
+    onlyIndexer?: boolean;
+    debugMode?: boolean;
 };
 export declare function getLogs(options: GetLogsOptions): Promise<EventLog[] | EventLog[][] | any[]>;
+export declare function toFilterTopic(topic: string | Interface): string;
 export type logCache = {
     logs: EventLog[];
     metadata: {

package/build/util/logs.js

@@ -1,17 +1,35 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getLogs = void 0;
+exports.toFilterTopic = exports.getLogs = void 0;
 const _1 = require(".");
 const ethers_1 = require("ethers");
 const blocks_1 = require("./blocks");
 const cache_1 = require("./cache");
 const debugLog_1 = require("./debugLog");
+const indexer_1 = require("./indexer");
 const tron_1 = require("../abi/tron");
 const currentVersion = 'v3';
 async function getLogs(options) {
     let { target, chain = 'ethereum', fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, // can be passed as input as extra filter arguments
     skipCache = false, skipCacheRead = false, entireLog = false, cacheInCloud = false, onlyArgs = false, keys = [], //  [Deprecated] This is just used to select only part of the logs
-    targets, flatten = true, } = options;
+    targets, flatten = true, skipIndexer = false, onlyIndexer = false, debugMode = false, } = options;
+    if (!skipIndexer && (0, indexer_1.isIndexerEnabled)(chain)) {
+        try {
+            const response = await (0, indexer_1.getLogs)({
+                ...options,
+                all: true,
+            });
+            return response;
+        }
+        catch (e) {
+            let message = e === null || e === void 0 ? void 0 : e.message;
+            (0, debugLog_1.debugLog)('Error in  indexer getLogs', message);
+        }
+    }
+    if (!debugMode)
+        debugMode = debugLog_1.DEBUG_LEVEL2;
+    if (onlyIndexer)
+        throw new Error('onlyIndexer is true, but indexer is not enabled or threw an error');
     // if (!target && !targets?.length) throw new Error('target|targets is required')
     if (!fromBlock && !fromTimestamp)
         throw new Error('fromBlock or fromTimestamp is required');
@@ -89,14 +107,22 @@ async function getLogs(options) {
         return logs;
     return logs.map((i) => iface.parseLog(i)).map((i) => onlyArgs ? i.args : i);
     async function addLogsToCache(fromBlock, toBlock) {
-        (0, debugLog_1.debugLog)('adding logs to cache: ', fromBlock, toBlock, target, topic);
+        const debugTimeKey = `getLogs-${chain}-${topic}-${target}_${Math.random()}-${fromBlock}-${toBlock}`;
+        if (debugMode)
+            (0, debugLog_1.debugLog)('adding logs to cache: ', fromBlock, toBlock, target, topic, chain);
         if (fromBlock > toBlock)
             return; // no data to add
         fromBlock = fromBlock - 10;
         toBlock = toBlock + 10;
+        if (debugMode)
+            console.time(debugTimeKey);
         let { output: logs } = await (0, _1.getLogs)({
             chain, target: target, topic: topic, keys, topics, fromBlock, toBlock,
         });
+        if (debugMode) {
+            console.timeEnd(debugTimeKey);
+            (0, debugLog_1.debugLog)('Logs pulled ' + chain, target, logs.length);
+        }
         caches.push({
             logs,
             metadata: { fromBlock, toBlock, }
@@ -151,16 +177,17 @@ async function getLogs(options) {
             extraKey += '-' + keys.join('-').toLowerCase();
         return `event-logs/${chain}/${(_a = target === null || target === void 0 ? void 0 : target.toLowerCase()) !== null && _a !== void 0 ? _a : null}-${extraKey}`;
     }
-    // we need to form filter topic with indexed keyword, else it messes up generated topic string
-    function toFilterTopic(topic) {
-        if (typeof topic === 'string') {
-            if (topic.startsWith('0x'))
-                return topic;
-            topic = new ethers_1.Interface([topic]);
-        }
-        const fragment = topic.fragments[0];
-        return (0, ethers_1.id)(`${fragment.name}(${fragment.inputs.map((i) => i.type).join(',')})`);
-    }
 }
 exports.getLogs = getLogs;
+// we need to form filter topic with indexed keyword, else it messes up generated topic string
+function toFilterTopic(topic) {
+    if (typeof topic === 'string') {
+        if (topic.startsWith('0x'))
+            return topic;
+        topic = new ethers_1.Interface([topic]);
+    }
+    const fragment = topic.fragments[0];
+    return (0, ethers_1.id)(`${fragment.name}(${fragment.inputs.map((i) => i.type).join(',')})`);
+}
+exports.toFilterTopic = toFilterTopic;
 exports.default = getLogs;

package/build/util/updateProviderList.js

@@ -67,8 +67,13 @@ async function main() {
     }
     fs_1.default.writeFileSync(__dirname + '/../providers.json', JSON.stringify(providerList));
 }
+const blacklist = [
+    'https://eth.api.onfinality.io/public'
+];
 function filterRPCs(rpc) {
     return rpc.filter((i) => {
+        if (blacklist.includes(i))
+            return false; // remove rpcs returning bad block heights etc
         if (i.endsWith('/demo'))
             return false; // remove demo rpc
         if (i.includes('$'))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.83",
+  "version": "5.0.85",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {