@defillama/sdk 5.0.179 → 5.0.182

package/build/util/LlamaProvider.js

@@ -60,7 +60,7 @@ class LlamaProvider extends ethers_1.FallbackProvider {
                 currentBlocks[url] = block;
             }
             catch (e) {
-                if (env_1.logLlamaProviderCalls)
+                if (env_1.ENV_CONSTANTS.LOG_LLAMA_PROVIDER_CALLS)
                     (0, debugLog_1.debugLog)(`${_this.chainName} skipping RPC ${url} is not working, error: ${e.message}`);
                 _this._removeProvider(url);
             }

package/build/util/blocks.js

@@ -134,6 +134,9 @@ let lastBlockCacheSave = Date.now();
 const blockCacheSaveInterval = 1000 * 60 * 5; // 5 minutes 
 const { data: blockTimeCache, saveCacheFile: saveBlockCacheFile } = (0, cache_1.getTempLocalCache)({ file: 'BlockCache.json', defaultData: {}, clearAfter: cache_1.ONE_WEEK, returnWithSaveFunction: true });
 function validateCurrentBlock(block, chain = 'ethereum') {
+    // sometimes the chain is halted or facing issues, so we skip the validation for those chains
+    if ((0, env_1.shouldSkipCurrentBlockValidation)(chain))
+        return;
     const provider = getExtraProvider(chain);
     const now = Math.floor(Date.now() / 1000);
     const minutesDiff = Math.floor((now - block.timestamp) / 60);
@@ -227,7 +230,7 @@ async function _lookupBlock(timestamp, extraParams) {
             ]);
             updateBlock(blocks);
         }
-        if (i > 2 && env_1.logGetBlockStats)
+        if (i > 2 && env_1.ENV_CONSTANTS.GET_BLOCK_STATS)
             (0, debugLog_1.debugLog)(`chain: ${chain} block: ${block.number} #calls: ${i} imprecision: ${Number((imprecision) / 60).toFixed(2)} (min) Time Taken: ${Number((Date.now() - time) / 1000).toFixed(2)} (in sec)`);
         if (chain !== "bsc" && // this check is there because bsc halted the chain for few days
             Math.abs(block.timestamp - timestamp) > 3600) {

package/build/util/cache.js

@@ -1,7 +1,4 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ONE_WEEK = exports.currentVersion = void 0;
 exports.readCache = readCache;
@@ -19,7 +16,6 @@ const zlib_1 = require("zlib");
 const util_1 = require("util");
 const env_1 = require("./env");
 const common_1 = require("./common");
-const crypto_1 = __importDefault(require("crypto"));
 const brotliOptions = {
     [zlib_1.constants.BROTLI_PARAM_MODE]: zlib_1.constants.BROTLI_MODE_TEXT,
     [zlib_1.constants.BROTLI_PARAM_QUALITY]: zlib_1.constants.BROTLI_MAX_QUALITY,
@@ -99,7 +95,7 @@ async function writeCache(file, data, options = {}) {
             (0, debugLog_1.debugLog)('Data too small to cache: ', file);
             return;
         }
-        if (isSameData(data, await readCache(file, { skipR2Cache: true, skipCompression: options.skipCompression, })))
+        if (!options.skipCompression && isSameData(data, await readCache(file, { skipR2Cache: true, skipCompression: options.skipCompression, })))
             return;
         if (options.skipCompression)
             file = `${file}-uncompressed`;
@@ -241,7 +237,7 @@ async function cachedFetch({ key, endpoint, postData, fetcher, fetchOptions, err
     if (!key) {
         if (!endpoint || postData || fetcher)
             throw new Error('Key is required for cachedFetch');
-        key = crypto_1.default.createHash('sha256').update(endpoint).digest('hex').substring(0, 32);
+        key = (0, common_1.getHash)(endpoint);
     }
     const fileKey = 'cached-fetch/' + key;
     try {

package/build/util/coins.js

@@ -0,0 +1,98 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPrices = getPrices;
+exports.getMcaps = getMcaps;
+const axios_1 = __importDefault(require("axios"));
+const env_1 = require("./env");
+const promisePool_1 = __importDefault(require("./promisePool"));
+const common_1 = require("./common");
+const _1 = require(".");
+const coinsApiKey = (0, env_1.getEnvValue)("COINS_API_KEY");
+const bodySize = 100;
+function getBodies(readKeys, timestamp) {
+    const chunks = (0, _1.sliceIntoChunks)(readKeys, bodySize);
+    return chunks.map(chunk => {
+        const body = { coins: chunk };
+        if (timestamp !== "now")
+            body.timestamp = timestamp;
+        return body;
+    });
+}
+async function restCallWrapper(request, retries = 3, name = "-") {
+    retries--;
+    try {
+        const res = await request();
+        return res;
+    }
+    catch (e) {
+        if (retries <= 0)
+            throw (0, common_1.formError)(e);
+    }
+    await (0, common_1.sleepRandom)(5000 + 10000, 5000);
+    return restCallWrapper(request, retries, name);
+}
+const priceCacheAll = {
+    now: {
+        "coingecko:tether": {
+            price: 1,
+            symbol: "USDT",
+            timestamp: Math.floor(Date.now() / 1e3 + 3600), // an hour from script start time
+        },
+    }
+};
+const mcapCacheAll = {};
+async function getPrices(readKeys, timestamp) {
+    return getPricesData(readKeys, timestamp, "price");
+}
+async function getPricesData(readKeys, timestamp, dataType) {
+    if (!readKeys.length)
+        return {};
+    const cacheAllObject = dataType === "price" ? priceCacheAll : mcapCacheAll;
+    const subRoute = dataType === "price" ? "prices" : "mcaps";
+    const aggregatedRes = {};
+    const timestampString = timestamp.toString();
+    const priceCache = cacheAllObject[timestampString] || {};
+    if (!cacheAllObject[timestampString])
+        cacheAllObject[timestampString] = priceCache;
+    // read data from cache where possible
+    readKeys = readKeys.filter((PK) => {
+        const pkNormalized = PK.toLowerCase();
+        let pkData = priceCache[pkNormalized];
+        if (pkData) {
+            aggregatedRes[PK] = { ...pkData, PK };
+            return false;
+        }
+        return true;
+    });
+    const bodies = getBodies(readKeys, timestamp);
+    await (0, promisePool_1.default)({
+        items: bodies,
+        concurrency: 10,
+        processor: async (body) => {
+            if (!body.coins.length)
+                return;
+            const res = await restCallWrapper(() => axios_1.default.post(`https://coins.llama.fi/${subRoute}`, body, {
+                headers: { "Content-Type": "application/json" },
+                params: { source: "internal", apikey: coinsApiKey },
+            }));
+            const data = dataType === "price" ? res.data.coins : res.data;
+            Object.entries(data).forEach(([PK, value]) => {
+                const pkNormalized = PK.toLowerCase();
+                priceCache[pkNormalized] = value;
+            });
+            body.coins.forEach((PK) => {
+                const pkNormalized = PK.toLowerCase();
+                const coinData = priceCache[pkNormalized];
+                if (coinData)
+                    aggregatedRes[PK] = { ...coinData, PK };
+            });
+        },
+    });
+    return aggregatedRes;
+}
+async function getMcaps(readKeys, timestamp) {
+    return getPricesData(readKeys, timestamp, "mcap");
+}

package/build/util/common.d.ts

@@ -30,4 +30,5 @@ export declare function evmToTronAddress(address: string): string;
 export declare function sleep(ms: number): Promise<unknown>;
 export declare function sleepRandom(max: number, min?: number): Promise<unknown>;
 export declare function fixTronCallParams(params: any): void;
+export declare function getHash(str: string | string[]): string;
 export {};

package/build/util/common.js

@@ -33,6 +33,9 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.convertToBigInt = convertToBigInt;
 exports.sumMultiBalanceOf = sumMultiBalanceOf;
@@ -52,10 +55,12 @@ exports.evmToTronAddress = evmToTronAddress;
 exports.sleep = sleep;
 exports.sleepRandom = sleepRandom;
 exports.fixTronCallParams = fixTronCallParams;
+exports.getHash = getHash;
 const axios_1 = __importStar(require("axios"));
 const ethers = __importStar(require("ethers"));
 const TronAddressFormatter = __importStar(require("tron-format-address"));
 const env_1 = require("./env");
+const crypto_1 = __importDefault(require("crypto"));
 function convertToBigInt(value) {
     const balance = value;
     if (!value)
@@ -229,7 +234,7 @@ function formErrorString(e, errorParams = {}) {
     }
     if (e.llamaRPCError) {
         const errorString = `Llama RPC error! method: ${e.method} `;
-        const errors = e.errors.map((i) => { var _a; return `- host: ${i === null || i === void 0 ? void 0 : i.host} error: ${(_a = i === null || i === void 0 ? void 0 : i.error) === null || _a === void 0 ? void 0 : _a.message}`; });
+        const errors = e.errors.map((i) => { var _a; return `- host: ${i === null || i === void 0 ? void 0 : i.host} error: ${typeof (i === null || i === void 0 ? void 0 : i.error) === 'string' ? i === null || i === void 0 ? void 0 : i.error : (_a = i === null || i === void 0 ? void 0 : i.error) === null || _a === void 0 ? void 0 : _a.message}`; });
         return errorString + '\n' + errors.join('\n');
     }
     let errorString = e.toString();
@@ -286,11 +291,20 @@ function formErrorString(e, errorParams = {}) {
     }
     return errorString;
 }
-function shortenString(str, length = env_1.defaultShortenStringLength) {
+function shortenString(str, length) {
+    if (!length) {
+        if (!isNaN(+env_1.ENV_CONSTANTS.DEFAULT_SHORTEN_STRING_LENGTH))
+            length = +env_1.ENV_CONSTANTS.DEFAULT_SHORTEN_STRING_LENGTH;
+        if (!length || isNaN(length))
+            length = 120;
+    }
     return str.length > length ? str.slice(0, length).concat('...') : str;
 }
 function formError(e, errorParams = {}) {
     var _a;
+    if (e instanceof Error && (0, env_1.getBoolEnvValue)('DEBUG_ERROR_STACK')) {
+        console.error(e);
+    }
     const errorParamsSet = Object.keys(errorParams).length;
     if ((e === null || e === void 0 ? void 0 : e._isCustomError) || (!errorParamsSet && typeof e === 'object' && !Object.keys(e).length))
         return e; // already formatted or vannila error
@@ -350,3 +364,8 @@ function fixTronCallParams(params) {
     if (Array.isArray(params.calls))
         params.calls.forEach(fixTronCallParams);
 }
+function getHash(str) {
+    if (Array.isArray(str))
+        str = str.join('-');
+    return crypto_1.default.createHash('sha256').update(str).digest('hex').substring(0, 32);
+}

package/build/util/computeTVL.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = computeTVL;
 exports.getUnique = getUnique;
 const generalUtil_1 = require("../generalUtil");
-const coinsApi_1 = require("./coinsApi");
+const coins_1 = require("./coins");
 const currentPriceCache = {};
 const priceCaches = {};
 let lastPriceUpdate = 0;
@@ -69,7 +69,7 @@ async function updatePriceCache(keys, timestamp) {
         "confidence": 1
     };
     const missingKeys = keys.filter(key => !pricesCache[key.toLowerCase()]);
-    const coins = await (0, coinsApi_1.getPrices)(missingKeys, timestamp !== null && timestamp !== void 0 ? timestamp : "now");
+    const coins = await (0, coins_1.getPrices)(missingKeys, timestamp !== null && timestamp !== void 0 ? timestamp : "now");
     for (const [token, data] of Object.entries(coins)) {
         pricesCache[token.toLowerCase()] = data;
     }

package/build/util/env.d.ts

@@ -1,12 +1,5 @@
-export declare const isDebugLevel3: boolean;
-export declare const logLlamaProviderCalls: boolean;
-export declare const logGetBlockStats: boolean;
-export declare const logGetLogsErrors: boolean;
-export declare const logGetLogsDebug: boolean;
-export declare const logGetLogsIndexer: boolean;
-export declare const defaultShortenStringLength: number;
 export declare const ENV_CONSTANTS: {
-    [key: string]: string | undefined;
+    [key: string]: string | boolean | undefined;
 };
 export declare function getChainId(chain: string, chainId: number): number;
 export declare function getMaxParallelRequests(chain: string): number;
@@ -21,3 +14,5 @@ export declare function getBatchMaxCount(chain: string): number;
 export declare function getArchivalRPCs(chain: string): string[];
 export declare function getChainRPCs(chain: string, defaultList?: string[]): string | undefined;
 export declare function getWhitelistedRPCs(chain: string): string[];
+export declare function shouldSkipCurrentBlockValidation(chain: string): boolean;
+export declare function getBoolEnvValue(key: string, defaultValue?: boolean): boolean;

package/build/util/env.js

@@ -1,7 +1,7 @@
 "use strict";
 var _a;
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ENV_CONSTANTS = exports.defaultShortenStringLength = exports.logGetLogsIndexer = exports.logGetLogsDebug = exports.logGetLogsErrors = exports.logGetBlockStats = exports.logLlamaProviderCalls = exports.isDebugLevel3 = void 0;
+exports.ENV_CONSTANTS = void 0;
 exports.getChainId = getChainId;
 exports.getMaxParallelRequests = getMaxParallelRequests;
 exports.getParallelGetLogsLimit = getParallelGetLogsLimit;
@@ -15,17 +15,13 @@ exports.getBatchMaxCount = getBatchMaxCount;
 exports.getArchivalRPCs = getArchivalRPCs;
 exports.getChainRPCs = getChainRPCs;
 exports.getWhitelistedRPCs = getWhitelistedRPCs;
+exports.shouldSkipCurrentBlockValidation = shouldSkipCurrentBlockValidation;
+exports.getBoolEnvValue = getBoolEnvValue;
 const whitelistedEnvConstants = [
     'R2_ENDPOINT', 'R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY', 'TRON_PRO_API_KEY', 'COINS_API_KEY',
     'ELASTICSEARCH_CONFIG', 'GRAPH_API_KEY', 'LLAMA_INDEXER_ENDPOINT', 'LLAMA_INDEXER_API_KEY', 'LLAMA_INDEXER_V2_ENDPOINT', 'LLAMA_INDEXER_V2_API_KEY',
 ];
-exports.isDebugLevel3 = process.env.LLAMA_SDK_DEBUG_LEVEL_3 === 'true' || false;
-exports.logLlamaProviderCalls = process.env.LLAMA_SDK_LOG_LLAMA_PROVIDER_CALLS === 'true' || exports.isDebugLevel3;
-exports.logGetBlockStats = process.env.LLAMA_SDK_LOG_GET_BLOCK_STATS === 'true' || exports.isDebugLevel3;
-exports.logGetLogsErrors = process.env.LLAMA_SDK_LOG_GET_LOGS_ERRORS === 'true' || exports.isDebugLevel3;
-exports.logGetLogsDebug = process.env.LLAMA_SDK_LOG_GET_LOGS_DEBUG === 'true' || exports.isDebugLevel3;
-exports.logGetLogsIndexer = process.env.LLAMA_SDK_LOG_GET_LOGS_INDEXER === 'true' || exports.isDebugLevel3;
-exports.defaultShortenStringLength = +((_a = process.env.LLAMA_SDK_DEFAULT_SHORTEN_STRING_LENGTH) !== null && _a !== void 0 ? _a : '120');
+const isDebugLevel3 = process.env.LLAMA_SDK_DEBUG_LEVEL_3 === 'true';
 const defaultEnvValues = {
     INTERNAL_SDK_CACHE_FILE: './cache.json',
     TRON_RPC_CONCURRENCY_LIMIT: '5',
@@ -43,7 +39,15 @@ const defaultEnvValues = {
 };
 const _ENV_CONSTANTS = {
     DEBUG_ENABLED: getEnvValue('DEBUG') === "true" || process.env.LLAMA_DEBUG_MODE ? 'true' : 'false',
-    DEBUG_LEVEL2: process.env.LLAMA_DEBUG_LEVEL2 ? 'true' : 'false',
+    DEBUG_ERROR_STACK: getEnvValue('DEBUG_ERROR_STACK') === "true",
+    DEBUG_LEVEL2: process.env.LLAMA_DEBUG_LEVEL2 || process.env.LLAMA_SDK_DEBUG_LEVEL_3 ? 'true' : 'false',
+    DEBUG_LEBEL_3: isDebugLevel3,
+    LOG_LLAMA_PROVIDER_CALLS: isDebugLevel3 || process.env.LLAMA_SDK_LOG_LLAMA_PROVIDER_CALLS === 'true',
+    GET_BLOCK_STATS: isDebugLevel3 || process.env.LLAMA_SDK_GET_BLOCK_STATS === 'true',
+    // GET_LOGS_ERRORS:  isDebugLevel3 || process.env.LLAMA_SDK_GET_LOGS_ERRORS === 'true',
+    GET_LOGS_DEBUG: isDebugLevel3 || process.env.LLAMA_SDK_GET_LOGS_DEBUG === 'true',
+    GET_LOGS_INDEXER: isDebugLevel3 || process.env.LLAMA_SDK_GET_LOGS_INDEXER === 'true',
+    DEFAULT_SHORTEN_STRING_LENGTH: (_a = process.env.LLAMA_SDK_DEFAULT_SHORTEN_STRING_LENGTH) !== null && _a !== void 0 ? _a : '120',
 };
 whitelistedEnvConstants.forEach(key => _ENV_CONSTANTS[key] = getEnvValue(key));
 Object.keys(defaultEnvValues).forEach(key => _ENV_CONSTANTS[key] = getEnvValue(key, defaultEnvValues[key]));
@@ -130,3 +134,18 @@ function getWhitelistedRPCs(chain) {
         return getEnvValue(key).split(',');
     return [];
 }
+let skipCurrentBlockValidationChainSet = null;
+function shouldSkipCurrentBlockValidation(chain) {
+    if (!skipCurrentBlockValidationChainSet) {
+        const envValue = getEnvValue('SKIP_CURRENT_BLOCK_VALIDATION_CHAINS', '');
+        const chainList = envValue ? envValue.split(',').map(c => c.trim()) : [];
+        skipCurrentBlockValidationChainSet = new Set(chainList);
+    }
+    return skipCurrentBlockValidationChainSet.has(chain);
+}
+function getBoolEnvValue(key, defaultValue = false) {
+    const envValue = getEnvValue(key);
+    if (envValue === undefined)
+        return defaultValue;
+    return envValue === 'true';
+}

package/build/util/index.js

@@ -55,7 +55,7 @@ async function getLogs(options) {
         catch (e) {
             options.provider = (0, general_1.getProvider)(options.chain, false);
             // if (provider === options.provider) throw e // the provider for archival and non-archival request is the same
-            if (env_1.logGetLogsDebug)
+            if (env_1.ENV_CONSTANTS.GET_LOGS_DEBUG)
                 (0, debugLog_1.debugLog)(`Error fetching logs for chain ${options.chain} ${(0, common_1.formError)(e, { chain: options.chain, target: options.target, })}, retrying...`);
             return getLimiter()(() => _getLogs(options));
         }
@@ -91,7 +91,7 @@ async function _getLogs(params) {
             currentBlock = nextBlock;
         }
         catch (e) {
-            if (env_1.logGetLogsDebug)
+            if (env_1.ENV_CONSTANTS.GET_LOGS_DEBUG)
                 (0, debugLog_1.debugLog)(`Error fetching logs for chain ${params.chain} blockSpread: ${blockSpread}. ${(0, common_1.formError)(e, { chain: params.chain, target: params.target, provider })}`);
             if (blockSpread >= 1001) {
                 // We got too many results

package/build/util/indexer.js

@@ -160,7 +160,7 @@ async function getTokens(address, { onlyWhitelisted = true, skipCacheRead = fals
         if (cache.timestamp && timeNow - cache.timestamp < THREE_DAYS)
             return cache.tokens;
     }
-    if (env_1.logGetLogsIndexer)
+    if (env_1.ENV_CONSTANTS.GET_LOGS_INDEXER)
         (0, debugLog_1.debugLog)("[Indexer] Pulling tokens for " + address);
     const tokens = (_b = cache.tokens) !== null && _b !== void 0 ? _b : {};
     const { data: { balances }, } = await axiosInstances.v1(`/balances`, {
@@ -197,7 +197,7 @@ async function getLogs(options) {
     let { all = true, limit = 1000, offset: initialOffset = 0, target, targets = [], flatten = true, debugMode = false, noTarget = false, maxBlockRange, processor, } = options;
     const { chain, fromBlock, toBlock, topic, topic1, topic2, topic3, transformLog, } = await (0, logs_1.getLogParams)(options, true);
     if (!debugMode)
-        debugMode = debugLog_1.DEBUG_LEVEL2 && env_1.logGetLogsIndexer;
+        debugMode = debugLog_1.DEBUG_LEVEL2 && !!env_1.ENV_CONSTANTS.GET_LOGS_INDEXER;
     const blockRange = toBlock - fromBlock;
     const effectiveMaxBlockRange = maxBlockRange !== null && maxBlockRange !== void 0 ? maxBlockRange : (noTarget ? 10000 : Infinity);
     if (noTarget && blockRange > 500000) {
@@ -213,7 +213,7 @@ async function getLogs(options) {
         const percentageMissing = ((toBlock - lastIndexedBlock) / (toBlock - fromBlock)) * 100;
         if (percentageMissing > 50) // more than 50% of the requested range is missing, throw an error and so we pull the logs through rpc calls
             throw new Error(`Indexer not up to date for ${chain}. Last indexed block: ${lastIndexedBlock}, requested block: ${toBlock}`);
-        if (env_1.logGetLogsIndexer)
+        if (env_1.ENV_CONSTANTS.GET_LOGS_INDEXER)
             (0, debugLog_1.debugLog)(`Indexer only partially up to date for ${chain}. Last indexed block: ${lastIndexedBlock}, requested block: ${toBlock}, missing ${Number(percentageMissing).toFixed(2)}%. Pulling part of the logs through RPC calls.`);
         // now we split the request into two parts, one that goes to the indexer and one that goes through rpc calls
         const breakBlock = lastIndexedBlock - 50; // we add a small buffer to avoid missing logs due to reorgs
@@ -329,7 +329,7 @@ async function getLogs(options) {
 async function getTokenTransfers({ chain = "ethereum", fromAddressFilter, fromBlock, toBlock, all = true, limit = 1000, offset = 0, target, targets = [], flatten = true, fromTimestamp, toTimestamp, debugMode = false, transferType = "in", token, tokens, }) {
     var _a, _b;
     if (!debugMode)
-        debugMode = debugLog_1.DEBUG_LEVEL2 && env_1.logGetLogsIndexer;
+        debugMode = debugLog_1.DEBUG_LEVEL2 && !!env_1.ENV_CONSTANTS.GET_LOGS_INDEXER;
     checkIndexerConfig("v2");
     const chainId = getChainId(chain, "v2");
     // -----------------------------------------------------------------------
@@ -446,7 +446,7 @@ async function getTokenTransfers({ chain = "ethereum", fromAddressFilter, fromBl
 async function getTransactions({ chain = "ethereum", addresses, transaction_hashes, from_block, to_block, all = true, limit = 1000, offset = 0, debugMode = false, transactionType = "from", }) {
     var _a, _b;
     if (!debugMode)
-        debugMode = debugLog_1.DEBUG_LEVEL2 && env_1.logGetLogsIndexer;
+        debugMode = debugLog_1.DEBUG_LEVEL2 && !!env_1.ENV_CONSTANTS.GET_LOGS_INDEXER;
     checkIndexerConfig("v2");
     const chainId = getChainId(chain, "v2");
     if ((!addresses || addresses.length === 0) && (!transaction_hashes || transaction_hashes.length === 0))

package/build/util/logs.js

@@ -46,7 +46,7 @@ async function getLogs(options) {
         }
     }
     if (!debugMode)
-        debugMode = debugLog_1.DEBUG_LEVEL2 && env_1.logGetLogsDebug;
+        debugMode = debugLog_1.DEBUG_LEVEL2 && !!env_1.ENV_CONSTANTS.GET_LOGS_DEBUG;
     /* ----------------------- Split by maxBlockRange ------------------------- */
     const blockRange = toBlock - fromBlock;
     if (maxBlockRange && blockRange > maxBlockRange) {
@@ -166,8 +166,42 @@ async function getLogs(options) {
                 merged.push(c);
         });
         caches = merged;
-        if (!skipCache) // we are skipping compression by default, so reads & writes are faster
-            await (0, cache_1.writeCache)(getFile(), { caches, version: currentVersion }, { skipR2CacheWrite: !cacheInCloud, skipCompression: !cacheInCloud });
+        if (!skipCache) {
+            let storeCaches = caches;
+            let randomLogLength = 0;
+            const totalLogCount = caches.reduce((acc, c) => {
+                if (c.logs.length)
+                    randomLogLength = JSON.stringify(c.logs[0]).length;
+                return acc + c.logs.length;
+            }, 0);
+            const fileSizeInMB = (totalLogCount * randomLogLength) / (1024 * 1024);
+            if (fileSizeInMB > 300) { // if cache size is larger than ~300MB, trim it down
+                console.log(`getLogs: large cache size detected ${totalLogCount} logs size: ${fileSizeInMB}MB, retaining only the latest 200k logs to limit memory usage target=${target} topic=${topic} fromBlock=${fromBlock} toBlock=${toBlock}`);
+                // retain only the latest logs that fit within 300,000 logs
+                if (caches.length > 1) {
+                    let cacheWithLatestFromBlock = caches[0];
+                    for (const c of caches) {
+                        if (c.metadata.fromBlock > cacheWithLatestFromBlock.metadata.fromBlock) {
+                            cacheWithLatestFromBlock = c;
+                        }
+                    }
+                    storeCaches = [cacheWithLatestFromBlock];
+                }
+                storeCaches = storeCaches.map(c => {
+                    const clone = { logs: [], metadata: JSON.parse(JSON.stringify(c.metadata)) };
+                    if (c.logs.length > 200000) {
+                        // sort the logs by block number descending and keep only the latest 200k logs
+                        c.logs.sort((a, b) => b.blockNumber - a.blockNumber);
+                        clone.logs = c.logs.slice(0, 200000);
+                        const lastLog = clone.logs[clone.logs.length - 1];
+                        clone.metadata.fromBlock = lastLog.blockNumber;
+                    }
+                    return clone;
+                });
+            }
+            // we are skipping compression by default, so reads & writes are faster
+            await (0, cache_1.writeCache)(getFile(), { caches: storeCaches, version: currentVersion }, { skipR2CacheWrite: !cacheInCloud, skipCompression: !cacheInCloud });
+        }
     }
     function dedupLogs(...arr) {
         const seen = new Set();
@@ -203,7 +237,12 @@ async function getLogs(options) {
             throw new Error("extraKey is required");
         if (keys.length)
             extraKey += "-" + keys.join("-").toLowerCase();
-        return `event-logs/${chain}/${(_a = target === null || target === void 0 ? void 0 : target.toLowerCase()) !== null && _a !== void 0 ? _a : null}-${extraKey}`;
+        let file = `${(_a = target === null || target === void 0 ? void 0 : target.toLowerCase()) !== null && _a !== void 0 ? _a : null}-${extraKey}`;
+        const normalKeyLength = '0x27e5cb712334e101b3c232eb0be198baaa595f5f-0xe8137aa901976cc8eaf1cef5dec491873faadc99d9720ccaec95673294a9d7c5-uncompressed'.length;
+        // sometimes, there is a lot of topics and we hit the file name length limit
+        if (file.length > normalKeyLength)
+            file = (0, common_1.getHash)(file);
+        return `event-logs/${chain}/${file}`;
     }
 }
 /* -------------------------------------------------------------------------- */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.179",
+  "version": "5.0.182",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {

package/build/util/coinsApi.js

@@ -1,125 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getPrices = getPrices;
-exports.getMcaps = getMcaps;
-const axios_1 = __importDefault(require("axios"));
-const env_1 = require("./env");
-const promisePool_1 = __importDefault(require("./promisePool"));
-const common_1 = require("./common");
-const coinsApiKey = (0, env_1.getEnvValue)("COINS_API_KEY");
-const bodySize = 2; // 100;
-function getBodies(readKeys, timestamp) {
-    const bodies = [];
-    for (let i = 0; i < readKeys.length; i += bodySize) {
-        const body = {
-            coins: readKeys.slice(i, i + bodySize),
-        };
-        if (timestamp !== "now")
-            body.timestamp = timestamp;
-        bodies.push(JSON.stringify(body));
-    }
-    return bodies;
-}
-async function restCallWrapper(request, retries = 3, name = "-") {
-    while (retries > 0) {
-        try {
-            const res = await request();
-            return res;
-        }
-        catch {
-            await (0, common_1.sleepRandom)(5000 + 10000, 5000);
-            restCallWrapper(request, retries--, name);
-        }
-    }
-    throw new Error(`couldnt work ${name} call after retries!`);
-}
-const priceCache = {
-    "coingecko:tether": {
-        price: 1,
-        symbol: "USDT",
-        timestamp: Math.floor(Date.now() / 1e3 + 3600), // an hour from script start time
-    },
-};
-async function getPrices(readKeys, timestamp) {
-    if (!readKeys.length)
-        return {};
-    const aggregatedRes = {};
-    // read data from cache where possible
-    readKeys = readKeys.filter((PK) => {
-        if (timestamp !== "now")
-            return true;
-        if (priceCache[PK]) {
-            aggregatedRes[PK] = { ...priceCache[PK], PK };
-            return false;
-        }
-        return true;
-    });
-    const bodies = getBodies(readKeys, timestamp);
-    const tokenData = [];
-    await (0, promisePool_1.default)({
-        items: bodies,
-        concurrency: 10,
-        processor: async (body) => {
-            const res = await restCallWrapper(() => axios_1.default.post(`https://coins.llama.fi/prices?source=internal${coinsApiKey ? `?apikey=${coinsApiKey}` : ""}`, body, {
-                headers: { "Content-Type": "application/json" },
-                params: { source: "internal", apikey: coinsApiKey },
-            }));
-            const data = (res.data.coins = Object.entries(res.data.coins).map(([PK, value]) => ({
-                ...value,
-                PK,
-            })));
-            tokenData.push(data);
-        },
-    });
-    const normalizedReadKeys = readKeys.map((k) => k.toLowerCase());
-    tokenData.map((batch) => {
-        batch.map((a) => {
-            if (!a.PK)
-                return;
-            const i = normalizedReadKeys.indexOf(a.PK.toLowerCase());
-            aggregatedRes[readKeys[i]] = a;
-        });
-    });
-    return aggregatedRes;
-}
-const mcapCache = {};
-async function getMcaps(readKeys, timestamp) {
-    if (!readKeys.length)
-        return {};
-    const aggregatedRes = {};
-    // read data from cache where possible
-    readKeys = readKeys.filter((PK) => {
-        if (timestamp !== "now")
-            return true;
-        if (mcapCache[PK]) {
-            aggregatedRes[PK] = { ...mcapCache[PK], PK };
-            return false;
-        }
-        return true;
-    });
-    const bodies = getBodies(readKeys, timestamp);
-    const tokenData = [];
-    await (0, promisePool_1.default)({
-        items: bodies,
-        concurrency: 10,
-        processor: async (body) => {
-            const res = await restCallWrapper(() => axios_1.default.post(`https://coins.llama.fi/mcaps${coinsApiKey ? `?apikey=${coinsApiKey}` : ""}`, body, {
-                headers: { "Content-Type": "application/json" },
-            }));
-            tokenData.push(res.data);
-        },
-    });
-    const normalizedReadKeys = readKeys.map((k) => k.toLowerCase());
-    tokenData.map((batch) => {
-        Object.keys(batch).map((a) => {
-            if (!batch[a].mcap)
-                return;
-            const i = normalizedReadKeys.indexOf(a.toLowerCase());
-            aggregatedRes[readKeys[i]] = batch[a];
-        });
-    });
-    return aggregatedRes;
-}