@defillama/sdk 5.0.146 → 5.0.148

package/build/util/logs.d.ts

@@ -1,5 +1,6 @@
-import { EventLog } from "ethers";
+import { EventLog, Interface } from "ethers";
 import { Address } from "../types";
+import { IndexerGetLogsOptions } from "./indexer";
 export type GetLogsOptions = {
     target?: Address;
     topic?: string;
@@ -24,6 +25,9 @@ export type GetLogsOptions = {
     debugMode?: boolean;
     noTarget?: boolean;
     parseLog?: boolean;
+    processor?: (logs: any[]) => Promise<void> | void;
+    maxBlockRange?: number;
+    allowParseFailure?: boolean;
 };
 export declare function getLogs(options: GetLogsOptions): Promise<EventLog[] | EventLog[][] | any[]>;
 export declare function toFilterTopic(topic: string): string;
@@ -35,3 +39,17 @@ export type logCache = {
     };
 };
 export default getLogs;
+type GetLogsParamsResponse = {
+    chain: string;
+    fromBlock: number;
+    toBlock: number;
+    target?: Address;
+    topic: string;
+    iface?: Interface;
+    transformLog: (log: any) => any;
+    topics?: (string | null)[];
+    topic1?: string;
+    topic2?: string;
+    topic3?: string;
+};
+export declare function getLogParams(options: GetLogsOptions | IndexerGetLogsOptions, isIndexerCall?: boolean): Promise<GetLogsParamsResponse>;

package/build/util/logs.js

@@ -1,198 +1,334 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toFilterTopic = exports.getLogs = void 0;
-const _1 = require(".");
+exports.getLogParams = exports.toFilterTopic = exports.getLogs = void 0;
 const ethers_1 = require("ethers");
+const _1 = require(".");
+const tron_1 = require("../abi/tron");
 const blocks_1 = require("./blocks");
 const cache_1 = require("./cache");
 const debugLog_1 = require("./debugLog");
 const indexer_1 = require("./indexer");
-const tron_1 = require("../abi/tron");
-const currentVersion = 'v3';
+const promisePool_1 = __importDefault(require("./promisePool"));
+const currentVersion = "v3";
+/* -------------------------------------------------------------------------- */
+/*                                   getLogs                                  */
+/* -------------------------------------------------------------------------- */
 async function getLogs(options) {
-    let { target, chain = 'ethereum', fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, // can be passed as input as extra filter arguments
-    skipCache = false, skipCacheRead = false, entireLog = false, cacheInCloud = false, onlyArgs = false, keys = [], //  [Deprecated] This is just used to select only part of the logs
-    targets, flatten = true, skipIndexer = false, onlyIndexer = false, debugMode = false, noTarget = false, // we sometimes want to query logs without a target, but it will an be expensive bug if target/targets were not passed by mistake, so this is a safety check
-    parseLog = false, } = options;
+    let { target, skipCache = false, skipCacheRead = false, cacheInCloud = false, keys = [], targets, flatten = true, skipIndexer = false, onlyIndexer = false, debugMode = false, processor, maxBlockRange, } = options;
+    const { fromBlock, toBlock, topics, topic, transformLog, chain } = await getLogParams(options, false);
+    /* ----------------------------- Indexer path ----------------------------- */
+    // If the caller demands indexer only but it is disabled, fail fast
+    if (onlyIndexer && !(0, indexer_1.isIndexerEnabled)(chain))
+        throw new Error("onlyIndexer is true, but indexer is not enabled");
     if (!skipIndexer && (0, indexer_1.isIndexerEnabled)(chain)) {
         try {
             const response = await (0, indexer_1.getLogs)({
                 ...options,
-                all: true,
+                all: true, // internal service – always fetch full set
+                parseLog: options.parseLog,
+                eventAbi: options.eventAbi,
+                onlyArgs: options.onlyArgs,
             });
             return response;
         }
         catch (e) {
-            let message = e === null || e === void 0 ? void 0 : e.message;
-            (0, debugLog_1.debugLog)('Error in  indexer getLogs', message);
+            (0, debugLog_1.debugLog)("Error in indexer getLogs", e.message);
+            // If caller insisted on indexer only, propagate the error
+            if (onlyIndexer)
+                throw e;
+            // Otherwise continue with RPC fallback
         }
     }
     if (!debugMode)
         debugMode = debugLog_1.DEBUG_LEVEL2;
-    if (onlyIndexer)
-        throw new Error('onlyIndexer is true, but indexer is not enabled or threw an error');
-    // if (!target && !targets?.length) throw new Error('target|targets is required')
-    if (!fromBlock && !fromTimestamp)
-        throw new Error('fromBlock or fromTimestamp is required');
-    if (!toBlock && !toTimestamp)
-        throw new Error('toBlock or toTimestamp is required');
-    if (!fromBlock)
-        fromBlock = await (0, blocks_1.getBlockNumber)(chain, fromTimestamp);
-    if (!toBlock)
-        toBlock = await (0, blocks_1.getBlockNumber)(chain, toTimestamp);
-    if (!fromBlock || !toBlock)
-        throw new Error('fromBlock and toBlock must be > 0');
-    if (!noTarget && !target && !(targets === null || targets === void 0 ? void 0 : targets.length)) {
-        throw new Error('target|targets is required or set the flag "noTarget" to true');
+    /* ----------------------- Split by maxBlockRange ------------------------- */
+    const blockRange = toBlock - fromBlock;
+    if (maxBlockRange && blockRange > maxBlockRange) {
+        const results = [];
+        for (let currentFromBlock = fromBlock; currentFromBlock <= toBlock; currentFromBlock += maxBlockRange) {
+            const currentToBlock = Math.min(currentFromBlock + maxBlockRange - 1, toBlock);
+            const chunk = await getLogs({
+                ...options,
+                fromBlock: currentFromBlock,
+                toBlock: currentToBlock,
+            });
+            results.push(chunk);
+        }
+        return flatten ? results.flat() : results;
     }
+    /* ------------------------ Multi-target parallel ------------------------- */
     if (targets === null || targets === void 0 ? void 0 : targets.length) {
-        const newOptions = { ...options, fromBlock, toBlock };
-        delete newOptions.targets;
-        const res = await Promise.all(targets.map(i => getLogs({ ...newOptions, target: i, skipIndexer: true })));
-        if (flatten)
-            return res.flat();
-        return res;
+        const baseOpts = { ...options, fromBlock, toBlock, skipIndexer: true };
+        delete baseOpts.targets;
+        const jobs = targets.map((address, idx) => ({ idx, address }));
+        const proc = async (job) => {
+            const logs = await getLogs({ ...baseOpts, target: job.address });
+            return { idx: job.idx, logs };
+        };
+        const results = await (0, promisePool_1.default)({ items: jobs, concurrency: 5, processor: proc });
+        const ordered = new Array(targets.length);
+        for (const { idx, logs } of results)
+            ordered[idx] = logs;
+        return flatten ? ordered.flat() : ordered;
     }
-    if (chain === 'tron')
+    /* ----------------------------- RPC fallback ----------------------------- */
+    if (chain === "tron")
         target = (0, tron_1.hexifyTarget)(target);
-    let iface;
-    if (eventAbi)
-        iface = new ethers_1.Interface([eventAbi]);
-    if ((!topics || !topics.length)) {
-        if (topic)
-            topic = toFilterTopic(topic);
-        else if (eventAbi)
-            topic = toFilterTopic(eventAbi);
-        else {
-            throw new Error('eventAbi | topic | topics are required');
-        }
-        topics = [topic];
-        if (extraTopics)
-            topics.push(...extraTopics);
-    }
-    let caches = await _getCache();
-    const isFirstRun = !caches.length;
-    if (isFirstRun || skipCacheRead) {
+    let caches = (await _getCache());
+    const firstRun = !caches.length;
+    if (firstRun || skipCacheRead) {
         await addLogsToCache(fromBlock, toBlock);
     }
     else {
-        let _fromBlock = fromBlock; // we need to keep a copy of fromBlock as it will be modified
-        let _toBlock = toBlock; // we need to keep a copy of toBlock as it will be modified
+        // try to fill gaps compared to what is already cached
+        let _from = fromBlock;
+        let _to = toBlock;
         const firstCache = caches[0];
         const lastCache = caches[caches.length - 1];
-        for (const cacheData of caches) {
-            const { fromBlock: cFirstBlock, toBlock: cToBlock } = cacheData.metadata;
-            if (_toBlock < firstCache.metadata.fromBlock || _fromBlock > lastCache.metadata.toBlock) { // no intersection with any cache
-                await addLogsToCache(_fromBlock, _toBlock);
+        for (const c of caches) {
+            const { fromBlock: cStart, toBlock: cEnd } = c.metadata;
+            if (_to < firstCache.metadata.fromBlock || _from > lastCache.metadata.toBlock) {
+                await addLogsToCache(_from, _to);
                 break;
             }
-            if (_fromBlock >= cToBlock)
-                continue; // request is after cache end
-            if (_fromBlock <= cFirstBlock) { // request is before cache start
-                await addLogsToCache(_fromBlock, Math.min(_toBlock, cFirstBlock - 1));
+            if (_from >= cEnd)
+                continue;
+            if (_from <= cStart) {
+                await addLogsToCache(_from, Math.min(_to, cStart - 1));
             }
-            if (_toBlock <= cToBlock) { // request ends before cache end
-                _fromBlock = cToBlock;
+            if (_to <= cEnd) {
+                _from = cEnd;
                 break;
             }
-            _fromBlock = cToBlock;
+            _from = cEnd;
         }
-        if (_fromBlock < _toBlock)
-            await addLogsToCache(_fromBlock, _toBlock);
+        if (_from < _to)
+            await addLogsToCache(_from, _to);
     }
-    const logs = [];
-    for (const cacheData of caches) {
-        const { fromBlock: cFirstBlock, toBlock: cToBlock } = cacheData.metadata;
-        if (fromBlock > cToBlock || toBlock < cFirstBlock)
-            continue; // no intersection with cache
-        logs.push(...cacheData.logs.filter((i) => i.blockNumber >= fromBlock && i.blockNumber <= toBlock));
+    /* ---------------------------- Return result ----------------------------- */
+    const out = [];
+    for (const c of caches) {
+        const { fromBlock: cStart, toBlock: cEnd } = c.metadata;
+        if (fromBlock > cEnd || toBlock < cStart)
+            continue;
+        const filtered = c.logs
+            .filter((i) => i.blockNumber >= fromBlock && i.blockNumber <= toBlock)
+            .map(transformLog);
+        if (processor)
+            await processor(filtered);
+        // Always include filtered logs in the return value, even when a processor is provided
+        out.push(...filtered);
     }
-    if (!eventAbi || entireLog) {
-        if (iface && parseLog) {
-            logs.forEach((i) => i.parsedLog = iface === null || iface === void 0 ? void 0 : iface.parseLog(i));
-        }
-        return logs;
-    }
-    return logs.map((i) => iface.parseLog(i)).map((i) => onlyArgs ? i.args : i);
+    return out;
+    /* ----------------------------------------------------------------------- */
+    /*                         Internal helper functions                       */
+    /* ----------------------------------------------------------------------- */
     async function addLogsToCache(fromBlock, toBlock) {
-        const debugTimeKey = `getLogs-${chain}-${topic}-${target}_${Math.random()}-${fromBlock}-${toBlock}`;
+        const dbgKey = `getLogs-${chain}-${topic}-${target}_${Math.random()}-${fromBlock}-${toBlock}`;
         if (debugMode)
-            (0, debugLog_1.debugLog)('adding logs to cache: ', fromBlock, toBlock, target, topic, chain);
+            (0, debugLog_1.debugLog)("adding logs to cache:", fromBlock, toBlock);
         if (fromBlock > toBlock)
-            return; // no data to add
-        fromBlock = fromBlock - 10;
+            return;
+        fromBlock = Math.max(fromBlock - 10, 0); // avoid negative block
         toBlock = toBlock + 10;
         if (debugMode)
-            console.time(debugTimeKey);
-        let { output: logs } = await (0, _1.getLogs)({
-            chain, target: target, topic: topic, keys, topics, fromBlock, toBlock,
+            console.time(dbgKey);
+        const { output: newLogs } = await (0, _1.getLogs)({
+            chain,
+            target: target,
+            topic: topic,
+            keys,
+            topics,
+            fromBlock,
+            toBlock,
         });
         if (debugMode) {
-            console.timeEnd(debugTimeKey);
-            (0, debugLog_1.debugLog)('Logs pulled ' + chain, target, logs.length);
+            console.timeEnd(dbgKey);
+            (0, debugLog_1.debugLog)("Logs pulled", chain, target, newLogs.length);
         }
-        caches.push({
-            logs,
-            metadata: { fromBlock, toBlock, }
-        });
+        caches.push({ logs: newLogs, metadata: { fromBlock, toBlock } });
         caches.sort((a, b) => a.metadata.fromBlock - b.metadata.fromBlock);
-        const mergedCaches = [caches[0]];
-        caches.slice(1).forEach(i => {
-            const last = mergedCaches[mergedCaches.length - 1];
-            if (last.metadata.toBlock + 1 > i.metadata.fromBlock) {
-                last.metadata.toBlock = i.metadata.toBlock;
-                last.logs = dedupLogs(last.logs, i.logs);
-            }
-            else {
-                mergedCaches.push(i);
+        // merge overlapping ranges
+        const merged = [caches[0]];
+        caches.slice(1).forEach((c) => {
+            const last = merged[merged.length - 1];
+            if (last.metadata.toBlock + 1 > c.metadata.fromBlock) {
+                last.metadata.toBlock = c.metadata.toBlock;
+                last.logs = dedupLogs(last.logs, c.logs);
             }
+            else
+                merged.push(c);
         });
-        caches = mergedCaches;
-        if (!skipCache)
-            await (0, cache_1.writeCache)(getFile(), { caches, version: currentVersion }, { skipR2CacheWrite: !cacheInCloud });
+        caches = merged;
+        if (!skipCache) // we are skipping compression by default, so reads & writes are faster
+            await (0, cache_1.writeCache)(getFile(), { caches, version: currentVersion }, { skipR2CacheWrite: !cacheInCloud, skipCompression: !cacheInCloud });
     }
-    function dedupLogs(...logs) {
-        const logIndices = new Set();
-        return logs.flat().filter((i) => {
+    function dedupLogs(...arr) {
+        const seen = new Set();
+        return arr.flat().filter((i) => {
             var _a;
-            let key = i.transactionHash + ((_a = i.logIndex) !== null && _a !== void 0 ? _a : i.index); // ethers v5 had logIndex, ethers v6 has index
-            if (!(i.hasOwnProperty('logIndex') || i.hasOwnProperty('index')) || !i.hasOwnProperty('transactionHash')) {
-                (0, debugLog_1.debugLog)(i, i.logIndex, i.index, i.transactionHash);
-                throw new Error('Missing crucial field: logIndex/index');
+            const key = `${i.transactionHash}-${(_a = i.logIndex) !== null && _a !== void 0 ? _a : i.index}`;
+            if (!(i.hasOwnProperty("logIndex") ||
+                i.hasOwnProperty("index")) ||
+                !i.hasOwnProperty("transactionHash")) {
+                (0, debugLog_1.debugLog)("Missing crucial field", i);
+                throw new Error("Missing crucial field: logIndex/index");
             }
-            if (logIndices.has(key))
+            if (seen.has(key))
                 return false;
-            logIndices.add(key);
+            seen.add(key);
             return true;
         });
     }
     async function _getCache() {
         const key = getFile();
-        const defaultRes = [];
+        const def = [];
         if (skipCache)
-            return defaultRes;
-        let cache = await (0, cache_1.readCache)(key, { skipR2Cache: !cacheInCloud });
+            return def;
+        let cache = await (0, cache_1.readCache)(key, { skipR2Cache: !cacheInCloud, skipCompression: !cacheInCloud, });
         if (!cache.caches || !cache.caches.length || cache.version !== currentVersion)
-            return defaultRes;
+            return def;
         return cache.caches;
     }
     function getFile() {
         var _a;
-        let extraKey = topics === null || topics === void 0 ? void 0 : topics.join('-').toLowerCase();
+        let extraKey = topics === null || topics === void 0 ? void 0 : topics.join("-").toLowerCase();
         if (!extraKey)
-            throw new Error('extraKey is required');
+            throw new Error("extraKey is required");
         if (keys.length)
-            extraKey += '-' + keys.join('-').toLowerCase();
+            extraKey += "-" + keys.join("-").toLowerCase();
         return `event-logs/${chain}/${(_a = target === null || target === void 0 ? void 0 : target.toLowerCase()) !== null && _a !== void 0 ? _a : null}-${extraKey}`;
     }
 }
 exports.getLogs = getLogs;
+/* -------------------------------------------------------------------------- */
+/*                           Utility helpers exported                         */
+/* -------------------------------------------------------------------------- */
 function toFilterTopic(topic) {
-    if (typeof topic === 'string' && topic.startsWith('0x')) {
+    if (topic.startsWith("0x"))
         return topic;
-    }
     const fragment = ethers_1.EventFragment.from(topic);
     return (0, ethers_1.id)(fragment.format());
 }
 exports.toFilterTopic = toFilterTopic;
 exports.default = getLogs;
+async function getLogParams(options, isIndexerCall = false) {
+    let { target, chain = "ethereum", fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, entireLog = false, onlyArgs = false, targets = [], noTarget = false, parseLog = false, allowParseFailure = false, processor, flatten = false, } = options;
+    if (eventAbi && !Object.prototype.hasOwnProperty.call(options, "parseLog"))
+        parseLog = true;
+    // keep raw log if entireLog=true
+    if (entireLog)
+        onlyArgs = false;
+    if (processor && targets.length > 1 && !flatten)
+        throw new Error("processor is not supported with multiple targets when flatten=false");
+    if (!fromBlock && !fromTimestamp)
+        throw new Error("fromBlock or fromTimestamp is required");
+    if (!toBlock && !toTimestamp)
+        throw new Error("toBlock or toTimestamp is required");
+    if (!fromBlock)
+        fromBlock = await (0, blocks_1.getBlockNumber)(chain, fromTimestamp);
+    if (!toBlock)
+        toBlock = await (0, blocks_1.getBlockNumber)(chain, toTimestamp);
+    if (!fromBlock || !toBlock)
+        throw new Error("fromBlock and toBlock must be > 0");
+    if (!noTarget && !target && !targets.length)
+        throw new Error('target|targets is required or set the flag "noTarget" to true');
+    let iface;
+    if (eventAbi)
+        iface = new ethers_1.Interface([eventAbi]);
+    if (!(topics === null || topics === void 0 ? void 0 : topics.length)) {
+        if (topic)
+            topic = toFilterTopic(topic);
+        else if (eventAbi)
+            topic = toFilterTopic(eventAbi);
+        else
+            throw new Error("eventAbi | topic | topics are required");
+        topics = [topic];
+        if (extraTopics)
+            topics.push(...extraTopics);
+    }
+    if (topics === null || topics === void 0 ? void 0 : topics.length)
+        topic = topics[0];
+    function transformLog(log) {
+        // normalize indexer payload
+        if (isIndexerCall) {
+            log.address = log.source;
+            log.logIndex = log.log_index;
+            log.index = log.log_index;
+            log.transactionHash = log.transaction_hash;
+            log.blockNumber = parseInt(log.block_number);
+            // Ensure topics are properly formatted
+            const topics = [log.topic0, log.topic1, log.topic2, log.topic3]
+                .filter(Boolean)
+                .map((t) => {
+                if (!t)
+                    return null;
+                const topic = t.startsWith("0x") ? t : `0x${t}`;
+                return ethers_1.ethers.zeroPadValue(topic, 32);
+            });
+            // Ensure first topic (event signature) is present
+            if (!topics[0]) {
+                return log;
+            }
+            log.topics = topics;
+            // Store original topics for later use
+            log._originalTopics = {
+                topic0: log.topic0,
+                topic1: log.topic1,
+                topic2: log.topic2,
+                topic3: log.topic3
+            };
+            // Store original transaction hash
+            log._originalTransactionHash = log.transaction_hash;
+            // Only delete fields that are not needed for parsing
+            [
+                "chain",
+                "log_index",
+            ].forEach((k) => delete log[k]);
+        }
+        if (log.blockNumber === undefined && log.block_number !== undefined)
+            log.blockNumber = parseInt(log.block_number);
+        if ((entireLog && !parseLog) || !iface) {
+            return log;
+        }
+        // Restore topics if they were lost
+        if (!log.topics && log._originalTopics) {
+            log.topics = [log._originalTopics.topic0, log._originalTopics.topic1, log._originalTopics.topic2, log._originalTopics.topic3]
+                .filter(Boolean)
+                .map((t) => {
+                if (!t)
+                    return null;
+                const topic = t.startsWith("0x") ? t : `0x${t}`;
+                return ethers_1.ethers.zeroPadValue(topic, 32);
+            });
+        }
+        // Restore transaction hash if it was lost
+        if (!log.transactionHash && log._originalTransactionHash) {
+            log.transactionHash = log._originalTransactionHash;
+        }
+        const parsed = iface.parseLog(log);
+        if (!parsed && !allowParseFailure)
+            throw new Error(`Failed to parse log: ${JSON.stringify(log.transactionHash)}`);
+        log.args = parsed === null || parsed === void 0 ? void 0 : parsed.args;
+        if (entireLog)
+            log.parsedLog = parsed;
+        return onlyArgs ? log.args : log;
+    }
+    return {
+        chain,
+        fromBlock,
+        toBlock,
+        target,
+        topic: topic,
+        topic1: topics[1],
+        topic2: topics[2],
+        topic3: topics[3],
+        iface,
+        transformLog,
+        topics,
+    };
+}
+exports.getLogParams = getLogParams;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.146",
+  "version": "5.0.148",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {