@defillama/sdk 5.0.236 → 5.0.238

package/build/util/indexer.js

@@ -94,6 +94,7 @@ const axiosInstances = {
     v2: createIndexerClient("v2"),
     v4: createIndexerClient("v4"),
 };
+const INDEXER_HEAD_BUFFER = 50;
 function checkIndexerConfig(version) {
     const { endpoint, apiKey } = indexerConfigs[version];
     if (!endpoint || !apiKey)
@@ -318,7 +319,7 @@ async function getLogs(options) {
     if (processor && typeof options.collect === "undefined" && clientStreaming) {
         options = { ...options, collect: false };
     }
-    const { chain, fromBlock, toBlock, topic, topic1, topic2, topic3, transformLog, } = await (0, logs_1.getLogParams)(options, true);
+    const { chain, fromBlock, toBlock, topic, topic1, topic2, topic3, topics, transformLog, } = await (0, logs_1.getLogParams)(options, true);
     const viemFastPath = 
     // Preserve the historical streaming shape, including entireLog: the viem
     // fast path exposes args without adding the buffered decoder's parsedLog.
@@ -331,9 +332,11 @@ async function getLogs(options) {
     if (processor && (targets === null || targets === void 0 ? void 0 : targets.length) > 1 && !flatten)
         throw new Error("processor is not supported with multiple targets when flatten=false");
     const blockRange = toBlock - fromBlock;
+    if (!Number.isSafeInteger(fromBlock) || !Number.isSafeInteger(toBlock) || fromBlock > toBlock)
+        throw new Error("getLogs requires integer block bounds with fromBlock <= toBlock");
     const effectiveMaxBlockRange = maxBlockRange !== null && maxBlockRange !== void 0 ? maxBlockRange : (noTarget ? 10000 : Infinity);
-    if (!(effectiveMaxBlockRange > 0))
-        throw new Error("maxBlockRange must be greater than zero");
+    if (!(effectiveMaxBlockRange > 0) || (effectiveMaxBlockRange !== Infinity && !Number.isSafeInteger(effectiveMaxBlockRange)))
+        throw new Error("maxBlockRange must be a positive integer");
     if (noTarget && blockRange > 500000) {
         throw new Error("When noTarget is true, block range must be less than 500k blocks. Please narrow down your block range.");
     }
@@ -342,21 +345,19 @@ async function getLogs(options) {
     checkIndexerConfig(indexerVersion);
     const chainId = getChainId(chain);
     const chainIndexStatus = await getChainIndexStatus(indexerVersion);
-    const lastIndexedBlock = (_b = (_a = chainIndexStatus[chain]) === null || _a === void 0 ? void 0 : _a.block) !== null && _b !== void 0 ? _b : 0;
+    const lastIndexedBlock = Number((_b = (_a = chainIndexStatus[chain]) === null || _a === void 0 ? void 0 : _a.block) !== null && _b !== void 0 ? _b : 0);
+    if (!Number.isSafeInteger(lastIndexedBlock) || lastIndexedBlock < 0)
+        throw new Error(`Indexer returned an invalid last indexed block for ${chain}`);
     if (lastIndexedBlock < toBlock) {
         const percentageMissing = ((toBlock - lastIndexedBlock) / (toBlock - fromBlock)) * 100;
         if (percentageMissing > 50)
             throw new Error(`Indexer not up to date for ${chain}. Last indexed block: ${lastIndexedBlock}, requested block: ${toBlock}`);
         if (env_1.ENV_CONSTANTS.GET_LOGS_INDEXER)
             (0, debugLog_1.debugLog)(`Indexer only partially up to date for ${chain}. Last indexed block: ${lastIndexedBlock}, requested block: ${toBlock}, missing ${Number(percentageMissing).toFixed(2)}%. Pulling part of the logs through RPC calls.`);
-        const breakBlock = lastIndexedBlock - 50; // small buffer
-        const indexerLogs = await getLogs({ ...options, fromBlock, toBlock: breakBlock });
-        const rpcLogs = await (0, logs_1.getLogs)({ ...options, fromBlock: breakBlock + 1, toBlock, skipIndexer: true });
-        // When flatten=false, preserve per-target buckets: merge RPC logs into their corresponding indexer bucket so the result still maintains one element per target.
-        if (!flatten) {
-            return indexerLogs.map((bucket, i) => bucket.concat(rpcLogs[i]));
-        }
-        return indexerLogs.concat(rpcLogs);
+        // An empty indexer prefix uses RPC for exactly the original window. The two
+        // inclusive ranges otherwise meet at breakBlock / breakBlock + 1.
+        const breakBlock = Math.max(fromBlock - 1, lastIndexedBlock - INDEXER_HEAD_BUFFER);
+        return getPartiallyIndexedLogs(breakBlock);
     }
     // Re-curse if the requested range is too large
     if (blockRange > effectiveMaxBlockRange) {
@@ -366,9 +367,10 @@ async function getLogs(options) {
             const chunk = await getLogs({ ...options, fromBlock: currentFromBlock, toBlock: currentToBlock });
             results.push(chunk);
         }
-        if (flatten || !(targets === null || targets === void 0 ? void 0 : targets.length))
+        const splitTargets = typeof target === "string" ? [target] : targets;
+        if (flatten || !splitTargets.length)
             return results.flat();
-        return targets.map((_, i) => results.map((r) => r[i]).flat());
+        return splitTargets.map((_, i) => results.map((r) => r[i]).flat());
     }
     let address = target;
     if (typeof target === "string")
@@ -548,6 +550,110 @@ async function getLogs(options) {
         return mapped;
     }
     return allLogsPairs.map(i => i.transformed);
+    async function getPartiallyIndexedLogs(breakBlock) {
+        if (!Number.isSafeInteger(limit) || limit < 0 || !Number.isSafeInteger(initialOffset) || initialOffset < 0)
+            throw new Error("Indexer pagination requires a non-negative integer limit and offset");
+        const requestedTargets = typeof target === "string" ? [target] : targets;
+        const uniqueTargets = [...new Set(requestedTargets.map(t => t.toLowerCase()))];
+        const splitByAddress = requestedTargets.length > 0 && !flatten;
+        const collected = splitByAddress ? requestedTargets.map(() => []) : [];
+        const targetBuckets = new Map();
+        requestedTargets.forEach((t, i) => {
+            var _a;
+            const key = t.toLowerCase();
+            targetBuckets.set(key, [...((_a = targetBuckets.get(key)) !== null && _a !== void 0 ? _a : []), i]);
+        });
+        let remainingOffset = initialOffset;
+        let remainingLimit = all ? Infinity : limit;
+        if (limit === 0)
+            return collected;
+        const rpcTransform = (await (0, logs_1.getLogParams)(options, false)).transformLog;
+        const finished = Symbol("partial log limit reached");
+        let seenBlock;
+        const seen = new Set();
+        async function consume(rawRows, isIndexer, start, end) {
+            var _a, _b, _c, _d;
+            const selected = [];
+            for (const row of rawRows) {
+                const block = Number((_a = row.blockNumber) !== null && _a !== void 0 ? _a : row.block_number);
+                if (!Number.isSafeInteger(block))
+                    throw new Error("Log returned without a valid block number");
+                if (block < start || block > end)
+                    continue;
+                if (block !== seenBlock) {
+                    seenBlock = block;
+                    seen.clear();
+                }
+                const hash = (_b = row.transactionHash) !== null && _b !== void 0 ? _b : row.transaction_hash;
+                const index = (_d = (_c = row.logIndex) !== null && _c !== void 0 ? _c : row.index) !== null && _d !== void 0 ? _d : row.log_index;
+                if (hash !== undefined && index !== undefined) {
+                    const key = `${String(hash).toLowerCase()}:${index}`;
+                    if (seen.has(key))
+                        continue;
+                    seen.add(key);
+                }
+                if (remainingOffset > 0) {
+                    remainingOffset--;
+                    continue;
+                }
+                if (remainingLimit <= 0)
+                    break;
+                selected.push(clientStreaming && !isIndexer ? { ...row } : row);
+                remainingLimit--;
+            }
+            if (selected.length) {
+                const transform = isIndexer ? transformLog : rpcTransform;
+                const startTime = Date.now();
+                const decoded = clientStreaming
+                    ? await decodeStreamingLogs(selected, transform, viemFastPath !== null && viemFastPath !== void 0 ? viemFastPath : transform.batch, options.onlyArgs)
+                    : await Promise.all(selected.map(row => transform(row)));
+                onDecodeStats === null || onDecodeStats === void 0 ? void 0 : onDecodeStats({ batchSize: selected.length, decodeTime: Date.now() - startTime, itemsDecoded: selected.length });
+                if (processor)
+                    await processor(decoded);
+                if (options.collect !== false) {
+                    if (splitByAddress)
+                        selected.forEach((row, i) => {
+                            var _a, _b, _c;
+                            for (const bucket of (_c = targetBuckets.get((_b = ((_a = row.source) !== null && _a !== void 0 ? _a : row.address)) === null || _b === void 0 ? void 0 : _b.toLowerCase())) !== null && _c !== void 0 ? _c : [])
+                                collected[bucket].push(decoded[i]);
+                        });
+                    else
+                        safePush(collected, decoded);
+                }
+            }
+            if (remainingLimit <= 0)
+                throw finished;
+        }
+        const rawOptions = {
+            ...options, chain, topic, topics, target: undefined, targets: uniqueTargets,
+            eventAbi: undefined, parseLog: false, entireLog: true, onlyArgs: false,
+            flatten: true, onDecodeStats: undefined,
+        };
+        try {
+            if (breakBlock >= fromBlock) {
+                await getLogs({
+                    ...rawOptions, fromBlock, toBlock: breakBlock, all: true, offset: 0, collect: false,
+                    processor: rows => consume(rows, true, fromBlock, breakBlock),
+                });
+            }
+            // RPC cache reads may extend the physical fetch by a buffer, but logs.ts
+            // filters to these exact bounds before any decoding or processing here.
+            for (let start = breakBlock + 1; start <= toBlock; start += effectiveMaxBlockRange) {
+                const end = Math.min(toBlock, start + effectiveMaxBlockRange - 1);
+                const rows = await (0, logs_1.getLogs)({
+                    ...rawOptions, fromBlock: start, toBlock: end, skipIndexer: true,
+                    maxBlockRange: undefined, processor: undefined,
+                });
+                rows.sort((a, b) => { var _a, _b; return a.blockNumber - b.blockNumber || ((_a = a.logIndex) !== null && _a !== void 0 ? _a : a.index) - ((_b = b.logIndex) !== null && _b !== void 0 ? _b : b.index); });
+                await consume(rows, false, start, end);
+            }
+        }
+        catch (error) {
+            if (error !== finished)
+                throw error;
+        }
+        return collected;
+    }
 }
 async function getTokenTransfers({ chain = "ethereum", fromAddressFilter, fromBlock, toBlock, all = true, limit = 1000, offset = 0, target, targets = [], flatten = true, fromTimestamp, toTimestamp, debugMode = false, transferType = "in", token, tokens, }) {
     var _a, _b;

package/build/util/logs.d.ts

@@ -1,6 +1,7 @@
 import { EventLog, Interface } from "ethers";
 import { Address } from "../types";
 import { IndexerGetLogsOptions } from "./indexer";
+export declare function getBufferedToBlock(toBlock: number, head?: number, buffer?: number): number;
 export type GetLogsOptions = {
     target?: Address;
     topic?: string;
@@ -30,6 +31,8 @@ export type GetLogsOptions = {
     allowParseFailure?: boolean;
 };
 export declare function getLogs(options: GetLogsOptions): Promise<EventLog[] | EventLog[][] | any[]>;
+export declare function maxLogsForStorage(bytesPerLog: number, cap?: number, budgetMB?: number): number;
+export declare function trimCachesForStorage(caches: logCache[], maxLogs?: number): logCache[];
 export declare function toFilterTopic(topic: string): string;
 export type logCache = {
     logs: EventLog[];

package/build/util/logs.js

@@ -3,7 +3,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.getBufferedToBlock = getBufferedToBlock;
 exports.getLogs = getLogs;
+exports.maxLogsForStorage = maxLogsForStorage;
+exports.trimCachesForStorage = trimCachesForStorage;
 exports.toFilterTopic = toFilterTopic;
 exports.getLogParams = getLogParams;
 const ethers_1 = require("ethers");
@@ -17,6 +20,15 @@ const env_1 = require("./env");
 const common_1 = require("./common");
 const logs_decode_shared_1 = require("./logs.decode.shared");
 const currentVersion = "v3";
+const BLOCK_BUFFER = 10;
+// widen the fetched range forward by BLOCK_BUFFER, without asking for blocks the chain does not
+// have yet and without ever returning less than the caller asked for
+function getBufferedToBlock(toBlock, head, buffer = BLOCK_BUFFER) {
+    const buffered = toBlock + buffer;
+    if (head === undefined)
+        return buffered;
+    return Math.min(buffered, Math.max(toBlock, head));
+}
 /* -------------------------------------------------------------------------- */
 /*                                   getLogs                                  */
 /* -------------------------------------------------------------------------- */
@@ -133,8 +145,16 @@ async function getLogs(options) {
             (0, debugLog_1.debugLog)("adding logs to cache:", fromBlock, toBlock);
         if (fromBlock > toBlock)
             return;
-        fromBlock = Math.max(fromBlock - 10, 0); // avoid negative block
-        toBlock = toBlock + 10;
+        fromBlock = Math.max(fromBlock - BLOCK_BUFFER, 0); // avoid negative block
+        // the forward buffer must not run past the chain head. Several RPCs reject a range that ends
+        // beyond their latest block outright, and since every RPC in the pool rejects it the whole call
+        // fails. If the head cannot be read (halted chain, provider error) keep the previous behaviour.
+        let head;
+        try {
+            head = await (0, blocks_1.getBlockNumber)(chain);
+        }
+        catch (e) { }
+        toBlock = getBufferedToBlock(toBlock, head);
         if (debugMode)
             console.time(dbgKey);
         const { output: newLogs } = await (0, _1.getLogs)({
@@ -156,8 +176,8 @@ async function getLogs(options) {
         const merged = [caches[0]];
         caches.slice(1).forEach((c) => {
             const last = merged[merged.length - 1];
-            if (last.metadata.toBlock + 1 > c.metadata.fromBlock) {
-                last.metadata.toBlock = c.metadata.toBlock;
+            if (last.metadata.toBlock + 1 >= c.metadata.fromBlock) {
+                last.metadata.toBlock = Math.max(last.metadata.toBlock, c.metadata.toBlock);
                 last.logs = dedupLogs(last.logs, c.logs);
             }
             else
@@ -174,28 +194,9 @@ async function getLogs(options) {
             }, 0);
             const fileSizeInMB = (totalLogCount * randomLogLength) / (1024 * 1024);
             if (fileSizeInMB > 300) { // if cache size is larger than ~300MB, trim it down
-                console.log(`getLogs: large cache size detected ${totalLogCount} logs size: ${fileSizeInMB}MB, retaining only the latest 200k logs to limit memory usage target=${target} topic=${topic} fromBlock=${fromBlock} toBlock=${toBlock}`);
-                // retain only the latest logs that fit within 300,000 logs
-                if (caches.length > 1) {
-                    let cacheWithLatestFromBlock = caches[0];
-                    for (const c of caches) {
-                        if (c.metadata.fromBlock > cacheWithLatestFromBlock.metadata.fromBlock) {
-                            cacheWithLatestFromBlock = c;
-                        }
-                    }
-                    storeCaches = [cacheWithLatestFromBlock];
-                }
-                storeCaches = storeCaches.map(c => {
-                    const clone = { logs: [], metadata: JSON.parse(JSON.stringify(c.metadata)) };
-                    if (c.logs.length > 200000) {
-                        // sort the logs by block number descending and keep only the latest 200k logs
-                        c.logs.sort((a, b) => b.blockNumber - a.blockNumber);
-                        clone.logs = c.logs.slice(0, 200000);
-                        const lastLog = clone.logs[clone.logs.length - 1];
-                        clone.metadata.fromBlock = lastLog.blockNumber;
-                    }
-                    return clone;
-                });
+                const maxLogs = maxLogsForStorage(randomLogLength);
+                console.log(`getLogs: large cache size detected ${totalLogCount} logs size: ${fileSizeInMB}MB, retaining only the latest ${maxLogs} logs to limit memory usage target=${target} topic=${topic} fromBlock=${fromBlock} toBlock=${toBlock}`);
+                storeCaches = trimCachesForStorage(storeCaches, maxLogs);
             }
             // we are skipping compression by default, so reads & writes are faster
             await (0, cache_1.writeCache)(getFile(), { caches: storeCaches, version: currentVersion }, { skipR2CacheWrite: !cacheInCloud, skipCompression: !cacheInCloud });
@@ -246,6 +247,36 @@ async function getLogs(options) {
 /* -------------------------------------------------------------------------- */
 /*                           Utility helpers exported                         */
 /* -------------------------------------------------------------------------- */
+// the log count that fits the same byte budget the size check uses, so a wide log trims before the flat cap
+function maxLogsForStorage(bytesPerLog, cap = 200000, budgetMB = 300) {
+    if (!bytesPerLog || bytesPerLog < 1)
+        return cap;
+    return Math.max(1, Math.min(cap, Math.floor((budgetMB * 1024 * 1024) / bytesPerLog)));
+}
+// keeps the newest range and at most maxLogs of its logs, narrowing the metadata to match what is kept
+function trimCachesForStorage(caches, maxLogs = 200000) {
+    if (!caches.length)
+        return caches;
+    let trimmed = caches;
+    if (caches.length > 1) {
+        let cacheWithLatestFromBlock = caches[0];
+        for (const c of caches) {
+            if (c.metadata.fromBlock > cacheWithLatestFromBlock.metadata.fromBlock) {
+                cacheWithLatestFromBlock = c;
+            }
+        }
+        trimmed = [cacheWithLatestFromBlock];
+    }
+    return trimmed.map(c => {
+        const clone = { logs: c.logs, metadata: JSON.parse(JSON.stringify(c.metadata)) };
+        if (c.logs.length > maxLogs) {
+            const sorted = [...c.logs].sort((a, b) => b.blockNumber - a.blockNumber);
+            clone.logs = sorted.slice(0, maxLogs);
+            clone.metadata.fromBlock = clone.logs[clone.logs.length - 1].blockNumber;
+        }
+        return clone;
+    });
+}
 function toFilterTopic(topic) {
     if (topic.startsWith("0x"))
         return topic;

package/build/util/updateProviderList.js

@@ -259,7 +259,8 @@ async function filterForWorkingRPCs(rpc, chain, chainId) {
                 if (data.result) {
                     const returnedChainId = parseInt(data.result, 16);
                     if (returnedChainId !== chainId) {
-                        console.log(`RPC ${i} for ${chain} returned invalid chainId ${returnedChainId}, expected ${chainId}`);
+                        console.log(`RPC ${i} for ${chain} returned invalid chainId ${returnedChainId}, expected ${chainId}, skipping`);
+                        return;
                     }
                     filteredRPCs.push(i);
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.236",
+  "version": "5.0.238",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {
@@ -11,8 +11,8 @@
     "test": "npm run load-providers && TEST_MODE=true LLAMA_DEBUG_MODE='' SDK_DEBUG='' LLAMA_DEBUG_LEVEL2=''  LLAMA_DEBUG_LEVEL3='' jest",
     "test-only": "echo 'skipped building providers.json' && TEST_MODE=true LLAMA_DEBUG_MODE='' SDK_DEBUG='' LLAMA_DEBUG_LEVEL2=''  LLAMA_DEBUG_LEVEL3='' jest",
     "test-debug": "TEST_MODE=true JEST_VERBOSE=true jest",
-    "test-indexer": "jest --runInBand --runTestsByPath src/util/indexer.test.ts src/util/indexer.e2e.test.ts src/util/indexerSql.test.ts src/util/indexerSql.parity.test.ts",
-    "test-indexer-unit": "jest --runInBand --runTestsByPath src/util/indexer.test.ts src/util/indexerSql.test.ts",
+    "test-indexer": "jest --runInBand --runTestsByPath src/util/indexer.test.ts src/util/indexer.partialSync.test.ts src/util/indexer.partialSync.e2e.test.ts src/util/indexer.e2e.test.ts src/util/indexerSql.test.ts src/util/indexerSql.parity.test.ts",
+    "test-indexer-unit": "jest --runInBand --runTestsByPath src/util/indexer.test.ts src/util/indexer.partialSync.test.ts src/util/indexerSql.test.ts",
     "update-providers": "ts-node src/util/updateProviderList.ts",
     "update-chain-labels": "ts-node src/util/updateChainLabels.ts",
     "remove-rpcs": "ts-node src/util/removeRedundantRPCs.ts",