@defillama/sdk 5.0.221 → 5.0.222

package/build/util/env.js

@@ -19,7 +19,8 @@ exports.shouldSkipBlockValidation = shouldSkipBlockValidation;
 exports.getBoolEnvValue = getBoolEnvValue;
 exports.whitelistedEnvConstants = [
     'R2_ENDPOINT', 'R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY', 'TRON_PRO_API_KEY', 'COINS_API_KEY', 'INTERNAL_API_KEY',
-    'ELASTICSEARCH_CONFIG', 'GRAPH_API_KEY', 'LLAMA_INDEXER_V2_ENDPOINT', 'LLAMA_INDEXER_V2_API_KEY', 'LLAMA_INDEXER_TIMEOUT_MS'
+    'ELASTICSEARCH_CONFIG', 'GRAPH_API_KEY', 'LLAMA_INDEXER_V2_ENDPOINT', 'LLAMA_INDEXER_V2_API_KEY', 'LLAMA_INDEXER_TIMEOUT_MS',
+    'LLAMA_INDEXER_V4_ENDPOINT', 'LLAMA_INDEXER_V4_API_KEY', 'LLAMA_INDEXER_PREFER_V4', 'LLAMA_INDEXER_V4_ONLY_CHAINS',
 ];
 const isDebugLevel3 = process.env.LLAMA_SDK_DEBUG_LEVEL_3 === 'true';
 const defaultEnvValues = {

package/build/util/indexer.d.ts

@@ -1,5 +1,7 @@
 import { Address } from "../types";
 import { GetTransactionOptions } from "./transactions";
+export type IndexerVersion = "v2" | "v4";
+export declare function getChainIndexerVersion(chain: string): IndexerVersion;
 export declare const supportedChainSet2: Set<string>;
 export type IndexerGetLogsOptions = {
     target?: Address;

package/build/util/indexer.js

@@ -2,8 +2,10 @@
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
+var _a, _b;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.supportedChainSet2 = void 0;
+exports.getChainIndexerVersion = getChainIndexerVersion;
 exports.getLogs = getLogs;
 exports.getTokenTransfers = getTokenTransfers;
 exports.getTransactions = getTransactions;
@@ -22,8 +24,12 @@ const debugLog_1 = require("./debugLog");
 const env_1 = require("./env");
 const logs_1 = require("./logs");
 const logs_decode_shared_1 = require("./logs.decode.shared");
-const LLAMA_INDEXER_ENDPOINT = (0, env_1.getEnvValue)("LLAMA_INDEXER_V2_ENDPOINT");
-const LLAMA_INDEXER_API_KEY = (0, env_1.getEnvValue)("LLAMA_INDEXER_V2_API_KEY");
+const LLAMA_INDEXER_V2_ENDPOINT = (0, env_1.getEnvValue)("LLAMA_INDEXER_V2_ENDPOINT");
+const LLAMA_INDEXER_V2_API_KEY = (0, env_1.getEnvValue)("LLAMA_INDEXER_V2_API_KEY");
+const LLAMA_INDEXER_V4_ENDPOINT = (0, env_1.getEnvValue)("LLAMA_INDEXER_V4_ENDPOINT");
+const LLAMA_INDEXER_V4_API_KEY = (_a = (0, env_1.getEnvValue)("LLAMA_INDEXER_V4_API_KEY")) !== null && _a !== void 0 ? _a : LLAMA_INDEXER_V2_API_KEY;
+// route every supported chain through v4 (v2 stays available as explicit config)
+const LLAMA_INDEXER_PREFER_V4 = (0, env_1.getEnvValue)("LLAMA_INDEXER_PREFER_V4") === "true";
 const addressChunkSize = +(0, env_1.getEnvValue)("LLAMA_INDEXER_ADDRESS_CHUNK_SIZE") || 100;
 const INDEXER_REQUEST_TIMEOUT_MS = +(0, env_1.getEnvValue)("LLAMA_INDEXER_TIMEOUT_MS");
 const indexerChainIdChainMapping = {
@@ -52,43 +58,88 @@ const indexerChainIdChainMapping = {
     4326: "megaeth",
     196: "xlayer"
 };
+const v4OnlyChainIdChainMapping = {
+    4663: "robinhood",
+};
+((_b = (0, env_1.getEnvValue)("LLAMA_INDEXER_V4_ONLY_CHAINS")) !== null && _b !== void 0 ? _b : "")
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter(Boolean)
+    .forEach((entry) => {
+    const [chainId, chainName] = entry.split(":").map((i) => i.trim());
+    if (!chainId || !chainName || isNaN(+chainId))
+        throw new Error(`Invalid LLAMA_INDEXER_V4_ONLY_CHAINS entry: ${entry}`);
+    v4OnlyChainIdChainMapping[+chainId] = chainName;
+});
+const allChainIdChainMapping = { ...indexerChainIdChainMapping, ...v4OnlyChainIdChainMapping };
+const v4OnlyChainSet = new Set(Object.values(v4OnlyChainIdChainMapping));
+const indexerConfigs = {
+    v2: { endpoint: LLAMA_INDEXER_V2_ENDPOINT, apiKey: LLAMA_INDEXER_V2_API_KEY },
+    v4: { endpoint: LLAMA_INDEXER_V4_ENDPOINT, apiKey: LLAMA_INDEXER_V4_API_KEY },
+};
 // http agents for streaming (keep-alive)
 const httpAgent = new http_1.default.Agent({ keepAlive: true, maxSockets: 256, maxFreeSockets: 64 });
 const httpsAgent = new https_1.default.Agent({ keepAlive: true, maxSockets: 256, maxFreeSockets: 64 });
-const axiosInstance = axios_1.default.create({
-    headers: { "x-api-key": LLAMA_INDEXER_API_KEY },
-    baseURL: LLAMA_INDEXER_ENDPOINT,
-    httpAgent,
-    httpsAgent,
-    timeout: INDEXER_REQUEST_TIMEOUT_MS,
-});
-function checkIndexerConfig() {
-    if (!LLAMA_INDEXER_ENDPOINT || !LLAMA_INDEXER_API_KEY)
-        throw new Error(`Llama Indexer URL/api key is not set`);
+const axiosInstances = {
+    v2: axios_1.default.create({
+        headers: { "x-api-key": LLAMA_INDEXER_V2_API_KEY },
+        baseURL: LLAMA_INDEXER_V2_ENDPOINT,
+        httpAgent,
+        httpsAgent,
+        timeout: INDEXER_REQUEST_TIMEOUT_MS,
+    }),
+    v4: axios_1.default.create({
+        headers: { "x-api-key": LLAMA_INDEXER_V4_API_KEY },
+        baseURL: LLAMA_INDEXER_V4_ENDPOINT,
+        httpAgent,
+        httpsAgent,
+        timeout: INDEXER_REQUEST_TIMEOUT_MS,
+    }),
+};
+function checkIndexerConfig(version) {
+    const { endpoint, apiKey } = indexerConfigs[version];
+    if (!endpoint || !apiKey)
+        throw new Error(`Llama Indexer (${version}) URL/api key is not set`);
+}
+// pick which indexer serves a given chain:
+// - v4-only chains (not indexed on v2) always go to v4
+// - everything else goes to v2, unless LLAMA_INDEXER_PREFER_V4 is set or v2 is not configured
+function getChainIndexerVersion(chain) {
+    if (v4OnlyChainSet.has(chain))
+        return "v4";
+    if (!LLAMA_INDEXER_V4_ENDPOINT)
+        return "v2";
+    if (LLAMA_INDEXER_PREFER_V4 || !LLAMA_INDEXER_V2_ENDPOINT)
+        return "v4";
+    return "v2";
 }
 function getChainId(chain) {
     var _a;
-    const chainId = (_a = Object.entries(indexerChainIdChainMapping).find(([, chainName]) => chainName === chain)) === null || _a === void 0 ? void 0 : _a[0];
+    const chainId = (_a = Object.entries(allChainIdChainMapping).find(([, chainName]) => chainName === chain)) === null || _a === void 0 ? void 0 : _a[0];
     if (!chainId)
         throw new Error("Chain not supported");
     return +chainId;
 }
 function getSupportedChains() {
-    return new Set(Object.values(indexerChainIdChainMapping));
+    return new Set(Object.values(allChainIdChainMapping));
 }
 exports.supportedChainSet2 = getSupportedChains();
-const state = { chainIndexStatus: {} };
+const syncStates = {
+    v2: { chainIndexStatus: {} },
+    v4: { chainIndexStatus: {} },
+};
 const cacheTime = 1 * 60 * 1000; // 1 min
-async function getChainIndexStatus() {
-    checkIndexerConfig();
+async function getChainIndexStatus(version) {
+    checkIndexerConfig(version);
+    const state = syncStates[version];
     if (state.timestamp && Date.now() - state.timestamp < cacheTime)
         return state.chainIndexStatus;
     state.timestamp = Date.now();
     state.chainIndexStatus = (async () => {
-        const { data: { syncStatus }, } = await axiosInstance.get(`/sync`).catch((e) => { throw (0, common_1.formError)(e); });
+        const { data: { syncStatus }, } = await axiosInstances[version].get(`/sync`).catch((e) => { throw (0, common_1.formError)(e); });
         const info = {};
         syncStatus.forEach((d) => {
-            const chain = indexerChainIdChainMapping[d.chain];
+            const chain = allChainIdChainMapping[d.chain];
             if (chain) {
                 info[chain] = {
                     block: d.lastIndexedBlock,
@@ -104,11 +155,11 @@ async function getChainIndexStatus() {
 // Streaming support (disabled by default, opt-in via clientStreaming: true)
 // ---------------------------------------------------------------------------
 async function streamJsonArrayFromIndexer(opts) {
-    const { path, arrayKey = "logs", onItem, onChunkStats, shouldStop } = opts;
+    const { path, version, arrayKey = "logs", onItem, onChunkStats, shouldStop } = opts;
     const controller = new AbortController();
     let bytesReceived = 0;
     let itemsProcessed = 0;
-    const res = await axiosInstance.get(path, {
+    const res = await axiosInstances[version].get(path, {
         responseType: "stream",
         timeout: 0,
         maxContentLength: Infinity,
@@ -198,9 +249,10 @@ async function getLogs(options) {
         throw new Error("When noTarget is true, block range must be less than 500k blocks. Please narrow down your block range.");
     }
     // Ensure the indexer is synced far enough
-    checkIndexerConfig();
+    const indexerVersion = getChainIndexerVersion(chain);
+    checkIndexerConfig(indexerVersion);
     const chainId = getChainId(chain);
-    const chainIndexStatus = await getChainIndexStatus();
+    const chainIndexStatus = await getChainIndexStatus(indexerVersion);
     const lastIndexedBlock = (_b = (_a = chainIndexStatus[chain]) === null || _a === void 0 ? void 0 : _a.block) !== null && _b !== void 0 ? _b : 0;
     if (lastIndexedBlock < toBlock) {
         const percentageMissing = ((toBlock - lastIndexedBlock) / (toBlock - fromBlock)) * 100;
@@ -261,7 +313,7 @@ async function getLogs(options) {
     // Streaming (opt-in) — NO server-side parsing; mirror legacy semantics
     // -----------------------------------------------------------------------
     if (clientStreaming) {
-        const useAll = options.all === true;
+        const useAll = all === true;
         const effectiveLimit = useAll ? Number.POSITIVE_INFINITY : ((_d = options.limit) !== null && _d !== void 0 ? _d : limit);
         const shouldLimit = !useAll && Number.isFinite(effectiveLimit);
         const effectiveOffset = useAll ? 0 : ((_f = (_e = options.offset) !== null && _e !== void 0 ? _e : initialOffset) !== null && _f !== void 0 ? _f : 0);
@@ -368,6 +420,7 @@ async function getLogs(options) {
             };
             await streamJsonArrayFromIndexer({
                 path: `/logs?${qs.toString()}`,
+                version: indexerVersion,
                 arrayKey: "logs",
                 onItem: (raw) => {
                     var _a, _b, _c;
@@ -437,7 +490,7 @@ async function getLogs(options) {
                 offset: chunkOffset,
                 noTarget,
             };
-            const { data: { logs: _logs, totalCount }, } = await axiosInstance(`/logs`, { params }).catch(e => { throw (0, common_1.formError)(e); });
+            const { data: { logs: _logs, totalCount }, } = await axiosInstances[indexerVersion](`/logs`, { params }).catch((e) => { throw (0, common_1.formError)(e); });
             const filtered = _logs.filter((l) => {
                 var _a, _b, _c;
                 const isWhitelisted = !addressSet.size || addressSet.has((_c = (_b = ((_a = l.source) !== null && _a !== void 0 ? _a : l.address)) === null || _b === void 0 ? void 0 : _b.toLowerCase) === null || _c === void 0 ? void 0 : _c.call(_b));
@@ -505,7 +558,8 @@ async function getTokenTransfers({ chain = "ethereum", fromAddressFilter, fromBl
     var _a, _b;
     if (!debugMode)
         debugMode = debugLog_1.DEBUG_LEVEL2 && !!env_1.ENV_CONSTANTS.GET_LOGS_INDEXER;
-    checkIndexerConfig();
+    const indexerVersion = getChainIndexerVersion(chain);
+    checkIndexerConfig(indexerVersion);
     const chainId = getChainId(chain);
     const fromFilterEnabled = !!(fromAddressFilter === null || fromAddressFilter === void 0 ? void 0 : fromAddressFilter.length);
     if (typeof fromAddressFilter === "string")
@@ -536,8 +590,8 @@ async function getTokenTransfers({ chain = "ethereum", fromAddressFilter, fromBl
         throw new Error("target|targets is required");
     targets = targets.map((t) => t.toLowerCase());
     const addresses = targets.join(",");
-    // Ensure v2 indexer is up-to-date
-    const chainIndexStatus = await getChainIndexStatus();
+    // Ensure the indexer is up-to-date
+    const chainIndexStatus = await getChainIndexStatus(indexerVersion);
     const lastIndexedBlock = (_b = (_a = chainIndexStatus[chain]) === null || _a === void 0 ? void 0 : _a.block) !== null && _b !== void 0 ? _b : 0;
     if (lastIndexedBlock < toBlock) {
         throw new Error(`Indexer not up to date for ${chain}. Last indexed block: ${lastIndexedBlock}, requested block: ${toBlock}`);
@@ -576,7 +630,7 @@ async function getTokenTransfers({ chain = "ethereum", fromAddressFilter, fromBl
             default:
                 throw new Error("Invalid transferType");
         }
-        const { data: { transfers: _logs }, } = await axiosInstance(`/token-transfers`, { params }).catch(e => { throw (0, common_1.formError)(e); });
+        const { data: { transfers: _logs }, } = await axiosInstances[indexerVersion](`/token-transfers`, { params }).catch((e) => { throw (0, common_1.formError)(e); });
         rawTransfers.push(..._logs);
         currentOffset += limit;
         hasMore = _logs.length === limit;
@@ -609,13 +663,14 @@ async function getTransactions({ chain = "ethereum", addresses, transaction_hash
     var _a, _b;
     if (!debugMode)
         debugMode = debugLog_1.DEBUG_LEVEL2 && !!env_1.ENV_CONSTANTS.GET_LOGS_INDEXER;
-    checkIndexerConfig();
+    const indexerVersion = getChainIndexerVersion(chain);
+    checkIndexerConfig(indexerVersion);
     const chainId = getChainId(chain);
     if ((!addresses || addresses.length === 0) && (!transaction_hashes || transaction_hashes.length === 0))
         throw new Error("You must provide at least 'addresses' or 'transaction_hashes'");
     if (!from_block || !to_block)
         throw new Error("'from_block' and 'to_block' are required to search for transactions");
-    const chainIndexStatus = await getChainIndexStatus();
+    const chainIndexStatus = await getChainIndexStatus(indexerVersion);
     const lastIndexedBlock = (_b = (_a = chainIndexStatus[chain]) === null || _a === void 0 ? void 0 : _a.block) !== null && _b !== void 0 ? _b : 0;
     if (to_block > lastIndexedBlock) {
         throw new Error(`Indexer not up to date for ${chain}. Last indexed block: ${lastIndexedBlock}, requested block: ${to_block}`);
@@ -646,7 +701,7 @@ async function getTransactions({ chain = "ethereum", addresses, transaction_hash
         (0, debugLog_1.debugLog)("[Indexer] Pulling transactions " + debugTimeKey);
         console.time(debugTimeKey);
     }
-    const { data: { transactions } } = await axiosInstance(`/transactions`, { params }).catch(e => { throw (0, common_1.formError)(e); });
+    const { data: { transactions } } = await axiosInstances[indexerVersion](`/transactions`, { params }).catch((e) => { throw (0, common_1.formError)(e); });
     if (debugMode) {
         console.timeEnd(debugTimeKey);
         (0, debugLog_1.debugLog)("Transactions pulled " + chain, addresses || transaction_hashes, (transactions === null || transactions === void 0 ? void 0 : transactions.length) || 0);
@@ -678,11 +733,13 @@ async function getTransactions({ chain = "ethereum", addresses, transaction_hash
     }));
 }
 function isIndexerEnabled(chain) {
-    if (!LLAMA_INDEXER_ENDPOINT)
+    if (!LLAMA_INDEXER_V2_ENDPOINT && !LLAMA_INDEXER_V4_ENDPOINT)
         return false;
-    if (chain && !exports.supportedChainSet2.has(chain))
+    if (!chain)
+        return true;
+    if (!exports.supportedChainSet2.has(chain))
         return false;
-    return true;
+    return !!indexerConfigs[getChainIndexerVersion(chain)].endpoint;
 }
 function isIndexer2Enabled(chain) {
     return isIndexerEnabled(chain);

package/build/util/indexerSql.d.ts

@@ -0,0 +1,2 @@
+export declare function isIndexerSqlEnabled(): boolean;
+export declare function queryClickhouse<T = any>(sql: string, params?: Record<string, unknown>, settings?: Record<string, string | number>): Promise<T[]>;

package/build/util/indexerSql.js

@@ -0,0 +1,71 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isIndexerSqlEnabled = isIndexerSqlEnabled;
+exports.queryClickhouse = queryClickhouse;
+const axios_1 = __importDefault(require("axios"));
+const http_1 = __importDefault(require("http"));
+const https_1 = __importDefault(require("https"));
+const env_1 = require("./env");
+const httpAgent = new http_1.default.Agent({ keepAlive: true, maxSockets: 64, maxFreeSockets: 16 });
+const httpsAgent = new https_1.default.Agent({ keepAlive: true, maxSockets: 64, maxFreeSockets: 16 });
+function sqlConfig() {
+    return {
+        endpoint: (0, env_1.getEnvValue)("LLAMA_INDEXER_SQL_ENDPOINT"),
+        user: (0, env_1.getEnvValue)("LLAMA_INDEXER_SQL_USER"),
+        password: (0, env_1.getEnvValue)("LLAMA_INDEXER_SQL_PASSWORD"),
+        timeoutMs: +(0, env_1.getEnvValue)("LLAMA_INDEXER_SQL_TIMEOUT_MS") || 180000,
+    };
+}
+function isIndexerSqlEnabled() {
+    const c = sqlConfig();
+    return !!(c.endpoint && c.user && c.password);
+}
+async function queryClickhouse(sql, params, settings) {
+    var _a, _b;
+    const { endpoint, user, password, timeoutMs } = sqlConfig();
+    if (!endpoint || !user || !password) {
+        throw new Error("Llama indexer SQL gateway not configured (LLAMA_INDEXER_SQL_ENDPOINT/USER/PASSWORD)");
+    }
+    const search = new URLSearchParams();
+    search.set("default_format", "JSONEachRow");
+    for (const [k, v] of Object.entries(params !== null && params !== void 0 ? params : {})) {
+        search.set(`param_${k}`, typeof v === "object" ? JSON.stringify(v) : String(v));
+    }
+    for (const [k, v] of Object.entries(settings !== null && settings !== void 0 ? settings : {})) {
+        search.set(k, String(v));
+    }
+    let res;
+    try {
+        res = await axios_1.default.post(`${endpoint}?${search.toString()}`, sql, {
+            auth: { username: user, password },
+            headers: { "Content-Type": "text/plain; charset=utf-8" },
+            responseType: "text",
+            transformResponse: (d) => d,
+            timeout: timeoutMs,
+            maxBodyLength: Infinity,
+            maxContentLength: Infinity,
+            httpAgent,
+            httpsAgent,
+        });
+    }
+    catch (e) {
+        const status = (_a = e === null || e === void 0 ? void 0 : e.response) === null || _a === void 0 ? void 0 : _a.status;
+        const body = typeof ((_b = e === null || e === void 0 ? void 0 : e.response) === null || _b === void 0 ? void 0 : _b.data) === "string" ? e.response.data.slice(0, 500) : "";
+        throw new Error(`indexer /sql query failed${status ? ` (HTTP ${status})` : ""}: ${body || (e === null || e === void 0 ? void 0 : e.message) || e}`);
+    }
+    return parseJsonEachRow(res.data);
+}
+function parseJsonEachRow(text) {
+    if (!text)
+        return [];
+    const out = [];
+    for (const line of text.split("\n")) {
+        if (!line)
+            continue;
+        out.push(JSON.parse(line));
+    }
+    return out;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.221",
+  "version": "5.0.222",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {
@@ -12,6 +12,7 @@
     "test": "npm run load-providers && TEST_MODE=true LLAMA_DEBUG_MODE='' SDK_DEBUG='' LLAMA_DEBUG_LEVEL2=''  LLAMA_DEBUG_LEVEL3='' jest",
     "test-only": "echo 'skipped building providers.json' && TEST_MODE=true LLAMA_DEBUG_MODE='' SDK_DEBUG='' LLAMA_DEBUG_LEVEL2=''  LLAMA_DEBUG_LEVEL3='' jest",
     "test-debug": "TEST_MODE=true jest",
+    "test-indexer": "TEST_MODE=true jest src/util/indexer.routing.test.ts src/util/indexer.test.ts src/util/indexer.robinhood.test.ts src/util/indexer.e2e.test.ts --runInBand",
     "update-providers": "ts-node src/util/updateProviderList.ts",
     "remove-rpcs": "ts-node src/util/removeRedundantRPCs.ts",
     "format": "prettier --write \"src/**/*.ts\"",