@defillama/sdk 5.0.240 → 5.0.244

package/build/util/updateProviderList.js

@@ -0,0 +1,275 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const providers_json_1 = __importDefault(require("../providers.json"));
+const fs_1 = __importDefault(require("fs"));
+const debugLog_1 = require("./debugLog");
+const promise_pool_1 = __importDefault(require("@supercharge/promise-pool"));
+const generalUtil_1 = require("../generalUtil");
+const chainUtils_1 = require("../util/chainUtils");
+const concurrentCheckChains = +(process.env.SDK_BUILD_CONCURRENT_CHAINS || 7);
+const chainRemovalThreshold = +(process.env.SDK_BUILD_CHAIN_REMOVAL_THRESHOLD || 20);
+const droppedTestnetChainsSet = new Set([]);
+const providerList = providers_json_1.default;
+async function getChainData() {
+    try {
+        const chainData = await (0, generalUtil_1.fetchJson)('https://chainlist.org/rpcs.json');
+        // fix for zksync era name clash
+        const eteriaListing = chainData.find((i) => i.chainId === 140);
+        if (eteriaListing)
+            eteriaListing.shortName = 'eteria';
+        return chainData;
+    }
+    catch (e) {
+        console.log('Failed to fetch chainlist.org, falling back to local copy');
+        // https://unpkg.com/@defillama/sdk@latest/build/providers.json
+        return require(__dirname + '/chainlistOrgCache.json');
+    }
+}
+async function main() {
+    await (0, chainUtils_1.updateChainLabels)();
+    const oldProviders = await (0, generalUtil_1.fetchJson)(`https://unpkg.com/@defillama/sdk@latest/build/providers.json`);
+    const currentChains = await (0, generalUtil_1.fetchJson)(`https://raw.githubusercontent.com/DefiLlama/DefiLlama-Adapters/refs/heads/main/projects/helper/chains.json`);
+    const currentChainsSet = new Set(currentChains);
+    let chainData = await getChainData();
+    const providerIDMap = {};
+    const chainIdToKeyMap = {};
+    Object.entries(providerList).forEach(([key, i]) => {
+        providerIDMap[i.chainId] = i.rpc;
+        chainIdToKeyMap[i.chainId] = key;
+    });
+    const shortNameToKeyMap = {};
+    Object.entries(chainShortNameMapping).forEach(([key, shortName]) => shortNameToKeyMap[shortName] = key);
+    chainData = chainData
+        .filter((i) => i.rpc.length)
+        // .filter((i: any) => !i.status || (i.status === 'active' || i.status === 'incubating'))
+        .filter((i) => i.shortName)
+        .filter((i) => {
+        var _a, _b;
+        if (currentChainsSet.has(i.shortName))
+            return true;
+        const bannedKeys = ['sepolia', 'test', 'goerli', 'devnet', 'holesky',];
+        const bannedRPCKeys = ['sepolia', 'goerli', 'devnet', 'holesky'];
+        const isTestnetKey = bannedKeys.some((j) => i.shortName.includes(j));
+        const hasTestnetRPC = i.rpc.some((rpc) => bannedRPCKeys.some((k) => rpc === null || rpc === void 0 ? void 0 : rpc.url.includes(k)));
+        const hasTestnetName = ((_a = i.name) === null || _a === void 0 ? void 0 : _a.toLowerCase().includes('testnet')) || ((_b = i.name) === null || _b === void 0 ? void 0 : _b.toLowerCase().includes('devnet'));
+        if (isTestnetKey || hasTestnetRPC || hasTestnetName) {
+            droppedTestnetChainsSet.add(i.shortName);
+            // console.log(`Excluding testnet chain ${i.name} (${i.shortName})`)
+        }
+        return !isTestnetKey && !hasTestnetRPC && !hasTestnetName;
+    });
+    // trim/remove mainnet|-mainnet|_mainnet from short names, preliminary rpc filtering
+    const rpcFilterRegex = /(wss\:|ws\:|terminet\.io|huobichain\.com|getblock\.io|bitstack\.com|nodereal.io\/v1|pokt\.network|histori\.xyz|chainstacklabs\.com|owlracle\.info|blastapi\.io)/;
+    chainData.forEach((i) => {
+        if (currentChainsSet.has(i.shortName)) {
+            // keep the short name as is
+        }
+        if (i.shortName.endsWith('-mainnet'))
+            i.shortName = i.shortName.slice(0, -8);
+        else if (i.shortName.endsWith('_mainnet'))
+            i.shortName = i.shortName.slice(0, -8);
+        else if (i.shortName.endsWith('mainnet'))
+            i.shortName = i.shortName.slice(0, -7);
+        i.rpc = i.rpc.filter((j) => {
+            if (!j.url)
+                return false;
+            if (rpcFilterRegex.test(j.url)) {
+                // console.log(`Removing blacklisted rpc ${j.url} from ${i.name}`)
+                return false; // remove anything with blacklisted words
+            }
+            return true;
+        });
+    });
+    // shuffle the array
+    chainData.sort(() => Math.random() - 0.5);
+    // chain keys this chain could end up under in providers.json
+    const getCandidateKeys = (i) => {
+        const keys = [i.shortName.toLowerCase().replace(/-/g, '_')];
+        if (chainIdToKeyMap[i.chainId])
+            keys.push(chainIdToKeyMap[i.chainId]);
+        if (shortNameToKeyMap[i.shortName])
+            keys.push(shortNameToKeyMap[i.shortName]);
+        return keys;
+    };
+    // only check rpc health for chains we track and that are not dead, the rest are only filtered statically
+    const shouldCheckRPCs = (i) => {
+        const keys = getCandidateKeys(i);
+        const isTracked = keys.some((k) => currentChainsSet.has(k));
+        const isDead = keys.some((k) => (0, chainUtils_1.isDeadChain)(k));
+        return isTracked && !isDead;
+    };
+    const skippedRPCChecks = { dead: 0, untracked: 0 };
+    await promise_pool_1.default
+        .withConcurrency(concurrentCheckChains)
+        .for(chainData)
+        .process(async (i) => {
+        var _a, _b;
+        const rpcs = i.rpc.map((j) => j.url);
+        if (shouldCheckRPCs(i)) {
+            i.rpc = await filterForWorkingRPCs(rpcs, i.name, i.chainId);
+        }
+        else {
+            if (getCandidateKeys(i).some((k) => (0, chainUtils_1.isDeadChain)(k)))
+                skippedRPCChecks.dead++;
+            else
+                skippedRPCChecks.untracked++;
+            i.rpc = rpcs.length < 4 ? rpcs : filterRPCs(rpcs);
+        }
+        if (!i.rpc.length)
+            return;
+        if (providerIDMap[i.chainId]) {
+            const isBEVM = i.chainId + '' === '11501'; // bevm chain id clashes with 
+            if (!isBEVM)
+                providerIDMap[i.chainId].push(...i.rpc);
+        }
+        else if (providerList[i.shortName.toLowerCase()]) {
+            (0, debugLog_1.debugLog)(`Duplicate short name ${i.chainId} for ${i.shortName}, doing nothing`);
+        }
+        else {
+            const label = i.shortName.toLowerCase().replace(/-/g, '_');
+            providerList[label] = {
+                rpc: i.rpc,
+                chainId: i.chainId
+            };
+            let explorer = (_b = (_a = i.explorers) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.url;
+            if (explorer)
+                providerList[label].explorer = explorer;
+        }
+    });
+    Object.entries(providerList).forEach(([key, i]) => {
+        i.rpc.forEach((j) => {
+            if (filterRPCs([j]).length === 0) {
+                (0, debugLog_1.debugLog)(`Removing rpc ${j} from ${key}`);
+            }
+        });
+        i.rpc = Array.from(new Set(filterRPCs(i.rpc)));
+    });
+    Object.keys(providerList).forEach((i) => {
+        if (!providerList[i].rpc.length)
+            delete providerList[i];
+    });
+    for (const [key, shorName] of Object.entries(chainShortNameMapping)) {
+        if (providerList[key])
+            continue;
+        providerList[key] = providerList[shorName];
+    }
+    // add from the old providers any missing rpcs that is in the tvl chain list
+    Object.entries(oldProviders).forEach(([key, i]) => {
+        if (!currentChainsSet.has(key))
+            return;
+        if (providerList[key])
+            return;
+        console.log(`Adding missing chain ${key} from old providers`);
+        providerList[key] = i;
+    });
+    const droppedChains = Object.keys(oldProviders).filter(oldChain => providerList[oldChain] === undefined);
+    const filteredDroppedChains = droppedChains.filter(i => !droppedTestnetChainsSet.has(i) && !i.includes('test'));
+    console.log('Dropped chains before testnet filtering:', droppedChains.length);
+    console.log('Dropped chains after testnet filtering:', filteredDroppedChains.length);
+    if (filteredDroppedChains.length > chainRemovalThreshold) {
+        throw new Error(`Following chains used to be included but is not anymore, can the devs fix please?\n${filteredDroppedChains.join(', ')}`);
+    }
+    delete providerList.bitcoin; // what were they smoking?
+    const rpcCount = Object.values(providerList).reduce((acc, i) => { var _a; return acc + ((_a = i === null || i === void 0 ? void 0 : i.rpc.length) !== null && _a !== void 0 ? _a : 0); }, 0);
+    const rpcCountOld = Object.values(oldProviders).reduce((acc, i) => { var _a; return acc + ((_a = i === null || i === void 0 ? void 0 : i.rpc.length) !== null && _a !== void 0 ? _a : 0); }, 0);
+    const newlyAddedChains = Object.keys(providerList).filter(chain => oldProviders[chain] === undefined);
+    console.log('Final provider list:');
+    console.log('Skipped rpc checks (dead chains):', skippedRPCChecks.dead);
+    console.log('Skipped rpc checks (untracked chains):', skippedRPCChecks.untracked);
+    console.log('Chain count:', Object.keys(providerList).length);
+    console.log('Newly added chains count:', newlyAddedChains.length);
+    console.log('Dropped chain count:', filteredDroppedChains.length);
+    console.log('RPC count:', rpcCount);
+    console.log('RPC count current:', rpcCountOld);
+    console.log('Dropped chains:', filteredDroppedChains.join(', '));
+    console.log('Newly added chains:', newlyAddedChains.join(', '));
+    fs_1.default.writeFileSync(__dirname + '/../providers.json', JSON.stringify(providerList));
+}
+const blacklist = [
+    'https://eth.api.onfinality.io/public'
+];
+// this is stricter filtering used only when chain has more than 10 rpcs
+function filterRPCs(rpc) {
+    return rpc.filter((i) => {
+        if (blacklist.includes(i))
+            return false; // remove rpcs returning bad block heights etc
+        if (i.endsWith('/demo'))
+            return false; // remove demo rpc
+        if (!i.startsWith('http'))
+            return false; // remove demo rpc
+        if (i.includes('$'))
+            return false; // remove anything where api key is injected
+        // reject websocket, http, testnet, devnet, and anything with '='
+        if (/(wss\:|ws\:|http\:|test|devnet|terminet|huobichain|getblock|bitstack|nodereal.io\/v1|pokt\.network|ankr\.com|histori\.xyz|chainstacklabs|owlracle|blastapi|routeme|\=)/.test(i))
+            return false; // remove anything with blacklisted words
+        return true;
+    }).map((i) => {
+        if (i.endsWith('/'))
+            return i.slice(0, -1);
+        return i;
+    });
+}
+const chainShortNameMapping = {
+    real: 're-al',
+    karura: 'karura_evm',
+    taiko: 'tko-mainnet',
+    bsquared: 'b2-mainnet',
+    ox_chain: 'ox-chain',
+    ailayer: 'ailayer-mainnet',
+    ontology: 'OntologyMainnet',
+    occ: 'edu_chain',
+    hyperliquid: 'hyper_evm',
+    hydradx: 'hdx',
+    nibiru: 'cataclysm_1',
+    fraxtal: 'frax',
+    elsm: 'ely',
+    tac: 'tacchain',
+    xone: 'xoc',
+    wchain: 'wcn',
+    hydragon: 'hydra',
+    camp: 'campmainnet',
+    mantra: 'mantrachain',
+    saga: 'sagaevm',
+    robinhood: 'robinhoodchain',
+    bcypher: 'bchyper',
+    area: 'areum',
+};
+async function filterForWorkingRPCs(rpc, chain, chainId) {
+    if (rpc.length < 4)
+        return rpc;
+    rpc = filterRPCs(rpc);
+    const filteredRPCs = [];
+    await (0, generalUtil_1.runInPromisePool)({
+        items: rpc,
+        concurrency: concurrentCheckChains,
+        permitFailure: true,
+        processor: async (i) => {
+            try {
+                const data = await (0, generalUtil_1.postJson)(i, {
+                    jsonrpc: '2.0',
+                    method: 'eth_chainId',
+                    params: [],
+                    id: 1
+                }, {
+                    timeout: 60000
+                });
+                if (data.result) {
+                    const returnedChainId = parseInt(data.result, 16);
+                    if (returnedChainId !== chainId) {
+                        console.log(`RPC ${i} for ${chain} returned invalid chainId ${returnedChainId}, expected ${chainId}, skipping`);
+                        return;
+                    }
+                    filteredRPCs.push(i);
+                }
+            }
+            catch (e) {
+                console.log('ignoring bad rpc', chain, e === null || e === void 0 ? void 0 : e.message);
+            }
+        }
+    });
+    return filteredRPCs;
+}
+main();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "5.0.240",
+  "version": "5.0.244",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {
@@ -16,7 +16,8 @@
     "update-providers": "ts-node src/util/updateProviderList.ts",
     "update-chain-labels": "ts-node src/util/updateChainLabels.ts",
     "remove-rpcs": "ts-node src/util/removeRedundantRPCs.ts",
-    "prepublishOnly": "npm run build"
+    "verify-package": "node scripts/verifyPackage.js",
+    "prepublishOnly": "npm run build && npm run verify-package"
   },
   "repository": {
     "type": "git",