@defillama/sdk 5.0.240 → 5.0.244

package/build/util/callStats.js

@@ -0,0 +1,385 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _a;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_MIN_GET_LOGS = exports.DEFAULT_MIN_CALLS = exports.ES_INDEX = void 0;
+exports.registerEndpoints = registerEndpoints;
+exports.resolveChain = resolveChain;
+exports.ignoreHost = ignoreHost;
+exports.recordMethod = recordMethod;
+exports.install = install;
+exports.getCurrent = getCurrent;
+exports.snapshot = snapshot;
+exports.mergeSnapshots = mergeSnapshots;
+exports.run = run;
+exports.getMinCalls = getMinCalls;
+exports.buildDocs = buildDocs;
+exports.report = report;
+/**
+ * Per-run counters for every outbound HTTP request a process makes, attributed to a chain
+ * and to the adapter run that made them, and flushed to Elasticsearch when the run ends.
+ *
+ * How it works
+ *  - `install()` wraps `http.request` / `http.get` / `https.request` / `https.get` and subscribes to
+ *    the `undici:request:create` diagnostics channel (global `fetch`). That catches axios, ethers,
+ *    @solana/web3.js, graphql-request, raw fetch... without touching any call site.
+ *  - The SDK registers the RPC urls it resolves for a chain (`registerEndpoints`, called from
+ *    LlamaProvider and the `chains/*` endpoint resolvers). A request whose url starts with a
+ *    registered endpoint is counted against that chain, everything else against its host only.
+ *  - `run(meta, fn)` executes `fn` inside an AsyncLocalStorage context, so concurrent adapter runs
+ *    in one process (the server's pooled children) each get their own counters.
+ *
+ * All state lives on `globalThis` under a well-known symbol: when two copies of the SDK are loaded
+ * (server's node_modules + the adapters repo's node_modules) they share one hook, one endpoint
+ * registry and one context, and nothing is counted twice.
+ */
+const async_hooks_1 = require("async_hooks");
+const http_1 = __importDefault(require("http"));
+const https_1 = __importDefault(require("https"));
+const diagnostics_channel_1 = __importDefault(require("diagnostics_channel"));
+const elastic_1 = require("./elastic");
+const debugLog_1 = require("./debugLog");
+const env_1 = require("./env");
+exports.ES_INDEX = 'adapter-call-stats';
+exports.DEFAULT_MIN_CALLS = 50;
+/** a chain bucket is also kept when it has at least this many getLogs requests, however few calls it made */
+exports.DEFAULT_MIN_GET_LOGS = 10;
+const STATE_KEY = Symbol.for('@defillama/sdk/callStats');
+function newStats(meta) {
+    return { meta, startTime: Date.now(), total: 0, byChain: {}, byHost: {}, unattributed: 0, byMethod: {} };
+}
+const shared = (_a = globalThis[STATE_KEY]) !== null && _a !== void 0 ? _a : (globalThis[STATE_KEY] = {
+    als: new async_hooks_1.AsyncLocalStorage(),
+    installed: false,
+    endpoints: new Map(),
+    global: newStats({ application: 'global', name: '-' }),
+    ignoredHosts: new Set(),
+    registered: new Set(),
+});
+// ---------------------------------------------------------------------------
+// endpoint registry
+// ---------------------------------------------------------------------------
+function parseUrl(url) {
+    try {
+        const u = new URL(url);
+        if (u.protocol !== 'http:' && u.protocol !== 'https:')
+            return;
+        return { host: u.host.toLowerCase(), path: u.pathname.replace(/\/+$/, '') };
+    }
+    catch {
+        return;
+    }
+}
+/**
+ * Remember that `endpoints` serve `chain`, so requests to them are counted against that chain.
+ * Called by the SDK wherever it resolves rpc urls; adapters can call it for endpoints they manage themselves.
+ */
+function registerEndpoints(chain, endpoints) {
+    if (!chain || !endpoints)
+        return;
+    chain = chain.toLowerCase();
+    // callers pass the same list on every request: skip the url parsing after the first time
+    const key = chain + '|' + (Array.isArray(endpoints) ? endpoints.join(',') : endpoints);
+    if (shared.registered.has(key))
+        return;
+    shared.registered.add(key);
+    const list = Array.isArray(endpoints) ? endpoints : endpoints.split(',');
+    for (const raw of list) {
+        const parsed = parseUrl(String(raw).trim());
+        if (!parsed)
+            continue;
+        let entries = shared.endpoints.get(parsed.host);
+        if (!entries) {
+            entries = [];
+            shared.endpoints.set(parsed.host, entries);
+        }
+        if (entries.some(e => e.path === parsed.path && e.chain === chain))
+            continue;
+        entries.push({ path: parsed.path, chain });
+        entries.sort((a, b) => b.path.length - a.path.length);
+    }
+}
+/** Chain a url is counted against, or undefined when it matches no registered endpoint. */
+function resolveChain(url) {
+    const parsed = parseUrl(url);
+    if (!parsed)
+        return;
+    const entries = shared.endpoints.get(parsed.host);
+    if (!entries)
+        return;
+    for (const entry of entries) {
+        if (!entry.path || parsed.path === entry.path || parsed.path.startsWith(entry.path + '/'))
+            return entry.chain;
+    }
+}
+/** Requests to these hosts are not counted (eg. the elasticsearch cluster the stats are written to). */
+function ignoreHost(url) {
+    const parsed = parseUrl(url);
+    if (parsed)
+        shared.ignoredHosts.add(parsed.host);
+}
+// ---------------------------------------------------------------------------
+// recording
+// ---------------------------------------------------------------------------
+function record(url) {
+    var _a, _b, _c;
+    const parsed = parseUrl(url);
+    if (!parsed || shared.ignoredHosts.has(parsed.host))
+        return;
+    const stats = (_a = shared.als.getStore()) !== null && _a !== void 0 ? _a : shared.global;
+    stats.total++;
+    stats.byHost[parsed.host] = ((_b = stats.byHost[parsed.host]) !== null && _b !== void 0 ? _b : 0) + 1;
+    const chain = resolveChain(url);
+    if (chain)
+        stats.byChain[chain] = ((_c = stats.byChain[chain]) !== null && _c !== void 0 ? _c : 0) + 1;
+    else
+        stats.unattributed++;
+}
+/**
+ * Count a logical rpc request (one attempt, so retries count) against `method` and `chain` in the current run.
+ * The http layer already counts the request itself; this adds the method breakdown (getLogs being the one that matters).
+ */
+function recordMethod(method, chain = 'unknown') {
+    var _a, _b, _c;
+    const stats = (_a = shared.als.getStore()) !== null && _a !== void 0 ? _a : shared.global;
+    const byChain = (_b = stats.byMethod[method]) !== null && _b !== void 0 ? _b : (stats.byMethod[method] = {});
+    chain = chain.toLowerCase();
+    byChain[chain] = ((_c = byChain[chain]) !== null && _c !== void 0 ? _c : 0) + 1;
+}
+function getLogsCount(stats, chain) {
+    var _a, _b, _c;
+    const byChain = (_b = (_a = stats.byMethod) === null || _a === void 0 ? void 0 : _a.getLogs) !== null && _b !== void 0 ? _b : {};
+    if (chain)
+        return (_c = byChain[chain]) !== null && _c !== void 0 ? _c : 0;
+    return Object.values(byChain).reduce((a, b) => a + b, 0);
+}
+/** method -> requests across chains */
+function methodTotals(stats) {
+    var _a;
+    const out = {};
+    for (const [method, byChain] of Object.entries((_a = stats.byMethod) !== null && _a !== void 0 ? _a : {}))
+        out[method] = Object.values(byChain).reduce((a, b) => a + b, 0);
+    return out;
+}
+/** method -> requests for one chain (only methods the chain used) */
+function methodsForChain(stats, chain) {
+    var _a;
+    const out = {};
+    for (const [method, byChain] of Object.entries((_a = stats.byMethod) !== null && _a !== void 0 ? _a : {}))
+        if (byChain[chain])
+            out[method] = byChain[chain];
+    return out;
+}
+function urlFromRequestArgs(args, defaultProtocol) {
+    let url;
+    let options;
+    if (typeof args[0] === 'string') {
+        url = args[0];
+        if (args[1] && typeof args[1] === 'object')
+            options = args[1];
+    }
+    else if (args[0] instanceof URL) {
+        url = args[0].href;
+        if (args[1] && typeof args[1] === 'object')
+            options = args[1];
+    }
+    else if (args[0] && typeof args[0] === 'object') {
+        options = args[0];
+    }
+    if (url && (!options || (!options.hostname && !options.host && !options.path)))
+        return url;
+    if (!options)
+        return url;
+    // options override the url (same precedence as node)
+    const base = url ? new URL(url) : undefined;
+    const protocol = options.protocol || (base === null || base === void 0 ? void 0 : base.protocol) || defaultProtocol;
+    const host = options.hostname || options.host || (base === null || base === void 0 ? void 0 : base.hostname) || 'localhost';
+    const port = options.port ? `:${options.port}` : (options.hostname || !options.host) && (base === null || base === void 0 ? void 0 : base.port) ? `:${base.port}` : '';
+    const path = options.path || (base ? base.pathname + base.search : '/');
+    return `${protocol}//${host}${host.includes(':') ? '' : port}${path}`;
+}
+function wrap(mod, name, defaultProtocol) {
+    const original = mod[name];
+    if (typeof original !== 'function')
+        return;
+    const patched = function (...args) {
+        try {
+            const url = urlFromRequestArgs(args, defaultProtocol);
+            if (url)
+                record(url);
+        }
+        catch { /* never break the request */ }
+        return original.apply(this, args);
+    };
+    Object.defineProperty(patched, 'name', { value: original.name });
+    mod[name] = patched;
+}
+/** Patch the node http layer. Idempotent, shared across sdk copies. Called by `run`. */
+function install() {
+    var _a, _b;
+    if (shared.installed)
+        return;
+    shared.installed = true;
+    wrap(http_1.default, 'request', 'http:');
+    wrap(http_1.default, 'get', 'http:');
+    wrap(https_1.default, 'request', 'https:');
+    wrap(https_1.default, 'get', 'https:');
+    try {
+        // global fetch (undici); node < 18.7 has no subscribe, fetch is then simply not counted
+        ;
+        (_b = (_a = diagnostics_channel_1.default).subscribe) === null || _b === void 0 ? void 0 : _b.call(_a, 'undici:request:create', (message) => {
+            var _a;
+            try {
+                const request = message === null || message === void 0 ? void 0 : message.request;
+                if (request === null || request === void 0 ? void 0 : request.origin)
+                    record(String(request.origin) + String((_a = request.path) !== null && _a !== void 0 ? _a : ''));
+            }
+            catch { /* ignore */ }
+        });
+    }
+    catch { /* ignore */ }
+    try {
+        const esConfig = (0, env_1.getEnvValue)('ELASTICSEARCH_CONFIG');
+        if (esConfig)
+            ignoreHost(JSON.parse(esConfig.replace(/\\"/g, '"')).host);
+    }
+    catch { /* ignore */ }
+}
+/** Stats of the current run context, or the process-wide bucket when called outside `run`. */
+function getCurrent() {
+    var _a;
+    return (_a = shared.als.getStore()) !== null && _a !== void 0 ? _a : shared.global;
+}
+function snapshot(stats = getCurrent()) {
+    const byMethod = {};
+    for (const [method, byChain] of Object.entries(stats.byMethod))
+        byMethod[method] = { ...byChain };
+    return { total: stats.total, byChain: { ...stats.byChain }, byHost: { ...stats.byHost }, unattributed: stats.unattributed, byMethod };
+}
+function mergeSnapshots(snapshots) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+    const out = { total: 0, byChain: {}, byHost: {}, unattributed: 0, byMethod: {} };
+    for (const s of snapshots) {
+        if (!s)
+            continue;
+        out.total += (_a = s.total) !== null && _a !== void 0 ? _a : 0;
+        out.unattributed += (_b = s.unattributed) !== null && _b !== void 0 ? _b : 0;
+        for (const [k, v] of Object.entries((_c = s.byChain) !== null && _c !== void 0 ? _c : {}))
+            out.byChain[k] = ((_d = out.byChain[k]) !== null && _d !== void 0 ? _d : 0) + v;
+        for (const [k, v] of Object.entries((_e = s.byHost) !== null && _e !== void 0 ? _e : {}))
+            out.byHost[k] = ((_f = out.byHost[k]) !== null && _f !== void 0 ? _f : 0) + v;
+        for (const [method, byChain] of Object.entries((_g = s.byMethod) !== null && _g !== void 0 ? _g : {})) {
+            const target = (_h = out.byMethod[method]) !== null && _h !== void 0 ? _h : (out.byMethod[method] = {});
+            for (const [k, v] of Object.entries(byChain))
+                target[k] = ((_j = target[k]) !== null && _j !== void 0 ? _j : 0) + v;
+        }
+    }
+    return out;
+}
+/**
+ * Run `fn` with its own counters and report them when it settles (success or failure).
+ * Nested calls get their own context; calls made by `fn` through any http client are attributed to it.
+ */
+async function run(meta, fn, options = {}) {
+    install();
+    const stats = newStats(meta);
+    return shared.als.run(stats, async () => {
+        let success = false;
+        try {
+            const result = await fn();
+            success = true;
+            return result;
+        }
+        finally {
+            try {
+                await report(stats, { success, runtime: (Date.now() - stats.startTime) / 1000, ...options });
+            }
+            catch (e) {
+                (0, debugLog_1.debugLog)('[callStats] failed to report', e === null || e === void 0 ? void 0 : e.message);
+            }
+        }
+    });
+}
+// ---------------------------------------------------------------------------
+// reporting
+// ---------------------------------------------------------------------------
+function sortedEntries(map, minCalls, limit) {
+    const entries = Object.entries(map).filter(([, calls]) => calls >= minCalls).map(([key, calls]) => ({ key, calls }));
+    entries.sort((a, b) => b.calls - a.calls);
+    return limit ? entries.slice(0, limit) : entries;
+}
+function getMinCalls(minCalls) {
+    if (minCalls !== undefined)
+        return minCalls;
+    return +(0, env_1.getEnvValue)('CALL_STATS_MIN_CALLS', String(exports.DEFAULT_MIN_CALLS));
+}
+function getMinGetLogs() {
+    return +(0, env_1.getEnvValue)('CALL_STATS_MIN_GET_LOGS', String(exports.DEFAULT_MIN_GET_LOGS));
+}
+/**
+ * Chains worth a row: at least `minCalls` http requests, or at least `minGetLogs` getLogs requests
+ * (a chain that makes 20 getLogs calls is heavier than one making 50 eth_calls). Most calls first.
+ */
+function chainEntries(stats, minCalls) {
+    var _a, _b;
+    const minGetLogs = getMinGetLogs();
+    const chains = new Set([...Object.keys(stats.byChain), ...Object.keys((_b = (_a = stats.byMethod) === null || _a === void 0 ? void 0 : _a.getLogs) !== null && _b !== void 0 ? _b : {})]);
+    const entries = [...chains]
+        .map(key => { var _a; return ({ key, calls: (_a = stats.byChain[key]) !== null && _a !== void 0 ? _a : 0, getLogs: getLogsCount(stats, key) }); })
+        .filter(i => i.calls >= minCalls || i.getLogs >= minGetLogs);
+    entries.sort((a, b) => b.calls - a.calls || b.getLogs - a.getLogs);
+    return entries;
+}
+/** Elasticsearch documents for a run: one `run` summary, one `chain` doc per chain and one `host` doc per unattributed host at or above `minCalls`. */
+function buildDocs(stats, meta, { success, runtime, minCalls } = {}) {
+    minCalls = getMinCalls(minCalls);
+    const timestamp = Date.now();
+    const chains = chainEntries(stats, minCalls);
+    const hosts = sortedEntries(stats.byHost, minCalls, 25);
+    const rpcCalls = Object.values(stats.byChain).reduce((a, b) => a + b, 0);
+    const getLogsCalls = getLogsCount(stats);
+    const unattributedHosts = sortedEntries(Object.fromEntries(Object.entries(stats.byHost).filter(([host]) => !shared.endpoints.has(host))), minCalls);
+    const docs = [];
+    if (stats.total < minCalls && getLogsCalls < getMinGetLogs())
+        return docs;
+    docs.push({
+        timestamp, type: 'run', metadata: meta, success, runtime,
+        calls: stats.total, rpcCalls, otherCalls: stats.unattributed, getLogsCalls,
+        methods: methodTotals(stats),
+        chains: chains.map(i => ({ chain: i.key, calls: i.calls, getLogs: i.getLogs })),
+        hosts: hosts.map(i => ({ host: i.key, calls: i.calls })),
+    });
+    for (const i of chains)
+        docs.push({ timestamp, type: 'chain', metadata: meta, chain: i.key, calls: i.calls, getLogs: i.getLogs, methods: methodsForChain(stats, i.key) });
+    for (const i of unattributedHosts)
+        docs.push({ timestamp, type: 'host', metadata: meta, host: i.key, calls: i.calls });
+    return docs;
+}
+/** Write the docs for `stats` to elasticsearch (no-op when ELASTICSEARCH_CONFIG is unset) and print the summary in debug mode. */
+async function report(stats, { success, runtime, minCalls, print, skipElastic, meta } = {}) {
+    var _a;
+    meta = (_a = meta !== null && meta !== void 0 ? meta : stats.meta) !== null && _a !== void 0 ? _a : { application: 'unknown', name: '-' };
+    minCalls = getMinCalls(minCalls);
+    const docs = buildDocs(stats, meta, { success, runtime, minCalls });
+    if (print)
+        printStats(stats, meta, minCalls, console.log, console.table);
+    else
+        printStats(stats, meta, minCalls, debugLog_1.debugLog, debugLog_1.debugTable);
+    if (skipElastic || !docs.length)
+        return;
+    await Promise.all(docs.map(doc => (0, elastic_1.writeLog)(exports.ES_INDEX, doc)));
+}
+function printStats(stats, meta, minCalls, log, table) {
+    const getLogsCalls = getLogsCount(stats);
+    if (stats.total < minCalls && getLogsCalls < getMinGetLogs())
+        return;
+    log(`[callStats] ${meta.application}/${meta.name}: ${stats.total} http requests (${stats.unattributed} not matching a known rpc, ${getLogsCalls} getLogs)`);
+    const rows = [
+        ...chainEntries(stats, minCalls).map(i => ({ chain: i.key, calls: i.calls, getLogs: i.getLogs })),
+        ...sortedEntries(Object.fromEntries(Object.entries(stats.byHost).filter(([host]) => !shared.endpoints.has(host))), minCalls).map(i => ({ chain: `host: ${i.key}`, calls: i.calls, getLogs: 0 })),
+    ];
+    if (rows.length)
+        table(rows);
+}