@defillama/sdk 5.0.240 → 5.0.244

package/build/util/cache.js

@@ -0,0 +1,309 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ONE_WEEK = exports.currentVersion = void 0;
+exports.readCache = readCache;
+exports.writeCache = writeCache;
+exports.parseCache = parseCache;
+exports.compressCache = compressCache;
+exports.getTempLocalCache = getTempLocalCache;
+exports.deleteCache = deleteCache;
+exports.readExpiringJsonCache = readExpiringJsonCache;
+exports.writeExpiringJsonCache = writeExpiringJsonCache;
+exports.cachedFetch = cachedFetch;
+const debugLog_1 = require("./debugLog");
+const r2_1 = require("./r2");
+const zlib_1 = require("zlib");
+const util_1 = require("util");
+const env_1 = require("./env");
+const common_1 = require("./common");
+const brotliOptions = {
+    [zlib_1.constants.BROTLI_PARAM_MODE]: zlib_1.constants.BROTLI_MODE_TEXT,
+    [zlib_1.constants.BROTLI_PARAM_QUALITY]: zlib_1.constants.BROTLI_MAX_QUALITY,
+};
+function zipAsync(data) {
+    return (0, util_1.promisify)(zlib_1.brotliCompress)(data, brotliOptions);
+}
+function unzipAsync(data) {
+    return (0, util_1.promisify)(zlib_1.brotliDecompress)(data, brotliOptions).then(r => r.toString());
+}
+const _fs = require('fs');
+const fs = _fs.promises;
+const path = require('path');
+const foldersCreated = {};
+exports.currentVersion = 'zlib-1.0';
+function getCacheRootFolder() {
+    const defaultCacheRootFolder = path.join(__dirname, 'local_cache');
+    return path.join((0, env_1.getEnvCacheFolder)(defaultCacheRootFolder), exports.currentVersion);
+}
+function getFilePath(file) {
+    return path.join(getCacheRootFolder(), file);
+}
+async function createSubPath(folderPath) {
+    const datRoot = getCacheRootFolder();
+    folderPath = folderPath.replace(datRoot, '');
+    if (foldersCreated[folderPath])
+        return;
+    try {
+        await fs.mkdir(path.join(datRoot, folderPath), { recursive: true });
+    }
+    catch (error) {
+        // sdk.log('Error creating folder path:', error)
+    }
+    foldersCreated[folderPath] = true;
+}
+const ONE_DAY_IN_MS = 1000 * 60 * 60 * 24;
+async function readCache(file, options = {}) {
+    if (options.skipCompression)
+        file = `${file}-uncompressed`;
+    try {
+        const data = await readFile();
+        return await parseCache(data, options);
+    }
+    catch (error) {
+        // debugLog('Error reading cache:', error)
+        return {};
+    }
+    async function readFile() {
+        var _a;
+        try {
+            const data = await readDataFromLocal((_a = options.readFromR2Cache) !== null && _a !== void 0 ? _a : false);
+            return data;
+        }
+        catch (error) {
+            // debugLog('Error reading cache:', error)
+            return readDataFromR2();
+        }
+    }
+    async function readDataFromLocal(readFromR2Cache) {
+        const filePath = getFilePath(file);
+        if (options.checkIfRecent) {
+            const stats = await fs.stat(filePath);
+            if (Date.now() - stats.mtimeMs > ONE_DAY_IN_MS)
+                throw new Error('File too old, read from R2 instead');
+        }
+        if (readFromR2Cache)
+            throw new Error('Read from R2 cache');
+        const data = await fs.readFile(filePath);
+        return data;
+    }
+    async function readDataFromR2() {
+        var _a;
+        if (options.skipR2Cache)
+            return;
+        try {
+            const r2Data = await (0, r2_1.getR2JSONString)(exports.currentVersion + '/' + file, { throwErrorOnFail: (_a = options.readFromR2Cache) !== null && _a !== void 0 ? _a : false });
+            if (r2Data) {
+                // `file` already carries the `-uncompressed` suffix (added above), so omit skipCompression here to avoid writeCache appending it again
+                await writeCache(file, r2Data, { alreadyCompressed: true, skipR2CacheWrite: true });
+                return r2Data;
+            }
+        }
+        catch (error) {
+            if (options.readFromR2Cache) {
+                // if we were forced to read from R2 cache, try reading from local cache if R2 fails
+                try {
+                    const data = await readDataFromLocal(false);
+                    return data;
+                }
+                catch (_e) { }
+            }
+            return {};
+        }
+    }
+}
+async function writeCache(file, data, options = {}) {
+    try {
+        const fileData = options.alreadyCompressed ? data : await compressCache(data, options);
+        if (!data || (typeof data === 'string' && data.length < 20) || fileData.length < 20) {
+            (0, debugLog_1.debugLog)('Data too small to cache: ', file);
+            return;
+        }
+        if (!options.skipCompression && isSameData(data, await readCache(file, { skipR2Cache: true, skipCompression: options.skipCompression, })))
+            return;
+        if (options.skipCompression)
+            file = `${file}-uncompressed`;
+        const filePath = getFilePath(file);
+        await createSubPath(path.dirname(filePath));
+        await fs.writeFile(filePath, fileData);
+        if (!options.skipR2CacheWrite) {
+            await (0, r2_1.storeR2JSONString)(exports.currentVersion + '/' + file, fileData);
+        }
+        return fileData;
+    }
+    catch (error) {
+        if (typeof data === 'object' && data !== null && Object.keys(data).length === 0)
+            return;
+        (0, debugLog_1.debugLog)('Error writing cache:', error);
+    }
+}
+async function parseCache(dataString, options = {}) {
+    var _a;
+    let dataBuffer;
+    if (typeof dataString === 'string')
+        dataBuffer = Buffer.from(dataString, 'base64');
+    else {
+        dataBuffer = dataString;
+        dataString = dataBuffer.toString('utf8');
+    }
+    let decompressed;
+    let _unzipAsync = (str) => options.skipCompression ? dataString : unzipAsync(str); // if skipCompression is true, we don't decompress the data
+    try {
+        decompressed = await _unzipAsync((_a = dataBuffer) !== null && _a !== void 0 ? _a : dataString);
+    }
+    catch (e) {
+        // dataString = dataString.toString('utf8')
+        const convertedDataString = Buffer.from(dataString, 'base64');
+        decompressed = await _unzipAsync(convertedDataString);
+    }
+    return JSON.parse(decompressed).llamaWrapped;
+}
+async function compressCache(data, options = {}) {
+    const dataString = JSON.stringify({ version: exports.currentVersion, llamaWrapped: data });
+    if (options.skipCompression)
+        return dataString;
+    const comressedCache = await zipAsync(dataString);
+    return comressedCache;
+}
+function isSameData(data1, data2) {
+    return JSON.stringify(data1) === JSON.stringify(data2);
+}
+exports.ONE_WEEK = 1000 * 60 * 60 * 24 * 7;
+function getTempLocalCache({ file, defaultData = {}, clearAfter = exports.ONE_WEEK, returnWithSaveFunction = false }) {
+    const now = Math.floor(Date.now() / 1e3);
+    let emptyFile = { created: now, data: defaultData };
+    let fileData = emptyFile;
+    const filePath = getFilePath(file);
+    let saveCacheFlag = false; // to ensure that we save only once on exit
+    createSubPath(path.dirname(filePath)); // create folder if not exists, this is async but we don't care
+    try {
+        const data = _fs.readFileSync(filePath);
+        fileData = JSON.parse(data.toString());
+        if (now - fileData.created > clearAfter)
+            fileData = emptyFile; // clear cache
+    }
+    catch (error) { }
+    // save cache on exit
+    process.on('exit', _save);
+    process.on('SIGTERM', _save);
+    process.on('SIGINT', _save);
+    if (returnWithSaveFunction) {
+        return {
+            data: fileData.data,
+            saveCacheFile: () => saveCache(true)
+        };
+    }
+    return fileData.data;
+    function _save() {
+        saveCache(false);
+    }
+    function saveCache(skipCacheFlagCheck = false) {
+        if (!skipCacheFlagCheck) {
+            if (saveCacheFlag)
+                return;
+            saveCacheFlag = true;
+        }
+        try {
+            // debugLog('Saving cache to file:', filePath)
+            removePromisesAndFunctions(fileData);
+            _fs.writeFileSync(filePath, JSON.stringify(fileData));
+        }
+        catch (error) {
+            try {
+                (0, debugLog_1.debugLog)('Error saving cache:', error === null || error === void 0 ? void 0 : error.toString());
+            }
+            catch (error) { }
+        }
+    }
+    function removePromisesAndFunctions(data) {
+        if (typeof data === 'object') {
+            for (const key in data) {
+                if (typeof data[key] === 'function')
+                    delete data[key];
+                if (data[key] instanceof Promise)
+                    delete data[key];
+                else
+                    removePromisesAndFunctions(data[key]);
+            }
+        }
+    }
+}
+// this removes data only from the file cache, not from the R2 cache
+async function deleteCache(file) {
+    const filePath = getFilePath(file);
+    await fs.unlink(filePath).catch(() => {
+        // ignore errors
+    });
+}
+async function readExpiringJsonCache(file) {
+    const options = { readFromR2Cache: false, skipR2Cache: true, skipCompression: true };
+    file = 'expiring/' + file;
+    const data = await readCache(file, options);
+    if (!data || Object.keys(data).length === 0)
+        return null;
+    if (data.expiryTimestamp < (Date.now() / 1e3)) {
+        await deleteCache(`${file}-uncompressed`);
+        return null;
+    }
+    return data.data;
+}
+async function writeExpiringJsonCache(file, data, { expireAfter = 60 * 60 * 24, // cache for 1 day by default
+expiryTimestamp, } = {}) {
+    file = 'expiring/' + file;
+    if (!expiryTimestamp)
+        expiryTimestamp = Math.floor(Date.now() / 1e3) + expireAfter;
+    const options = { skipR2CacheWrite: true, skipCompression: true };
+    await writeCache(file, { data, expiryTimestamp }, options);
+}
+async function cachedFetch({ key, endpoint, postData, fetcher, fetchOptions, errorOnNoCache = true, cacheMaxAge = 0, writeCacheOptions = {
+    skipCompression: true,
+    skipR2CacheWrite: true,
+}, readCacheOptions = {
+    skipCompression: true,
+    skipR2Cache: true,
+}, }) {
+    if (!key) {
+        if (!endpoint || postData || fetcher)
+            throw new Error('Key is required for cachedFetch');
+        key = (0, common_1.getHash)(endpoint);
+    }
+    const fileKey = 'cached-fetch/' + key;
+    // check local cache first
+    if (cacheMaxAge > 0) {
+        try {
+            const filePath = getFilePath(readCacheOptions.skipCompression ? `${fileKey}-uncompressed` : fileKey);
+            const stats = await fs.stat(filePath);
+            if (Date.now() - stats.mtimeMs < cacheMaxAge) {
+                const cachedData = await readCache(fileKey, readCacheOptions);
+                if (cachedData && Object.keys(cachedData).length > 0)
+                    return cachedData;
+            }
+        }
+        catch (error) {
+            // cache miss or stat error, proceed to fetch
+        }
+    }
+    try {
+        let data;
+        if (fetcher)
+            data = await fetcher();
+        else {
+            if (!endpoint)
+                throw new Error('Either fetcher or endpoint is required for cachedFetch');
+            if (postData)
+                data = await (0, common_1.postJson)(endpoint, postData, fetchOptions);
+            else
+                data = await (0, common_1.fetchJson)(endpoint, fetchOptions);
+        }
+        await writeCache(fileKey, data, writeCacheOptions);
+        return data;
+    }
+    catch (error) {
+        const errorMessage = (0, common_1.formError)(error).message;
+        (0, debugLog_1.debugLog)('Error in cachedFetch:', errorMessage);
+        // if fetch fails, read from cache (regardless of age)
+        const cachedData = await readCache(fileKey, readCacheOptions);
+        if (!cachedData && errorOnNoCache)
+            throw new Error('Fetch failed and no cache available');
+        return cachedData;
+    }
+}

package/build/util/callStats.d.ts

@@ -0,0 +1,74 @@
+export declare const ES_INDEX = "adapter-call-stats";
+export declare const DEFAULT_MIN_CALLS = 50;
+/** a chain bucket is also kept when it has at least this many getLogs requests, however few calls it made */
+export declare const DEFAULT_MIN_GET_LOGS = 10;
+export type CallStatsMeta = {
+    /** 'tvl' | 'dimensions' | ... */
+    application: string;
+    /** protocol / module name */
+    name: string;
+    [key: string]: any;
+};
+export type CallStats = {
+    meta: CallStatsMeta;
+    startTime: number;
+    total: number;
+    /** chain -> requests to a registered rpc endpoint of that chain */
+    byChain: Record<string, number>;
+    /** host -> requests, every request */
+    byHost: Record<string, number>;
+    /** requests that matched no registered endpoint */
+    unattributed: number;
+    /**
+     * logical rpc method -> chain -> requests, recorded by the SDK at the call site (`recordMethod`):
+     * every LlamaProvider attempt (`call`, `getLogs`, `getBlock`, ...), starknet `getLogs`, and `indexerGetLogs`
+     */
+    byMethod: Record<string, Record<string, number>>;
+};
+export type CallStatsSnapshot = Pick<CallStats, 'total' | 'byChain' | 'byHost' | 'unattributed' | 'byMethod'>;
+/**
+ * Remember that `endpoints` serve `chain`, so requests to them are counted against that chain.
+ * Called by the SDK wherever it resolves rpc urls; adapters can call it for endpoints they manage themselves.
+ */
+export declare function registerEndpoints(chain: string | undefined, endpoints: string | string[] | undefined): void;
+/** Chain a url is counted against, or undefined when it matches no registered endpoint. */
+export declare function resolveChain(url: string): string | undefined;
+/** Requests to these hosts are not counted (eg. the elasticsearch cluster the stats are written to). */
+export declare function ignoreHost(url: string): void;
+/**
+ * Count a logical rpc request (one attempt, so retries count) against `method` and `chain` in the current run.
+ * The http layer already counts the request itself; this adds the method breakdown (getLogs being the one that matters).
+ */
+export declare function recordMethod(method: string, chain?: string): void;
+/** Patch the node http layer. Idempotent, shared across sdk copies. Called by `run`. */
+export declare function install(): void;
+export type RunOptions = {
+    /** chain / host buckets below this are dropped, and nothing is written when the total is below it (default 50, env LLAMA_SDK_CALL_STATS_MIN_CALLS) */
+    minCalls?: number;
+    /** print the summary table to the console (always, not only in debug mode) */
+    print?: boolean;
+    /** skip the elasticsearch write */
+    skipElastic?: boolean;
+};
+/** Stats of the current run context, or the process-wide bucket when called outside `run`. */
+export declare function getCurrent(): CallStats;
+export declare function snapshot(stats?: CallStats): CallStatsSnapshot;
+export declare function mergeSnapshots(snapshots: CallStatsSnapshot[]): CallStatsSnapshot;
+/**
+ * Run `fn` with its own counters and report them when it settles (success or failure).
+ * Nested calls get their own context; calls made by `fn` through any http client are attributed to it.
+ */
+export declare function run<T>(meta: CallStatsMeta, fn: () => Promise<T>, options?: RunOptions): Promise<T>;
+export declare function getMinCalls(minCalls?: number): number;
+/** Elasticsearch documents for a run: one `run` summary, one `chain` doc per chain and one `host` doc per unattributed host at or above `minCalls`. */
+export declare function buildDocs(stats: CallStatsSnapshot, meta: CallStatsMeta, { success, runtime, minCalls }?: {
+    success?: boolean;
+    runtime?: number;
+    minCalls?: number;
+}): any[];
+/** Write the docs for `stats` to elasticsearch (no-op when ELASTICSEARCH_CONFIG is unset) and print the summary in debug mode. */
+export declare function report(stats: CallStats | CallStatsSnapshot, { success, runtime, minCalls, print, skipElastic, meta }?: RunOptions & {
+    success?: boolean;
+    runtime?: number;
+    meta?: CallStatsMeta;
+}): Promise<void>;