@defillama/sdk 4.0.47 → 4.0.49

package/build/util/cache.d.ts

@@ -1,11 +1,12 @@
-export declare function getCache(options: CacheOptions): any;
-export declare function setCache(options: CacheOptions): void;
-export declare function saveCache(): void;
-export declare function startCache(_cache?: {}): void;
-export declare function retriveCache(): any;
-export type CacheOptions = {
-    address: string;
-    abi: string;
-    chain?: string;
-    value?: any;
-};
+export declare function readCache(file: string, options?: ReadCacheOptions): Promise<any>;
+interface WriteCacheOptions {
+    skipR2CacheWrite?: boolean;
+    alreadyCompressed?: boolean;
+}
+interface ReadCacheOptions {
+    skipR2Cache?: boolean;
+}
+export declare function writeCache(file: string, data: any, options?: WriteCacheOptions): Promise<string | undefined>;
+export declare function parseCache(dataString: string | object): Promise<any>;
+export declare function compressCache(data: any): Promise<any>;
+export {};

package/build/util/cache.js

@@ -1,68 +1,217 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
     }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.retriveCache = exports.startCache = exports.saveCache = exports.setCache = exports.getCache = void 0;
-var fs = __importStar(require("fs"));
-// const cacheFile = __dirname + '/../../../sc-cache1.json'
-var cacheFile = process.env.CACHE_FILE;
-var cache = {};
-// if (cacheFile) cache = require(cacheFile) // disabled for now
-function getCache(options) {
-    var address = options.address, abi = options.abi, _a = options.chain, chain = _a === void 0 ? "ethereum" : _a;
-    if (!address)
-        return;
-    address = address.slice(2).toLowerCase();
-    if (!cache[chain])
-        cache[chain] = {};
-    if (!cache[chain][abi])
-        cache[chain][abi] = {};
-    return cache[chain][abi][address];
+exports.compressCache = exports.parseCache = exports.writeCache = exports.readCache = void 0;
+var debugLog_1 = require("./debugLog");
+var r2_1 = require("./r2");
+var fs = require('fs').promises;
+var path = require('path');
+var lzma = require('lzma-native');
+var promisify = require('util').promisify;
+var zipAsync = promisify(lzma.compress);
+var unzipAsync = promisify(lzma.decompress);
+var foldersCreated = {};
+var currentVersion = 'llama-zip-1.0.0';
+function getCacheRootFolder() {
+    return process.env.TVL_LOCAL_CACHE_ROOT_FOLDER || path.join(__dirname, 'local_cache');
+}
+function getFilePath(file) {
+    return path.join(getCacheRootFolder(), file);
+}
+function createSubPath(folderPath) {
+    return __awaiter(this, void 0, void 0, function () {
+        var datRoot, error_1;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    datRoot = getCacheRootFolder();
+                    folderPath = folderPath.replace(datRoot, '');
+                    if (foldersCreated[folderPath])
+                        return [2 /*return*/];
+                    _a.label = 1;
+                case 1:
+                    _a.trys.push([1, 3, , 4]);
+                    return [4 /*yield*/, fs.mkdir(path.join(datRoot, folderPath), { recursive: true })];
+                case 2:
+                    _a.sent();
+                    return [3 /*break*/, 4];
+                case 3:
+                    error_1 = _a.sent();
+                    return [3 /*break*/, 4];
+                case 4:
+                    foldersCreated[folderPath] = true;
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+function readCache(file, options) {
+    if (options === void 0) { options = {}; }
+    return __awaiter(this, void 0, void 0, function () {
+        function readFile(file, options) {
+            if (options === void 0) { options = {}; }
+            return __awaiter(this, void 0, void 0, function () {
+                var filePath, data, error_3, r2Data;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0:
+                            filePath = getFilePath(file);
+                            _a.label = 1;
+                        case 1:
+                            _a.trys.push([1, 3, , 7]);
+                            return [4 /*yield*/, fs.readFile(filePath)];
+                        case 2:
+                            data = _a.sent();
+                            return [2 /*return*/, data];
+                        case 3:
+                            error_3 = _a.sent();
+                            // debugLog('Error reading cache:', error)
+                            if (options.skipR2Cache)
+                                return [2 /*return*/];
+                            return [4 /*yield*/, (0, r2_1.getR2JSONString)(file)];
+                        case 4:
+                            r2Data = _a.sent();
+                            if (!r2Data) return [3 /*break*/, 6];
+                            return [4 /*yield*/, writeCache(file, r2Data, { alreadyCompressed: true, skipR2CacheWrite: true })];
+                        case 5:
+                            _a.sent();
+                            return [2 /*return*/, r2Data];
+                        case 6: return [3 /*break*/, 7];
+                        case 7: return [2 /*return*/];
+                    }
+                });
+            });
+        }
+        var data, error_2;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    _a.trys.push([0, 3, , 4]);
+                    return [4 /*yield*/, readFile(file, options)];
+                case 1:
+                    data = _a.sent();
+                    return [4 /*yield*/, parseCache(data)];
+                case 2: return [2 /*return*/, _a.sent()];
+                case 3:
+                    error_2 = _a.sent();
+                    // debugLog('Error reading cache:', error)
+                    return [2 /*return*/, {}];
+                case 4: return [2 /*return*/];
+            }
+        });
+    });
 }
-exports.getCache = getCache;
-function setCache(options) {
-    var address = options.address, abi = options.abi, _a = options.chain, chain = _a === void 0 ? "ethereum" : _a, value = options.value;
-    if (!address)
-        return;
-    address = address.slice(2).toLowerCase();
-    if (!cache[chain])
-        cache[chain] = {};
-    if (!cache[chain][abi])
-        cache[chain][abi] = {};
-    cache[chain][abi][address] = value;
+exports.readCache = readCache;
+function writeCache(file, data, options) {
+    if (options === void 0) { options = {}; }
+    return __awaiter(this, void 0, void 0, function () {
+        var fileData, _a, _b, _c, filePath;
+        return __generator(this, function (_d) {
+            switch (_d.label) {
+                case 0:
+                    if (!options.alreadyCompressed) return [3 /*break*/, 1];
+                    _a = data;
+                    return [3 /*break*/, 3];
+                case 1: return [4 /*yield*/, compressCache(data)];
+                case 2:
+                    _a = _d.sent();
+                    _d.label = 3;
+                case 3:
+                    fileData = _a;
+                    if (!data || (typeof data === 'string' && data.length < 50) || fileData.length < 120) {
+                        (0, debugLog_1.debugLog)('Data too small to cache: ', file);
+                        return [2 /*return*/];
+                    }
+                    _b = isSameData;
+                    _c = [data];
+                    return [4 /*yield*/, readCache(file, { skipR2Cache: true })];
+                case 4:
+                    if (_b.apply(void 0, _c.concat([_d.sent()])))
+                        return [2 /*return*/];
+                    filePath = getFilePath(file);
+                    return [4 /*yield*/, createSubPath(path.dirname(filePath))];
+                case 5:
+                    _d.sent();
+                    return [4 /*yield*/, fs.writeFile(filePath, fileData)];
+                case 6:
+                    _d.sent();
+                    if (!!options.skipR2CacheWrite) return [3 /*break*/, 8];
+                    return [4 /*yield*/, (0, r2_1.storeR2JSONString)(file, fileData)];
+                case 7:
+                    _d.sent();
+                    _d.label = 8;
+                case 8: return [2 /*return*/, fileData];
+            }
+        });
+    });
 }
-exports.setCache = setCache;
-function saveCache() {
-    fs.writeFileSync(cacheFile || './cache.json', JSON.stringify(cache));
+exports.writeCache = writeCache;
+function parseCache(dataString) {
+    return __awaiter(this, void 0, void 0, function () {
+        var decompressed;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    if (typeof dataString === 'string')
+                        dataString = Buffer.from(dataString, 'base64');
+                    return [4 /*yield*/, unzipAsync(dataString)];
+                case 1:
+                    decompressed = _a.sent();
+                    return [2 /*return*/, JSON.parse(decompressed).llamaWrapped];
+            }
+        });
+    });
 }
-exports.saveCache = saveCache;
-function startCache(_cache) {
-    if (_cache === void 0) { _cache = {}; }
-    cache = _cache;
+exports.parseCache = parseCache;
+function compressCache(data) {
+    return __awaiter(this, void 0, void 0, function () {
+        var comressedCache;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0: return [4 /*yield*/, zipAsync(JSON.stringify({ version: currentVersion, llamaWrapped: data }), 9)];
+                case 1:
+                    comressedCache = _a.sent();
+                    return [2 /*return*/, comressedCache];
+            }
+        });
+    });
 }
-exports.startCache = startCache;
-function retriveCache() {
-    return cache;
+exports.compressCache = compressCache;
+function isSameData(data1, data2) {
+    return JSON.stringify(data1) === JSON.stringify(data2);
 }
-exports.retriveCache = retriveCache;

package/build/util/internal-cache.d.ts

@@ -0,0 +1,11 @@
+export declare function getCache(options: CacheOptions): any;
+export declare function setCache(options: CacheOptions): void;
+export declare function saveCache(): void;
+export declare function startCache(_cache?: {}): void;
+export declare function retriveCache(): any;
+export type CacheOptions = {
+    address: string;
+    abi: string;
+    chain?: string;
+    value?: any;
+};

package/build/util/internal-cache.js

@@ -0,0 +1,68 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retriveCache = exports.startCache = exports.saveCache = exports.setCache = exports.getCache = void 0;
+var fs = __importStar(require("fs"));
+// const cacheFile = __dirname + '/../../../sc-cache1.json'
+var cacheFile = process.env.INTERNAL_SDK_CACHE_FILE;
+var cache = {};
+// if (cacheFile) cache = require(cacheFile) // disabled for now
+function getCache(options) {
+    var address = options.address, abi = options.abi, _a = options.chain, chain = _a === void 0 ? "ethereum" : _a;
+    if (!address)
+        return;
+    address = address.slice(2).toLowerCase();
+    if (!cache[chain])
+        cache[chain] = {};
+    if (!cache[chain][abi])
+        cache[chain][abi] = {};
+    return cache[chain][abi][address];
+}
+exports.getCache = getCache;
+function setCache(options) {
+    var address = options.address, abi = options.abi, _a = options.chain, chain = _a === void 0 ? "ethereum" : _a, value = options.value;
+    if (!address)
+        return;
+    address = address.slice(2).toLowerCase();
+    if (!cache[chain])
+        cache[chain] = {};
+    if (!cache[chain][abi])
+        cache[chain][abi] = {};
+    cache[chain][abi][address] = value;
+}
+exports.setCache = setCache;
+function saveCache() {
+    fs.writeFileSync(cacheFile || './cache.json', JSON.stringify(cache));
+}
+exports.saveCache = saveCache;
+function startCache(_cache) {
+    if (_cache === void 0) { _cache = {}; }
+    cache = _cache;
+}
+exports.startCache = startCache;
+function retriveCache() {
+    return cache;
+}
+exports.retriveCache = retriveCache;

package/build/util/r2.d.ts

@@ -0,0 +1,3 @@
+/// <reference types="node" />
+export declare function storeR2JSONString(filename: string, body: string | Buffer): Promise<any>;
+export declare function getR2JSONString(filename: string): Promise<any>;

package/build/util/r2.js

@@ -0,0 +1,113 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getR2JSONString = exports.storeR2JSONString = void 0;
+var _a = require("@aws-sdk/client-s3"), S3Client = _a.S3Client, PutObjectCommand = _a.PutObjectCommand, GetObjectCommand = _a.GetObjectCommand;
+var _b = process.env, R2_ENDPOINT = _b.R2_ENDPOINT, R2_ACCESS_KEY_ID = _b.R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY = _b.R2_SECRET_ACCESS_KEY;
+var datasetBucket = "defillama-datasets";
+var publicBucketUrl = "https://defillama-datasets.llama.fi";
+var R2;
+if (R2_ENDPOINT && R2_ACCESS_KEY_ID && R2_SECRET_ACCESS_KEY) {
+    R2 = new S3Client({
+        region: "auto",
+        endpoint: "https://" + R2_ENDPOINT,
+        credentials: {
+            accessKeyId: R2_ACCESS_KEY_ID,
+            secretAccessKey: R2_SECRET_ACCESS_KEY,
+        },
+    });
+}
+var getKey = function (filename) { return filename.replace(/(:|'|#)/g, '/'); };
+function storeR2JSONString(filename, body) {
+    return __awaiter(this, void 0, void 0, function () {
+        var command;
+        return __generator(this, function (_a) {
+            if (typeof body !== 'string')
+                body = body.toString('base64');
+            if (!R2)
+                return [2 /*return*/];
+            command = new PutObjectCommand({
+                Bucket: datasetBucket,
+                Key: getKey(filename),
+                Body: body,
+                ContentType: "application/json",
+            });
+            return [2 /*return*/, R2.send(command)];
+        });
+    });
+}
+exports.storeR2JSONString = storeR2JSONString;
+function getR2JSONString(filename) {
+    return __awaiter(this, void 0, void 0, function () {
+        function _fetchData() {
+            return __awaiter(this, void 0, void 0, function () {
+                var response;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0: return [4 /*yield*/, fetch("".concat(publicBucketUrl, "/").concat(filename))];
+                        case 1:
+                            response = _a.sent();
+                            return [2 /*return*/, response.text()];
+                    }
+                });
+            });
+        }
+        var command, Body, e_1;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    _a.trys.push([0, 5, , 6]);
+                    if (!!R2) return [3 /*break*/, 2];
+                    return [4 /*yield*/, _fetchData()];
+                case 1: return [2 /*return*/, _a.sent()];
+                case 2:
+                    command = new GetObjectCommand({ Bucket: datasetBucket, Key: getKey(filename), });
+                    return [4 /*yield*/, R2.send(command)];
+                case 3:
+                    Body = (_a.sent()).Body;
+                    return [4 /*yield*/, Body.transformToString()];
+                case 4: return [2 /*return*/, _a.sent()];
+                case 5:
+                    e_1 = _a.sent();
+                    return [2 /*return*/, {}];
+                case 6: return [2 /*return*/];
+            }
+        });
+    });
+}
+exports.getR2JSONString = getR2JSONString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@defillama/sdk",
-  "version": "4.0.47",
+  "version": "4.0.49",
   "description": "SDK for calculating TVL",
   "main": "build/index.js",
   "scripts": {
@@ -23,8 +23,10 @@
     "build"
   ],
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.400.0",
     "@supercharge/promise-pool": "^2.1.0",
     "ethers": "^5.4.5",
+    "lzma-native": "^8.0.6",
     "node-fetch": "^2.6.7",
     "tron-format-address": "^0.1.11"
   },