@defillama/sdk 5.0.10 → 5.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/build/Balances.d.ts +10 -0
  2. package/build/Balances.js +51 -107
  3. package/build/ChainApi.d.ts +1 -0
  4. package/build/ChainApi.js +152 -253
  5. package/build/abi/abi2.js +47 -144
  6. package/build/abi/convertResults.js +9 -29
  7. package/build/abi/index.js +220 -326
  8. package/build/abi/multicall.js +115 -217
  9. package/build/abi/multicall3.js +59 -123
  10. package/build/abi/rpcCall.js +43 -106
  11. package/build/abi/tron.js +153 -283
  12. package/build/api.js +3 -4
  13. package/build/computeTVL/blocks.d.ts +1 -1
  14. package/build/computeTVL/blocks.js +2 -1
  15. package/build/computeTVL/humanizeNumber.js +6 -7
  16. package/build/computeTVL/prices.js +55 -145
  17. package/build/erc20/index.js +34 -122
  18. package/build/eth/index.js +37 -107
  19. package/build/general.js +22 -23
  20. package/build/generalUtil.js +68 -127
  21. package/build/index.js +2 -2
  22. package/build/providers.json +1 -1
  23. package/build/util/blocks.d.ts +5 -2
  24. package/build/util/blocks.js +234 -373
  25. package/build/util/blockscout.js +13 -58
  26. package/build/util/cache.js +112 -274
  27. package/build/util/computeTVL.js +72 -150
  28. package/build/util/cosmos.js +21 -75
  29. package/build/util/debugLog.js +5 -13
  30. package/build/util/env.d.ts +3 -2
  31. package/build/util/env.js +30 -35
  32. package/build/util/index.d.ts +5 -3
  33. package/build/util/index.js +105 -134
  34. package/build/util/internal-cache.js +7 -8
  35. package/build/util/logs.d.ts +1 -2
  36. package/build/util/logs.js +148 -236
  37. package/build/util/promisePool.js +11 -62
  38. package/build/util/r2.js +36 -95
  39. package/build/util/updateProviderList.js +23 -83
  40. package/package.json +2 -3
@@ -1,247 +1,159 @@
1
1
  "use strict";
2
- var __assign = (this && this.__assign) || function () {
3
- __assign = Object.assign || function(t) {
4
- for (var s, i = 1, n = arguments.length; i < n; i++) {
5
- s = arguments[i];
6
- for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
7
- t[p] = s[p];
8
- }
9
- return t;
10
- };
11
- return __assign.apply(this, arguments);
12
- };
13
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
14
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
15
- return new (P || (P = Promise))(function (resolve, reject) {
16
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
17
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
18
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
19
- step((generator = generator.apply(thisArg, _arguments || [])).next());
20
- });
21
- };
22
- var __generator = (this && this.__generator) || function (thisArg, body) {
23
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
24
- return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
25
- function verb(n) { return function (v) { return step([n, v]); }; }
26
- function step(op) {
27
- if (f) throw new TypeError("Generator is already executing.");
28
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
29
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
30
- if (y = 0, t) op = [op[0] & 2, t.value];
31
- switch (op[0]) {
32
- case 0: case 1: t = op; break;
33
- case 4: _.label++; return { value: op[1], done: false };
34
- case 5: _.label++; y = op[1]; op = [0]; continue;
35
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
36
- default:
37
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
38
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
39
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
40
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
41
- if (t[2]) _.ops.pop();
42
- _.trys.pop(); continue;
43
- }
44
- op = body.call(thisArg, _);
45
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
46
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
47
- }
48
- };
49
- var __importDefault = (this && this.__importDefault) || function (mod) {
50
- return (mod && mod.__esModule) ? mod : { "default": mod };
51
- };
52
2
  Object.defineProperty(exports, "__esModule", { value: true });
53
3
  exports.getLogs = void 0;
54
- var _1 = require(".");
55
- var ethers_1 = require("ethers");
56
- var blocks_1 = require("./blocks");
57
- var cache_1 = require("./cache");
58
- var debugLog_1 = require("./debugLog");
59
- var p_limit_1 = __importDefault(require("p-limit"));
60
- var env_1 = require("./env");
61
- var currentVersion = 'v2';
62
- function getLogs(options) {
63
- return __awaiter(this, void 0, void 0, function () {
64
- function addLogsToCache(fromBlock, toBlock) {
65
- return __awaiter(this, void 0, void 0, function () {
66
- var metadata, logs, logIndices;
67
- var _a;
68
- return __generator(this, function (_b) {
69
- switch (_b.label) {
70
- case 0:
71
- metadata = cacheData.metadata;
72
- return [4 /*yield*/, (0, _1.getLogs)({
73
- chain: chain,
74
- target: target, topic: topic,
75
- keys: keys,
76
- topics: topics,
77
- fromBlock: fromBlock,
78
- toBlock: toBlock,
79
- })];
80
- case 1:
81
- logs = (_b.sent()).output;
82
- (_a = cacheData.logs).push.apply(_a, logs);
83
- logIndices = new Set();
84
- cacheData.logs = cacheData.logs.filter(function (i) {
85
- var _a;
86
- var key = i.transactionHash + ((_a = i.logIndex) !== null && _a !== void 0 ? _a : i.index); // ethers v5 had logIndex, ethers v6 has index
87
- if (!(i.hasOwnProperty('logIndex') || i.hasOwnProperty('index')) || !i.hasOwnProperty('transactionHash')) {
88
- (0, debugLog_1.debugLog)(i, i.logIndex, i.index, i.transactionHash);
89
- throw new Error('Missing crucial field');
90
- }
91
- if (logIndices.has(key))
92
- return false;
93
- logIndices.add(key);
94
- return true;
95
- });
96
- if (!metadata.fromBlock || fromBlock < metadata.fromBlock)
97
- metadata.fromBlock = fromBlock;
98
- if (!metadata.toBlock || toBlock > metadata.toBlock)
99
- metadata.toBlock = toBlock;
100
- if (!!skipCache) return [3 /*break*/, 3];
101
- return [4 /*yield*/, (0, cache_1.writeCache)(getFile(), cacheData, { skipR2CacheWrite: !cacheInCloud })];
102
- case 2:
103
- _b.sent();
104
- _b.label = 3;
105
- case 3: return [2 /*return*/];
106
- }
107
- });
108
- });
109
- }
110
- function _getCache() {
111
- return __awaiter(this, void 0, void 0, function () {
112
- var key, defaultRes, cache;
113
- return __generator(this, function (_a) {
114
- switch (_a.label) {
115
- case 0:
116
- key = getFile();
117
- defaultRes = {
118
- logs: [],
119
- };
120
- if (skipCache || skipCacheRead)
121
- return [2 /*return*/, defaultRes];
122
- return [4 /*yield*/, (0, cache_1.readCache)(key, { skipR2Cache: !cacheInCloud })];
123
- case 1:
124
- cache = _a.sent();
125
- if (!cache.metadata || cache.metadata.version !== currentVersion)
126
- return [2 /*return*/, defaultRes];
127
- return [2 /*return*/, cache];
128
- }
129
- });
130
- });
131
- }
132
- function getFile() {
133
- var extraKey = topics === null || topics === void 0 ? void 0 : topics.join('-').toLowerCase();
134
- if (!extraKey)
135
- throw new Error('extraKey is required');
136
- if (keys.length)
137
- extraKey += '-' + keys.join('-').toLowerCase();
138
- return "event-logs/".concat(chain, "/").concat(target.toLowerCase(), "-").concat(extraKey);
4
+ const _1 = require(".");
5
+ const ethers_1 = require("ethers");
6
+ const blocks_1 = require("./blocks");
7
+ const cache_1 = require("./cache");
8
+ const debugLog_1 = require("./debugLog");
9
+ const currentVersion = 'v3';
10
+ async function getLogs(options) {
11
+ let { target, chain = 'ethereum', fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, // can be passed as input as extra filter arguments
12
+ skipCache = false, skipCacheRead = false, entireLog = false, cacheInCloud = false, onlyArgs = false, keys = [], // [Deprecated] This is just used to select only part of the logs
13
+ targets, flatten = true, } = options;
14
+ if (!target && !(targets === null || targets === void 0 ? void 0 : targets.length))
15
+ throw new Error('target|targets is required');
16
+ if (!fromBlock && !fromTimestamp)
17
+ throw new Error('fromBlock or fromTimestamp is required');
18
+ if (!toBlock && !toTimestamp)
19
+ throw new Error('toBlock or toTimestamp is required');
20
+ if (!fromBlock)
21
+ fromBlock = await (0, blocks_1.getBlockNumber)(chain, fromTimestamp);
22
+ if (!toBlock)
23
+ toBlock = await (0, blocks_1.getBlockNumber)(chain, toTimestamp);
24
+ if (!fromBlock || !toBlock)
25
+ throw new Error('fromBlock and toBlock must be > 0');
26
+ if (targets === null || targets === void 0 ? void 0 : targets.length) {
27
+ const newOptions = { ...options, fromBlock, toBlock };
28
+ delete newOptions.targets;
29
+ const res = await Promise.all(targets.map(i => getLogs({ ...newOptions, target: i })));
30
+ if (flatten)
31
+ return res.flat();
32
+ return res;
33
+ }
34
+ let iface;
35
+ if (eventAbi)
36
+ iface = new ethers_1.Interface([eventAbi]);
37
+ if ((!topics || !topics.length)) {
38
+ if (topic)
39
+ topic = toFilterTopic(topic);
40
+ else if (eventAbi)
41
+ topic = toFilterTopic(iface);
42
+ else {
43
+ throw new Error('eventAbi | topic | topics are required');
139
44
  }
140
- // we need to form filter topic with indexed keyword, else it messes up generated topic string
141
- function toFilterTopic(topic) {
142
- if (typeof topic === 'string') {
143
- if (topic.startsWith('0x'))
144
- return topic;
145
- topic = new ethers_1.Interface([topic]);
45
+ topics = [topic];
46
+ if (extraTopics)
47
+ topics.push(...extraTopics);
48
+ }
49
+ let caches = await _getCache();
50
+ const isFirstRun = !caches.length;
51
+ if (isFirstRun || skipCacheRead) {
52
+ await addLogsToCache(fromBlock, toBlock);
53
+ }
54
+ else {
55
+ let _fromBlock = fromBlock; // we need to keep a copy of fromBlock as it will be modified
56
+ let _toBlock = toBlock; // we need to keep a copy of toBlock as it will be modified
57
+ const firstCache = caches[0];
58
+ const lastCache = caches[caches.length - 1];
59
+ for (const cacheData of caches) {
60
+ const { fromBlock: cFirstBlock, toBlock: cToBlock } = cacheData.metadata;
61
+ if (_toBlock < firstCache.metadata.fromBlock || _fromBlock > lastCache.metadata.toBlock) { // no intersection with any cache
62
+ await addLogsToCache(_fromBlock, _toBlock);
63
+ break;
64
+ }
65
+ if (_fromBlock >= cToBlock)
66
+ continue; // request is after cache end
67
+ if (_fromBlock <= cFirstBlock) { // request is before cache start
68
+ await addLogsToCache(_fromBlock, Math.min(_toBlock, cFirstBlock - 1));
146
69
  }
147
- var fragment = topic.fragments[0];
148
- return (0, ethers_1.id)("".concat(fragment.name, "(").concat(fragment.inputs.map(function (i) { return i.type; }).join(','), ")"));
70
+ if (_toBlock <= cToBlock)
71
+ break; // request ends before cache end
72
+ _fromBlock = cToBlock;
149
73
  }
150
- var target, _a, chain, fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, _b, skipCache, _c, skipCacheRead, _d, entireLog, _e, cacheInCloud, _f, onlyArgs, _g, keys, targets, _h, flatten, limiter, newOptions_1, res, iface, cacheData, isFirstRun, metadata_1, logs;
151
- return __generator(this, function (_j) {
152
- switch (_j.label) {
153
- case 0:
154
- target = options.target, _a = options.chain, chain = _a === void 0 ? 'ethereum' : _a, fromBlock = options.fromBlock, toBlock = options.toBlock, fromTimestamp = options.fromTimestamp, toTimestamp = options.toTimestamp, eventAbi = options.eventAbi, topic = options.topic, topics = options.topics, extraTopics = options.extraTopics, _b = options.skipCache, skipCache = _b === void 0 ? false : _b, _c = options.skipCacheRead, skipCacheRead = _c === void 0 ? false : _c, _d = options.entireLog, entireLog = _d === void 0 ? false : _d, _e = options.cacheInCloud, cacheInCloud = _e === void 0 ? false : _e, _f = options.onlyArgs, onlyArgs = _f === void 0 ? false : _f, _g = options.keys, keys = _g === void 0 ? [] : _g, targets = options.targets, _h = options.flatten, flatten = _h === void 0 ? true : _h;
155
- if (!target && !(targets === null || targets === void 0 ? void 0 : targets.length))
156
- throw new Error('target|targets is required');
157
- if (!fromBlock && !fromTimestamp)
158
- throw new Error('fromBlock or fromTimestamp is required');
159
- if (!toBlock && !toTimestamp)
160
- throw new Error('toBlock or toTimestamp is required');
161
- limiter = getChainLimiter(chain);
162
- if (!!fromBlock) return [3 /*break*/, 2];
163
- return [4 /*yield*/, limiter(function () { return (0, blocks_1.getBlock)(chain, fromTimestamp); })];
164
- case 1:
165
- fromBlock = (_j.sent()).block;
166
- _j.label = 2;
167
- case 2:
168
- if (!!toBlock) return [3 /*break*/, 4];
169
- return [4 /*yield*/, limiter(function () { return (0, blocks_1.getBlock)(chain, toTimestamp); })];
170
- case 3:
171
- toBlock = (_j.sent()).block;
172
- _j.label = 4;
173
- case 4:
174
- if (!(targets === null || targets === void 0 ? void 0 : targets.length)) return [3 /*break*/, 6];
175
- newOptions_1 = __assign(__assign({}, options), { fromBlock: fromBlock, toBlock: toBlock });
176
- delete newOptions_1.targets;
177
- return [4 /*yield*/, Promise.all(targets.map(function (i) { return getLogs(__assign(__assign({}, newOptions_1), { target: i })); }))];
178
- case 5:
179
- res = _j.sent();
180
- if (flatten)
181
- return [2 /*return*/, res.flat()];
182
- return [2 /*return*/, res];
183
- case 6:
184
- if (eventAbi)
185
- iface = new ethers_1.Interface([eventAbi]);
186
- if ((!topics || !topics.length)) {
187
- if (topic)
188
- topic = toFilterTopic(topic);
189
- else if (eventAbi)
190
- topic = toFilterTopic(iface);
191
- else {
192
- throw new Error('eventAbi | topic | topics are required');
193
- }
194
- topics = [topic];
195
- if (extraTopics)
196
- topics.push.apply(topics, extraTopics);
197
- }
198
- return [4 /*yield*/, _getCache()];
199
- case 7:
200
- cacheData = _j.sent();
201
- isFirstRun = !cacheData.metadata;
202
- if (isFirstRun)
203
- cacheData.metadata = { version: currentVersion };
204
- if (!isFirstRun) return [3 /*break*/, 9];
205
- return [4 /*yield*/, limiter(function () { return addLogsToCache(fromBlock, toBlock); })];
206
- case 8:
207
- _j.sent();
208
- return [3 /*break*/, 13];
209
- case 9:
210
- metadata_1 = cacheData.metadata;
211
- if (!(fromBlock < metadata_1.fromBlock)) return [3 /*break*/, 11];
212
- return [4 /*yield*/, limiter(function () { return addLogsToCache(fromBlock, metadata_1.fromBlock); })];
213
- case 10:
214
- _j.sent();
215
- _j.label = 11;
216
- case 11:
217
- if (!(toBlock > metadata_1.toBlock)) return [3 /*break*/, 13];
218
- return [4 /*yield*/, limiter(function () { return addLogsToCache(metadata_1.toBlock, toBlock); })];
219
- case 12:
220
- _j.sent();
221
- _j.label = 13;
222
- case 13:
223
- logs = cacheData.logs.filter(function (i) {
224
- if (i.blockNumber < fromBlock)
225
- return false;
226
- if (i.blockNumber > toBlock)
227
- return false;
228
- return true;
229
- });
230
- if (!eventAbi || entireLog)
231
- return [2 /*return*/, logs];
232
- return [2 /*return*/, logs.map(function (i) { return iface.parseLog(i); }).map(function (i) { return onlyArgs ? i.args : i; })];
74
+ }
75
+ const logs = [];
76
+ for (const cacheData of caches) {
77
+ const { fromBlock: cFirstBlock, toBlock: cToBlock } = cacheData.metadata;
78
+ if (fromBlock > cToBlock || toBlock < cFirstBlock)
79
+ continue; // no intersection with cache
80
+ logs.push(...cacheData.logs.filter((i) => i.blockNumber >= fromBlock && i.blockNumber <= toBlock));
81
+ }
82
+ if (!eventAbi || entireLog)
83
+ return logs;
84
+ return logs.map((i) => iface.parseLog(i)).map((i) => onlyArgs ? i.args : i);
85
+ async function addLogsToCache(fromBlock, toBlock) {
86
+ (0, debugLog_1.debugLog)('adding logs to cache: ', fromBlock, toBlock, target, topic);
87
+ if (fromBlock > toBlock)
88
+ return; // no data to add
89
+ fromBlock = fromBlock - 10;
90
+ toBlock = toBlock + 10;
91
+ let { output: logs } = await (0, _1.getLogs)({
92
+ chain, target: target, topic: topic, keys, topics, fromBlock, toBlock,
93
+ });
94
+ caches.push({
95
+ logs,
96
+ metadata: { fromBlock, toBlock, }
97
+ });
98
+ caches.sort((a, b) => a.metadata.fromBlock - b.metadata.fromBlock);
99
+ const mergedCaches = [caches[0]];
100
+ caches.slice(1).forEach(i => {
101
+ const last = mergedCaches[mergedCaches.length - 1];
102
+ if (last.metadata.toBlock + 1 > i.metadata.fromBlock) {
103
+ last.metadata.toBlock = i.metadata.toBlock;
104
+ last.logs = dedupLogs(last.logs, i.logs);
105
+ }
106
+ else {
107
+ mergedCaches.push(i);
233
108
  }
234
109
  });
235
- });
236
- }
237
- exports.getLogs = getLogs;
238
- var chainLimiters = {};
239
- function getChainLimiter(chain) {
240
- if (!chainLimiters[chain])
241
- chainLimiters[chain] = createLimiter(chain);
242
- return chainLimiters[chain];
243
- function createLimiter(chain) {
244
- return (0, p_limit_1.default)((0, env_1.getParallelGetLogsLimit)(chain));
110
+ caches = mergedCaches;
111
+ if (!skipCache)
112
+ await (0, cache_1.writeCache)(getFile(), { caches, version: currentVersion }, { skipR2CacheWrite: !cacheInCloud });
113
+ }
114
+ function dedupLogs(...logs) {
115
+ const logIndices = new Set();
116
+ return logs.flat().filter((i) => {
117
+ var _a;
118
+ let key = i.transactionHash + ((_a = i.logIndex) !== null && _a !== void 0 ? _a : i.index); // ethers v5 had logIndex, ethers v6 has index
119
+ if (!(i.hasOwnProperty('logIndex') || i.hasOwnProperty('index')) || !i.hasOwnProperty('transactionHash')) {
120
+ (0, debugLog_1.debugLog)(i, i.logIndex, i.index, i.transactionHash);
121
+ throw new Error('Missing crucial field: logIndex/index');
122
+ }
123
+ if (logIndices.has(key))
124
+ return false;
125
+ logIndices.add(key);
126
+ return true;
127
+ });
128
+ }
129
+ async function _getCache() {
130
+ const key = getFile();
131
+ const defaultRes = [];
132
+ if (skipCache)
133
+ return defaultRes;
134
+ let cache = await (0, cache_1.readCache)(key, { skipR2Cache: !cacheInCloud });
135
+ if (!cache.caches || !cache.caches.length || cache.version !== currentVersion)
136
+ return defaultRes;
137
+ return cache.caches;
138
+ }
139
+ function getFile() {
140
+ let extraKey = topics === null || topics === void 0 ? void 0 : topics.join('-').toLowerCase();
141
+ if (!extraKey)
142
+ throw new Error('extraKey is required');
143
+ if (keys.length)
144
+ extraKey += '-' + keys.join('-').toLowerCase();
145
+ return `event-logs/${chain}/${target.toLowerCase()}-${extraKey}`;
146
+ }
147
+ // we need to form filter topic with indexed keyword, else it messes up generated topic string
148
+ function toFilterTopic(topic) {
149
+ if (typeof topic === 'string') {
150
+ if (topic.startsWith('0x'))
151
+ return topic;
152
+ topic = new ethers_1.Interface([topic]);
153
+ }
154
+ const fragment = topic.fragments[0];
155
+ return (0, ethers_1.id)(`${fragment.name}(${fragment.inputs.map((i) => i.type).join(',')})`);
245
156
  }
246
157
  }
158
+ exports.getLogs = getLogs;
247
159
  exports.default = getLogs;
@@ -1,67 +1,16 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __generator = (this && this.__generator) || function (thisArg, body) {
12
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
13
- return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14
- function verb(n) { return function (v) { return step([n, v]); }; }
15
- function step(op) {
16
- if (f) throw new TypeError("Generator is already executing.");
17
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
18
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19
- if (y = 0, t) op = [op[0] & 2, t.value];
20
- switch (op[0]) {
21
- case 0: case 1: t = op; break;
22
- case 4: _.label++; return { value: op[1], done: false };
23
- case 5: _.label++; y = op[1]; op = [0]; continue;
24
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
25
- default:
26
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30
- if (t[2]) _.ops.pop();
31
- _.trys.pop(); continue;
32
- }
33
- op = body.call(thisArg, _);
34
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36
- }
37
- };
38
2
  Object.defineProperty(exports, "__esModule", { value: true });
39
- var promise_pool_1 = require("@supercharge/promise-pool");
40
- var generalUtil_1 = require("../generalUtil");
3
+ const promise_pool_1 = require("@supercharge/promise-pool");
4
+ const generalUtil_1 = require("../generalUtil");
41
5
  generalUtil_1.formError;
42
- function runInPromisePool(params) {
43
- return __awaiter(this, void 0, void 0, function () {
44
- var _a, results, errors, flatResults;
45
- var _this = this;
46
- return __generator(this, function (_b) {
47
- switch (_b.label) {
48
- case 0: return [4 /*yield*/, promise_pool_1.PromisePool
49
- .withConcurrency(params.concurrency)
50
- .for(params.items)
51
- .process(function (item, i) { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
52
- switch (_a.label) {
53
- case 0: return [4 /*yield*/, params.processor(item, i)];
54
- case 1: return [2 /*return*/, [_a.sent(), i]];
55
- }
56
- }); }); })];
57
- case 1:
58
- _a = _b.sent(), results = _a.results, errors = _a.errors;
59
- if (errors.length)
60
- throw (0, generalUtil_1.formError)(null, { promisePoolErrors: errors });
61
- flatResults = results.sort(function (a, b) { return a[1] - b[1]; }).map((function (i) { return i[0]; }));
62
- return [2 /*return*/, flatResults];
63
- }
64
- });
65
- });
6
+ async function runInPromisePool(params) {
7
+ const { results, errors } = await promise_pool_1.PromisePool
8
+ .withConcurrency(params.concurrency)
9
+ .for(params.items)
10
+ .process(async (item, i) => [await params.processor(item, i), i]);
11
+ if (errors.length)
12
+ throw (0, generalUtil_1.formError)(null, { promisePoolErrors: errors });
13
+ const flatResults = results.sort((a, b) => a[1] - b[1]).map((i => i[0]));
14
+ return flatResults;
66
15
  }
67
16
  exports.default = runInPromisePool;
package/build/util/r2.js CHANGED
@@ -1,48 +1,16 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __generator = (this && this.__generator) || function (thisArg, body) {
12
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
13
- return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14
- function verb(n) { return function (v) { return step([n, v]); }; }
15
- function step(op) {
16
- if (f) throw new TypeError("Generator is already executing.");
17
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
18
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19
- if (y = 0, t) op = [op[0] & 2, t.value];
20
- switch (op[0]) {
21
- case 0: case 1: t = op; break;
22
- case 4: _.label++; return { value: op[1], done: false };
23
- case 5: _.label++; y = op[1]; op = [0]; continue;
24
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
25
- default:
26
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30
- if (t[2]) _.ops.pop();
31
- _.trys.pop(); continue;
32
- }
33
- op = body.call(thisArg, _);
34
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36
- }
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
37
4
  };
38
5
  Object.defineProperty(exports, "__esModule", { value: true });
39
6
  exports.getR2JSONString = exports.storeR2JSONString = void 0;
40
- var env_1 = require("./env");
41
- var _a = require("@aws-sdk/client-s3"), S3Client = _a.S3Client, PutObjectCommand = _a.PutObjectCommand, GetObjectCommand = _a.GetObjectCommand;
42
- var R2_ENDPOINT = env_1.ENV_CONSTANTS.R2_ENDPOINT, R2_ACCESS_KEY_ID = env_1.ENV_CONSTANTS.R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY = env_1.ENV_CONSTANTS.R2_SECRET_ACCESS_KEY;
43
- var datasetBucket = "defillama-datasets";
44
- var publicBucketUrl = "https://defillama-datasets.llama.fi";
45
- var R2;
7
+ const axios_1 = __importDefault(require("axios"));
8
+ const env_1 = require("./env");
9
+ const { S3Client, PutObjectCommand, GetObjectCommand, } = require("@aws-sdk/client-s3");
10
+ const { R2_ENDPOINT, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY } = env_1.ENV_CONSTANTS;
11
+ const datasetBucket = "defillama-datasets";
12
+ const publicBucketUrl = "https://defillama-datasets.llama.fi";
13
+ let R2;
46
14
  if (R2_ENDPOINT && R2_ACCESS_KEY_ID && R2_SECRET_ACCESS_KEY) {
47
15
  R2 = new S3Client({
48
16
  region: "auto",
@@ -53,62 +21,35 @@ if (R2_ENDPOINT && R2_ACCESS_KEY_ID && R2_SECRET_ACCESS_KEY) {
53
21
  },
54
22
  });
55
23
  }
56
- var getKey = function (filename) { return filename.replace(/(:|'|#)/g, '/'); };
57
- function storeR2JSONString(filename, body) {
58
- return __awaiter(this, void 0, void 0, function () {
59
- var command;
60
- return __generator(this, function (_a) {
61
- if (typeof body !== 'string')
62
- body = body.toString('base64');
63
- if (!R2)
64
- return [2 /*return*/];
65
- command = new PutObjectCommand({
66
- Bucket: datasetBucket,
67
- Key: getKey(filename),
68
- Body: body,
69
- ContentType: "application/json",
70
- });
71
- return [2 /*return*/, R2.send(command)];
72
- });
24
+ const getKey = (filename) => filename.replace(/(:|'|#)/g, '/');
25
+ async function storeR2JSONString(filename, body) {
26
+ if (typeof body !== 'string')
27
+ body = body.toString('base64');
28
+ if (!R2)
29
+ return;
30
+ const command = new PutObjectCommand({
31
+ Bucket: datasetBucket,
32
+ Key: getKey(filename),
33
+ Body: body,
34
+ ContentType: "application/json",
73
35
  });
36
+ return R2.send(command);
74
37
  }
75
38
  exports.storeR2JSONString = storeR2JSONString;
76
- function getR2JSONString(filename) {
77
- return __awaiter(this, void 0, void 0, function () {
78
- function _fetchData() {
79
- return __awaiter(this, void 0, void 0, function () {
80
- var response;
81
- return __generator(this, function (_a) {
82
- switch (_a.label) {
83
- case 0: return [4 /*yield*/, fetch("".concat(publicBucketUrl, "/").concat(filename))];
84
- case 1:
85
- response = _a.sent();
86
- return [2 /*return*/, response.text()];
87
- }
88
- });
89
- });
90
- }
91
- var command, Body, e_1;
92
- return __generator(this, function (_a) {
93
- switch (_a.label) {
94
- case 0:
95
- _a.trys.push([0, 5, , 6]);
96
- if (!!R2) return [3 /*break*/, 2];
97
- return [4 /*yield*/, _fetchData()];
98
- case 1: return [2 /*return*/, _a.sent()];
99
- case 2:
100
- command = new GetObjectCommand({ Bucket: datasetBucket, Key: getKey(filename), });
101
- return [4 /*yield*/, R2.send(command)];
102
- case 3:
103
- Body = (_a.sent()).Body;
104
- return [4 /*yield*/, Body.transformToString()];
105
- case 4: return [2 /*return*/, _a.sent()];
106
- case 5:
107
- e_1 = _a.sent();
108
- return [2 /*return*/, {}];
109
- case 6: return [2 /*return*/];
110
- }
111
- });
112
- });
39
+ async function getR2JSONString(filename) {
40
+ try {
41
+ if (!R2)
42
+ return await _fetchData();
43
+ const command = new GetObjectCommand({ Bucket: datasetBucket, Key: getKey(filename), });
44
+ const { Body } = await R2.send(command);
45
+ return await Body.transformToString();
46
+ }
47
+ catch (e) {
48
+ return {};
49
+ }
50
+ async function _fetchData() {
51
+ const response = await (0, axios_1.default)(`${publicBucketUrl}/${filename}`);
52
+ return response.data;
53
+ }
113
54
  }
114
55
  exports.getR2JSONString = getR2JSONString;