@defillama/sdk 5.0.9 → 5.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,241 +1,168 @@
1
1
  "use strict";
2
- var __assign = (this && this.__assign) || function () {
3
- __assign = Object.assign || function(t) {
4
- for (var s, i = 1, n = arguments.length; i < n; i++) {
5
- s = arguments[i];
6
- for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
7
- t[p] = s[p];
8
- }
9
- return t;
10
- };
11
- return __assign.apply(this, arguments);
12
- };
13
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
14
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
15
- return new (P || (P = Promise))(function (resolve, reject) {
16
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
17
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
18
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
19
- step((generator = generator.apply(thisArg, _arguments || [])).next());
20
- });
21
- };
22
- var __generator = (this && this.__generator) || function (thisArg, body) {
23
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
24
- return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
25
- function verb(n) { return function (v) { return step([n, v]); }; }
26
- function step(op) {
27
- if (f) throw new TypeError("Generator is already executing.");
28
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
29
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
30
- if (y = 0, t) op = [op[0] & 2, t.value];
31
- switch (op[0]) {
32
- case 0: case 1: t = op; break;
33
- case 4: _.label++; return { value: op[1], done: false };
34
- case 5: _.label++; y = op[1]; op = [0]; continue;
35
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
36
- default:
37
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
38
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
39
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
40
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
41
- if (t[2]) _.ops.pop();
42
- _.trys.pop(); continue;
43
- }
44
- op = body.call(thisArg, _);
45
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
46
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
47
- }
48
- };
49
2
  var __importDefault = (this && this.__importDefault) || function (mod) {
50
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
51
4
  };
52
5
  Object.defineProperty(exports, "__esModule", { value: true });
53
6
  exports.getLogs = void 0;
54
- var _1 = require(".");
55
- var ethers_1 = require("ethers");
56
- var blocks_1 = require("./blocks");
57
- var cache_1 = require("./cache");
58
- var debugLog_1 = require("./debugLog");
59
- var p_limit_1 = __importDefault(require("p-limit"));
60
- var env_1 = require("./env");
61
- var currentVersion = 'v2';
62
- function getLogs(options) {
63
- return __awaiter(this, void 0, void 0, function () {
64
- function addLogsToCache(fromBlock, toBlock) {
65
- return __awaiter(this, void 0, void 0, function () {
66
- var metadata, logs, logIndices;
67
- var _a;
68
- return __generator(this, function (_b) {
69
- switch (_b.label) {
70
- case 0:
71
- metadata = cacheData.metadata;
72
- return [4 /*yield*/, (0, _1.getLogs)({
73
- chain: chain,
74
- target: target, topic: topic,
75
- keys: keys,
76
- topics: topics,
77
- fromBlock: fromBlock,
78
- toBlock: toBlock,
79
- })];
80
- case 1:
81
- logs = (_b.sent()).output;
82
- (_a = cacheData.logs).push.apply(_a, logs);
83
- logIndices = new Set();
84
- cacheData.logs = cacheData.logs.filter(function (i) {
85
- var _a;
86
- var key = i.transactionHash + ((_a = i.logIndex) !== null && _a !== void 0 ? _a : i.index); // ethers v5 had logIndex, ethers v6 has index
87
- if (!(i.hasOwnProperty('logIndex') || i.hasOwnProperty('index')) || !i.hasOwnProperty('transactionHash')) {
88
- (0, debugLog_1.debugLog)(i, i.logIndex, i.index, i.transactionHash);
89
- throw new Error('Missing crucial field');
90
- }
91
- if (logIndices.has(key))
92
- return false;
93
- logIndices.add(key);
94
- return true;
95
- });
96
- if (!metadata.fromBlock || fromBlock < metadata.fromBlock)
97
- metadata.fromBlock = fromBlock;
98
- if (!metadata.toBlock || toBlock > metadata.toBlock)
99
- metadata.toBlock = toBlock;
100
- if (!!skipCache) return [3 /*break*/, 3];
101
- return [4 /*yield*/, (0, cache_1.writeCache)(getFile(), cacheData, { skipR2CacheWrite: !cacheInCloud })];
102
- case 2:
103
- _b.sent();
104
- _b.label = 3;
105
- case 3: return [2 /*return*/];
106
- }
107
- });
108
- });
109
- }
110
- function _getCache() {
111
- return __awaiter(this, void 0, void 0, function () {
112
- var key, defaultRes, cache;
113
- return __generator(this, function (_a) {
114
- switch (_a.label) {
115
- case 0:
116
- key = getFile();
117
- defaultRes = {
118
- logs: [],
119
- };
120
- if (skipCache || skipCacheRead)
121
- return [2 /*return*/, defaultRes];
122
- return [4 /*yield*/, (0, cache_1.readCache)(key, { skipR2Cache: !cacheInCloud })];
123
- case 1:
124
- cache = _a.sent();
125
- if (!cache.metadata || cache.metadata.version !== currentVersion)
126
- return [2 /*return*/, defaultRes];
127
- return [2 /*return*/, cache];
128
- }
129
- });
130
- });
131
- }
132
- function getFile() {
133
- var extraKey = topics === null || topics === void 0 ? void 0 : topics.join('-').toLowerCase();
134
- if (!extraKey)
135
- throw new Error('extraKey is required');
136
- if (keys.length)
137
- extraKey += '-' + keys.join('-').toLowerCase();
138
- return "event-logs/".concat(chain, "/").concat(target.toLowerCase(), "-").concat(extraKey);
7
+ const _1 = require(".");
8
+ const ethers_1 = require("ethers");
9
+ const blocks_1 = require("./blocks");
10
+ const cache_1 = require("./cache");
11
+ const debugLog_1 = require("./debugLog");
12
+ const p_limit_1 = __importDefault(require("p-limit"));
13
+ const env_1 = require("./env");
14
+ const currentVersion = 'v3';
15
+ async function getLogs(options) {
16
+ let { target, chain = 'ethereum', fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, // can be passed as input as extra filter arguments
17
+ skipCache = false, skipCacheRead = false, entireLog = false, cacheInCloud = false, onlyArgs = false, keys = [], // [Deprecated] This is just used to select only part of the logs
18
+ targets, flatten = true, } = options;
19
+ if (!target && !(targets === null || targets === void 0 ? void 0 : targets.length))
20
+ throw new Error('target|targets is required');
21
+ if (!fromBlock && !fromTimestamp)
22
+ throw new Error('fromBlock or fromTimestamp is required');
23
+ if (!toBlock && !toTimestamp)
24
+ throw new Error('toBlock or toTimestamp is required');
25
+ const limiter = getChainLimiter(chain);
26
+ if (!fromBlock)
27
+ fromBlock = (await limiter(() => (0, blocks_1.getBlock)(chain, fromTimestamp))).block;
28
+ if (!toBlock)
29
+ toBlock = (await limiter(() => (0, blocks_1.getBlock)(chain, toTimestamp))).block;
30
+ if (!fromBlock || !toBlock)
31
+ throw new Error('fromBlock and toBlock must be > 0');
32
+ if (targets === null || targets === void 0 ? void 0 : targets.length) {
33
+ const newOptions = { ...options, fromBlock, toBlock };
34
+ delete newOptions.targets;
35
+ const res = await Promise.all(targets.map(i => getLogs({ ...newOptions, target: i })));
36
+ if (flatten)
37
+ return res.flat();
38
+ return res;
39
+ }
40
+ let iface;
41
+ if (eventAbi)
42
+ iface = new ethers_1.Interface([eventAbi]);
43
+ if ((!topics || !topics.length)) {
44
+ if (topic)
45
+ topic = toFilterTopic(topic);
46
+ else if (eventAbi)
47
+ topic = toFilterTopic(iface);
48
+ else {
49
+ throw new Error('eventAbi | topic | topics are required');
139
50
  }
140
- // we need to form filter topic with indexed keyword, else it messes up generated topic string
141
- function toFilterTopic(topic) {
142
- if (typeof topic === 'string') {
143
- if (topic.startsWith('0x'))
144
- return topic;
145
- topic = new ethers_1.Interface([topic]);
51
+ topics = [topic];
52
+ if (extraTopics)
53
+ topics.push(...extraTopics);
54
+ }
55
+ let caches = await _getCache();
56
+ const isFirstRun = !caches.length;
57
+ if (isFirstRun || skipCacheRead) {
58
+ await limiter(() => addLogsToCache(fromBlock, toBlock));
59
+ }
60
+ else {
61
+ let _fromBlock = fromBlock; // we need to keep a copy of fromBlock as it will be modified
62
+ let _toBlock = toBlock; // we need to keep a copy of toBlock as it will be modified
63
+ const firstCache = caches[0];
64
+ const lastCache = caches[caches.length - 1];
65
+ for (const cacheData of caches) {
66
+ const { fromBlock: cFirstBlock, toBlock: cToBlock } = cacheData.metadata;
67
+ if (_toBlock < firstCache.metadata.fromBlock || _fromBlock > lastCache.metadata.toBlock) { // no intersection with any cache
68
+ await limiter(() => addLogsToCache(_fromBlock, _toBlock));
69
+ break;
70
+ }
71
+ if (_fromBlock >= cToBlock)
72
+ continue; // request is after cache end
73
+ if (_fromBlock <= cFirstBlock) { // request is before cache start
74
+ await limiter(() => addLogsToCache(_fromBlock, Math.min(_toBlock, cFirstBlock - 1)));
146
75
  }
147
- var fragment = topic.fragments[0];
148
- return (0, ethers_1.id)("".concat(fragment.name, "(").concat(fragment.inputs.map(function (i) { return i.type; }).join(','), ")"));
76
+ if (_toBlock <= cToBlock)
77
+ break; // request ends before cache end
78
+ _fromBlock = cToBlock;
149
79
  }
150
- var target, _a, chain, fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, _b, skipCache, _c, skipCacheRead, _d, entireLog, _e, cacheInCloud, _f, onlyArgs, _g, keys, targets, _h, flatten, limiter, newOptions_1, res, iface, cacheData, isFirstRun, metadata_1, logs;
151
- return __generator(this, function (_j) {
152
- switch (_j.label) {
153
- case 0:
154
- target = options.target, _a = options.chain, chain = _a === void 0 ? 'ethereum' : _a, fromBlock = options.fromBlock, toBlock = options.toBlock, fromTimestamp = options.fromTimestamp, toTimestamp = options.toTimestamp, eventAbi = options.eventAbi, topic = options.topic, topics = options.topics, extraTopics = options.extraTopics, _b = options.skipCache, skipCache = _b === void 0 ? false : _b, _c = options.skipCacheRead, skipCacheRead = _c === void 0 ? false : _c, _d = options.entireLog, entireLog = _d === void 0 ? false : _d, _e = options.cacheInCloud, cacheInCloud = _e === void 0 ? false : _e, _f = options.onlyArgs, onlyArgs = _f === void 0 ? false : _f, _g = options.keys, keys = _g === void 0 ? [] : _g, targets = options.targets, _h = options.flatten, flatten = _h === void 0 ? true : _h;
155
- if (!target && !(targets === null || targets === void 0 ? void 0 : targets.length))
156
- throw new Error('target|targets is required');
157
- if (!fromBlock && !fromTimestamp)
158
- throw new Error('fromBlock or fromTimestamp is required');
159
- if (!toBlock && !toTimestamp)
160
- throw new Error('toBlock or toTimestamp is required');
161
- limiter = getChainLimiter(chain);
162
- if (!!fromBlock) return [3 /*break*/, 2];
163
- return [4 /*yield*/, limiter(function () { return (0, blocks_1.getBlock)(chain, fromTimestamp); })];
164
- case 1:
165
- fromBlock = (_j.sent()).block;
166
- _j.label = 2;
167
- case 2:
168
- if (!!toBlock) return [3 /*break*/, 4];
169
- return [4 /*yield*/, limiter(function () { return (0, blocks_1.getBlock)(chain, toTimestamp); })];
170
- case 3:
171
- toBlock = (_j.sent()).block;
172
- _j.label = 4;
173
- case 4:
174
- if (!(targets === null || targets === void 0 ? void 0 : targets.length)) return [3 /*break*/, 6];
175
- newOptions_1 = __assign(__assign({}, options), { fromBlock: fromBlock, toBlock: toBlock });
176
- delete newOptions_1.targets;
177
- return [4 /*yield*/, Promise.all(targets.map(function (i) { return getLogs(__assign(__assign({}, newOptions_1), { target: i })); }))];
178
- case 5:
179
- res = _j.sent();
180
- if (flatten)
181
- return [2 /*return*/, res.flat()];
182
- return [2 /*return*/, res];
183
- case 6:
184
- if (eventAbi)
185
- iface = new ethers_1.Interface([eventAbi]);
186
- if ((!topics || !topics.length)) {
187
- if (topic)
188
- topic = toFilterTopic(topic);
189
- else if (eventAbi)
190
- topic = toFilterTopic(iface);
191
- else {
192
- throw new Error('eventAbi | topic | topics are required');
193
- }
194
- topics = [topic];
195
- if (extraTopics)
196
- topics.push.apply(topics, extraTopics);
197
- }
198
- return [4 /*yield*/, _getCache()];
199
- case 7:
200
- cacheData = _j.sent();
201
- isFirstRun = !cacheData.metadata;
202
- if (isFirstRun)
203
- cacheData.metadata = { version: currentVersion };
204
- if (!isFirstRun) return [3 /*break*/, 9];
205
- return [4 /*yield*/, limiter(function () { return addLogsToCache(fromBlock, toBlock); })];
206
- case 8:
207
- _j.sent();
208
- return [3 /*break*/, 13];
209
- case 9:
210
- metadata_1 = cacheData.metadata;
211
- if (!(fromBlock < metadata_1.fromBlock)) return [3 /*break*/, 11];
212
- return [4 /*yield*/, limiter(function () { return addLogsToCache(fromBlock, metadata_1.fromBlock); })];
213
- case 10:
214
- _j.sent();
215
- _j.label = 11;
216
- case 11:
217
- if (!(toBlock > metadata_1.toBlock)) return [3 /*break*/, 13];
218
- return [4 /*yield*/, limiter(function () { return addLogsToCache(metadata_1.toBlock, toBlock); })];
219
- case 12:
220
- _j.sent();
221
- _j.label = 13;
222
- case 13:
223
- logs = cacheData.logs.filter(function (i) {
224
- if (i.blockNumber < fromBlock)
225
- return false;
226
- if (i.blockNumber > toBlock)
227
- return false;
228
- return true;
229
- });
230
- if (!eventAbi || entireLog)
231
- return [2 /*return*/, logs];
232
- return [2 /*return*/, logs.map(function (i) { return iface.parseLog(i); }).map(function (i) { return onlyArgs ? i.args : i; })];
80
+ }
81
+ const logs = [];
82
+ for (const cacheData of caches) {
83
+ const { fromBlock: cFirstBlock, toBlock: cToBlock } = cacheData.metadata;
84
+ if (fromBlock > cToBlock || toBlock < cFirstBlock)
85
+ continue; // no intersection with cache
86
+ logs.push(...cacheData.logs.filter((i) => i.blockNumber >= fromBlock && i.blockNumber <= toBlock));
87
+ }
88
+ if (!eventAbi || entireLog)
89
+ return logs;
90
+ return logs.map((i) => iface.parseLog(i)).map((i) => onlyArgs ? i.args : i);
91
+ async function addLogsToCache(fromBlock, toBlock) {
92
+ (0, debugLog_1.debugLog)('adding logs to cache: ', fromBlock, toBlock, target, topic);
93
+ if (fromBlock > toBlock)
94
+ return; // no data to add
95
+ fromBlock = fromBlock - 10;
96
+ toBlock = toBlock + 10;
97
+ let { output: logs } = await (0, _1.getLogs)({
98
+ chain, target: target, topic: topic, keys, topics, fromBlock, toBlock,
99
+ });
100
+ caches.push({
101
+ logs,
102
+ metadata: { fromBlock, toBlock, }
103
+ });
104
+ caches.sort((a, b) => a.metadata.fromBlock - b.metadata.fromBlock);
105
+ const mergedCaches = [caches[0]];
106
+ caches.slice(1).forEach(i => {
107
+ const last = mergedCaches[mergedCaches.length - 1];
108
+ if (last.metadata.toBlock + 1 > i.metadata.fromBlock) {
109
+ last.metadata.toBlock = i.metadata.toBlock;
110
+ last.logs = dedupLogs(last.logs, i.logs);
111
+ }
112
+ else {
113
+ mergedCaches.push(i);
114
+ }
115
+ });
116
+ caches = mergedCaches;
117
+ if (!skipCache)
118
+ await (0, cache_1.writeCache)(getFile(), { caches, version: currentVersion }, { skipR2CacheWrite: !cacheInCloud });
119
+ }
120
+ function dedupLogs(...logs) {
121
+ const logIndices = new Set();
122
+ return logs.flat().filter((i) => {
123
+ var _a;
124
+ let key = i.transactionHash + ((_a = i.logIndex) !== null && _a !== void 0 ? _a : i.index); // ethers v5 had logIndex, ethers v6 has index
125
+ if (!(i.hasOwnProperty('logIndex') || i.hasOwnProperty('index')) || !i.hasOwnProperty('transactionHash')) {
126
+ (0, debugLog_1.debugLog)(i, i.logIndex, i.index, i.transactionHash);
127
+ throw new Error('Missing crucial field: logIndex/index');
233
128
  }
129
+ if (logIndices.has(key))
130
+ return false;
131
+ logIndices.add(key);
132
+ return true;
234
133
  });
235
- });
134
+ }
135
+ async function _getCache() {
136
+ const key = getFile();
137
+ const defaultRes = [];
138
+ if (skipCache)
139
+ return defaultRes;
140
+ let cache = await (0, cache_1.readCache)(key, { skipR2Cache: !cacheInCloud });
141
+ if (!cache.caches || !cache.caches.length || cache.version !== currentVersion)
142
+ return defaultRes;
143
+ return cache.caches;
144
+ }
145
+ function getFile() {
146
+ let extraKey = topics === null || topics === void 0 ? void 0 : topics.join('-').toLowerCase();
147
+ if (!extraKey)
148
+ throw new Error('extraKey is required');
149
+ if (keys.length)
150
+ extraKey += '-' + keys.join('-').toLowerCase();
151
+ return `event-logs/${chain}/${target.toLowerCase()}-${extraKey}`;
152
+ }
153
+ // we need to form filter topic with indexed keyword, else it messes up generated topic string
154
+ function toFilterTopic(topic) {
155
+ if (typeof topic === 'string') {
156
+ if (topic.startsWith('0x'))
157
+ return topic;
158
+ topic = new ethers_1.Interface([topic]);
159
+ }
160
+ const fragment = topic.fragments[0];
161
+ return (0, ethers_1.id)(`${fragment.name}(${fragment.inputs.map((i) => i.type).join(',')})`);
162
+ }
236
163
  }
237
164
  exports.getLogs = getLogs;
238
- var chainLimiters = {};
165
+ const chainLimiters = {};
239
166
  function getChainLimiter(chain) {
240
167
  if (!chainLimiters[chain])
241
168
  chainLimiters[chain] = createLimiter(chain);
@@ -1,67 +1,16 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __generator = (this && this.__generator) || function (thisArg, body) {
12
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
13
- return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14
- function verb(n) { return function (v) { return step([n, v]); }; }
15
- function step(op) {
16
- if (f) throw new TypeError("Generator is already executing.");
17
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
18
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19
- if (y = 0, t) op = [op[0] & 2, t.value];
20
- switch (op[0]) {
21
- case 0: case 1: t = op; break;
22
- case 4: _.label++; return { value: op[1], done: false };
23
- case 5: _.label++; y = op[1]; op = [0]; continue;
24
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
25
- default:
26
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30
- if (t[2]) _.ops.pop();
31
- _.trys.pop(); continue;
32
- }
33
- op = body.call(thisArg, _);
34
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36
- }
37
- };
38
2
  Object.defineProperty(exports, "__esModule", { value: true });
39
- var promise_pool_1 = require("@supercharge/promise-pool");
40
- var generalUtil_1 = require("../generalUtil");
3
+ const promise_pool_1 = require("@supercharge/promise-pool");
4
+ const generalUtil_1 = require("../generalUtil");
41
5
  generalUtil_1.formError;
42
- function runInPromisePool(params) {
43
- return __awaiter(this, void 0, void 0, function () {
44
- var _a, results, errors, flatResults;
45
- var _this = this;
46
- return __generator(this, function (_b) {
47
- switch (_b.label) {
48
- case 0: return [4 /*yield*/, promise_pool_1.PromisePool
49
- .withConcurrency(params.concurrency)
50
- .for(params.items)
51
- .process(function (item, i) { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
52
- switch (_a.label) {
53
- case 0: return [4 /*yield*/, params.processor(item, i)];
54
- case 1: return [2 /*return*/, [_a.sent(), i]];
55
- }
56
- }); }); })];
57
- case 1:
58
- _a = _b.sent(), results = _a.results, errors = _a.errors;
59
- if (errors.length)
60
- throw (0, generalUtil_1.formError)(null, { promisePoolErrors: errors });
61
- flatResults = results.sort(function (a, b) { return a[1] - b[1]; }).map((function (i) { return i[0]; }));
62
- return [2 /*return*/, flatResults];
63
- }
64
- });
65
- });
6
+ async function runInPromisePool(params) {
7
+ const { results, errors } = await promise_pool_1.PromisePool
8
+ .withConcurrency(params.concurrency)
9
+ .for(params.items)
10
+ .process(async (item, i) => [await params.processor(item, i), i]);
11
+ if (errors.length)
12
+ throw (0, generalUtil_1.formError)(null, { promisePoolErrors: errors });
13
+ const flatResults = results.sort((a, b) => a[1] - b[1]).map((i => i[0]));
14
+ return flatResults;
66
15
  }
67
16
  exports.default = runInPromisePool;
package/build/util/r2.js CHANGED
@@ -1,48 +1,16 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __generator = (this && this.__generator) || function (thisArg, body) {
12
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
13
- return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14
- function verb(n) { return function (v) { return step([n, v]); }; }
15
- function step(op) {
16
- if (f) throw new TypeError("Generator is already executing.");
17
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
18
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19
- if (y = 0, t) op = [op[0] & 2, t.value];
20
- switch (op[0]) {
21
- case 0: case 1: t = op; break;
22
- case 4: _.label++; return { value: op[1], done: false };
23
- case 5: _.label++; y = op[1]; op = [0]; continue;
24
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
25
- default:
26
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30
- if (t[2]) _.ops.pop();
31
- _.trys.pop(); continue;
32
- }
33
- op = body.call(thisArg, _);
34
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36
- }
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
37
4
  };
38
5
  Object.defineProperty(exports, "__esModule", { value: true });
39
6
  exports.getR2JSONString = exports.storeR2JSONString = void 0;
40
- var env_1 = require("./env");
41
- var _a = require("@aws-sdk/client-s3"), S3Client = _a.S3Client, PutObjectCommand = _a.PutObjectCommand, GetObjectCommand = _a.GetObjectCommand;
42
- var R2_ENDPOINT = env_1.ENV_CONSTANTS.R2_ENDPOINT, R2_ACCESS_KEY_ID = env_1.ENV_CONSTANTS.R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY = env_1.ENV_CONSTANTS.R2_SECRET_ACCESS_KEY;
43
- var datasetBucket = "defillama-datasets";
44
- var publicBucketUrl = "https://defillama-datasets.llama.fi";
45
- var R2;
7
+ const axios_1 = __importDefault(require("axios"));
8
+ const env_1 = require("./env");
9
+ const { S3Client, PutObjectCommand, GetObjectCommand, } = require("@aws-sdk/client-s3");
10
+ const { R2_ENDPOINT, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY } = env_1.ENV_CONSTANTS;
11
+ const datasetBucket = "defillama-datasets";
12
+ const publicBucketUrl = "https://defillama-datasets.llama.fi";
13
+ let R2;
46
14
  if (R2_ENDPOINT && R2_ACCESS_KEY_ID && R2_SECRET_ACCESS_KEY) {
47
15
  R2 = new S3Client({
48
16
  region: "auto",
@@ -53,62 +21,35 @@ if (R2_ENDPOINT && R2_ACCESS_KEY_ID && R2_SECRET_ACCESS_KEY) {
53
21
  },
54
22
  });
55
23
  }
56
- var getKey = function (filename) { return filename.replace(/(:|'|#)/g, '/'); };
57
- function storeR2JSONString(filename, body) {
58
- return __awaiter(this, void 0, void 0, function () {
59
- var command;
60
- return __generator(this, function (_a) {
61
- if (typeof body !== 'string')
62
- body = body.toString('base64');
63
- if (!R2)
64
- return [2 /*return*/];
65
- command = new PutObjectCommand({
66
- Bucket: datasetBucket,
67
- Key: getKey(filename),
68
- Body: body,
69
- ContentType: "application/json",
70
- });
71
- return [2 /*return*/, R2.send(command)];
72
- });
24
+ const getKey = (filename) => filename.replace(/(:|'|#)/g, '/');
25
+ async function storeR2JSONString(filename, body) {
26
+ if (typeof body !== 'string')
27
+ body = body.toString('base64');
28
+ if (!R2)
29
+ return;
30
+ const command = new PutObjectCommand({
31
+ Bucket: datasetBucket,
32
+ Key: getKey(filename),
33
+ Body: body,
34
+ ContentType: "application/json",
73
35
  });
36
+ return R2.send(command);
74
37
  }
75
38
  exports.storeR2JSONString = storeR2JSONString;
76
- function getR2JSONString(filename) {
77
- return __awaiter(this, void 0, void 0, function () {
78
- function _fetchData() {
79
- return __awaiter(this, void 0, void 0, function () {
80
- var response;
81
- return __generator(this, function (_a) {
82
- switch (_a.label) {
83
- case 0: return [4 /*yield*/, fetch("".concat(publicBucketUrl, "/").concat(filename))];
84
- case 1:
85
- response = _a.sent();
86
- return [2 /*return*/, response.text()];
87
- }
88
- });
89
- });
90
- }
91
- var command, Body, e_1;
92
- return __generator(this, function (_a) {
93
- switch (_a.label) {
94
- case 0:
95
- _a.trys.push([0, 5, , 6]);
96
- if (!!R2) return [3 /*break*/, 2];
97
- return [4 /*yield*/, _fetchData()];
98
- case 1: return [2 /*return*/, _a.sent()];
99
- case 2:
100
- command = new GetObjectCommand({ Bucket: datasetBucket, Key: getKey(filename), });
101
- return [4 /*yield*/, R2.send(command)];
102
- case 3:
103
- Body = (_a.sent()).Body;
104
- return [4 /*yield*/, Body.transformToString()];
105
- case 4: return [2 /*return*/, _a.sent()];
106
- case 5:
107
- e_1 = _a.sent();
108
- return [2 /*return*/, {}];
109
- case 6: return [2 /*return*/];
110
- }
111
- });
112
- });
39
+ async function getR2JSONString(filename) {
40
+ try {
41
+ if (!R2)
42
+ return await _fetchData();
43
+ const command = new GetObjectCommand({ Bucket: datasetBucket, Key: getKey(filename), });
44
+ const { Body } = await R2.send(command);
45
+ return await Body.transformToString();
46
+ }
47
+ catch (e) {
48
+ return {};
49
+ }
50
+ async function _fetchData() {
51
+ const response = await (0, axios_1.default)(`${publicBucketUrl}/${filename}`);
52
+ return response.data;
53
+ }
113
54
  }
114
55
  exports.getR2JSONString = getR2JSONString;