@defillama/sdk 5.0.10 → 5.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/Balances.d.ts +5 -0
- package/build/Balances.js +47 -107
- package/build/ChainApi.d.ts +1 -0
- package/build/ChainApi.js +152 -253
- package/build/abi/abi2.js +47 -144
- package/build/abi/convertResults.js +9 -29
- package/build/abi/index.js +220 -326
- package/build/abi/multicall.js +115 -217
- package/build/abi/multicall3.js +59 -123
- package/build/abi/rpcCall.js +43 -106
- package/build/abi/tron.js +153 -283
- package/build/api.js +3 -4
- package/build/computeTVL/humanizeNumber.js +6 -7
- package/build/computeTVL/prices.js +55 -145
- package/build/erc20/index.js +34 -122
- package/build/eth/index.js +37 -107
- package/build/general.js +22 -23
- package/build/generalUtil.js +68 -127
- package/build/index.js +2 -2
- package/build/providers.json +1 -1
- package/build/util/blocks.js +212 -371
- package/build/util/blockscout.js +13 -58
- package/build/util/cache.js +112 -274
- package/build/util/computeTVL.js +70 -150
- package/build/util/cosmos.js +21 -75
- package/build/util/debugLog.js +5 -13
- package/build/util/env.d.ts +1 -1
- package/build/util/env.js +19 -32
- package/build/util/index.js +72 -132
- package/build/util/internal-cache.js +7 -8
- package/build/util/logs.d.ts +1 -2
- package/build/util/logs.js +152 -225
- package/build/util/promisePool.js +11 -62
- package/build/util/r2.js +36 -95
- package/build/util/updateProviderList.js +23 -83
- package/package.json +2 -3
package/build/util/logs.js
CHANGED
|
@@ -1,241 +1,168 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __assign = (this && this.__assign) || function () {
|
|
3
|
-
__assign = Object.assign || function(t) {
|
|
4
|
-
for (var s, i = 1, n = arguments.length; i < n; i++) {
|
|
5
|
-
s = arguments[i];
|
|
6
|
-
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
|
|
7
|
-
t[p] = s[p];
|
|
8
|
-
}
|
|
9
|
-
return t;
|
|
10
|
-
};
|
|
11
|
-
return __assign.apply(this, arguments);
|
|
12
|
-
};
|
|
13
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
14
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
15
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
16
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
17
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
18
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
19
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
20
|
-
});
|
|
21
|
-
};
|
|
22
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
23
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
24
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
25
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
26
|
-
function step(op) {
|
|
27
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
28
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
29
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
30
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
31
|
-
switch (op[0]) {
|
|
32
|
-
case 0: case 1: t = op; break;
|
|
33
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
34
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
35
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
36
|
-
default:
|
|
37
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
38
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
39
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
40
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
41
|
-
if (t[2]) _.ops.pop();
|
|
42
|
-
_.trys.pop(); continue;
|
|
43
|
-
}
|
|
44
|
-
op = body.call(thisArg, _);
|
|
45
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
46
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
47
|
-
}
|
|
48
|
-
};
|
|
49
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
50
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
51
4
|
};
|
|
52
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
53
6
|
exports.getLogs = void 0;
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
function getLogs(options) {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
metadata.fromBlock = fromBlock;
|
|
98
|
-
if (!metadata.toBlock || toBlock > metadata.toBlock)
|
|
99
|
-
metadata.toBlock = toBlock;
|
|
100
|
-
if (!!skipCache) return [3 /*break*/, 3];
|
|
101
|
-
return [4 /*yield*/, (0, cache_1.writeCache)(getFile(), cacheData, { skipR2CacheWrite: !cacheInCloud })];
|
|
102
|
-
case 2:
|
|
103
|
-
_b.sent();
|
|
104
|
-
_b.label = 3;
|
|
105
|
-
case 3: return [2 /*return*/];
|
|
106
|
-
}
|
|
107
|
-
});
|
|
108
|
-
});
|
|
109
|
-
}
|
|
110
|
-
function _getCache() {
|
|
111
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
112
|
-
var key, defaultRes, cache;
|
|
113
|
-
return __generator(this, function (_a) {
|
|
114
|
-
switch (_a.label) {
|
|
115
|
-
case 0:
|
|
116
|
-
key = getFile();
|
|
117
|
-
defaultRes = {
|
|
118
|
-
logs: [],
|
|
119
|
-
};
|
|
120
|
-
if (skipCache || skipCacheRead)
|
|
121
|
-
return [2 /*return*/, defaultRes];
|
|
122
|
-
return [4 /*yield*/, (0, cache_1.readCache)(key, { skipR2Cache: !cacheInCloud })];
|
|
123
|
-
case 1:
|
|
124
|
-
cache = _a.sent();
|
|
125
|
-
if (!cache.metadata || cache.metadata.version !== currentVersion)
|
|
126
|
-
return [2 /*return*/, defaultRes];
|
|
127
|
-
return [2 /*return*/, cache];
|
|
128
|
-
}
|
|
129
|
-
});
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
function getFile() {
|
|
133
|
-
var extraKey = topics === null || topics === void 0 ? void 0 : topics.join('-').toLowerCase();
|
|
134
|
-
if (!extraKey)
|
|
135
|
-
throw new Error('extraKey is required');
|
|
136
|
-
if (keys.length)
|
|
137
|
-
extraKey += '-' + keys.join('-').toLowerCase();
|
|
138
|
-
return "event-logs/".concat(chain, "/").concat(target.toLowerCase(), "-").concat(extraKey);
|
|
7
|
+
const _1 = require(".");
|
|
8
|
+
const ethers_1 = require("ethers");
|
|
9
|
+
const blocks_1 = require("./blocks");
|
|
10
|
+
const cache_1 = require("./cache");
|
|
11
|
+
const debugLog_1 = require("./debugLog");
|
|
12
|
+
const p_limit_1 = __importDefault(require("p-limit"));
|
|
13
|
+
const env_1 = require("./env");
|
|
14
|
+
const currentVersion = 'v3';
|
|
15
|
+
async function getLogs(options) {
|
|
16
|
+
let { target, chain = 'ethereum', fromBlock, toBlock, fromTimestamp, toTimestamp, eventAbi, topic, topics, extraTopics, // can be passed as input as extra filter arguments
|
|
17
|
+
skipCache = false, skipCacheRead = false, entireLog = false, cacheInCloud = false, onlyArgs = false, keys = [], // [Deprecated] This is just used to select only part of the logs
|
|
18
|
+
targets, flatten = true, } = options;
|
|
19
|
+
if (!target && !(targets === null || targets === void 0 ? void 0 : targets.length))
|
|
20
|
+
throw new Error('target|targets is required');
|
|
21
|
+
if (!fromBlock && !fromTimestamp)
|
|
22
|
+
throw new Error('fromBlock or fromTimestamp is required');
|
|
23
|
+
if (!toBlock && !toTimestamp)
|
|
24
|
+
throw new Error('toBlock or toTimestamp is required');
|
|
25
|
+
const limiter = getChainLimiter(chain);
|
|
26
|
+
if (!fromBlock)
|
|
27
|
+
fromBlock = (await limiter(() => (0, blocks_1.getBlock)(chain, fromTimestamp))).block;
|
|
28
|
+
if (!toBlock)
|
|
29
|
+
toBlock = (await limiter(() => (0, blocks_1.getBlock)(chain, toTimestamp))).block;
|
|
30
|
+
if (!fromBlock || !toBlock)
|
|
31
|
+
throw new Error('fromBlock and toBlock must be > 0');
|
|
32
|
+
if (targets === null || targets === void 0 ? void 0 : targets.length) {
|
|
33
|
+
const newOptions = { ...options, fromBlock, toBlock };
|
|
34
|
+
delete newOptions.targets;
|
|
35
|
+
const res = await Promise.all(targets.map(i => getLogs({ ...newOptions, target: i })));
|
|
36
|
+
if (flatten)
|
|
37
|
+
return res.flat();
|
|
38
|
+
return res;
|
|
39
|
+
}
|
|
40
|
+
let iface;
|
|
41
|
+
if (eventAbi)
|
|
42
|
+
iface = new ethers_1.Interface([eventAbi]);
|
|
43
|
+
if ((!topics || !topics.length)) {
|
|
44
|
+
if (topic)
|
|
45
|
+
topic = toFilterTopic(topic);
|
|
46
|
+
else if (eventAbi)
|
|
47
|
+
topic = toFilterTopic(iface);
|
|
48
|
+
else {
|
|
49
|
+
throw new Error('eventAbi | topic | topics are required');
|
|
139
50
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
51
|
+
topics = [topic];
|
|
52
|
+
if (extraTopics)
|
|
53
|
+
topics.push(...extraTopics);
|
|
54
|
+
}
|
|
55
|
+
let caches = await _getCache();
|
|
56
|
+
const isFirstRun = !caches.length;
|
|
57
|
+
if (isFirstRun || skipCacheRead) {
|
|
58
|
+
await limiter(() => addLogsToCache(fromBlock, toBlock));
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
let _fromBlock = fromBlock; // we need to keep a copy of fromBlock as it will be modified
|
|
62
|
+
let _toBlock = toBlock; // we need to keep a copy of toBlock as it will be modified
|
|
63
|
+
const firstCache = caches[0];
|
|
64
|
+
const lastCache = caches[caches.length - 1];
|
|
65
|
+
for (const cacheData of caches) {
|
|
66
|
+
const { fromBlock: cFirstBlock, toBlock: cToBlock } = cacheData.metadata;
|
|
67
|
+
if (_toBlock < firstCache.metadata.fromBlock || _fromBlock > lastCache.metadata.toBlock) { // no intersection with any cache
|
|
68
|
+
await limiter(() => addLogsToCache(_fromBlock, _toBlock));
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
if (_fromBlock >= cToBlock)
|
|
72
|
+
continue; // request is after cache end
|
|
73
|
+
if (_fromBlock <= cFirstBlock) { // request is before cache start
|
|
74
|
+
await limiter(() => addLogsToCache(_fromBlock, Math.min(_toBlock, cFirstBlock - 1)));
|
|
146
75
|
}
|
|
147
|
-
|
|
148
|
-
|
|
76
|
+
if (_toBlock <= cToBlock)
|
|
77
|
+
break; // request ends before cache end
|
|
78
|
+
_fromBlock = cToBlock;
|
|
149
79
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
return [4 /*yield*/, _getCache()];
|
|
199
|
-
case 7:
|
|
200
|
-
cacheData = _j.sent();
|
|
201
|
-
isFirstRun = !cacheData.metadata;
|
|
202
|
-
if (isFirstRun)
|
|
203
|
-
cacheData.metadata = { version: currentVersion };
|
|
204
|
-
if (!isFirstRun) return [3 /*break*/, 9];
|
|
205
|
-
return [4 /*yield*/, limiter(function () { return addLogsToCache(fromBlock, toBlock); })];
|
|
206
|
-
case 8:
|
|
207
|
-
_j.sent();
|
|
208
|
-
return [3 /*break*/, 13];
|
|
209
|
-
case 9:
|
|
210
|
-
metadata_1 = cacheData.metadata;
|
|
211
|
-
if (!(fromBlock < metadata_1.fromBlock)) return [3 /*break*/, 11];
|
|
212
|
-
return [4 /*yield*/, limiter(function () { return addLogsToCache(fromBlock, metadata_1.fromBlock); })];
|
|
213
|
-
case 10:
|
|
214
|
-
_j.sent();
|
|
215
|
-
_j.label = 11;
|
|
216
|
-
case 11:
|
|
217
|
-
if (!(toBlock > metadata_1.toBlock)) return [3 /*break*/, 13];
|
|
218
|
-
return [4 /*yield*/, limiter(function () { return addLogsToCache(metadata_1.toBlock, toBlock); })];
|
|
219
|
-
case 12:
|
|
220
|
-
_j.sent();
|
|
221
|
-
_j.label = 13;
|
|
222
|
-
case 13:
|
|
223
|
-
logs = cacheData.logs.filter(function (i) {
|
|
224
|
-
if (i.blockNumber < fromBlock)
|
|
225
|
-
return false;
|
|
226
|
-
if (i.blockNumber > toBlock)
|
|
227
|
-
return false;
|
|
228
|
-
return true;
|
|
229
|
-
});
|
|
230
|
-
if (!eventAbi || entireLog)
|
|
231
|
-
return [2 /*return*/, logs];
|
|
232
|
-
return [2 /*return*/, logs.map(function (i) { return iface.parseLog(i); }).map(function (i) { return onlyArgs ? i.args : i; })];
|
|
80
|
+
}
|
|
81
|
+
const logs = [];
|
|
82
|
+
for (const cacheData of caches) {
|
|
83
|
+
const { fromBlock: cFirstBlock, toBlock: cToBlock } = cacheData.metadata;
|
|
84
|
+
if (fromBlock > cToBlock || toBlock < cFirstBlock)
|
|
85
|
+
continue; // no intersection with cache
|
|
86
|
+
logs.push(...cacheData.logs.filter((i) => i.blockNumber >= fromBlock && i.blockNumber <= toBlock));
|
|
87
|
+
}
|
|
88
|
+
if (!eventAbi || entireLog)
|
|
89
|
+
return logs;
|
|
90
|
+
return logs.map((i) => iface.parseLog(i)).map((i) => onlyArgs ? i.args : i);
|
|
91
|
+
async function addLogsToCache(fromBlock, toBlock) {
|
|
92
|
+
(0, debugLog_1.debugLog)('adding logs to cache: ', fromBlock, toBlock, target, topic);
|
|
93
|
+
if (fromBlock > toBlock)
|
|
94
|
+
return; // no data to add
|
|
95
|
+
fromBlock = fromBlock - 10;
|
|
96
|
+
toBlock = toBlock + 10;
|
|
97
|
+
let { output: logs } = await (0, _1.getLogs)({
|
|
98
|
+
chain, target: target, topic: topic, keys, topics, fromBlock, toBlock,
|
|
99
|
+
});
|
|
100
|
+
caches.push({
|
|
101
|
+
logs,
|
|
102
|
+
metadata: { fromBlock, toBlock, }
|
|
103
|
+
});
|
|
104
|
+
caches.sort((a, b) => a.metadata.fromBlock - b.metadata.fromBlock);
|
|
105
|
+
const mergedCaches = [caches[0]];
|
|
106
|
+
caches.slice(1).forEach(i => {
|
|
107
|
+
const last = mergedCaches[mergedCaches.length - 1];
|
|
108
|
+
if (last.metadata.toBlock + 1 > i.metadata.fromBlock) {
|
|
109
|
+
last.metadata.toBlock = i.metadata.toBlock;
|
|
110
|
+
last.logs = dedupLogs(last.logs, i.logs);
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
mergedCaches.push(i);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
caches = mergedCaches;
|
|
117
|
+
if (!skipCache)
|
|
118
|
+
await (0, cache_1.writeCache)(getFile(), { caches, version: currentVersion }, { skipR2CacheWrite: !cacheInCloud });
|
|
119
|
+
}
|
|
120
|
+
function dedupLogs(...logs) {
|
|
121
|
+
const logIndices = new Set();
|
|
122
|
+
return logs.flat().filter((i) => {
|
|
123
|
+
var _a;
|
|
124
|
+
let key = i.transactionHash + ((_a = i.logIndex) !== null && _a !== void 0 ? _a : i.index); // ethers v5 had logIndex, ethers v6 has index
|
|
125
|
+
if (!(i.hasOwnProperty('logIndex') || i.hasOwnProperty('index')) || !i.hasOwnProperty('transactionHash')) {
|
|
126
|
+
(0, debugLog_1.debugLog)(i, i.logIndex, i.index, i.transactionHash);
|
|
127
|
+
throw new Error('Missing crucial field: logIndex/index');
|
|
233
128
|
}
|
|
129
|
+
if (logIndices.has(key))
|
|
130
|
+
return false;
|
|
131
|
+
logIndices.add(key);
|
|
132
|
+
return true;
|
|
234
133
|
});
|
|
235
|
-
}
|
|
134
|
+
}
|
|
135
|
+
async function _getCache() {
|
|
136
|
+
const key = getFile();
|
|
137
|
+
const defaultRes = [];
|
|
138
|
+
if (skipCache)
|
|
139
|
+
return defaultRes;
|
|
140
|
+
let cache = await (0, cache_1.readCache)(key, { skipR2Cache: !cacheInCloud });
|
|
141
|
+
if (!cache.caches || !cache.caches.length || cache.version !== currentVersion)
|
|
142
|
+
return defaultRes;
|
|
143
|
+
return cache.caches;
|
|
144
|
+
}
|
|
145
|
+
function getFile() {
|
|
146
|
+
let extraKey = topics === null || topics === void 0 ? void 0 : topics.join('-').toLowerCase();
|
|
147
|
+
if (!extraKey)
|
|
148
|
+
throw new Error('extraKey is required');
|
|
149
|
+
if (keys.length)
|
|
150
|
+
extraKey += '-' + keys.join('-').toLowerCase();
|
|
151
|
+
return `event-logs/${chain}/${target.toLowerCase()}-${extraKey}`;
|
|
152
|
+
}
|
|
153
|
+
// we need to form filter topic with indexed keyword, else it messes up generated topic string
|
|
154
|
+
function toFilterTopic(topic) {
|
|
155
|
+
if (typeof topic === 'string') {
|
|
156
|
+
if (topic.startsWith('0x'))
|
|
157
|
+
return topic;
|
|
158
|
+
topic = new ethers_1.Interface([topic]);
|
|
159
|
+
}
|
|
160
|
+
const fragment = topic.fragments[0];
|
|
161
|
+
return (0, ethers_1.id)(`${fragment.name}(${fragment.inputs.map((i) => i.type).join(',')})`);
|
|
162
|
+
}
|
|
236
163
|
}
|
|
237
164
|
exports.getLogs = getLogs;
|
|
238
|
-
|
|
165
|
+
const chainLimiters = {};
|
|
239
166
|
function getChainLimiter(chain) {
|
|
240
167
|
if (!chainLimiters[chain])
|
|
241
168
|
chainLimiters[chain] = createLimiter(chain);
|
|
@@ -1,67 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
12
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
13
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
14
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
15
|
-
function step(op) {
|
|
16
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
17
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
18
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
19
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
20
|
-
switch (op[0]) {
|
|
21
|
-
case 0: case 1: t = op; break;
|
|
22
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
23
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
24
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
25
|
-
default:
|
|
26
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
27
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
28
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
29
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
30
|
-
if (t[2]) _.ops.pop();
|
|
31
|
-
_.trys.pop(); continue;
|
|
32
|
-
}
|
|
33
|
-
op = body.call(thisArg, _);
|
|
34
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
35
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
|
|
40
|
-
|
|
3
|
+
const promise_pool_1 = require("@supercharge/promise-pool");
|
|
4
|
+
const generalUtil_1 = require("../generalUtil");
|
|
41
5
|
generalUtil_1.formError;
|
|
42
|
-
function runInPromisePool(params) {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
.process(function (item, i) { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
|
|
52
|
-
switch (_a.label) {
|
|
53
|
-
case 0: return [4 /*yield*/, params.processor(item, i)];
|
|
54
|
-
case 1: return [2 /*return*/, [_a.sent(), i]];
|
|
55
|
-
}
|
|
56
|
-
}); }); })];
|
|
57
|
-
case 1:
|
|
58
|
-
_a = _b.sent(), results = _a.results, errors = _a.errors;
|
|
59
|
-
if (errors.length)
|
|
60
|
-
throw (0, generalUtil_1.formError)(null, { promisePoolErrors: errors });
|
|
61
|
-
flatResults = results.sort(function (a, b) { return a[1] - b[1]; }).map((function (i) { return i[0]; }));
|
|
62
|
-
return [2 /*return*/, flatResults];
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
});
|
|
6
|
+
async function runInPromisePool(params) {
|
|
7
|
+
const { results, errors } = await promise_pool_1.PromisePool
|
|
8
|
+
.withConcurrency(params.concurrency)
|
|
9
|
+
.for(params.items)
|
|
10
|
+
.process(async (item, i) => [await params.processor(item, i), i]);
|
|
11
|
+
if (errors.length)
|
|
12
|
+
throw (0, generalUtil_1.formError)(null, { promisePoolErrors: errors });
|
|
13
|
+
const flatResults = results.sort((a, b) => a[1] - b[1]).map((i => i[0]));
|
|
14
|
+
return flatResults;
|
|
66
15
|
}
|
|
67
16
|
exports.default = runInPromisePool;
|
package/build/util/r2.js
CHANGED
|
@@ -1,48 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var
|
|
3
|
-
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
12
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
13
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
14
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
15
|
-
function step(op) {
|
|
16
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
17
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
18
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
19
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
20
|
-
switch (op[0]) {
|
|
21
|
-
case 0: case 1: t = op; break;
|
|
22
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
23
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
24
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
25
|
-
default:
|
|
26
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
27
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
28
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
29
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
30
|
-
if (t[2]) _.ops.pop();
|
|
31
|
-
_.trys.pop(); continue;
|
|
32
|
-
}
|
|
33
|
-
op = body.call(thisArg, _);
|
|
34
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
35
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
36
|
-
}
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
4
|
};
|
|
38
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
6
|
exports.getR2JSONString = exports.storeR2JSONString = void 0;
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
7
|
+
const axios_1 = __importDefault(require("axios"));
|
|
8
|
+
const env_1 = require("./env");
|
|
9
|
+
const { S3Client, PutObjectCommand, GetObjectCommand, } = require("@aws-sdk/client-s3");
|
|
10
|
+
const { R2_ENDPOINT, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY } = env_1.ENV_CONSTANTS;
|
|
11
|
+
const datasetBucket = "defillama-datasets";
|
|
12
|
+
const publicBucketUrl = "https://defillama-datasets.llama.fi";
|
|
13
|
+
let R2;
|
|
46
14
|
if (R2_ENDPOINT && R2_ACCESS_KEY_ID && R2_SECRET_ACCESS_KEY) {
|
|
47
15
|
R2 = new S3Client({
|
|
48
16
|
region: "auto",
|
|
@@ -53,62 +21,35 @@ if (R2_ENDPOINT && R2_ACCESS_KEY_ID && R2_SECRET_ACCESS_KEY) {
|
|
|
53
21
|
},
|
|
54
22
|
});
|
|
55
23
|
}
|
|
56
|
-
|
|
57
|
-
function storeR2JSONString(filename, body) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
Key: getKey(filename),
|
|
68
|
-
Body: body,
|
|
69
|
-
ContentType: "application/json",
|
|
70
|
-
});
|
|
71
|
-
return [2 /*return*/, R2.send(command)];
|
|
72
|
-
});
|
|
24
|
+
const getKey = (filename) => filename.replace(/(:|'|#)/g, '/');
|
|
25
|
+
async function storeR2JSONString(filename, body) {
|
|
26
|
+
if (typeof body !== 'string')
|
|
27
|
+
body = body.toString('base64');
|
|
28
|
+
if (!R2)
|
|
29
|
+
return;
|
|
30
|
+
const command = new PutObjectCommand({
|
|
31
|
+
Bucket: datasetBucket,
|
|
32
|
+
Key: getKey(filename),
|
|
33
|
+
Body: body,
|
|
34
|
+
ContentType: "application/json",
|
|
73
35
|
});
|
|
36
|
+
return R2.send(command);
|
|
74
37
|
}
|
|
75
38
|
exports.storeR2JSONString = storeR2JSONString;
|
|
76
|
-
function getR2JSONString(filename) {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
return
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
var command, Body, e_1;
|
|
92
|
-
return __generator(this, function (_a) {
|
|
93
|
-
switch (_a.label) {
|
|
94
|
-
case 0:
|
|
95
|
-
_a.trys.push([0, 5, , 6]);
|
|
96
|
-
if (!!R2) return [3 /*break*/, 2];
|
|
97
|
-
return [4 /*yield*/, _fetchData()];
|
|
98
|
-
case 1: return [2 /*return*/, _a.sent()];
|
|
99
|
-
case 2:
|
|
100
|
-
command = new GetObjectCommand({ Bucket: datasetBucket, Key: getKey(filename), });
|
|
101
|
-
return [4 /*yield*/, R2.send(command)];
|
|
102
|
-
case 3:
|
|
103
|
-
Body = (_a.sent()).Body;
|
|
104
|
-
return [4 /*yield*/, Body.transformToString()];
|
|
105
|
-
case 4: return [2 /*return*/, _a.sent()];
|
|
106
|
-
case 5:
|
|
107
|
-
e_1 = _a.sent();
|
|
108
|
-
return [2 /*return*/, {}];
|
|
109
|
-
case 6: return [2 /*return*/];
|
|
110
|
-
}
|
|
111
|
-
});
|
|
112
|
-
});
|
|
39
|
+
async function getR2JSONString(filename) {
|
|
40
|
+
try {
|
|
41
|
+
if (!R2)
|
|
42
|
+
return await _fetchData();
|
|
43
|
+
const command = new GetObjectCommand({ Bucket: datasetBucket, Key: getKey(filename), });
|
|
44
|
+
const { Body } = await R2.send(command);
|
|
45
|
+
return await Body.transformToString();
|
|
46
|
+
}
|
|
47
|
+
catch (e) {
|
|
48
|
+
return {};
|
|
49
|
+
}
|
|
50
|
+
async function _fetchData() {
|
|
51
|
+
const response = await (0, axios_1.default)(`${publicBucketUrl}/${filename}`);
|
|
52
|
+
return response.data;
|
|
53
|
+
}
|
|
113
54
|
}
|
|
114
55
|
exports.getR2JSONString = getR2JSONString;
|