envio 3.6.1 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +197 -9
- package/package.json +6 -6
- package/src/Batch.res +64 -25
- package/src/Batch.res.mjs +69 -46
- package/src/ChainFetching.res +13 -17
- package/src/ChainFetching.res.mjs +9 -14
- package/src/ChainState.res +160 -119
- package/src/ChainState.res.mjs +75 -46
- package/src/ChainState.resi +15 -12
- package/src/Config.res +9 -2
- package/src/Config.res.mjs +4 -4
- package/src/Core.res +5 -0
- package/src/Ecosystem.res +0 -1
- package/src/Envio.res +16 -3
- package/src/EventConfigBuilder.res +155 -40
- package/src/EventConfigBuilder.res.mjs +76 -24
- package/src/EventProcessing.res +4 -8
- package/src/EventProcessing.res.mjs +4 -4
- package/src/EventUtils.res +3 -47
- package/src/FetchState.res +124 -104
- package/src/FetchState.res.mjs +116 -100
- package/src/HandlerRegister.res +108 -2
- package/src/HandlerRegister.res.mjs +59 -10
- package/src/HandlerRegister.resi +4 -0
- package/src/Internal.res +84 -24
- package/src/Internal.res.mjs +33 -2
- package/src/LazyLoader.res +12 -11
- package/src/LazyLoader.res.mjs +13 -7
- package/src/Main.res +22 -9
- package/src/Main.res.mjs +18 -5
- package/src/MemoryStorage.res +724 -0
- package/src/MemoryStorage.res.mjs +601 -0
- package/src/PgStorage.res +18 -7
- package/src/PgStorage.res.mjs +14 -11
- package/src/ReorgDetection.res +0 -222
- package/src/ReorgDetection.res.mjs +0 -163
- package/src/Rollback.res +14 -15
- package/src/Rollback.res.mjs +4 -5
- package/src/SimulateItems.res +2 -2
- package/src/Utils.res +3 -0
- package/src/bindings/ClickHouse.res +16 -11
- package/src/bindings/ClickHouse.res.mjs +10 -10
- package/src/bindings/Vitest.res +1 -1
- package/src/sources/BlockStore.res +115 -5
- package/src/sources/BlockStore.res.mjs +25 -0
- package/src/sources/Evm.res +0 -1
- package/src/sources/Evm.res.mjs +0 -1
- package/src/sources/EvmHyperSyncSource.res +19 -84
- package/src/sources/EvmHyperSyncSource.res.mjs +24 -60
- package/src/sources/Fuel.res +24 -4
- package/src/sources/Fuel.res.mjs +23 -3
- package/src/sources/FuelHyperSync.res +8 -3
- package/src/sources/FuelHyperSync.res.mjs +5 -4
- package/src/sources/FuelHyperSync.resi +3 -1
- package/src/sources/FuelHyperSyncClient.res +7 -10
- package/src/sources/FuelHyperSyncSource.res +18 -45
- package/src/sources/FuelHyperSyncSource.res.mjs +18 -35
- package/src/sources/HyperSync.res +49 -229
- package/src/sources/HyperSync.res.mjs +74 -191
- package/src/sources/HyperSync.resi +11 -35
- package/src/sources/HyperSyncClient.res +9 -79
- package/src/sources/HyperSyncClient.res.mjs +2 -6
- package/src/sources/RequestStat.res +5 -0
- package/src/sources/RequestStat.res.mjs +2 -0
- package/src/sources/RpcSource.res +149 -43
- package/src/sources/RpcSource.res.mjs +104 -41
- package/src/sources/SimulateSource.res +8 -5
- package/src/sources/SimulateSource.res.mjs +6 -6
- package/src/sources/Source.res +89 -16
- package/src/sources/Source.res.mjs +50 -1
- package/src/sources/SourceManager.res +255 -50
- package/src/sources/SourceManager.res.mjs +149 -55
- package/src/sources/SourceManager.resi +2 -6
- package/src/sources/Svm.res +0 -7
- package/src/sources/Svm.res.mjs +0 -8
- package/src/sources/SvmHyperSyncClient.res +21 -16
- package/src/sources/SvmHyperSyncClient.res.mjs +3 -4
- package/src/sources/SvmHyperSyncSource.res +25 -117
- package/src/sources/SvmHyperSyncSource.res.mjs +8 -121
- package/svm.schema.json +3 -2
|
@@ -1,59 +1,65 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
1
|
+
// A napi error carries nothing but a message, so the native clients signal the
|
|
2
|
+
// recoverable conditions SourceManager knows how to retry as a `PREFIX:<int>`
|
|
3
|
+
// marker. Keep in sync with `request_stats.rs`.
|
|
4
|
+
let rateLimitedPrefix = "RATE_LIMITED:"
|
|
5
|
+
let behindHeadPrefix = "SOURCE_BEHIND_HEAD:"
|
|
6
|
+
|
|
7
|
+
let markerValue = (msg, ~prefix) =>
|
|
8
|
+
msg->String.slice(~start=prefix->String.length, ~end=msg->String.length)->Int.fromString
|
|
9
|
+
|
|
10
|
+
let mapNativeFailure = (failure: Source.nativeRequestFailure) => {
|
|
11
|
+
switch failure.message {
|
|
12
|
+
| Some(msg) if msg->String.startsWith(rateLimitedPrefix) =>
|
|
13
|
+
Source.RateLimited({
|
|
14
|
+
resetMs: msg->markerValue(~prefix=rateLimitedPrefix)->Option.getOr(1000),
|
|
15
|
+
requestStats: failure.requestStats,
|
|
16
|
+
})
|
|
17
|
+
| Some(msg) if msg->String.startsWith(behindHeadPrefix) =>
|
|
18
|
+
Source.SourceBehindHead({
|
|
19
|
+
blockNumber: msg->markerValue(~prefix=behindHeadPrefix)->Option.getOr(0),
|
|
20
|
+
requestStats: failure.requestStats,
|
|
21
|
+
})
|
|
22
|
+
| _ => failure.cause
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let mapNativeFailureExn = exn => exn->Source.unpackNativeRequestFailure->mapNativeFailure
|
|
27
|
+
|
|
28
|
+
// Every HyperSync client paginates block hashes in Rust and returns the page
|
|
29
|
+
// store with the timings of the requests it took. A failure arrives as the
|
|
30
|
+
// native envelope, which carries those same timings, so the caller records them
|
|
31
|
+
// either way.
|
|
32
|
+
let makeGetBlockHashes = (
|
|
33
|
+
~query: (~blockNumbers: array<int>) => promise<(BlockStore.t, array<Source.requestStat>)>,
|
|
34
|
+
) =>
|
|
35
|
+
async (~blockNumbers, ~logger as _) => {
|
|
36
|
+
let (result, requestStats) = try {
|
|
37
|
+
let (blockStore, requestStats) = await query(~blockNumbers)
|
|
38
|
+
(Ok(blockStore), requestStats)
|
|
39
|
+
} catch {
|
|
40
|
+
| exn =>
|
|
41
|
+
let failure = exn->Source.unpackNativeRequestFailure
|
|
42
|
+
(Error(failure->mapNativeFailure), failure.requestStats)
|
|
13
43
|
}
|
|
44
|
+
{Source.result, requestStats}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
let reraiseIfRecoverable = exn =>
|
|
48
|
+
switch exn->mapNativeFailureExn {
|
|
49
|
+
| (Source.RateLimited(_) | Source.SourceBehindHead(_)) as exn => throw(exn)
|
|
14
50
|
| _ => ()
|
|
15
51
|
}
|
|
16
52
|
|
|
17
53
|
type logsQueryPage = {
|
|
18
54
|
items: array<HyperSyncClient.EventItems.item>,
|
|
19
|
-
// Headers for every returned block, deduplicated by block number.
|
|
20
|
-
blocks: array<HyperSyncClient.EventItems.blockHeader>,
|
|
21
55
|
nextBlock: int,
|
|
22
56
|
archiveHeight: int,
|
|
23
|
-
rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
|
|
24
57
|
// Page store owning this page's raw transactions.
|
|
25
58
|
transactionStore: TransactionStore.t,
|
|
26
59
|
// Page store owning this page's raw blocks.
|
|
27
60
|
blockStore: BlockStore.t,
|
|
28
61
|
}
|
|
29
62
|
|
|
30
|
-
type missingParams = {
|
|
31
|
-
queryName: string,
|
|
32
|
-
missingParams: array<string>,
|
|
33
|
-
}
|
|
34
|
-
type queryError = UnexpectedMissingParams(missingParams)
|
|
35
|
-
|
|
36
|
-
exception HyperSyncQueryError(queryError)
|
|
37
|
-
|
|
38
|
-
let queryErrorToExn = queryError => {
|
|
39
|
-
HyperSyncQueryError(queryError)
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
let queryErrorToMsq = (e: queryError): string => {
|
|
43
|
-
switch e {
|
|
44
|
-
| UnexpectedMissingParams({queryName, missingParams}) =>
|
|
45
|
-
`${queryName} query failed due to unexpected missing params on response:
|
|
46
|
-
${missingParams->Array.joinUnsafe(", ")}`
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
type queryResponse<'a> = result<'a, queryError>
|
|
51
|
-
let mapExn = (queryResponse: queryResponse<'a>) =>
|
|
52
|
-
switch queryResponse {
|
|
53
|
-
| Ok(v) => Ok(v)
|
|
54
|
-
| Error(err) => err->queryErrorToExn->Error
|
|
55
|
-
}
|
|
56
|
-
|
|
57
63
|
module GetLogs = {
|
|
58
64
|
type error =
|
|
59
65
|
| UnexpectedMissingParams({missingParams: array<string>})
|
|
@@ -109,7 +115,7 @@ module GetLogs = {
|
|
|
109
115
|
) {
|
|
110
116
|
| res => res
|
|
111
117
|
| exception exn =>
|
|
112
|
-
|
|
118
|
+
reraiseIfRecoverable(exn)
|
|
113
119
|
switch extractMissingParams(exn) {
|
|
114
120
|
| Some(missingParams) => throw(Error(UnexpectedMissingParams({missingParams: missingParams})))
|
|
115
121
|
| None => throw(exn)
|
|
@@ -122,196 +128,10 @@ module GetLogs = {
|
|
|
122
128
|
|
|
123
129
|
{
|
|
124
130
|
items: res.items,
|
|
125
|
-
blocks: res.blocks,
|
|
126
131
|
nextBlock: res.nextBlock,
|
|
127
132
|
archiveHeight: res.archiveHeight->Option.getOr(0), //Archive Height is only None if height is 0
|
|
128
|
-
rollbackGuard: res.rollbackGuard,
|
|
129
133
|
transactionStore,
|
|
130
134
|
blockStore,
|
|
131
135
|
}
|
|
132
136
|
}
|
|
133
137
|
}
|
|
134
|
-
|
|
135
|
-
module BlockData = {
|
|
136
|
-
let makeRequestBody = (~fromBlock, ~toBlock): HyperSyncClient.QueryTypes.query => {
|
|
137
|
-
fromBlock,
|
|
138
|
-
toBlockExclusive: toBlock + 1,
|
|
139
|
-
fieldSelection: {
|
|
140
|
-
block: [Number, Hash, Timestamp],
|
|
141
|
-
},
|
|
142
|
-
includeAllBlocks: true,
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
let convertResponse = (res: HyperSyncClient.queryResponse): queryResponse<
|
|
146
|
-
array<ReorgDetection.blockDataWithTimestamp>,
|
|
147
|
-
> => {
|
|
148
|
-
res.data.blocks
|
|
149
|
-
->Array.map(block => {
|
|
150
|
-
switch block {
|
|
151
|
-
| {number: blockNumber, hash: blockHash, timestamp: blockTimestamp} =>
|
|
152
|
-
Ok(
|
|
153
|
-
(
|
|
154
|
-
{
|
|
155
|
-
blockTimestamp,
|
|
156
|
-
blockNumber,
|
|
157
|
-
blockHash,
|
|
158
|
-
}: ReorgDetection.blockDataWithTimestamp
|
|
159
|
-
),
|
|
160
|
-
)
|
|
161
|
-
| _ =>
|
|
162
|
-
let missingParams =
|
|
163
|
-
[
|
|
164
|
-
block.number->Utils.Option.mapNone("block.number"),
|
|
165
|
-
block.timestamp->Utils.Option.mapNone("block.timestamp"),
|
|
166
|
-
block.hash->Utils.Option.mapNone("block.hash"),
|
|
167
|
-
]->Array.filterMap(p => p)
|
|
168
|
-
|
|
169
|
-
Error(
|
|
170
|
-
UnexpectedMissingParams({
|
|
171
|
-
queryName: "query block data HyperSync",
|
|
172
|
-
missingParams,
|
|
173
|
-
}),
|
|
174
|
-
)
|
|
175
|
-
}
|
|
176
|
-
})
|
|
177
|
-
->Utils.Array.transposeResults
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
let rec queryBlockData = async (
|
|
181
|
-
~client: HyperSyncClient.t,
|
|
182
|
-
~fromBlock,
|
|
183
|
-
~toBlock,
|
|
184
|
-
~sourceName,
|
|
185
|
-
~chainId,
|
|
186
|
-
~logger,
|
|
187
|
-
~requestStats: array<Source.requestStat>,
|
|
188
|
-
): queryResponse<array<ReorgDetection.blockDataWithTimestamp>> => {
|
|
189
|
-
let body = makeRequestBody(~fromBlock, ~toBlock)
|
|
190
|
-
|
|
191
|
-
let logger = Logging.createChildFrom(
|
|
192
|
-
~logger,
|
|
193
|
-
~params={
|
|
194
|
-
"logType": "HyperSync get block hash query",
|
|
195
|
-
"fromBlock": fromBlock,
|
|
196
|
-
"toBlock": toBlock,
|
|
197
|
-
},
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
let timerRef = Performance.now()
|
|
201
|
-
let maybeSuccessfulRes = switch await client.get(~query=body) {
|
|
202
|
-
| exception exn =>
|
|
203
|
-
reraisIfRateLimited(exn)
|
|
204
|
-
None
|
|
205
|
-
| res if res.nextBlock <= fromBlock => None
|
|
206
|
-
| res => Some(res)
|
|
207
|
-
}
|
|
208
|
-
requestStats
|
|
209
|
-
->Array.push({Source.method: "getBlockHashes", seconds: timerRef->Performance.secondsSince})
|
|
210
|
-
->ignore
|
|
211
|
-
|
|
212
|
-
// If the block is not found, retry the query. This can occur since replicas of hypersync might not have caught up yet
|
|
213
|
-
switch maybeSuccessfulRes {
|
|
214
|
-
| None => {
|
|
215
|
-
let delayMilliseconds = 100
|
|
216
|
-
logger->Logging.childInfo(
|
|
217
|
-
`Block #${fromBlock->Int.toString} not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${delayMilliseconds->Int.toString}ms.`,
|
|
218
|
-
)
|
|
219
|
-
await Time.resolvePromiseAfterDelay(~delayMilliseconds)
|
|
220
|
-
await queryBlockData(
|
|
221
|
-
~client,
|
|
222
|
-
~fromBlock,
|
|
223
|
-
~toBlock,
|
|
224
|
-
~sourceName,
|
|
225
|
-
~chainId,
|
|
226
|
-
~logger,
|
|
227
|
-
~requestStats,
|
|
228
|
-
)
|
|
229
|
-
}
|
|
230
|
-
| Some(res) =>
|
|
231
|
-
switch res->convertResponse {
|
|
232
|
-
| Error(_) as err => err
|
|
233
|
-
| Ok(datas) if res.nextBlock <= toBlock => {
|
|
234
|
-
let restRes = await queryBlockData(
|
|
235
|
-
~client,
|
|
236
|
-
~fromBlock=res.nextBlock,
|
|
237
|
-
~toBlock,
|
|
238
|
-
~sourceName,
|
|
239
|
-
~chainId,
|
|
240
|
-
~logger,
|
|
241
|
-
~requestStats,
|
|
242
|
-
)
|
|
243
|
-
restRes->Result.map(rest => datas->Array.concat(rest))
|
|
244
|
-
}
|
|
245
|
-
| Ok(_) as ok => ok
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
let queryBlockDataMulti = async (
|
|
251
|
-
~client: HyperSyncClient.t,
|
|
252
|
-
~blockNumbers,
|
|
253
|
-
~sourceName,
|
|
254
|
-
~chainId,
|
|
255
|
-
~logger,
|
|
256
|
-
): (queryResponse<array<ReorgDetection.blockDataWithTimestamp>>, array<Source.requestStat>) => {
|
|
257
|
-
let requestStats = []
|
|
258
|
-
let result = switch blockNumbers->Array.get(0) {
|
|
259
|
-
| None => Ok([])
|
|
260
|
-
| Some(firstBlock) => {
|
|
261
|
-
let fromBlock = ref(firstBlock)
|
|
262
|
-
let toBlock = ref(firstBlock)
|
|
263
|
-
let set = Utils.Set.make()
|
|
264
|
-
for idx in 0 to blockNumbers->Array.length - 1 {
|
|
265
|
-
let blockNumber = blockNumbers->Array.getUnsafe(idx)
|
|
266
|
-
if blockNumber < fromBlock.contents {
|
|
267
|
-
fromBlock := blockNumber
|
|
268
|
-
}
|
|
269
|
-
if blockNumber > toBlock.contents {
|
|
270
|
-
toBlock := blockNumber
|
|
271
|
-
}
|
|
272
|
-
set->Utils.Set.add(blockNumber)->ignore
|
|
273
|
-
}
|
|
274
|
-
if toBlock.contents - fromBlock.contents > 1000 {
|
|
275
|
-
JsError.throwWithMessage(
|
|
276
|
-
`Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ${fromBlock.contents->Int.toString}-${toBlock.contents->Int.toString}`,
|
|
277
|
-
)
|
|
278
|
-
}
|
|
279
|
-
let res = await queryBlockData(
|
|
280
|
-
~fromBlock=fromBlock.contents,
|
|
281
|
-
~toBlock=toBlock.contents,
|
|
282
|
-
~client,
|
|
283
|
-
~sourceName,
|
|
284
|
-
~chainId,
|
|
285
|
-
~logger,
|
|
286
|
-
~requestStats,
|
|
287
|
-
)
|
|
288
|
-
let filtered = res->Result.map(datas => {
|
|
289
|
-
datas->Array.filter(data => set->Utils.Set.delete(data.blockNumber))
|
|
290
|
-
})
|
|
291
|
-
if set->Utils.Set.size > 0 {
|
|
292
|
-
JsError.throwWithMessage(
|
|
293
|
-
`Invalid response. Failed to get block data for block numbers: ${set
|
|
294
|
-
->Utils.Set.toArray
|
|
295
|
-
->Array.joinUnsafe(", ")}`,
|
|
296
|
-
)
|
|
297
|
-
}
|
|
298
|
-
filtered
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
(result, requestStats)
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) => {
|
|
306
|
-
let requestStats = []
|
|
307
|
-
BlockData.queryBlockData(
|
|
308
|
-
~client,
|
|
309
|
-
~fromBlock=blockNumber,
|
|
310
|
-
~toBlock=blockNumber,
|
|
311
|
-
~sourceName,
|
|
312
|
-
~chainId,
|
|
313
|
-
~logger,
|
|
314
|
-
~requestStats,
|
|
315
|
-
)->Promise.thenResolve(res => (res->Result.map(res => res->Array.get(0)), requestStats))
|
|
316
|
-
}
|
|
317
|
-
let queryBlockDataMulti = BlockData.queryBlockDataMulti
|
|
@@ -1,62 +1,89 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
|
-
import * as Time from "../Time.res.mjs";
|
|
4
|
-
import * as Utils from "../Utils.res.mjs";
|
|
5
3
|
import * as Source from "./Source.res.mjs";
|
|
6
|
-
import * as Logging from "../Logging.res.mjs";
|
|
7
4
|
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
8
|
-
import * as Performance from "../bindings/Performance.res.mjs";
|
|
9
5
|
import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
|
|
10
6
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
11
7
|
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
12
8
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
13
|
-
import * as Stdlib_Result from "@rescript/runtime/lib/es6/Stdlib_Result.js";
|
|
14
|
-
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
15
9
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
16
10
|
|
|
17
|
-
|
|
18
|
-
let e = Primitive_exceptions.internalToException(exn);
|
|
19
|
-
if (e.RE_EXN_ID !== "JsExn") {
|
|
20
|
-
return;
|
|
21
|
-
}
|
|
22
|
-
let msg = Stdlib_JsExn.message(e._1);
|
|
23
|
-
if (msg === undefined) {
|
|
24
|
-
return;
|
|
25
|
-
}
|
|
26
|
-
if (!msg.startsWith("RATE_LIMITED:")) {
|
|
27
|
-
return;
|
|
28
|
-
}
|
|
29
|
-
let resetMs = Stdlib_Option.getOr(Stdlib_Int.fromString(msg.slice(13, msg.length), undefined), 1000);
|
|
30
|
-
throw {
|
|
31
|
-
RE_EXN_ID: Source.RateLimited,
|
|
32
|
-
resetMs: resetMs,
|
|
33
|
-
Error: new Error()
|
|
34
|
-
};
|
|
35
|
-
}
|
|
11
|
+
let rateLimitedPrefix = "RATE_LIMITED:";
|
|
36
12
|
|
|
37
|
-
let
|
|
13
|
+
let behindHeadPrefix = "SOURCE_BEHIND_HEAD:";
|
|
38
14
|
|
|
39
|
-
function
|
|
40
|
-
|
|
41
|
-
return match.queryName + ` query failed due to unexpected missing params on response:
|
|
42
|
-
` + match.missingParams.join(", ");
|
|
15
|
+
function markerValue(msg, prefix) {
|
|
16
|
+
return Stdlib_Int.fromString(msg.slice(prefix.length, msg.length), undefined);
|
|
43
17
|
}
|
|
44
18
|
|
|
45
|
-
function
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
19
|
+
function mapNativeFailure(failure) {
|
|
20
|
+
let msg = failure.message;
|
|
21
|
+
if (msg !== undefined) {
|
|
22
|
+
if (msg.startsWith(rateLimitedPrefix)) {
|
|
23
|
+
return {
|
|
24
|
+
RE_EXN_ID: Source.RateLimited,
|
|
25
|
+
_1: {
|
|
26
|
+
resetMs: Stdlib_Option.getOr(markerValue(msg, rateLimitedPrefix), 1000),
|
|
27
|
+
requestStats: failure.requestStats
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
} else if (msg.startsWith(behindHeadPrefix)) {
|
|
31
|
+
return {
|
|
32
|
+
RE_EXN_ID: Source.SourceBehindHead,
|
|
33
|
+
blockNumber: Stdlib_Option.getOr(markerValue(msg, behindHeadPrefix), 0),
|
|
34
|
+
requestStats: failure.requestStats
|
|
35
|
+
};
|
|
36
|
+
} else {
|
|
37
|
+
return failure.cause;
|
|
38
|
+
}
|
|
51
39
|
} else {
|
|
40
|
+
return failure.cause;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function mapNativeFailureExn(exn) {
|
|
45
|
+
return mapNativeFailure(Source.unpackNativeRequestFailure(exn));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function makeGetBlockHashes(query) {
|
|
49
|
+
return async (blockNumbers, param) => {
|
|
50
|
+
let match;
|
|
51
|
+
try {
|
|
52
|
+
let match$1 = await query(blockNumbers);
|
|
53
|
+
match = [
|
|
54
|
+
{
|
|
55
|
+
TAG: "Ok",
|
|
56
|
+
_0: match$1[0]
|
|
57
|
+
},
|
|
58
|
+
match$1[1]
|
|
59
|
+
];
|
|
60
|
+
} catch (raw_exn) {
|
|
61
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
62
|
+
let failure = Source.unpackNativeRequestFailure(exn);
|
|
63
|
+
match = [
|
|
64
|
+
{
|
|
65
|
+
TAG: "Error",
|
|
66
|
+
_0: mapNativeFailure(failure)
|
|
67
|
+
},
|
|
68
|
+
failure.requestStats
|
|
69
|
+
];
|
|
70
|
+
}
|
|
52
71
|
return {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
RE_EXN_ID: HyperSyncQueryError,
|
|
56
|
-
_1: queryResponse._0
|
|
57
|
-
}
|
|
72
|
+
result: match[0],
|
|
73
|
+
requestStats: match[1]
|
|
58
74
|
};
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function reraiseIfRecoverable(exn) {
|
|
79
|
+
let exn$1 = mapNativeFailure(Source.unpackNativeRequestFailure(exn));
|
|
80
|
+
if (exn$1.RE_EXN_ID === Source.RateLimited) {
|
|
81
|
+
throw exn$1;
|
|
82
|
+
}
|
|
83
|
+
if (exn$1.RE_EXN_ID !== Source.SourceBehindHead) {
|
|
84
|
+
return;
|
|
59
85
|
}
|
|
86
|
+
throw exn$1;
|
|
60
87
|
}
|
|
61
88
|
|
|
62
89
|
let $$Error = /* @__PURE__ */Primitive_exceptions.create("HyperSync.GetLogs.Error");
|
|
@@ -95,7 +122,7 @@ async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes
|
|
|
95
122
|
match = await client.getEventItems(query$1, addressSet);
|
|
96
123
|
} catch (raw_exn) {
|
|
97
124
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
98
|
-
|
|
125
|
+
reraiseIfRecoverable(exn);
|
|
99
126
|
let missingParams = extractMissingParams(exn);
|
|
100
127
|
if (missingParams !== undefined) {
|
|
101
128
|
throw {
|
|
@@ -119,10 +146,8 @@ async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes
|
|
|
119
146
|
}
|
|
120
147
|
return {
|
|
121
148
|
items: res.items,
|
|
122
|
-
blocks: res.blocks,
|
|
123
149
|
nextBlock: res.nextBlock,
|
|
124
150
|
archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0),
|
|
125
|
-
rollbackGuard: res.rollbackGuard,
|
|
126
151
|
transactionStore: match[1],
|
|
127
152
|
blockStore: match[2]
|
|
128
153
|
};
|
|
@@ -134,153 +159,11 @@ let GetLogs = {
|
|
|
134
159
|
query: query
|
|
135
160
|
};
|
|
136
161
|
|
|
137
|
-
function makeRequestBody(fromBlock, toBlock) {
|
|
138
|
-
return {
|
|
139
|
-
fromBlock: fromBlock,
|
|
140
|
-
toBlock: toBlock + 1 | 0,
|
|
141
|
-
fieldSelection: {
|
|
142
|
-
block: [
|
|
143
|
-
"Number",
|
|
144
|
-
"Hash",
|
|
145
|
-
"Timestamp"
|
|
146
|
-
]
|
|
147
|
-
},
|
|
148
|
-
includeAllBlocks: true
|
|
149
|
-
};
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
function convertResponse(res) {
|
|
153
|
-
return Utils.$$Array.transposeResults(res.data.blocks.map(block => {
|
|
154
|
-
let blockNumber = block.number;
|
|
155
|
-
if (blockNumber !== undefined) {
|
|
156
|
-
let blockHash = block.hash;
|
|
157
|
-
if (blockHash !== undefined) {
|
|
158
|
-
let blockTimestamp = block.timestamp;
|
|
159
|
-
if (blockTimestamp !== undefined) {
|
|
160
|
-
return {
|
|
161
|
-
TAG: "Ok",
|
|
162
|
-
_0: {
|
|
163
|
-
blockHash: blockHash,
|
|
164
|
-
blockNumber: blockNumber,
|
|
165
|
-
blockTimestamp: blockTimestamp
|
|
166
|
-
}
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
let missingParams = Stdlib_Array.filterMap([
|
|
172
|
-
Utils.Option.mapNone(block.number, "block.number"),
|
|
173
|
-
Utils.Option.mapNone(block.timestamp, "block.timestamp"),
|
|
174
|
-
Utils.Option.mapNone(block.hash, "block.hash")
|
|
175
|
-
], p => p);
|
|
176
|
-
return {
|
|
177
|
-
TAG: "Error",
|
|
178
|
-
_0: {
|
|
179
|
-
TAG: "UnexpectedMissingParams",
|
|
180
|
-
_0: {
|
|
181
|
-
queryName: "query block data HyperSync",
|
|
182
|
-
missingParams: missingParams
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
};
|
|
186
|
-
}));
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger, requestStats) {
|
|
190
|
-
let body = makeRequestBody(fromBlock, toBlock);
|
|
191
|
-
let logger$1 = Logging.createChildFrom(logger, {
|
|
192
|
-
logType: "HyperSync get block hash query",
|
|
193
|
-
fromBlock: fromBlock,
|
|
194
|
-
toBlock: toBlock
|
|
195
|
-
});
|
|
196
|
-
let timerRef = Performance.now();
|
|
197
|
-
let maybeSuccessfulRes;
|
|
198
|
-
let exit = 0;
|
|
199
|
-
let res;
|
|
200
|
-
try {
|
|
201
|
-
res = await client.get(body);
|
|
202
|
-
exit = 1;
|
|
203
|
-
} catch (raw_exn) {
|
|
204
|
-
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
205
|
-
reraisIfRateLimited(exn);
|
|
206
|
-
maybeSuccessfulRes = undefined;
|
|
207
|
-
}
|
|
208
|
-
if (exit === 1) {
|
|
209
|
-
maybeSuccessfulRes = res.nextBlock <= fromBlock ? undefined : res;
|
|
210
|
-
}
|
|
211
|
-
requestStats.push({
|
|
212
|
-
method: "getBlockHashes",
|
|
213
|
-
seconds: Performance.secondsSince(timerRef)
|
|
214
|
-
});
|
|
215
|
-
if (maybeSuccessfulRes !== undefined) {
|
|
216
|
-
let err = convertResponse(maybeSuccessfulRes);
|
|
217
|
-
if (err.TAG !== "Ok") {
|
|
218
|
-
return err;
|
|
219
|
-
}
|
|
220
|
-
if (maybeSuccessfulRes.nextBlock > toBlock) {
|
|
221
|
-
return err;
|
|
222
|
-
}
|
|
223
|
-
let datas = err._0;
|
|
224
|
-
let restRes = await queryBlockData(client, maybeSuccessfulRes.nextBlock, toBlock, sourceName, chainId, logger$1, requestStats);
|
|
225
|
-
return Stdlib_Result.map(restRes, rest => datas.concat(rest));
|
|
226
|
-
}
|
|
227
|
-
Logging.childInfo(logger$1, `Block #` + fromBlock.toString() + ` not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ` + (100).toString() + `ms.`);
|
|
228
|
-
await Time.resolvePromiseAfterDelay(100);
|
|
229
|
-
return await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger$1, requestStats);
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
async function queryBlockDataMulti(client, blockNumbers, sourceName, chainId, logger) {
|
|
233
|
-
let requestStats = [];
|
|
234
|
-
let firstBlock = blockNumbers[0];
|
|
235
|
-
let result;
|
|
236
|
-
if (firstBlock !== undefined) {
|
|
237
|
-
let fromBlock = firstBlock;
|
|
238
|
-
let toBlock = firstBlock;
|
|
239
|
-
let set = new Set();
|
|
240
|
-
for (let idx = 0, idx_finish = blockNumbers.length; idx < idx_finish; ++idx) {
|
|
241
|
-
let blockNumber = blockNumbers[idx];
|
|
242
|
-
if (blockNumber < fromBlock) {
|
|
243
|
-
fromBlock = blockNumber;
|
|
244
|
-
}
|
|
245
|
-
if (blockNumber > toBlock) {
|
|
246
|
-
toBlock = blockNumber;
|
|
247
|
-
}
|
|
248
|
-
set.add(blockNumber);
|
|
249
|
-
}
|
|
250
|
-
if ((toBlock - fromBlock | 0) > 1000) {
|
|
251
|
-
Stdlib_JsError.throwWithMessage(`Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ` + fromBlock.toString() + `-` + toBlock.toString());
|
|
252
|
-
}
|
|
253
|
-
let res = await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger, requestStats);
|
|
254
|
-
let filtered = Stdlib_Result.map(res, datas => datas.filter(data => set.delete(data.blockNumber)));
|
|
255
|
-
if (set.size > 0) {
|
|
256
|
-
Stdlib_JsError.throwWithMessage(`Invalid response. Failed to get block data for block numbers: ` + Array.from(set).join(", "));
|
|
257
|
-
}
|
|
258
|
-
result = filtered;
|
|
259
|
-
} else {
|
|
260
|
-
result = {
|
|
261
|
-
TAG: "Ok",
|
|
262
|
-
_0: []
|
|
263
|
-
};
|
|
264
|
-
}
|
|
265
|
-
return [
|
|
266
|
-
result,
|
|
267
|
-
requestStats
|
|
268
|
-
];
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
function queryBlockData$1(client, blockNumber, sourceName, chainId, logger) {
|
|
272
|
-
let requestStats = [];
|
|
273
|
-
return queryBlockData(client, blockNumber, blockNumber, sourceName, chainId, logger, requestStats).then(res => [
|
|
274
|
-
Stdlib_Result.map(res, res => res[0]),
|
|
275
|
-
requestStats
|
|
276
|
-
]);
|
|
277
|
-
}
|
|
278
|
-
|
|
279
162
|
export {
|
|
280
|
-
|
|
163
|
+
mapNativeFailure,
|
|
164
|
+
mapNativeFailureExn,
|
|
165
|
+
reraiseIfRecoverable,
|
|
166
|
+
makeGetBlockHashes,
|
|
281
167
|
GetLogs,
|
|
282
|
-
queryBlockData$1 as queryBlockData,
|
|
283
|
-
queryBlockDataMulti,
|
|
284
|
-
mapExn,
|
|
285
168
|
}
|
|
286
|
-
/*
|
|
169
|
+
/* Source Not a pure module */
|
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
type logsQueryPage = {
|
|
2
2
|
items: array<HyperSyncClient.EventItems.item>,
|
|
3
|
-
blocks: array<HyperSyncClient.EventItems.blockHeader>,
|
|
4
3
|
nextBlock: int,
|
|
5
4
|
archiveHeight: int,
|
|
6
|
-
rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
|
|
7
5
|
transactionStore: TransactionStore.t,
|
|
8
6
|
blockStore: BlockStore.t,
|
|
9
7
|
}
|
|
10
8
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
let queryErrorToMsq: queryError => string
|
|
9
|
+
// Map a native client's `PREFIX:<int>` failure marker onto the exception
|
|
10
|
+
// SourceManager retries on (`RateLimited` / `SourceBehindHead`), falling back to
|
|
11
|
+
// the original cause.
|
|
12
|
+
let mapNativeFailure: Source.nativeRequestFailure => exn
|
|
13
|
+
let mapNativeFailureExn: exn => exn
|
|
14
|
+
let reraiseIfRecoverable: exn => unit
|
|
19
15
|
|
|
20
|
-
|
|
16
|
+
// Wrap a native client's paginated block-hash query as a `Source.getBlockHashes`,
|
|
17
|
+
// keeping the request timings whether it succeeds or fails.
|
|
18
|
+
let makeGetBlockHashes: (
|
|
19
|
+
~query: (~blockNumbers: array<int>) => promise<(BlockStore.t, array<Source.requestStat>)>,
|
|
20
|
+
) => (~blockNumbers: array<int>, ~logger: Pino.t) => promise<Source.getBlockHashesResponse>
|
|
21
21
|
|
|
22
22
|
module GetLogs: {
|
|
23
23
|
type error =
|
|
@@ -38,27 +38,3 @@ module GetLogs: {
|
|
|
38
38
|
~clientFilteredContracts: option<array<string>>,
|
|
39
39
|
) => promise<logsQueryPage>
|
|
40
40
|
}
|
|
41
|
-
|
|
42
|
-
let queryBlockData: (
|
|
43
|
-
~client: HyperSyncClient.t,
|
|
44
|
-
~blockNumber: int,
|
|
45
|
-
~sourceName: string,
|
|
46
|
-
~chainId: ChainId.t,
|
|
47
|
-
~logger: Pino.t,
|
|
48
|
-
) => promise<(
|
|
49
|
-
queryResponse<option<ReorgDetection.blockDataWithTimestamp>>,
|
|
50
|
-
array<Source.requestStat>,
|
|
51
|
-
)>
|
|
52
|
-
|
|
53
|
-
let queryBlockDataMulti: (
|
|
54
|
-
~client: HyperSyncClient.t,
|
|
55
|
-
~blockNumbers: array<int>,
|
|
56
|
-
~sourceName: string,
|
|
57
|
-
~chainId: ChainId.t,
|
|
58
|
-
~logger: Pino.t,
|
|
59
|
-
) => promise<(
|
|
60
|
-
queryResponse<array<ReorgDetection.blockDataWithTimestamp>>,
|
|
61
|
-
array<Source.requestStat>,
|
|
62
|
-
)>
|
|
63
|
-
|
|
64
|
-
let mapExn: queryResponse<'a> => result<'a, exn>
|