envio 3.6.1 → 3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/index.d.ts +197 -9
  2. package/package.json +6 -6
  3. package/src/Batch.res +64 -25
  4. package/src/Batch.res.mjs +69 -46
  5. package/src/ChainFetching.res +13 -17
  6. package/src/ChainFetching.res.mjs +9 -14
  7. package/src/ChainState.res +160 -119
  8. package/src/ChainState.res.mjs +75 -46
  9. package/src/ChainState.resi +15 -12
  10. package/src/Config.res +9 -2
  11. package/src/Config.res.mjs +4 -4
  12. package/src/Core.res +5 -0
  13. package/src/Ecosystem.res +0 -1
  14. package/src/Envio.res +16 -3
  15. package/src/EventConfigBuilder.res +155 -40
  16. package/src/EventConfigBuilder.res.mjs +76 -24
  17. package/src/EventProcessing.res +4 -8
  18. package/src/EventProcessing.res.mjs +4 -4
  19. package/src/EventUtils.res +3 -47
  20. package/src/FetchState.res +124 -104
  21. package/src/FetchState.res.mjs +116 -100
  22. package/src/HandlerRegister.res +108 -2
  23. package/src/HandlerRegister.res.mjs +59 -10
  24. package/src/HandlerRegister.resi +4 -0
  25. package/src/Internal.res +84 -24
  26. package/src/Internal.res.mjs +33 -2
  27. package/src/LazyLoader.res +12 -11
  28. package/src/LazyLoader.res.mjs +13 -7
  29. package/src/Main.res +22 -9
  30. package/src/Main.res.mjs +18 -5
  31. package/src/MemoryStorage.res +724 -0
  32. package/src/MemoryStorage.res.mjs +601 -0
  33. package/src/PgStorage.res +18 -7
  34. package/src/PgStorage.res.mjs +14 -11
  35. package/src/ReorgDetection.res +0 -222
  36. package/src/ReorgDetection.res.mjs +0 -163
  37. package/src/Rollback.res +14 -15
  38. package/src/Rollback.res.mjs +4 -5
  39. package/src/SimulateItems.res +2 -2
  40. package/src/Utils.res +3 -0
  41. package/src/bindings/ClickHouse.res +16 -11
  42. package/src/bindings/ClickHouse.res.mjs +10 -10
  43. package/src/bindings/Vitest.res +1 -1
  44. package/src/sources/BlockStore.res +115 -5
  45. package/src/sources/BlockStore.res.mjs +25 -0
  46. package/src/sources/Evm.res +0 -1
  47. package/src/sources/Evm.res.mjs +0 -1
  48. package/src/sources/EvmHyperSyncSource.res +19 -84
  49. package/src/sources/EvmHyperSyncSource.res.mjs +24 -60
  50. package/src/sources/Fuel.res +24 -4
  51. package/src/sources/Fuel.res.mjs +23 -3
  52. package/src/sources/FuelHyperSync.res +8 -3
  53. package/src/sources/FuelHyperSync.res.mjs +5 -4
  54. package/src/sources/FuelHyperSync.resi +3 -1
  55. package/src/sources/FuelHyperSyncClient.res +7 -10
  56. package/src/sources/FuelHyperSyncSource.res +18 -45
  57. package/src/sources/FuelHyperSyncSource.res.mjs +18 -35
  58. package/src/sources/HyperSync.res +49 -229
  59. package/src/sources/HyperSync.res.mjs +74 -191
  60. package/src/sources/HyperSync.resi +11 -35
  61. package/src/sources/HyperSyncClient.res +9 -79
  62. package/src/sources/HyperSyncClient.res.mjs +2 -6
  63. package/src/sources/RequestStat.res +5 -0
  64. package/src/sources/RequestStat.res.mjs +2 -0
  65. package/src/sources/RpcSource.res +149 -43
  66. package/src/sources/RpcSource.res.mjs +104 -41
  67. package/src/sources/SimulateSource.res +8 -5
  68. package/src/sources/SimulateSource.res.mjs +6 -6
  69. package/src/sources/Source.res +89 -16
  70. package/src/sources/Source.res.mjs +50 -1
  71. package/src/sources/SourceManager.res +255 -50
  72. package/src/sources/SourceManager.res.mjs +149 -55
  73. package/src/sources/SourceManager.resi +2 -6
  74. package/src/sources/Svm.res +0 -7
  75. package/src/sources/Svm.res.mjs +0 -8
  76. package/src/sources/SvmHyperSyncClient.res +21 -16
  77. package/src/sources/SvmHyperSyncClient.res.mjs +3 -4
  78. package/src/sources/SvmHyperSyncSource.res +25 -117
  79. package/src/sources/SvmHyperSyncSource.res.mjs +8 -121
  80. package/svm.schema.json +3 -2
@@ -1,59 +1,65 @@
1
- let reraisIfRateLimited = exn =>
2
- switch exn->JsExn.anyToExnInternal {
3
- | JsExn(e) =>
4
- switch e->JsExn.message {
5
- | Some(msg) if msg->String.startsWith("RATE_LIMITED:") =>
6
- let resetMs =
7
- msg
8
- ->String.slice(~start=13, ~end=msg->String.length)
9
- ->Int.fromString
10
- ->Option.getOr(1000)
11
- throw(Source.RateLimited({resetMs: resetMs}))
12
- | _ => ()
1
+ // A napi error carries nothing but a message, so the native clients signal the
2
+ // recoverable conditions SourceManager knows how to retry as a `PREFIX:<int>`
3
+ // marker. Keep in sync with `request_stats.rs`.
4
+ let rateLimitedPrefix = "RATE_LIMITED:"
5
+ let behindHeadPrefix = "SOURCE_BEHIND_HEAD:"
6
+
7
+ let markerValue = (msg, ~prefix) =>
8
+ msg->String.slice(~start=prefix->String.length, ~end=msg->String.length)->Int.fromString
9
+
10
+ let mapNativeFailure = (failure: Source.nativeRequestFailure) => {
11
+ switch failure.message {
12
+ | Some(msg) if msg->String.startsWith(rateLimitedPrefix) =>
13
+ Source.RateLimited({
14
+ resetMs: msg->markerValue(~prefix=rateLimitedPrefix)->Option.getOr(1000),
15
+ requestStats: failure.requestStats,
16
+ })
17
+ | Some(msg) if msg->String.startsWith(behindHeadPrefix) =>
18
+ Source.SourceBehindHead({
19
+ blockNumber: msg->markerValue(~prefix=behindHeadPrefix)->Option.getOr(0),
20
+ requestStats: failure.requestStats,
21
+ })
22
+ | _ => failure.cause
23
+ }
24
+ }
25
+
26
+ let mapNativeFailureExn = exn => exn->Source.unpackNativeRequestFailure->mapNativeFailure
27
+
28
+ // Every HyperSync client paginates block hashes in Rust and returns the page
29
+ // store with the timings of the requests it took. A failure arrives as the
30
+ // native envelope, which carries those same timings, so the caller records them
31
+ // either way.
32
+ let makeGetBlockHashes = (
33
+ ~query: (~blockNumbers: array<int>) => promise<(BlockStore.t, array<Source.requestStat>)>,
34
+ ) =>
35
+ async (~blockNumbers, ~logger as _) => {
36
+ let (result, requestStats) = try {
37
+ let (blockStore, requestStats) = await query(~blockNumbers)
38
+ (Ok(blockStore), requestStats)
39
+ } catch {
40
+ | exn =>
41
+ let failure = exn->Source.unpackNativeRequestFailure
42
+ (Error(failure->mapNativeFailure), failure.requestStats)
13
43
  }
44
+ {Source.result, requestStats}
45
+ }
46
+
47
+ let reraiseIfRecoverable = exn =>
48
+ switch exn->mapNativeFailureExn {
49
+ | (Source.RateLimited(_) | Source.SourceBehindHead(_)) as exn => throw(exn)
14
50
  | _ => ()
15
51
  }
16
52
 
17
53
  type logsQueryPage = {
18
54
  items: array<HyperSyncClient.EventItems.item>,
19
- // Headers for every returned block, deduplicated by block number.
20
- blocks: array<HyperSyncClient.EventItems.blockHeader>,
21
55
  nextBlock: int,
22
56
  archiveHeight: int,
23
- rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
24
57
  // Page store owning this page's raw transactions.
25
58
  transactionStore: TransactionStore.t,
26
59
  // Page store owning this page's raw blocks.
27
60
  blockStore: BlockStore.t,
28
61
  }
29
62
 
30
- type missingParams = {
31
- queryName: string,
32
- missingParams: array<string>,
33
- }
34
- type queryError = UnexpectedMissingParams(missingParams)
35
-
36
- exception HyperSyncQueryError(queryError)
37
-
38
- let queryErrorToExn = queryError => {
39
- HyperSyncQueryError(queryError)
40
- }
41
-
42
- let queryErrorToMsq = (e: queryError): string => {
43
- switch e {
44
- | UnexpectedMissingParams({queryName, missingParams}) =>
45
- `${queryName} query failed due to unexpected missing params on response:
46
- ${missingParams->Array.joinUnsafe(", ")}`
47
- }
48
- }
49
-
50
- type queryResponse<'a> = result<'a, queryError>
51
- let mapExn = (queryResponse: queryResponse<'a>) =>
52
- switch queryResponse {
53
- | Ok(v) => Ok(v)
54
- | Error(err) => err->queryErrorToExn->Error
55
- }
56
-
57
63
  module GetLogs = {
58
64
  type error =
59
65
  | UnexpectedMissingParams({missingParams: array<string>})
@@ -109,7 +115,7 @@ module GetLogs = {
109
115
  ) {
110
116
  | res => res
111
117
  | exception exn =>
112
- reraisIfRateLimited(exn)
118
+ reraiseIfRecoverable(exn)
113
119
  switch extractMissingParams(exn) {
114
120
  | Some(missingParams) => throw(Error(UnexpectedMissingParams({missingParams: missingParams})))
115
121
  | None => throw(exn)
@@ -122,196 +128,10 @@ module GetLogs = {
122
128
 
123
129
  {
124
130
  items: res.items,
125
- blocks: res.blocks,
126
131
  nextBlock: res.nextBlock,
127
132
  archiveHeight: res.archiveHeight->Option.getOr(0), //Archive Height is only None if height is 0
128
- rollbackGuard: res.rollbackGuard,
129
133
  transactionStore,
130
134
  blockStore,
131
135
  }
132
136
  }
133
137
  }
134
-
135
- module BlockData = {
136
- let makeRequestBody = (~fromBlock, ~toBlock): HyperSyncClient.QueryTypes.query => {
137
- fromBlock,
138
- toBlockExclusive: toBlock + 1,
139
- fieldSelection: {
140
- block: [Number, Hash, Timestamp],
141
- },
142
- includeAllBlocks: true,
143
- }
144
-
145
- let convertResponse = (res: HyperSyncClient.queryResponse): queryResponse<
146
- array<ReorgDetection.blockDataWithTimestamp>,
147
- > => {
148
- res.data.blocks
149
- ->Array.map(block => {
150
- switch block {
151
- | {number: blockNumber, hash: blockHash, timestamp: blockTimestamp} =>
152
- Ok(
153
- (
154
- {
155
- blockTimestamp,
156
- blockNumber,
157
- blockHash,
158
- }: ReorgDetection.blockDataWithTimestamp
159
- ),
160
- )
161
- | _ =>
162
- let missingParams =
163
- [
164
- block.number->Utils.Option.mapNone("block.number"),
165
- block.timestamp->Utils.Option.mapNone("block.timestamp"),
166
- block.hash->Utils.Option.mapNone("block.hash"),
167
- ]->Array.filterMap(p => p)
168
-
169
- Error(
170
- UnexpectedMissingParams({
171
- queryName: "query block data HyperSync",
172
- missingParams,
173
- }),
174
- )
175
- }
176
- })
177
- ->Utils.Array.transposeResults
178
- }
179
-
180
- let rec queryBlockData = async (
181
- ~client: HyperSyncClient.t,
182
- ~fromBlock,
183
- ~toBlock,
184
- ~sourceName,
185
- ~chainId,
186
- ~logger,
187
- ~requestStats: array<Source.requestStat>,
188
- ): queryResponse<array<ReorgDetection.blockDataWithTimestamp>> => {
189
- let body = makeRequestBody(~fromBlock, ~toBlock)
190
-
191
- let logger = Logging.createChildFrom(
192
- ~logger,
193
- ~params={
194
- "logType": "HyperSync get block hash query",
195
- "fromBlock": fromBlock,
196
- "toBlock": toBlock,
197
- },
198
- )
199
-
200
- let timerRef = Performance.now()
201
- let maybeSuccessfulRes = switch await client.get(~query=body) {
202
- | exception exn =>
203
- reraisIfRateLimited(exn)
204
- None
205
- | res if res.nextBlock <= fromBlock => None
206
- | res => Some(res)
207
- }
208
- requestStats
209
- ->Array.push({Source.method: "getBlockHashes", seconds: timerRef->Performance.secondsSince})
210
- ->ignore
211
-
212
- // If the block is not found, retry the query. This can occur since replicas of hypersync might not have caught up yet
213
- switch maybeSuccessfulRes {
214
- | None => {
215
- let delayMilliseconds = 100
216
- logger->Logging.childInfo(
217
- `Block #${fromBlock->Int.toString} not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${delayMilliseconds->Int.toString}ms.`,
218
- )
219
- await Time.resolvePromiseAfterDelay(~delayMilliseconds)
220
- await queryBlockData(
221
- ~client,
222
- ~fromBlock,
223
- ~toBlock,
224
- ~sourceName,
225
- ~chainId,
226
- ~logger,
227
- ~requestStats,
228
- )
229
- }
230
- | Some(res) =>
231
- switch res->convertResponse {
232
- | Error(_) as err => err
233
- | Ok(datas) if res.nextBlock <= toBlock => {
234
- let restRes = await queryBlockData(
235
- ~client,
236
- ~fromBlock=res.nextBlock,
237
- ~toBlock,
238
- ~sourceName,
239
- ~chainId,
240
- ~logger,
241
- ~requestStats,
242
- )
243
- restRes->Result.map(rest => datas->Array.concat(rest))
244
- }
245
- | Ok(_) as ok => ok
246
- }
247
- }
248
- }
249
-
250
- let queryBlockDataMulti = async (
251
- ~client: HyperSyncClient.t,
252
- ~blockNumbers,
253
- ~sourceName,
254
- ~chainId,
255
- ~logger,
256
- ): (queryResponse<array<ReorgDetection.blockDataWithTimestamp>>, array<Source.requestStat>) => {
257
- let requestStats = []
258
- let result = switch blockNumbers->Array.get(0) {
259
- | None => Ok([])
260
- | Some(firstBlock) => {
261
- let fromBlock = ref(firstBlock)
262
- let toBlock = ref(firstBlock)
263
- let set = Utils.Set.make()
264
- for idx in 0 to blockNumbers->Array.length - 1 {
265
- let blockNumber = blockNumbers->Array.getUnsafe(idx)
266
- if blockNumber < fromBlock.contents {
267
- fromBlock := blockNumber
268
- }
269
- if blockNumber > toBlock.contents {
270
- toBlock := blockNumber
271
- }
272
- set->Utils.Set.add(blockNumber)->ignore
273
- }
274
- if toBlock.contents - fromBlock.contents > 1000 {
275
- JsError.throwWithMessage(
276
- `Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ${fromBlock.contents->Int.toString}-${toBlock.contents->Int.toString}`,
277
- )
278
- }
279
- let res = await queryBlockData(
280
- ~fromBlock=fromBlock.contents,
281
- ~toBlock=toBlock.contents,
282
- ~client,
283
- ~sourceName,
284
- ~chainId,
285
- ~logger,
286
- ~requestStats,
287
- )
288
- let filtered = res->Result.map(datas => {
289
- datas->Array.filter(data => set->Utils.Set.delete(data.blockNumber))
290
- })
291
- if set->Utils.Set.size > 0 {
292
- JsError.throwWithMessage(
293
- `Invalid response. Failed to get block data for block numbers: ${set
294
- ->Utils.Set.toArray
295
- ->Array.joinUnsafe(", ")}`,
296
- )
297
- }
298
- filtered
299
- }
300
- }
301
- (result, requestStats)
302
- }
303
- }
304
-
305
- let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) => {
306
- let requestStats = []
307
- BlockData.queryBlockData(
308
- ~client,
309
- ~fromBlock=blockNumber,
310
- ~toBlock=blockNumber,
311
- ~sourceName,
312
- ~chainId,
313
- ~logger,
314
- ~requestStats,
315
- )->Promise.thenResolve(res => (res->Result.map(res => res->Array.get(0)), requestStats))
316
- }
317
- let queryBlockDataMulti = BlockData.queryBlockDataMulti
@@ -1,62 +1,89 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
- import * as Time from "../Time.res.mjs";
4
- import * as Utils from "../Utils.res.mjs";
5
3
  import * as Source from "./Source.res.mjs";
6
- import * as Logging from "../Logging.res.mjs";
7
4
  import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
8
- import * as Performance from "../bindings/Performance.res.mjs";
9
5
  import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
10
6
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
11
7
  import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
12
8
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
13
- import * as Stdlib_Result from "@rescript/runtime/lib/es6/Stdlib_Result.js";
14
- import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
15
9
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
16
10
 
17
- function reraisIfRateLimited(exn) {
18
- let e = Primitive_exceptions.internalToException(exn);
19
- if (e.RE_EXN_ID !== "JsExn") {
20
- return;
21
- }
22
- let msg = Stdlib_JsExn.message(e._1);
23
- if (msg === undefined) {
24
- return;
25
- }
26
- if (!msg.startsWith("RATE_LIMITED:")) {
27
- return;
28
- }
29
- let resetMs = Stdlib_Option.getOr(Stdlib_Int.fromString(msg.slice(13, msg.length), undefined), 1000);
30
- throw {
31
- RE_EXN_ID: Source.RateLimited,
32
- resetMs: resetMs,
33
- Error: new Error()
34
- };
35
- }
11
+ let rateLimitedPrefix = "RATE_LIMITED:";
36
12
 
37
- let HyperSyncQueryError = /* @__PURE__ */Primitive_exceptions.create("HyperSync.HyperSyncQueryError");
13
+ let behindHeadPrefix = "SOURCE_BEHIND_HEAD:";
38
14
 
39
- function queryErrorToMsq(e) {
40
- let match = e._0;
41
- return match.queryName + ` query failed due to unexpected missing params on response:
42
- ` + match.missingParams.join(", ");
15
+ function markerValue(msg, prefix) {
16
+ return Stdlib_Int.fromString(msg.slice(prefix.length, msg.length), undefined);
43
17
  }
44
18
 
45
- function mapExn(queryResponse) {
46
- if (queryResponse.TAG === "Ok") {
47
- return {
48
- TAG: "Ok",
49
- _0: queryResponse._0
50
- };
19
+ function mapNativeFailure(failure) {
20
+ let msg = failure.message;
21
+ if (msg !== undefined) {
22
+ if (msg.startsWith(rateLimitedPrefix)) {
23
+ return {
24
+ RE_EXN_ID: Source.RateLimited,
25
+ _1: {
26
+ resetMs: Stdlib_Option.getOr(markerValue(msg, rateLimitedPrefix), 1000),
27
+ requestStats: failure.requestStats
28
+ }
29
+ };
30
+ } else if (msg.startsWith(behindHeadPrefix)) {
31
+ return {
32
+ RE_EXN_ID: Source.SourceBehindHead,
33
+ blockNumber: Stdlib_Option.getOr(markerValue(msg, behindHeadPrefix), 0),
34
+ requestStats: failure.requestStats
35
+ };
36
+ } else {
37
+ return failure.cause;
38
+ }
51
39
  } else {
40
+ return failure.cause;
41
+ }
42
+ }
43
+
44
+ function mapNativeFailureExn(exn) {
45
+ return mapNativeFailure(Source.unpackNativeRequestFailure(exn));
46
+ }
47
+
48
+ function makeGetBlockHashes(query) {
49
+ return async (blockNumbers, param) => {
50
+ let match;
51
+ try {
52
+ let match$1 = await query(blockNumbers);
53
+ match = [
54
+ {
55
+ TAG: "Ok",
56
+ _0: match$1[0]
57
+ },
58
+ match$1[1]
59
+ ];
60
+ } catch (raw_exn) {
61
+ let exn = Primitive_exceptions.internalToException(raw_exn);
62
+ let failure = Source.unpackNativeRequestFailure(exn);
63
+ match = [
64
+ {
65
+ TAG: "Error",
66
+ _0: mapNativeFailure(failure)
67
+ },
68
+ failure.requestStats
69
+ ];
70
+ }
52
71
  return {
53
- TAG: "Error",
54
- _0: {
55
- RE_EXN_ID: HyperSyncQueryError,
56
- _1: queryResponse._0
57
- }
72
+ result: match[0],
73
+ requestStats: match[1]
58
74
  };
75
+ };
76
+ }
77
+
78
+ function reraiseIfRecoverable(exn) {
79
+ let exn$1 = mapNativeFailure(Source.unpackNativeRequestFailure(exn));
80
+ if (exn$1.RE_EXN_ID === Source.RateLimited) {
81
+ throw exn$1;
82
+ }
83
+ if (exn$1.RE_EXN_ID !== Source.SourceBehindHead) {
84
+ return;
59
85
  }
86
+ throw exn$1;
60
87
  }
61
88
 
62
89
  let $$Error = /* @__PURE__ */Primitive_exceptions.create("HyperSync.GetLogs.Error");
@@ -95,7 +122,7 @@ async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes
95
122
  match = await client.getEventItems(query$1, addressSet);
96
123
  } catch (raw_exn) {
97
124
  let exn = Primitive_exceptions.internalToException(raw_exn);
98
- reraisIfRateLimited(exn);
125
+ reraiseIfRecoverable(exn);
99
126
  let missingParams = extractMissingParams(exn);
100
127
  if (missingParams !== undefined) {
101
128
  throw {
@@ -119,10 +146,8 @@ async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes
119
146
  }
120
147
  return {
121
148
  items: res.items,
122
- blocks: res.blocks,
123
149
  nextBlock: res.nextBlock,
124
150
  archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0),
125
- rollbackGuard: res.rollbackGuard,
126
151
  transactionStore: match[1],
127
152
  blockStore: match[2]
128
153
  };
@@ -134,153 +159,11 @@ let GetLogs = {
134
159
  query: query
135
160
  };
136
161
 
137
- function makeRequestBody(fromBlock, toBlock) {
138
- return {
139
- fromBlock: fromBlock,
140
- toBlock: toBlock + 1 | 0,
141
- fieldSelection: {
142
- block: [
143
- "Number",
144
- "Hash",
145
- "Timestamp"
146
- ]
147
- },
148
- includeAllBlocks: true
149
- };
150
- }
151
-
152
- function convertResponse(res) {
153
- return Utils.$$Array.transposeResults(res.data.blocks.map(block => {
154
- let blockNumber = block.number;
155
- if (blockNumber !== undefined) {
156
- let blockHash = block.hash;
157
- if (blockHash !== undefined) {
158
- let blockTimestamp = block.timestamp;
159
- if (blockTimestamp !== undefined) {
160
- return {
161
- TAG: "Ok",
162
- _0: {
163
- blockHash: blockHash,
164
- blockNumber: blockNumber,
165
- blockTimestamp: blockTimestamp
166
- }
167
- };
168
- }
169
- }
170
- }
171
- let missingParams = Stdlib_Array.filterMap([
172
- Utils.Option.mapNone(block.number, "block.number"),
173
- Utils.Option.mapNone(block.timestamp, "block.timestamp"),
174
- Utils.Option.mapNone(block.hash, "block.hash")
175
- ], p => p);
176
- return {
177
- TAG: "Error",
178
- _0: {
179
- TAG: "UnexpectedMissingParams",
180
- _0: {
181
- queryName: "query block data HyperSync",
182
- missingParams: missingParams
183
- }
184
- }
185
- };
186
- }));
187
- }
188
-
189
- async function queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger, requestStats) {
190
- let body = makeRequestBody(fromBlock, toBlock);
191
- let logger$1 = Logging.createChildFrom(logger, {
192
- logType: "HyperSync get block hash query",
193
- fromBlock: fromBlock,
194
- toBlock: toBlock
195
- });
196
- let timerRef = Performance.now();
197
- let maybeSuccessfulRes;
198
- let exit = 0;
199
- let res;
200
- try {
201
- res = await client.get(body);
202
- exit = 1;
203
- } catch (raw_exn) {
204
- let exn = Primitive_exceptions.internalToException(raw_exn);
205
- reraisIfRateLimited(exn);
206
- maybeSuccessfulRes = undefined;
207
- }
208
- if (exit === 1) {
209
- maybeSuccessfulRes = res.nextBlock <= fromBlock ? undefined : res;
210
- }
211
- requestStats.push({
212
- method: "getBlockHashes",
213
- seconds: Performance.secondsSince(timerRef)
214
- });
215
- if (maybeSuccessfulRes !== undefined) {
216
- let err = convertResponse(maybeSuccessfulRes);
217
- if (err.TAG !== "Ok") {
218
- return err;
219
- }
220
- if (maybeSuccessfulRes.nextBlock > toBlock) {
221
- return err;
222
- }
223
- let datas = err._0;
224
- let restRes = await queryBlockData(client, maybeSuccessfulRes.nextBlock, toBlock, sourceName, chainId, logger$1, requestStats);
225
- return Stdlib_Result.map(restRes, rest => datas.concat(rest));
226
- }
227
- Logging.childInfo(logger$1, `Block #` + fromBlock.toString() + ` not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ` + (100).toString() + `ms.`);
228
- await Time.resolvePromiseAfterDelay(100);
229
- return await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger$1, requestStats);
230
- }
231
-
232
- async function queryBlockDataMulti(client, blockNumbers, sourceName, chainId, logger) {
233
- let requestStats = [];
234
- let firstBlock = blockNumbers[0];
235
- let result;
236
- if (firstBlock !== undefined) {
237
- let fromBlock = firstBlock;
238
- let toBlock = firstBlock;
239
- let set = new Set();
240
- for (let idx = 0, idx_finish = blockNumbers.length; idx < idx_finish; ++idx) {
241
- let blockNumber = blockNumbers[idx];
242
- if (blockNumber < fromBlock) {
243
- fromBlock = blockNumber;
244
- }
245
- if (blockNumber > toBlock) {
246
- toBlock = blockNumber;
247
- }
248
- set.add(blockNumber);
249
- }
250
- if ((toBlock - fromBlock | 0) > 1000) {
251
- Stdlib_JsError.throwWithMessage(`Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ` + fromBlock.toString() + `-` + toBlock.toString());
252
- }
253
- let res = await queryBlockData(client, fromBlock, toBlock, sourceName, chainId, logger, requestStats);
254
- let filtered = Stdlib_Result.map(res, datas => datas.filter(data => set.delete(data.blockNumber)));
255
- if (set.size > 0) {
256
- Stdlib_JsError.throwWithMessage(`Invalid response. Failed to get block data for block numbers: ` + Array.from(set).join(", "));
257
- }
258
- result = filtered;
259
- } else {
260
- result = {
261
- TAG: "Ok",
262
- _0: []
263
- };
264
- }
265
- return [
266
- result,
267
- requestStats
268
- ];
269
- }
270
-
271
- function queryBlockData$1(client, blockNumber, sourceName, chainId, logger) {
272
- let requestStats = [];
273
- return queryBlockData(client, blockNumber, blockNumber, sourceName, chainId, logger, requestStats).then(res => [
274
- Stdlib_Result.map(res, res => res[0]),
275
- requestStats
276
- ]);
277
- }
278
-
279
162
  export {
280
- queryErrorToMsq,
163
+ mapNativeFailure,
164
+ mapNativeFailureExn,
165
+ reraiseIfRecoverable,
166
+ makeGetBlockHashes,
281
167
  GetLogs,
282
- queryBlockData$1 as queryBlockData,
283
- queryBlockDataMulti,
284
- mapExn,
285
168
  }
286
- /* Time Not a pure module */
169
+ /* Source Not a pure module */
@@ -1,23 +1,23 @@
1
1
  type logsQueryPage = {
2
2
  items: array<HyperSyncClient.EventItems.item>,
3
- blocks: array<HyperSyncClient.EventItems.blockHeader>,
4
3
  nextBlock: int,
5
4
  archiveHeight: int,
6
- rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
7
5
  transactionStore: TransactionStore.t,
8
6
  blockStore: BlockStore.t,
9
7
  }
10
8
 
11
- type missingParams = {
12
- queryName: string,
13
- missingParams: array<string>,
14
- }
15
-
16
- type queryError = UnexpectedMissingParams(missingParams)
17
-
18
- let queryErrorToMsq: queryError => string
9
+ // Map a native client's `PREFIX:<int>` failure marker onto the exception
10
+ // SourceManager retries on (`RateLimited` / `SourceBehindHead`), falling back to
11
+ // the original cause.
12
+ let mapNativeFailure: Source.nativeRequestFailure => exn
13
+ let mapNativeFailureExn: exn => exn
14
+ let reraiseIfRecoverable: exn => unit
19
15
 
20
- type queryResponse<'a> = result<'a, queryError>
16
+ // Wrap a native client's paginated block-hash query as a `Source.getBlockHashes`,
17
+ // keeping the request timings whether it succeeds or fails.
18
+ let makeGetBlockHashes: (
19
+ ~query: (~blockNumbers: array<int>) => promise<(BlockStore.t, array<Source.requestStat>)>,
20
+ ) => (~blockNumbers: array<int>, ~logger: Pino.t) => promise<Source.getBlockHashesResponse>
21
21
 
22
22
  module GetLogs: {
23
23
  type error =
@@ -38,27 +38,3 @@ module GetLogs: {
38
38
  ~clientFilteredContracts: option<array<string>>,
39
39
  ) => promise<logsQueryPage>
40
40
  }
41
-
42
- let queryBlockData: (
43
- ~client: HyperSyncClient.t,
44
- ~blockNumber: int,
45
- ~sourceName: string,
46
- ~chainId: ChainId.t,
47
- ~logger: Pino.t,
48
- ) => promise<(
49
- queryResponse<option<ReorgDetection.blockDataWithTimestamp>>,
50
- array<Source.requestStat>,
51
- )>
52
-
53
- let queryBlockDataMulti: (
54
- ~client: HyperSyncClient.t,
55
- ~blockNumbers: array<int>,
56
- ~sourceName: string,
57
- ~chainId: ChainId.t,
58
- ~logger: Pino.t,
59
- ) => promise<(
60
- queryResponse<array<ReorgDetection.blockDataWithTimestamp>>,
61
- array<Source.requestStat>,
62
- )>
63
-
64
- let mapExn: queryResponse<'a> => result<'a, exn>