envio 3.3.0-alpha.9 → 3.3.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/index.d.ts +19 -0
  2. package/licenses/CLA.md +35 -0
  3. package/licenses/EULA.md +67 -0
  4. package/licenses/LICENSE.md +45 -0
  5. package/licenses/README.md +35 -0
  6. package/package.json +9 -8
  7. package/src/Batch.res.mjs +1 -1
  8. package/src/BatchProcessing.res +0 -7
  9. package/src/BatchProcessing.res.mjs +1 -8
  10. package/src/ChainFetching.res +30 -18
  11. package/src/ChainFetching.res.mjs +23 -13
  12. package/src/ChainState.res +134 -55
  13. package/src/ChainState.res.mjs +78 -36
  14. package/src/ChainState.resi +14 -2
  15. package/src/Config.res +9 -5
  16. package/src/Config.res.mjs +2 -2
  17. package/src/Core.res +20 -0
  18. package/src/Core.res.mjs +12 -0
  19. package/src/CrossChainState.res +115 -34
  20. package/src/CrossChainState.res.mjs +37 -14
  21. package/src/EffectState.res +100 -0
  22. package/src/EffectState.res.mjs +74 -0
  23. package/src/EffectState.resi +32 -0
  24. package/src/Envio.res +25 -7
  25. package/src/Envio.res.mjs +15 -19
  26. package/src/EventConfigBuilder.res +21 -46
  27. package/src/EventConfigBuilder.res.mjs +18 -25
  28. package/src/EventProcessing.res +8 -5
  29. package/src/EventProcessing.res.mjs +2 -1
  30. package/src/FetchState.res +814 -467
  31. package/src/FetchState.res.mjs +541 -369
  32. package/src/HandlerRegister.res +3 -1
  33. package/src/HandlerRegister.res.mjs +3 -2
  34. package/src/InMemoryStore.res +14 -26
  35. package/src/InMemoryStore.res.mjs +6 -19
  36. package/src/IndexerState.res +15 -39
  37. package/src/IndexerState.res.mjs +18 -29
  38. package/src/IndexerState.resi +2 -10
  39. package/src/Internal.res +98 -14
  40. package/src/Internal.res.mjs +96 -2
  41. package/src/LoadLayer.res +44 -24
  42. package/src/LoadLayer.res.mjs +43 -20
  43. package/src/LoadLayer.resi +1 -0
  44. package/src/LogSelection.res +0 -62
  45. package/src/LogSelection.res.mjs +0 -74
  46. package/src/Metrics.res +1 -1
  47. package/src/Metrics.res.mjs +1 -1
  48. package/src/Persistence.res +12 -3
  49. package/src/PgStorage.res +155 -82
  50. package/src/PgStorage.res.mjs +130 -77
  51. package/src/Prometheus.res +20 -10
  52. package/src/Prometheus.res.mjs +22 -10
  53. package/src/PruneStaleHistory.res +146 -35
  54. package/src/PruneStaleHistory.res.mjs +114 -20
  55. package/src/RawEvent.res +2 -2
  56. package/src/Rollback.res +32 -13
  57. package/src/Rollback.res.mjs +24 -11
  58. package/src/SimulateDeadInputTracker.res +1 -1
  59. package/src/SimulateItems.res +38 -6
  60. package/src/SimulateItems.res.mjs +28 -9
  61. package/src/TestIndexer.res +2 -2
  62. package/src/TestIndexer.res.mjs +2 -2
  63. package/src/TopicFilter.res +1 -25
  64. package/src/TopicFilter.res.mjs +0 -74
  65. package/src/UserContext.res +62 -23
  66. package/src/UserContext.res.mjs +26 -6
  67. package/src/Writing.res +60 -21
  68. package/src/Writing.res.mjs +27 -9
  69. package/src/bindings/NodeJs.res +5 -0
  70. package/src/bindings/Viem.res +0 -5
  71. package/src/sources/EventRouter.res +0 -21
  72. package/src/sources/EventRouter.res.mjs +0 -12
  73. package/src/sources/EvmChain.res +2 -27
  74. package/src/sources/EvmChain.res.mjs +2 -23
  75. package/src/sources/EvmRpcClient.res +58 -23
  76. package/src/sources/EvmRpcClient.res.mjs +11 -5
  77. package/src/sources/HyperSync.res +9 -38
  78. package/src/sources/HyperSync.res.mjs +16 -28
  79. package/src/sources/HyperSync.resi +2 -2
  80. package/src/sources/HyperSyncClient.res +88 -11
  81. package/src/sources/HyperSyncClient.res.mjs +39 -6
  82. package/src/sources/HyperSyncSource.res +18 -199
  83. package/src/sources/HyperSyncSource.res.mjs +9 -128
  84. package/src/sources/Rpc.res +0 -32
  85. package/src/sources/Rpc.res.mjs +1 -46
  86. package/src/sources/RpcSource.res +129 -522
  87. package/src/sources/RpcSource.res.mjs +161 -366
  88. package/src/sources/SimulateSource.res +37 -19
  89. package/src/sources/SimulateSource.res.mjs +27 -10
  90. package/src/sources/SourceManager.res +3 -7
  91. package/src/sources/SourceManager.res.mjs +2 -7
  92. package/src/sources/SourceManager.resi +0 -2
  93. package/src/sources/SvmHyperSyncSource.res +13 -3
  94. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
@@ -1,7 +1,5 @@
1
1
  open Source
2
2
 
3
- exception QueryTimout(string)
4
-
5
3
  // eth_getTransactionByHash/eth_getTransactionReceipt returning null is usually
6
4
  // transient: a load-balanced provider can route the lookup to a node that
7
5
  // hasn't caught up with the one that served eth_getLogs. Must stay retryable,
@@ -74,454 +72,91 @@ let getKnownRawBlockWithBackoff = async (
74
72
  }
75
73
  result.contents->Option.getOrThrow
76
74
  }
77
- // Pulls the provider error message out of either a parsed Rpc.JsonRpcError or
78
- // the raw napi JsExn shape (`error.error.message`), so classifiers below don't
79
- // each have to re-derive it.
75
+ // Pulls the underlying provider error message back out of a caught exn, for
76
+ // logging/debugging. Provider JSON-RPC errors surface as `Rpc.JsonRpcError`;
77
+ // the paging retry decision (see `parseGetNextPageRetryError` below) surfaces
78
+ // as a napi `JsExn` whose message is the JSON payload `EvmRpcClient.getNextPage`
79
+ // throws, carrying the classified message (if any) under `errorMessage`.
80
80
  let getErrorMessage = (exn: exn): option<string> =>
81
81
  switch exn {
82
82
  | Rpc.JsonRpcError({message}) => Some(message)
83
- | JsExn(error) =>
84
- try {
85
- let message: string = (error->Obj.magic)["error"]["message"]
86
- message->S.assertOrThrow(S.string)
87
- Some(message)
88
- } catch {
89
- | _ => None
90
- }
91
- | _ => None
92
- }
93
-
94
- // Deterministic "the range returned too much data" errors that carry no numeric
95
- // block-range suggestion (HyperRPC's 50k-log cap, response-size and result-count
96
- // limits). They depend on log density, not on a fixed block window, so waiting
97
- // never helps — the same range always re-trips the same cap. The reaction is to
98
- // shrink the range and retry immediately, ratcheting the max range down.
99
- let isResponseTooLargeError = {
100
- // Provider-specific "too many logs" messages, matched loosely since each names
101
- // it differently: HyperRPC ("More than 50000 logs returned"), ZkEVM ("query
102
- // returned more than N results"), LlamaRPC ("query exceeds max results"),
103
- // 1RPC ("response size should not..."), Optimism ("(backend) response too
104
- // large"), Arbitrum ("logs matched by query exceeds limit"), Ankr ("block
105
- // range is too wide"). Kept as one block rather than per-line comments — the
106
- // ReScript formatter doesn't preserve comments attached to regex literals in
107
- // an array.
108
- let patterns = [
109
- /more than \d+ logs/i,
110
- /\d+ logs returned/i,
111
- /too many logs/i,
112
- /query returned more than \d+ results/i,
113
- /query exceeds max results/i,
114
- /response size should not/i,
115
- /(backend )?response too large/i,
116
- /logs matched by query exceeds limit/i,
117
- /block range is too wide/i,
118
- ]
119
- (exn: exn) =>
120
- switch exn->getErrorMessage {
121
- | Some(message) => patterns->Array.some(re => re->RegExp.test(message))
122
- | None => false
123
- }
124
- }
125
-
126
- let getSuggestedBlockIntervalFromExn = {
127
- // Unknown provider: "retry with the range 123-456"
128
- let suggestedRangeRegExp = /retry with the range (\d+)-(\d+)/
129
-
130
- // QuickNode, 1RPC, Blast: "limited to a 1000 blocks range"
131
- let blockRangeLimitRegExp = /limited to a (\d+) blocks range/
132
-
133
- // Alchemy: "up to a 500 block range"
134
- let alchemyRangeRegExp = /up to a (\d+) block range/
135
-
136
- // Cloudflare: "Max range: 3500"
137
- let cloudflareRangeRegExp = /Max range: (\d+)/
138
-
139
- // Thirdweb: "Maximum allowed number of requested blocks is 3500"
140
- let thirdwebRangeRegExp = /Maximum allowed number of requested blocks is (\d+)/
141
-
142
- // BlockPI: "limited to 2000 block"
143
- let blockpiRangeRegExp = /limited to (\d+) block/
144
-
145
- // Base: "block range too large" - fixed 2000 block limit
146
- let baseRangeRegExp = /block range too large/
147
-
148
- // evm-rpc.sei-apis.com: "block range too large (2000), maximum allowed is 1000 blocks"
149
- let maxAllowedBlocksRegExp = /maximum allowed is (\d+) blocks/
150
-
151
- // Blast (paid): "exceeds the range allowed for your plan (5000 > 3000)"
152
- let blastPaidRegExp = /exceeds the range allowed for your plan \(\d+ > (\d+)\)/
153
-
154
- // Chainstack: "Block range limit exceeded" - 10000 block limit
155
- let chainstackRegExp = /Block range limit exceeded./
156
-
157
- // Coinbase: "please limit the query to at most 1000 blocks"
158
- let coinbaseRegExp = /please limit the query to at most (\d+) blocks/
159
-
160
- // PublicNode: "maximum block range: 2000"
161
- let publicNodeRegExp = /maximum block range: (\d+)/
162
-
163
- // Hyperliquid: "query exceeds max block range 1000"
164
- let hyperliquidRegExp = /query exceeds max block range (\d+)/
165
-
166
- // TODO: Reproduce how the error message looks like
167
- // when we send request with numeric block range instead of hex
168
- // Infura, ZkSync: "Try with this block range [0x123,0x456]"
169
-
170
- let parseMessageForBlockRange = (message: string) => {
171
- // Helper to extract block range from regex match
172
- let extractBlockRange = (execResult, ~isMaxRange) =>
173
- switch execResult->RegExp.Result.matches {
174
- | [Some(blockRangeLimit)] =>
175
- switch blockRangeLimit->Int.fromString {
176
- | Some(blockRangeLimit) if blockRangeLimit > 0 => Some(blockRangeLimit, isMaxRange)
177
- | _ => None
178
- }
179
- | _ => None
180
- }
181
-
182
- // Try each regex pattern in order
183
- switch suggestedRangeRegExp->RegExp.exec(message) {
184
- | Some(execResult) =>
185
- switch execResult->RegExp.Result.matches {
186
- | [Some(fromBlock), Some(toBlock)] =>
187
- switch (fromBlock->Int.fromString, toBlock->Int.fromString) {
188
- | (Some(fromBlock), Some(toBlock)) if toBlock >= fromBlock =>
189
- Some(toBlock - fromBlock + 1, false)
83
+ | JsExn(e) =>
84
+ switch e->JsExn.message {
85
+ | Some(msg) =>
86
+ switch msg->JSON.parseOrThrow->JSON.Decode.object {
87
+ | exception _ => None
88
+ | None => None
89
+ | Some(obj) =>
90
+ switch obj->Dict.get("errorMessage") {
91
+ | Some(String(message)) => Some(message)
190
92
  | _ => None
191
93
  }
192
- | _ => None
193
- }
194
- | None =>
195
- // Try each provider's specific error pattern
196
- switch blockRangeLimitRegExp->RegExp.exec(message) {
197
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
198
- | None =>
199
- switch alchemyRangeRegExp->RegExp.exec(message) {
200
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
201
- | None =>
202
- switch cloudflareRangeRegExp->RegExp.exec(message) {
203
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
204
- | None =>
205
- switch thirdwebRangeRegExp->RegExp.exec(message) {
206
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
207
- | None =>
208
- switch blockpiRangeRegExp->RegExp.exec(message) {
209
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
210
- | None =>
211
- switch maxAllowedBlocksRegExp->RegExp.exec(message) {
212
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
213
- | None =>
214
- switch baseRangeRegExp->RegExp.exec(message) {
215
- | Some(_) => Some(2000, true)
216
- | None =>
217
- switch blastPaidRegExp->RegExp.exec(message) {
218
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
219
- | None =>
220
- switch chainstackRegExp->RegExp.exec(message) {
221
- | Some(_) => Some(10000, true)
222
- | None =>
223
- switch coinbaseRegExp->RegExp.exec(message) {
224
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
225
- | None =>
226
- switch publicNodeRegExp->RegExp.exec(message) {
227
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
228
- | None =>
229
- switch hyperliquidRegExp->RegExp.exec(message) {
230
- | Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
231
- | None => None
232
- }
233
- }
234
- }
235
- }
236
- }
237
- }
238
- }
239
- }
240
- }
241
- }
242
- }
243
94
  }
244
- }
245
- }
246
-
247
- (exn): option<(
248
- // The suggested block range
249
- int,
250
- // Whether it's the max range that the provider allows
251
- bool,
252
- )> =>
253
- switch exn->getErrorMessage {
254
- | Some(message) => parseMessageForBlockRange(message)
255
95
  | None => None
256
96
  }
257
- }
258
-
259
- type eventBatchQuery = {
260
- items: array<EvmRpcClient.rpcEventItem>,
261
- latestFetchedBlockInfo: blockInfo,
262
- }
263
-
264
- let maxSuggestedBlockIntervalKey = "max"
265
-
266
- let getSourceMaxBlockInterval = (mutSuggestedBlockIntervals, ~intervalCeiling) =>
267
- mutSuggestedBlockIntervals
268
- ->Utils.Dict.dangerouslyGetNonOption(maxSuggestedBlockIntervalKey)
269
- ->Option.getOr(intervalCeiling)
270
-
271
- type logSelection = {
272
- addresses: option<array<Address.t>>,
273
- topicQuery: Rpc.GetLogs.topicQuery,
274
- }
275
-
276
- // A log can satisfy more than one selection when a single event's `where` is an
277
- // OR of param groups, so dedup the fanned-out responses by (blockNumber,
278
- // logIndex) — unique per chain — keeping the first occurrence.
279
- let mergeAndDedupItems = (itemsPerSelection: array<array<EvmRpcClient.rpcEventItem>>) => {
280
- let seen = Utils.Set.make()
281
- let merged = []
282
- itemsPerSelection->Array.forEach(items =>
283
- items->Array.forEach((item: EvmRpcClient.rpcEventItem) => {
284
- let key = `${item.log.blockNumber->Int.toString}-${item.log.logIndex->Int.toString}`
285
- if seen->Utils.Set.has(key)->not {
286
- seen->Utils.Set.add(key)->ignore
287
- merged->Array.push(item)->ignore
288
- }
289
- })
290
- )
291
- merged
292
- }
293
-
294
- let getNextPage = (
295
- ~fromBlock,
296
- ~toBlock,
297
- ~logSelections: array<logSelection>,
298
- ~loadBlock,
299
- ~syncConfig as sc: Config.sourceSync,
300
- ~rpcClient: EvmRpcClient.t,
301
- ~mutSuggestedBlockIntervals,
302
- ~partitionId,
303
- ~recordRequest: (~method: string, ~seconds: float) => unit,
304
- ): promise<eventBatchQuery> => {
305
- //If the query hangs for longer than this, reject this promise to reduce the block interval
306
- let queryTimoutPromise =
307
- Time.resolvePromiseAfterDelay(~delayMilliseconds=sc.queryTimeoutMillis)->Promise.then(() =>
308
- Promise.reject(
309
- QueryTimout(`Query took longer than ${Int.toString(sc.queryTimeoutMillis / 1000)} seconds`),
310
- )
311
- )
312
-
313
- let latestFetchedBlockPromise = loadBlock(toBlock)
314
-
315
- let queryLogs = ({addresses, topicQuery}: logSelection) => {
316
- let timerRef = Performance.now()
317
- rpcClient.getLogs({
318
- fromBlock,
319
- toBlock,
320
- ?addresses,
321
- topics: topicQuery->Array.map(filter =>
322
- switch filter {
323
- | Rpc.GetLogs.Null => Nullable.null
324
- | Single(topic) => Nullable.make([topic])
325
- | Multiple(topics) => Nullable.make(topics)
326
- }
327
- ),
328
- })->Promise.thenResolve(items => {
329
- recordRequest(~method="eth_getLogs", ~seconds=timerRef->Performance.secondsSince)
330
- items
331
- })
332
- }
333
-
334
- let logsPromise = switch logSelections {
335
- | [] =>
336
- latestFetchedBlockPromise->Promise.thenResolve((latestFetchedBlockInfo): eventBatchQuery => {
337
- items: [],
338
- latestFetchedBlockInfo,
339
- })
340
- // Fast path: a single selection needs no cross-request merge or dedup.
341
- | [logSelection] =>
342
- logSelection
343
- ->queryLogs
344
- ->Promise.then(async items => {
345
- {
346
- items,
347
- latestFetchedBlockInfo: await latestFetchedBlockPromise,
348
- }
349
- })
350
- | _ =>
351
- logSelections
352
- ->Array.map(queryLogs)
353
- ->Promise.all
354
- ->Promise.then(async itemsPerSelection => {
355
- {
356
- items: itemsPerSelection->mergeAndDedupItems,
357
- latestFetchedBlockInfo: await latestFetchedBlockPromise,
358
- }
359
- })
360
- }
361
-
362
- [queryTimoutPromise, logsPromise]
363
- ->Promise.race
364
- ->Promise.catch(err => {
365
- let executedBlockInterval = toBlock - fromBlock + 1
366
- let shrunkBlockInterval = Pervasives.max(
367
- 1,
368
- (executedBlockInterval->Int.toFloat *. sc.backoffMultiplicative)->Float.toInt,
369
- )
370
-
371
- let throwFailedGettingItems = retry =>
372
- throw(Source.GetItemsError(FailedGettingItems({exn: err, attemptedToBlock: toBlock, retry})))
373
- let throwResize = interval =>
374
- throwFailedGettingItems(WithSuggestedToBlock({toBlock: fromBlock + interval - 1}))
375
-
376
- switch getSuggestedBlockIntervalFromExn(err) {
377
- // "limited to N blocks" — a structural cap on the whole source; only tighten.
378
- | Some((interval, true)) =>
379
- let capped = Pervasives.min(
380
- mutSuggestedBlockIntervals->getSourceMaxBlockInterval(~intervalCeiling=sc.intervalCeiling),
381
- interval,
382
- )
383
- mutSuggestedBlockIntervals->Dict.set(maxSuggestedBlockIntervalKey, capped)
384
- throwResize(capped)
385
- // A one-off suggested range ("retry with range X-Y") — apply to this partition.
386
- | Some((interval, false)) =>
387
- mutSuggestedBlockIntervals->Dict.set(partitionId, interval)
388
- throwResize(interval)
389
- // Density cap with no suggested number (too many logs / response too large):
390
- // shrink THIS partition and retry immediately (no wait); acceleration
391
- // re-adapts on the next successful query. The interval>1 guard avoids a
392
- // no-progress tight loop on a single over-cap block.
393
- | None if executedBlockInterval > 1 && err->isResponseTooLargeError =>
394
- mutSuggestedBlockIntervals->Dict.set(partitionId, shrunkBlockInterval)
395
- throwResize(shrunkBlockInterval)
396
- // Transient/unknown — shrink this partition and back off.
397
- | None =>
398
- mutSuggestedBlockIntervals->Dict.set(partitionId, shrunkBlockInterval)
399
- throwFailedGettingItems(
400
- WithBackoff({
401
- message: `Failed getting data for the block range. Will try smaller block range for the next attempt.`,
402
- backoffMillis: sc.backoffMillis,
403
- }),
404
- )
405
- }
406
- })
407
- }
408
-
409
- type selectionConfig = {
410
- getLogSelectionsOrThrow: (
411
- ~addressesByContractName: dict<array<Address.t>>,
412
- ) => array<logSelection>,
413
- }
414
-
415
- let getSelectionConfig = (selection: FetchState.selection) => {
416
- let evmOnEventRegistrations =
417
- selection.onEventRegistrations->(
418
- Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>
419
- )
420
-
421
- if evmOnEventRegistrations->Utils.Array.isEmpty {
422
- throw(
423
- Source.GetItemsError(
424
- UnsupportedSelection({
425
- message: "Invalid events configuration for the partition. Nothing to fetch. Please, report to the Envio team.",
426
- }),
427
- ),
428
- )
97
+ | _ => None
429
98
  }
430
99
 
431
- // eth_getLogs takes one address list and one topic selection per request, so
432
- // fan out to one request per selection. Each address-bound event is grouped by
433
- // its contract and later scoped to that contract's own addresses — pooling all
434
- // contracts' addresses would let one contract's query fetch a sibling's logs,
435
- // which route back by address and bypass the sibling's filter (routing never
436
- // re-applies it). Pure-wildcard events carry no address constraint, so they're
437
- // pooled and resolved once.
438
- let noAddressTopicSelections = []
439
- let byContract = Dict.make()
440
- let wildcardByContract = Dict.make()
441
- let contractNames = Utils.Set.make()
442
-
443
- evmOnEventRegistrations->Array.forEach(reg => {
444
- let contractName = reg.eventConfig.contractName
445
- let {isWildcard, dependsOnAddresses, resolvedWhere} = reg
446
- if dependsOnAddresses {
447
- contractNames->Utils.Set.add(contractName)->ignore
448
- (isWildcard ? wildcardByContract : byContract)->Utils.Dict.pushMany(
449
- contractName,
450
- resolvedWhere.topicSelections,
451
- )
452
- } else {
453
- noAddressTopicSelections
454
- ->Array.pushMany(
455
- resolvedWhere.topicSelections->LogSelection.materializeTopicSelections(~addresses=[]),
456
- )
457
- ->ignore
458
- }
459
- })
460
-
461
- // `compressTopicSelections` folds the filter-less events into a single topic0
462
- // OR-set, keeping the common case at one request.
463
- let toLogSelections = (~addresses, topicSelections): array<logSelection> =>
464
- topicSelections
465
- ->LogSelection.compressTopicSelections
466
- ->Array.map(topicSelection => {
467
- addresses,
468
- topicQuery: topicSelection->Rpc.GetLogs.mapTopicQuery,
469
- })
470
-
471
- // Address-independent, so resolve once (the wildcard partition reuses this).
472
- let noAddressLogSelections = toLogSelections(~addresses=None, noAddressTopicSelections)
473
100
 
474
- let getLogSelectionsOrThrow = if contractNames->Utils.Set.size === 0 {
475
- (~addressesByContractName as _) => noAddressLogSelections
476
- } else {
477
- (~addressesByContractName): array<logSelection> => {
478
- let logSelections = noAddressLogSelections->Array.copy
479
- contractNames->Utils.Set.forEach(contractName => {
480
- switch addressesByContractName->Utils.Dict.dangerouslyGetNonOption(contractName) {
481
- | None
482
- | Some([]) => ()
483
- | Some(addresses) =>
484
- // Non-wildcard filters, scoped to this contract's addresses.
485
- switch byContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
486
- | Some(topicSelections) =>
487
- logSelections
488
- ->Array.pushMany(
489
- toLogSelections(
490
- ~addresses=Some(addresses),
491
- topicSelections->LogSelection.materializeTopicSelections(~addresses),
492
- ),
101
+ // `EvmRpcClient.getNextPage` throws a napi error whose message is a JSON
102
+ // payload describing the retry decision:
103
+ // `{"kind":"Retry","attemptedToBlock":int,"errorMessage":string|null,
104
+ // "requestStats":[{"method":string,"seconds":float}],"retry":
105
+ // {"tag":"WithSuggestedToBlock","toBlock":int} |
106
+ // {"tag":"WithBackoff","message":string,"backoffMillis":int}}`.
107
+ let parseGetNextPageRetryError = (exn: exn): option<(
108
+ int,
109
+ Source.getItemsRetry,
110
+ array<Source.requestStat>,
111
+ )> =>
112
+ switch exn {
113
+ | JsExn(e) =>
114
+ switch e->JsExn.message {
115
+ | Some(msg) =>
116
+ switch msg->JSON.parseOrThrow->JSON.Decode.object {
117
+ | exception _ => None
118
+ | None => None
119
+ | Some(obj) =>
120
+ switch (obj->Dict.get("kind"), obj->Dict.get("attemptedToBlock"), obj->Dict.get("retry")) {
121
+ | (Some(String("Retry")), Some(Number(attemptedToBlock)), Some(Object(retryObj))) =>
122
+ let requestStats = switch obj->Dict.get("requestStats") {
123
+ | Some(Array(stats)) =>
124
+ stats->Array.filterMap(s =>
125
+ switch s->JSON.Decode.object {
126
+ | Some(o) =>
127
+ switch (o->Dict.get("method"), o->Dict.get("seconds")) {
128
+ | (Some(String(method)), Some(Number(seconds))) => Some({Source.method, seconds})
129
+ | _ => None
130
+ }
131
+ | None => None
132
+ }
493
133
  )
494
- ->ignore
495
- | None => ()
134
+ | _ => []
496
135
  }
497
-
498
- // Wildcard-by-address filters fold the address into the topics,
499
- // so they still match any address.
500
- switch wildcardByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
501
- | Some(topicSelections) =>
502
- logSelections
503
- ->Array.pushMany(
504
- toLogSelections(
505
- ~addresses=None,
506
- topicSelections->LogSelection.materializeTopicSelections(~addresses),
507
- ),
508
- )
509
- ->ignore
510
- | None => ()
136
+ let retry = switch retryObj->Dict.get("tag") {
137
+ | Some(String("WithSuggestedToBlock")) =>
138
+ switch retryObj->Dict.get("toBlock") {
139
+ | Some(Number(toBlock)) =>
140
+ Some(Source.WithSuggestedToBlock({toBlock: toBlock->Float.toInt}))
141
+ | _ => None
142
+ }
143
+ | Some(String("WithBackoff")) =>
144
+ switch (retryObj->Dict.get("message"), retryObj->Dict.get("backoffMillis")) {
145
+ | (Some(String(message)), Some(Number(backoffMillis))) =>
146
+ Some(Source.WithBackoff({message, backoffMillis: backoffMillis->Float.toInt}))
147
+ | _ => None
148
+ }
149
+ | _ => None
511
150
  }
151
+ retry->Option.map(retry => (attemptedToBlock->Float.toInt, retry, requestStats))
152
+ | _ => None
512
153
  }
513
- })
514
- logSelections
154
+ }
155
+ | None => None
515
156
  }
157
+ | _ => None
516
158
  }
517
159
 
518
- {
519
- getLogSelectionsOrThrow: getLogSelectionsOrThrow,
520
- }
521
- }
522
-
523
- let memoGetSelectionConfig = () => Utils.WeakMap.memoize(getSelectionConfig)
524
-
525
160
  // Type-erase a schema for storage in the field registry
526
161
  external toFieldSchema: S.t<'a> => S.t<JSON.t> = "%identity"
527
162
 
@@ -980,8 +615,8 @@ type options = {
980
615
  syncConfig: Config.sourceSync,
981
616
  url: string,
982
617
  chain: ChainMap.Chain.t,
983
- eventRouter: EventRouter.t<Internal.evmOnEventRegistration>,
984
- allEventParams: array<HyperSyncClient.Decoder.eventParamsInput>,
618
+ // The chain's registrations, indexed by their sequential `index`.
619
+ onEventRegistrations: array<Internal.evmOnEventRegistration>,
985
620
  lowercaseAddresses: bool,
986
621
  ws?: string,
987
622
  headers?: dict<string>,
@@ -993,8 +628,7 @@ let make = (
993
628
  syncConfig,
994
629
  url,
995
630
  chain,
996
- eventRouter,
997
- allEventParams,
631
+ onEventRegistrations,
998
632
  lowercaseAddresses,
999
633
  ?ws,
1000
634
  ?headers,
@@ -1010,19 +644,14 @@ let make = (
1010
644
  }
1011
645
  let name = `RPC (${urlHost})`
1012
646
 
1013
- let getSelectionConfig = memoGetSelectionConfig()
1014
-
1015
- // Per-partition adaptive block interval (AIMD), keyed by partitionId. The
1016
- // `max` key holds a source-wide ceiling that only ever tightens, set by
1017
- // structural provider limits ("limited to N blocks"). A partition's own entry
1018
- // can go stale when partitions merge/split — acceptable, it re-adapts.
1019
- let mutSuggestedBlockIntervals = Dict.make()
1020
-
1021
647
  let client = Rpc.makeClient(url, ~headers?)
1022
648
  let rpcClient = EvmRpcClient.make(
1023
649
  ~url,
1024
- ~allEventParams,
650
+ ~eventRegistrations=HyperSyncClient.Registration.fromOnEventRegistrations(
651
+ onEventRegistrations,
652
+ ),
1025
653
  ~checksumAddresses=!lowercaseAddresses,
654
+ ~syncConfig,
1026
655
  ~headers?,
1027
656
  )
1028
657
 
@@ -1171,7 +800,7 @@ let make = (
1171
800
  ~fromBlock,
1172
801
  ~toBlock,
1173
802
  ~addressesByContractName,
1174
- ~contractNameByAddress,
803
+ ~contractNameByAddress as _,
1175
804
  ~knownHeight,
1176
805
  ~partitionId,
1177
806
  ~selection: FetchState.selection,
@@ -1181,27 +810,12 @@ let make = (
1181
810
  ) => {
1182
811
  let startFetchingBatchTimeRef = Performance.now()
1183
812
 
1184
- let sourceMaxBlockInterval =
1185
- mutSuggestedBlockIntervals->getSourceMaxBlockInterval(
1186
- ~intervalCeiling=syncConfig.intervalCeiling,
1187
- )
1188
- let suggestedBlockInterval = Pervasives.min(
1189
- mutSuggestedBlockIntervals
1190
- ->Utils.Dict.dangerouslyGetNonOption(partitionId)
1191
- ->Option.getOr(syncConfig.initialBlockInterval),
1192
- sourceMaxBlockInterval,
1193
- )
1194
-
1195
813
  // Always have a toBlock for an RPC worker
1196
814
  let toBlock = switch toBlock {
1197
815
  | Some(toBlock) => Pervasives.min(toBlock, knownHeight)
1198
816
  | None => knownHeight
1199
817
  }
1200
818
 
1201
- let suggestedToBlock = Pervasives.min(fromBlock + suggestedBlockInterval - 1, toBlock)
1202
- //Defensively ensure we never query a target block below fromBlock
1203
- ->Pervasives.max(fromBlock)
1204
-
1205
819
  let firstBlockParentPromise =
1206
820
  fromBlock > 0
1207
821
  ? blockLoader.contents
@@ -1209,67 +823,64 @@ let make = (
1209
823
  ->Promise.thenResolve(json => Some(parseBlockInfo(json)))
1210
824
  : Promise.resolve(None)
1211
825
 
1212
- let {getLogSelectionsOrThrow} = getSelectionConfig(selection)
1213
- let logSelections = getLogSelectionsOrThrow(~addressesByContractName)
1214
-
1215
- let {items, latestFetchedBlockInfo} = await getNextPage(
1216
- ~fromBlock,
1217
- ~toBlock=suggestedToBlock,
1218
- ~logSelections,
1219
- ~loadBlock=blockNumber =>
1220
- blockLoader.contents
1221
- ->LazyLoader.get(blockNumber)
1222
- ->Promise.thenResolve(parseBlockInfo),
1223
- ~syncConfig,
1224
- ~rpcClient,
1225
- ~mutSuggestedBlockIntervals,
1226
- ~partitionId,
1227
- ~recordRequest,
1228
- )
1229
-
1230
- let executedBlockInterval = suggestedToBlock - fromBlock + 1
1231
-
1232
- // Grow this partition's interval only when the full suggested range was
1233
- // actually applied (not clamped by a hard toBlock). The min clamps to the
1234
- // source-wide ceiling, which also stops growth once a structural cap tightened it.
1235
- // See: https://en.wikipedia.org/wiki/Additive_increase/multiplicative_decrease
1236
- if executedBlockInterval >= suggestedBlockInterval {
1237
- mutSuggestedBlockIntervals->Dict.set(
1238
- partitionId,
1239
- Pervasives.min(
1240
- executedBlockInterval + syncConfig.accelerationAdditive,
1241
- sourceMaxBlockInterval,
826
+ if selection.onEventRegistrations->Utils.Array.isEmpty {
827
+ throw(
828
+ Source.GetItemsError(
829
+ UnsupportedSelection({
830
+ message: "Invalid events configuration for the partition. Nothing to fetch. Please, report to the Envio team.",
831
+ }),
1242
832
  ),
1243
833
  )
1244
834
  }
1245
835
 
1246
- let parsedQueueItems = await items
1247
- ->Array.filterMap(({log, params: maybeDecodedEvent}: EvmRpcClient.rpcEventItem) => {
1248
- let topic0 = log.topics[0]->Option.getOr("0x0")
1249
- let routedAddress = if lowercaseAddresses {
1250
- log.address->Address.Evm.fromAddressLowercaseOrThrow
1251
- } else {
1252
- log.address->Address.Evm.fromAddressOrThrow
836
+ let {items, toBlock: queriedToBlock, requestStats} = try await rpcClient.getNextPage({
837
+ fromBlock,
838
+ toBlockCeiling: toBlock,
839
+ partitionId,
840
+ registrationIndexes: selection.onEventRegistrations->Array.map(reg => reg.index),
841
+ addressesByContractName,
842
+ }) catch {
843
+ | exn =>
844
+ switch exn->parseGetNextPageRetryError {
845
+ | Some((attemptedToBlock, retry, requestStats)) =>
846
+ requestStats->Array.forEach(stat =>
847
+ recordRequest(~method=stat.method, ~seconds=stat.seconds)
848
+ )
849
+ throw(Source.GetItemsError(FailedGettingItems({exn, attemptedToBlock, retry})))
850
+ | None =>
851
+ throw(
852
+ Source.GetItemsError(
853
+ FailedGettingItems({
854
+ exn,
855
+ attemptedToBlock: toBlock,
856
+ retry: WithBackoff({
857
+ message: "Unexpected issue while fetching events from the RPC client. Attempt a retry.",
858
+ backoffMillis: switch retry {
859
+ | 0 => 500
860
+ | _ => 1000 * retry
861
+ },
862
+ }),
863
+ }),
864
+ ),
865
+ )
1253
866
  }
867
+ }
868
+ requestStats->Array.forEach(stat => recordRequest(~method=stat.method, ~seconds=stat.seconds))
1254
869
 
1255
- switch eventRouter->EventRouter.get(
1256
- ~tag=EventRouter.getEvmEventId(~sighash=topic0, ~topicCount=log.topics->Array.length),
1257
- ~contractNameByAddress,
1258
- ~contractAddress=routedAddress,
1259
- ) {
1260
- | None => None
1261
- | Some(onEventRegistration) =>
1262
- let eventConfig =
1263
- onEventRegistration.eventConfig->(
1264
- Utils.magic: Internal.eventConfig => Internal.evmEventConfig
1265
- )
1266
- switch maybeDecodedEvent
1267
- ->Nullable.toOption
1268
- ->Option.flatMap(Dict.get(_, eventConfig.contractName)) {
1269
- | Some(decoded) =>
1270
- Some(
1271
- (
1272
- async () => {
870
+ let latestFetchedBlockInfo = await blockLoader.contents
871
+ ->LazyLoader.get(queriedToBlock)
872
+ ->Promise.thenResolve(parseBlockInfo)
873
+
874
+ let parsedQueueItems = await items
875
+ ->Array.map(({log, onEventRegistrationIndex, params: decoded}: EvmRpcClient.rpcEventItem) => {
876
+ // `log.address` comes back already normalized to the client's casing.
877
+ let onEventRegistration = onEventRegistrations->Array.getUnsafe(onEventRegistrationIndex)
878
+ let eventConfig =
879
+ onEventRegistration.eventConfig->(
880
+ Utils.magic: Internal.eventConfig => Internal.evmEventConfig
881
+ )
882
+ (
883
+ async () => {
1273
884
  let (block, transaction) = try await Promise.all2((
1274
885
  log->getEventBlockOrThrow(~selectedBlockFields=eventConfig.selectedBlockFields),
1275
886
  log->getEventTransactionOrThrow(
@@ -1321,16 +932,12 @@ let make = (
1321
932
  params: decoded,
1322
933
  block,
1323
934
  transaction,
1324
- srcAddress: routedAddress,
935
+ srcAddress: log.address,
1325
936
  logIndex: log.logIndex,
1326
937
  }->Evm.fromPayload,
1327
938
  })
1328
- }
1329
- )(),
1330
- )
1331
- | None => None
1332
939
  }
1333
- }
940
+ )()
1334
941
  })
1335
942
  ->Promise.all
1336
943