envio 3.3.0-alpha.4 → 3.3.0-alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/evm.schema.json +10 -0
  2. package/package.json +7 -7
  3. package/src/Batch.res.mjs +1 -1
  4. package/src/BatchProcessing.res +5 -0
  5. package/src/BatchProcessing.res.mjs +6 -0
  6. package/src/ChainState.res +41 -7
  7. package/src/ChainState.res.mjs +22 -18
  8. package/src/ChainState.resi +3 -1
  9. package/src/Config.res +3 -0
  10. package/src/Config.res.mjs +3 -1
  11. package/src/Core.res +0 -3
  12. package/src/ExitOnCaughtUp.res +10 -2
  13. package/src/ExitOnCaughtUp.res.mjs +10 -1
  14. package/src/FetchState.res +63 -133
  15. package/src/FetchState.res.mjs +107 -166
  16. package/src/IndexerState.res +4 -0
  17. package/src/IndexerState.res.mjs +8 -1
  18. package/src/IndexerState.resi +1 -0
  19. package/src/IndexingAddresses.res +105 -0
  20. package/src/IndexingAddresses.res.mjs +100 -0
  21. package/src/IndexingAddresses.resi +32 -0
  22. package/src/Main.res +4 -15
  23. package/src/Main.res.mjs +6 -14
  24. package/src/Metrics.res +33 -0
  25. package/src/Metrics.res.mjs +39 -0
  26. package/src/Prometheus.res +2 -20
  27. package/src/Prometheus.res.mjs +83 -95
  28. package/src/SimulateDeadInputTracker.res +85 -0
  29. package/src/SimulateDeadInputTracker.res.mjs +73 -0
  30. package/src/SimulateDeadInputTracker.resi +12 -0
  31. package/src/SimulateItems.res +23 -42
  32. package/src/SimulateItems.res.mjs +12 -30
  33. package/src/TestIndexer.res +3 -27
  34. package/src/TestIndexer.res.mjs +2 -9
  35. package/src/bindings/Viem.res +0 -41
  36. package/src/bindings/Viem.res.mjs +1 -43
  37. package/src/sources/Evm.res +7 -36
  38. package/src/sources/Evm.res.mjs +4 -36
  39. package/src/sources/EvmChain.res +4 -2
  40. package/src/sources/EvmChain.res.mjs +3 -2
  41. package/src/sources/EvmRpcClient.res +36 -5
  42. package/src/sources/EvmRpcClient.res.mjs +7 -4
  43. package/src/sources/HyperSyncClient.res +0 -35
  44. package/src/sources/HyperSyncClient.res.mjs +1 -8
  45. package/src/sources/Rpc.res +15 -47
  46. package/src/sources/Rpc.res.mjs +25 -56
  47. package/src/sources/RpcSource.res +289 -204
  48. package/src/sources/RpcSource.res.mjs +293 -303
  49. package/src/sources/SimulateSource.res +1 -0
  50. package/src/sources/SimulateSource.res.mjs +2 -1
  51. package/src/sources/Source.res +4 -0
  52. package/src/sources/Svm.res +7 -15
  53. package/src/sources/Svm.res.mjs +4 -15
  54. package/src/sources/TransactionStore.res +14 -2
  55. package/src/sources/TransactionStore.res.mjs +10 -2
@@ -74,6 +74,47 @@ let getKnownRawBlockWithBackoff = async (
74
74
  }
75
75
  result.contents->Option.getOrThrow
76
76
  }
77
+ // Pulls the provider error message out of either a parsed Rpc.JsonRpcError or
78
+ // the raw napi JsExn shape (`error.error.message`), so classifiers below don't
79
+ // each have to re-derive it.
80
+ let getErrorMessage = (exn: exn): option<string> =>
81
+ switch exn {
82
+ | Rpc.JsonRpcError({message}) => Some(message)
83
+ | JsExn(error) =>
84
+ try {
85
+ let message: string = (error->Obj.magic)["error"]["message"]
86
+ message->S.assertOrThrow(S.string)
87
+ Some(message)
88
+ } catch {
89
+ | _ => None
90
+ }
91
+ | _ => None
92
+ }
93
+
94
+ // Deterministic "the range returned too much data" errors that carry no numeric
95
+ // block-range suggestion (HyperRPC's 50k-log cap, response-size and result-count
96
+ // limits). They depend on log density, not on a fixed block window, so waiting
97
+ // never helps — the same range always re-trips the same cap. The reaction is to
98
+ // shrink the range and retry immediately, ratcheting the max range down.
99
+ let isResponseTooLargeError = {
100
+ let patterns = [
101
+ /more than \d+ logs/i, // HyperRPC: "More than 50000 logs returned"
102
+ /\d+ logs returned/i,
103
+ /too many logs/i,
104
+ /query returned more than \d+ results/i, // ZkEVM
105
+ /query exceeds max results/i, // LlamaRPC
106
+ /response size should not/i, // 1RPC
107
+ /(backend )?response too large/i, // Optimism
108
+ /logs matched by query exceeds limit/i, // Arbitrum
109
+ /block range is too wide/i, // Ankr
110
+ ]
111
+ (exn: exn) =>
112
+ switch exn->getErrorMessage {
113
+ | Some(message) => patterns->Array.some(re => re->RegExp.test(message))
114
+ | None => false
115
+ }
116
+ }
117
+
77
118
  let getSuggestedBlockIntervalFromExn = {
78
119
  // Unknown provider: "retry with the range 123-456"
79
120
  let suggestedRangeRegExp = /retry with the range (\d+)-(\d+)/
@@ -118,14 +159,6 @@ let getSuggestedBlockIntervalFromExn = {
118
159
  // when we send request with numeric block range instead of hex
119
160
  // Infura, ZkSync: "Try with this block range [0x123,0x456]"
120
161
 
121
- // Future handling needed for these providers that don't suggest ranges:
122
- // - Ankr: "block range is too wide"
123
- // - 1RPC: "response size should not greater than 10000000 bytes"
124
- // - ZkEVM: "query returned more than 10000 results"
125
- // - LlamaRPC: "query exceeds max results"
126
- // - Optimism: "backend response too large" or "Block range is too large"
127
- // - Arbitrum: "logs matched by query exceeds limit of 10000"
128
-
129
162
  let parseMessageForBlockRange = (message: string) => {
130
163
  // Helper to extract block range from regex match
131
164
  let extractBlockRange = (execResult, ~isMaxRange) =>
@@ -209,35 +242,54 @@ let getSuggestedBlockIntervalFromExn = {
209
242
  // Whether it's the max range that the provider allows
210
243
  bool,
211
244
  )> =>
212
- switch exn {
213
- | Rpc.JsonRpcError({message}) => parseMessageForBlockRange(message)
214
- | JsExn(error) =>
215
- try {
216
- let message: string = (error->Obj.magic)["error"]["message"]
217
- message->S.assertOrThrow(S.string)
218
- parseMessageForBlockRange(message)
219
- } catch {
220
- | _ => None
221
- }
222
- | _ => None
245
+ switch exn->getErrorMessage {
246
+ | Some(message) => parseMessageForBlockRange(message)
247
+ | None => None
223
248
  }
224
249
  }
225
250
 
226
251
  type eventBatchQuery = {
227
- logs: array<Rpc.GetLogs.log>,
252
+ items: array<EvmRpcClient.rpcEventItem>,
228
253
  latestFetchedBlockInfo: blockInfo,
229
254
  }
230
255
 
231
256
  let maxSuggestedBlockIntervalKey = "max"
232
257
 
258
+ let getSourceMaxBlockInterval = (mutSuggestedBlockIntervals, ~intervalCeiling) =>
259
+ mutSuggestedBlockIntervals
260
+ ->Utils.Dict.dangerouslyGetNonOption(maxSuggestedBlockIntervalKey)
261
+ ->Option.getOr(intervalCeiling)
262
+
263
+ type logSelection = {
264
+ addresses: option<array<Address.t>>,
265
+ topicQuery: Rpc.GetLogs.topicQuery,
266
+ }
267
+
268
+ // A log can satisfy more than one selection when a single event's `where` is an
269
+ // OR of param groups, so dedup the fanned-out responses by (blockNumber,
270
+ // logIndex) — unique per chain — keeping the first occurrence.
271
+ let mergeAndDedupItems = (itemsPerSelection: array<array<EvmRpcClient.rpcEventItem>>) => {
272
+ let seen = Utils.Set.make()
273
+ let merged = []
274
+ itemsPerSelection->Array.forEach(items =>
275
+ items->Array.forEach((item: EvmRpcClient.rpcEventItem) => {
276
+ let key = `${item.log.blockNumber->Int.toString}-${item.log.logIndex->Int.toString}`
277
+ if seen->Utils.Set.has(key)->not {
278
+ seen->Utils.Set.add(key)->ignore
279
+ merged->Array.push(item)->ignore
280
+ }
281
+ })
282
+ )
283
+ merged
284
+ }
285
+
233
286
  let getNextPage = (
234
287
  ~fromBlock,
235
288
  ~toBlock,
236
- ~addresses,
237
- ~topicQuery,
289
+ ~logSelections: array<logSelection>,
238
290
  ~loadBlock,
239
291
  ~syncConfig as sc: Config.sourceSync,
240
- ~client,
292
+ ~rpcClient: EvmRpcClient.t,
241
293
  ~mutSuggestedBlockIntervals,
242
294
  ~partitionId,
243
295
  ~sourceName,
@@ -252,90 +304,109 @@ let getNextPage = (
252
304
  )
253
305
 
254
306
  let latestFetchedBlockPromise = loadBlock(toBlock)
255
- Prometheus.SourceRequestCount.increment(~sourceName, ~chainId, ~method="eth_getLogs")
256
- let logsPromise = Rpc.getLogs(
257
- ~client,
258
- ~param={
259
- address: ?addresses,
260
- topics: topicQuery,
307
+
308
+ let queryLogs = ({addresses, topicQuery}: logSelection) => {
309
+ Prometheus.SourceRequestCount.increment(~sourceName, ~chainId, ~method="eth_getLogs")
310
+ rpcClient.getLogs({
261
311
  fromBlock,
262
312
  toBlock,
263
- },
264
- )->Promise.then(async logs => {
265
- {
266
- logs,
267
- latestFetchedBlockInfo: await latestFetchedBlockPromise,
268
- }
269
- })
313
+ ?addresses,
314
+ topics: topicQuery->Array.map(filter =>
315
+ switch filter {
316
+ | Rpc.GetLogs.Null => Nullable.null
317
+ | Single(topic) => Nullable.make([topic])
318
+ | Multiple(topics) => Nullable.make(topics)
319
+ }
320
+ ),
321
+ })
322
+ }
323
+
324
+ let logsPromise = switch logSelections {
325
+ | [] =>
326
+ latestFetchedBlockPromise->Promise.thenResolve((latestFetchedBlockInfo): eventBatchQuery => {
327
+ items: [],
328
+ latestFetchedBlockInfo,
329
+ })
330
+ // Fast path: a single selection needs no cross-request merge or dedup.
331
+ | [logSelection] =>
332
+ logSelection
333
+ ->queryLogs
334
+ ->Promise.then(async items => {
335
+ {
336
+ items,
337
+ latestFetchedBlockInfo: await latestFetchedBlockPromise,
338
+ }
339
+ })
340
+ | _ =>
341
+ logSelections
342
+ ->Array.map(queryLogs)
343
+ ->Promise.all
344
+ ->Promise.then(async itemsPerSelection => {
345
+ {
346
+ items: itemsPerSelection->mergeAndDedupItems,
347
+ latestFetchedBlockInfo: await latestFetchedBlockPromise,
348
+ }
349
+ })
350
+ }
270
351
 
271
352
  [queryTimoutPromise, logsPromise]
272
353
  ->Promise.race
273
354
  ->Promise.catch(err => {
355
+ let executedBlockInterval = toBlock - fromBlock + 1
356
+ let shrunkBlockInterval =
357
+ Pervasives.max(1, (executedBlockInterval->Int.toFloat *. sc.backoffMultiplicative)->Float.toInt)
358
+
359
+ let throwFailedGettingItems = retry =>
360
+ throw(Source.GetItemsError(FailedGettingItems({exn: err, attemptedToBlock: toBlock, retry})))
361
+ let throwResize = interval =>
362
+ throwFailedGettingItems(WithSuggestedToBlock({toBlock: fromBlock + interval - 1}))
363
+
274
364
  switch getSuggestedBlockIntervalFromExn(err) {
275
- | Some((nextBlockIntervalTry, isMaxRange)) =>
276
- mutSuggestedBlockIntervals->Dict.set(
277
- isMaxRange ? maxSuggestedBlockIntervalKey : partitionId,
278
- nextBlockIntervalTry,
279
- )
280
- throw(
281
- Source.GetItemsError(
282
- FailedGettingItems({
283
- exn: err,
284
- attemptedToBlock: toBlock,
285
- retry: WithSuggestedToBlock({
286
- toBlock: fromBlock + nextBlockIntervalTry - 1,
287
- }),
288
- }),
289
- ),
365
+ // "limited to N blocks" — a structural cap on the whole source; only tighten.
366
+ | Some((interval, true)) =>
367
+ let capped = Pervasives.min(
368
+ mutSuggestedBlockIntervals->getSourceMaxBlockInterval(~intervalCeiling=sc.intervalCeiling),
369
+ interval,
290
370
  )
371
+ mutSuggestedBlockIntervals->Dict.set(maxSuggestedBlockIntervalKey, capped)
372
+ throwResize(capped)
373
+ // A one-off suggested range ("retry with range X-Y") — apply to this partition.
374
+ | Some((interval, false)) =>
375
+ mutSuggestedBlockIntervals->Dict.set(partitionId, interval)
376
+ throwResize(interval)
377
+ // Density cap with no suggested number (too many logs / response too large):
378
+ // shrink THIS partition and retry immediately (no wait); acceleration
379
+ // re-adapts on the next successful query. The interval>1 guard avoids a
380
+ // no-progress tight loop on a single over-cap block.
381
+ | None if executedBlockInterval > 1 && err->isResponseTooLargeError =>
382
+ mutSuggestedBlockIntervals->Dict.set(partitionId, shrunkBlockInterval)
383
+ throwResize(shrunkBlockInterval)
384
+ // Transient/unknown — shrink this partition and back off.
291
385
  | None =>
292
- let executedBlockInterval = toBlock - fromBlock + 1
293
- let nextBlockIntervalTry =
294
- (executedBlockInterval->Int.toFloat *. sc.backoffMultiplicative)->Float.toInt
295
- mutSuggestedBlockIntervals->Dict.set(partitionId, nextBlockIntervalTry)
296
- throw(
297
- Source.GetItemsError(
298
- Source.FailedGettingItems({
299
- exn: err,
300
- attemptedToBlock: toBlock,
301
- retry: WithBackoff({
302
- message: `Failed getting data for the block range. Will try smaller block range for the next attempt.`,
303
- backoffMillis: sc.backoffMillis,
304
- }),
305
- }),
306
- ),
386
+ mutSuggestedBlockIntervals->Dict.set(partitionId, shrunkBlockInterval)
387
+ throwFailedGettingItems(
388
+ WithBackoff({
389
+ message: `Failed getting data for the block range. Will try smaller block range for the next attempt.`,
390
+ backoffMillis: sc.backoffMillis,
391
+ }),
307
392
  )
308
393
  }
309
394
  })
310
395
  }
311
396
 
312
- type logSelection = {
313
- addresses: option<array<Address.t>>,
314
- topicQuery: Rpc.GetLogs.topicQuery,
315
- }
316
-
317
397
  type selectionConfig = {
318
- getLogSelectionOrThrow: (~addressesByContractName: dict<array<Address.t>>) => logSelection,
398
+ getLogSelectionsOrThrow: (
399
+ ~addressesByContractName: dict<array<Address.t>>,
400
+ ) => array<logSelection>,
319
401
  }
320
402
 
321
403
  let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
322
- let staticTopicSelections = []
323
- let dynamicEventFilters = []
324
-
325
- selection.eventConfigs
326
- ->(Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>)
327
- ->Array.forEach(({getEventFiltersOrThrow}) => {
328
- switch getEventFiltersOrThrow(chain) {
329
- | Static(s) => staticTopicSelections->Array.pushMany(s)->ignore
330
- | Dynamic(fn) => dynamicEventFilters->Array.push(fn)->ignore
331
- }
332
- })
404
+ let evmEventConfigs =
405
+ selection.eventConfigs->(
406
+ Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
407
+ )
333
408
 
334
- let getLogSelectionOrThrow = switch (
335
- staticTopicSelections->LogSelection.compressTopicSelections,
336
- dynamicEventFilters,
337
- ) {
338
- | ([], []) =>
409
+ if evmEventConfigs->Utils.Array.isEmpty {
339
410
  throw(
340
411
  Source.GetItemsError(
341
412
  UnsupportedSelection({
@@ -343,48 +414,110 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
343
414
  }),
344
415
  ),
345
416
  )
346
- | ([topicSelection], []) => {
347
- let topicQuery = topicSelection->Rpc.GetLogs.mapTopicQuery
348
- (~addressesByContractName) => {
349
- addresses: switch addressesByContractName->FetchState.addressesByContractNameGetAll {
350
- | [] => None
351
- | addresses => Some(addresses)
352
- },
353
- topicQuery,
417
+ }
418
+
419
+ // eth_getLogs takes one address list and one topic selection per request, so
420
+ // fan out to one request per selection. Each address-bound event is grouped by
421
+ // its contract and later scoped to that contract's own addresses — pooling all
422
+ // contracts' addresses would let one contract's query fetch a sibling's logs,
423
+ // which route back by address and bypass the sibling's filter (routing never
424
+ // re-applies it). Pure-wildcard events carry no address constraint, so they're
425
+ // pooled and resolved once.
426
+ let noAddressTopicSelections = []
427
+ let staticByContract = Dict.make()
428
+ let dynamicByContract = Dict.make()
429
+ let dynamicWildcardByContract = Dict.make()
430
+ let contractNames = Utils.Set.make()
431
+
432
+ evmEventConfigs->Array.forEach(({
433
+ contractName,
434
+ isWildcard,
435
+ dependsOnAddresses,
436
+ getEventFiltersOrThrow,
437
+ }) => {
438
+ let eventFilters = getEventFiltersOrThrow(chain)
439
+ if dependsOnAddresses {
440
+ contractNames->Utils.Set.add(contractName)->ignore
441
+ switch eventFilters {
442
+ | Internal.Static(topicSelections) =>
443
+ staticByContract->Utils.Dict.pushMany(contractName, topicSelections)
444
+ | Dynamic(fn) =>
445
+ (isWildcard ? dynamicWildcardByContract : dynamicByContract)->Utils.Dict.push(
446
+ contractName,
447
+ fn,
448
+ )
354
449
  }
450
+ } else {
451
+ noAddressTopicSelections
452
+ ->Array.pushMany(
453
+ switch eventFilters {
454
+ | Static(s) => s
455
+ | Dynamic(fn) => fn([])
456
+ },
457
+ )
458
+ ->ignore
355
459
  }
356
- | ([], [dynamicEventFilter]) if selection.eventConfigs->Array.length === 1 =>
357
- let eventConfig = selection.eventConfigs->Utils.Array.firstUnsafe
460
+ })
358
461
 
359
- (~addressesByContractName) => {
360
- let addresses = addressesByContractName->FetchState.addressesByContractNameGetAll
361
- {
362
- addresses: eventConfig.isWildcard ? None : Some(addresses),
363
- topicQuery: switch dynamicEventFilter(addresses) {
364
- | [topicSelection] => topicSelection->Rpc.GetLogs.mapTopicQuery
365
- | _ =>
366
- throw(
367
- Source.GetItemsError(
368
- UnsupportedSelection({
369
- message: "RPC data-source currently doesn't support an array of event filters. Please, create a GitHub issue if it's a blocker for you.",
370
- }),
371
- ),
372
- )
373
- },
374
- }
462
+ // `compressTopicSelections` folds the filter-less events into a single topic0
463
+ // OR-set, keeping the common case at one request.
464
+ let toLogSelections = (~addresses, topicSelections): array<logSelection> =>
465
+ topicSelections
466
+ ->LogSelection.compressTopicSelections
467
+ ->Array.map(topicSelection => {
468
+ addresses,
469
+ topicQuery: topicSelection->Rpc.GetLogs.mapTopicQuery,
470
+ })
471
+
472
+ // Address-independent, so resolve once (the wildcard partition reuses this).
473
+ let noAddressLogSelections = toLogSelections(~addresses=None, noAddressTopicSelections)
474
+
475
+ let getLogSelectionsOrThrow = if contractNames->Utils.Set.size === 0 {
476
+ (~addressesByContractName as _) => noAddressLogSelections
477
+ } else {
478
+ (~addressesByContractName): array<logSelection> => {
479
+ let logSelections = noAddressLogSelections->Array.copy
480
+ contractNames->Utils.Set.forEach(contractName => {
481
+ switch addressesByContractName->Utils.Dict.dangerouslyGetNonOption(contractName) {
482
+ | None
483
+ | Some([]) => ()
484
+ | Some(addresses) =>
485
+ // Static + dynamic non-wildcard filters, scoped to this contract's addresses.
486
+ let addressedTopicSelections = []
487
+ switch staticByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
488
+ | Some(s) => addressedTopicSelections->Array.pushMany(s)->ignore
489
+ | None => ()
490
+ }
491
+ switch dynamicByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
492
+ | Some(fns) =>
493
+ fns->Array.forEach(fn =>
494
+ addressedTopicSelections->Array.pushMany(fn(addresses))->ignore
495
+ )
496
+ | None => ()
497
+ }
498
+ logSelections
499
+ ->Array.pushMany(toLogSelections(~addresses=Some(addresses), addressedTopicSelections))
500
+ ->ignore
501
+
502
+ // Dynamic wildcard-by-address filters fold the address into the topics,
503
+ // so they still match any address.
504
+ switch dynamicWildcardByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
505
+ | Some(fns) =>
506
+ logSelections
507
+ ->Array.pushMany(
508
+ toLogSelections(~addresses=None, fns->Array.flatMap(fn => fn(addresses))),
509
+ )
510
+ ->ignore
511
+ | None => ()
512
+ }
513
+ }
514
+ })
515
+ logSelections
375
516
  }
376
- | _ =>
377
- throw(
378
- Source.GetItemsError(
379
- UnsupportedSelection({
380
- message: "RPC data-source currently supports event filters only when there's a single wildcard event. Please, create a GitHub issue if it's a blocker for you.",
381
- }),
382
- ),
383
- )
384
517
  }
385
518
 
386
519
  {
387
- getLogSelectionOrThrow: getLogSelectionOrThrow,
520
+ getLogSelectionsOrThrow: getLogSelectionsOrThrow,
388
521
  }
389
522
  }
390
523
 
@@ -855,6 +988,7 @@ type options = {
855
988
  allEventParams: array<HyperSyncClient.Decoder.eventParamsInput>,
856
989
  lowercaseAddresses: bool,
857
990
  ws?: string,
991
+ headers?: dict<string>,
858
992
  }
859
993
 
860
994
  let make = (
@@ -867,6 +1001,7 @@ let make = (
867
1001
  allEventParams,
868
1002
  lowercaseAddresses,
869
1003
  ?ws,
1004
+ ?headers,
870
1005
  }: options,
871
1006
  ): t => {
872
1007
  let chainId = chain->ChainMap.Chain.toChainId
@@ -881,10 +1016,19 @@ let make = (
881
1016
 
882
1017
  let getSelectionConfig = memoGetSelectionConfig(~chain)
883
1018
 
1019
+ // Per-partition adaptive block interval (AIMD), keyed by partitionId. The
1020
+ // `max` key holds a source-wide ceiling that only ever tightens, set by
1021
+ // structural provider limits ("limited to N blocks"). A partition's own entry
1022
+ // can go stale when partitions merge/split — acceptable, it re-adapts.
884
1023
  let mutSuggestedBlockIntervals = Dict.make()
885
1024
 
886
- let client = Rpc.makeClient(url)
887
- let rpcClient = EvmRpcClient.make(~url)
1025
+ let client = Rpc.makeClient(url, ~headers?)
1026
+ let rpcClient = EvmRpcClient.make(
1027
+ ~url,
1028
+ ~allEventParams,
1029
+ ~checksumAddresses=!lowercaseAddresses,
1030
+ ~headers?,
1031
+ )
888
1032
 
889
1033
  let makeTransactionLoader = () =>
890
1034
  LazyLoader.make(
@@ -1000,36 +1144,6 @@ let make = (
1000
1144
  ~lowercaseAddresses,
1001
1145
  )
1002
1146
 
1003
- let convertLogToHyperSyncEvent = (log: Rpc.GetLogs.log): HyperSyncClient.ResponseTypes.event => {
1004
- let hyperSyncLog: HyperSyncClient.ResponseTypes.log = {
1005
- removed: log.removed,
1006
- index: log.logIndex,
1007
- transactionIndex: log.transactionIndex,
1008
- transactionHash: log.transactionHash,
1009
- blockHash: log.blockHash,
1010
- blockNumber: log.blockNumber,
1011
- address: log.address,
1012
- data: log.data,
1013
- topics: log.topics->(Utils.magic: array<string> => array<Nullable.t<EvmTypes.Hex.t>>),
1014
- }
1015
- {log: hyperSyncLog}
1016
- }
1017
-
1018
- let hscDecoder: ref<option<HyperSyncClient.Decoder.tWithParams>> = ref(None)
1019
- let getHscDecoder = () => {
1020
- switch hscDecoder.contents {
1021
- | Some(decoder) => decoder
1022
- | None => {
1023
- let decoder = HyperSyncClient.Decoder.fromParams(
1024
- allEventParams,
1025
- ~checksumAddresses=!lowercaseAddresses,
1026
- )
1027
- hscDecoder := Some(decoder)
1028
- decoder
1029
- }
1030
- }
1031
- }
1032
-
1033
1147
  let getItemsOrThrow = async (
1034
1148
  ~fromBlock,
1035
1149
  ~toBlock,
@@ -1043,15 +1157,14 @@ let make = (
1043
1157
  ) => {
1044
1158
  let startFetchingBatchTimeRef = Performance.now()
1045
1159
 
1046
- let suggestedBlockInterval = switch mutSuggestedBlockIntervals->Utils.Dict.dangerouslyGetNonOption(
1047
- maxSuggestedBlockIntervalKey,
1048
- ) {
1049
- | Some(maxSuggestedBlockInterval) => maxSuggestedBlockInterval
1050
- | None =>
1160
+ let sourceMaxBlockInterval =
1161
+ mutSuggestedBlockIntervals->getSourceMaxBlockInterval(~intervalCeiling=syncConfig.intervalCeiling)
1162
+ let suggestedBlockInterval = Pervasives.min(
1051
1163
  mutSuggestedBlockIntervals
1052
1164
  ->Utils.Dict.dangerouslyGetNonOption(partitionId)
1053
- ->Option.getOr(syncConfig.initialBlockInterval)
1054
- }
1165
+ ->Option.getOr(syncConfig.initialBlockInterval),
1166
+ sourceMaxBlockInterval,
1167
+ )
1055
1168
 
1056
1169
  // Always have a toBlock for an RPC worker
1057
1170
  let toBlock = switch toBlock {
@@ -1070,20 +1183,19 @@ let make = (
1070
1183
  ->Promise.thenResolve(json => Some(parseBlockInfo(json)))
1071
1184
  : Promise.resolve(None)
1072
1185
 
1073
- let {getLogSelectionOrThrow} = getSelectionConfig(selection)
1074
- let {addresses, topicQuery} = getLogSelectionOrThrow(~addressesByContractName)
1186
+ let {getLogSelectionsOrThrow} = getSelectionConfig(selection)
1187
+ let logSelections = getLogSelectionsOrThrow(~addressesByContractName)
1075
1188
 
1076
- let {logs, latestFetchedBlockInfo} = await getNextPage(
1189
+ let {items, latestFetchedBlockInfo} = await getNextPage(
1077
1190
  ~fromBlock,
1078
1191
  ~toBlock=suggestedToBlock,
1079
- ~addresses,
1080
- ~topicQuery,
1192
+ ~logSelections,
1081
1193
  ~loadBlock=blockNumber =>
1082
1194
  blockLoader.contents
1083
1195
  ->LazyLoader.get(blockNumber)
1084
1196
  ->Promise.thenResolve(parseBlockInfo),
1085
1197
  ~syncConfig,
1086
- ~client,
1198
+ ~rpcClient,
1087
1199
  ~mutSuggestedBlockIntervals,
1088
1200
  ~partitionId,
1089
1201
  ~sourceName=name,
@@ -1092,49 +1204,22 @@ let make = (
1092
1204
 
1093
1205
  let executedBlockInterval = suggestedToBlock - fromBlock + 1
1094
1206
 
1095
- // Increase the suggested block interval only when it was actually applied
1096
- // and we didn't query to a hard toBlock
1097
- // We also don't care about it when we have a hard max block interval
1098
- if (
1099
- executedBlockInterval >= suggestedBlockInterval &&
1100
- !(mutSuggestedBlockIntervals->Dict.has(maxSuggestedBlockIntervalKey))
1101
- ) {
1102
- // Increase batch size going forward, but do not increase past a configured maximum
1103
- // See: https://en.wikipedia.org/wiki/Additive_increase/multiplicative_decrease
1207
+ // Grow this partition's interval only when the full suggested range was
1208
+ // actually applied (not clamped by a hard toBlock). The min clamps to the
1209
+ // source-wide ceiling, which also stops growth once a structural cap tightened it.
1210
+ // See: https://en.wikipedia.org/wiki/Additive_increase/multiplicative_decrease
1211
+ if executedBlockInterval >= suggestedBlockInterval {
1104
1212
  mutSuggestedBlockIntervals->Dict.set(
1105
1213
  partitionId,
1106
1214
  Pervasives.min(
1107
1215
  executedBlockInterval + syncConfig.accelerationAdditive,
1108
- syncConfig.intervalCeiling,
1109
- ),
1110
- )
1111
- }
1112
-
1113
- // Convert RPC logs to HyperSync events
1114
- let hyperSyncEvents = logs->Array.map(convertLogToHyperSyncEvent)
1115
-
1116
- // Decode using HyperSyncClient decoder
1117
- let parsedEvents = try await getHscDecoder().decodeLogs(hyperSyncEvents) catch {
1118
- | exn =>
1119
- throw(
1120
- Source.GetItemsError(
1121
- FailedGettingItems({
1122
- exn,
1123
- attemptedToBlock: toBlock,
1124
- retry: ImpossibleForTheQuery({
1125
- message: "Failed to parse events using hypersync client decoder. Please double-check your ABI.",
1126
- }),
1127
- }),
1216
+ sourceMaxBlockInterval,
1128
1217
  ),
1129
1218
  )
1130
1219
  }
1131
1220
 
1132
- let parsedQueueItems = await logs
1133
- ->Array.zip(parsedEvents)
1134
- ->Array.filterMap(((
1135
- log: Rpc.GetLogs.log,
1136
- maybeDecodedEvent: Nullable.t<dict<Internal.eventParams>>,
1137
- )) => {
1221
+ let parsedQueueItems = await items
1222
+ ->Array.filterMap(({log, params: maybeDecodedEvent}: EvmRpcClient.rpcEventItem) => {
1138
1223
  let topic0 = log.topics[0]->Option.getOr("0x0")
1139
1224
  let routedAddress = if lowercaseAddresses {
1140
1225
  log.address->Address.Evm.fromAddressLowercaseOrThrow
@@ -1242,7 +1327,7 @@ let make = (
1242
1327
  | Some(b) => pushBlockInfo(b)
1243
1328
  | None => ()
1244
1329
  }
1245
- logs->Array.forEach(log =>
1330
+ items->Array.forEach(({log}) =>
1246
1331
  blockHashes
1247
1332
  ->Array.push({ReorgDetection.blockNumber: log.blockNumber, blockHash: log.blockHash})
1248
1333
  ->ignore