envio 3.3.0-alpha.6 → 3.3.0-alpha.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/index.d.ts +23 -5
  2. package/package.json +6 -6
  3. package/src/Address.res +5 -2
  4. package/src/Address.res.mjs +3 -1
  5. package/src/Api.res +1 -1
  6. package/src/Api.res.mjs +1 -1
  7. package/src/ChainFetching.res +34 -34
  8. package/src/ChainFetching.res.mjs +29 -29
  9. package/src/ChainState.res +328 -165
  10. package/src/ChainState.res.mjs +205 -100
  11. package/src/ChainState.resi +16 -7
  12. package/src/Config.res +55 -27
  13. package/src/Config.res.mjs +46 -7
  14. package/src/ContractRegisterContext.res +2 -12
  15. package/src/ContractRegisterContext.res.mjs +3 -5
  16. package/src/Core.res +7 -0
  17. package/src/CrossChainState.res +54 -79
  18. package/src/CrossChainState.res.mjs +45 -60
  19. package/src/CrossChainState.resi +1 -1
  20. package/src/Ecosystem.res +3 -3
  21. package/src/Ecosystem.res.mjs +3 -3
  22. package/src/Envio.res +14 -9
  23. package/src/EventConfigBuilder.res +105 -56
  24. package/src/EventConfigBuilder.res.mjs +69 -33
  25. package/src/EventProcessing.res +19 -15
  26. package/src/EventProcessing.res.mjs +13 -12
  27. package/src/FetchState.res +383 -182
  28. package/src/FetchState.res.mjs +260 -264
  29. package/src/HandlerLoader.res +7 -112
  30. package/src/HandlerLoader.res.mjs +1 -87
  31. package/src/HandlerRegister.res +209 -6
  32. package/src/HandlerRegister.res.mjs +90 -5
  33. package/src/HandlerRegister.resi +13 -2
  34. package/src/IndexerState.res +6 -3
  35. package/src/IndexerState.res.mjs +3 -3
  36. package/src/IndexerState.resi +1 -1
  37. package/src/IndexingAddresses.res +10 -7
  38. package/src/IndexingAddresses.res.mjs +8 -7
  39. package/src/IndexingAddresses.resi +3 -1
  40. package/src/Internal.res +82 -36
  41. package/src/Internal.res.mjs +11 -1
  42. package/src/Main.res +22 -23
  43. package/src/Main.res.mjs +15 -17
  44. package/src/Metrics.res +43 -2
  45. package/src/Metrics.res.mjs +39 -3
  46. package/src/Prometheus.res +0 -35
  47. package/src/Prometheus.res.mjs +33 -68
  48. package/src/RawEvent.res +7 -2
  49. package/src/RawEvent.res.mjs +4 -4
  50. package/src/SimulateItems.res +26 -8
  51. package/src/SimulateItems.res.mjs +11 -14
  52. package/src/TestIndexer.res +1 -1
  53. package/src/TestIndexer.res.mjs +1 -1
  54. package/src/sources/BlockStore.res +46 -0
  55. package/src/sources/BlockStore.res.mjs +24 -0
  56. package/src/sources/EventRouter.res +19 -12
  57. package/src/sources/EventRouter.res.mjs +7 -5
  58. package/src/sources/Evm.res +54 -4
  59. package/src/sources/Evm.res.mjs +43 -4
  60. package/src/sources/EvmChain.res +14 -18
  61. package/src/sources/EvmChain.res.mjs +12 -13
  62. package/src/sources/FieldMask.res +39 -0
  63. package/src/sources/FieldMask.res.mjs +42 -0
  64. package/src/sources/Fuel.res +5 -2
  65. package/src/sources/Fuel.res.mjs +4 -3
  66. package/src/sources/HyperFuelSource.res +32 -40
  67. package/src/sources/HyperFuelSource.res.mjs +53 -45
  68. package/src/sources/HyperSync.res +35 -9
  69. package/src/sources/HyperSync.res.mjs +52 -35
  70. package/src/sources/HyperSync.resi +11 -3
  71. package/src/sources/HyperSyncClient.res +15 -4
  72. package/src/sources/HyperSyncHeightStream.res +1 -8
  73. package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
  74. package/src/sources/HyperSyncSource.res +39 -58
  75. package/src/sources/HyperSyncSource.res.mjs +44 -44
  76. package/src/sources/RpcSource.res +114 -92
  77. package/src/sources/RpcSource.res.mjs +77 -46
  78. package/src/sources/RpcWebSocketHeightStream.res +0 -5
  79. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
  80. package/src/sources/SimulateSource.res +6 -3
  81. package/src/sources/SimulateSource.res.mjs +13 -5
  82. package/src/sources/Source.res +25 -5
  83. package/src/sources/SourceManager.res +66 -3
  84. package/src/sources/SourceManager.res.mjs +55 -7
  85. package/src/sources/SourceManager.resi +10 -0
  86. package/src/sources/Svm.res +15 -10
  87. package/src/sources/Svm.res.mjs +24 -7
  88. package/src/sources/SvmHyperSyncClient.res +5 -6
  89. package/src/sources/SvmHyperSyncSource.res +84 -41
  90. package/src/sources/SvmHyperSyncSource.res.mjs +90 -43
  91. package/src/sources/TransactionStore.res +6 -107
  92. package/src/sources/TransactionStore.res.mjs +5 -86
  93. package/svm.schema.json +19 -0
package/src/Config.res CHANGED
@@ -198,6 +198,7 @@ let svmEventDescriptorSchema = S.schema(s =>
198
198
  "discriminator": s.matches(S.option(S.string)),
199
199
  "discriminatorByteLen": s.matches(S.int),
200
200
  "transactionFields": s.matches(S.array(Internal.svmTransactionFieldSchema)),
201
+ "blockFields": s.matches(S.option(S.array(Internal.svmBlockFieldSchema))),
201
202
  "includeLogs": s.matches(S.bool),
202
203
  "accountFilters": s.matches(
203
204
  S.option(
@@ -654,7 +655,6 @@ let fromPublic = (publicConfigJson: JSON.t) => {
654
655
  ~events: option<array<_>>,
655
656
  ~abi,
656
657
  ~chainId: int,
657
- ~startBlock: option<int>,
658
658
  ~addresses: array<string>,
659
659
  ~svmDefinedTypes: JSON.t=JSON.Null,
660
660
  ) => {
@@ -677,10 +677,6 @@ let fromPublic = (publicConfigJson: JSON.t) => {
677
677
  ~kind=fuelKind,
678
678
  ~sighash,
679
679
  ~rawAbi=abi->(Utils.magic: EvmTypes.Abi.t => JSON.t),
680
- ~isWildcard=false,
681
- ~handler=None,
682
- ~contractRegister=None,
683
- ~startBlock?,
684
680
  ) :> Internal.eventConfig)
685
681
  | None =>
686
682
  JsError.throwWithMessage(
@@ -706,6 +702,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
706
702
  "discriminator": option<string>,
707
703
  "discriminatorByteLen": int,
708
704
  "transactionFields": array<Internal.svmTransactionField>,
705
+ "blockFields": option<array<Internal.svmBlockField>>,
709
706
  "includeLogs": bool,
710
707
  "accountFilters": option<
711
708
  array<array<{"position": int, "values": array<string>}>>,
@@ -741,16 +738,13 @@ let fromPublic = (publicConfigJson: JSON.t) => {
741
738
  ~discriminator=svm["discriminator"],
742
739
  ~discriminatorByteLen=svm["discriminatorByteLen"],
743
740
  ~transactionFields=svm["transactionFields"],
741
+ ~blockFields=?svm["blockFields"],
744
742
  ~includeLogs=svm["includeLogs"],
745
743
  ~accountFilters,
746
744
  ~isInner=svm["isInner"],
747
- ~isWildcard=false,
748
- ~handler=None,
749
- ~contractRegister=None,
750
745
  ~accounts=svm["accounts"]->Option.getOr([]),
751
746
  ~args=svm["args"]->Option.getOr(JSON.Null),
752
747
  ~definedTypes=svmDefinedTypes,
753
- ~startBlock?,
754
748
  ) :> Internal.eventConfig)
755
749
  | _ =>
756
750
  (EventConfigBuilder.buildEvmEventConfig(
@@ -758,15 +752,8 @@ let fromPublic = (publicConfigJson: JSON.t) => {
758
752
  ~eventName,
759
753
  ~sighash,
760
754
  ~params,
761
- ~isWildcard=false,
762
- ~handler=None,
763
- ~contractRegister=None,
764
- ~eventFilters=None,
765
- ~probeChainId=chainId,
766
- ~onEventBlockFilterSchema=ecosystem.onEventBlockFilterSchema,
767
755
  ~blockFields=?eventItem["blockFields"],
768
756
  ~transactionFields=?eventItem["transactionFields"],
769
- ~startBlock?,
770
757
  ~globalBlockFieldsSet,
771
758
  ~globalTransactionFieldsSet,
772
759
  ) :> Internal.eventConfig)
@@ -818,17 +805,14 @@ let fromPublic = (publicConfigJson: JSON.t) => {
818
805
  let addresses = rawAddresses->Array.map(parseAddress)
819
806
  let startBlock = chainContract->Option.flatMap(cc => cc["startBlock"])
820
807
 
821
- // Build event configs from JSON (field selections resolved inline)
822
- // chainId is threaded in so the where-callback detection probe
823
- // exercises the callback with this chain's real id — handlers
824
- // that branch on `chain.id` are taken through the same path
825
- // they will follow at runtime.
808
+ // Build event definitions from JSON (field selections resolved
809
+ // inline). Handlers and per-chain `where` filters are layered on
810
+ // later as `onEventRegistration`s in `ChainState.makeInternal`.
826
811
  let events = buildContractEvents(
827
812
  ~contractName=capitalizedName,
828
813
  ~events=contractData["events"],
829
814
  ~abi=contractData["abi"],
830
815
  ~chainId,
831
- ~startBlock,
832
816
  ~addresses=rawAddresses,
833
817
  ~svmDefinedTypes=contractData["svmAbi"]
834
818
  ->Option.map(a => a["definedTypes"])
@@ -1018,6 +1002,50 @@ let fromPublic = (publicConfigJson: JSON.t) => {
1018
1002
  }
1019
1003
  }
1020
1004
 
1005
+ // Canonicalize a user-provided address to the configured casing so it matches
1006
+ // addresses parsed from config.yaml during routing. HyperSync/RPC data arrives
1007
+ // already canonical; only user-land input (simulate srcAddress, contractRegister
1008
+ // add) can carry arbitrary casing and needs this before comparison.
1009
+ let normalizeUserAddress = (config: t, address: Address.t): Address.t =>
1010
+ switch config.ecosystem.name {
1011
+ | Ecosystem.Evm =>
1012
+ if config.lowercaseAddresses {
1013
+ address->Address.Evm.fromAddressLowercaseOrThrow
1014
+ } else {
1015
+ address->Address.Evm.fromAddressOrThrow
1016
+ }
1017
+ // TODO: Ideally we should do the same for other ecosystems
1018
+ | Ecosystem.Fuel | Ecosystem.Svm => address
1019
+ }
1020
+
1021
+ // Relaxed counterpart to normalizeUserAddress, used only for simulate srcAddress.
1022
+ // Tests commonly use short placeholder strings like "0xfoo" as a stand-in
1023
+ // identity (e.g. for a wildcard event whose srcAddress doesn't need to resolve
1024
+ // to any real contract), so only the "0x" prefix is enforced here. Under
1025
+ // address_format: checksum, a real address is checksummed even if the input
1026
+ // casing doesn't match (getAddress doesn't require the input to already be
1027
+ // checksummed) — a placeholder that isn't valid hex falls back unchanged.
1028
+ let normalizeSimulateAddress = (config: t, address: Address.t): Address.t =>
1029
+ switch config.ecosystem.name {
1030
+ | Ecosystem.Evm =>
1031
+ let str = address->Address.toString
1032
+ if !(str->String.startsWith("0x")) {
1033
+ JsError.throwWithMessage(
1034
+ `simulate: srcAddress "${str}" is invalid. Expected a string starting with "0x".`,
1035
+ )
1036
+ }
1037
+ if config.lowercaseAddresses {
1038
+ address->Address.Evm.fromAddressLowercaseOrThrow
1039
+ } else {
1040
+ try {
1041
+ address->Address.Evm.fromAddressOrThrow
1042
+ } catch {
1043
+ | _ => address
1044
+ }
1045
+ }
1046
+ | Ecosystem.Fuel | Ecosystem.Svm => address
1047
+ }
1048
+
1021
1049
  // Look up an event config by (contract, event) name. When `chainId` is given,
1022
1050
  // returns that chain's per-chain event config (matters for where-callback
1023
1051
  // probe detection, which runs with the chain's real id). Without `chainId`,
@@ -1259,11 +1287,11 @@ let throwIfIncompatible = (
1259
1287
  }
1260
1288
  }
1261
1289
 
1262
- // The returned value is a pure function of the JSON no handler
1263
- // registrations are applied here. Post-registration configs come from
1264
- // `HandlerLoader.applyRegistrations`. That purity is what lets this
1265
- // memoize without invalidation.
1266
- let loadWithoutRegistrations = () =>
1290
+ // The returned value is a pure function of the JSON: it holds only event
1291
+ // definitions, never handler/contractRegister/where state (that's layered on
1292
+ // separately as `HandlerRegister.registrationsByChainId`). That purity is
1293
+ // what lets this memoize without invalidation.
1294
+ let load = () =>
1267
1295
  switch cached.contents {
1268
1296
  | Some(c) => c
1269
1297
  | None => {
@@ -113,6 +113,7 @@ let svmEventDescriptorSchema = S$RescriptSchema.schema(s => ({
113
113
  discriminator: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
114
114
  discriminatorByteLen: s.m(S$RescriptSchema.int),
115
115
  transactionFields: s.m(S$RescriptSchema.array(Internal.svmTransactionFieldSchema)),
116
+ blockFields: s.m(S$RescriptSchema.option(S$RescriptSchema.array(Internal.svmBlockFieldSchema))),
116
117
  includeLogs: s.m(S$RescriptSchema.bool),
117
118
  accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
118
119
  position: s.m(S$RescriptSchema.int),
@@ -490,7 +491,7 @@ function fromPublic(publicConfigJson) {
490
491
  };
491
492
  });
492
493
  }
493
- let buildContractEvents = (contractName, events, abi, chainId, startBlock, addresses, svmDefinedTypesOpt) => {
494
+ let buildContractEvents = (contractName, events, abi, chainId, addresses, svmDefinedTypesOpt) => {
494
495
  let svmDefinedTypes = svmDefinedTypesOpt !== undefined ? svmDefinedTypesOpt : null;
495
496
  if (events !== undefined) {
496
497
  return events.map(eventItem => {
@@ -500,10 +501,10 @@ function fromPublic(publicConfigJson) {
500
501
  let kind = eventItem.kind;
501
502
  switch (ecosystemName) {
502
503
  case "evm" :
503
- return EventConfigBuilder.buildEvmEventConfig(contractName, eventName, sighash, params, false, undefined, undefined, undefined, chainId, ecosystem.onEventBlockFilterSchema, eventItem.blockFields, eventItem.transactionFields, startBlock, Primitive_option.some(globalBlockFieldsSet), Primitive_option.some(globalTransactionFieldsSet));
504
+ return EventConfigBuilder.buildEvmEventConfig(contractName, eventName, sighash, params, eventItem.blockFields, eventItem.transactionFields, Primitive_option.some(globalBlockFieldsSet), Primitive_option.some(globalTransactionFieldsSet));
504
505
  case "fuel" :
505
506
  if (kind !== undefined) {
506
- return EventConfigBuilder.buildFuelEventConfig(contractName, eventName, kind, sighash, abi, false, undefined, undefined, startBlock);
507
+ return EventConfigBuilder.buildFuelEventConfig(contractName, eventName, kind, sighash, abi);
507
508
  } else {
508
509
  return Stdlib_JsError.throwWithMessage(`Fuel event ` + contractName + `.` + eventName + ` is missing "kind" in internal config`);
509
510
  }
@@ -518,7 +519,7 @@ function fromPublic(publicConfigJson) {
518
519
  position: af.position,
519
520
  values: af.values
520
521
  })));
521
- return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeLogs, svm.transactionFields, accountFilters, svm.isInner, false, undefined, undefined, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes, startBlock);
522
+ return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeLogs, svm.transactionFields, svm.blockFields, accountFilters, svm.isInner, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes);
522
523
  }
523
524
  });
524
525
  } else {
@@ -559,7 +560,7 @@ function fromPublic(publicConfigJson) {
559
560
  let rawAddresses = Stdlib_Option.getOr(Stdlib_Option.flatMap(chainContract, cc => cc.addresses), []);
560
561
  let addresses = rawAddresses.map(parseAddress);
561
562
  let startBlock = Stdlib_Option.flatMap(chainContract, cc => cc.startBlock);
562
- let events = buildContractEvents(capitalizedName, contractData.events, contractData.abi, chainId, startBlock, rawAddresses, Stdlib_Option.getOr(Stdlib_Option.map(contractData.svmAbi, a => a.definedTypes), null));
563
+ let events = buildContractEvents(capitalizedName, contractData.events, contractData.abi, chainId, rawAddresses, Stdlib_Option.getOr(Stdlib_Option.map(contractData.svmAbi, a => a.definedTypes), null));
563
564
  return {
564
565
  name: capitalizedName,
565
566
  abi: contractData.abi,
@@ -706,6 +707,42 @@ function fromPublic(publicConfigJson) {
706
707
  };
707
708
  }
708
709
 
710
+ function normalizeUserAddress(config, address) {
711
+ let match = config.ecosystem.name;
712
+ switch (match) {
713
+ case "evm" :
714
+ if (config.lowercaseAddresses) {
715
+ return Address.Evm.fromAddressLowercaseOrThrow(address);
716
+ } else {
717
+ return Address.Evm.fromAddressOrThrow(address);
718
+ }
719
+ case "fuel" :
720
+ case "svm" :
721
+ return address;
722
+ }
723
+ }
724
+
725
+ function normalizeSimulateAddress(config, address) {
726
+ let match = config.ecosystem.name;
727
+ switch (match) {
728
+ case "evm" :
729
+ if (!address.startsWith("0x")) {
730
+ Stdlib_JsError.throwWithMessage(`simulate: srcAddress "` + address + `" is invalid. Expected a string starting with "0x".`);
731
+ }
732
+ if (config.lowercaseAddresses) {
733
+ return Address.Evm.fromAddressLowercaseOrThrow(address);
734
+ }
735
+ try {
736
+ return Address.Evm.fromAddressOrThrow(address);
737
+ } catch (exn) {
738
+ return address;
739
+ }
740
+ case "fuel" :
741
+ case "svm" :
742
+ return address;
743
+ }
744
+ }
745
+
709
746
  function getEventConfig(config, contractName, eventName, chainId) {
710
747
  let chains;
711
748
  if (chainId !== undefined) {
@@ -975,7 +1012,7 @@ function throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhou
975
1012
  Stdlib_JsError.throwWithMessage(`The following config changes are incompatible with the existing indexer data:\n\n` + bullets + `\n\nPick one:\n 1. ` + padTo(option1, col) + `# resume indexing where it left off\n 2. ` + padTo(resetCommand, col) + `# delete all indexed data and start over` + option3);
976
1013
  }
977
1014
 
978
- function loadWithoutRegistrations() {
1015
+ function load() {
979
1016
  let c = cached.contents;
980
1017
  if (c !== undefined) {
981
1018
  return c;
@@ -1012,6 +1049,8 @@ export {
1012
1049
  publicConfigStorageSchema,
1013
1050
  publicConfigSchema,
1014
1051
  fromPublic,
1052
+ normalizeUserAddress,
1053
+ normalizeSimulateAddress,
1015
1054
  getEventConfig,
1016
1055
  shouldSaveHistory,
1017
1056
  shouldPruneHistory,
@@ -1022,7 +1061,7 @@ export {
1022
1061
  canonicalJson,
1023
1062
  diffPaths,
1024
1063
  throwIfIncompatible,
1025
- loadWithoutRegistrations,
1064
+ load,
1026
1065
  getPgUserEntities,
1027
1066
  }
1028
1067
  /* schema Not a pure module */
@@ -22,18 +22,8 @@ let makeAddFunction = (~params: contractRegisterParams, ~contractName: string):
22
22
  ~logger=Ecosystem.getItemLogger(params.item, ~ecosystem=params.config.ecosystem),
23
23
  )
24
24
  }
25
- let validatedAddress = if params.config.ecosystem.name === Evm {
26
- // The value is passed from the user-land,
27
- // so we need to validate and checksum/lowercase the address.
28
- if params.config.lowercaseAddresses {
29
- contractAddress->Address.Evm.fromAddressLowercaseOrThrow
30
- } else {
31
- contractAddress->Address.Evm.fromAddressOrThrow
32
- }
33
- } else {
34
- // TODO: Ideally we should do the same for other ecosystems
35
- contractAddress
36
- }
25
+ // The value is passed from user-land, so validate and checksum/lowercase it.
26
+ let validatedAddress = params.config->Config.normalizeUserAddress(contractAddress)
37
27
 
38
28
  params.onRegister(~item=params.item, ~contractAddress=validatedAddress, ~contractName)
39
29
  }
@@ -1,6 +1,6 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
- import * as Address from "./Address.res.mjs";
3
+ import * as Config from "./Config.res.mjs";
4
4
  import * as ChainMap from "./ChainMap.res.mjs";
5
5
  import * as Ecosystem from "./Ecosystem.res.mjs";
6
6
  import * as EntityFilter from "./db/EntityFilter.res.mjs";
@@ -12,9 +12,7 @@ function makeAddFunction(params, contractName) {
12
12
  if (params.isResolved) {
13
13
  ErrorHandling.mkLogAndRaise(Ecosystem.getItemLogger(params.item, params.config.ecosystem), undefined, new Error(`Impossible to access context.chain after the contract register is resolved. Make sure you didn't miss an await in the handler.`));
14
14
  }
15
- let validatedAddress = params.config.ecosystem.name === "evm" ? (
16
- params.config.lowercaseAddresses ? Address.Evm.fromAddressLowercaseOrThrow(contractAddress) : Address.Evm.fromAddressOrThrow(contractAddress)
17
- ) : contractAddress;
15
+ let validatedAddress = Config.normalizeUserAddress(params.config, contractAddress);
18
16
  params.onRegister(params.item, validatedAddress, contractName);
19
17
  };
20
18
  }
@@ -73,4 +71,4 @@ export {
73
71
  getContractRegisterContext,
74
72
  getContractRegisterArgs,
75
73
  }
76
- /* Address Not a pure module */
74
+ /* Config Not a pure module */
package/src/Core.res CHANGED
@@ -9,6 +9,7 @@ type evmRpcClientCtor
9
9
  type svmHypersyncClientCtor
10
10
  type hyperfuelClientCtor
11
11
  type transactionStoreCtor
12
+ type blockStoreCtor
12
13
 
13
14
  type addon = {
14
15
  getConfigJson: (~configPath: Null.t<string>, ~directory: Null.t<string>) => string,
@@ -23,10 +24,16 @@ type addon = {
23
24
  hyperfuelClient: hyperfuelClientCtor,
24
25
  @as("TransactionStore")
25
26
  transactionStore: transactionStoreCtor,
27
+ @as("BlockStore")
28
+ blockStore: blockStoreCtor,
26
29
  // Ordered transaction-field names exposed for the field-code contract test
27
30
  // (the ReScript `transactionFields` arrays must match the Rust ordinals).
28
31
  evmTransactionFieldNames: unit => array<string>,
29
32
  svmTransactionFieldNames: unit => array<string>,
33
+ // Ordered block-field names for the same contract test (`blockFields` arrays
34
+ // must match the Rust ordinals).
35
+ evmBlockFieldNames: unit => array<string>,
36
+ svmBlockFieldNames: unit => array<string>,
30
37
  }
31
38
 
32
39
  @module("node:module") external createRequire: string => {..} = "createRequire"
@@ -19,7 +19,7 @@ type t = {
19
19
  let calculateTargetBufferSize = () =>
20
20
  switch Env.targetBufferSize {
21
21
  | Some(size) => size
22
- | None => 100_000
22
+ | None => 50_000
23
23
  }
24
24
 
25
25
  let make = (
@@ -135,13 +135,13 @@ let enterReorgThreshold = (crossChainState: t) => {
135
135
  // whole indexer. A chain is marked caught up only once EVERY chain is caught up
136
136
  // (reached endblock or fetched/processed to head) with no processable events
137
137
  // left — so no chain flips to ready while another is still backfilling.
138
- let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
138
+ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
139
139
  let chainIds = crossChainState.chainIds
140
140
 
141
141
  let everyChainCaughtUp = ref(true)
142
142
  for i in 0 to chainIds->Array.length - 1 {
143
143
  let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
144
- cs->ChainState.applyBatchProgress(~batch)
144
+ cs->ChainState.applyBatchProgress(~batch, ~blockTimestampName)
145
145
  if !(cs->ChainState.hasProcessedToEndblock || cs->ChainState.isProgressAtHead) {
146
146
  everyChainCaughtUp := false
147
147
  }
@@ -190,96 +190,71 @@ let totalReservedSize = (crossChainState: t) => {
190
190
  total.contents
191
191
  }
192
192
 
193
- // Furthest-behind first: order candidate queries by the chain progress % at
194
- // their fromBlock, so the most behind ranges across all chains are admitted
195
- // before the rest.
196
- let compareByProgress = (a: FetchState.query, b: FetchState.query) =>
197
- Float.compare(a.progress, b.progress)
198
-
199
193
  // Dispatch a fetch tick across the whole indexer from one shared pool of
200
- // ~targetBufferSize ready events. Every chain proposes its candidate queries
201
- // (each carrying an estimated response size) against the full free budget; the
202
- // candidates are then pooled, ordered by chain progress (furthest-behind first),
203
- // and admitted until the budget is consumed. So the budget is split per query
204
- // across chains rather than per chain a chain that can only use a little
205
- // leaves the rest for the others automatically.
194
+ // ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
195
+ // first, hand each the budget remaining at that point (plus its own
196
+ // already-reserved share, since a chain's pending queries aren't "spent" —
197
+ // they're this chain's), let it turn that into queries sized against its own
198
+ // chain-density-derived target block, then subtract what it actually used
199
+ // before moving to the next chain. So a chain that can only use a little
200
+ // (density too low, or already caught up) leaves the rest for the others
201
+ // automatically — unlike a per-query pool, this can starve a near-head chain
202
+ // while an earlier one is deep in backfill, which is the intended tradeoff.
206
203
  let checkAndFetch = async (
207
204
  crossChainState: t,
208
205
  ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
209
206
  ) => {
210
- let remaining = Pervasives.max(
211
- 0,
212
- crossChainState.targetBufferSize -
213
- crossChainState->totalReadyCount -
214
- crossChainState->totalReservedSize->Float.toInt,
207
+ let remaining = ref(
208
+ Pervasives.max(
209
+ 0.,
210
+ crossChainState.targetBufferSize->Int.toFloat -.
211
+ crossChainState->totalReadyCount->Int.toFloat -.
212
+ crossChainState->totalReservedSize,
213
+ ),
215
214
  )
216
215
 
217
- let chainIds = crossChainState.chainIds
218
216
  let actionByChain = Dict.make()
219
- // Candidate queries from every chain. Each query carries its chain id and the
220
- // chain progress % at its fromBlock (the admission sort key), set here so the
221
- // pool can be ordered without a side tuple per query.
222
- let candidates = []
223
- for i in 0 to chainIds->Array.length - 1 {
224
- let chainId = chainIds->Array.getUnsafe(i)
225
- let cs = crossChainState->getChainState(chainId)
226
- switch cs->ChainState.getNextQuery(~budget=remaining) {
217
+ crossChainState
218
+ ->priorityOrder
219
+ ->Array.forEach(cs => {
220
+ let chainId = (cs->ChainState.chainConfig).id
221
+ let chainTargetItems = remaining.contents +. cs->ChainState.pendingBudget
222
+ switch cs->ChainState.getNextQuery(~chainTargetItems) {
227
223
  | (WaitingForNewBlock | NothingToQuery) as action =>
228
224
  actionByChain->Utils.Dict.setByInt(chainId, action)
229
- | Ready(queries) =>
230
- // Default to NothingToQuery; replaced below if any candidate is admitted.
231
- actionByChain->Utils.Dict.setByInt(chainId, FetchState.NothingToQuery)
232
- queries->Array.forEach(query => {
233
- query.chainId = chainId
234
- query.progress = cs->ChainState.getProgressPercentageAt(~blockNumber=query.fromBlock)
235
- candidates->Array.push(query)
236
- })
225
+ | Ready(queries) => {
226
+ let consumed =
227
+ queries->Array.reduce(0., (acc, query: FetchState.query) => acc +. query.itemsTarget)
228
+
229
+ let partitions = Dict.make()
230
+ queries->Array.forEach((query: FetchState.query) =>
231
+ partitions->Dict.set(
232
+ query.partitionId,
233
+ {
234
+ "fromBlock": query.fromBlock,
235
+ "targetBlock": query.toBlock,
236
+ "targetEvents": query.itemsTarget->Math.round->Float.toInt,
237
+ },
238
+ )
239
+ )
240
+ Logging.trace({
241
+ "msg": "Started querying",
242
+ "chainId": chainId,
243
+ "partitions": partitions,
244
+ })
245
+
246
+ actionByChain->Utils.Dict.setByInt(chainId, FetchState.Ready(queries))
247
+ // Mark the queries in flight and reserve their size against the shared
248
+ // budget; released as each response lands in handleQueryResult.
249
+ cs->ChainState.startFetchingQueries(~queries)
250
+ remaining := Pervasives.max(0., remaining.contents -. consumed)
251
+ }
237
252
  }
238
- }
239
-
240
- candidates->Array.sort(compareByProgress)
241
-
242
- // Admit furthest-behind first until the budget runs out. The condition is
243
- // checked before each query, so as long as there's any budget left we admit
244
- // the next one even when its estimate alone exceeds the remainder — otherwise a
245
- // chain whose only query is bigger than the budget would never make progress.
246
- let admittedByChain = Dict.make()
247
- let running = ref(0.)
248
- let remainingF = remaining->Int.toFloat
249
- let idx = ref(0)
250
- while running.contents < remainingF && idx.contents < candidates->Array.length {
251
- let query = candidates->Array.getUnsafe(idx.contents)
252
- admittedByChain->Utils.Dict.push(query.chainId->Int.toString, query)
253
- running := running.contents +. query.estResponseSize
254
- idx := idx.contents + 1
255
- }
256
- admittedByChain->Dict.forEachWithKey((queries, chainId) => {
257
- let partitions = Dict.make()
258
- queries->Array.forEach((query: FetchState.query) =>
259
- partitions->Dict.set(
260
- query.partitionId,
261
- {
262
- "fromBlock": query.fromBlock,
263
- "targetBlock": query.toBlock,
264
- },
265
- )
266
- )
267
- Logging.trace({
268
- "msg": "Started querying",
269
- "chainId": chainId->Int.fromString->Option.getUnsafe,
270
- "partitions": partitions,
271
- })
272
- actionByChain->Dict.set(chainId, FetchState.Ready(queries))
273
- // Mark the admitted queries in flight and reserve their size against the
274
- // shared budget; released as each response lands in handleQueryResult.
275
- crossChainState
276
- ->getChainState(chainId->Int.fromString->Option.getUnsafe)
277
- ->ChainState.startFetchingQueries(~queries)
278
253
  })
279
254
 
280
255
  let promises = []
281
- for i in 0 to chainIds->Array.length - 1 {
282
- let chainId = chainIds->Array.getUnsafe(i)
256
+ for i in 0 to crossChainState.chainIds->Array.length - 1 {
257
+ let chainId = crossChainState.chainIds->Array.getUnsafe(i)
283
258
  switch actionByChain->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
284
259
  | Some(NothingToQuery)
285
260
  | None => ()