envio 3.5.0-alpha.1 → 3.5.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/index.d.ts +10 -6
  2. package/package.json +6 -6
  3. package/src/BatchProcessing.res +13 -0
  4. package/src/BatchProcessing.res.mjs +4 -0
  5. package/src/ChainFetching.res +5 -5
  6. package/src/ChainFetching.res.mjs +3 -4
  7. package/src/ChainState.res +38 -21
  8. package/src/ChainState.res.mjs +17 -24
  9. package/src/ChainState.resi +1 -5
  10. package/src/Change.res +3 -3
  11. package/src/Config.res +0 -4
  12. package/src/Config.res.mjs +0 -10
  13. package/src/Core.res +3 -0
  14. package/src/EntityId.res +15 -0
  15. package/src/EntityId.res.mjs +9 -0
  16. package/src/EventConfigBuilder.res +1 -58
  17. package/src/EventConfigBuilder.res.mjs +1 -33
  18. package/src/FetchState.res +345 -459
  19. package/src/FetchState.res.mjs +269 -430
  20. package/src/HandlerRegister.res +45 -0
  21. package/src/HandlerRegister.res.mjs +43 -0
  22. package/src/InMemoryStore.res +11 -4
  23. package/src/InMemoryTable.res +12 -10
  24. package/src/InMemoryTable.res.mjs +6 -5
  25. package/src/IndexerState.res +59 -2
  26. package/src/IndexerState.res.mjs +48 -3
  27. package/src/IndexerState.resi +2 -0
  28. package/src/Internal.res +17 -8
  29. package/src/LogSelection.res +2 -1
  30. package/src/Metrics.res +37 -6
  31. package/src/Metrics.res.mjs +5 -1
  32. package/src/Persistence.res +1 -1
  33. package/src/PgStorage.res +55 -22
  34. package/src/PgStorage.res.mjs +25 -17
  35. package/src/SimulateItems.res +2 -2
  36. package/src/TestIndexer.res +6 -5
  37. package/src/TestIndexer.res.mjs +3 -2
  38. package/src/UserContext.res +3 -3
  39. package/src/Writing.res +12 -2
  40. package/src/Writing.res.mjs +13 -2
  41. package/src/bindings/ClickHouse.res +5 -3
  42. package/src/bindings/ClickHouse.res.mjs +2 -5
  43. package/src/bindings/Vitest.res +22 -1
  44. package/src/bindings/Vitest.res.mjs +15 -0
  45. package/src/db/EntityHistory.res +16 -7
  46. package/src/db/EntityHistory.res.mjs +7 -6
  47. package/src/db/Table.res +40 -9
  48. package/src/db/Table.res.mjs +44 -6
  49. package/src/sources/AddressSet.res +48 -0
  50. package/src/sources/AddressSet.res.mjs +11 -0
  51. package/src/sources/AddressStore.res +152 -0
  52. package/src/sources/AddressStore.res.mjs +97 -0
  53. package/src/sources/EvmChain.res +3 -0
  54. package/src/sources/EvmChain.res.mjs +4 -2
  55. package/src/sources/EvmHyperSyncSource.res +6 -3
  56. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  57. package/src/sources/EvmRpcClient.res +6 -3
  58. package/src/sources/EvmRpcClient.res.mjs +3 -3
  59. package/src/sources/FuelHyperSync.res +4 -3
  60. package/src/sources/FuelHyperSync.res.mjs +3 -3
  61. package/src/sources/FuelHyperSync.resi +2 -1
  62. package/src/sources/FuelHyperSyncClient.res +12 -6
  63. package/src/sources/FuelHyperSyncClient.res.mjs +2 -2
  64. package/src/sources/FuelHyperSyncSource.res +7 -4
  65. package/src/sources/FuelHyperSyncSource.res.mjs +3 -3
  66. package/src/sources/HyperSync.res +6 -4
  67. package/src/sources/HyperSync.res.mjs +2 -3
  68. package/src/sources/HyperSync.resi +1 -1
  69. package/src/sources/HyperSyncClient.res +18 -9
  70. package/src/sources/HyperSyncClient.res.mjs +4 -4
  71. package/src/sources/RpcSource.res +15 -10
  72. package/src/sources/RpcSource.res.mjs +3 -4
  73. package/src/sources/SimulateSource.res +26 -11
  74. package/src/sources/SimulateSource.res.mjs +13 -5
  75. package/src/sources/Source.res +4 -2
  76. package/src/sources/SourceManager.res +2 -3
  77. package/src/sources/SourceManager.res.mjs +2 -3
  78. package/src/sources/Svm.res +1 -2
  79. package/src/sources/Svm.res.mjs +1 -1
  80. package/src/sources/SvmHyperSyncClient.res +12 -3
  81. package/src/sources/SvmHyperSyncClient.res.mjs +2 -2
  82. package/src/sources/SvmHyperSyncSource.res +10 -5
  83. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  84. package/src/IndexingAddresses.res +0 -151
  85. package/src/IndexingAddresses.res.mjs +0 -130
  86. package/src/IndexingAddresses.resi +0 -39
@@ -15,9 +15,9 @@ type selection = {
15
15
  dependsOnAddresses: bool,
16
16
  // Contract names this query fetches address-free even though their
17
17
  // registrations depend on addresses. The source passes these to the client so
18
- // their log selections carry no server-side address filter and routing accepts
19
- // any emitter; the JS `clientAddressFilter` still gates each item by registered
20
- // address + effectiveStartBlock. Absent for normal partitions.
18
+ // their log selections carry no server-side address filter; the client then
19
+ // gates each item against the chain's address store instead of the
20
+ // partition's set. Absent for normal partitions.
21
21
  clientFilteredContracts?: array<string>,
22
22
  }
23
23
 
@@ -49,10 +49,13 @@ type partition = {
49
49
  // which added all its events to the queue
50
50
  latestFetchedBlock: blockNumberAndTimestamp,
51
51
  selection: selection,
52
- addressesByContractName: dict<array<Address.t>>,
52
+ // The partition's slice of the chain's address index. Ordered by
53
+ // (effectiveStartBlock, address) inside Rust, so a partition layout is a pure
54
+ // function of which addresses it holds — not of the order they arrived in.
55
+ addresses: AddressSet.t,
53
56
  mergeBlock: option<int>,
54
57
  // When set, partition indexes a single dynamic contract type.
55
- // The addressesByContractName must contain only addresses for this contract.
58
+ // `addresses` must then contain only addresses for this contract.
56
59
  dynamicContract: option<string>,
57
60
  // Mutated in place and shared across fetchState versions (updateInternal
58
61
  // copies the record, not this array): startFetchingQueries inserts,
@@ -92,26 +95,10 @@ type query = {
92
95
  // budget is reserved/consumed in and that sizes every query.
93
96
  itemsEst: int,
94
97
  selection: selection,
95
- addressesByContractName: dict<array<Address.t>>,
98
+ addresses: AddressSet.t,
96
99
  }
97
100
 
98
- // Invert addressesByContractName into address→contractName for log-ownership
99
- // routing. 1:1 today (each address belongs to one contract), so no key
100
- // collisions. Memoized on the addressesByContractName object so a partition's
101
- // many responses share one derivation and a large factory never rebuilds the
102
- // whole index; sound because the dict is immutable after construction (every
103
- // mutation produces a new dict).
104
- let deriveContractNameByAddress: dict<array<Address.t>> => dict<
105
- string,
106
- > = Utils.WeakMap.memoize(addressesByContractName => {
107
- let result = Dict.make()
108
- addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
109
- for i in 0 to addresses->Array.length - 1 {
110
- result->Dict.set(addresses->Array.getUnsafe(i)->Address.toString, contractName)
111
- }
112
- })
113
- result
114
- })
101
+ let withAddresses = (p: partition, addresses: AddressSet.t) => {...p, addresses}
115
102
 
116
103
  // itemsEst for a query over [fromBlock, toBlock] at the given event density
117
104
  // (items/block). toBlock None is the open-ended tail, capped at
@@ -200,10 +187,7 @@ module OptimizedPartitions = {
200
187
  ~maxAddrInPartition: int,
201
188
  ~nextPartitionIndexRef: ref<int>,
202
189
  ) => {
203
- let combinedAddresses =
204
- p1.addressesByContractName
205
- ->Dict.getUnsafe(contractName)
206
- ->Array.concat(p2.addressesByContractName->Dict.getUnsafe(contractName))
190
+ let combinedAddresses = p1.addresses->AddressSet.merge(p2.addresses)
207
191
 
208
192
  let p1Below = p1.latestFetchedBlock.blockNumber < potentialMergeBlock
209
193
  let p2Below = p2.latestFetchedBlock.blockNumber < potentialMergeBlock
@@ -240,7 +224,7 @@ module OptimizedPartitions = {
240
224
  selection: p1.selection,
241
225
  latestFetchedBlock: {blockNumber: potentialMergeBlock, blockTimestamp: 0},
242
226
  mergeBlock: None,
243
- addressesByContractName: Dict.make(), // set below
227
+ addresses: p1.addresses, // replaced below
244
228
  mutPendingQueries: [],
245
229
  sourceRangeCapacity: minRange,
246
230
  prevSourceRangeCapacity: minRange,
@@ -250,29 +234,28 @@ module OptimizedPartitions = {
250
234
  }
251
235
 
252
236
  // Apply address split on the continuing partition
253
- if combinedAddresses->Array.length > maxAddrInPartition {
254
- let addressesFull = combinedAddresses->Array.slice(~start=0, ~end=maxAddrInPartition)
255
- let addressesRest = combinedAddresses->Array.slice(~start=maxAddrInPartition)
256
- let abcFull = Dict.make()
257
- abcFull->Dict.set(contractName, addressesFull)
258
- let abcRest = Dict.make()
259
- abcRest->Dict.set(contractName, addressesRest)
260
- completed->Array.push({...continuingBase, addressesByContractName: abcFull})->ignore
237
+ if combinedAddresses->AddressSet.size > maxAddrInPartition {
238
+ completed
239
+ ->Array.push(
240
+ continuingBase->withAddresses(
241
+ combinedAddresses->AddressSet.slice(~offset=0, ~limit=Some(maxAddrInPartition)),
242
+ ),
243
+ )
244
+ ->ignore
261
245
  let restId = nextPartitionIndexRef.contents->Int.toString
262
246
  nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
263
247
  completed
264
248
  ->Array.push({
265
- ...continuingBase,
249
+ ...continuingBase->withAddresses(
250
+ combinedAddresses->AddressSet.slice(~offset=maxAddrInPartition, ~limit=None),
251
+ ),
266
252
  id: restId,
267
- addressesByContractName: abcRest,
268
253
  mutPendingQueries: [],
269
254
  })
270
255
  ->ignore
271
256
  completed
272
257
  } else {
273
- let abc = Dict.make()
274
- abc->Dict.set(contractName, combinedAddresses)
275
- completed->Array.push({...continuingBase, addressesByContractName: abc})->ignore
258
+ completed->Array.push(continuingBase->withAddresses(combinedAddresses))->ignore
276
259
  completed
277
260
  }
278
261
  }
@@ -320,7 +303,7 @@ module OptimizedPartitions = {
320
303
  {mergeBlock: Some(_)} =>
321
304
  newPartitions->Array.push(p)->ignore
322
305
  | {dynamicContract: Some(contractName)} =>
323
- let pAddressesCount = p.addressesByContractName->Dict.getUnsafe(contractName)->Array.length
306
+ let pAddressesCount = p.addresses->AddressSet.countFor(contractName)
324
307
  // Compute merge block: last pending query's toBlock, or lfb if idle
325
308
  let potentialMergeBlock = switch p.mutPendingQueries->Utils.Array.last {
326
309
  | Some({isChunk: true, toBlock: Some(toBlock)}) => Some(toBlock)
@@ -664,8 +647,6 @@ type t = {
664
647
  startBlock: int,
665
648
  endBlock: option<int>,
666
649
  normalSelection: selection,
667
- // By contract name
668
- contractConfigs: dict<IndexingAddresses.contractConfig>,
669
650
  // Not used for logic - only metadata
670
651
  chainId: int,
671
652
  // The block number of the latest block which was added to the queue
@@ -686,8 +667,7 @@ type t = {
686
667
  knownHeight: int,
687
668
  firstEventBlock: option<int>,
688
669
  // Per-contract registered-address count past which a dynamic contract is
689
- // switched to client-side filtering. None disables the switch
690
- // (sources that can't filter client-side, e.g. SVM/Fuel), leaving every
670
+ // switched to client-side filtering. None disables the switch, leaving every
691
671
  // contract filtered server-side.
692
672
  clientFilterAddressThreshold: option<int>,
693
673
  }
@@ -944,7 +924,6 @@ let updateInternal = (
944
924
  let updatedFetchState = {
945
925
  startBlock: fetchState.startBlock,
946
926
  endBlock: fetchState.endBlock,
947
- contractConfigs: fetchState.contractConfigs,
948
927
  normalSelection: fetchState.normalSelection,
949
928
  chainId: fetchState.chainId,
950
929
  onBlockRegistrations: fetchState.onBlockRegistrations,
@@ -965,40 +944,6 @@ let updateInternal = (
965
944
  updatedFetchState
966
945
  }
967
946
 
968
- let warnDifferentContractType = (
969
- fetchState,
970
- ~existingContract: indexingAddress,
971
- ~dc: indexingAddress,
972
- ) => {
973
- let logger = Logging.createChild(
974
- ~params={
975
- "chainId": fetchState.chainId,
976
- "contractAddress": dc.address->Address.toString,
977
- "existingContractType": existingContract.contractName,
978
- "newContractType": dc.contractName,
979
- },
980
- )
981
- logger->Logging.childWarn(`Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`)
982
- }
983
-
984
- let addressesByContractNameCount = (addressesByContractName: dict<array<Address.t>>) => {
985
- let numAddresses = ref(0)
986
- addressesByContractName->Utils.Dict.forEach(addresses => {
987
- numAddresses := numAddresses.contents + addresses->Array.length
988
- })
989
- numAddresses.contents
990
- }
991
-
992
- let addressesByContractNameGetAll = (addressesByContractName: dict<array<Address.t>>) => {
993
- let all = []
994
- addressesByContractName->Utils.Dict.forEach(addresses => {
995
- for idx in 0 to addresses->Array.length - 1 {
996
- all->Array.push(addresses->Array.getUnsafe(idx))->ignore
997
- }
998
- })
999
- all
1000
- }
1001
-
1002
947
  // Move a contract to client-side address filtering, recording why.
1003
948
  let addClientFilteredContract = (
1004
949
  clientFilteredContracts: Utils.Set.t<string>,
@@ -1020,6 +965,24 @@ let addClientFilteredContract = (
1020
965
  )
1021
966
  }
1022
967
 
968
+ // The block a partition will have fetched to once everything on its pending
969
+ // queue lands — not the block it has fetched to now. A backfill has to reach it
970
+ // rather than the frontier: a query that was routed before this batch's
971
+ // addresses reached the address store carries no events for them, yet it still
972
+ // advances the frontier over its range on arrival, and nothing else would ever
973
+ // refetch it. An in-flight open-ended query has no toBlock of its own; it can't
974
+ // return past the chain's known height, so that bounds it.
975
+ let claimedFetchedBlock = (p: partition, ~knownHeight) =>
976
+ p.mutPendingQueries->Array.reduce(p.latestFetchedBlock.blockNumber, (max, q) =>
977
+ Pervasives.max(
978
+ max,
979
+ switch q.fetchedBlock {
980
+ | Some({blockNumber}) => blockNumber
981
+ | None => q.toBlock->Option.getOr(knownHeight)
982
+ },
983
+ )
984
+ )
985
+
1023
986
  // Fold every client-filtered contract's server-side partitions into client-side
1024
987
  // fetching without tearing down established state:
1025
988
  // - The standing address-free partition (no mergeBlock) keeps its id, frontier,
@@ -1027,14 +990,14 @@ let addClientFilteredContract = (
1027
990
  // must be added does its selection change — under a fresh id, so in-flight
1028
991
  // responses built from the old selection are orphaned instead of advancing
1029
992
  // the frontier past ranges the new contract wasn't fetched for.
1030
- // - Partitions absorbed below the standing frontier (single-contract dynamic
1031
- // partitions and config partitions all of whose contracts are
1032
- // client-filtered, plus any prior backfill) become one bounded backfill
1033
- // partition covering [their min frontier, standing frontier]: getNextQuery
1034
- // caps its queries at mergeBlock and handleQueryResponse deletes it on
1035
- // arrival. The overlap it re-delivers is deduped by mergeIntoBuffer, and the
1036
- // re-fetch doubles as history for freshly registered addresses: events
1037
- // dropped before the address was registered now pass the clientAddressFilter.
993
+ // - Partitions absorbed below the standing partition's claimed block
994
+ // (single-contract dynamic partitions and config partitions all of whose
995
+ // contracts are client-filtered, plus any prior backfill) become one bounded
996
+ // backfill partition covering [their min frontier, that claimed block]:
997
+ // getNextQuery caps its queries at mergeBlock and handleQueryResponse deletes
998
+ // it on arrival. The overlap it re-delivers is deduped by mergeIntoBuffer, and
999
+ // the re-fetch doubles as history for freshly registered addresses: events
1000
+ // dropped before the address was registered now pass the address gate.
1038
1001
  // - A partition mixing client-filtered and server-side contracts stays,
1039
1002
  // stripped of the client-filtered contracts' addresses — the address-free
1040
1003
  // partition covers those logs, so fetching them server-side too would only
@@ -1050,6 +1013,8 @@ let collapseClientFilteredContracts = (
1050
1013
  ~clientFilteredContracts: Utils.Set.t<string>,
1051
1014
  ~normalSelection: selection,
1052
1015
  ~nextPartitionIndexRef: ref<int>,
1016
+ ~addressStore: AddressStore.t,
1017
+ ~knownHeight: int,
1053
1018
  ) => {
1054
1019
  if clientFilteredContracts->Utils.Set.size === 0 {
1055
1020
  partitions
@@ -1069,18 +1034,18 @@ let collapseClientFilteredContracts = (
1069
1034
  backfills->Array.push(p)->ignore
1070
1035
  | {selection: {dependsOnAddresses: false}} => absorbedPartitions->Array.push(p)->ignore
1071
1036
  | _ =>
1072
- let contractNames = p.addressesByContractName->Dict.keysToArray
1073
- let clientFilteredNames =
1074
- contractNames->Array.filter(c => clientFilteredContracts->Utils.Set.has(c))
1075
- if clientFilteredNames->Utils.Array.isEmpty {
1037
+ let contractNames = p.addresses->AddressSet.contractNames
1038
+ let serverSideNames =
1039
+ contractNames->Array.filter(c => !(clientFilteredContracts->Utils.Set.has(c)))
1040
+ if serverSideNames->Array.length === contractNames->Array.length {
1076
1041
  kept->Array.push(p)->ignore
1077
- } else if clientFilteredNames->Array.length === contractNames->Array.length {
1042
+ } else if serverSideNames->Utils.Array.isEmpty {
1078
1043
  absorbedPartitions->Array.push(p)->ignore
1079
1044
  } else {
1080
- let stripped = p.addressesByContractName->Utils.Dict.shallowCopy
1081
- clientFilteredNames->Array.forEach(c => stripped->Utils.Dict.deleteInPlace(c))
1082
1045
  strippedFrontiers->Array.push(p.latestFetchedBlock)->ignore
1083
- kept->Array.push({...p, addressesByContractName: stripped})->ignore
1046
+ kept
1047
+ ->Array.push(p->withAddresses(p.addresses->AddressSet.filterByContracts(serverSideNames)))
1048
+ ->ignore
1084
1049
  }
1085
1050
  }
1086
1051
  )
@@ -1150,7 +1115,7 @@ let collapseClientFilteredContracts = (
1150
1115
  id,
1151
1116
  latestFetchedBlock: minFrontier,
1152
1117
  selection: newSelection,
1153
- addressesByContractName: Dict.make(),
1118
+ addresses: addressStore->AddressStore.emptySet,
1154
1119
  mergeBlock: None,
1155
1120
  dynamicContract: None,
1156
1121
  mutPendingQueries: [],
@@ -1171,8 +1136,12 @@ let collapseClientFilteredContracts = (
1171
1136
  standing
1172
1137
  }
1173
1138
  kept->Array.push(standingOut)->ignore
1139
+ // Read off standingOut, not standing: a selection change orphans the
1140
+ // in-flight queries (fresh id, empty queue), so there the frontier is
1141
+ // all the partition will ever claim.
1142
+ let catchUpToBlock = standingOut->claimedFetchedBlock(~knownHeight)
1174
1143
  switch minFrontierRef.contents {
1175
- | Some(minFrontier) if minFrontier.blockNumber < standing.latestFetchedBlock.blockNumber =>
1144
+ | Some(minFrontier) if minFrontier.blockNumber < catchUpToBlock =>
1176
1145
  let id = nextPartitionIndexRef.contents->Int.toString
1177
1146
  nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
1178
1147
  kept
@@ -1180,8 +1149,8 @@ let collapseClientFilteredContracts = (
1180
1149
  id,
1181
1150
  latestFetchedBlock: minFrontier,
1182
1151
  selection: newSelection,
1183
- addressesByContractName: Dict.make(),
1184
- mergeBlock: Some(standing.latestFetchedBlock.blockNumber),
1152
+ addresses: addressStore->AddressStore.emptySet,
1153
+ mergeBlock: Some(catchUpToBlock),
1185
1154
  dynamicContract: None,
1186
1155
  mutPendingQueries: [],
1187
1156
  // Same query shape as the standing partition, so inherit its
@@ -1192,8 +1161,9 @@ let collapseClientFilteredContracts = (
1192
1161
  latestSourceRangeCapacityUpdateBlock: 0,
1193
1162
  })
1194
1163
  ->ignore
1195
- // An absorbed frontier at/above the standing frontier needs no
1196
- // backfill: the standing partition hasn't fetched past there yet.
1164
+ // An absorbed frontier at/above the claimed block needs no backfill:
1165
+ // nothing has fetched past there yet, so the standing partition still
1166
+ // covers it going forward.
1197
1167
  | _ => ()
1198
1168
  }
1199
1169
  kept
@@ -1209,8 +1179,11 @@ Phase 2: Merge non-dynamic partitions together to reduce unnecessary concurrency
1209
1179
  Returns OptimizedPartitions.t directly.
1210
1180
  (Dynamic partitions are merged by OptimizedPartitions.make automatically)
1211
1181
  */
1212
- let createPartitionsFromIndexingAddresses = (
1213
- ~registeringContractsByContract: dict<dict<indexingAddress>>,
1182
+ let createPartitions = (
1183
+ // One set per contract, holding the addresses that still need a partition:
1184
+ // a batch's fresh registrations, or the survivors of a rolled-back partition.
1185
+ ~registeringSetsByContract: dict<AddressSet.t>,
1186
+ ~addressStore: AddressStore.t,
1214
1187
  ~dynamicContracts: Utils.Set.t<string>,
1215
1188
  ~clientFilteredContracts: Utils.Set.t<string>,
1216
1189
  ~normalSelection: selection,
@@ -1218,6 +1191,7 @@ let createPartitionsFromIndexingAddresses = (
1218
1191
  ~nextPartitionIndex: int,
1219
1192
  ~existingPartitions: array<partition>,
1220
1193
  ~progressBlockNumber: int,
1194
+ ~knownHeight: int,
1221
1195
  ): // Floor for latestFetchedBlock (use progressBlockNumber from make, or 0 for registerDynamicContracts)
1222
1196
  OptimizedPartitions.t => {
1223
1197
  let nextPartitionIndexRef = ref(nextPartitionIndex)
@@ -1226,90 +1200,67 @@ OptimizedPartitions.t => {
1226
1200
  let dynamicPartitions = []
1227
1201
  let nonDynamicPartitions = []
1228
1202
 
1229
- let contractNames = registeringContractsByContract->Dict.keysToArray
1203
+ let contractNames = registeringSetsByContract->Dict.keysToArray
1230
1204
  for cIdx in 0 to contractNames->Array.length - 1 {
1231
1205
  let contractName = contractNames->Array.getUnsafe(cIdx)
1232
- let registeringContracts = registeringContractsByContract->Dict.getUnsafe(contractName)
1233
- let addresses =
1234
- registeringContracts->Dict.keysToArray->(Utils.magic: array<string> => array<Address.t>)
1206
+ let contractSet = registeringSetsByContract->Dict.getUnsafe(contractName)
1235
1207
 
1236
1208
  let isDynamic = dynamicContracts->Utils.Set.has(contractName)
1237
1209
  let partitions = isDynamic ? dynamicPartitions : nonDynamicPartitions
1238
1210
 
1239
- let byStartBlock = Dict.make()
1240
- for jdx in 0 to addresses->Array.length - 1 {
1241
- let address = addresses->Array.getUnsafe(jdx)
1242
- let indexingContract = registeringContracts->Dict.getUnsafe(address->Address.toString)
1243
- byStartBlock->Utils.Dict.push(indexingContract.effectiveStartBlock->Int.toString, address)
1244
- }
1245
-
1246
- // Will be in ASC order by JS spec
1247
- let ascKeys = byStartBlock->Dict.keysToArray
1248
- let initialKey = ascKeys->Utils.Array.firstUnsafe
1249
-
1250
- let startBlockRef = ref(initialKey->Int.fromString->Option.getUnsafe)
1251
- let addressesRef = ref(byStartBlock->Dict.getUnsafe(initialKey))
1252
-
1253
- for idx in 0 to ascKeys->Array.length - 1 {
1254
- let maybeNextStartBlockKey =
1255
- ascKeys->Array.getUnsafe(idx + 1)->(Utils.magic: string => option<string>)
1256
-
1257
- let shouldAllocateNewPartition = switch maybeNextStartBlockKey {
1258
- | None => true
1259
- | Some(nextStartBlockKey) => {
1260
- let nextStartBlock = nextStartBlockKey->Int.fromString->Option.getUnsafe
1261
- let shouldJoinCurrentStartBlock =
1262
- nextStartBlock - startBlockRef.contents < OptimizedPartitions.tooFarBlockRange
1263
-
1264
- // Addresses with different start blocks within range share a partition;
1265
- // events before each address's effectiveStartBlock are dropped on the
1266
- // client side (the event's clientAddressFilter for address-valued
1267
- // params).
1268
- if shouldJoinCurrentStartBlock {
1269
- addressesRef :=
1270
- addressesRef.contents->Array.concat(byStartBlock->Dict.getUnsafe(nextStartBlockKey))
1271
- false
1272
- } else {
1273
- true
1274
- }
1211
+ // A set is ordered by effectiveStartBlock, so its start-block groups are
1212
+ // ascending and each group's addresses are a contiguous slice.
1213
+ let groups = contractSet->AddressSet.startBlockGroups
1214
+ let offsetRef = ref(0)
1215
+ let groupIdx = ref(0)
1216
+ while groupIdx.contents < groups->Array.length {
1217
+ let startBlock = (groups->Array.getUnsafe(groupIdx.contents)).startBlock
1218
+ // Addresses with different start blocks within range share a partition;
1219
+ // events before each address's effectiveStartBlock are dropped by the
1220
+ // source's address gate.
1221
+ let countRef = ref(0)
1222
+ let nextIdx = ref(groupIdx.contents)
1223
+ let joining = ref(true)
1224
+ while joining.contents && nextIdx.contents < groups->Array.length {
1225
+ let group = groups->Array.getUnsafe(nextIdx.contents)
1226
+ if group.startBlock - startBlock < OptimizedPartitions.tooFarBlockRange {
1227
+ countRef := countRef.contents + group.count
1228
+ nextIdx := nextIdx.contents + 1
1229
+ } else {
1230
+ joining := false
1275
1231
  }
1276
1232
  }
1277
1233
 
1278
- if shouldAllocateNewPartition {
1279
- let latestFetchedBlock = {
1280
- blockNumber: Pervasives.max(startBlockRef.contents - 1, progressBlockNumber),
1281
- blockTimestamp: 0,
1282
- }
1283
- while addressesRef.contents->Array.length > 0 {
1284
- let pAddresses = addressesRef.contents->Array.slice(~start=0, ~end=maxAddrInPartition)
1285
- addressesRef.contents = addressesRef.contents->Array.slice(~start=maxAddrInPartition)
1286
-
1287
- let addressesByContractName = Dict.make()
1288
- addressesByContractName->Dict.set(contractName, pAddresses)
1289
- partitions->Array.push({
1290
- id: nextPartitionIndexRef.contents->Int.toString,
1291
- latestFetchedBlock,
1292
- selection: normalSelection,
1293
- dynamicContract: isDynamic ? Some(contractName) : None,
1294
- addressesByContractName,
1295
- mergeBlock: None,
1296
- mutPendingQueries: [],
1297
- sourceRangeCapacity: 0,
1298
- prevSourceRangeCapacity: 0,
1299
- eventDensity: None,
1300
- latestSourceRangeCapacityUpdateBlock: 0,
1301
- })
1302
- nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
1303
- }
1304
-
1305
- switch maybeNextStartBlockKey {
1306
- | None => ()
1307
- | Some(nextStartBlockKey) => {
1308
- startBlockRef := nextStartBlockKey->Int.fromString->Option.getUnsafe
1309
- addressesRef := byStartBlock->Dict.getUnsafe(nextStartBlockKey)
1310
- }
1311
- }
1234
+ let latestFetchedBlock = {
1235
+ blockNumber: Pervasives.max(startBlock - 1, progressBlockNumber),
1236
+ blockTimestamp: 0,
1237
+ }
1238
+ let remainingRef = ref(countRef.contents)
1239
+ let chunkOffsetRef = ref(offsetRef.contents)
1240
+ while remainingRef.contents > 0 {
1241
+ let take = Pervasives.min(remainingRef.contents, maxAddrInPartition)
1242
+ let pAddresses =
1243
+ contractSet->AddressSet.slice(~offset=chunkOffsetRef.contents, ~limit=Some(take))
1244
+ partitions->Array.push({
1245
+ id: nextPartitionIndexRef.contents->Int.toString,
1246
+ latestFetchedBlock,
1247
+ selection: normalSelection,
1248
+ dynamicContract: isDynamic ? Some(contractName) : None,
1249
+ addresses: pAddresses,
1250
+ mergeBlock: None,
1251
+ mutPendingQueries: [],
1252
+ sourceRangeCapacity: 0,
1253
+ prevSourceRangeCapacity: 0,
1254
+ eventDensity: None,
1255
+ latestSourceRangeCapacityUpdateBlock: 0,
1256
+ })
1257
+ nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
1258
+ chunkOffsetRef := chunkOffsetRef.contents + take
1259
+ remainingRef := remainingRef.contents - take
1312
1260
  }
1261
+
1262
+ offsetRef := offsetRef.contents + countRef.contents
1263
+ groupIdx := nextIdx.contents
1313
1264
  }
1314
1265
  }
1315
1266
 
@@ -1329,29 +1280,14 @@ OptimizedPartitions.t => {
1329
1280
  let currentPBlock = currentP.latestFetchedBlock.blockNumber
1330
1281
  let nextPBlock = nextP.latestFetchedBlock.blockNumber
1331
1282
 
1332
- // Compute total count WITHOUT mutating any arrays
1333
- let totalCount =
1334
- currentP.addressesByContractName->addressesByContractNameCount +
1335
- nextP.addressesByContractName->addressesByContractNameCount
1283
+ let totalCount = currentP.addresses->AddressSet.size + nextP.addresses->AddressSet.size
1336
1284
 
1337
1285
  if totalCount > maxAddrInPartition {
1338
1286
  // Exceeds address limit - don't merge, keep partitions separate
1339
1287
  mergedNonDynamic->Array.push(currentP)->ignore
1340
1288
  currentPRef := nextP
1341
1289
  } else {
1342
- // Build merged addresses using Array.concat (non-mutating)
1343
- let mergedAddresses = nextP.addressesByContractName->Utils.Dict.shallowCopy
1344
- let currentContractNames = currentP.addressesByContractName->Dict.keysToArray
1345
- for jdx in 0 to currentContractNames->Array.length - 1 {
1346
- let cn = currentContractNames->Array.getUnsafe(jdx)
1347
- let currentAddrs = currentP.addressesByContractName->Dict.getUnsafe(cn)
1348
- switch mergedAddresses->Utils.Dict.dangerouslyGetNonOption(cn) {
1349
- | Some(existingAddrs) =>
1350
- // Use concat (non-mutating) to avoid corrupting nextP's arrays
1351
- mergedAddresses->Dict.set(cn, existingAddrs->Array.concat(currentAddrs))
1352
- | None => mergedAddresses->Dict.set(cn, currentAddrs)
1353
- }
1354
- }
1290
+ let mergedAddresses = nextP.addresses->AddressSet.merge(currentP.addresses)
1355
1291
 
1356
1292
  let isTooFar = currentPBlock + OptimizedPartitions.tooFarBlockRange < nextPBlock
1357
1293
 
@@ -1363,16 +1299,10 @@ OptimizedPartitions.t => {
1363
1299
  mergeBlock: currentPBlock < nextPBlock ? Some(nextPBlock) : None,
1364
1300
  })
1365
1301
  ->ignore
1366
- currentPRef := {
1367
- ...nextP,
1368
- addressesByContractName: mergedAddresses,
1369
- }
1302
+ currentPRef := nextP->withAddresses(mergedAddresses)
1370
1303
  } else {
1371
1304
  // Close: push next's addresses into current
1372
- currentPRef := {
1373
- ...currentP,
1374
- addressesByContractName: mergedAddresses,
1375
- }
1305
+ currentPRef := currentP->withAddresses(mergedAddresses)
1376
1306
  }
1377
1307
  }
1378
1308
 
@@ -1393,6 +1323,8 @@ OptimizedPartitions.t => {
1393
1323
  ~clientFilteredContracts,
1394
1324
  ~normalSelection,
1395
1325
  ~nextPartitionIndexRef,
1326
+ ~addressStore,
1327
+ ~knownHeight,
1396
1328
  )
1397
1329
  OptimizedPartitions.make(
1398
1330
  ~partitions=allPartitions,
@@ -1405,7 +1337,14 @@ OptimizedPartitions.t => {
1405
1337
 
1406
1338
  let registerDynamicContracts = (
1407
1339
  fetchState: t,
1408
- ~indexingAddresses: IndexingAddresses.t,
1340
+ ~addressStore: AddressStore.t,
1341
+ // How far an in-flight open-ended query may end up claiming. These
1342
+ // registrations usually come out of a response that is applied right after
1343
+ // this call, and an unbounded query can reach past the height known when it
1344
+ // was dispatched — so the caller folds that response's own frontier and
1345
+ // height in here. Defaults to what the fetch state knows, which is all
1346
+ // there is to go on when no response is being applied.
1347
+ ~claimCeiling=fetchState.knownHeight,
1409
1348
  // These are raw items which might have dynamic contracts received from contractRegister call.
1410
1349
  // Might contain duplicates which we should filter out
1411
1350
  items: array<Internal.item>,
@@ -1417,167 +1356,130 @@ let registerDynamicContracts = (
1417
1356
  )
1418
1357
  }
1419
1358
 
1420
- let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
1421
- let earliestRegisteringEventBlockNumber = ref(%raw(`Infinity`))
1422
- // Addresses registered for contracts without matching events. These are not
1423
- // added to partitions, but they are tracked on indexingAddresses
1424
- // so that later conflicting registrations are detected, and are persisted
1425
- // to envio_addresses so they can be picked up on restart with updated config.
1426
- let noEventsAddresses: dict<indexingAddress> = Dict.make()
1427
- // Batch-level view of all addresses registered so far (across contracts,
1428
- // including no-events ones), so two contracts registering the same address
1429
- // within one batch conflict the same way as against indexingAddresses.
1430
- let registeringAddresses: dict<indexingAddress> = Dict.make()
1431
-
1359
+ // Flatten every item's dcs into one batch, remembering where each came from
1360
+ // so the rejected ones can be spliced back out of their item (which is what
1361
+ // keeps them from being persisted twice).
1362
+ let registrations: array<AddressStore.registration> = []
1363
+ let sourceDcs: array<Internal.dcs> = []
1364
+ let sourceIndexes: array<int> = []
1432
1365
  for itemIdx in 0 to items->Array.length - 1 {
1433
- let item = items->Array.getUnsafe(itemIdx)
1434
- switch item->Internal.getItemDcs {
1366
+ switch items->Array.getUnsafe(itemIdx)->Internal.getItemDcs {
1435
1367
  | None => ()
1436
1368
  | Some(dcs) =>
1437
- let idx = ref(0)
1438
- while idx.contents < dcs->Array.length {
1439
- let dc = dcs->Array.getUnsafe(idx.contents)
1440
-
1441
- let shouldRemove = ref(false)
1442
-
1443
- switch fetchState.contractConfigs->Utils.Dict.dangerouslyGetNonOption(dc.contractName) {
1444
- | Some({startBlock: contractStartBlock}) =>
1445
- let dcWithStartBlock: indexingAddress = {
1446
- address: dc.address,
1447
- contractName: dc.contractName,
1448
- registrationBlock: dc.registrationBlock,
1449
- effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
1450
- ~registrationBlock=dc.registrationBlock,
1451
- ~contractStartBlock,
1452
- ),
1453
- }
1454
- // Prevent registering already indexing contracts
1455
- switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
1456
- | Some(existingContract) =>
1457
- // FIXME: Instead of filtering out duplicates,
1458
- // we should check the block number first.
1459
- // If new registration with earlier block number
1460
- // we should register it for the missing block range
1461
- if existingContract.contractName != dc.contractName {
1462
- fetchState->warnDifferentContractType(~existingContract, ~dc=dcWithStartBlock)
1463
- } else if existingContract.effectiveStartBlock > dcWithStartBlock.effectiveStartBlock {
1464
- let logger = Logging.createChild(
1465
- ~params={
1466
- "chainId": fetchState.chainId,
1467
- "contractAddress": dc.address->Address.toString,
1468
- "existingBlockNumber": existingContract.effectiveStartBlock,
1469
- "newBlockNumber": dcWithStartBlock.effectiveStartBlock,
1470
- },
1471
- )
1472
- logger->Logging.childWarn(`Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`)
1473
- }
1474
- shouldRemove := true
1475
- | None =>
1476
- let shouldUpdate = switch registeringAddresses->Utils.Dict.dangerouslyGetNonOption(
1477
- dc.address->Address.toString,
1478
- ) {
1479
- | Some(registeringContract) if registeringContract.contractName != dc.contractName =>
1480
- fetchState->warnDifferentContractType(
1481
- ~existingContract=registeringContract,
1482
- ~dc=dcWithStartBlock,
1483
- )
1484
- false
1485
- | Some(_) => // Since the DC is registered by an earlier item in the query
1486
- // FIXME: This unsafely relies on the asc order of the items
1487
- // which is 99% true, but there were cases when the source ordering was wrong
1488
- false
1489
- | None => true
1490
- }
1491
- if shouldUpdate {
1492
- earliestRegisteringEventBlockNumber :=
1493
- Pervasives.min(
1494
- earliestRegisteringEventBlockNumber.contents,
1495
- dcWithStartBlock.effectiveStartBlock,
1496
- )
1497
- registeringContractsByContract
1498
- ->Utils.Dict.getOrInsertEmptyDict(dc.contractName)
1499
- ->Dict.set(dc.address->Address.toString, dcWithStartBlock)
1500
- registeringAddresses->Dict.set(dc.address->Address.toString, dcWithStartBlock)
1501
- } else {
1502
- shouldRemove := true
1503
- }
1504
- }
1505
- | None =>
1506
- let dcAsIndexingAddress: indexingAddress = {
1507
- address: dc.address,
1508
- contractName: dc.contractName,
1509
- registrationBlock: dc.registrationBlock,
1510
- effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
1511
- ~registrationBlock=dc.registrationBlock,
1512
- ~contractStartBlock=None,
1513
- ),
1514
- }
1515
- // Prevent duplicate logging/persistence when the same address is
1516
- // already tracked on fetchState, either from the db on startup or
1517
- // from an earlier registration in this batch.
1518
- switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
1519
- | Some(existingContract) =>
1520
- if existingContract.contractName != dc.contractName {
1521
- fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
1522
- }
1523
- shouldRemove := true
1524
- | None =>
1525
- switch registeringAddresses->Utils.Dict.dangerouslyGetNonOption(
1526
- dc.address->Address.toString,
1527
- ) {
1528
- | Some(existingContract) =>
1529
- if existingContract.contractName != dc.contractName {
1530
- fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
1531
- }
1532
- // Otherwise already queued for persistence by an earlier item in this batch.
1533
- shouldRemove := true
1534
- | None =>
1535
- let logger = Logging.createChild(
1536
- ~params={
1537
- "chainId": fetchState.chainId,
1538
- "contractAddress": dc.address->Address.toString,
1539
- "contractName": dc.contractName,
1540
- },
1541
- )
1542
- // Persist the address to the db so a future config change that
1543
- // adds events for this contract can pick it up on restart, but
1544
- // skip partition registration since there's nothing to fetch.
1545
- logger->Logging.childWarn(`Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`)
1546
- noEventsAddresses->Dict.set(dc.address->Address.toString, dcAsIndexingAddress)
1547
- registeringAddresses->Dict.set(dc.address->Address.toString, dcAsIndexingAddress)
1548
- }
1549
- }
1550
- }
1369
+ for dcIdx in 0 to dcs->Array.length - 1 {
1370
+ let dc = dcs->Array.getUnsafe(dcIdx)
1371
+ registrations
1372
+ ->Array.push({
1373
+ address: dc.address,
1374
+ contractName: dc.contractName,
1375
+ registrationBlock: dc.registrationBlock,
1376
+ })
1377
+ ->ignore
1378
+ sourceDcs->Array.push(dcs)->ignore
1379
+ sourceIndexes->Array.push(dcIdx)->ignore
1380
+ }
1381
+ }
1382
+ }
1551
1383
 
1552
- if shouldRemove.contents {
1553
- // Remove the DC from item to prevent it from saving to the db
1554
- let _ = dcs->Array.splice(~start=idx.contents, ~remove=1, ~insert=[])
1555
- // Don't increment idx - next element shifted into current position
1556
- } else {
1557
- idx := idx.contents + 1
1558
- }
1384
+ // Ids are handed out in registration order, so a cursor taken now selects
1385
+ // exactly what this batch adds.
1386
+ let idCursor = addressStore->AddressStore.nextId
1387
+ // The store resolves each address against both what it already holds and the
1388
+ // batch's own earlier entries, so two contracts claiming one address inside a
1389
+ // single batch conflict the same way as across batches.
1390
+ let verdicts = addressStore->AddressStore.registerBatch(registrations)
1391
+
1392
+ let warn = (~contractAddress, ~params, message) =>
1393
+ Logging.createChild(
1394
+ ~params={
1395
+ "chainId": fetchState.chainId,
1396
+ "contractAddress": contractAddress,
1397
+ "details": params,
1398
+ },
1399
+ )->Logging.childWarn(message)
1400
+
1401
+ let registeringContractNames = []
1402
+ let hasNoEventsUpdatesRef = ref(false)
1403
+ for idx in 0 to verdicts->Array.length - 1 {
1404
+ let registration = registrations->Array.getUnsafe(idx)
1405
+ let contractAddress = registration.address->Address.toString
1406
+ let keep = switch verdicts->Array.getUnsafe(idx) {
1407
+ | Added(_) =>
1408
+ if !(registeringContractNames->Array.includes(registration.contractName)) {
1409
+ registeringContractNames->Array.push(registration.contractName)->ignore
1559
1410
  }
1411
+ true
1412
+ | NoEvents(_) =>
1413
+ hasNoEventsUpdatesRef := true
1414
+ // Persist the address to the db so a future config change that adds
1415
+ // events for this contract can pick it up on restart, but skip partition
1416
+ // registration since there's nothing to fetch.
1417
+ warn(
1418
+ ~contractAddress,
1419
+ ~params={"contractName": registration.contractName},
1420
+ `Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`,
1421
+ )
1422
+ true
1423
+ | Conflict({existingContractName}) =>
1424
+ warn(
1425
+ ~contractAddress,
1426
+ ~params={
1427
+ "existingContractType": existingContractName,
1428
+ "newContractType": registration.contractName,
1429
+ },
1430
+ `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`,
1431
+ )
1432
+ false
1433
+ | Duplicate({effectiveStartBlock, existingEffectiveStartBlock}) =>
1434
+ // FIXME: Instead of filtering out duplicates, we should check the block
1435
+ // number first. If a new registration has an earlier block number we
1436
+ // should register it for the missing block range.
1437
+ if existingEffectiveStartBlock > effectiveStartBlock {
1438
+ warn(
1439
+ ~contractAddress,
1440
+ ~params={
1441
+ "existingBlockNumber": existingEffectiveStartBlock,
1442
+ "newBlockNumber": effectiveStartBlock,
1443
+ },
1444
+ `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`,
1445
+ )
1446
+ }
1447
+ false
1448
+ | Invalid =>
1449
+ warn(
1450
+ ~contractAddress,
1451
+ ~params={"contractName": registration.contractName},
1452
+ `Skipping contract registration: Not a valid address for this chain's ecosystem.`,
1453
+ )
1454
+ false
1455
+ }
1456
+ if !keep {
1457
+ // Mark for removal; the splice happens below, back to front, so earlier
1458
+ // indexes stay valid.
1459
+ sourceIndexes->Array.setUnsafe(idx, -1 - sourceIndexes->Array.getUnsafe(idx))
1460
+ }
1461
+ }
1462
+ for idx in verdicts->Array.length - 1 downto 0 {
1463
+ let marked = sourceIndexes->Array.getUnsafe(idx)
1464
+ if marked < 0 {
1465
+ let _ =
1466
+ sourceDcs->Array.getUnsafe(idx)->Array.splice(~start=-1 - marked, ~remove=1, ~insert=[])
1560
1467
  }
1561
1468
  }
1562
1469
 
1563
- let dcContractNamesToStore = registeringContractsByContract->Dict.keysToArray
1564
- let hasNoEventsUpdates = !(noEventsAddresses->Utils.Dict.isEmpty)
1565
- switch (dcContractNamesToStore, hasNoEventsUpdates) {
1566
- // Dont update anything when everything was filter out
1470
+ switch (registeringContractNames, hasNoEventsUpdatesRef.contents) {
1471
+ // Dont update anything when everything was filtered out
1567
1472
  | ([], false) => fetchState
1568
- | ([], true) =>
1569
- // Only dcs for contracts without events. Track them on
1570
- // indexingAddresses so subsequent registrations see them, but don't touch
1571
- // partitions since there's nothing to fetch for them.
1572
- indexingAddresses->IndexingAddresses.register(noEventsAddresses)
1573
- fetchState
1473
+ // Only dcs for contracts without events. The store already tracks them so
1474
+ // subsequent registrations see them, but there are no partitions to touch.
1475
+ | ([], true) => fetchState
1574
1476
  | (_, _) => {
1575
1477
  let newPartitions = []
1576
1478
  let dynamicContractsRef = ref(fetchState.optimizedPartitions.dynamicContracts)
1577
1479
  let mutExistingPartitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
1578
1480
 
1579
- for idx in 0 to dcContractNamesToStore->Array.length - 1 {
1580
- let contractName = dcContractNamesToStore->Array.getUnsafe(idx)
1481
+ for idx in 0 to registeringContractNames->Array.length - 1 {
1482
+ let contractName = registeringContractNames->Array.getUnsafe(idx)
1581
1483
 
1582
1484
  // When a new contract name is added as a dynamic contract for the first time (not in dynamicContracts set):
1583
1485
  // Walks through existing partitions that have addresses for this contract name
@@ -1590,12 +1492,12 @@ let registerDynamicContracts = (
1590
1492
 
1591
1493
  for idx in 0 to mutExistingPartitions->Array.length - 1 {
1592
1494
  let p = mutExistingPartitions->Array.getUnsafe(idx)
1593
- switch p.addressesByContractName->Utils.Dict.dangerouslyGetNonOption(contractName) {
1594
- | None => () // Skip partitions which don't have our contract
1595
- | Some(addresses) =>
1495
+ switch p.addresses->AddressSet.countFor(contractName) {
1496
+ | 0 => () // Skip partitions which don't have our contract
1497
+ | _ =>
1596
1498
  // Also filter out partitions which are 100% not mergable
1597
1499
  if p.selection.dependsOnAddresses && p.mergeBlock === None {
1598
- let allPartitionContractNames = p.addressesByContractName->Dict.keysToArray
1500
+ let allPartitionContractNames = p.addresses->AddressSet.contractNames
1599
1501
  switch allPartitionContractNames {
1600
1502
  | [_] =>
1601
1503
  mutExistingPartitions->Array.setUnsafe(
@@ -1618,26 +1520,20 @@ let registerDynamicContracts = (
1618
1520
  (fetchState.optimizedPartitions.nextPartitionIndex +
1619
1521
  newPartitions->Array.length)->Int.toString
1620
1522
 
1621
- let restAddressesByContractName =
1622
- p.addressesByContractName->Utils.Dict.shallowCopy
1623
- restAddressesByContractName->Utils.Dict.deleteInPlace(contractName)
1624
-
1523
+ let restNames =
1524
+ allPartitionContractNames->Array.filter(name => name !== contractName)
1625
1525
  mutExistingPartitions->Array.setUnsafe(
1626
1526
  idx,
1627
- {
1628
- ...p,
1629
- addressesByContractName: restAddressesByContractName,
1630
- },
1527
+ p->withAddresses(p.addresses->AddressSet.filterByContracts(restNames)),
1631
1528
  )
1632
1529
 
1633
- let addressesByContractName = Dict.make()
1634
- addressesByContractName->Dict.set(contractName, addresses)
1530
+ let splitAddresses = p.addresses->AddressSet.filterByContracts([contractName])
1635
1531
  newPartitions->Array.push({
1636
1532
  id: newPartitionId,
1637
1533
  latestFetchedBlock: p.latestFetchedBlock,
1638
1534
  selection: fetchState.normalSelection,
1639
1535
  dynamicContract: Some(contractName),
1640
- addressesByContractName,
1536
+ addresses: splitAddresses,
1641
1537
  mergeBlock: None,
1642
1538
  mutPendingQueries: p.mutPendingQueries,
1643
1539
  sourceRangeCapacity: p.sourceRangeCapacity,
@@ -1652,16 +1548,11 @@ let registerDynamicContracts = (
1652
1548
  }
1653
1549
  }
1654
1550
  }
1655
-
1656
- let registeringContracts = registeringContractsByContract->Dict.getUnsafe(contractName)
1657
- indexingAddresses->IndexingAddresses.register(registeringContracts)
1658
1551
  }
1659
- // Include no-events dcs so later batches detect conflicts against them.
1660
- indexingAddresses->IndexingAddresses.register(noEventsAddresses)
1661
1552
 
1662
1553
  // Switch any dynamic contract that has just crossed the server-side
1663
1554
  // address threshold to client-side filtering. Sticky: the set only grows, and
1664
- // collapse in createPartitionsFromIndexingAddresses folds the contract's
1555
+ // collapse in createPartitions folds the contract's
1665
1556
  // partitions into the single address-free partition.
1666
1557
  // Clone the sticky set before mutating so this update owns its copy and
1667
1558
  // older fetchState snapshots keep theirs (Utils.Set.add mutates in place).
@@ -1674,7 +1565,7 @@ let registerDynamicContracts = (
1674
1565
  dynamicContractsRef.contents
1675
1566
  ->Utils.Set.toArray
1676
1567
  ->Array.forEach(contractName => {
1677
- let addressCount = indexingAddresses->IndexingAddresses.contractCount(~contractName)
1568
+ let addressCount = addressStore->AddressStore.contractCount(contractName)
1678
1569
  if !(clientFilteredContracts->Utils.Set.has(contractName)) && addressCount > threshold {
1679
1570
  clientFilteredContracts->addClientFilteredContract(
1680
1571
  ~contractName,
@@ -1688,8 +1579,19 @@ let registerDynamicContracts = (
1688
1579
  | None => fetchState.optimizedPartitions.clientFilteredContracts
1689
1580
  }
1690
1581
 
1691
- let optimizedPartitions = createPartitionsFromIndexingAddresses(
1692
- ~registeringContractsByContract,
1582
+ // Only this batch's additions need partitions; everything already
1583
+ // registered has one.
1584
+ let registeringSetsByContract = Dict.make()
1585
+ registeringContractNames->Array.forEach(contractName => {
1586
+ registeringSetsByContract->Dict.set(
1587
+ contractName,
1588
+ addressStore->AddressStore.makeSet(~contractName, ~options={minId: idCursor}),
1589
+ )
1590
+ })
1591
+
1592
+ let optimizedPartitions = createPartitions(
1593
+ ~registeringSetsByContract,
1594
+ ~addressStore,
1693
1595
  ~dynamicContracts=dynamicContractsRef.contents,
1694
1596
  ~clientFilteredContracts,
1695
1597
  ~normalSelection=fetchState.normalSelection,
@@ -1698,6 +1600,7 @@ let registerDynamicContracts = (
1698
1600
  newPartitions->Array.length,
1699
1601
  ~existingPartitions=mutExistingPartitions->Array.concat(newPartitions),
1700
1602
  ~progressBlockNumber=0,
1603
+ ~knownHeight=Pervasives.max(claimCeiling, fetchState.knownHeight),
1701
1604
  )
1702
1605
 
1703
1606
  fetchState->updateInternal(~optimizedPartitions)
@@ -1705,33 +1608,6 @@ let registerDynamicContracts = (
1705
1608
  }
1706
1609
  }
1707
1610
 
1708
- // Drop events an address-param filter rejects. A merged partition may
1709
- // over-fetch a wildcard event whose indexed address param references an
1710
- // address registered after the log's block; `clientAddressFilter` is the
1711
- // param-level analogue of the srcAddress effectiveStartBlock check.
1712
- let filterByClientAddress = (
1713
- items: array<Internal.item>,
1714
- ~indexingAddresses: IndexingAddresses.t,
1715
- ): array<Internal.item> =>
1716
- items->Array.filter(item =>
1717
- switch item {
1718
- | Internal.Event({payload, blockNumber}) as item =>
1719
- let reg = (item->Internal.castUnsafeEventItem).onEventRegistration
1720
- switch reg.clientAddressFilter {
1721
- | Some(filter) =>
1722
- filter(
1723
- payload,
1724
- blockNumber,
1725
- indexingAddresses->IndexingAddresses.forContract(
1726
- ~contractName=reg.eventConfig.contractName,
1727
- ),
1728
- )
1729
- | None => true
1730
- }
1731
- | _ => true
1732
- }
1733
- )
1734
-
1735
1611
  /*
1736
1612
  Updates fetchState with a response for a given query.
1737
1613
  Returns Error if the partition with given query cannot be found (unexpected)
@@ -1831,7 +1707,7 @@ let pushDensityPricedQuery = (
1831
1707
  ~density,
1832
1708
  ~chainTargetBlock,
1833
1709
  ~selection,
1834
- ~addressesByContractName,
1710
+ ~addresses,
1835
1711
  ) => {
1836
1712
  let itemsEst = densityItemsTarget(~density, ~fromBlock, ~toBlock, ~chainTargetBlock)
1837
1713
  queries
@@ -1846,7 +1722,7 @@ let pushDensityPricedQuery = (
1846
1722
  | None => Some(itemsEst)
1847
1723
  },
1848
1724
  itemsEst,
1849
- addressesByContractName,
1725
+ addresses,
1850
1726
  })
1851
1727
  ->ignore
1852
1728
  itemsEst
@@ -1875,7 +1751,7 @@ let pushGapFillQueries = (
1875
1751
  ~partition: partition,
1876
1752
  ~partitionBudget: float,
1877
1753
  ~selection: selection,
1878
- ~addressesByContractName: dict<array<Address.t>>,
1754
+ ~addresses: AddressSet.t,
1879
1755
  ) => {
1880
1756
  // Gaps past the chain's target block wait: they regenerate from the
1881
1757
  // pending-walk each tick and fill once the target reaches them. The lagged
@@ -1899,7 +1775,7 @@ let pushGapFillQueries = (
1899
1775
  ~density,
1900
1776
  ~chainTargetBlock,
1901
1777
  ~selection,
1902
- ~addressesByContractName,
1778
+ ~addresses,
1903
1779
  )
1904
1780
  ->ignore
1905
1781
  switch (partition->getTrustedDensity, maybeChunkRange) {
@@ -1921,7 +1797,7 @@ let pushGapFillQueries = (
1921
1797
  ~density,
1922
1798
  ~chainTargetBlock,
1923
1799
  ~selection,
1924
- ~addressesByContractName,
1800
+ ~addresses,
1925
1801
  )
1926
1802
  ->ignore
1927
1803
  chunkFromBlock := chunkToBlock + 1
@@ -1994,7 +1870,7 @@ let walkPartitionPending = (
1994
1870
  ~partition=p,
1995
1871
  ~partitionBudget,
1996
1872
  ~selection=p.selection,
1997
- ~addressesByContractName=p.addressesByContractName,
1873
+ ~addresses=p.addresses,
1998
1874
  )
1999
1875
  chunksUsedThisCall := chunksUsedThisCall.contents + (candidates->Array.length - beforeLen)
2000
1876
  }
@@ -2097,7 +1973,7 @@ let pushForwardCandidates = (
2097
1973
  ~density,
2098
1974
  ~chainTargetBlock,
2099
1975
  ~selection=p.selection,
2100
- ~addressesByContractName=p.addressesByContractName,
1976
+ ~addresses=p.addresses,
2101
1977
  )
2102
1978
  generatedItems := generatedItems.contents +. itemsEst->Int.toFloat
2103
1979
  chunkFromBlock := chunkToBlock + 1
@@ -2132,7 +2008,7 @@ let pushForwardCandidates = (
2132
2008
  // keeps a server cap at its estimate to protect the shared buffer.
2133
2009
  itemsTarget: Some(itemsEst),
2134
2010
  itemsEst,
2135
- addressesByContractName: p.addressesByContractName,
2011
+ addresses: p.addresses,
2136
2012
  })
2137
2013
  ->ignore
2138
2014
  }
@@ -2456,7 +2332,7 @@ let make = (
2456
2332
  ~startBlock,
2457
2333
  ~endBlock,
2458
2334
  ~onEventRegistrations: array<Internal.onEventRegistration>,
2459
- ~contractConfigs: dict<IndexingAddresses.contractConfig>,
2335
+ ~addressStore: AddressStore.t,
2460
2336
  ~addresses: array<Internal.indexingAddress>,
2461
2337
  ~maxAddrInPartition,
2462
2338
  ~chainId,
@@ -2496,7 +2372,7 @@ let make = (
2496
2372
  dependsOnAddresses: false,
2497
2373
  onEventRegistrations: notDependingOnAddresses,
2498
2374
  },
2499
- addressesByContractName: Dict.make(),
2375
+ addresses: addressStore->AddressStore.emptySet,
2500
2376
  mergeBlock: None,
2501
2377
  dynamicContract: None,
2502
2378
  mutPendingQueries: [],
@@ -2512,9 +2388,22 @@ let make = (
2512
2388
  onEventRegistrations: normalRegistrations,
2513
2389
  }
2514
2390
 
2515
- let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
2391
+ // Every address the chain indexes goes into the store — including ones whose
2392
+ // contract has no address-dependent events, so a later registration of the
2393
+ // same address still conflicts and the address is still persisted.
2394
+ addressStore
2395
+ ->AddressStore.registerBatch(
2396
+ addresses->Array.map((contract): AddressStore.registration => {
2397
+ address: contract.address,
2398
+ contractName: contract.contractName,
2399
+ registrationBlock: contract.registrationBlock,
2400
+ }),
2401
+ )
2402
+ ->ignore
2403
+
2516
2404
  let dynamicContracts = Utils.Set.make()
2517
2405
  let clientFilteredContracts = Utils.Set.make()
2406
+ let registeringSetsByContract = Dict.make()
2518
2407
 
2519
2408
  addresses->Array.forEach(contract => {
2520
2409
  let contractName = contract.contractName
@@ -2522,12 +2411,12 @@ let make = (
2522
2411
  // Only addresses whose contract has events that depend on addresses get
2523
2412
  // registered for active fetching via partitions.
2524
2413
  if contractNamesWithNormalEvents->Utils.Set.has(contractName) {
2525
- registeringContractsByContract
2526
- ->Utils.Dict.getOrInsertEmptyDict(contractName)
2527
- ->Dict.set(
2528
- contract.address->Address.toString,
2529
- IndexingAddresses.makeIndexingAddress(~contract, ~contractConfigs),
2530
- )
2414
+ if !(registeringSetsByContract->Dict.has(contractName)) {
2415
+ registeringSetsByContract->Dict.set(
2416
+ contractName,
2417
+ addressStore->AddressStore.makeSet(~contractName),
2418
+ )
2419
+ }
2531
2420
 
2532
2421
  // Detect dynamic contracts by registrationBlock
2533
2422
  if contract.registrationBlock !== -1 {
@@ -2541,8 +2430,8 @@ let make = (
2541
2430
  // address list, or a dynamic contract restored from a large persisted set.
2542
2431
  switch clientFilterAddressThreshold {
2543
2432
  | Some(threshold) =>
2544
- registeringContractsByContract->Utils.Dict.forEachWithKey((addrs, contractName) => {
2545
- let addressCount = addrs->Utils.Dict.size
2433
+ registeringSetsByContract->Utils.Dict.forEachWithKey((set, contractName) => {
2434
+ let addressCount = set->AddressSet.size
2546
2435
  if addressCount > threshold {
2547
2436
  clientFilteredContracts->addClientFilteredContract(
2548
2437
  ~contractName,
@@ -2555,8 +2444,9 @@ let make = (
2555
2444
  | None => ()
2556
2445
  }
2557
2446
 
2558
- let optimizedPartitions = createPartitionsFromIndexingAddresses(
2559
- ~registeringContractsByContract,
2447
+ let optimizedPartitions = createPartitions(
2448
+ ~registeringSetsByContract,
2449
+ ~addressStore,
2560
2450
  ~dynamicContracts,
2561
2451
  ~clientFilteredContracts,
2562
2452
  ~normalSelection,
@@ -2564,6 +2454,7 @@ let make = (
2564
2454
  ~nextPartitionIndex=partitions->Array.length,
2565
2455
  ~existingPartitions=partitions, // wildcard partition(s) if any
2566
2456
  ~progressBlockNumber,
2457
+ ~knownHeight,
2567
2458
  )
2568
2459
 
2569
2460
  if (
@@ -2604,7 +2495,6 @@ let make = (
2604
2495
 
2605
2496
  let fetchState = {
2606
2497
  optimizedPartitions,
2607
- contractConfigs,
2608
2498
  chainId,
2609
2499
  startBlock,
2610
2500
  endBlock,
@@ -2657,16 +2547,29 @@ Always recreates optimized partitions to avoid duplicate addresses:
2657
2547
  - Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
2658
2548
  - Non-wildcard with lfb > target: delete, track addresses for recreation
2659
2549
  */
2660
- let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetBlockNumber) => {
2661
- // Step 1: Prune addresses registered after the target block. The pruned index is
2662
- // then the source of truth for partition cleanup below — an address survives iff
2663
- // it's still in the index.
2664
- indexingAddresses->IndexingAddresses.rollbackInPlace(~targetBlockNumber)
2550
+ let rollback = (fetchState: t, ~addressStore: AddressStore.t, ~targetBlockNumber) => {
2551
+ // Step 1: Prune addresses registered after the target block. The pruned store
2552
+ // is then the source of truth for partition cleanup below — an address
2553
+ // survives iff `filterByRegistrationBlock` keeps it.
2554
+ addressStore->AddressStore.rollback(targetBlockNumber)->ignore
2665
2555
 
2666
2556
  // Step 2: Categorize partitions
2667
2557
  let keptPartitions = []
2668
2558
  let nextKeptIdRef = ref(0)
2669
- let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
2559
+ let registeringSetsByContract: dict<AddressSet.t> = Dict.make()
2560
+ let collectForRecreation = (set: AddressSet.t) =>
2561
+ set
2562
+ ->AddressSet.contractNames
2563
+ ->Array.forEach(contractName => {
2564
+ let contractSet = set->AddressSet.filterByContracts([contractName])
2565
+ registeringSetsByContract->Dict.set(
2566
+ contractName,
2567
+ switch registeringSetsByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
2568
+ | Some(existing) => existing->AddressSet.merge(contractSet)
2569
+ | None => contractSet
2570
+ },
2571
+ )
2572
+ })
2670
2573
 
2671
2574
  let partitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
2672
2575
  for idx in 0 to partitions->Array.length - 1 {
@@ -2689,46 +2592,27 @@ let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetB
2689
2592
 
2690
2593
  // Non-wildcard with lfb > target: delete, collect addresses for recreation
2691
2594
  | _ if p.latestFetchedBlock.blockNumber > targetBlockNumber =>
2692
- p.addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
2693
- addresses->Array.forEach(address => {
2694
- switch indexingAddresses->IndexingAddresses.get(address->Address.toString) {
2695
- | Some(indexingContract) =>
2696
- let registeringContracts =
2697
- registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
2698
- registeringContracts->Dict.set(address->Address.toString, indexingContract)
2699
- | None => ()
2700
- }
2701
- })
2702
- })
2595
+ collectForRecreation(p.addresses->AddressSet.filterByRegistrationBlock(targetBlockNumber))
2703
2596
 
2704
2597
  // Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
2705
- | {addressesByContractName} => {
2598
+ | _ => {
2706
2599
  // Cap mergeBlock at target
2707
2600
  let mergeBlock = switch p.mergeBlock {
2708
2601
  | Some(mergeBlock) if mergeBlock > targetBlockNumber => Some(targetBlockNumber)
2709
2602
  | other => other
2710
2603
  }
2711
2604
 
2712
- // Drop addresses pruned from the index
2713
- let rollbackedAddressesByContractName = Dict.make()
2714
- addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
2715
- let keptAddresses =
2716
- addresses->Array.filter(address =>
2717
- indexingAddresses->IndexingAddresses.get(address->Address.toString)->Option.isSome
2718
- )
2719
- if keptAddresses->Array.length > 0 {
2720
- rollbackedAddressesByContractName->Dict.set(contractName, keptAddresses)
2721
- }
2722
- })
2605
+ // Drop addresses pruned from the store
2606
+ let rollbackedAddresses =
2607
+ p.addresses->AddressSet.filterByRegistrationBlock(targetBlockNumber)
2723
2608
 
2724
- if !(rollbackedAddressesByContractName->Utils.Dict.isEmpty) {
2609
+ if !(rollbackedAddresses->AddressSet.isEmpty) {
2725
2610
  let id = nextKeptIdRef.contents->Int.toString
2726
2611
  nextKeptIdRef := nextKeptIdRef.contents + 1
2727
2612
  keptPartitions
2728
2613
  ->Array.push({
2729
- ...p,
2614
+ ...p->withAddresses(rollbackedAddresses),
2730
2615
  id,
2731
- addressesByContractName: rollbackedAddressesByContractName,
2732
2616
  mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, ~targetBlockNumber),
2733
2617
  mergeBlock,
2734
2618
  })
@@ -2739,8 +2623,9 @@ let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetB
2739
2623
  }
2740
2624
 
2741
2625
  // Step 3: Recreate partitions from deleted partition addresses
2742
- let optimizedPartitions = createPartitionsFromIndexingAddresses(
2743
- ~registeringContractsByContract,
2626
+ let optimizedPartitions = createPartitions(
2627
+ ~registeringSetsByContract,
2628
+ ~addressStore,
2744
2629
  ~dynamicContracts=fetchState.optimizedPartitions.dynamicContracts,
2745
2630
  ~clientFilteredContracts=fetchState.optimizedPartitions.clientFilteredContracts,
2746
2631
  ~normalSelection=fetchState.normalSelection,
@@ -2748,6 +2633,7 @@ let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetB
2748
2633
  ~nextPartitionIndex=nextKeptIdRef.contents,
2749
2634
  ~existingPartitions=keptPartitions,
2750
2635
  ~progressBlockNumber=targetBlockNumber,
2636
+ ~knownHeight=fetchState.knownHeight,
2751
2637
  )
2752
2638
 
2753
2639
  // Step 4: Update state