envio 3.5.0-alpha.1 → 3.5.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +10 -6
- package/package.json +6 -6
- package/src/BatchProcessing.res +13 -0
- package/src/BatchProcessing.res.mjs +4 -0
- package/src/ChainFetching.res +5 -5
- package/src/ChainFetching.res.mjs +3 -4
- package/src/ChainState.res +38 -21
- package/src/ChainState.res.mjs +17 -24
- package/src/ChainState.resi +1 -5
- package/src/Change.res +3 -3
- package/src/Config.res +0 -4
- package/src/Config.res.mjs +0 -10
- package/src/Core.res +3 -0
- package/src/EntityId.res +15 -0
- package/src/EntityId.res.mjs +9 -0
- package/src/EventConfigBuilder.res +1 -58
- package/src/EventConfigBuilder.res.mjs +1 -33
- package/src/FetchState.res +345 -459
- package/src/FetchState.res.mjs +269 -430
- package/src/HandlerRegister.res +45 -0
- package/src/HandlerRegister.res.mjs +43 -0
- package/src/InMemoryStore.res +11 -4
- package/src/InMemoryTable.res +12 -10
- package/src/InMemoryTable.res.mjs +6 -5
- package/src/IndexerState.res +59 -2
- package/src/IndexerState.res.mjs +48 -3
- package/src/IndexerState.resi +2 -0
- package/src/Internal.res +17 -8
- package/src/LogSelection.res +2 -1
- package/src/Metrics.res +37 -6
- package/src/Metrics.res.mjs +5 -1
- package/src/Persistence.res +1 -1
- package/src/PgStorage.res +55 -22
- package/src/PgStorage.res.mjs +25 -17
- package/src/SimulateItems.res +2 -2
- package/src/TestIndexer.res +6 -5
- package/src/TestIndexer.res.mjs +3 -2
- package/src/UserContext.res +3 -3
- package/src/Writing.res +12 -2
- package/src/Writing.res.mjs +13 -2
- package/src/bindings/ClickHouse.res +5 -3
- package/src/bindings/ClickHouse.res.mjs +2 -5
- package/src/bindings/Vitest.res +22 -1
- package/src/bindings/Vitest.res.mjs +15 -0
- package/src/db/EntityHistory.res +16 -7
- package/src/db/EntityHistory.res.mjs +7 -6
- package/src/db/Table.res +40 -9
- package/src/db/Table.res.mjs +44 -6
- package/src/sources/AddressSet.res +48 -0
- package/src/sources/AddressSet.res.mjs +11 -0
- package/src/sources/AddressStore.res +152 -0
- package/src/sources/AddressStore.res.mjs +97 -0
- package/src/sources/EvmChain.res +3 -0
- package/src/sources/EvmChain.res.mjs +4 -2
- package/src/sources/EvmHyperSyncSource.res +6 -3
- package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
- package/src/sources/EvmRpcClient.res +6 -3
- package/src/sources/EvmRpcClient.res.mjs +3 -3
- package/src/sources/FuelHyperSync.res +4 -3
- package/src/sources/FuelHyperSync.res.mjs +3 -3
- package/src/sources/FuelHyperSync.resi +2 -1
- package/src/sources/FuelHyperSyncClient.res +12 -6
- package/src/sources/FuelHyperSyncClient.res.mjs +2 -2
- package/src/sources/FuelHyperSyncSource.res +7 -4
- package/src/sources/FuelHyperSyncSource.res.mjs +3 -3
- package/src/sources/HyperSync.res +6 -4
- package/src/sources/HyperSync.res.mjs +2 -3
- package/src/sources/HyperSync.resi +1 -1
- package/src/sources/HyperSyncClient.res +18 -9
- package/src/sources/HyperSyncClient.res.mjs +4 -4
- package/src/sources/RpcSource.res +15 -10
- package/src/sources/RpcSource.res.mjs +3 -4
- package/src/sources/SimulateSource.res +26 -11
- package/src/sources/SimulateSource.res.mjs +13 -5
- package/src/sources/Source.res +4 -2
- package/src/sources/SourceManager.res +2 -3
- package/src/sources/SourceManager.res.mjs +2 -3
- package/src/sources/Svm.res +1 -2
- package/src/sources/Svm.res.mjs +1 -1
- package/src/sources/SvmHyperSyncClient.res +12 -3
- package/src/sources/SvmHyperSyncClient.res.mjs +2 -2
- package/src/sources/SvmHyperSyncSource.res +10 -5
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
- package/src/IndexingAddresses.res +0 -151
- package/src/IndexingAddresses.res.mjs +0 -130
- package/src/IndexingAddresses.resi +0 -39
package/src/FetchState.res
CHANGED
|
@@ -15,9 +15,9 @@ type selection = {
|
|
|
15
15
|
dependsOnAddresses: bool,
|
|
16
16
|
// Contract names this query fetches address-free even though their
|
|
17
17
|
// registrations depend on addresses. The source passes these to the client so
|
|
18
|
-
// their log selections carry no server-side address filter
|
|
19
|
-
//
|
|
20
|
-
//
|
|
18
|
+
// their log selections carry no server-side address filter; the client then
|
|
19
|
+
// gates each item against the chain's address store instead of the
|
|
20
|
+
// partition's set. Absent for normal partitions.
|
|
21
21
|
clientFilteredContracts?: array<string>,
|
|
22
22
|
}
|
|
23
23
|
|
|
@@ -49,10 +49,13 @@ type partition = {
|
|
|
49
49
|
// which added all its events to the queue
|
|
50
50
|
latestFetchedBlock: blockNumberAndTimestamp,
|
|
51
51
|
selection: selection,
|
|
52
|
-
|
|
52
|
+
// The partition's slice of the chain's address index. Ordered by
|
|
53
|
+
// (effectiveStartBlock, address) inside Rust, so a partition layout is a pure
|
|
54
|
+
// function of which addresses it holds — not of the order they arrived in.
|
|
55
|
+
addresses: AddressSet.t,
|
|
53
56
|
mergeBlock: option<int>,
|
|
54
57
|
// When set, partition indexes a single dynamic contract type.
|
|
55
|
-
//
|
|
58
|
+
// `addresses` must then contain only addresses for this contract.
|
|
56
59
|
dynamicContract: option<string>,
|
|
57
60
|
// Mutated in place and shared across fetchState versions (updateInternal
|
|
58
61
|
// copies the record, not this array): startFetchingQueries inserts,
|
|
@@ -92,26 +95,10 @@ type query = {
|
|
|
92
95
|
// budget is reserved/consumed in and that sizes every query.
|
|
93
96
|
itemsEst: int,
|
|
94
97
|
selection: selection,
|
|
95
|
-
|
|
98
|
+
addresses: AddressSet.t,
|
|
96
99
|
}
|
|
97
100
|
|
|
98
|
-
|
|
99
|
-
// routing. 1:1 today (each address belongs to one contract), so no key
|
|
100
|
-
// collisions. Memoized on the addressesByContractName object so a partition's
|
|
101
|
-
// many responses share one derivation and a large factory never rebuilds the
|
|
102
|
-
// whole index; sound because the dict is immutable after construction (every
|
|
103
|
-
// mutation produces a new dict).
|
|
104
|
-
let deriveContractNameByAddress: dict<array<Address.t>> => dict<
|
|
105
|
-
string,
|
|
106
|
-
> = Utils.WeakMap.memoize(addressesByContractName => {
|
|
107
|
-
let result = Dict.make()
|
|
108
|
-
addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
|
|
109
|
-
for i in 0 to addresses->Array.length - 1 {
|
|
110
|
-
result->Dict.set(addresses->Array.getUnsafe(i)->Address.toString, contractName)
|
|
111
|
-
}
|
|
112
|
-
})
|
|
113
|
-
result
|
|
114
|
-
})
|
|
101
|
+
let withAddresses = (p: partition, addresses: AddressSet.t) => {...p, addresses}
|
|
115
102
|
|
|
116
103
|
// itemsEst for a query over [fromBlock, toBlock] at the given event density
|
|
117
104
|
// (items/block). toBlock None is the open-ended tail, capped at
|
|
@@ -200,10 +187,7 @@ module OptimizedPartitions = {
|
|
|
200
187
|
~maxAddrInPartition: int,
|
|
201
188
|
~nextPartitionIndexRef: ref<int>,
|
|
202
189
|
) => {
|
|
203
|
-
let combinedAddresses =
|
|
204
|
-
p1.addressesByContractName
|
|
205
|
-
->Dict.getUnsafe(contractName)
|
|
206
|
-
->Array.concat(p2.addressesByContractName->Dict.getUnsafe(contractName))
|
|
190
|
+
let combinedAddresses = p1.addresses->AddressSet.merge(p2.addresses)
|
|
207
191
|
|
|
208
192
|
let p1Below = p1.latestFetchedBlock.blockNumber < potentialMergeBlock
|
|
209
193
|
let p2Below = p2.latestFetchedBlock.blockNumber < potentialMergeBlock
|
|
@@ -240,7 +224,7 @@ module OptimizedPartitions = {
|
|
|
240
224
|
selection: p1.selection,
|
|
241
225
|
latestFetchedBlock: {blockNumber: potentialMergeBlock, blockTimestamp: 0},
|
|
242
226
|
mergeBlock: None,
|
|
243
|
-
|
|
227
|
+
addresses: p1.addresses, // replaced below
|
|
244
228
|
mutPendingQueries: [],
|
|
245
229
|
sourceRangeCapacity: minRange,
|
|
246
230
|
prevSourceRangeCapacity: minRange,
|
|
@@ -250,29 +234,28 @@ module OptimizedPartitions = {
|
|
|
250
234
|
}
|
|
251
235
|
|
|
252
236
|
// Apply address split on the continuing partition
|
|
253
|
-
if combinedAddresses->
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
237
|
+
if combinedAddresses->AddressSet.size > maxAddrInPartition {
|
|
238
|
+
completed
|
|
239
|
+
->Array.push(
|
|
240
|
+
continuingBase->withAddresses(
|
|
241
|
+
combinedAddresses->AddressSet.slice(~offset=0, ~limit=Some(maxAddrInPartition)),
|
|
242
|
+
),
|
|
243
|
+
)
|
|
244
|
+
->ignore
|
|
261
245
|
let restId = nextPartitionIndexRef.contents->Int.toString
|
|
262
246
|
nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
|
|
263
247
|
completed
|
|
264
248
|
->Array.push({
|
|
265
|
-
...continuingBase
|
|
249
|
+
...continuingBase->withAddresses(
|
|
250
|
+
combinedAddresses->AddressSet.slice(~offset=maxAddrInPartition, ~limit=None),
|
|
251
|
+
),
|
|
266
252
|
id: restId,
|
|
267
|
-
addressesByContractName: abcRest,
|
|
268
253
|
mutPendingQueries: [],
|
|
269
254
|
})
|
|
270
255
|
->ignore
|
|
271
256
|
completed
|
|
272
257
|
} else {
|
|
273
|
-
|
|
274
|
-
abc->Dict.set(contractName, combinedAddresses)
|
|
275
|
-
completed->Array.push({...continuingBase, addressesByContractName: abc})->ignore
|
|
258
|
+
completed->Array.push(continuingBase->withAddresses(combinedAddresses))->ignore
|
|
276
259
|
completed
|
|
277
260
|
}
|
|
278
261
|
}
|
|
@@ -320,7 +303,7 @@ module OptimizedPartitions = {
|
|
|
320
303
|
{mergeBlock: Some(_)} =>
|
|
321
304
|
newPartitions->Array.push(p)->ignore
|
|
322
305
|
| {dynamicContract: Some(contractName)} =>
|
|
323
|
-
let pAddressesCount = p.
|
|
306
|
+
let pAddressesCount = p.addresses->AddressSet.countFor(contractName)
|
|
324
307
|
// Compute merge block: last pending query's toBlock, or lfb if idle
|
|
325
308
|
let potentialMergeBlock = switch p.mutPendingQueries->Utils.Array.last {
|
|
326
309
|
| Some({isChunk: true, toBlock: Some(toBlock)}) => Some(toBlock)
|
|
@@ -664,8 +647,6 @@ type t = {
|
|
|
664
647
|
startBlock: int,
|
|
665
648
|
endBlock: option<int>,
|
|
666
649
|
normalSelection: selection,
|
|
667
|
-
// By contract name
|
|
668
|
-
contractConfigs: dict<IndexingAddresses.contractConfig>,
|
|
669
650
|
// Not used for logic - only metadata
|
|
670
651
|
chainId: int,
|
|
671
652
|
// The block number of the latest block which was added to the queue
|
|
@@ -686,8 +667,7 @@ type t = {
|
|
|
686
667
|
knownHeight: int,
|
|
687
668
|
firstEventBlock: option<int>,
|
|
688
669
|
// Per-contract registered-address count past which a dynamic contract is
|
|
689
|
-
// switched to client-side filtering. None disables the switch
|
|
690
|
-
// (sources that can't filter client-side, e.g. SVM/Fuel), leaving every
|
|
670
|
+
// switched to client-side filtering. None disables the switch, leaving every
|
|
691
671
|
// contract filtered server-side.
|
|
692
672
|
clientFilterAddressThreshold: option<int>,
|
|
693
673
|
}
|
|
@@ -944,7 +924,6 @@ let updateInternal = (
|
|
|
944
924
|
let updatedFetchState = {
|
|
945
925
|
startBlock: fetchState.startBlock,
|
|
946
926
|
endBlock: fetchState.endBlock,
|
|
947
|
-
contractConfigs: fetchState.contractConfigs,
|
|
948
927
|
normalSelection: fetchState.normalSelection,
|
|
949
928
|
chainId: fetchState.chainId,
|
|
950
929
|
onBlockRegistrations: fetchState.onBlockRegistrations,
|
|
@@ -965,40 +944,6 @@ let updateInternal = (
|
|
|
965
944
|
updatedFetchState
|
|
966
945
|
}
|
|
967
946
|
|
|
968
|
-
let warnDifferentContractType = (
|
|
969
|
-
fetchState,
|
|
970
|
-
~existingContract: indexingAddress,
|
|
971
|
-
~dc: indexingAddress,
|
|
972
|
-
) => {
|
|
973
|
-
let logger = Logging.createChild(
|
|
974
|
-
~params={
|
|
975
|
-
"chainId": fetchState.chainId,
|
|
976
|
-
"contractAddress": dc.address->Address.toString,
|
|
977
|
-
"existingContractType": existingContract.contractName,
|
|
978
|
-
"newContractType": dc.contractName,
|
|
979
|
-
},
|
|
980
|
-
)
|
|
981
|
-
logger->Logging.childWarn(`Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`)
|
|
982
|
-
}
|
|
983
|
-
|
|
984
|
-
let addressesByContractNameCount = (addressesByContractName: dict<array<Address.t>>) => {
|
|
985
|
-
let numAddresses = ref(0)
|
|
986
|
-
addressesByContractName->Utils.Dict.forEach(addresses => {
|
|
987
|
-
numAddresses := numAddresses.contents + addresses->Array.length
|
|
988
|
-
})
|
|
989
|
-
numAddresses.contents
|
|
990
|
-
}
|
|
991
|
-
|
|
992
|
-
let addressesByContractNameGetAll = (addressesByContractName: dict<array<Address.t>>) => {
|
|
993
|
-
let all = []
|
|
994
|
-
addressesByContractName->Utils.Dict.forEach(addresses => {
|
|
995
|
-
for idx in 0 to addresses->Array.length - 1 {
|
|
996
|
-
all->Array.push(addresses->Array.getUnsafe(idx))->ignore
|
|
997
|
-
}
|
|
998
|
-
})
|
|
999
|
-
all
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
947
|
// Move a contract to client-side address filtering, recording why.
|
|
1003
948
|
let addClientFilteredContract = (
|
|
1004
949
|
clientFilteredContracts: Utils.Set.t<string>,
|
|
@@ -1020,6 +965,24 @@ let addClientFilteredContract = (
|
|
|
1020
965
|
)
|
|
1021
966
|
}
|
|
1022
967
|
|
|
968
|
+
// The block a partition will have fetched to once everything on its pending
|
|
969
|
+
// queue lands — not the block it has fetched to now. A backfill has to reach it
|
|
970
|
+
// rather than the frontier: a query that was routed before this batch's
|
|
971
|
+
// addresses reached the address store carries no events for them, yet it still
|
|
972
|
+
// advances the frontier over its range on arrival, and nothing else would ever
|
|
973
|
+
// refetch it. An in-flight open-ended query has no toBlock of its own; it can't
|
|
974
|
+
// return past the chain's known height, so that bounds it.
|
|
975
|
+
let claimedFetchedBlock = (p: partition, ~knownHeight) =>
|
|
976
|
+
p.mutPendingQueries->Array.reduce(p.latestFetchedBlock.blockNumber, (max, q) =>
|
|
977
|
+
Pervasives.max(
|
|
978
|
+
max,
|
|
979
|
+
switch q.fetchedBlock {
|
|
980
|
+
| Some({blockNumber}) => blockNumber
|
|
981
|
+
| None => q.toBlock->Option.getOr(knownHeight)
|
|
982
|
+
},
|
|
983
|
+
)
|
|
984
|
+
)
|
|
985
|
+
|
|
1023
986
|
// Fold every client-filtered contract's server-side partitions into client-side
|
|
1024
987
|
// fetching without tearing down established state:
|
|
1025
988
|
// - The standing address-free partition (no mergeBlock) keeps its id, frontier,
|
|
@@ -1027,14 +990,14 @@ let addClientFilteredContract = (
|
|
|
1027
990
|
// must be added does its selection change — under a fresh id, so in-flight
|
|
1028
991
|
// responses built from the old selection are orphaned instead of advancing
|
|
1029
992
|
// the frontier past ranges the new contract wasn't fetched for.
|
|
1030
|
-
// - Partitions absorbed below the standing
|
|
1031
|
-
// partitions and config partitions all of whose
|
|
1032
|
-
// client-filtered, plus any prior backfill) become one bounded
|
|
1033
|
-
// partition covering [their min frontier,
|
|
1034
|
-
// caps its queries at mergeBlock and handleQueryResponse deletes
|
|
1035
|
-
// arrival. The overlap it re-delivers is deduped by mergeIntoBuffer, and
|
|
1036
|
-
// re-fetch doubles as history for freshly registered addresses: events
|
|
1037
|
-
// dropped before the address was registered now pass the
|
|
993
|
+
// - Partitions absorbed below the standing partition's claimed block
|
|
994
|
+
// (single-contract dynamic partitions and config partitions all of whose
|
|
995
|
+
// contracts are client-filtered, plus any prior backfill) become one bounded
|
|
996
|
+
// backfill partition covering [their min frontier, that claimed block]:
|
|
997
|
+
// getNextQuery caps its queries at mergeBlock and handleQueryResponse deletes
|
|
998
|
+
// it on arrival. The overlap it re-delivers is deduped by mergeIntoBuffer, and
|
|
999
|
+
// the re-fetch doubles as history for freshly registered addresses: events
|
|
1000
|
+
// dropped before the address was registered now pass the address gate.
|
|
1038
1001
|
// - A partition mixing client-filtered and server-side contracts stays,
|
|
1039
1002
|
// stripped of the client-filtered contracts' addresses — the address-free
|
|
1040
1003
|
// partition covers those logs, so fetching them server-side too would only
|
|
@@ -1050,6 +1013,8 @@ let collapseClientFilteredContracts = (
|
|
|
1050
1013
|
~clientFilteredContracts: Utils.Set.t<string>,
|
|
1051
1014
|
~normalSelection: selection,
|
|
1052
1015
|
~nextPartitionIndexRef: ref<int>,
|
|
1016
|
+
~addressStore: AddressStore.t,
|
|
1017
|
+
~knownHeight: int,
|
|
1053
1018
|
) => {
|
|
1054
1019
|
if clientFilteredContracts->Utils.Set.size === 0 {
|
|
1055
1020
|
partitions
|
|
@@ -1069,18 +1034,18 @@ let collapseClientFilteredContracts = (
|
|
|
1069
1034
|
backfills->Array.push(p)->ignore
|
|
1070
1035
|
| {selection: {dependsOnAddresses: false}} => absorbedPartitions->Array.push(p)->ignore
|
|
1071
1036
|
| _ =>
|
|
1072
|
-
let contractNames = p.
|
|
1073
|
-
let
|
|
1074
|
-
contractNames->Array.filter(c => clientFilteredContracts->Utils.Set.has(c))
|
|
1075
|
-
if
|
|
1037
|
+
let contractNames = p.addresses->AddressSet.contractNames
|
|
1038
|
+
let serverSideNames =
|
|
1039
|
+
contractNames->Array.filter(c => !(clientFilteredContracts->Utils.Set.has(c)))
|
|
1040
|
+
if serverSideNames->Array.length === contractNames->Array.length {
|
|
1076
1041
|
kept->Array.push(p)->ignore
|
|
1077
|
-
} else if
|
|
1042
|
+
} else if serverSideNames->Utils.Array.isEmpty {
|
|
1078
1043
|
absorbedPartitions->Array.push(p)->ignore
|
|
1079
1044
|
} else {
|
|
1080
|
-
let stripped = p.addressesByContractName->Utils.Dict.shallowCopy
|
|
1081
|
-
clientFilteredNames->Array.forEach(c => stripped->Utils.Dict.deleteInPlace(c))
|
|
1082
1045
|
strippedFrontiers->Array.push(p.latestFetchedBlock)->ignore
|
|
1083
|
-
kept
|
|
1046
|
+
kept
|
|
1047
|
+
->Array.push(p->withAddresses(p.addresses->AddressSet.filterByContracts(serverSideNames)))
|
|
1048
|
+
->ignore
|
|
1084
1049
|
}
|
|
1085
1050
|
}
|
|
1086
1051
|
)
|
|
@@ -1150,7 +1115,7 @@ let collapseClientFilteredContracts = (
|
|
|
1150
1115
|
id,
|
|
1151
1116
|
latestFetchedBlock: minFrontier,
|
|
1152
1117
|
selection: newSelection,
|
|
1153
|
-
|
|
1118
|
+
addresses: addressStore->AddressStore.emptySet,
|
|
1154
1119
|
mergeBlock: None,
|
|
1155
1120
|
dynamicContract: None,
|
|
1156
1121
|
mutPendingQueries: [],
|
|
@@ -1171,8 +1136,12 @@ let collapseClientFilteredContracts = (
|
|
|
1171
1136
|
standing
|
|
1172
1137
|
}
|
|
1173
1138
|
kept->Array.push(standingOut)->ignore
|
|
1139
|
+
// Read off standingOut, not standing: a selection change orphans the
|
|
1140
|
+
// in-flight queries (fresh id, empty queue), so there the frontier is
|
|
1141
|
+
// all the partition will ever claim.
|
|
1142
|
+
let catchUpToBlock = standingOut->claimedFetchedBlock(~knownHeight)
|
|
1174
1143
|
switch minFrontierRef.contents {
|
|
1175
|
-
| Some(minFrontier) if minFrontier.blockNumber <
|
|
1144
|
+
| Some(minFrontier) if minFrontier.blockNumber < catchUpToBlock =>
|
|
1176
1145
|
let id = nextPartitionIndexRef.contents->Int.toString
|
|
1177
1146
|
nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
|
|
1178
1147
|
kept
|
|
@@ -1180,8 +1149,8 @@ let collapseClientFilteredContracts = (
|
|
|
1180
1149
|
id,
|
|
1181
1150
|
latestFetchedBlock: minFrontier,
|
|
1182
1151
|
selection: newSelection,
|
|
1183
|
-
|
|
1184
|
-
mergeBlock: Some(
|
|
1152
|
+
addresses: addressStore->AddressStore.emptySet,
|
|
1153
|
+
mergeBlock: Some(catchUpToBlock),
|
|
1185
1154
|
dynamicContract: None,
|
|
1186
1155
|
mutPendingQueries: [],
|
|
1187
1156
|
// Same query shape as the standing partition, so inherit its
|
|
@@ -1192,8 +1161,9 @@ let collapseClientFilteredContracts = (
|
|
|
1192
1161
|
latestSourceRangeCapacityUpdateBlock: 0,
|
|
1193
1162
|
})
|
|
1194
1163
|
->ignore
|
|
1195
|
-
// An absorbed frontier at/above the
|
|
1196
|
-
//
|
|
1164
|
+
// An absorbed frontier at/above the claimed block needs no backfill:
|
|
1165
|
+
// nothing has fetched past there yet, so the standing partition still
|
|
1166
|
+
// covers it going forward.
|
|
1197
1167
|
| _ => ()
|
|
1198
1168
|
}
|
|
1199
1169
|
kept
|
|
@@ -1209,8 +1179,11 @@ Phase 2: Merge non-dynamic partitions together to reduce unnecessary concurrency
|
|
|
1209
1179
|
Returns OptimizedPartitions.t directly.
|
|
1210
1180
|
(Dynamic partitions are merged by OptimizedPartitions.make automatically)
|
|
1211
1181
|
*/
|
|
1212
|
-
let
|
|
1213
|
-
|
|
1182
|
+
let createPartitions = (
|
|
1183
|
+
// One set per contract, holding the addresses that still need a partition:
|
|
1184
|
+
// a batch's fresh registrations, or the survivors of a rolled-back partition.
|
|
1185
|
+
~registeringSetsByContract: dict<AddressSet.t>,
|
|
1186
|
+
~addressStore: AddressStore.t,
|
|
1214
1187
|
~dynamicContracts: Utils.Set.t<string>,
|
|
1215
1188
|
~clientFilteredContracts: Utils.Set.t<string>,
|
|
1216
1189
|
~normalSelection: selection,
|
|
@@ -1218,6 +1191,7 @@ let createPartitionsFromIndexingAddresses = (
|
|
|
1218
1191
|
~nextPartitionIndex: int,
|
|
1219
1192
|
~existingPartitions: array<partition>,
|
|
1220
1193
|
~progressBlockNumber: int,
|
|
1194
|
+
~knownHeight: int,
|
|
1221
1195
|
): // Floor for latestFetchedBlock (use progressBlockNumber from make, or 0 for registerDynamicContracts)
|
|
1222
1196
|
OptimizedPartitions.t => {
|
|
1223
1197
|
let nextPartitionIndexRef = ref(nextPartitionIndex)
|
|
@@ -1226,90 +1200,67 @@ OptimizedPartitions.t => {
|
|
|
1226
1200
|
let dynamicPartitions = []
|
|
1227
1201
|
let nonDynamicPartitions = []
|
|
1228
1202
|
|
|
1229
|
-
let contractNames =
|
|
1203
|
+
let contractNames = registeringSetsByContract->Dict.keysToArray
|
|
1230
1204
|
for cIdx in 0 to contractNames->Array.length - 1 {
|
|
1231
1205
|
let contractName = contractNames->Array.getUnsafe(cIdx)
|
|
1232
|
-
let
|
|
1233
|
-
let addresses =
|
|
1234
|
-
registeringContracts->Dict.keysToArray->(Utils.magic: array<string> => array<Address.t>)
|
|
1206
|
+
let contractSet = registeringSetsByContract->Dict.getUnsafe(contractName)
|
|
1235
1207
|
|
|
1236
1208
|
let isDynamic = dynamicContracts->Utils.Set.has(contractName)
|
|
1237
1209
|
let partitions = isDynamic ? dynamicPartitions : nonDynamicPartitions
|
|
1238
1210
|
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
| Some(nextStartBlockKey) => {
|
|
1260
|
-
let nextStartBlock = nextStartBlockKey->Int.fromString->Option.getUnsafe
|
|
1261
|
-
let shouldJoinCurrentStartBlock =
|
|
1262
|
-
nextStartBlock - startBlockRef.contents < OptimizedPartitions.tooFarBlockRange
|
|
1263
|
-
|
|
1264
|
-
// Addresses with different start blocks within range share a partition;
|
|
1265
|
-
// events before each address's effectiveStartBlock are dropped on the
|
|
1266
|
-
// client side (the event's clientAddressFilter for address-valued
|
|
1267
|
-
// params).
|
|
1268
|
-
if shouldJoinCurrentStartBlock {
|
|
1269
|
-
addressesRef :=
|
|
1270
|
-
addressesRef.contents->Array.concat(byStartBlock->Dict.getUnsafe(nextStartBlockKey))
|
|
1271
|
-
false
|
|
1272
|
-
} else {
|
|
1273
|
-
true
|
|
1274
|
-
}
|
|
1211
|
+
// A set is ordered by effectiveStartBlock, so its start-block groups are
|
|
1212
|
+
// ascending and each group's addresses are a contiguous slice.
|
|
1213
|
+
let groups = contractSet->AddressSet.startBlockGroups
|
|
1214
|
+
let offsetRef = ref(0)
|
|
1215
|
+
let groupIdx = ref(0)
|
|
1216
|
+
while groupIdx.contents < groups->Array.length {
|
|
1217
|
+
let startBlock = (groups->Array.getUnsafe(groupIdx.contents)).startBlock
|
|
1218
|
+
// Addresses with different start blocks within range share a partition;
|
|
1219
|
+
// events before each address's effectiveStartBlock are dropped by the
|
|
1220
|
+
// source's address gate.
|
|
1221
|
+
let countRef = ref(0)
|
|
1222
|
+
let nextIdx = ref(groupIdx.contents)
|
|
1223
|
+
let joining = ref(true)
|
|
1224
|
+
while joining.contents && nextIdx.contents < groups->Array.length {
|
|
1225
|
+
let group = groups->Array.getUnsafe(nextIdx.contents)
|
|
1226
|
+
if group.startBlock - startBlock < OptimizedPartitions.tooFarBlockRange {
|
|
1227
|
+
countRef := countRef.contents + group.count
|
|
1228
|
+
nextIdx := nextIdx.contents + 1
|
|
1229
|
+
} else {
|
|
1230
|
+
joining := false
|
|
1275
1231
|
}
|
|
1276
1232
|
}
|
|
1277
1233
|
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
switch maybeNextStartBlockKey {
|
|
1306
|
-
| None => ()
|
|
1307
|
-
| Some(nextStartBlockKey) => {
|
|
1308
|
-
startBlockRef := nextStartBlockKey->Int.fromString->Option.getUnsafe
|
|
1309
|
-
addressesRef := byStartBlock->Dict.getUnsafe(nextStartBlockKey)
|
|
1310
|
-
}
|
|
1311
|
-
}
|
|
1234
|
+
let latestFetchedBlock = {
|
|
1235
|
+
blockNumber: Pervasives.max(startBlock - 1, progressBlockNumber),
|
|
1236
|
+
blockTimestamp: 0,
|
|
1237
|
+
}
|
|
1238
|
+
let remainingRef = ref(countRef.contents)
|
|
1239
|
+
let chunkOffsetRef = ref(offsetRef.contents)
|
|
1240
|
+
while remainingRef.contents > 0 {
|
|
1241
|
+
let take = Pervasives.min(remainingRef.contents, maxAddrInPartition)
|
|
1242
|
+
let pAddresses =
|
|
1243
|
+
contractSet->AddressSet.slice(~offset=chunkOffsetRef.contents, ~limit=Some(take))
|
|
1244
|
+
partitions->Array.push({
|
|
1245
|
+
id: nextPartitionIndexRef.contents->Int.toString,
|
|
1246
|
+
latestFetchedBlock,
|
|
1247
|
+
selection: normalSelection,
|
|
1248
|
+
dynamicContract: isDynamic ? Some(contractName) : None,
|
|
1249
|
+
addresses: pAddresses,
|
|
1250
|
+
mergeBlock: None,
|
|
1251
|
+
mutPendingQueries: [],
|
|
1252
|
+
sourceRangeCapacity: 0,
|
|
1253
|
+
prevSourceRangeCapacity: 0,
|
|
1254
|
+
eventDensity: None,
|
|
1255
|
+
latestSourceRangeCapacityUpdateBlock: 0,
|
|
1256
|
+
})
|
|
1257
|
+
nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
|
|
1258
|
+
chunkOffsetRef := chunkOffsetRef.contents + take
|
|
1259
|
+
remainingRef := remainingRef.contents - take
|
|
1312
1260
|
}
|
|
1261
|
+
|
|
1262
|
+
offsetRef := offsetRef.contents + countRef.contents
|
|
1263
|
+
groupIdx := nextIdx.contents
|
|
1313
1264
|
}
|
|
1314
1265
|
}
|
|
1315
1266
|
|
|
@@ -1329,29 +1280,14 @@ OptimizedPartitions.t => {
|
|
|
1329
1280
|
let currentPBlock = currentP.latestFetchedBlock.blockNumber
|
|
1330
1281
|
let nextPBlock = nextP.latestFetchedBlock.blockNumber
|
|
1331
1282
|
|
|
1332
|
-
|
|
1333
|
-
let totalCount =
|
|
1334
|
-
currentP.addressesByContractName->addressesByContractNameCount +
|
|
1335
|
-
nextP.addressesByContractName->addressesByContractNameCount
|
|
1283
|
+
let totalCount = currentP.addresses->AddressSet.size + nextP.addresses->AddressSet.size
|
|
1336
1284
|
|
|
1337
1285
|
if totalCount > maxAddrInPartition {
|
|
1338
1286
|
// Exceeds address limit - don't merge, keep partitions separate
|
|
1339
1287
|
mergedNonDynamic->Array.push(currentP)->ignore
|
|
1340
1288
|
currentPRef := nextP
|
|
1341
1289
|
} else {
|
|
1342
|
-
|
|
1343
|
-
let mergedAddresses = nextP.addressesByContractName->Utils.Dict.shallowCopy
|
|
1344
|
-
let currentContractNames = currentP.addressesByContractName->Dict.keysToArray
|
|
1345
|
-
for jdx in 0 to currentContractNames->Array.length - 1 {
|
|
1346
|
-
let cn = currentContractNames->Array.getUnsafe(jdx)
|
|
1347
|
-
let currentAddrs = currentP.addressesByContractName->Dict.getUnsafe(cn)
|
|
1348
|
-
switch mergedAddresses->Utils.Dict.dangerouslyGetNonOption(cn) {
|
|
1349
|
-
| Some(existingAddrs) =>
|
|
1350
|
-
// Use concat (non-mutating) to avoid corrupting nextP's arrays
|
|
1351
|
-
mergedAddresses->Dict.set(cn, existingAddrs->Array.concat(currentAddrs))
|
|
1352
|
-
| None => mergedAddresses->Dict.set(cn, currentAddrs)
|
|
1353
|
-
}
|
|
1354
|
-
}
|
|
1290
|
+
let mergedAddresses = nextP.addresses->AddressSet.merge(currentP.addresses)
|
|
1355
1291
|
|
|
1356
1292
|
let isTooFar = currentPBlock + OptimizedPartitions.tooFarBlockRange < nextPBlock
|
|
1357
1293
|
|
|
@@ -1363,16 +1299,10 @@ OptimizedPartitions.t => {
|
|
|
1363
1299
|
mergeBlock: currentPBlock < nextPBlock ? Some(nextPBlock) : None,
|
|
1364
1300
|
})
|
|
1365
1301
|
->ignore
|
|
1366
|
-
currentPRef :=
|
|
1367
|
-
...nextP,
|
|
1368
|
-
addressesByContractName: mergedAddresses,
|
|
1369
|
-
}
|
|
1302
|
+
currentPRef := nextP->withAddresses(mergedAddresses)
|
|
1370
1303
|
} else {
|
|
1371
1304
|
// Close: push next's addresses into current
|
|
1372
|
-
currentPRef :=
|
|
1373
|
-
...currentP,
|
|
1374
|
-
addressesByContractName: mergedAddresses,
|
|
1375
|
-
}
|
|
1305
|
+
currentPRef := currentP->withAddresses(mergedAddresses)
|
|
1376
1306
|
}
|
|
1377
1307
|
}
|
|
1378
1308
|
|
|
@@ -1393,6 +1323,8 @@ OptimizedPartitions.t => {
|
|
|
1393
1323
|
~clientFilteredContracts,
|
|
1394
1324
|
~normalSelection,
|
|
1395
1325
|
~nextPartitionIndexRef,
|
|
1326
|
+
~addressStore,
|
|
1327
|
+
~knownHeight,
|
|
1396
1328
|
)
|
|
1397
1329
|
OptimizedPartitions.make(
|
|
1398
1330
|
~partitions=allPartitions,
|
|
@@ -1405,7 +1337,14 @@ OptimizedPartitions.t => {
|
|
|
1405
1337
|
|
|
1406
1338
|
let registerDynamicContracts = (
|
|
1407
1339
|
fetchState: t,
|
|
1408
|
-
~
|
|
1340
|
+
~addressStore: AddressStore.t,
|
|
1341
|
+
// How far an in-flight open-ended query may end up claiming. These
|
|
1342
|
+
// registrations usually come out of a response that is applied right after
|
|
1343
|
+
// this call, and an unbounded query can reach past the height known when it
|
|
1344
|
+
// was dispatched — so the caller folds that response's own frontier and
|
|
1345
|
+
// height in here. Defaults to what the fetch state knows, which is all
|
|
1346
|
+
// there is to go on when no response is being applied.
|
|
1347
|
+
~claimCeiling=fetchState.knownHeight,
|
|
1409
1348
|
// These are raw items which might have dynamic contracts received from contractRegister call.
|
|
1410
1349
|
// Might contain duplicates which we should filter out
|
|
1411
1350
|
items: array<Internal.item>,
|
|
@@ -1417,167 +1356,130 @@ let registerDynamicContracts = (
|
|
|
1417
1356
|
)
|
|
1418
1357
|
}
|
|
1419
1358
|
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
//
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
let noEventsAddresses: dict<indexingAddress> = Dict.make()
|
|
1427
|
-
// Batch-level view of all addresses registered so far (across contracts,
|
|
1428
|
-
// including no-events ones), so two contracts registering the same address
|
|
1429
|
-
// within one batch conflict the same way as against indexingAddresses.
|
|
1430
|
-
let registeringAddresses: dict<indexingAddress> = Dict.make()
|
|
1431
|
-
|
|
1359
|
+
// Flatten every item's dcs into one batch, remembering where each came from
|
|
1360
|
+
// so the rejected ones can be spliced back out of their item (which is what
|
|
1361
|
+
// keeps them from being persisted twice).
|
|
1362
|
+
let registrations: array<AddressStore.registration> = []
|
|
1363
|
+
let sourceDcs: array<Internal.dcs> = []
|
|
1364
|
+
let sourceIndexes: array<int> = []
|
|
1432
1365
|
for itemIdx in 0 to items->Array.length - 1 {
|
|
1433
|
-
|
|
1434
|
-
switch item->Internal.getItemDcs {
|
|
1366
|
+
switch items->Array.getUnsafe(itemIdx)->Internal.getItemDcs {
|
|
1435
1367
|
| None => ()
|
|
1436
1368
|
| Some(dcs) =>
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
~contractStartBlock,
|
|
1452
|
-
),
|
|
1453
|
-
}
|
|
1454
|
-
// Prevent registering already indexing contracts
|
|
1455
|
-
switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
|
|
1456
|
-
| Some(existingContract) =>
|
|
1457
|
-
// FIXME: Instead of filtering out duplicates,
|
|
1458
|
-
// we should check the block number first.
|
|
1459
|
-
// If new registration with earlier block number
|
|
1460
|
-
// we should register it for the missing block range
|
|
1461
|
-
if existingContract.contractName != dc.contractName {
|
|
1462
|
-
fetchState->warnDifferentContractType(~existingContract, ~dc=dcWithStartBlock)
|
|
1463
|
-
} else if existingContract.effectiveStartBlock > dcWithStartBlock.effectiveStartBlock {
|
|
1464
|
-
let logger = Logging.createChild(
|
|
1465
|
-
~params={
|
|
1466
|
-
"chainId": fetchState.chainId,
|
|
1467
|
-
"contractAddress": dc.address->Address.toString,
|
|
1468
|
-
"existingBlockNumber": existingContract.effectiveStartBlock,
|
|
1469
|
-
"newBlockNumber": dcWithStartBlock.effectiveStartBlock,
|
|
1470
|
-
},
|
|
1471
|
-
)
|
|
1472
|
-
logger->Logging.childWarn(`Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`)
|
|
1473
|
-
}
|
|
1474
|
-
shouldRemove := true
|
|
1475
|
-
| None =>
|
|
1476
|
-
let shouldUpdate = switch registeringAddresses->Utils.Dict.dangerouslyGetNonOption(
|
|
1477
|
-
dc.address->Address.toString,
|
|
1478
|
-
) {
|
|
1479
|
-
| Some(registeringContract) if registeringContract.contractName != dc.contractName =>
|
|
1480
|
-
fetchState->warnDifferentContractType(
|
|
1481
|
-
~existingContract=registeringContract,
|
|
1482
|
-
~dc=dcWithStartBlock,
|
|
1483
|
-
)
|
|
1484
|
-
false
|
|
1485
|
-
| Some(_) => // Since the DC is registered by an earlier item in the query
|
|
1486
|
-
// FIXME: This unsafely relies on the asc order of the items
|
|
1487
|
-
// which is 99% true, but there were cases when the source ordering was wrong
|
|
1488
|
-
false
|
|
1489
|
-
| None => true
|
|
1490
|
-
}
|
|
1491
|
-
if shouldUpdate {
|
|
1492
|
-
earliestRegisteringEventBlockNumber :=
|
|
1493
|
-
Pervasives.min(
|
|
1494
|
-
earliestRegisteringEventBlockNumber.contents,
|
|
1495
|
-
dcWithStartBlock.effectiveStartBlock,
|
|
1496
|
-
)
|
|
1497
|
-
registeringContractsByContract
|
|
1498
|
-
->Utils.Dict.getOrInsertEmptyDict(dc.contractName)
|
|
1499
|
-
->Dict.set(dc.address->Address.toString, dcWithStartBlock)
|
|
1500
|
-
registeringAddresses->Dict.set(dc.address->Address.toString, dcWithStartBlock)
|
|
1501
|
-
} else {
|
|
1502
|
-
shouldRemove := true
|
|
1503
|
-
}
|
|
1504
|
-
}
|
|
1505
|
-
| None =>
|
|
1506
|
-
let dcAsIndexingAddress: indexingAddress = {
|
|
1507
|
-
address: dc.address,
|
|
1508
|
-
contractName: dc.contractName,
|
|
1509
|
-
registrationBlock: dc.registrationBlock,
|
|
1510
|
-
effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
|
|
1511
|
-
~registrationBlock=dc.registrationBlock,
|
|
1512
|
-
~contractStartBlock=None,
|
|
1513
|
-
),
|
|
1514
|
-
}
|
|
1515
|
-
// Prevent duplicate logging/persistence when the same address is
|
|
1516
|
-
// already tracked on fetchState, either from the db on startup or
|
|
1517
|
-
// from an earlier registration in this batch.
|
|
1518
|
-
switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
|
|
1519
|
-
| Some(existingContract) =>
|
|
1520
|
-
if existingContract.contractName != dc.contractName {
|
|
1521
|
-
fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
|
|
1522
|
-
}
|
|
1523
|
-
shouldRemove := true
|
|
1524
|
-
| None =>
|
|
1525
|
-
switch registeringAddresses->Utils.Dict.dangerouslyGetNonOption(
|
|
1526
|
-
dc.address->Address.toString,
|
|
1527
|
-
) {
|
|
1528
|
-
| Some(existingContract) =>
|
|
1529
|
-
if existingContract.contractName != dc.contractName {
|
|
1530
|
-
fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
|
|
1531
|
-
}
|
|
1532
|
-
// Otherwise already queued for persistence by an earlier item in this batch.
|
|
1533
|
-
shouldRemove := true
|
|
1534
|
-
| None =>
|
|
1535
|
-
let logger = Logging.createChild(
|
|
1536
|
-
~params={
|
|
1537
|
-
"chainId": fetchState.chainId,
|
|
1538
|
-
"contractAddress": dc.address->Address.toString,
|
|
1539
|
-
"contractName": dc.contractName,
|
|
1540
|
-
},
|
|
1541
|
-
)
|
|
1542
|
-
// Persist the address to the db so a future config change that
|
|
1543
|
-
// adds events for this contract can pick it up on restart, but
|
|
1544
|
-
// skip partition registration since there's nothing to fetch.
|
|
1545
|
-
logger->Logging.childWarn(`Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`)
|
|
1546
|
-
noEventsAddresses->Dict.set(dc.address->Address.toString, dcAsIndexingAddress)
|
|
1547
|
-
registeringAddresses->Dict.set(dc.address->Address.toString, dcAsIndexingAddress)
|
|
1548
|
-
}
|
|
1549
|
-
}
|
|
1550
|
-
}
|
|
1369
|
+
for dcIdx in 0 to dcs->Array.length - 1 {
|
|
1370
|
+
let dc = dcs->Array.getUnsafe(dcIdx)
|
|
1371
|
+
registrations
|
|
1372
|
+
->Array.push({
|
|
1373
|
+
address: dc.address,
|
|
1374
|
+
contractName: dc.contractName,
|
|
1375
|
+
registrationBlock: dc.registrationBlock,
|
|
1376
|
+
})
|
|
1377
|
+
->ignore
|
|
1378
|
+
sourceDcs->Array.push(dcs)->ignore
|
|
1379
|
+
sourceIndexes->Array.push(dcIdx)->ignore
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1551
1383
|
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1384
|
+
// Ids are handed out in registration order, so a cursor taken now selects
|
|
1385
|
+
// exactly what this batch adds.
|
|
1386
|
+
let idCursor = addressStore->AddressStore.nextId
|
|
1387
|
+
// The store resolves each address against both what it already holds and the
|
|
1388
|
+
// batch's own earlier entries, so two contracts claiming one address inside a
|
|
1389
|
+
// single batch conflict the same way as across batches.
|
|
1390
|
+
let verdicts = addressStore->AddressStore.registerBatch(registrations)
|
|
1391
|
+
|
|
1392
|
+
let warn = (~contractAddress, ~params, message) =>
|
|
1393
|
+
Logging.createChild(
|
|
1394
|
+
~params={
|
|
1395
|
+
"chainId": fetchState.chainId,
|
|
1396
|
+
"contractAddress": contractAddress,
|
|
1397
|
+
"details": params,
|
|
1398
|
+
},
|
|
1399
|
+
)->Logging.childWarn(message)
|
|
1400
|
+
|
|
1401
|
+
let registeringContractNames = []
|
|
1402
|
+
let hasNoEventsUpdatesRef = ref(false)
|
|
1403
|
+
for idx in 0 to verdicts->Array.length - 1 {
|
|
1404
|
+
let registration = registrations->Array.getUnsafe(idx)
|
|
1405
|
+
let contractAddress = registration.address->Address.toString
|
|
1406
|
+
let keep = switch verdicts->Array.getUnsafe(idx) {
|
|
1407
|
+
| Added(_) =>
|
|
1408
|
+
if !(registeringContractNames->Array.includes(registration.contractName)) {
|
|
1409
|
+
registeringContractNames->Array.push(registration.contractName)->ignore
|
|
1559
1410
|
}
|
|
1411
|
+
true
|
|
1412
|
+
| NoEvents(_) =>
|
|
1413
|
+
hasNoEventsUpdatesRef := true
|
|
1414
|
+
// Persist the address to the db so a future config change that adds
|
|
1415
|
+
// events for this contract can pick it up on restart, but skip partition
|
|
1416
|
+
// registration since there's nothing to fetch.
|
|
1417
|
+
warn(
|
|
1418
|
+
~contractAddress,
|
|
1419
|
+
~params={"contractName": registration.contractName},
|
|
1420
|
+
`Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`,
|
|
1421
|
+
)
|
|
1422
|
+
true
|
|
1423
|
+
| Conflict({existingContractName}) =>
|
|
1424
|
+
warn(
|
|
1425
|
+
~contractAddress,
|
|
1426
|
+
~params={
|
|
1427
|
+
"existingContractType": existingContractName,
|
|
1428
|
+
"newContractType": registration.contractName,
|
|
1429
|
+
},
|
|
1430
|
+
`Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`,
|
|
1431
|
+
)
|
|
1432
|
+
false
|
|
1433
|
+
| Duplicate({effectiveStartBlock, existingEffectiveStartBlock}) =>
|
|
1434
|
+
// FIXME: Instead of filtering out duplicates, we should check the block
|
|
1435
|
+
// number first. If a new registration has an earlier block number we
|
|
1436
|
+
// should register it for the missing block range.
|
|
1437
|
+
if existingEffectiveStartBlock > effectiveStartBlock {
|
|
1438
|
+
warn(
|
|
1439
|
+
~contractAddress,
|
|
1440
|
+
~params={
|
|
1441
|
+
"existingBlockNumber": existingEffectiveStartBlock,
|
|
1442
|
+
"newBlockNumber": effectiveStartBlock,
|
|
1443
|
+
},
|
|
1444
|
+
`Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`,
|
|
1445
|
+
)
|
|
1446
|
+
}
|
|
1447
|
+
false
|
|
1448
|
+
| Invalid =>
|
|
1449
|
+
warn(
|
|
1450
|
+
~contractAddress,
|
|
1451
|
+
~params={"contractName": registration.contractName},
|
|
1452
|
+
`Skipping contract registration: Not a valid address for this chain's ecosystem.`,
|
|
1453
|
+
)
|
|
1454
|
+
false
|
|
1455
|
+
}
|
|
1456
|
+
if !keep {
|
|
1457
|
+
// Mark for removal; the splice happens below, back to front, so earlier
|
|
1458
|
+
// indexes stay valid.
|
|
1459
|
+
sourceIndexes->Array.setUnsafe(idx, -1 - sourceIndexes->Array.getUnsafe(idx))
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
for idx in verdicts->Array.length - 1 downto 0 {
|
|
1463
|
+
let marked = sourceIndexes->Array.getUnsafe(idx)
|
|
1464
|
+
if marked < 0 {
|
|
1465
|
+
let _ =
|
|
1466
|
+
sourceDcs->Array.getUnsafe(idx)->Array.splice(~start=-1 - marked, ~remove=1, ~insert=[])
|
|
1560
1467
|
}
|
|
1561
1468
|
}
|
|
1562
1469
|
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
switch (dcContractNamesToStore, hasNoEventsUpdates) {
|
|
1566
|
-
// Dont update anything when everything was filter out
|
|
1470
|
+
switch (registeringContractNames, hasNoEventsUpdatesRef.contents) {
|
|
1471
|
+
// Dont update anything when everything was filtered out
|
|
1567
1472
|
| ([], false) => fetchState
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
// partitions since there's nothing to fetch for them.
|
|
1572
|
-
indexingAddresses->IndexingAddresses.register(noEventsAddresses)
|
|
1573
|
-
fetchState
|
|
1473
|
+
// Only dcs for contracts without events. The store already tracks them so
|
|
1474
|
+
// subsequent registrations see them, but there are no partitions to touch.
|
|
1475
|
+
| ([], true) => fetchState
|
|
1574
1476
|
| (_, _) => {
|
|
1575
1477
|
let newPartitions = []
|
|
1576
1478
|
let dynamicContractsRef = ref(fetchState.optimizedPartitions.dynamicContracts)
|
|
1577
1479
|
let mutExistingPartitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
|
|
1578
1480
|
|
|
1579
|
-
for idx in 0 to
|
|
1580
|
-
let contractName =
|
|
1481
|
+
for idx in 0 to registeringContractNames->Array.length - 1 {
|
|
1482
|
+
let contractName = registeringContractNames->Array.getUnsafe(idx)
|
|
1581
1483
|
|
|
1582
1484
|
// When a new contract name is added as a dynamic contract for the first time (not in dynamicContracts set):
|
|
1583
1485
|
// Walks through existing partitions that have addresses for this contract name
|
|
@@ -1590,12 +1492,12 @@ let registerDynamicContracts = (
|
|
|
1590
1492
|
|
|
1591
1493
|
for idx in 0 to mutExistingPartitions->Array.length - 1 {
|
|
1592
1494
|
let p = mutExistingPartitions->Array.getUnsafe(idx)
|
|
1593
|
-
switch p.
|
|
1594
|
-
|
|
|
1595
|
-
|
|
|
1495
|
+
switch p.addresses->AddressSet.countFor(contractName) {
|
|
1496
|
+
| 0 => () // Skip partitions which don't have our contract
|
|
1497
|
+
| _ =>
|
|
1596
1498
|
// Also filter out partitions which are 100% not mergable
|
|
1597
1499
|
if p.selection.dependsOnAddresses && p.mergeBlock === None {
|
|
1598
|
-
let allPartitionContractNames = p.
|
|
1500
|
+
let allPartitionContractNames = p.addresses->AddressSet.contractNames
|
|
1599
1501
|
switch allPartitionContractNames {
|
|
1600
1502
|
| [_] =>
|
|
1601
1503
|
mutExistingPartitions->Array.setUnsafe(
|
|
@@ -1618,26 +1520,20 @@ let registerDynamicContracts = (
|
|
|
1618
1520
|
(fetchState.optimizedPartitions.nextPartitionIndex +
|
|
1619
1521
|
newPartitions->Array.length)->Int.toString
|
|
1620
1522
|
|
|
1621
|
-
let
|
|
1622
|
-
|
|
1623
|
-
restAddressesByContractName->Utils.Dict.deleteInPlace(contractName)
|
|
1624
|
-
|
|
1523
|
+
let restNames =
|
|
1524
|
+
allPartitionContractNames->Array.filter(name => name !== contractName)
|
|
1625
1525
|
mutExistingPartitions->Array.setUnsafe(
|
|
1626
1526
|
idx,
|
|
1627
|
-
|
|
1628
|
-
...p,
|
|
1629
|
-
addressesByContractName: restAddressesByContractName,
|
|
1630
|
-
},
|
|
1527
|
+
p->withAddresses(p.addresses->AddressSet.filterByContracts(restNames)),
|
|
1631
1528
|
)
|
|
1632
1529
|
|
|
1633
|
-
let
|
|
1634
|
-
addressesByContractName->Dict.set(contractName, addresses)
|
|
1530
|
+
let splitAddresses = p.addresses->AddressSet.filterByContracts([contractName])
|
|
1635
1531
|
newPartitions->Array.push({
|
|
1636
1532
|
id: newPartitionId,
|
|
1637
1533
|
latestFetchedBlock: p.latestFetchedBlock,
|
|
1638
1534
|
selection: fetchState.normalSelection,
|
|
1639
1535
|
dynamicContract: Some(contractName),
|
|
1640
|
-
|
|
1536
|
+
addresses: splitAddresses,
|
|
1641
1537
|
mergeBlock: None,
|
|
1642
1538
|
mutPendingQueries: p.mutPendingQueries,
|
|
1643
1539
|
sourceRangeCapacity: p.sourceRangeCapacity,
|
|
@@ -1652,16 +1548,11 @@ let registerDynamicContracts = (
|
|
|
1652
1548
|
}
|
|
1653
1549
|
}
|
|
1654
1550
|
}
|
|
1655
|
-
|
|
1656
|
-
let registeringContracts = registeringContractsByContract->Dict.getUnsafe(contractName)
|
|
1657
|
-
indexingAddresses->IndexingAddresses.register(registeringContracts)
|
|
1658
1551
|
}
|
|
1659
|
-
// Include no-events dcs so later batches detect conflicts against them.
|
|
1660
|
-
indexingAddresses->IndexingAddresses.register(noEventsAddresses)
|
|
1661
1552
|
|
|
1662
1553
|
// Switch any dynamic contract that has just crossed the server-side
|
|
1663
1554
|
// address threshold to client-side filtering. Sticky: the set only grows, and
|
|
1664
|
-
// collapse in
|
|
1555
|
+
// collapse in createPartitions folds the contract's
|
|
1665
1556
|
// partitions into the single address-free partition.
|
|
1666
1557
|
// Clone the sticky set before mutating so this update owns its copy and
|
|
1667
1558
|
// older fetchState snapshots keep theirs (Utils.Set.add mutates in place).
|
|
@@ -1674,7 +1565,7 @@ let registerDynamicContracts = (
|
|
|
1674
1565
|
dynamicContractsRef.contents
|
|
1675
1566
|
->Utils.Set.toArray
|
|
1676
1567
|
->Array.forEach(contractName => {
|
|
1677
|
-
let addressCount =
|
|
1568
|
+
let addressCount = addressStore->AddressStore.contractCount(contractName)
|
|
1678
1569
|
if !(clientFilteredContracts->Utils.Set.has(contractName)) && addressCount > threshold {
|
|
1679
1570
|
clientFilteredContracts->addClientFilteredContract(
|
|
1680
1571
|
~contractName,
|
|
@@ -1688,8 +1579,19 @@ let registerDynamicContracts = (
|
|
|
1688
1579
|
| None => fetchState.optimizedPartitions.clientFilteredContracts
|
|
1689
1580
|
}
|
|
1690
1581
|
|
|
1691
|
-
|
|
1692
|
-
|
|
1582
|
+
// Only this batch's additions need partitions; everything already
|
|
1583
|
+
// registered has one.
|
|
1584
|
+
let registeringSetsByContract = Dict.make()
|
|
1585
|
+
registeringContractNames->Array.forEach(contractName => {
|
|
1586
|
+
registeringSetsByContract->Dict.set(
|
|
1587
|
+
contractName,
|
|
1588
|
+
addressStore->AddressStore.makeSet(~contractName, ~options={minId: idCursor}),
|
|
1589
|
+
)
|
|
1590
|
+
})
|
|
1591
|
+
|
|
1592
|
+
let optimizedPartitions = createPartitions(
|
|
1593
|
+
~registeringSetsByContract,
|
|
1594
|
+
~addressStore,
|
|
1693
1595
|
~dynamicContracts=dynamicContractsRef.contents,
|
|
1694
1596
|
~clientFilteredContracts,
|
|
1695
1597
|
~normalSelection=fetchState.normalSelection,
|
|
@@ -1698,6 +1600,7 @@ let registerDynamicContracts = (
|
|
|
1698
1600
|
newPartitions->Array.length,
|
|
1699
1601
|
~existingPartitions=mutExistingPartitions->Array.concat(newPartitions),
|
|
1700
1602
|
~progressBlockNumber=0,
|
|
1603
|
+
~knownHeight=Pervasives.max(claimCeiling, fetchState.knownHeight),
|
|
1701
1604
|
)
|
|
1702
1605
|
|
|
1703
1606
|
fetchState->updateInternal(~optimizedPartitions)
|
|
@@ -1705,33 +1608,6 @@ let registerDynamicContracts = (
|
|
|
1705
1608
|
}
|
|
1706
1609
|
}
|
|
1707
1610
|
|
|
1708
|
-
// Drop events an address-param filter rejects. A merged partition may
|
|
1709
|
-
// over-fetch a wildcard event whose indexed address param references an
|
|
1710
|
-
// address registered after the log's block; `clientAddressFilter` is the
|
|
1711
|
-
// param-level analogue of the srcAddress effectiveStartBlock check.
|
|
1712
|
-
let filterByClientAddress = (
|
|
1713
|
-
items: array<Internal.item>,
|
|
1714
|
-
~indexingAddresses: IndexingAddresses.t,
|
|
1715
|
-
): array<Internal.item> =>
|
|
1716
|
-
items->Array.filter(item =>
|
|
1717
|
-
switch item {
|
|
1718
|
-
| Internal.Event({payload, blockNumber}) as item =>
|
|
1719
|
-
let reg = (item->Internal.castUnsafeEventItem).onEventRegistration
|
|
1720
|
-
switch reg.clientAddressFilter {
|
|
1721
|
-
| Some(filter) =>
|
|
1722
|
-
filter(
|
|
1723
|
-
payload,
|
|
1724
|
-
blockNumber,
|
|
1725
|
-
indexingAddresses->IndexingAddresses.forContract(
|
|
1726
|
-
~contractName=reg.eventConfig.contractName,
|
|
1727
|
-
),
|
|
1728
|
-
)
|
|
1729
|
-
| None => true
|
|
1730
|
-
}
|
|
1731
|
-
| _ => true
|
|
1732
|
-
}
|
|
1733
|
-
)
|
|
1734
|
-
|
|
1735
1611
|
/*
|
|
1736
1612
|
Updates fetchState with a response for a given query.
|
|
1737
1613
|
Returns Error if the partition with given query cannot be found (unexpected)
|
|
@@ -1831,7 +1707,7 @@ let pushDensityPricedQuery = (
|
|
|
1831
1707
|
~density,
|
|
1832
1708
|
~chainTargetBlock,
|
|
1833
1709
|
~selection,
|
|
1834
|
-
~
|
|
1710
|
+
~addresses,
|
|
1835
1711
|
) => {
|
|
1836
1712
|
let itemsEst = densityItemsTarget(~density, ~fromBlock, ~toBlock, ~chainTargetBlock)
|
|
1837
1713
|
queries
|
|
@@ -1846,7 +1722,7 @@ let pushDensityPricedQuery = (
|
|
|
1846
1722
|
| None => Some(itemsEst)
|
|
1847
1723
|
},
|
|
1848
1724
|
itemsEst,
|
|
1849
|
-
|
|
1725
|
+
addresses,
|
|
1850
1726
|
})
|
|
1851
1727
|
->ignore
|
|
1852
1728
|
itemsEst
|
|
@@ -1875,7 +1751,7 @@ let pushGapFillQueries = (
|
|
|
1875
1751
|
~partition: partition,
|
|
1876
1752
|
~partitionBudget: float,
|
|
1877
1753
|
~selection: selection,
|
|
1878
|
-
~
|
|
1754
|
+
~addresses: AddressSet.t,
|
|
1879
1755
|
) => {
|
|
1880
1756
|
// Gaps past the chain's target block wait: they regenerate from the
|
|
1881
1757
|
// pending-walk each tick and fill once the target reaches them. The lagged
|
|
@@ -1899,7 +1775,7 @@ let pushGapFillQueries = (
|
|
|
1899
1775
|
~density,
|
|
1900
1776
|
~chainTargetBlock,
|
|
1901
1777
|
~selection,
|
|
1902
|
-
~
|
|
1778
|
+
~addresses,
|
|
1903
1779
|
)
|
|
1904
1780
|
->ignore
|
|
1905
1781
|
switch (partition->getTrustedDensity, maybeChunkRange) {
|
|
@@ -1921,7 +1797,7 @@ let pushGapFillQueries = (
|
|
|
1921
1797
|
~density,
|
|
1922
1798
|
~chainTargetBlock,
|
|
1923
1799
|
~selection,
|
|
1924
|
-
~
|
|
1800
|
+
~addresses,
|
|
1925
1801
|
)
|
|
1926
1802
|
->ignore
|
|
1927
1803
|
chunkFromBlock := chunkToBlock + 1
|
|
@@ -1994,7 +1870,7 @@ let walkPartitionPending = (
|
|
|
1994
1870
|
~partition=p,
|
|
1995
1871
|
~partitionBudget,
|
|
1996
1872
|
~selection=p.selection,
|
|
1997
|
-
~
|
|
1873
|
+
~addresses=p.addresses,
|
|
1998
1874
|
)
|
|
1999
1875
|
chunksUsedThisCall := chunksUsedThisCall.contents + (candidates->Array.length - beforeLen)
|
|
2000
1876
|
}
|
|
@@ -2097,7 +1973,7 @@ let pushForwardCandidates = (
|
|
|
2097
1973
|
~density,
|
|
2098
1974
|
~chainTargetBlock,
|
|
2099
1975
|
~selection=p.selection,
|
|
2100
|
-
~
|
|
1976
|
+
~addresses=p.addresses,
|
|
2101
1977
|
)
|
|
2102
1978
|
generatedItems := generatedItems.contents +. itemsEst->Int.toFloat
|
|
2103
1979
|
chunkFromBlock := chunkToBlock + 1
|
|
@@ -2132,7 +2008,7 @@ let pushForwardCandidates = (
|
|
|
2132
2008
|
// keeps a server cap at its estimate to protect the shared buffer.
|
|
2133
2009
|
itemsTarget: Some(itemsEst),
|
|
2134
2010
|
itemsEst,
|
|
2135
|
-
|
|
2011
|
+
addresses: p.addresses,
|
|
2136
2012
|
})
|
|
2137
2013
|
->ignore
|
|
2138
2014
|
}
|
|
@@ -2456,7 +2332,7 @@ let make = (
|
|
|
2456
2332
|
~startBlock,
|
|
2457
2333
|
~endBlock,
|
|
2458
2334
|
~onEventRegistrations: array<Internal.onEventRegistration>,
|
|
2459
|
-
~
|
|
2335
|
+
~addressStore: AddressStore.t,
|
|
2460
2336
|
~addresses: array<Internal.indexingAddress>,
|
|
2461
2337
|
~maxAddrInPartition,
|
|
2462
2338
|
~chainId,
|
|
@@ -2496,7 +2372,7 @@ let make = (
|
|
|
2496
2372
|
dependsOnAddresses: false,
|
|
2497
2373
|
onEventRegistrations: notDependingOnAddresses,
|
|
2498
2374
|
},
|
|
2499
|
-
|
|
2375
|
+
addresses: addressStore->AddressStore.emptySet,
|
|
2500
2376
|
mergeBlock: None,
|
|
2501
2377
|
dynamicContract: None,
|
|
2502
2378
|
mutPendingQueries: [],
|
|
@@ -2512,9 +2388,22 @@ let make = (
|
|
|
2512
2388
|
onEventRegistrations: normalRegistrations,
|
|
2513
2389
|
}
|
|
2514
2390
|
|
|
2515
|
-
|
|
2391
|
+
// Every address the chain indexes goes into the store — including ones whose
|
|
2392
|
+
// contract has no address-dependent events, so a later registration of the
|
|
2393
|
+
// same address still conflicts and the address is still persisted.
|
|
2394
|
+
addressStore
|
|
2395
|
+
->AddressStore.registerBatch(
|
|
2396
|
+
addresses->Array.map((contract): AddressStore.registration => {
|
|
2397
|
+
address: contract.address,
|
|
2398
|
+
contractName: contract.contractName,
|
|
2399
|
+
registrationBlock: contract.registrationBlock,
|
|
2400
|
+
}),
|
|
2401
|
+
)
|
|
2402
|
+
->ignore
|
|
2403
|
+
|
|
2516
2404
|
let dynamicContracts = Utils.Set.make()
|
|
2517
2405
|
let clientFilteredContracts = Utils.Set.make()
|
|
2406
|
+
let registeringSetsByContract = Dict.make()
|
|
2518
2407
|
|
|
2519
2408
|
addresses->Array.forEach(contract => {
|
|
2520
2409
|
let contractName = contract.contractName
|
|
@@ -2522,12 +2411,12 @@ let make = (
|
|
|
2522
2411
|
// Only addresses whose contract has events that depend on addresses get
|
|
2523
2412
|
// registered for active fetching via partitions.
|
|
2524
2413
|
if contractNamesWithNormalEvents->Utils.Set.has(contractName) {
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2414
|
+
if !(registeringSetsByContract->Dict.has(contractName)) {
|
|
2415
|
+
registeringSetsByContract->Dict.set(
|
|
2416
|
+
contractName,
|
|
2417
|
+
addressStore->AddressStore.makeSet(~contractName),
|
|
2418
|
+
)
|
|
2419
|
+
}
|
|
2531
2420
|
|
|
2532
2421
|
// Detect dynamic contracts by registrationBlock
|
|
2533
2422
|
if contract.registrationBlock !== -1 {
|
|
@@ -2541,8 +2430,8 @@ let make = (
|
|
|
2541
2430
|
// address list, or a dynamic contract restored from a large persisted set.
|
|
2542
2431
|
switch clientFilterAddressThreshold {
|
|
2543
2432
|
| Some(threshold) =>
|
|
2544
|
-
|
|
2545
|
-
let addressCount =
|
|
2433
|
+
registeringSetsByContract->Utils.Dict.forEachWithKey((set, contractName) => {
|
|
2434
|
+
let addressCount = set->AddressSet.size
|
|
2546
2435
|
if addressCount > threshold {
|
|
2547
2436
|
clientFilteredContracts->addClientFilteredContract(
|
|
2548
2437
|
~contractName,
|
|
@@ -2555,8 +2444,9 @@ let make = (
|
|
|
2555
2444
|
| None => ()
|
|
2556
2445
|
}
|
|
2557
2446
|
|
|
2558
|
-
let optimizedPartitions =
|
|
2559
|
-
~
|
|
2447
|
+
let optimizedPartitions = createPartitions(
|
|
2448
|
+
~registeringSetsByContract,
|
|
2449
|
+
~addressStore,
|
|
2560
2450
|
~dynamicContracts,
|
|
2561
2451
|
~clientFilteredContracts,
|
|
2562
2452
|
~normalSelection,
|
|
@@ -2564,6 +2454,7 @@ let make = (
|
|
|
2564
2454
|
~nextPartitionIndex=partitions->Array.length,
|
|
2565
2455
|
~existingPartitions=partitions, // wildcard partition(s) if any
|
|
2566
2456
|
~progressBlockNumber,
|
|
2457
|
+
~knownHeight,
|
|
2567
2458
|
)
|
|
2568
2459
|
|
|
2569
2460
|
if (
|
|
@@ -2604,7 +2495,6 @@ let make = (
|
|
|
2604
2495
|
|
|
2605
2496
|
let fetchState = {
|
|
2606
2497
|
optimizedPartitions,
|
|
2607
|
-
contractConfigs,
|
|
2608
2498
|
chainId,
|
|
2609
2499
|
startBlock,
|
|
2610
2500
|
endBlock,
|
|
@@ -2657,16 +2547,29 @@ Always recreates optimized partitions to avoid duplicate addresses:
|
|
|
2657
2547
|
- Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
|
|
2658
2548
|
- Non-wildcard with lfb > target: delete, track addresses for recreation
|
|
2659
2549
|
*/
|
|
2660
|
-
let rollback = (fetchState: t, ~
|
|
2661
|
-
// Step 1: Prune addresses registered after the target block. The pruned
|
|
2662
|
-
// then the source of truth for partition cleanup below — an address
|
|
2663
|
-
//
|
|
2664
|
-
|
|
2550
|
+
let rollback = (fetchState: t, ~addressStore: AddressStore.t, ~targetBlockNumber) => {
|
|
2551
|
+
// Step 1: Prune addresses registered after the target block. The pruned store
|
|
2552
|
+
// is then the source of truth for partition cleanup below — an address
|
|
2553
|
+
// survives iff `filterByRegistrationBlock` keeps it.
|
|
2554
|
+
addressStore->AddressStore.rollback(targetBlockNumber)->ignore
|
|
2665
2555
|
|
|
2666
2556
|
// Step 2: Categorize partitions
|
|
2667
2557
|
let keptPartitions = []
|
|
2668
2558
|
let nextKeptIdRef = ref(0)
|
|
2669
|
-
let
|
|
2559
|
+
let registeringSetsByContract: dict<AddressSet.t> = Dict.make()
|
|
2560
|
+
let collectForRecreation = (set: AddressSet.t) =>
|
|
2561
|
+
set
|
|
2562
|
+
->AddressSet.contractNames
|
|
2563
|
+
->Array.forEach(contractName => {
|
|
2564
|
+
let contractSet = set->AddressSet.filterByContracts([contractName])
|
|
2565
|
+
registeringSetsByContract->Dict.set(
|
|
2566
|
+
contractName,
|
|
2567
|
+
switch registeringSetsByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
|
|
2568
|
+
| Some(existing) => existing->AddressSet.merge(contractSet)
|
|
2569
|
+
| None => contractSet
|
|
2570
|
+
},
|
|
2571
|
+
)
|
|
2572
|
+
})
|
|
2670
2573
|
|
|
2671
2574
|
let partitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
|
|
2672
2575
|
for idx in 0 to partitions->Array.length - 1 {
|
|
@@ -2689,46 +2592,27 @@ let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetB
|
|
|
2689
2592
|
|
|
2690
2593
|
// Non-wildcard with lfb > target: delete, collect addresses for recreation
|
|
2691
2594
|
| _ if p.latestFetchedBlock.blockNumber > targetBlockNumber =>
|
|
2692
|
-
p.
|
|
2693
|
-
addresses->Array.forEach(address => {
|
|
2694
|
-
switch indexingAddresses->IndexingAddresses.get(address->Address.toString) {
|
|
2695
|
-
| Some(indexingContract) =>
|
|
2696
|
-
let registeringContracts =
|
|
2697
|
-
registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
|
|
2698
|
-
registeringContracts->Dict.set(address->Address.toString, indexingContract)
|
|
2699
|
-
| None => ()
|
|
2700
|
-
}
|
|
2701
|
-
})
|
|
2702
|
-
})
|
|
2595
|
+
collectForRecreation(p.addresses->AddressSet.filterByRegistrationBlock(targetBlockNumber))
|
|
2703
2596
|
|
|
2704
2597
|
// Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
|
|
2705
|
-
|
|
|
2598
|
+
| _ => {
|
|
2706
2599
|
// Cap mergeBlock at target
|
|
2707
2600
|
let mergeBlock = switch p.mergeBlock {
|
|
2708
2601
|
| Some(mergeBlock) if mergeBlock > targetBlockNumber => Some(targetBlockNumber)
|
|
2709
2602
|
| other => other
|
|
2710
2603
|
}
|
|
2711
2604
|
|
|
2712
|
-
// Drop addresses pruned from the
|
|
2713
|
-
let
|
|
2714
|
-
|
|
2715
|
-
let keptAddresses =
|
|
2716
|
-
addresses->Array.filter(address =>
|
|
2717
|
-
indexingAddresses->IndexingAddresses.get(address->Address.toString)->Option.isSome
|
|
2718
|
-
)
|
|
2719
|
-
if keptAddresses->Array.length > 0 {
|
|
2720
|
-
rollbackedAddressesByContractName->Dict.set(contractName, keptAddresses)
|
|
2721
|
-
}
|
|
2722
|
-
})
|
|
2605
|
+
// Drop addresses pruned from the store
|
|
2606
|
+
let rollbackedAddresses =
|
|
2607
|
+
p.addresses->AddressSet.filterByRegistrationBlock(targetBlockNumber)
|
|
2723
2608
|
|
|
2724
|
-
if !(
|
|
2609
|
+
if !(rollbackedAddresses->AddressSet.isEmpty) {
|
|
2725
2610
|
let id = nextKeptIdRef.contents->Int.toString
|
|
2726
2611
|
nextKeptIdRef := nextKeptIdRef.contents + 1
|
|
2727
2612
|
keptPartitions
|
|
2728
2613
|
->Array.push({
|
|
2729
|
-
...p,
|
|
2614
|
+
...p->withAddresses(rollbackedAddresses),
|
|
2730
2615
|
id,
|
|
2731
|
-
addressesByContractName: rollbackedAddressesByContractName,
|
|
2732
2616
|
mutPendingQueries: rollbackPendingQueries(p.mutPendingQueries, ~targetBlockNumber),
|
|
2733
2617
|
mergeBlock,
|
|
2734
2618
|
})
|
|
@@ -2739,8 +2623,9 @@ let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetB
|
|
|
2739
2623
|
}
|
|
2740
2624
|
|
|
2741
2625
|
// Step 3: Recreate partitions from deleted partition addresses
|
|
2742
|
-
let optimizedPartitions =
|
|
2743
|
-
~
|
|
2626
|
+
let optimizedPartitions = createPartitions(
|
|
2627
|
+
~registeringSetsByContract,
|
|
2628
|
+
~addressStore,
|
|
2744
2629
|
~dynamicContracts=fetchState.optimizedPartitions.dynamicContracts,
|
|
2745
2630
|
~clientFilteredContracts=fetchState.optimizedPartitions.clientFilteredContracts,
|
|
2746
2631
|
~normalSelection=fetchState.normalSelection,
|
|
@@ -2748,6 +2633,7 @@ let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetB
|
|
|
2748
2633
|
~nextPartitionIndex=nextKeptIdRef.contents,
|
|
2749
2634
|
~existingPartitions=keptPartitions,
|
|
2750
2635
|
~progressBlockNumber=targetBlockNumber,
|
|
2636
|
+
~knownHeight=fetchState.knownHeight,
|
|
2751
2637
|
)
|
|
2752
2638
|
|
|
2753
2639
|
// Step 4: Update state
|