envio 3.0.0-alpha.21 → 3.0.0-alpha.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/bin.mjs +2 -48
- package/evm.schema.json +67 -0
- package/fuel.schema.json +67 -0
- package/index.d.ts +822 -38
- package/index.js +5 -3
- package/package.json +10 -8
- package/rescript.json +5 -9
- package/src/Address.res +4 -5
- package/src/Address.res.mjs +9 -12
- package/src/Api.res +15 -0
- package/src/Api.res.mjs +20 -0
- package/src/Batch.res +32 -34
- package/src/Batch.res.mjs +172 -187
- package/src/Bin.res +89 -0
- package/src/Bin.res.mjs +97 -0
- package/src/ChainFetcher.res +33 -57
- package/src/ChainFetcher.res.mjs +197 -227
- package/src/ChainManager.res +6 -14
- package/src/ChainManager.res.mjs +74 -85
- package/src/ChainMap.res +14 -16
- package/src/ChainMap.res.mjs +38 -38
- package/src/Config.res +193 -135
- package/src/Config.res.mjs +566 -592
- package/src/Core.res +182 -0
- package/src/Core.res.mjs +207 -0
- package/src/Ecosystem.res +25 -4
- package/src/Ecosystem.res.mjs +12 -13
- package/src/Env.res +20 -13
- package/src/Env.res.mjs +124 -113
- package/src/EnvSafe.res +269 -0
- package/src/EnvSafe.res.mjs +296 -0
- package/src/EnvSafe.resi +18 -0
- package/src/Envio.res +37 -26
- package/src/Envio.res.mjs +59 -60
- package/src/ErrorHandling.res +2 -2
- package/src/ErrorHandling.res.mjs +15 -15
- package/src/EventConfigBuilder.res +219 -81
- package/src/EventConfigBuilder.res.mjs +259 -202
- package/src/EventProcessing.res +27 -38
- package/src/EventProcessing.res.mjs +165 -183
- package/src/EventUtils.res +11 -11
- package/src/EventUtils.res.mjs +21 -22
- package/src/EvmTypes.res +0 -1
- package/src/EvmTypes.res.mjs +5 -5
- package/src/FetchState.res +360 -256
- package/src/FetchState.res.mjs +958 -914
- package/src/GlobalState.res +365 -351
- package/src/GlobalState.res.mjs +958 -992
- package/src/GlobalStateManager.res +1 -2
- package/src/GlobalStateManager.res.mjs +36 -44
- package/src/HandlerLoader.res +107 -23
- package/src/HandlerLoader.res.mjs +128 -38
- package/src/HandlerRegister.res +127 -103
- package/src/HandlerRegister.res.mjs +164 -164
- package/src/HandlerRegister.resi +12 -4
- package/src/Hasura.res +35 -22
- package/src/Hasura.res.mjs +158 -167
- package/src/InMemoryStore.res +20 -27
- package/src/InMemoryStore.res.mjs +64 -80
- package/src/InMemoryTable.res +34 -39
- package/src/InMemoryTable.res.mjs +165 -170
- package/src/Internal.res +52 -33
- package/src/Internal.res.mjs +84 -81
- package/src/LazyLoader.res.mjs +55 -61
- package/src/LoadLayer.res +77 -78
- package/src/LoadLayer.res.mjs +160 -189
- package/src/LoadManager.res +16 -21
- package/src/LoadManager.res.mjs +79 -84
- package/src/LogSelection.res +236 -68
- package/src/LogSelection.res.mjs +211 -141
- package/src/Logging.res +13 -9
- package/src/Logging.res.mjs +130 -143
- package/src/Main.res +430 -51
- package/src/Main.res.mjs +530 -271
- package/src/Persistence.res +80 -84
- package/src/Persistence.res.mjs +131 -132
- package/src/PgStorage.res +294 -167
- package/src/PgStorage.res.mjs +799 -817
- package/src/Prometheus.res +50 -58
- package/src/Prometheus.res.mjs +345 -373
- package/src/ReorgDetection.res +22 -24
- package/src/ReorgDetection.res.mjs +100 -106
- package/src/SafeCheckpointTracking.res +7 -7
- package/src/SafeCheckpointTracking.res.mjs +40 -43
- package/src/SimulateItems.res +41 -49
- package/src/SimulateItems.res.mjs +257 -272
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +22 -26
- package/src/TableIndices.res +1 -2
- package/src/TableIndices.res.mjs +42 -48
- package/src/TestIndexer.res +196 -189
- package/src/TestIndexer.res.mjs +536 -536
- package/src/TestIndexerProxyStorage.res +16 -16
- package/src/TestIndexerProxyStorage.res.mjs +99 -122
- package/src/TestIndexerWorker.res +4 -0
- package/src/TestIndexerWorker.res.mjs +7 -0
- package/src/Throttler.res +3 -3
- package/src/Throttler.res.mjs +23 -24
- package/src/Time.res +1 -1
- package/src/Time.res.mjs +18 -21
- package/src/TopicFilter.res +3 -3
- package/src/TopicFilter.res.mjs +29 -30
- package/src/UserContext.res +93 -54
- package/src/UserContext.res.mjs +197 -182
- package/src/Utils.res +141 -86
- package/src/Utils.res.mjs +334 -295
- package/src/bindings/BigDecimal.res +0 -2
- package/src/bindings/BigDecimal.res.mjs +19 -23
- package/src/bindings/ClickHouse.res +28 -27
- package/src/bindings/ClickHouse.res.mjs +243 -240
- package/src/bindings/DateFns.res +11 -11
- package/src/bindings/DateFns.res.mjs +7 -7
- package/src/bindings/EventSource.res.mjs +2 -2
- package/src/bindings/Express.res +2 -5
- package/src/bindings/Hrtime.res +2 -2
- package/src/bindings/Hrtime.res.mjs +30 -32
- package/src/bindings/Lodash.res.mjs +1 -1
- package/src/bindings/NodeJs.res +14 -9
- package/src/bindings/NodeJs.res.mjs +20 -20
- package/src/bindings/Pino.res +8 -10
- package/src/bindings/Pino.res.mjs +40 -43
- package/src/bindings/Postgres.res +7 -5
- package/src/bindings/Postgres.res.mjs +9 -9
- package/src/bindings/PromClient.res +17 -2
- package/src/bindings/PromClient.res.mjs +30 -7
- package/src/bindings/SDSL.res.mjs +2 -2
- package/src/bindings/Viem.res +4 -4
- package/src/bindings/Viem.res.mjs +20 -22
- package/src/bindings/Vitest.res +1 -1
- package/src/bindings/Vitest.res.mjs +2 -2
- package/src/bindings/WebSocket.res +1 -1
- package/src/db/EntityHistory.res +9 -3
- package/src/db/EntityHistory.res.mjs +84 -59
- package/src/db/InternalTable.res +62 -60
- package/src/db/InternalTable.res.mjs +271 -203
- package/src/db/Schema.res +1 -2
- package/src/db/Schema.res.mjs +28 -32
- package/src/db/Table.res +28 -27
- package/src/db/Table.res.mjs +276 -292
- package/src/sources/EventRouter.res +21 -16
- package/src/sources/EventRouter.res.mjs +55 -57
- package/src/sources/Evm.res +17 -1
- package/src/sources/Evm.res.mjs +16 -8
- package/src/sources/EvmChain.res +15 -17
- package/src/sources/EvmChain.res.mjs +40 -42
- package/src/sources/Fuel.res +14 -1
- package/src/sources/Fuel.res.mjs +16 -8
- package/src/sources/FuelSDK.res +1 -1
- package/src/sources/FuelSDK.res.mjs +6 -8
- package/src/sources/HyperFuel.res +8 -10
- package/src/sources/HyperFuel.res.mjs +113 -123
- package/src/sources/HyperFuelClient.res.mjs +6 -7
- package/src/sources/HyperFuelSource.res +19 -20
- package/src/sources/HyperFuelSource.res.mjs +339 -356
- package/src/sources/HyperSync.res +11 -13
- package/src/sources/HyperSync.res.mjs +206 -220
- package/src/sources/HyperSyncClient.res +5 -7
- package/src/sources/HyperSyncClient.res.mjs +70 -75
- package/src/sources/HyperSyncHeightStream.res +8 -9
- package/src/sources/HyperSyncHeightStream.res.mjs +78 -86
- package/src/sources/HyperSyncJsonApi.res +18 -15
- package/src/sources/HyperSyncJsonApi.res.mjs +201 -231
- package/src/sources/HyperSyncSource.res +17 -21
- package/src/sources/HyperSyncSource.res.mjs +268 -290
- package/src/sources/Rpc.res +5 -5
- package/src/sources/Rpc.res.mjs +168 -192
- package/src/sources/RpcSource.res +166 -167
- package/src/sources/RpcSource.res.mjs +972 -1046
- package/src/sources/RpcWebSocketHeightStream.res +10 -11
- package/src/sources/RpcWebSocketHeightStream.res.mjs +131 -145
- package/src/sources/SimulateSource.res +1 -1
- package/src/sources/SimulateSource.res.mjs +35 -38
- package/src/sources/Source.res +1 -1
- package/src/sources/Source.res.mjs +3 -3
- package/src/sources/SourceManager.res +39 -20
- package/src/sources/SourceManager.res.mjs +340 -371
- package/src/sources/SourceManager.resi +2 -1
- package/src/sources/Svm.res +12 -5
- package/src/sources/Svm.res.mjs +44 -41
- package/src/tui/Tui.res +23 -12
- package/src/tui/Tui.res.mjs +292 -290
- package/src/tui/bindings/Ink.res +2 -4
- package/src/tui/bindings/Ink.res.mjs +35 -41
- package/src/tui/components/BufferedProgressBar.res +7 -7
- package/src/tui/components/BufferedProgressBar.res.mjs +46 -46
- package/src/tui/components/CustomHooks.res +1 -2
- package/src/tui/components/CustomHooks.res.mjs +102 -122
- package/src/tui/components/Messages.res +1 -2
- package/src/tui/components/Messages.res.mjs +38 -42
- package/src/tui/components/SyncETA.res +10 -11
- package/src/tui/components/SyncETA.res.mjs +178 -196
- package/src/tui/components/TuiData.res +1 -1
- package/src/tui/components/TuiData.res.mjs +7 -6
- package/src/vendored/Rest.res +52 -66
- package/src/vendored/Rest.res.mjs +324 -364
- package/svm.schema.json +67 -0
- package/src/Address.gen.ts +0 -8
- package/src/Config.gen.ts +0 -19
- package/src/Envio.gen.ts +0 -55
- package/src/EvmTypes.gen.ts +0 -6
- package/src/InMemoryStore.gen.ts +0 -6
- package/src/Internal.gen.ts +0 -64
- package/src/PgStorage.gen.ts +0 -10
- package/src/PgStorage.res.d.mts +0 -5
- package/src/Types.ts +0 -56
- package/src/bindings/BigDecimal.gen.ts +0 -14
- package/src/bindings/BigDecimal.res.d.mts +0 -5
- package/src/bindings/BigInt.gen.ts +0 -10
- package/src/bindings/BigInt.res +0 -70
- package/src/bindings/BigInt.res.d.mts +0 -5
- package/src/bindings/BigInt.res.mjs +0 -154
- package/src/bindings/Ethers.res.d.mts +0 -5
- package/src/bindings/Pino.gen.ts +0 -17
- package/src/bindings/Postgres.gen.ts +0 -8
- package/src/bindings/Postgres.res.d.mts +0 -5
- package/src/bindings/Promise.res +0 -67
- package/src/bindings/Promise.res.mjs +0 -26
- package/src/db/InternalTable.gen.ts +0 -36
- package/src/sources/HyperSyncClient.gen.ts +0 -19
package/src/FetchState.res
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
|
-
|
|
1
|
+
type contractConfig = {filterByAddresses: bool, startBlock: option<int>}
|
|
2
2
|
|
|
3
|
-
type
|
|
3
|
+
type indexingAddress = {
|
|
4
|
+
...Internal.indexingAddress,
|
|
5
|
+
effectiveStartBlock: int,
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
let deriveEffectiveStartBlock = (~registrationBlock: int, ~contractStartBlock: option<int>) => {
|
|
9
|
+
Pervasives.max(Pervasives.max(registrationBlock, 0), contractStartBlock->Option.getOr(0))
|
|
10
|
+
}
|
|
4
11
|
|
|
5
12
|
type blockNumberAndTimestamp = {
|
|
6
13
|
blockNumber: int,
|
|
@@ -56,7 +63,7 @@ type query = {
|
|
|
56
63
|
isChunk: bool,
|
|
57
64
|
selection: selection,
|
|
58
65
|
addressesByContractName: dict<array<Address.t>>,
|
|
59
|
-
|
|
66
|
+
indexingAddresses: dict<indexingAddress>,
|
|
60
67
|
}
|
|
61
68
|
|
|
62
69
|
// Calculate the chunk range from history using min-of-last-3-ranges heuristic
|
|
@@ -70,7 +77,9 @@ let getMinHistoryRange = (p: partition) => {
|
|
|
70
77
|
let getMinQueryRange = (partitions: array<partition>) => {
|
|
71
78
|
let min = ref(0)
|
|
72
79
|
for i in 0 to partitions->Array.length - 1 {
|
|
73
|
-
let p = partitions->
|
|
80
|
+
let p = partitions->Array.getUnsafe(i)
|
|
81
|
+
|
|
82
|
+
// Even if it's fetching, set dynamicContract field
|
|
74
83
|
let a = p.prevQueryRange
|
|
75
84
|
let b = p.prevPrevQueryRange
|
|
76
85
|
if a > 0 && (min.contents == 0 || a < min.contents) {
|
|
@@ -101,9 +110,9 @@ module OptimizedPartitions = {
|
|
|
101
110
|
|
|
102
111
|
@inline
|
|
103
112
|
let getOrThrow = (optimizedPartitions: t, ~partitionId) => {
|
|
104
|
-
switch optimizedPartitions.entities->
|
|
113
|
+
switch optimizedPartitions.entities->Dict.get(partitionId) {
|
|
105
114
|
| Some(p) => p
|
|
106
|
-
| None =>
|
|
115
|
+
| None => JsError.throwWithMessage(`Unexpected case: Couldn't find partition ${partitionId}`)
|
|
107
116
|
}
|
|
108
117
|
}
|
|
109
118
|
|
|
@@ -121,8 +130,8 @@ module OptimizedPartitions = {
|
|
|
121
130
|
) => {
|
|
122
131
|
let combinedAddresses =
|
|
123
132
|
p1.addressesByContractName
|
|
124
|
-
->
|
|
125
|
-
->
|
|
133
|
+
->Dict.getUnsafe(contractName)
|
|
134
|
+
->Array.concat(p2.addressesByContractName->Dict.getUnsafe(contractName))
|
|
126
135
|
|
|
127
136
|
let p1Below = p1.latestFetchedBlock.blockNumber < potentialMergeBlock
|
|
128
137
|
let p2Below = p2.latestFetchedBlock.blockNumber < potentialMergeBlock
|
|
@@ -133,15 +142,15 @@ module OptimizedPartitions = {
|
|
|
133
142
|
let continuingBase = switch (p1Below, p2Below) {
|
|
134
143
|
| (false, false) => p1
|
|
135
144
|
| (false, true) =>
|
|
136
|
-
completed->
|
|
145
|
+
completed->Array.push({...p2, mergeBlock: Some(potentialMergeBlock)})->ignore
|
|
137
146
|
p1
|
|
138
147
|
| (true, false) =>
|
|
139
|
-
completed->
|
|
148
|
+
completed->Array.push({...p1, mergeBlock: Some(potentialMergeBlock)})->ignore
|
|
140
149
|
p2
|
|
141
150
|
| (true, true) =>
|
|
142
|
-
completed->
|
|
143
|
-
completed->
|
|
144
|
-
let newId = nextPartitionIndexRef.contents->
|
|
151
|
+
completed->Array.push({...p1, mergeBlock: Some(potentialMergeBlock)})->ignore
|
|
152
|
+
completed->Array.push({...p2, mergeBlock: Some(potentialMergeBlock)})->ignore
|
|
153
|
+
let newId = nextPartitionIndexRef.contents->Int.toString
|
|
145
154
|
nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
|
|
146
155
|
let minRange = getMinQueryRange([p1, p2])
|
|
147
156
|
{
|
|
@@ -150,7 +159,7 @@ module OptimizedPartitions = {
|
|
|
150
159
|
selection: p1.selection,
|
|
151
160
|
latestFetchedBlock: {blockNumber: potentialMergeBlock, blockTimestamp: 0},
|
|
152
161
|
mergeBlock: None,
|
|
153
|
-
addressesByContractName:
|
|
162
|
+
addressesByContractName: Dict.make(), // set below
|
|
154
163
|
mutPendingQueries: [],
|
|
155
164
|
prevQueryRange: minRange,
|
|
156
165
|
prevPrevQueryRange: minRange,
|
|
@@ -159,18 +168,18 @@ module OptimizedPartitions = {
|
|
|
159
168
|
}
|
|
160
169
|
|
|
161
170
|
// Apply address split on the continuing partition
|
|
162
|
-
if combinedAddresses->
|
|
163
|
-
let addressesFull = combinedAddresses->
|
|
164
|
-
let addressesRest = combinedAddresses->
|
|
165
|
-
let abcFull =
|
|
166
|
-
abcFull->
|
|
167
|
-
let abcRest =
|
|
168
|
-
abcRest->
|
|
169
|
-
completed->
|
|
170
|
-
let restId = nextPartitionIndexRef.contents->
|
|
171
|
+
if combinedAddresses->Array.length > maxAddrInPartition {
|
|
172
|
+
let addressesFull = combinedAddresses->Array.slice(~start=0, ~end=maxAddrInPartition)
|
|
173
|
+
let addressesRest = combinedAddresses->Array.slice(~start=maxAddrInPartition)
|
|
174
|
+
let abcFull = Dict.make()
|
|
175
|
+
abcFull->Dict.set(contractName, addressesFull)
|
|
176
|
+
let abcRest = Dict.make()
|
|
177
|
+
abcRest->Dict.set(contractName, addressesRest)
|
|
178
|
+
completed->Array.push({...continuingBase, addressesByContractName: abcFull})->ignore
|
|
179
|
+
let restId = nextPartitionIndexRef.contents->Int.toString
|
|
171
180
|
nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
|
|
172
181
|
completed
|
|
173
|
-
->
|
|
182
|
+
->Array.push({
|
|
174
183
|
...continuingBase,
|
|
175
184
|
id: restId,
|
|
176
185
|
addressesByContractName: abcRest,
|
|
@@ -179,9 +188,9 @@ module OptimizedPartitions = {
|
|
|
179
188
|
->ignore
|
|
180
189
|
completed
|
|
181
190
|
} else {
|
|
182
|
-
let abc =
|
|
183
|
-
abc->
|
|
184
|
-
completed->
|
|
191
|
+
let abc = Dict.make()
|
|
192
|
+
abc->Dict.set(contractName, combinedAddresses)
|
|
193
|
+
completed->Array.push({...continuingBase, addressesByContractName: abc})->ignore
|
|
185
194
|
completed
|
|
186
195
|
}
|
|
187
196
|
}
|
|
@@ -193,7 +202,8 @@ module OptimizedPartitions = {
|
|
|
193
202
|
// quering the same block range multiple times
|
|
194
203
|
let tooFarBlockRange = 20_000
|
|
195
204
|
|
|
196
|
-
let ascSortFn = (a, b) =>
|
|
205
|
+
let ascSortFn = (a, b) =>
|
|
206
|
+
Int.compare(a.latestFetchedBlock.blockNumber, b.latestFetchedBlock.blockNumber)
|
|
197
207
|
|
|
198
208
|
/**
|
|
199
209
|
* Optimizes partitions by finding opportunities to merge partitions that
|
|
@@ -209,11 +219,11 @@ module OptimizedPartitions = {
|
|
|
209
219
|
~dynamicContracts: Utils.Set.t<string>,
|
|
210
220
|
) => {
|
|
211
221
|
let newPartitions = []
|
|
212
|
-
let mergingPartitions =
|
|
222
|
+
let mergingPartitions = Dict.make()
|
|
213
223
|
let nextPartitionIndexRef = ref(nextPartitionIndex)
|
|
214
224
|
|
|
215
225
|
for idx in 0 to partitions->Array.length - 1 {
|
|
216
|
-
let p = partitions->
|
|
226
|
+
let p = partitions->Array.getUnsafe(idx)
|
|
217
227
|
switch p {
|
|
218
228
|
// Since it's not a dynamic contract partition,
|
|
219
229
|
// there's no need for merge logic
|
|
@@ -225,10 +235,9 @@ module OptimizedPartitions = {
|
|
|
225
235
|
// TODO: Although there might be cases with too far away mergeBlock,
|
|
226
236
|
// which is worth merging
|
|
227
237
|
{mergeBlock: Some(_)} =>
|
|
228
|
-
newPartitions->
|
|
238
|
+
newPartitions->Array.push(p)->ignore
|
|
229
239
|
| {dynamicContract: Some(contractName)} =>
|
|
230
|
-
let pAddressesCount =
|
|
231
|
-
p.addressesByContractName->Js.Dict.unsafeGet(contractName)->Js.Array2.length
|
|
240
|
+
let pAddressesCount = p.addressesByContractName->Dict.getUnsafe(contractName)->Array.length
|
|
232
241
|
// Compute merge block: last pending query's toBlock, or lfb if idle
|
|
233
242
|
let potentialMergeBlock = switch p.mutPendingQueries->Utils.Array.last {
|
|
234
243
|
| Some({isChunk: true, toBlock: Some(toBlock)}) => Some(toBlock)
|
|
@@ -236,10 +245,10 @@ module OptimizedPartitions = {
|
|
|
236
245
|
| None => Some(p.latestFetchedBlock.blockNumber)
|
|
237
246
|
}
|
|
238
247
|
switch potentialMergeBlock {
|
|
239
|
-
| None => newPartitions->
|
|
248
|
+
| None => newPartitions->Array.push(p)->ignore
|
|
240
249
|
| Some(potentialMergeBlock) =>
|
|
241
250
|
if pAddressesCount >= maxAddrInPartition {
|
|
242
|
-
newPartitions->
|
|
251
|
+
newPartitions->Array.push(p)->ignore
|
|
243
252
|
} else {
|
|
244
253
|
let partitionsByMergeBlock =
|
|
245
254
|
mergingPartitions->Utils.Dict.getOrInsertEmptyDict(contractName)
|
|
@@ -256,7 +265,7 @@ module OptimizedPartitions = {
|
|
|
256
265
|
~nextPartitionIndexRef,
|
|
257
266
|
)
|
|
258
267
|
for i in 0 to result->Array.length - 2 {
|
|
259
|
-
newPartitions->
|
|
268
|
+
newPartitions->Array.push(result->Array.getUnsafe(i))->ignore
|
|
260
269
|
}
|
|
261
270
|
partitionsByMergeBlock->Utils.Dict.setByInt(
|
|
262
271
|
potentialMergeBlock,
|
|
@@ -269,36 +278,36 @@ module OptimizedPartitions = {
|
|
|
269
278
|
}
|
|
270
279
|
}
|
|
271
280
|
|
|
272
|
-
let merginDynamicContracts = mergingPartitions->
|
|
281
|
+
let merginDynamicContracts = mergingPartitions->Dict.keysToArray
|
|
273
282
|
for idx in 0 to merginDynamicContracts->Array.length - 1 {
|
|
274
|
-
let contractName = merginDynamicContracts->
|
|
275
|
-
let partitionsByMergeBlock = mergingPartitions->
|
|
283
|
+
let contractName = merginDynamicContracts->Array.getUnsafe(idx)
|
|
284
|
+
let partitionsByMergeBlock = mergingPartitions->Dict.getUnsafe(contractName)
|
|
276
285
|
// JS engine automatically sorts number keys in objects
|
|
277
|
-
let ascPartitionKeys = partitionsByMergeBlock->
|
|
286
|
+
let ascPartitionKeys = partitionsByMergeBlock->Dict.keysToArray
|
|
278
287
|
|
|
279
288
|
// But -1 is placed last...
|
|
280
|
-
if ascPartitionKeys->
|
|
289
|
+
if ascPartitionKeys->Array.getUnsafe(ascPartitionKeys->Array.length - 1) === "-1" {
|
|
281
290
|
ascPartitionKeys
|
|
282
|
-
->
|
|
291
|
+
->Array.unshift(ascPartitionKeys->Array.pop->Option.getUnsafe)
|
|
283
292
|
->ignore
|
|
284
293
|
}
|
|
285
294
|
let currentPRef = ref(
|
|
286
|
-
partitionsByMergeBlock->
|
|
295
|
+
partitionsByMergeBlock->Dict.getUnsafe(ascPartitionKeys->Utils.Array.firstUnsafe),
|
|
287
296
|
)
|
|
288
297
|
let currentPMergeBlockRef = ref(
|
|
289
298
|
ascPartitionKeys->Utils.Array.firstUnsafe->Int.fromString->Option.getUnsafe,
|
|
290
299
|
)
|
|
291
300
|
let nextJdx = ref(1)
|
|
292
301
|
while nextJdx.contents < ascPartitionKeys->Array.length {
|
|
293
|
-
let nextKey = ascPartitionKeys->
|
|
302
|
+
let nextKey = ascPartitionKeys->Array.getUnsafe(nextJdx.contents)
|
|
294
303
|
let currentP = currentPRef.contents
|
|
295
|
-
let nextP = partitionsByMergeBlock->
|
|
304
|
+
let nextP = partitionsByMergeBlock->Dict.getUnsafe(nextKey)
|
|
296
305
|
let nextPMergeBlock = nextKey->Int.fromString->Option.getUnsafe
|
|
297
306
|
let currentPMergeBlock = currentPMergeBlockRef.contents
|
|
298
307
|
|
|
299
308
|
let isTooFar = currentPMergeBlock + tooFarBlockRange < nextPMergeBlock
|
|
300
309
|
if isTooFar {
|
|
301
|
-
newPartitions->
|
|
310
|
+
newPartitions->Array.push(currentP)->ignore
|
|
302
311
|
currentPRef := nextP
|
|
303
312
|
currentPMergeBlockRef := nextPMergeBlock
|
|
304
313
|
} else {
|
|
@@ -311,7 +320,7 @@ module OptimizedPartitions = {
|
|
|
311
320
|
~nextPartitionIndexRef,
|
|
312
321
|
)
|
|
313
322
|
for i in 0 to result->Array.length - 2 {
|
|
314
|
-
newPartitions->
|
|
323
|
+
newPartitions->Array.push(result->Array.getUnsafe(i))->ignore
|
|
315
324
|
}
|
|
316
325
|
currentPRef := result->Utils.Array.lastUnsafe
|
|
317
326
|
currentPMergeBlockRef := nextPMergeBlock
|
|
@@ -320,19 +329,19 @@ module OptimizedPartitions = {
|
|
|
320
329
|
nextJdx := nextJdx.contents + 1
|
|
321
330
|
}
|
|
322
331
|
|
|
323
|
-
newPartitions->
|
|
332
|
+
newPartitions->Array.push(currentPRef.contents)->ignore
|
|
324
333
|
}
|
|
325
334
|
|
|
326
335
|
// Sort partitions by latestFetchedBlock ascending
|
|
327
|
-
let _ = newPartitions->
|
|
336
|
+
let _ = newPartitions->Array.sort(ascSortFn)
|
|
328
337
|
|
|
329
338
|
let partitionsCount = newPartitions->Array.length
|
|
330
339
|
let idsInAscOrder = Belt.Array.makeUninitializedUnsafe(partitionsCount)
|
|
331
|
-
let entities =
|
|
340
|
+
let entities = Dict.make()
|
|
332
341
|
for idx in 0 to partitionsCount - 1 {
|
|
333
|
-
let p = newPartitions->
|
|
334
|
-
idsInAscOrder->
|
|
335
|
-
entities->
|
|
342
|
+
let p = newPartitions->Array.getUnsafe(idx)
|
|
343
|
+
idsInAscOrder->Array.setUnsafe(idx, p.id)
|
|
344
|
+
entities->Dict.set(p.id, p)
|
|
336
345
|
}
|
|
337
346
|
|
|
338
347
|
{
|
|
@@ -361,7 +370,7 @@ module OptimizedPartitions = {
|
|
|
361
370
|
pq.fetchedBlock !== None && pq.fromBlock <= latestFetchedBlock.contents.blockNumber + 1
|
|
362
371
|
}
|
|
363
372
|
) {
|
|
364
|
-
let removedQuery = mutPendingQueries->
|
|
373
|
+
let removedQuery = mutPendingQueries->Array.shift->Option.getUnsafe
|
|
365
374
|
latestFetchedBlock := removedQuery.fetchedBlock->Option.getUnsafe
|
|
366
375
|
}
|
|
367
376
|
|
|
@@ -372,7 +381,7 @@ module OptimizedPartitions = {
|
|
|
372
381
|
let idxRef = ref(0)
|
|
373
382
|
let pendingQueryRef = ref(None)
|
|
374
383
|
while idxRef.contents < p.mutPendingQueries->Array.length && pendingQueryRef.contents === None {
|
|
375
|
-
let pq = p.mutPendingQueries->
|
|
384
|
+
let pq = p.mutPendingQueries->Array.getUnsafe(idxRef.contents)
|
|
376
385
|
if pq.fromBlock === fromBlock {
|
|
377
386
|
pendingQueryRef := Some(pq)
|
|
378
387
|
}
|
|
@@ -381,7 +390,7 @@ module OptimizedPartitions = {
|
|
|
381
390
|
switch pendingQueryRef.contents {
|
|
382
391
|
| Some(pq) => pq
|
|
383
392
|
| None =>
|
|
384
|
-
|
|
393
|
+
JsError.throwWithMessage(
|
|
385
394
|
`Pending query not found for partition ${p.id} fromBlock ${fromBlock->Int.toString}`,
|
|
386
395
|
)
|
|
387
396
|
}
|
|
@@ -448,12 +457,12 @@ module OptimizedPartitions = {
|
|
|
448
457
|
: p.latestBlockRangeUpdateBlock,
|
|
449
458
|
}
|
|
450
459
|
|
|
451
|
-
mutEntities->
|
|
460
|
+
mutEntities->Dict.set(p.id, updatedMainPartition)
|
|
452
461
|
}
|
|
453
462
|
|
|
454
463
|
// Re-optimize to maintain sorted order and apply optimizations
|
|
455
464
|
make(
|
|
456
|
-
~partitions=mutEntities->
|
|
465
|
+
~partitions=mutEntities->Dict.valuesToArray,
|
|
457
466
|
~maxAddrInPartition=optimizedPartitions.maxAddrInPartition,
|
|
458
467
|
~nextPartitionIndex=optimizedPartitions.nextPartitionIndex,
|
|
459
468
|
~dynamicContracts=optimizedPartitions.dynamicContracts,
|
|
@@ -463,7 +472,7 @@ module OptimizedPartitions = {
|
|
|
463
472
|
@inline
|
|
464
473
|
let getLatestFullyFetchedBlock = (optimizedPartitions: t) => {
|
|
465
474
|
switch optimizedPartitions.idsInAscOrder->Array.get(0) {
|
|
466
|
-
| Some(id) => Some((optimizedPartitions.entities->
|
|
475
|
+
| Some(id) => Some((optimizedPartitions.entities->Dict.getUnsafe(id)).latestFetchedBlock)
|
|
467
476
|
| None => None
|
|
468
477
|
}
|
|
469
478
|
}
|
|
@@ -475,7 +484,7 @@ type t = {
|
|
|
475
484
|
endBlock: option<int>,
|
|
476
485
|
normalSelection: selection,
|
|
477
486
|
// By address
|
|
478
|
-
|
|
487
|
+
indexingAddresses: dict<indexingAddress>,
|
|
479
488
|
// By contract name
|
|
480
489
|
contractConfigs: dict<contractConfig>,
|
|
481
490
|
// Not used for logic - only metadata
|
|
@@ -533,18 +542,18 @@ let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
|
|
|
533
542
|
Comparitor for two events from the same chain. No need for chain id or timestamp
|
|
534
543
|
*/
|
|
535
544
|
let compareBufferItem = (a: Internal.item, b: Internal.item) => {
|
|
536
|
-
let
|
|
537
|
-
if
|
|
538
|
-
a->Internal.getItemLogIndex
|
|
545
|
+
let blockOrdering = Int.compare(a->Internal.getItemBlockNumber, b->Internal.getItemBlockNumber)
|
|
546
|
+
if blockOrdering === Ordering.equal {
|
|
547
|
+
Int.compare(a->Internal.getItemLogIndex, b->Internal.getItemLogIndex)
|
|
539
548
|
} else {
|
|
540
|
-
|
|
549
|
+
blockOrdering
|
|
541
550
|
}
|
|
542
551
|
}
|
|
543
552
|
|
|
544
553
|
// Some big number which should be bigger than any log index
|
|
545
554
|
let blockItemLogIndex = 16777216
|
|
546
555
|
|
|
547
|
-
let numAddresses = fetchState => fetchState.
|
|
556
|
+
let numAddresses = fetchState => fetchState.indexingAddresses->Utils.Dict.size
|
|
548
557
|
|
|
549
558
|
/*
|
|
550
559
|
Update fetchState, merge registers and recompute derived values.
|
|
@@ -553,7 +562,7 @@ Runs partition optimization when partitions change.
|
|
|
553
562
|
let updateInternal = (
|
|
554
563
|
fetchState: t,
|
|
555
564
|
~optimizedPartitions=fetchState.optimizedPartitions,
|
|
556
|
-
~
|
|
565
|
+
~indexingAddresses=fetchState.indexingAddresses,
|
|
557
566
|
~mutItems=?,
|
|
558
567
|
~blockLag=fetchState.blockLag,
|
|
559
568
|
~knownHeight=fetchState.knownHeight,
|
|
@@ -603,7 +612,7 @@ let updateInternal = (
|
|
|
603
612
|
latestOnBlockBlockNumber := blockNumber
|
|
604
613
|
|
|
605
614
|
for configIdx in 0 to onBlockConfigs->Array.length - 1 {
|
|
606
|
-
let onBlockConfig = onBlockConfigs->
|
|
615
|
+
let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
|
|
607
616
|
|
|
608
617
|
let handlerStartBlock = switch onBlockConfig.startBlock {
|
|
609
618
|
| Some(startBlock) => startBlock
|
|
@@ -644,14 +653,17 @@ let updateInternal = (
|
|
|
644
653
|
targetBufferSize: fetchState.targetBufferSize,
|
|
645
654
|
optimizedPartitions,
|
|
646
655
|
latestOnBlockBlockNumber,
|
|
647
|
-
|
|
656
|
+
indexingAddresses,
|
|
648
657
|
blockLag,
|
|
649
658
|
knownHeight,
|
|
650
659
|
buffer: switch mutItemsRef.contents {
|
|
651
660
|
// Theoretically it could be faster to asume that
|
|
652
661
|
// the items are sorted, but there are cases
|
|
653
662
|
// when the data source returns them unsorted
|
|
654
|
-
| Some(mutItems) =>
|
|
663
|
+
| Some(mutItems) => {
|
|
664
|
+
mutItems->Array.sort(compareBufferItem)
|
|
665
|
+
mutItems
|
|
666
|
+
}
|
|
655
667
|
| None => fetchState.buffer
|
|
656
668
|
},
|
|
657
669
|
firstEventBlock: fetchState.firstEventBlock,
|
|
@@ -669,7 +681,7 @@ let updateInternal = (
|
|
|
669
681
|
~blockNumber=updatedFetchState->bufferBlockNumber,
|
|
670
682
|
~chainId=fetchState.chainId,
|
|
671
683
|
)
|
|
672
|
-
if
|
|
684
|
+
if indexingAddresses !== fetchState.indexingAddresses {
|
|
673
685
|
Prometheus.IndexingAddresses.set(
|
|
674
686
|
~addressesCount=updatedFetchState->numAddresses,
|
|
675
687
|
~chainId=fetchState.chainId,
|
|
@@ -681,8 +693,8 @@ let updateInternal = (
|
|
|
681
693
|
|
|
682
694
|
let warnDifferentContractType = (
|
|
683
695
|
fetchState,
|
|
684
|
-
~existingContract:
|
|
685
|
-
~dc:
|
|
696
|
+
~existingContract: indexingAddress,
|
|
697
|
+
~dc: indexingAddress,
|
|
686
698
|
) => {
|
|
687
699
|
let logger = Logging.createChild(
|
|
688
700
|
~params={
|
|
@@ -697,23 +709,20 @@ let warnDifferentContractType = (
|
|
|
697
709
|
|
|
698
710
|
let addressesByContractNameCount = (addressesByContractName: dict<array<Address.t>>) => {
|
|
699
711
|
let numAddresses = ref(0)
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
numAddresses :=
|
|
704
|
-
numAddresses.contents + addressesByContractName->Js.Dict.unsafeGet(contractName)->Array.length
|
|
705
|
-
}
|
|
712
|
+
addressesByContractName->Utils.Dict.forEach(addresses => {
|
|
713
|
+
numAddresses := numAddresses.contents + addresses->Array.length
|
|
714
|
+
})
|
|
706
715
|
numAddresses.contents
|
|
707
716
|
}
|
|
708
717
|
|
|
709
718
|
let addressesByContractNameGetAll = (addressesByContractName: dict<array<Address.t>>) => {
|
|
710
|
-
let all =
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
}
|
|
716
|
-
all
|
|
719
|
+
let all = []
|
|
720
|
+
addressesByContractName->Utils.Dict.forEach(addresses => {
|
|
721
|
+
for idx in 0 to addresses->Array.length - 1 {
|
|
722
|
+
all->Array.push(addresses->Array.getUnsafe(idx))->ignore
|
|
723
|
+
}
|
|
724
|
+
})
|
|
725
|
+
all
|
|
717
726
|
}
|
|
718
727
|
|
|
719
728
|
/**
|
|
@@ -724,7 +733,7 @@ Returns OptimizedPartitions.t directly.
|
|
|
724
733
|
(Dynamic partitions are merged by OptimizedPartitions.make automatically)
|
|
725
734
|
*/
|
|
726
735
|
let createPartitionsFromIndexingAddresses = (
|
|
727
|
-
~registeringContractsByContract: dict<dict<
|
|
736
|
+
~registeringContractsByContract: dict<dict<indexingAddress>>,
|
|
728
737
|
~contractConfigs: dict<contractConfig>,
|
|
729
738
|
~dynamicContracts: Utils.Set.t<string>,
|
|
730
739
|
~normalSelection: selection,
|
|
@@ -740,36 +749,36 @@ OptimizedPartitions.t => {
|
|
|
740
749
|
let dynamicPartitions = []
|
|
741
750
|
let nonDynamicPartitions = []
|
|
742
751
|
|
|
743
|
-
let contractNames = registeringContractsByContract->
|
|
744
|
-
for cIdx in 0 to contractNames->
|
|
745
|
-
let contractName = contractNames->
|
|
746
|
-
let registeringContracts = registeringContractsByContract->
|
|
752
|
+
let contractNames = registeringContractsByContract->Dict.keysToArray
|
|
753
|
+
for cIdx in 0 to contractNames->Array.length - 1 {
|
|
754
|
+
let contractName = contractNames->Array.getUnsafe(cIdx)
|
|
755
|
+
let registeringContracts = registeringContractsByContract->Dict.getUnsafe(contractName)
|
|
747
756
|
let addresses =
|
|
748
|
-
registeringContracts->
|
|
757
|
+
registeringContracts->Dict.keysToArray->(Utils.magic: array<string> => array<Address.t>)
|
|
749
758
|
|
|
750
759
|
// Can unsafely get it, because we already filtered out the contracts
|
|
751
760
|
// that don't have any events to fetch
|
|
752
|
-
let contractConfig = contractConfigs->
|
|
761
|
+
let contractConfig = contractConfigs->Dict.getUnsafe(contractName)
|
|
753
762
|
let isDynamic = dynamicContracts->Utils.Set.has(contractName)
|
|
754
763
|
let partitions = isDynamic ? dynamicPartitions : nonDynamicPartitions
|
|
755
764
|
|
|
756
|
-
let byStartBlock =
|
|
765
|
+
let byStartBlock = Dict.make()
|
|
757
766
|
for jdx in 0 to addresses->Array.length - 1 {
|
|
758
|
-
let address = addresses->
|
|
759
|
-
let indexingContract = registeringContracts->
|
|
760
|
-
byStartBlock->Utils.Dict.push(indexingContract.
|
|
767
|
+
let address = addresses->Array.getUnsafe(jdx)
|
|
768
|
+
let indexingContract = registeringContracts->Dict.getUnsafe(address->Address.toString)
|
|
769
|
+
byStartBlock->Utils.Dict.push(indexingContract.effectiveStartBlock->Int.toString, address)
|
|
761
770
|
}
|
|
762
771
|
|
|
763
772
|
// Will be in ASC order by JS spec
|
|
764
|
-
let ascKeys = byStartBlock->
|
|
773
|
+
let ascKeys = byStartBlock->Dict.keysToArray
|
|
765
774
|
let initialKey = ascKeys->Utils.Array.firstUnsafe
|
|
766
775
|
|
|
767
776
|
let startBlockRef = ref(initialKey->Int.fromString->Option.getUnsafe)
|
|
768
|
-
let addressesRef = ref(byStartBlock->
|
|
777
|
+
let addressesRef = ref(byStartBlock->Dict.getUnsafe(initialKey))
|
|
769
778
|
|
|
770
|
-
for idx in 0 to ascKeys->
|
|
779
|
+
for idx in 0 to ascKeys->Array.length - 1 {
|
|
771
780
|
let maybeNextStartBlockKey =
|
|
772
|
-
ascKeys->
|
|
781
|
+
ascKeys->Array.getUnsafe(idx + 1)->(Utils.magic: string => option<string>)
|
|
773
782
|
|
|
774
783
|
// For this case we can't filter out events earlier than contract registration
|
|
775
784
|
// on the client side, so we need to keep the old logic of creating
|
|
@@ -785,14 +794,12 @@ OptimizedPartitions.t => {
|
|
|
785
794
|
nextStartBlock - startBlockRef.contents < OptimizedPartitions.tooFarBlockRange
|
|
786
795
|
|
|
787
796
|
// If dynamic contract registration are close to eachother
|
|
788
|
-
// and it's possible to use dc.
|
|
797
|
+
// and it's possible to use dc.effectiveStartBlock to filter out events on client side
|
|
789
798
|
// then we can optimize the number of partitions,
|
|
790
799
|
// by putting dcs with different startBlocks in the same partition
|
|
791
800
|
if shouldJoinCurrentStartBlock {
|
|
792
801
|
addressesRef :=
|
|
793
|
-
addressesRef.contents->Array.concat(
|
|
794
|
-
byStartBlock->Js.Dict.unsafeGet(nextStartBlockKey),
|
|
795
|
-
)
|
|
802
|
+
addressesRef.contents->Array.concat(byStartBlock->Dict.getUnsafe(nextStartBlockKey))
|
|
796
803
|
false
|
|
797
804
|
} else {
|
|
798
805
|
true
|
|
@@ -807,12 +814,11 @@ OptimizedPartitions.t => {
|
|
|
807
814
|
blockTimestamp: 0,
|
|
808
815
|
}
|
|
809
816
|
while addressesRef.contents->Array.length > 0 {
|
|
810
|
-
let pAddresses =
|
|
811
|
-
|
|
812
|
-
addressesRef.contents = addressesRef.contents->Js.Array2.sliceFrom(maxAddrInPartition)
|
|
817
|
+
let pAddresses = addressesRef.contents->Array.slice(~start=0, ~end=maxAddrInPartition)
|
|
818
|
+
addressesRef.contents = addressesRef.contents->Array.slice(~start=maxAddrInPartition)
|
|
813
819
|
|
|
814
|
-
let addressesByContractName =
|
|
815
|
-
addressesByContractName->
|
|
820
|
+
let addressesByContractName = Dict.make()
|
|
821
|
+
addressesByContractName->Dict.set(contractName, pAddresses)
|
|
816
822
|
partitions->Array.push({
|
|
817
823
|
id: nextPartitionIndexRef.contents->Int.toString,
|
|
818
824
|
latestFetchedBlock,
|
|
@@ -832,7 +838,7 @@ OptimizedPartitions.t => {
|
|
|
832
838
|
| None => ()
|
|
833
839
|
| Some(nextStartBlockKey) => {
|
|
834
840
|
startBlockRef := nextStartBlockKey->Int.fromString->Option.getUnsafe
|
|
835
|
-
addressesRef := byStartBlock->
|
|
841
|
+
addressesRef := byStartBlock->Dict.getUnsafe(nextStartBlockKey)
|
|
836
842
|
}
|
|
837
843
|
}
|
|
838
844
|
}
|
|
@@ -844,13 +850,13 @@ OptimizedPartitions.t => {
|
|
|
844
850
|
|
|
845
851
|
if nonDynamicPartitions->Array.length > 0 {
|
|
846
852
|
// Sort non-dynamic partitions by latestFetchedBlock ascending
|
|
847
|
-
let _ = nonDynamicPartitions->
|
|
853
|
+
let _ = nonDynamicPartitions->Array.sort(OptimizedPartitions.ascSortFn)
|
|
848
854
|
|
|
849
|
-
let currentPRef = ref(nonDynamicPartitions->
|
|
855
|
+
let currentPRef = ref(nonDynamicPartitions->Array.getUnsafe(0))
|
|
850
856
|
let nextIdx = ref(1)
|
|
851
857
|
|
|
852
858
|
while nextIdx.contents < nonDynamicPartitions->Array.length {
|
|
853
|
-
let nextP = nonDynamicPartitions->
|
|
859
|
+
let nextP = nonDynamicPartitions->Array.getUnsafe(nextIdx.contents)
|
|
854
860
|
let currentP = currentPRef.contents
|
|
855
861
|
let currentPBlock = currentP.latestFetchedBlock.blockNumber
|
|
856
862
|
let nextPBlock = nextP.latestFetchedBlock.blockNumber
|
|
@@ -862,33 +868,34 @@ OptimizedPartitions.t => {
|
|
|
862
868
|
|
|
863
869
|
if totalCount > maxAddrInPartition {
|
|
864
870
|
// Exceeds address limit - don't merge, keep partitions separate
|
|
865
|
-
mergedNonDynamic->
|
|
871
|
+
mergedNonDynamic->Array.push(currentP)->ignore
|
|
866
872
|
currentPRef := nextP
|
|
867
873
|
} else {
|
|
868
874
|
// Build merged addresses using Array.concat (non-mutating)
|
|
869
875
|
let mergedAddresses = nextP.addressesByContractName->Utils.Dict.shallowCopy
|
|
870
|
-
let currentContractNames = currentP.addressesByContractName->
|
|
871
|
-
for jdx in 0 to currentContractNames->
|
|
872
|
-
let cn = currentContractNames->
|
|
873
|
-
let currentAddrs = currentP.addressesByContractName->
|
|
876
|
+
let currentContractNames = currentP.addressesByContractName->Dict.keysToArray
|
|
877
|
+
for jdx in 0 to currentContractNames->Array.length - 1 {
|
|
878
|
+
let cn = currentContractNames->Array.getUnsafe(jdx)
|
|
879
|
+
let currentAddrs = currentP.addressesByContractName->Dict.getUnsafe(cn)
|
|
874
880
|
switch mergedAddresses->Utils.Dict.dangerouslyGetNonOption(cn) {
|
|
875
881
|
| Some(existingAddrs) =>
|
|
876
882
|
// Use concat (non-mutating) to avoid corrupting nextP's arrays
|
|
877
|
-
mergedAddresses->
|
|
878
|
-
| None => mergedAddresses->
|
|
883
|
+
mergedAddresses->Dict.set(cn, existingAddrs->Array.concat(currentAddrs))
|
|
884
|
+
| None => mergedAddresses->Dict.set(cn, currentAddrs)
|
|
879
885
|
}
|
|
880
886
|
}
|
|
881
887
|
|
|
882
|
-
let nextContractName =
|
|
888
|
+
let nextContractName =
|
|
889
|
+
nextP.addressesByContractName->Dict.keysToArray->Utils.Array.firstUnsafe
|
|
883
890
|
let hasFilterByAddresses = (
|
|
884
|
-
contractConfigs->
|
|
891
|
+
contractConfigs->Dict.getUnsafe(nextContractName)
|
|
885
892
|
).filterByAddresses
|
|
886
893
|
let isTooFar = currentPBlock + OptimizedPartitions.tooFarBlockRange < nextPBlock
|
|
887
894
|
|
|
888
895
|
if isTooFar || hasFilterByAddresses {
|
|
889
896
|
// Too far or address-filtered: mergeBlock on current, merge addresses into next
|
|
890
897
|
mergedNonDynamic
|
|
891
|
-
->
|
|
898
|
+
->Array.push({
|
|
892
899
|
...currentP,
|
|
893
900
|
mergeBlock: currentPBlock < nextPBlock ? Some(nextPBlock) : None,
|
|
894
901
|
})
|
|
@@ -909,14 +916,14 @@ OptimizedPartitions.t => {
|
|
|
909
916
|
nextIdx := nextIdx.contents + 1
|
|
910
917
|
}
|
|
911
918
|
|
|
912
|
-
mergedNonDynamic->
|
|
919
|
+
mergedNonDynamic->Array.push(currentPRef.contents)->ignore
|
|
913
920
|
}
|
|
914
921
|
|
|
915
|
-
let mergedPartitions = mergedNonDynamic->
|
|
922
|
+
let mergedPartitions = mergedNonDynamic->Array.concat(dynamicPartitions)
|
|
916
923
|
|
|
917
924
|
// Final step: concat existing partitions with phase 1+2 result and call OptimizedPartitions.make
|
|
918
925
|
OptimizedPartitions.make(
|
|
919
|
-
~partitions=existingPartitions->
|
|
926
|
+
~partitions=existingPartitions->Array.concat(mergedPartitions),
|
|
920
927
|
~maxAddrInPartition,
|
|
921
928
|
~nextPartitionIndex=nextPartitionIndexRef.contents,
|
|
922
929
|
~dynamicContracts,
|
|
@@ -931,31 +938,45 @@ let registerDynamicContracts = (
|
|
|
931
938
|
) => {
|
|
932
939
|
if fetchState.normalSelection.eventConfigs->Utils.Array.isEmpty {
|
|
933
940
|
// Can the normalSelection be empty?
|
|
934
|
-
|
|
941
|
+
JsError.throwWithMessage(
|
|
935
942
|
"Invalid configuration. No events to fetch for the dynamic contract registration.",
|
|
936
943
|
)
|
|
937
944
|
}
|
|
938
945
|
|
|
939
|
-
let
|
|
940
|
-
let registeringContractsByContract: dict<dict<
|
|
946
|
+
let indexingAddresses = fetchState.indexingAddresses
|
|
947
|
+
let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
|
|
941
948
|
let earliestRegisteringEventBlockNumber = ref(%raw(`Infinity`))
|
|
942
949
|
let hasDCWithFilterByAddresses = ref(false)
|
|
950
|
+
// Addresses registered for contracts without matching events. These are not
|
|
951
|
+
// added to partitions, but they are tracked on fetchState.indexingAddresses
|
|
952
|
+
// so that later conflicting registrations are detected, and are persisted
|
|
953
|
+
// to envio_addresses so they can be picked up on restart with updated config.
|
|
954
|
+
let noEventsAddresses: dict<indexingAddress> = Dict.make()
|
|
943
955
|
|
|
944
956
|
for itemIdx in 0 to items->Array.length - 1 {
|
|
945
|
-
let item = items->
|
|
957
|
+
let item = items->Array.getUnsafe(itemIdx)
|
|
946
958
|
switch item->Internal.getItemDcs {
|
|
947
959
|
| None => ()
|
|
948
960
|
| Some(dcs) =>
|
|
949
961
|
let idx = ref(0)
|
|
950
962
|
while idx.contents < dcs->Array.length {
|
|
951
|
-
let dc = dcs->
|
|
963
|
+
let dc = dcs->Array.getUnsafe(idx.contents)
|
|
952
964
|
|
|
953
965
|
let shouldRemove = ref(false)
|
|
954
966
|
|
|
955
967
|
switch fetchState.contractConfigs->Utils.Dict.dangerouslyGetNonOption(dc.contractName) {
|
|
956
|
-
| Some({filterByAddresses}) =>
|
|
968
|
+
| Some({filterByAddresses, startBlock: contractStartBlock}) =>
|
|
969
|
+
let dcWithStartBlock: indexingAddress = {
|
|
970
|
+
address: dc.address,
|
|
971
|
+
contractName: dc.contractName,
|
|
972
|
+
registrationBlock: dc.registrationBlock,
|
|
973
|
+
effectiveStartBlock: deriveEffectiveStartBlock(
|
|
974
|
+
~registrationBlock=dc.registrationBlock,
|
|
975
|
+
~contractStartBlock,
|
|
976
|
+
),
|
|
977
|
+
}
|
|
957
978
|
// Prevent registering already indexing contracts
|
|
958
|
-
switch
|
|
979
|
+
switch indexingAddresses->Utils.Dict.dangerouslyGetNonOption(
|
|
959
980
|
dc.address->Address.toString,
|
|
960
981
|
) {
|
|
961
982
|
| Some(existingContract) =>
|
|
@@ -964,14 +985,14 @@ let registerDynamicContracts = (
|
|
|
964
985
|
// If new registration with earlier block number
|
|
965
986
|
// we should register it for the missing block range
|
|
966
987
|
if existingContract.contractName != dc.contractName {
|
|
967
|
-
fetchState->warnDifferentContractType(~existingContract, ~dc)
|
|
968
|
-
} else if existingContract.
|
|
988
|
+
fetchState->warnDifferentContractType(~existingContract, ~dc=dcWithStartBlock)
|
|
989
|
+
} else if existingContract.effectiveStartBlock > dcWithStartBlock.effectiveStartBlock {
|
|
969
990
|
let logger = Logging.createChild(
|
|
970
991
|
~params={
|
|
971
992
|
"chainId": fetchState.chainId,
|
|
972
993
|
"contractAddress": dc.address->Address.toString,
|
|
973
|
-
"existingBlockNumber": existingContract.
|
|
974
|
-
"newBlockNumber":
|
|
994
|
+
"existingBlockNumber": existingContract.effectiveStartBlock,
|
|
995
|
+
"newBlockNumber": dcWithStartBlock.effectiveStartBlock,
|
|
975
996
|
},
|
|
976
997
|
)
|
|
977
998
|
logger->Logging.childWarn(`Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`)
|
|
@@ -984,7 +1005,10 @@ let registerDynamicContracts = (
|
|
|
984
1005
|
dc.address->Address.toString,
|
|
985
1006
|
) {
|
|
986
1007
|
| Some(registeringContract) if registeringContract.contractName != dc.contractName =>
|
|
987
|
-
fetchState->warnDifferentContractType(
|
|
1008
|
+
fetchState->warnDifferentContractType(
|
|
1009
|
+
~existingContract=registeringContract,
|
|
1010
|
+
~dc=dcWithStartBlock,
|
|
1011
|
+
)
|
|
988
1012
|
false
|
|
989
1013
|
| Some(_) => // Since the DC is registered by an earlier item in the query
|
|
990
1014
|
// FIXME: This unsafely relies on the asc order of the items
|
|
@@ -996,28 +1020,63 @@ let registerDynamicContracts = (
|
|
|
996
1020
|
}
|
|
997
1021
|
if shouldUpdate {
|
|
998
1022
|
earliestRegisteringEventBlockNumber :=
|
|
999
|
-
Pervasives.min(
|
|
1000
|
-
|
|
1023
|
+
Pervasives.min(
|
|
1024
|
+
earliestRegisteringEventBlockNumber.contents,
|
|
1025
|
+
dcWithStartBlock.effectiveStartBlock,
|
|
1026
|
+
)
|
|
1027
|
+
registeringContracts->Dict.set(dc.address->Address.toString, dcWithStartBlock)
|
|
1001
1028
|
} else {
|
|
1002
1029
|
shouldRemove := true
|
|
1003
1030
|
}
|
|
1004
1031
|
}
|
|
1005
|
-
| None =>
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1032
|
+
| None =>
|
|
1033
|
+
let dcAsIndexingAddress: indexingAddress = {
|
|
1034
|
+
address: dc.address,
|
|
1035
|
+
contractName: dc.contractName,
|
|
1036
|
+
registrationBlock: dc.registrationBlock,
|
|
1037
|
+
effectiveStartBlock: deriveEffectiveStartBlock(
|
|
1038
|
+
~registrationBlock=dc.registrationBlock,
|
|
1039
|
+
~contractStartBlock=None,
|
|
1040
|
+
),
|
|
1041
|
+
}
|
|
1042
|
+
// Prevent duplicate logging/persistence when the same address is
|
|
1043
|
+
// already tracked on fetchState, either from the db on startup or
|
|
1044
|
+
// from an earlier registration in this batch.
|
|
1045
|
+
switch indexingAddresses->Utils.Dict.dangerouslyGetNonOption(
|
|
1046
|
+
dc.address->Address.toString,
|
|
1047
|
+
) {
|
|
1048
|
+
| Some(existingContract) =>
|
|
1049
|
+
if existingContract.contractName != dc.contractName {
|
|
1050
|
+
fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
|
|
1051
|
+
}
|
|
1014
1052
|
shouldRemove := true
|
|
1053
|
+
| None =>
|
|
1054
|
+
switch noEventsAddresses->Utils.Dict.dangerouslyGetNonOption(
|
|
1055
|
+
dc.address->Address.toString,
|
|
1056
|
+
) {
|
|
1057
|
+
| Some(_) =>
|
|
1058
|
+
// Already queued for persistence by an earlier item in this batch.
|
|
1059
|
+
shouldRemove := true
|
|
1060
|
+
| None =>
|
|
1061
|
+
let logger = Logging.createChild(
|
|
1062
|
+
~params={
|
|
1063
|
+
"chainId": fetchState.chainId,
|
|
1064
|
+
"contractAddress": dc.address->Address.toString,
|
|
1065
|
+
"contractName": dc.contractName,
|
|
1066
|
+
},
|
|
1067
|
+
)
|
|
1068
|
+
// Persist the address to the db so a future config change that
|
|
1069
|
+
// adds events for this contract can pick it up on restart, but
|
|
1070
|
+
// skip partition registration since there's nothing to fetch.
|
|
1071
|
+
logger->Logging.childWarn(`Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`)
|
|
1072
|
+
noEventsAddresses->Dict.set(dc.address->Address.toString, dcAsIndexingAddress)
|
|
1073
|
+
}
|
|
1015
1074
|
}
|
|
1016
1075
|
}
|
|
1017
1076
|
|
|
1018
1077
|
if shouldRemove.contents {
|
|
1019
1078
|
// Remove the DC from item to prevent it from saving to the db
|
|
1020
|
-
let _ = dcs->
|
|
1079
|
+
let _ = dcs->Array.splice(~start=idx.contents, ~remove=1, ~insert=[])
|
|
1021
1080
|
// Don't increment idx - next element shifted into current position
|
|
1022
1081
|
} else {
|
|
1023
1082
|
idx := idx.contents + 1
|
|
@@ -1026,18 +1085,26 @@ let registerDynamicContracts = (
|
|
|
1026
1085
|
}
|
|
1027
1086
|
}
|
|
1028
1087
|
|
|
1029
|
-
let dcContractNamesToStore = registeringContractsByContract->
|
|
1030
|
-
|
|
1088
|
+
let dcContractNamesToStore = registeringContractsByContract->Dict.keysToArray
|
|
1089
|
+
let hasNoEventsUpdates = !(noEventsAddresses->Utils.Dict.isEmpty)
|
|
1090
|
+
switch (dcContractNamesToStore, hasNoEventsUpdates) {
|
|
1031
1091
|
// Dont update anything when everything was filter out
|
|
1032
|
-
| [] => fetchState
|
|
1033
|
-
|
|
|
1092
|
+
| ([], false) => fetchState
|
|
1093
|
+
| ([], true) =>
|
|
1094
|
+
// Only dcs for contracts without events. Track them on
|
|
1095
|
+
// indexingAddresses so subsequent registrations see them, but don't touch
|
|
1096
|
+
// partitions since there's nothing to fetch for them.
|
|
1097
|
+
let newIndexingContracts = indexingAddresses->Utils.Dict.shallowCopy
|
|
1098
|
+
let _ = Utils.Dict.mergeInPlace(newIndexingContracts, noEventsAddresses)
|
|
1099
|
+
fetchState->updateInternal(~indexingAddresses=newIndexingContracts)
|
|
1100
|
+
| (_, _) => {
|
|
1034
1101
|
let newPartitions = []
|
|
1035
|
-
let
|
|
1102
|
+
let newIndexingAddresses = indexingAddresses->Utils.Dict.shallowCopy
|
|
1036
1103
|
let dynamicContractsRef = ref(fetchState.optimizedPartitions.dynamicContracts)
|
|
1037
|
-
let mutExistingPartitions = fetchState.optimizedPartitions.entities->
|
|
1104
|
+
let mutExistingPartitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
|
|
1038
1105
|
|
|
1039
|
-
for idx in 0 to dcContractNamesToStore->
|
|
1040
|
-
let contractName = dcContractNamesToStore->
|
|
1106
|
+
for idx in 0 to dcContractNamesToStore->Array.length - 1 {
|
|
1107
|
+
let contractName = dcContractNamesToStore->Array.getUnsafe(idx)
|
|
1041
1108
|
|
|
1042
1109
|
// When a new contract name is added as a dynamic contract for the first time (not in dynamicContracts set):
|
|
1043
1110
|
// Walks through existing partitions that have addresses for this contract name
|
|
@@ -1048,19 +1115,18 @@ let registerDynamicContracts = (
|
|
|
1048
1115
|
if !(dynamicContractsRef.contents->Utils.Set.has(contractName)) {
|
|
1049
1116
|
dynamicContractsRef := dynamicContractsRef.contents->Utils.Set.immutableAdd(contractName)
|
|
1050
1117
|
|
|
1051
|
-
for idx in 0 to mutExistingPartitions->
|
|
1052
|
-
let p = mutExistingPartitions->
|
|
1118
|
+
for idx in 0 to mutExistingPartitions->Array.length - 1 {
|
|
1119
|
+
let p = mutExistingPartitions->Array.getUnsafe(idx)
|
|
1053
1120
|
switch p.addressesByContractName->Utils.Dict.dangerouslyGetNonOption(contractName) {
|
|
1054
1121
|
| None => () // Skip partitions which don't have our contract
|
|
1055
1122
|
| Some(addresses) =>
|
|
1056
1123
|
// Also filter out partitions which are 100% not mergable
|
|
1057
1124
|
if p.selection.dependsOnAddresses && p.mergeBlock === None {
|
|
1058
|
-
let allPartitionContractNames = p.addressesByContractName->
|
|
1125
|
+
let allPartitionContractNames = p.addressesByContractName->Dict.keysToArray
|
|
1059
1126
|
switch allPartitionContractNames {
|
|
1060
1127
|
| [_] =>
|
|
1061
|
-
mutExistingPartitions->
|
|
1128
|
+
mutExistingPartitions->Array.setUnsafe(
|
|
1062
1129
|
idx,
|
|
1063
|
-
// Even if it's fetching, set dynamicContract field
|
|
1064
1130
|
{
|
|
1065
1131
|
...p,
|
|
1066
1132
|
dynamicContract: Some(contractName),
|
|
@@ -1083,7 +1149,7 @@ let registerDynamicContracts = (
|
|
|
1083
1149
|
p.addressesByContractName->Utils.Dict.shallowCopy
|
|
1084
1150
|
restAddressesByContractName->Utils.Dict.deleteInPlace(contractName)
|
|
1085
1151
|
|
|
1086
|
-
mutExistingPartitions->
|
|
1152
|
+
mutExistingPartitions->Array.setUnsafe(
|
|
1087
1153
|
idx,
|
|
1088
1154
|
{
|
|
1089
1155
|
...p,
|
|
@@ -1091,8 +1157,8 @@ let registerDynamicContracts = (
|
|
|
1091
1157
|
},
|
|
1092
1158
|
)
|
|
1093
1159
|
|
|
1094
|
-
let addressesByContractName =
|
|
1095
|
-
addressesByContractName->
|
|
1160
|
+
let addressesByContractName = Dict.make()
|
|
1161
|
+
addressesByContractName->Dict.set(contractName, addresses)
|
|
1096
1162
|
newPartitions->Array.push({
|
|
1097
1163
|
id: newPartitionId,
|
|
1098
1164
|
latestFetchedBlock: p.latestFetchedBlock,
|
|
@@ -1113,9 +1179,11 @@ let registerDynamicContracts = (
|
|
|
1113
1179
|
}
|
|
1114
1180
|
}
|
|
1115
1181
|
|
|
1116
|
-
let registeringContracts = registeringContractsByContract->
|
|
1117
|
-
let _ = Utils.Dict.mergeInPlace(
|
|
1182
|
+
let registeringContracts = registeringContractsByContract->Dict.getUnsafe(contractName)
|
|
1183
|
+
let _ = Utils.Dict.mergeInPlace(newIndexingAddresses, registeringContracts)
|
|
1118
1184
|
}
|
|
1185
|
+
// Include no-events dcs so later batches detect conflicts against them.
|
|
1186
|
+
let _ = Utils.Dict.mergeInPlace(newIndexingAddresses, noEventsAddresses)
|
|
1119
1187
|
|
|
1120
1188
|
let optimizedPartitions = createPartitionsFromIndexingAddresses(
|
|
1121
1189
|
~registeringContractsByContract,
|
|
@@ -1125,11 +1193,11 @@ let registerDynamicContracts = (
|
|
|
1125
1193
|
~maxAddrInPartition=fetchState.optimizedPartitions.maxAddrInPartition,
|
|
1126
1194
|
~nextPartitionIndex=fetchState.optimizedPartitions.nextPartitionIndex +
|
|
1127
1195
|
newPartitions->Array.length,
|
|
1128
|
-
~existingPartitions=mutExistingPartitions->
|
|
1196
|
+
~existingPartitions=mutExistingPartitions->Array.concat(newPartitions),
|
|
1129
1197
|
~progressBlockNumber=0,
|
|
1130
1198
|
)
|
|
1131
1199
|
|
|
1132
|
-
fetchState->updateInternal(~optimizedPartitions, ~
|
|
1200
|
+
fetchState->updateInternal(~optimizedPartitions, ~indexingAddresses=newIndexingAddresses)
|
|
1133
1201
|
}
|
|
1134
1202
|
}
|
|
1135
1203
|
}
|
|
@@ -1169,7 +1237,7 @@ type nextQuery =
|
|
|
1169
1237
|
|
|
1170
1238
|
let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) => {
|
|
1171
1239
|
for qIdx in 0 to queries->Array.length - 1 {
|
|
1172
|
-
let q = queries->
|
|
1240
|
+
let q = queries->Array.getUnsafe(qIdx)
|
|
1173
1241
|
let p = optimizedPartitions->OptimizedPartitions.getOrThrow(~partitionId=q.partitionId)
|
|
1174
1242
|
|
|
1175
1243
|
let pq = {
|
|
@@ -1184,8 +1252,8 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
|
|
|
1184
1252
|
let inserted = ref(false)
|
|
1185
1253
|
let i = ref(0)
|
|
1186
1254
|
while i.contents < p.mutPendingQueries->Array.length && !inserted.contents {
|
|
1187
|
-
if (p.mutPendingQueries->
|
|
1188
|
-
p.mutPendingQueries->
|
|
1255
|
+
if (p.mutPendingQueries->Array.getUnsafe(i.contents)).fromBlock > q.fromBlock {
|
|
1256
|
+
p.mutPendingQueries->Array.splice(~start=i.contents, ~remove=0, ~insert=[pq])->ignore
|
|
1189
1257
|
inserted := true
|
|
1190
1258
|
}
|
|
1191
1259
|
i := i.contents + 1
|
|
@@ -1206,7 +1274,7 @@ let pushQueriesForRange = (
|
|
|
1206
1274
|
~maybeChunkRange: option<int>,
|
|
1207
1275
|
~selection: selection,
|
|
1208
1276
|
~addressesByContractName: dict<array<Address.t>>,
|
|
1209
|
-
~
|
|
1277
|
+
~indexingAddresses: dict<indexingAddress>,
|
|
1210
1278
|
) => {
|
|
1211
1279
|
if rangeFromBlock <= maxQueryBlockNumber {
|
|
1212
1280
|
switch rangeEndBlock {
|
|
@@ -1221,7 +1289,7 @@ let pushQueriesForRange = (
|
|
|
1221
1289
|
selection,
|
|
1222
1290
|
isChunk: false,
|
|
1223
1291
|
addressesByContractName,
|
|
1224
|
-
|
|
1292
|
+
indexingAddresses,
|
|
1225
1293
|
})
|
|
1226
1294
|
| Some(chunkRange) =>
|
|
1227
1295
|
let maxBlock = switch rangeEndBlock {
|
|
@@ -1238,7 +1306,7 @@ let pushQueriesForRange = (
|
|
|
1238
1306
|
isChunk: true,
|
|
1239
1307
|
selection,
|
|
1240
1308
|
addressesByContractName,
|
|
1241
|
-
|
|
1309
|
+
indexingAddresses,
|
|
1242
1310
|
})
|
|
1243
1311
|
queries->Array.push({
|
|
1244
1312
|
partitionId,
|
|
@@ -1247,7 +1315,7 @@ let pushQueriesForRange = (
|
|
|
1247
1315
|
isChunk: true,
|
|
1248
1316
|
selection,
|
|
1249
1317
|
addressesByContractName,
|
|
1250
|
-
|
|
1318
|
+
indexingAddresses,
|
|
1251
1319
|
})
|
|
1252
1320
|
} else {
|
|
1253
1321
|
// Not enough room for 2 chunks, fall back to a single query
|
|
@@ -1258,7 +1326,7 @@ let pushQueriesForRange = (
|
|
|
1258
1326
|
selection,
|
|
1259
1327
|
isChunk: rangeEndBlock !== None,
|
|
1260
1328
|
addressesByContractName,
|
|
1261
|
-
|
|
1329
|
+
indexingAddresses,
|
|
1262
1330
|
})
|
|
1263
1331
|
}
|
|
1264
1332
|
}
|
|
@@ -1271,7 +1339,7 @@ let getNextQuery = (
|
|
|
1271
1339
|
buffer,
|
|
1272
1340
|
optimizedPartitions,
|
|
1273
1341
|
targetBufferSize,
|
|
1274
|
-
|
|
1342
|
+
indexingAddresses,
|
|
1275
1343
|
blockLag,
|
|
1276
1344
|
latestOnBlockBlockNumber,
|
|
1277
1345
|
knownHeight,
|
|
@@ -1309,12 +1377,12 @@ let getNextQuery = (
|
|
|
1309
1377
|
|
|
1310
1378
|
let queries = []
|
|
1311
1379
|
|
|
1312
|
-
let partitionsCount = optimizedPartitions.idsInAscOrder->
|
|
1380
|
+
let partitionsCount = optimizedPartitions.idsInAscOrder->Array.length
|
|
1313
1381
|
let idxRef = ref(0)
|
|
1314
1382
|
while idxRef.contents < partitionsCount {
|
|
1315
1383
|
let idx = idxRef.contents
|
|
1316
|
-
let partitionId = optimizedPartitions.idsInAscOrder->
|
|
1317
|
-
let p = optimizedPartitions.entities->
|
|
1384
|
+
let partitionId = optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
|
|
1385
|
+
let p = optimizedPartitions.entities->Dict.getUnsafe(partitionId)
|
|
1318
1386
|
|
|
1319
1387
|
let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber
|
|
1320
1388
|
let hasPendingQueries = p.mutPendingQueries->Utils.Array.notEmpty
|
|
@@ -1354,7 +1422,7 @@ let getNextQuery = (
|
|
|
1354
1422
|
let canContinue = ref(true)
|
|
1355
1423
|
let pqIdx = ref(0)
|
|
1356
1424
|
while pqIdx.contents < p.mutPendingQueries->Array.length && canContinue.contents {
|
|
1357
|
-
let pq = p.mutPendingQueries->
|
|
1425
|
+
let pq = p.mutPendingQueries->Array.getUnsafe(pqIdx.contents)
|
|
1358
1426
|
|
|
1359
1427
|
// Gap before this pending query → create queries for the gap range
|
|
1360
1428
|
if pq.fromBlock > cursor.contents {
|
|
@@ -1367,7 +1435,7 @@ let getNextQuery = (
|
|
|
1367
1435
|
~maybeChunkRange,
|
|
1368
1436
|
~selection=p.selection,
|
|
1369
1437
|
~addressesByContractName=p.addressesByContractName,
|
|
1370
|
-
~
|
|
1438
|
+
~indexingAddresses,
|
|
1371
1439
|
)
|
|
1372
1440
|
}
|
|
1373
1441
|
switch pq {
|
|
@@ -1391,7 +1459,7 @@ let getNextQuery = (
|
|
|
1391
1459
|
~maybeChunkRange,
|
|
1392
1460
|
~selection=p.selection,
|
|
1393
1461
|
~addressesByContractName=p.addressesByContractName,
|
|
1394
|
-
~
|
|
1462
|
+
~indexingAddresses,
|
|
1395
1463
|
)
|
|
1396
1464
|
}
|
|
1397
1465
|
|
|
@@ -1407,8 +1475,8 @@ let getNextQuery = (
|
|
|
1407
1475
|
} else {
|
|
1408
1476
|
// Enforce concurrency limit: sort by fromBlock and take the first concurrencyLimit
|
|
1409
1477
|
let queries = if queries->Array.length > concurrencyLimit {
|
|
1410
|
-
queries->
|
|
1411
|
-
queries->
|
|
1478
|
+
queries->Array.sort((a, b) => Int.compare(a.fromBlock, b.fromBlock))
|
|
1479
|
+
queries->Array.slice(~start=0, ~end=concurrencyLimit)
|
|
1412
1480
|
} else {
|
|
1413
1481
|
queries
|
|
1414
1482
|
}
|
|
@@ -1421,7 +1489,7 @@ let getTimestampAt = (fetchState: t, ~index) => {
|
|
|
1421
1489
|
switch fetchState.buffer->Belt.Array.get(index) {
|
|
1422
1490
|
| Some(Event({timestamp})) => timestamp
|
|
1423
1491
|
| Some(Block(_)) =>
|
|
1424
|
-
|
|
1492
|
+
JsError.throwWithMessage("Block handlers are not supported for ordered multichain mode.")
|
|
1425
1493
|
| None => (fetchState->bufferBlock).blockTimestamp
|
|
1426
1494
|
}
|
|
1427
1495
|
}
|
|
@@ -1463,7 +1531,7 @@ let make = (
|
|
|
1463
1531
|
~startBlock,
|
|
1464
1532
|
~endBlock,
|
|
1465
1533
|
~eventConfigs: array<Internal.eventConfig>,
|
|
1466
|
-
~
|
|
1534
|
+
~addresses: array<Internal.indexingAddress>,
|
|
1467
1535
|
~maxAddrInPartition,
|
|
1468
1536
|
~chainId,
|
|
1469
1537
|
~targetBufferSize,
|
|
@@ -1481,18 +1549,31 @@ let make = (
|
|
|
1481
1549
|
let notDependingOnAddresses = []
|
|
1482
1550
|
let normalEventConfigs = []
|
|
1483
1551
|
let contractNamesWithNormalEvents = Utils.Set.make()
|
|
1484
|
-
let
|
|
1485
|
-
let contractConfigs =
|
|
1552
|
+
let indexingAddresses = Dict.make()
|
|
1553
|
+
let contractConfigs = Dict.make()
|
|
1486
1554
|
|
|
1487
1555
|
eventConfigs->Array.forEach(ec => {
|
|
1488
1556
|
switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(ec.contractName) {
|
|
1489
|
-
| Some({filterByAddresses}) =>
|
|
1490
|
-
contractConfigs->
|
|
1557
|
+
| Some({filterByAddresses, startBlock}) =>
|
|
1558
|
+
contractConfigs->Dict.set(
|
|
1491
1559
|
ec.contractName,
|
|
1492
|
-
{
|
|
1560
|
+
{
|
|
1561
|
+
filterByAddresses: filterByAddresses || ec.filterByAddresses,
|
|
1562
|
+
startBlock: switch (startBlock, ec.startBlock) {
|
|
1563
|
+
| (Some(a), Some(b)) => Some(Pervasives.min(a, b))
|
|
1564
|
+
| (Some(_) as s, None) | (None, Some(_) as s) => s
|
|
1565
|
+
| (None, None) => None
|
|
1566
|
+
},
|
|
1567
|
+
},
|
|
1493
1568
|
)
|
|
1494
1569
|
| None =>
|
|
1495
|
-
contractConfigs->
|
|
1570
|
+
contractConfigs->Dict.set(
|
|
1571
|
+
ec.contractName,
|
|
1572
|
+
{
|
|
1573
|
+
filterByAddresses: ec.filterByAddresses,
|
|
1574
|
+
startBlock: ec.startBlock,
|
|
1575
|
+
},
|
|
1576
|
+
)
|
|
1496
1577
|
}
|
|
1497
1578
|
|
|
1498
1579
|
if ec.dependsOnAddresses {
|
|
@@ -1513,7 +1594,7 @@ let make = (
|
|
|
1513
1594
|
dependsOnAddresses: false,
|
|
1514
1595
|
eventConfigs: notDependingOnAddresses,
|
|
1515
1596
|
},
|
|
1516
|
-
addressesByContractName:
|
|
1597
|
+
addressesByContractName: Dict.make(),
|
|
1517
1598
|
mergeBlock: None,
|
|
1518
1599
|
dynamicContract: None,
|
|
1519
1600
|
mutPendingQueries: [],
|
|
@@ -1528,27 +1609,44 @@ let make = (
|
|
|
1528
1609
|
eventConfigs: normalEventConfigs,
|
|
1529
1610
|
}
|
|
1530
1611
|
|
|
1531
|
-
let registeringContractsByContract: dict<dict<
|
|
1612
|
+
let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
|
|
1532
1613
|
let dynamicContracts = Utils.Set.make()
|
|
1533
1614
|
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1615
|
+
addresses->Array.forEach(contract => {
|
|
1616
|
+
let contractName = contract.contractName
|
|
1617
|
+
let contractStartBlock = switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(
|
|
1618
|
+
contractName,
|
|
1619
|
+
) {
|
|
1620
|
+
| Some({startBlock}) => startBlock
|
|
1621
|
+
| None => None
|
|
1622
|
+
}
|
|
1623
|
+
let ia: indexingAddress = {
|
|
1624
|
+
address: contract.address,
|
|
1625
|
+
contractName: contract.contractName,
|
|
1626
|
+
registrationBlock: contract.registrationBlock,
|
|
1627
|
+
effectiveStartBlock: deriveEffectiveStartBlock(
|
|
1628
|
+
~registrationBlock=contract.registrationBlock,
|
|
1629
|
+
~contractStartBlock,
|
|
1630
|
+
),
|
|
1631
|
+
}
|
|
1632
|
+
// Track the address on fetchState regardless of whether it currently has
|
|
1633
|
+
// matching events. This way, if the config is updated later to add events
|
|
1634
|
+
// for this contract, the address is already known.
|
|
1635
|
+
indexingAddresses->Dict.set(contract.address->Address.toString, ia)
|
|
1636
|
+
|
|
1637
|
+
// Only addresses whose contract has events that depend on addresses get
|
|
1638
|
+
// registered for active fetching via partitions.
|
|
1639
|
+
if contractNamesWithNormalEvents->Utils.Set.has(contractName) {
|
|
1640
|
+
let registeringContracts =
|
|
1641
|
+
registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
|
|
1642
|
+
registeringContracts->Dict.set(contract.address->Address.toString, ia)
|
|
1643
|
+
|
|
1644
|
+
// Detect dynamic contracts by registrationBlock
|
|
1645
|
+
if contract.registrationBlock !== -1 {
|
|
1646
|
+
dynamicContracts->Utils.Set.add(contractName)->ignore
|
|
1549
1647
|
}
|
|
1550
|
-
}
|
|
1551
|
-
}
|
|
1648
|
+
}
|
|
1649
|
+
})
|
|
1552
1650
|
|
|
1553
1651
|
let optimizedPartitions = createPartitionsFromIndexingAddresses(
|
|
1554
1652
|
~registeringContractsByContract,
|
|
@@ -1562,12 +1660,17 @@ let make = (
|
|
|
1562
1660
|
)
|
|
1563
1661
|
|
|
1564
1662
|
if optimizedPartitions->OptimizedPartitions.count === 0 && onBlockConfigs->Utils.Array.isEmpty {
|
|
1565
|
-
|
|
1566
|
-
|
|
1663
|
+
JsError.throwWithMessage(
|
|
1664
|
+
`Invalid configuration: Nothing to fetch on chain ${chainId->Int.toString}. ` ++
|
|
1665
|
+
`addresses=${addresses->Array.length->Int.toString}, ` ++
|
|
1666
|
+
`eventConfigs=${eventConfigs->Array.length->Int.toString}, ` ++
|
|
1667
|
+
`normalEventConfigs=${normalEventConfigs
|
|
1668
|
+
->Array.length
|
|
1669
|
+
->Int.toString}. ` ++ `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`,
|
|
1567
1670
|
)
|
|
1568
1671
|
}
|
|
1569
1672
|
|
|
1570
|
-
let numAddresses =
|
|
1673
|
+
let numAddresses = indexingAddresses->Utils.Dict.size
|
|
1571
1674
|
Prometheus.IndexingAddresses.set(~addressesCount=numAddresses, ~chainId)
|
|
1572
1675
|
Prometheus.IndexingPartitions.set(
|
|
1573
1676
|
~partitionsCount=optimizedPartitions->OptimizedPartitions.count,
|
|
@@ -1588,7 +1691,7 @@ let make = (
|
|
|
1588
1691
|
endBlock,
|
|
1589
1692
|
latestOnBlockBlockNumber: progressBlockNumber,
|
|
1590
1693
|
normalSelection,
|
|
1591
|
-
|
|
1694
|
+
indexingAddresses,
|
|
1592
1695
|
blockLag,
|
|
1593
1696
|
onBlockConfigs,
|
|
1594
1697
|
targetBufferSize,
|
|
@@ -1605,18 +1708,19 @@ let rollbackPendingQueries = (mutPendingQueries: array<pendingQuery>, ~targetBlo
|
|
|
1605
1708
|
// - Cap fetchedBlock at target where fetchedBlock > target
|
|
1606
1709
|
let adjusted = []
|
|
1607
1710
|
for qIdx in 0 to mutPendingQueries->Array.length - 1 {
|
|
1608
|
-
let pq = mutPendingQueries->
|
|
1711
|
+
let pq = mutPendingQueries->Array.getUnsafe(qIdx)
|
|
1609
1712
|
if pq.fromBlock <= targetBlockNumber {
|
|
1610
1713
|
switch pq.fetchedBlock {
|
|
1611
1714
|
| Some({blockNumber}) if blockNumber > targetBlockNumber =>
|
|
1612
1715
|
adjusted
|
|
1613
|
-
->
|
|
1716
|
+
->Array.push({
|
|
1614
1717
|
...pq,
|
|
1615
1718
|
fetchedBlock: Some({blockNumber: targetBlockNumber, blockTimestamp: 0}),
|
|
1616
1719
|
})
|
|
1617
1720
|
->ignore
|
|
1618
|
-
| Some(_) => adjusted->
|
|
1619
|
-
| None =>
|
|
1721
|
+
| Some(_) => adjusted->Array.push(pq)->ignore
|
|
1722
|
+
| None =>
|
|
1723
|
+
JsError.throwWithMessage("Internal error: Must not have a fetching query during rollback")
|
|
1620
1724
|
}
|
|
1621
1725
|
}
|
|
1622
1726
|
}
|
|
@@ -1631,36 +1735,36 @@ Always recreates optimized partitions to avoid duplicate addresses:
|
|
|
1631
1735
|
- Non-wildcard with lfb > target: delete, track addresses for recreation
|
|
1632
1736
|
*/
|
|
1633
1737
|
let rollback = (fetchState: t, ~targetBlockNumber) => {
|
|
1634
|
-
// Step 1: Build addressesToRemove and surviving
|
|
1738
|
+
// Step 1: Build addressesToRemove and surviving indexingAddresses
|
|
1635
1739
|
let addressesToRemove = Utils.Set.make()
|
|
1636
|
-
let
|
|
1740
|
+
let indexingAddresses = Dict.make()
|
|
1637
1741
|
|
|
1638
|
-
fetchState.
|
|
1639
|
-
->
|
|
1742
|
+
fetchState.indexingAddresses
|
|
1743
|
+
->Dict.keysToArray
|
|
1640
1744
|
->Array.forEach(address => {
|
|
1641
|
-
let indexingContract = fetchState.
|
|
1642
|
-
|
|
1643
|
-
| Some(registrationBlock) if registrationBlock > targetBlockNumber =>
|
|
1745
|
+
let indexingContract = fetchState.indexingAddresses->Dict.getUnsafe(address)
|
|
1746
|
+
if indexingContract.registrationBlock > targetBlockNumber {
|
|
1644
1747
|
let _ = addressesToRemove->Utils.Set.add(address->Address.unsafeFromString)
|
|
1645
|
-
|
|
1748
|
+
} else {
|
|
1749
|
+
indexingAddresses->Dict.set(address, indexingContract)
|
|
1646
1750
|
}
|
|
1647
1751
|
})
|
|
1648
1752
|
|
|
1649
1753
|
// Step 2: Categorize partitions
|
|
1650
1754
|
let keptPartitions = []
|
|
1651
1755
|
let nextKeptIdRef = ref(0)
|
|
1652
|
-
let registeringContractsByContract: dict<dict<
|
|
1756
|
+
let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
|
|
1653
1757
|
|
|
1654
|
-
let partitions = fetchState.optimizedPartitions.entities->
|
|
1758
|
+
let partitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
|
|
1655
1759
|
for idx in 0 to partitions->Array.length - 1 {
|
|
1656
|
-
let p = partitions->
|
|
1760
|
+
let p = partitions->Array.getUnsafe(idx)
|
|
1657
1761
|
switch p {
|
|
1658
1762
|
// Wildcard: rollback latestFetchedBlock and adjust pending queries
|
|
1659
1763
|
| {selection: {dependsOnAddresses: false}} =>
|
|
1660
1764
|
let id = nextKeptIdRef.contents->Int.toString
|
|
1661
1765
|
nextKeptIdRef := nextKeptIdRef.contents + 1
|
|
1662
1766
|
keptPartitions
|
|
1663
|
-
->
|
|
1767
|
+
->Array.push({
|
|
1664
1768
|
...p,
|
|
1665
1769
|
id,
|
|
1666
1770
|
latestFetchedBlock: p.latestFetchedBlock.blockNumber > targetBlockNumber
|
|
@@ -1676,15 +1780,15 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
|
|
|
1676
1780
|
addresses->Array.forEach(address => {
|
|
1677
1781
|
if (
|
|
1678
1782
|
!(addressesToRemove->Utils.Set.has(address)) &&
|
|
1679
|
-
|
|
1783
|
+
indexingAddresses
|
|
1680
1784
|
->Utils.Dict.dangerouslyGetNonOption(address->Address.toString)
|
|
1681
1785
|
->Option.isSome
|
|
1682
1786
|
) {
|
|
1683
1787
|
let registeringContracts =
|
|
1684
1788
|
registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
|
|
1685
|
-
registeringContracts->
|
|
1789
|
+
registeringContracts->Dict.set(
|
|
1686
1790
|
address->Address.toString,
|
|
1687
|
-
|
|
1791
|
+
indexingAddresses->Dict.getUnsafe(address->Address.toString),
|
|
1688
1792
|
)
|
|
1689
1793
|
}
|
|
1690
1794
|
})
|
|
@@ -1699,20 +1803,20 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
|
|
|
1699
1803
|
}
|
|
1700
1804
|
|
|
1701
1805
|
// Remove addresses that should be removed
|
|
1702
|
-
let rollbackedAddressesByContractName =
|
|
1806
|
+
let rollbackedAddressesByContractName = Dict.make()
|
|
1703
1807
|
addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
|
|
1704
1808
|
let keptAddresses =
|
|
1705
|
-
addresses->Array.
|
|
1809
|
+
addresses->Array.filter(address => !(addressesToRemove->Utils.Set.has(address)))
|
|
1706
1810
|
if keptAddresses->Array.length > 0 {
|
|
1707
|
-
rollbackedAddressesByContractName->
|
|
1811
|
+
rollbackedAddressesByContractName->Dict.set(contractName, keptAddresses)
|
|
1708
1812
|
}
|
|
1709
1813
|
})
|
|
1710
1814
|
|
|
1711
|
-
if rollbackedAddressesByContractName->
|
|
1815
|
+
if !(rollbackedAddressesByContractName->Utils.Dict.isEmpty) {
|
|
1712
1816
|
let id = nextKeptIdRef.contents->Int.toString
|
|
1713
1817
|
nextKeptIdRef := nextKeptIdRef.contents + 1
|
|
1714
1818
|
keptPartitions
|
|
1715
|
-
->
|
|
1819
|
+
->Array.push({
|
|
1716
1820
|
...p,
|
|
1717
1821
|
id,
|
|
1718
1822
|
addressesByContractName: rollbackedAddressesByContractName,
|
|
@@ -1747,8 +1851,8 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
|
|
|
1747
1851
|
),
|
|
1748
1852
|
}->updateInternal(
|
|
1749
1853
|
~optimizedPartitions,
|
|
1750
|
-
~
|
|
1751
|
-
~mutItems=fetchState.buffer->Array.
|
|
1854
|
+
~indexingAddresses,
|
|
1855
|
+
~mutItems=fetchState.buffer->Array.filter(item =>
|
|
1752
1856
|
switch item {
|
|
1753
1857
|
| Event({blockNumber})
|
|
1754
1858
|
| Block({blockNumber}) => blockNumber
|
|
@@ -1765,13 +1869,13 @@ let resetPendingQueries = (fetchState: t) => {
|
|
|
1765
1869
|
let newEntities = fetchState.optimizedPartitions.entities->Utils.Dict.shallowCopy
|
|
1766
1870
|
|
|
1767
1871
|
for idx in 0 to fetchState.optimizedPartitions.idsInAscOrder->Array.length - 1 {
|
|
1768
|
-
let partitionId = fetchState.optimizedPartitions.idsInAscOrder->
|
|
1769
|
-
let partition = fetchState.optimizedPartitions.entities->
|
|
1872
|
+
let partitionId = fetchState.optimizedPartitions.idsInAscOrder->Array.getUnsafe(idx)
|
|
1873
|
+
let partition = fetchState.optimizedPartitions.entities->Dict.getUnsafe(partitionId)
|
|
1770
1874
|
|
|
1771
1875
|
if partition.mutPendingQueries->Array.length > 0 {
|
|
1772
1876
|
// Keep only completed queries (with fetchedBlock)
|
|
1773
|
-
let kept = partition.mutPendingQueries->Array.
|
|
1774
|
-
newEntities->
|
|
1877
|
+
let kept = partition.mutPendingQueries->Array.filter(pq => pq.fetchedBlock !== None)
|
|
1878
|
+
newEntities->Dict.set(partitionId, {...partition, mutPendingQueries: kept})
|
|
1775
1879
|
}
|
|
1776
1880
|
}
|
|
1777
1881
|
|
|
@@ -1838,26 +1942,26 @@ let sortForUnorderedBatch = {
|
|
|
1838
1942
|
}
|
|
1839
1943
|
|
|
1840
1944
|
(fetchStates: array<t>, ~batchSizeTarget: int) => {
|
|
1841
|
-
fetchStates
|
|
1842
|
-
->Array.
|
|
1843
|
-
->Js.Array2.sortInPlaceWith((a: t, b: t) => {
|
|
1945
|
+
let copied = fetchStates->Array.copy
|
|
1946
|
+
copied->Array.sort((a: t, b: t) => {
|
|
1844
1947
|
switch (a->hasFullBatch(~batchSizeTarget), b->hasFullBatch(~batchSizeTarget)) {
|
|
1845
1948
|
| (true, true)
|
|
1846
1949
|
| (false, false) => {
|
|
1847
1950
|
let aProgress = a->getProgressPercentage
|
|
1848
1951
|
let bProgress = b->getProgressPercentage
|
|
1849
1952
|
if aProgress < bProgress {
|
|
1850
|
-
|
|
1953
|
+
Ordering.less
|
|
1851
1954
|
} else if aProgress > bProgress {
|
|
1852
|
-
|
|
1955
|
+
Ordering.greater
|
|
1853
1956
|
} else {
|
|
1854
|
-
|
|
1957
|
+
Ordering.equal
|
|
1855
1958
|
}
|
|
1856
1959
|
}
|
|
1857
|
-
| (true, false) =>
|
|
1858
|
-
| (false, true) =>
|
|
1960
|
+
| (true, false) => Ordering.less
|
|
1961
|
+
| (false, true) => Ordering.greater
|
|
1859
1962
|
}
|
|
1860
1963
|
})
|
|
1964
|
+
copied
|
|
1861
1965
|
}
|
|
1862
1966
|
}
|
|
1863
1967
|
|