envio 3.3.0-alpha.3 → 3.3.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +10 -0
- package/package.json +7 -7
- package/src/Batch.res.mjs +1 -1
- package/src/Bin.res +3 -0
- package/src/Bin.res.mjs +4 -0
- package/src/ChainFetching.res +1 -3
- package/src/ChainFetching.res.mjs +3 -3
- package/src/ChainState.res +44 -19
- package/src/ChainState.res.mjs +25 -23
- package/src/ChainState.resi +3 -3
- package/src/Config.res +3 -0
- package/src/Config.res.mjs +3 -1
- package/src/Core.res +0 -3
- package/src/Ecosystem.res +0 -4
- package/src/EventConfigBuilder.res +3 -0
- package/src/EventConfigBuilder.res.mjs +7 -1
- package/src/FetchState.res +73 -160
- package/src/FetchState.res.mjs +120 -222
- package/src/IndexingAddresses.res +105 -0
- package/src/IndexingAddresses.res.mjs +100 -0
- package/src/IndexingAddresses.resi +32 -0
- package/src/Internal.res +5 -0
- package/src/Main.res +21 -24
- package/src/Main.res.mjs +21 -20
- package/src/Metrics.res +33 -0
- package/src/Metrics.res.mjs +39 -0
- package/src/Prometheus.res +2 -20
- package/src/Prometheus.res.mjs +83 -95
- package/src/SimulateItems.res +4 -0
- package/src/TestIndexer.res +43 -17
- package/src/TestIndexer.res.mjs +16 -9
- package/src/bindings/Viem.res +0 -41
- package/src/bindings/Viem.res.mjs +1 -43
- package/src/sources/Evm.res +17 -39
- package/src/sources/Evm.res.mjs +8 -40
- package/src/sources/EvmChain.res +3 -1
- package/src/sources/EvmChain.res.mjs +2 -1
- package/src/sources/EvmRpcClient.res +36 -5
- package/src/sources/EvmRpcClient.res.mjs +7 -4
- package/src/sources/Fuel.res +0 -2
- package/src/sources/Fuel.res.mjs +0 -1
- package/src/sources/HyperSyncClient.res +0 -35
- package/src/sources/HyperSyncClient.res.mjs +1 -8
- package/src/sources/Rpc.res +15 -47
- package/src/sources/Rpc.res.mjs +25 -56
- package/src/sources/RpcSource.res +30 -73
- package/src/sources/RpcSource.res.mjs +24 -73
- package/src/sources/SourceManager.res +1 -1
- package/src/sources/SourceManager.res.mjs +1 -1
- package/src/sources/Svm.res +10 -17
- package/src/sources/Svm.res.mjs +6 -18
- package/src/sources/TransactionStore.res +98 -76
- package/src/sources/TransactionStore.res.mjs +49 -35
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
|
+
import * as Evm from "./sources/Evm.res.mjs";
|
|
4
|
+
import * as Svm from "./sources/Svm.res.mjs";
|
|
3
5
|
import * as Utils from "./Utils.res.mjs";
|
|
4
6
|
import * as Address from "./Address.res.mjs";
|
|
5
7
|
import * as FuelSDK from "./sources/FuelSDK.res.mjs";
|
|
@@ -294,6 +296,7 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcar
|
|
|
294
296
|
let filterByAddresses = match.filterByAddresses;
|
|
295
297
|
let resolvedStartBlock = whereStartBlock !== undefined ? whereStartBlock : startBlock;
|
|
296
298
|
let match$1 = resolveFieldSelection(blockFields, transactionFields, globalBlockFieldsSet, globalTransactionFieldsSet);
|
|
299
|
+
let selectedTransactionFields = match$1[1];
|
|
297
300
|
return {
|
|
298
301
|
id: sighash + "_" + topicCount.toString(),
|
|
299
302
|
name: eventName,
|
|
@@ -307,7 +310,8 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcar
|
|
|
307
310
|
paramsRawEventSchema: buildParamsSchema(params),
|
|
308
311
|
simulateParamsSchema: buildSimulateParamsSchema(params),
|
|
309
312
|
startBlock: resolvedStartBlock,
|
|
310
|
-
selectedTransactionFields:
|
|
313
|
+
selectedTransactionFields: selectedTransactionFields,
|
|
314
|
+
transactionFieldMask: Evm.eventTransactionFieldMask(selectedTransactionFields),
|
|
311
315
|
getEventFiltersOrThrow: match.getEventFiltersOrThrow,
|
|
312
316
|
selectedBlockFields: match$1[0],
|
|
313
317
|
sighash: sighash,
|
|
@@ -337,6 +341,7 @@ function buildSvmInstructionEventConfig(contractName, instructionName, programId
|
|
|
337
341
|
simulateParamsSchema: paramsSchema,
|
|
338
342
|
startBlock: startBlock,
|
|
339
343
|
selectedTransactionFields: selectedTransactionFields,
|
|
344
|
+
transactionFieldMask: Svm.eventTransactionFieldMask(selectedTransactionFields),
|
|
340
345
|
programId: programId,
|
|
341
346
|
discriminator: discriminator,
|
|
342
347
|
discriminatorByteLen: discriminatorByteLen,
|
|
@@ -406,6 +411,7 @@ function buildFuelEventConfig(contractName, eventName, kind, sighash, rawAbi, is
|
|
|
406
411
|
simulateParamsSchema: paramsSchema,
|
|
407
412
|
startBlock: startBlock,
|
|
408
413
|
selectedTransactionFields: new Set(),
|
|
414
|
+
transactionFieldMask: 0,
|
|
409
415
|
kind: fuelKind
|
|
410
416
|
};
|
|
411
417
|
}
|
package/src/FetchState.res
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
|
-
type contractConfig = {startBlock: option<int>}
|
|
2
|
-
|
|
3
1
|
type indexingAddress = Internal.indexingContract
|
|
4
2
|
|
|
5
|
-
let deriveEffectiveStartBlock = (~registrationBlock: int, ~contractStartBlock: option<int>) => {
|
|
6
|
-
Pervasives.max(Pervasives.max(registrationBlock, 0), contractStartBlock->Option.getOr(0))
|
|
7
|
-
}
|
|
8
|
-
|
|
9
3
|
type blockNumberAndTimestamp = {
|
|
10
4
|
blockNumber: int,
|
|
11
5
|
blockTimestamp: int,
|
|
@@ -41,12 +35,6 @@ type partition = {
|
|
|
41
35
|
latestFetchedBlock: blockNumberAndTimestamp,
|
|
42
36
|
selection: selection,
|
|
43
37
|
addressesByContractName: dict<array<Address.t>>,
|
|
44
|
-
// Reverse index address→contractName, derived from addressesByContractName.
|
|
45
|
-
// Used by EventRouter to resolve a log's owning contract from the partition
|
|
46
|
-
// that fetched it (instead of a chain-wide snapshot). Always recomputed in
|
|
47
|
-
// OptimizedPartitions.make, so partition literals can seed it with an empty
|
|
48
|
-
// dict.
|
|
49
|
-
contractNameByAddress: dict<string>,
|
|
50
38
|
mergeBlock: option<int>,
|
|
51
39
|
// When set, partition indexes a single dynamic contract type.
|
|
52
40
|
// The addressesByContractName must contain only addresses for this contract.
|
|
@@ -82,16 +70,17 @@ type query = {
|
|
|
82
70
|
mutable progress: float,
|
|
83
71
|
selection: selection,
|
|
84
72
|
addressesByContractName: dict<array<Address.t>>,
|
|
85
|
-
// The owning partition's reverse index, referenced (not copied) so routing
|
|
86
|
-
// resolves ownership without a chain-wide snapshot.
|
|
87
|
-
contractNameByAddress: dict<string>,
|
|
88
73
|
}
|
|
89
74
|
|
|
90
|
-
// Invert addressesByContractName into address→contractName
|
|
91
|
-
// address belongs to one contract), so no key
|
|
92
|
-
|
|
75
|
+
// Invert addressesByContractName into address→contractName for log-ownership
|
|
76
|
+
// routing. 1:1 today (each address belongs to one contract), so no key
|
|
77
|
+
// collisions. Memoized on the addressesByContractName object so a partition's
|
|
78
|
+
// many responses share one derivation and a large factory never rebuilds the
|
|
79
|
+
// whole index; sound because the dict is immutable after construction (every
|
|
80
|
+
// mutation produces a new dict).
|
|
81
|
+
let deriveContractNameByAddress: dict<array<Address.t>> => dict<
|
|
93
82
|
string,
|
|
94
|
-
> => {
|
|
83
|
+
> = Utils.WeakMap.memoize(addressesByContractName => {
|
|
95
84
|
let result = Dict.make()
|
|
96
85
|
addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
|
|
97
86
|
for i in 0 to addresses->Array.length - 1 {
|
|
@@ -99,7 +88,7 @@ let deriveContractNameByAddress = (addressesByContractName: dict<array<Address.t
|
|
|
99
88
|
}
|
|
100
89
|
})
|
|
101
90
|
result
|
|
102
|
-
}
|
|
91
|
+
})
|
|
103
92
|
|
|
104
93
|
// Default estimate for a query whose partition hasn't responded yet, so the
|
|
105
94
|
// shared budget still accounts for unknown queries instead of treating them as
|
|
@@ -214,7 +203,6 @@ module OptimizedPartitions = {
|
|
|
214
203
|
latestFetchedBlock: {blockNumber: potentialMergeBlock, blockTimestamp: 0},
|
|
215
204
|
mergeBlock: None,
|
|
216
205
|
addressesByContractName: Dict.make(), // set below
|
|
217
|
-
contractNameByAddress: Dict.make(), // derived in make
|
|
218
206
|
mutPendingQueries: [],
|
|
219
207
|
prevQueryRange: minRange,
|
|
220
208
|
prevPrevQueryRange: minRange,
|
|
@@ -397,12 +385,7 @@ module OptimizedPartitions = {
|
|
|
397
385
|
for idx in 0 to partitionsCount - 1 {
|
|
398
386
|
let p = newPartitions->Array.getUnsafe(idx)
|
|
399
387
|
idsInAscOrder->Array.setUnsafe(idx, p.id)
|
|
400
|
-
|
|
401
|
-
// construction paths (literals, spreads, merges, splits) stay consistent.
|
|
402
|
-
entities->Dict.set(
|
|
403
|
-
p.id,
|
|
404
|
-
{...p, contractNameByAddress: deriveContractNameByAddress(p.addressesByContractName)},
|
|
405
|
-
)
|
|
388
|
+
entities->Dict.set(p.id, p)
|
|
406
389
|
}
|
|
407
390
|
|
|
408
391
|
{
|
|
@@ -547,10 +530,8 @@ type t = {
|
|
|
547
530
|
startBlock: int,
|
|
548
531
|
endBlock: option<int>,
|
|
549
532
|
normalSelection: selection,
|
|
550
|
-
// By address
|
|
551
|
-
indexingAddresses: dict<indexingAddress>,
|
|
552
533
|
// By contract name
|
|
553
|
-
contractConfigs: dict<contractConfig>,
|
|
534
|
+
contractConfigs: dict<IndexingAddresses.contractConfig>,
|
|
554
535
|
// Not used for logic - only metadata
|
|
555
536
|
chainId: int,
|
|
556
537
|
// The block number of the latest block which was added to the queue
|
|
@@ -603,6 +584,25 @@ let bufferBlock = ({optimizedPartitions, latestOnBlockBlockNumber}: t) => {
|
|
|
603
584
|
}
|
|
604
585
|
}
|
|
605
586
|
|
|
587
|
+
// Number of buffered items at or below the ready frontier (processable now,
|
|
588
|
+
// i.e. not stuck behind a gap from a lagging partition or out-of-order chunk).
|
|
589
|
+
// The buffer is kept sorted, so binary-search the frontier in O(log n).
|
|
590
|
+
let bufferReadyCount = (fetchState: t) => {
|
|
591
|
+
let frontier = fetchState->bufferBlockNumber
|
|
592
|
+
let buffer = fetchState.buffer
|
|
593
|
+
let lo = ref(0)
|
|
594
|
+
let hi = ref(buffer->Array.length)
|
|
595
|
+
while lo.contents < hi.contents {
|
|
596
|
+
let mid = (lo.contents + hi.contents) / 2
|
|
597
|
+
if buffer->Array.getUnsafe(mid)->Internal.getItemBlockNumber <= frontier {
|
|
598
|
+
lo := mid + 1
|
|
599
|
+
} else {
|
|
600
|
+
hi := mid
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
lo.contents
|
|
604
|
+
}
|
|
605
|
+
|
|
606
606
|
/*
|
|
607
607
|
Comparitor for two events from the same chain. No need for chain id or timestamp
|
|
608
608
|
*/
|
|
@@ -618,8 +618,6 @@ let compareBufferItem = (a: Internal.item, b: Internal.item) => {
|
|
|
618
618
|
// Some big number which should be bigger than any log index
|
|
619
619
|
let blockItemLogIndex = 16777216
|
|
620
620
|
|
|
621
|
-
let numAddresses = fetchState => fetchState.indexingAddresses->Utils.Dict.size
|
|
622
|
-
|
|
623
621
|
// Appends Block items produced by the onBlock handlers for every block in
|
|
624
622
|
// (fromBlock, maxBlockNumber] into mutItems and returns the new
|
|
625
623
|
// latestOnBlockBlockNumber pointer. maxOnBlockBufferSize bounds how many items
|
|
@@ -684,7 +682,6 @@ Runs partition optimization when partitions change.
|
|
|
684
682
|
let updateInternal = (
|
|
685
683
|
fetchState: t,
|
|
686
684
|
~optimizedPartitions=fetchState.optimizedPartitions,
|
|
687
|
-
~indexingAddresses=fetchState.indexingAddresses,
|
|
688
685
|
~mutItems=?,
|
|
689
686
|
~blockLag=fetchState.blockLag,
|
|
690
687
|
~knownHeight=fetchState.knownHeight,
|
|
@@ -740,7 +737,6 @@ let updateInternal = (
|
|
|
740
737
|
maxOnBlockBufferSize: fetchState.maxOnBlockBufferSize,
|
|
741
738
|
optimizedPartitions,
|
|
742
739
|
latestOnBlockBlockNumber,
|
|
743
|
-
indexingAddresses,
|
|
744
740
|
blockLag,
|
|
745
741
|
knownHeight,
|
|
746
742
|
buffer: switch mutItemsRef.contents {
|
|
@@ -768,12 +764,6 @@ let updateInternal = (
|
|
|
768
764
|
~blockNumber=updatedFetchState->bufferBlockNumber,
|
|
769
765
|
~chainId=fetchState.chainId,
|
|
770
766
|
)
|
|
771
|
-
if indexingAddresses !== fetchState.indexingAddresses {
|
|
772
|
-
Prometheus.IndexingAddresses.set(
|
|
773
|
-
~addressesCount=updatedFetchState->numAddresses,
|
|
774
|
-
~chainId=fetchState.chainId,
|
|
775
|
-
)
|
|
776
|
-
}
|
|
777
767
|
|
|
778
768
|
updatedFetchState
|
|
779
769
|
}
|
|
@@ -901,7 +891,6 @@ OptimizedPartitions.t => {
|
|
|
901
891
|
selection: normalSelection,
|
|
902
892
|
dynamicContract: isDynamic ? Some(contractName) : None,
|
|
903
893
|
addressesByContractName,
|
|
904
|
-
contractNameByAddress: Dict.make(), // derived in make
|
|
905
894
|
mergeBlock: None,
|
|
906
895
|
mutPendingQueries: [],
|
|
907
896
|
prevQueryRange: 0,
|
|
@@ -1005,6 +994,7 @@ OptimizedPartitions.t => {
|
|
|
1005
994
|
|
|
1006
995
|
let registerDynamicContracts = (
|
|
1007
996
|
fetchState: t,
|
|
997
|
+
~indexingAddresses: IndexingAddresses.t,
|
|
1008
998
|
// These are raw items which might have dynamic contracts received from contractRegister call.
|
|
1009
999
|
// Might contain duplicates which we should filter out
|
|
1010
1000
|
items: array<Internal.item>,
|
|
@@ -1016,11 +1006,10 @@ let registerDynamicContracts = (
|
|
|
1016
1006
|
)
|
|
1017
1007
|
}
|
|
1018
1008
|
|
|
1019
|
-
let indexingAddresses = fetchState.indexingAddresses
|
|
1020
1009
|
let registeringContractsByContract: dict<dict<indexingAddress>> = Dict.make()
|
|
1021
1010
|
let earliestRegisteringEventBlockNumber = ref(%raw(`Infinity`))
|
|
1022
1011
|
// Addresses registered for contracts without matching events. These are not
|
|
1023
|
-
// added to partitions, but they are tracked on
|
|
1012
|
+
// added to partitions, but they are tracked on indexingAddresses
|
|
1024
1013
|
// so that later conflicting registrations are detected, and are persisted
|
|
1025
1014
|
// to envio_addresses so they can be picked up on restart with updated config.
|
|
1026
1015
|
let noEventsAddresses: dict<indexingAddress> = Dict.make()
|
|
@@ -1046,15 +1035,13 @@ let registerDynamicContracts = (
|
|
|
1046
1035
|
address: dc.address,
|
|
1047
1036
|
contractName: dc.contractName,
|
|
1048
1037
|
registrationBlock: dc.registrationBlock,
|
|
1049
|
-
effectiveStartBlock: deriveEffectiveStartBlock(
|
|
1038
|
+
effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
|
|
1050
1039
|
~registrationBlock=dc.registrationBlock,
|
|
1051
1040
|
~contractStartBlock,
|
|
1052
1041
|
),
|
|
1053
1042
|
}
|
|
1054
1043
|
// Prevent registering already indexing contracts
|
|
1055
|
-
switch indexingAddresses->
|
|
1056
|
-
dc.address->Address.toString,
|
|
1057
|
-
) {
|
|
1044
|
+
switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
|
|
1058
1045
|
| Some(existingContract) =>
|
|
1059
1046
|
// FIXME: Instead of filtering out duplicates,
|
|
1060
1047
|
// we should check the block number first.
|
|
@@ -1109,7 +1096,7 @@ let registerDynamicContracts = (
|
|
|
1109
1096
|
address: dc.address,
|
|
1110
1097
|
contractName: dc.contractName,
|
|
1111
1098
|
registrationBlock: dc.registrationBlock,
|
|
1112
|
-
effectiveStartBlock: deriveEffectiveStartBlock(
|
|
1099
|
+
effectiveStartBlock: IndexingAddresses.deriveEffectiveStartBlock(
|
|
1113
1100
|
~registrationBlock=dc.registrationBlock,
|
|
1114
1101
|
~contractStartBlock=None,
|
|
1115
1102
|
),
|
|
@@ -1117,9 +1104,7 @@ let registerDynamicContracts = (
|
|
|
1117
1104
|
// Prevent duplicate logging/persistence when the same address is
|
|
1118
1105
|
// already tracked on fetchState, either from the db on startup or
|
|
1119
1106
|
// from an earlier registration in this batch.
|
|
1120
|
-
switch indexingAddresses->
|
|
1121
|
-
dc.address->Address.toString,
|
|
1122
|
-
) {
|
|
1107
|
+
switch indexingAddresses->IndexingAddresses.get(dc.address->Address.toString) {
|
|
1123
1108
|
| Some(existingContract) =>
|
|
1124
1109
|
if existingContract.contractName != dc.contractName {
|
|
1125
1110
|
fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
|
|
@@ -1173,12 +1158,10 @@ let registerDynamicContracts = (
|
|
|
1173
1158
|
// Only dcs for contracts without events. Track them on
|
|
1174
1159
|
// indexingAddresses so subsequent registrations see them, but don't touch
|
|
1175
1160
|
// partitions since there's nothing to fetch for them.
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
fetchState->updateInternal(~indexingAddresses=newIndexingContracts)
|
|
1161
|
+
indexingAddresses->IndexingAddresses.register(noEventsAddresses)
|
|
1162
|
+
fetchState
|
|
1179
1163
|
| (_, _) => {
|
|
1180
1164
|
let newPartitions = []
|
|
1181
|
-
let newIndexingAddresses = indexingAddresses->Utils.Dict.shallowCopy
|
|
1182
1165
|
let dynamicContractsRef = ref(fetchState.optimizedPartitions.dynamicContracts)
|
|
1183
1166
|
let mutExistingPartitions = fetchState.optimizedPartitions.entities->Dict.valuesToArray
|
|
1184
1167
|
|
|
@@ -1244,7 +1227,6 @@ let registerDynamicContracts = (
|
|
|
1244
1227
|
selection: fetchState.normalSelection,
|
|
1245
1228
|
dynamicContract: Some(contractName),
|
|
1246
1229
|
addressesByContractName,
|
|
1247
|
-
contractNameByAddress: Dict.make(), // derived in make
|
|
1248
1230
|
mergeBlock: None,
|
|
1249
1231
|
mutPendingQueries: p.mutPendingQueries,
|
|
1250
1232
|
prevQueryRange: p.prevQueryRange,
|
|
@@ -1261,10 +1243,10 @@ let registerDynamicContracts = (
|
|
|
1261
1243
|
}
|
|
1262
1244
|
|
|
1263
1245
|
let registeringContracts = registeringContractsByContract->Dict.getUnsafe(contractName)
|
|
1264
|
-
|
|
1246
|
+
indexingAddresses->IndexingAddresses.register(registeringContracts)
|
|
1265
1247
|
}
|
|
1266
1248
|
// Include no-events dcs so later batches detect conflicts against them.
|
|
1267
|
-
|
|
1249
|
+
indexingAddresses->IndexingAddresses.register(noEventsAddresses)
|
|
1268
1250
|
|
|
1269
1251
|
let optimizedPartitions = createPartitionsFromIndexingAddresses(
|
|
1270
1252
|
~registeringContractsByContract,
|
|
@@ -1277,7 +1259,7 @@ let registerDynamicContracts = (
|
|
|
1277
1259
|
~progressBlockNumber=0,
|
|
1278
1260
|
)
|
|
1279
1261
|
|
|
1280
|
-
fetchState->updateInternal(~optimizedPartitions
|
|
1262
|
+
fetchState->updateInternal(~optimizedPartitions)
|
|
1281
1263
|
}
|
|
1282
1264
|
}
|
|
1283
1265
|
}
|
|
@@ -1290,6 +1272,7 @@ newItems are ordered earliest to latest (as they are returned from the worker)
|
|
|
1290
1272
|
*/
|
|
1291
1273
|
let handleQueryResult = (
|
|
1292
1274
|
fetchState: t,
|
|
1275
|
+
~indexingAddresses: IndexingAddresses.t,
|
|
1293
1276
|
~query: query,
|
|
1294
1277
|
~latestFetchedBlock: blockNumberAndTimestamp,
|
|
1295
1278
|
~newItems,
|
|
@@ -1302,7 +1285,8 @@ let handleQueryResult = (
|
|
|
1302
1285
|
switch item {
|
|
1303
1286
|
| Internal.Event({eventConfig, payload, blockNumber}) =>
|
|
1304
1287
|
switch eventConfig.clientAddressFilter {
|
|
1305
|
-
| Some(filter) =>
|
|
1288
|
+
| Some(filter) =>
|
|
1289
|
+
filter(payload, blockNumber, indexingAddresses->IndexingAddresses.rawForFilter)
|
|
1306
1290
|
| None => true
|
|
1307
1291
|
}
|
|
1308
1292
|
| _ => true
|
|
@@ -1396,7 +1380,6 @@ let pushQueriesForRange = (
|
|
|
1396
1380
|
chainId: 0,
|
|
1397
1381
|
progress: 0.,
|
|
1398
1382
|
addressesByContractName,
|
|
1399
|
-
contractNameByAddress: partition.contractNameByAddress,
|
|
1400
1383
|
})
|
|
1401
1384
|
| Some(chunkRange) =>
|
|
1402
1385
|
let maxBlock = switch rangeEndBlock {
|
|
@@ -1426,7 +1409,6 @@ let pushQueriesForRange = (
|
|
|
1426
1409
|
chainId: 0,
|
|
1427
1410
|
progress: 0.,
|
|
1428
1411
|
addressesByContractName,
|
|
1429
|
-
contractNameByAddress: partition.contractNameByAddress,
|
|
1430
1412
|
})
|
|
1431
1413
|
chunkFromBlock := chunkToBlock + 1
|
|
1432
1414
|
chunkIdx := chunkIdx.contents + 1
|
|
@@ -1448,7 +1430,6 @@ let pushQueriesForRange = (
|
|
|
1448
1430
|
chainId: 0,
|
|
1449
1431
|
progress: 0.,
|
|
1450
1432
|
addressesByContractName,
|
|
1451
|
-
contractNameByAddress: partition.contractNameByAddress,
|
|
1452
1433
|
})
|
|
1453
1434
|
}
|
|
1454
1435
|
}
|
|
@@ -1478,11 +1459,16 @@ let getNextQuery = (
|
|
|
1478
1459
|
!isOnBlockBehindTheHead,
|
|
1479
1460
|
)
|
|
1480
1461
|
|
|
1481
|
-
//
|
|
1482
|
-
// flight) so processing always has buffer without ballooning memory.
|
|
1483
|
-
//
|
|
1462
|
+
// Fetch at most `budget` items past the ready frontier (plus what's already
|
|
1463
|
+
// in flight) so processing always has buffer without ballooning memory.
|
|
1464
|
+
// budget already excludes items at/below the frontier (they're in the shared
|
|
1465
|
+
// totalReadyCount), so offset the index by bufferReadyCount — otherwise the
|
|
1466
|
+
// ready prefix is subtracted twice and the buffer caps at a fraction of its
|
|
1467
|
+
// target. A partition that fetched further is skipped until the buffer drains.
|
|
1484
1468
|
let maxQueryBlockNumber = {
|
|
1485
|
-
switch buffer->Array.get(
|
|
1469
|
+
switch buffer->Array.get(
|
|
1470
|
+
fetchState->bufferReadyCount + budget + chainPendingBudget->Float.toInt - 1,
|
|
1471
|
+
) {
|
|
1486
1472
|
| Some(item) =>
|
|
1487
1473
|
// Just in case check that we don't query beyond the current block
|
|
1488
1474
|
Pervasives.min(item->Internal.getItemBlockNumber, knownHeight)
|
|
@@ -1641,6 +1627,7 @@ let make = (
|
|
|
1641
1627
|
~startBlock,
|
|
1642
1628
|
~endBlock,
|
|
1643
1629
|
~eventConfigs: array<Internal.eventConfig>,
|
|
1630
|
+
~contractConfigs: dict<IndexingAddresses.contractConfig>,
|
|
1644
1631
|
~addresses: array<Internal.indexingAddress>,
|
|
1645
1632
|
~maxAddrInPartition,
|
|
1646
1633
|
~chainId,
|
|
@@ -1659,31 +1646,8 @@ let make = (
|
|
|
1659
1646
|
let notDependingOnAddresses = []
|
|
1660
1647
|
let normalEventConfigs = []
|
|
1661
1648
|
let contractNamesWithNormalEvents = Utils.Set.make()
|
|
1662
|
-
let indexingAddresses = Dict.make()
|
|
1663
|
-
let contractConfigs: dict<contractConfig> = Dict.make()
|
|
1664
1649
|
|
|
1665
1650
|
eventConfigs->Array.forEach(ec => {
|
|
1666
|
-
switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(ec.contractName) {
|
|
1667
|
-
| Some({startBlock}) =>
|
|
1668
|
-
contractConfigs->Dict.set(
|
|
1669
|
-
ec.contractName,
|
|
1670
|
-
{
|
|
1671
|
-
startBlock: switch (startBlock, ec.startBlock) {
|
|
1672
|
-
| (Some(a), Some(b)) => Some(Pervasives.min(a, b))
|
|
1673
|
-
| (Some(_) as s, None) | (None, Some(_) as s) => s
|
|
1674
|
-
| (None, None) => None
|
|
1675
|
-
},
|
|
1676
|
-
},
|
|
1677
|
-
)
|
|
1678
|
-
| None =>
|
|
1679
|
-
contractConfigs->Dict.set(
|
|
1680
|
-
ec.contractName,
|
|
1681
|
-
{
|
|
1682
|
-
startBlock: ec.startBlock,
|
|
1683
|
-
},
|
|
1684
|
-
)
|
|
1685
|
-
}
|
|
1686
|
-
|
|
1687
1651
|
if ec.dependsOnAddresses {
|
|
1688
1652
|
normalEventConfigs->Array.push(ec)
|
|
1689
1653
|
contractNamesWithNormalEvents->Utils.Set.add(ec.contractName)->ignore
|
|
@@ -1703,7 +1667,6 @@ let make = (
|
|
|
1703
1667
|
eventConfigs: notDependingOnAddresses,
|
|
1704
1668
|
},
|
|
1705
1669
|
addressesByContractName: Dict.make(),
|
|
1706
|
-
contractNameByAddress: Dict.make(), // derived in make
|
|
1707
1670
|
mergeBlock: None,
|
|
1708
1671
|
dynamicContract: None,
|
|
1709
1672
|
mutPendingQueries: [],
|
|
@@ -1724,32 +1687,16 @@ let make = (
|
|
|
1724
1687
|
|
|
1725
1688
|
addresses->Array.forEach(contract => {
|
|
1726
1689
|
let contractName = contract.contractName
|
|
1727
|
-
let contractStartBlock = switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(
|
|
1728
|
-
contractName,
|
|
1729
|
-
) {
|
|
1730
|
-
| Some({startBlock}) => startBlock
|
|
1731
|
-
| None => None
|
|
1732
|
-
}
|
|
1733
|
-
let ia: indexingAddress = {
|
|
1734
|
-
address: contract.address,
|
|
1735
|
-
contractName: contract.contractName,
|
|
1736
|
-
registrationBlock: contract.registrationBlock,
|
|
1737
|
-
effectiveStartBlock: deriveEffectiveStartBlock(
|
|
1738
|
-
~registrationBlock=contract.registrationBlock,
|
|
1739
|
-
~contractStartBlock,
|
|
1740
|
-
),
|
|
1741
|
-
}
|
|
1742
|
-
// Track the address on fetchState regardless of whether it currently has
|
|
1743
|
-
// matching events. This way, if the config is updated later to add events
|
|
1744
|
-
// for this contract, the address is already known.
|
|
1745
|
-
indexingAddresses->Dict.set(contract.address->Address.toString, ia)
|
|
1746
1690
|
|
|
1747
1691
|
// Only addresses whose contract has events that depend on addresses get
|
|
1748
1692
|
// registered for active fetching via partitions.
|
|
1749
1693
|
if contractNamesWithNormalEvents->Utils.Set.has(contractName) {
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1694
|
+
registeringContractsByContract
|
|
1695
|
+
->Utils.Dict.getOrInsertEmptyDict(contractName)
|
|
1696
|
+
->Dict.set(
|
|
1697
|
+
contract.address->Address.toString,
|
|
1698
|
+
IndexingAddresses.makeIndexingAddress(~contract, ~contractConfigs),
|
|
1699
|
+
)
|
|
1753
1700
|
|
|
1754
1701
|
// Detect dynamic contracts by registrationBlock
|
|
1755
1702
|
if contract.registrationBlock !== -1 {
|
|
@@ -1809,7 +1756,6 @@ let make = (
|
|
|
1809
1756
|
endBlock,
|
|
1810
1757
|
latestOnBlockBlockNumber,
|
|
1811
1758
|
normalSelection,
|
|
1812
|
-
indexingAddresses,
|
|
1813
1759
|
blockLag,
|
|
1814
1760
|
onBlockConfigs,
|
|
1815
1761
|
maxOnBlockBufferSize,
|
|
@@ -1818,8 +1764,6 @@ let make = (
|
|
|
1818
1764
|
firstEventBlock,
|
|
1819
1765
|
}
|
|
1820
1766
|
|
|
1821
|
-
let numAddresses = indexingAddresses->Utils.Dict.size
|
|
1822
|
-
Prometheus.IndexingAddresses.set(~addressesCount=numAddresses, ~chainId)
|
|
1823
1767
|
Prometheus.IndexingPartitions.set(
|
|
1824
1768
|
~partitionsCount=optimizedPartitions->OptimizedPartitions.count,
|
|
1825
1769
|
~chainId,
|
|
@@ -1836,25 +1780,6 @@ let make = (
|
|
|
1836
1780
|
|
|
1837
1781
|
let bufferSize = ({buffer}: t) => buffer->Array.length
|
|
1838
1782
|
|
|
1839
|
-
// Number of buffered items at or below the ready frontier (processable now,
|
|
1840
|
-
// i.e. not stuck behind a gap from a lagging partition or out-of-order chunk).
|
|
1841
|
-
// The buffer is kept sorted, so binary-search the frontier in O(log n).
|
|
1842
|
-
let bufferReadyCount = (fetchState: t) => {
|
|
1843
|
-
let frontier = fetchState->bufferBlockNumber
|
|
1844
|
-
let buffer = fetchState.buffer
|
|
1845
|
-
let lo = ref(0)
|
|
1846
|
-
let hi = ref(buffer->Array.length)
|
|
1847
|
-
while lo.contents < hi.contents {
|
|
1848
|
-
let mid = (lo.contents + hi.contents) / 2
|
|
1849
|
-
if buffer->Array.getUnsafe(mid)->Internal.getItemBlockNumber <= frontier {
|
|
1850
|
-
lo := mid + 1
|
|
1851
|
-
} else {
|
|
1852
|
-
hi := mid
|
|
1853
|
-
}
|
|
1854
|
-
}
|
|
1855
|
-
lo.contents
|
|
1856
|
-
}
|
|
1857
|
-
|
|
1858
1783
|
let rollbackPendingQueries = (mutPendingQueries: array<pendingQuery>, ~targetBlockNumber) => {
|
|
1859
1784
|
// - Remove queries where fromBlock > target
|
|
1860
1785
|
// - Cap fetchedBlock at target where fetchedBlock > target
|
|
@@ -1886,18 +1811,11 @@ Always recreates optimized partitions to avoid duplicate addresses:
|
|
|
1886
1811
|
- Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
|
|
1887
1812
|
- Non-wildcard with lfb > target: delete, track addresses for recreation
|
|
1888
1813
|
*/
|
|
1889
|
-
let rollback = (fetchState: t, ~targetBlockNumber) => {
|
|
1890
|
-
// Step 1:
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
fetchState.indexingAddresses->Utils.Dict.forEachWithKey((indexingContract, address) => {
|
|
1895
|
-
if indexingContract.registrationBlock > targetBlockNumber {
|
|
1896
|
-
let _ = addressesToRemove->Utils.Set.add(address->Address.unsafeFromString)
|
|
1897
|
-
} else {
|
|
1898
|
-
indexingAddresses->Dict.set(address, indexingContract)
|
|
1899
|
-
}
|
|
1900
|
-
})
|
|
1814
|
+
let rollback = (fetchState: t, ~indexingAddresses: IndexingAddresses.t, ~targetBlockNumber) => {
|
|
1815
|
+
// Step 1: Prune addresses registered after the target block. The pruned index is
|
|
1816
|
+
// then the source of truth for partition cleanup below — an address survives iff
|
|
1817
|
+
// it's still in the index.
|
|
1818
|
+
indexingAddresses->IndexingAddresses.rollbackInPlace(~targetBlockNumber)
|
|
1901
1819
|
|
|
1902
1820
|
// Step 2: Categorize partitions
|
|
1903
1821
|
let keptPartitions = []
|
|
@@ -1927,18 +1845,12 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
|
|
|
1927
1845
|
| _ if p.latestFetchedBlock.blockNumber > targetBlockNumber =>
|
|
1928
1846
|
p.addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
|
|
1929
1847
|
addresses->Array.forEach(address => {
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
indexingAddresses
|
|
1933
|
-
->Utils.Dict.dangerouslyGetNonOption(address->Address.toString)
|
|
1934
|
-
->Option.isSome
|
|
1935
|
-
) {
|
|
1848
|
+
switch indexingAddresses->IndexingAddresses.get(address->Address.toString) {
|
|
1849
|
+
| Some(indexingContract) =>
|
|
1936
1850
|
let registeringContracts =
|
|
1937
1851
|
registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(contractName)
|
|
1938
|
-
registeringContracts->Dict.set(
|
|
1939
|
-
|
|
1940
|
-
indexingAddresses->Dict.getUnsafe(address->Address.toString),
|
|
1941
|
-
)
|
|
1852
|
+
registeringContracts->Dict.set(address->Address.toString, indexingContract)
|
|
1853
|
+
| None => ()
|
|
1942
1854
|
}
|
|
1943
1855
|
})
|
|
1944
1856
|
})
|
|
@@ -1951,11 +1863,13 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
|
|
|
1951
1863
|
| other => other
|
|
1952
1864
|
}
|
|
1953
1865
|
|
|
1954
|
-
//
|
|
1866
|
+
// Drop addresses pruned from the index
|
|
1955
1867
|
let rollbackedAddressesByContractName = Dict.make()
|
|
1956
1868
|
addressesByContractName->Utils.Dict.forEachWithKey((addresses, contractName) => {
|
|
1957
1869
|
let keptAddresses =
|
|
1958
|
-
addresses->Array.filter(address =>
|
|
1870
|
+
addresses->Array.filter(address =>
|
|
1871
|
+
indexingAddresses->IndexingAddresses.get(address->Address.toString)->Option.isSome
|
|
1872
|
+
)
|
|
1959
1873
|
if keptAddresses->Array.length > 0 {
|
|
1960
1874
|
rollbackedAddressesByContractName->Dict.set(contractName, keptAddresses)
|
|
1961
1875
|
}
|
|
@@ -1999,7 +1913,6 @@ let rollback = (fetchState: t, ~targetBlockNumber) => {
|
|
|
1999
1913
|
),
|
|
2000
1914
|
}->updateInternal(
|
|
2001
1915
|
~optimizedPartitions,
|
|
2002
|
-
~indexingAddresses,
|
|
2003
1916
|
~mutItems=fetchState.buffer->Array.filter(item =>
|
|
2004
1917
|
switch item {
|
|
2005
1918
|
| Event({blockNumber})
|