envio 3.1.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +83 -11
- package/fuel.schema.json +83 -11
- package/index.d.ts +184 -3
- package/package.json +6 -6
- package/src/Batch.res +2 -2
- package/src/ChainFetcher.res +27 -3
- package/src/ChainFetcher.res.mjs +17 -3
- package/src/ChainManager.res +163 -0
- package/src/ChainManager.res.mjs +136 -0
- package/src/Config.res +213 -30
- package/src/Config.res.mjs +102 -41
- package/src/Core.res +16 -10
- package/src/Ecosystem.res +0 -3
- package/src/Env.res +2 -2
- package/src/Env.res.mjs +2 -2
- package/src/Envio.res +101 -2
- package/src/Envio.res.mjs +2 -3
- package/src/EventConfigBuilder.res +52 -0
- package/src/EventConfigBuilder.res.mjs +32 -0
- package/src/EventUtils.res +2 -2
- package/src/FetchState.res +126 -71
- package/src/FetchState.res.mjs +73 -51
- package/src/GlobalState.res +219 -363
- package/src/GlobalState.res.mjs +314 -491
- package/src/GlobalStateManager.res +49 -59
- package/src/GlobalStateManager.res.mjs +5 -4
- package/src/GlobalStateManager.resi +1 -1
- package/src/HandlerLoader.res +12 -1
- package/src/HandlerLoader.res.mjs +6 -1
- package/src/HandlerRegister.res +9 -9
- package/src/HandlerRegister.res.mjs +9 -9
- package/src/Hasura.res +102 -32
- package/src/Hasura.res.mjs +88 -34
- package/src/InMemoryStore.res +10 -1
- package/src/InMemoryStore.res.mjs +4 -1
- package/src/InMemoryTable.res +83 -136
- package/src/InMemoryTable.res.mjs +57 -86
- package/src/Internal.res +54 -5
- package/src/Internal.res.mjs +2 -8
- package/src/LazyLoader.res +2 -2
- package/src/LazyLoader.res.mjs +3 -3
- package/src/LoadLayer.res +47 -60
- package/src/LoadLayer.res.mjs +28 -50
- package/src/LoadLayer.resi +2 -5
- package/src/LogSelection.res +4 -4
- package/src/LogSelection.res.mjs +5 -7
- package/src/Logging.res +1 -1
- package/src/Main.res +61 -2
- package/src/Main.res.mjs +37 -1
- package/src/Persistence.res +3 -16
- package/src/PgStorage.res +125 -114
- package/src/PgStorage.res.mjs +112 -95
- package/src/Ports.res +5 -0
- package/src/Ports.res.mjs +9 -0
- package/src/Prometheus.res +3 -3
- package/src/Prometheus.res.mjs +4 -4
- package/src/ReorgDetection.res +4 -4
- package/src/ReorgDetection.res.mjs +4 -5
- package/src/SafeCheckpointTracking.res +16 -16
- package/src/SafeCheckpointTracking.res.mjs +2 -2
- package/src/SimulateItems.res +10 -14
- package/src/SimulateItems.res.mjs +5 -2
- package/src/Sink.res +1 -1
- package/src/Sink.res.mjs +1 -2
- package/src/SvmTypes.res +9 -0
- package/src/SvmTypes.res.mjs +14 -0
- package/src/TestIndexer.res +17 -57
- package/src/TestIndexer.res.mjs +14 -48
- package/src/TestIndexerProxyStorage.res +23 -23
- package/src/TestIndexerProxyStorage.res.mjs +12 -15
- package/src/Throttler.res +2 -2
- package/src/Time.res +2 -2
- package/src/Time.res.mjs +2 -2
- package/src/UserContext.res +19 -118
- package/src/UserContext.res.mjs +10 -66
- package/src/Utils.res +15 -15
- package/src/Utils.res.mjs +7 -8
- package/src/adapters/MarkBatchProcessedAdapter.res +5 -0
- package/src/adapters/MarkBatchProcessedAdapter.res.mjs +14 -0
- package/src/bindings/BigDecimal.res +1 -1
- package/src/bindings/BigDecimal.res.mjs +2 -2
- package/src/bindings/ClickHouse.res +8 -6
- package/src/bindings/ClickHouse.res.mjs +5 -5
- package/src/bindings/Hrtime.res +1 -1
- package/src/bindings/Pino.res +2 -2
- package/src/bindings/Pino.res.mjs +3 -4
- package/src/db/EntityFilter.res +410 -0
- package/src/db/EntityFilter.res.mjs +424 -0
- package/src/db/EntityHistory.res +1 -1
- package/src/db/EntityHistory.res.mjs +1 -1
- package/src/db/InternalTable.res +10 -10
- package/src/db/InternalTable.res.mjs +41 -45
- package/src/db/Schema.res +2 -2
- package/src/db/Schema.res.mjs +3 -3
- package/src/db/Table.res +106 -22
- package/src/db/Table.res.mjs +84 -35
- package/src/sources/EventRouter.res +67 -2
- package/src/sources/EventRouter.res.mjs +45 -3
- package/src/sources/Evm.res +0 -7
- package/src/sources/Evm.res.mjs +0 -15
- package/src/sources/EvmChain.res +1 -1
- package/src/sources/EvmChain.res.mjs +1 -2
- package/src/sources/EvmRpcClient.res +42 -0
- package/src/sources/EvmRpcClient.res.mjs +64 -0
- package/src/sources/Fuel.res +0 -7
- package/src/sources/Fuel.res.mjs +0 -15
- package/src/sources/HyperFuelSource.res +5 -4
- package/src/sources/HyperFuelSource.res.mjs +2 -2
- package/src/sources/HyperSyncClient.res +9 -5
- package/src/sources/HyperSyncClient.res.mjs +2 -2
- package/src/sources/HyperSyncHeightStream.res +2 -2
- package/src/sources/HyperSyncHeightStream.res.mjs +2 -2
- package/src/sources/HyperSyncSource.res +10 -9
- package/src/sources/HyperSyncSource.res.mjs +4 -4
- package/src/sources/Rpc.res +1 -5
- package/src/sources/Rpc.res.mjs +1 -9
- package/src/sources/RpcSource.res +57 -21
- package/src/sources/RpcSource.res.mjs +47 -20
- package/src/sources/RpcWebSocketHeightStream.res +1 -1
- package/src/sources/SourceManager.res +3 -2
- package/src/sources/SourceManager.res.mjs +1 -1
- package/src/sources/Svm.res +3 -10
- package/src/sources/Svm.res.mjs +4 -18
- package/src/sources/SvmHyperSyncClient.res +265 -0
- package/src/sources/SvmHyperSyncClient.res.mjs +28 -0
- package/src/sources/SvmHyperSyncSource.res +638 -0
- package/src/sources/SvmHyperSyncSource.res.mjs +557 -0
- package/src/tui/Tui.res +9 -2
- package/src/tui/Tui.res.mjs +18 -3
- package/src/tui/components/BufferedProgressBar.res +2 -2
- package/src/tui/components/TuiData.res +3 -0
- package/svm.schema.json +523 -14
- package/src/TableIndices.res +0 -115
- package/src/TableIndices.res.mjs +0 -144
|
@@ -288,6 +288,37 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcar
|
|
|
288
288
|
};
|
|
289
289
|
}
|
|
290
290
|
|
|
291
|
+
function buildSvmInstructionEventConfig(contractName, instructionName, programId, discriminator, discriminatorByteLen, includeTransaction, includeLogs, includeTokenBalances, accountFilters, isInner, isWildcard, handler, contractRegister, accountsOpt, argsOpt, definedTypesOpt, startBlock) {
|
|
292
|
+
let accounts = accountsOpt !== undefined ? accountsOpt : [];
|
|
293
|
+
let args = argsOpt !== undefined ? argsOpt : null;
|
|
294
|
+
let definedTypes = definedTypesOpt !== undefined ? definedTypesOpt : null;
|
|
295
|
+
let paramsSchema = Utils.Schema.coerceToJsonPgType(S$RescriptSchema.json(false));
|
|
296
|
+
return {
|
|
297
|
+
id: discriminator !== undefined ? discriminator : "none",
|
|
298
|
+
name: instructionName,
|
|
299
|
+
contractName: contractName,
|
|
300
|
+
isWildcard: isWildcard,
|
|
301
|
+
filterByAddresses: false,
|
|
302
|
+
dependsOnAddresses: !isWildcard,
|
|
303
|
+
handler: handler,
|
|
304
|
+
contractRegister: contractRegister,
|
|
305
|
+
paramsRawEventSchema: paramsSchema,
|
|
306
|
+
simulateParamsSchema: paramsSchema,
|
|
307
|
+
startBlock: startBlock,
|
|
308
|
+
programId: programId,
|
|
309
|
+
discriminator: discriminator,
|
|
310
|
+
discriminatorByteLen: discriminatorByteLen,
|
|
311
|
+
includeTransaction: includeTransaction,
|
|
312
|
+
includeLogs: includeLogs,
|
|
313
|
+
includeTokenBalances: includeTokenBalances,
|
|
314
|
+
accountFilters: accountFilters,
|
|
315
|
+
isInner: isInner,
|
|
316
|
+
accounts: accounts,
|
|
317
|
+
args: args,
|
|
318
|
+
definedTypes: definedTypes
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
|
|
291
322
|
function buildFuelEventConfig(contractName, eventName, kind, sighash, rawAbi, isWildcard, handler, contractRegister, startBlock) {
|
|
292
323
|
let fuelKind;
|
|
293
324
|
switch (kind) {
|
|
@@ -363,6 +394,7 @@ export {
|
|
|
363
394
|
alwaysIncludedBlockFields,
|
|
364
395
|
resolveFieldSelection,
|
|
365
396
|
buildEvmEventConfig,
|
|
397
|
+
buildSvmInstructionEventConfig,
|
|
366
398
|
buildFuelEventConfig,
|
|
367
399
|
}
|
|
368
400
|
/* paramMetaSchema Not a pure module */
|
package/src/EventUtils.res
CHANGED
|
@@ -45,8 +45,8 @@ let packEventIndex = (~blockNumber, ~logIndex) => {
|
|
|
45
45
|
// let logIndexMask = 65535->BigInt.fromInt
|
|
46
46
|
// let logIndex = packedEventIndex->BigInt.bitwiseAnd(logIndexMask)
|
|
47
47
|
// {
|
|
48
|
-
// blockNumber: blockNumber->BigInt.toString->
|
|
49
|
-
// logIndex: logIndex->BigInt.toString->
|
|
48
|
+
// blockNumber: blockNumber->BigInt.toString->Int.fromString->Option.getUnsafe,
|
|
49
|
+
// logIndex: logIndex->BigInt.toString->Int.fromString->Option.getUnsafe,
|
|
50
50
|
// }
|
|
51
51
|
// }
|
|
52
52
|
|
package/src/FetchState.res
CHANGED
|
@@ -336,7 +336,7 @@ module OptimizedPartitions = {
|
|
|
336
336
|
let _ = newPartitions->Array.sort(ascSortFn)
|
|
337
337
|
|
|
338
338
|
let partitionsCount = newPartitions->Array.length
|
|
339
|
-
let idsInAscOrder =
|
|
339
|
+
let idsInAscOrder = Utils.Array.jsArrayCreate(partitionsCount)
|
|
340
340
|
let entities = Dict.make()
|
|
341
341
|
for idx in 0 to partitionsCount - 1 {
|
|
342
342
|
let p = newPartitions->Array.getUnsafe(idx)
|
|
@@ -555,6 +555,63 @@ let blockItemLogIndex = 16777216
|
|
|
555
555
|
|
|
556
556
|
let numAddresses = fetchState => fetchState.indexingAddresses->Utils.Dict.size
|
|
557
557
|
|
|
558
|
+
// Appends Block items produced by the onBlock handlers for every block in
|
|
559
|
+
// (fromBlock, maxBlockNumber] into mutItems and returns the new
|
|
560
|
+
// latestOnBlockBlockNumber pointer. targetBufferSize bounds how many items
|
|
561
|
+
// are generated at once to prevent OOM.
|
|
562
|
+
let appendOnBlockItems = (
|
|
563
|
+
~mutItems: array<Internal.item>,
|
|
564
|
+
~onBlockConfigs: array<Internal.onBlockConfig>,
|
|
565
|
+
~indexerStartBlock,
|
|
566
|
+
~fromBlock,
|
|
567
|
+
~maxBlockNumber,
|
|
568
|
+
~targetBufferSize,
|
|
569
|
+
) => {
|
|
570
|
+
let newItemsCounter = ref(0)
|
|
571
|
+
let latestOnBlockBlockNumber = ref(fromBlock)
|
|
572
|
+
|
|
573
|
+
// Simply iterate over every block
|
|
574
|
+
// could have a better algorithm to iterate over blocks in a more efficient way
|
|
575
|
+
// but raw loops are fast enough
|
|
576
|
+
while (
|
|
577
|
+
latestOnBlockBlockNumber.contents < maxBlockNumber &&
|
|
578
|
+
// Additional safeguard to prevent OOM
|
|
579
|
+
newItemsCounter.contents <= targetBufferSize
|
|
580
|
+
) {
|
|
581
|
+
let blockNumber = latestOnBlockBlockNumber.contents + 1
|
|
582
|
+
latestOnBlockBlockNumber := blockNumber
|
|
583
|
+
|
|
584
|
+
for configIdx in 0 to onBlockConfigs->Array.length - 1 {
|
|
585
|
+
let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
|
|
586
|
+
|
|
587
|
+
let handlerStartBlock = switch onBlockConfig.startBlock {
|
|
588
|
+
| Some(startBlock) => startBlock
|
|
589
|
+
| None => indexerStartBlock
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
if (
|
|
593
|
+
blockNumber >= handlerStartBlock &&
|
|
594
|
+
switch onBlockConfig.endBlock {
|
|
595
|
+
| Some(endBlock) => blockNumber <= endBlock
|
|
596
|
+
| None => true
|
|
597
|
+
} &&
|
|
598
|
+
(blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0
|
|
599
|
+
) {
|
|
600
|
+
mutItems->Array.push(
|
|
601
|
+
Block({
|
|
602
|
+
onBlockConfig,
|
|
603
|
+
blockNumber,
|
|
604
|
+
logIndex: blockItemLogIndex + onBlockConfig.index,
|
|
605
|
+
}),
|
|
606
|
+
)
|
|
607
|
+
newItemsCounter := newItemsCounter.contents + 1
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
latestOnBlockBlockNumber.contents
|
|
613
|
+
}
|
|
614
|
+
|
|
558
615
|
/*
|
|
559
616
|
Update fetchState, merge registers and recompute derived values.
|
|
560
617
|
Runs partition optimization when partitions change.
|
|
@@ -582,7 +639,7 @@ let updateInternal = (
|
|
|
582
639
|
let maxBlockNumber = switch switch mutItemsRef.contents {
|
|
583
640
|
| Some(mutItems) => mutItems
|
|
584
641
|
| None => fetchState.buffer
|
|
585
|
-
}->
|
|
642
|
+
}->Array.get(fetchState.targetBufferSize - 1) {
|
|
586
643
|
| Some(item) => item->Internal.getItemBlockNumber
|
|
587
644
|
| None =>
|
|
588
645
|
switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
@@ -597,49 +654,14 @@ let updateInternal = (
|
|
|
597
654
|
}
|
|
598
655
|
mutItemsRef := Some(mutItems)
|
|
599
656
|
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
// Additional safeguard to prevent OOM
|
|
609
|
-
newItemsCounter.contents <= fetchState.targetBufferSize
|
|
610
|
-
) {
|
|
611
|
-
let blockNumber = latestOnBlockBlockNumber.contents + 1
|
|
612
|
-
latestOnBlockBlockNumber := blockNumber
|
|
613
|
-
|
|
614
|
-
for configIdx in 0 to onBlockConfigs->Array.length - 1 {
|
|
615
|
-
let onBlockConfig = onBlockConfigs->Array.getUnsafe(configIdx)
|
|
616
|
-
|
|
617
|
-
let handlerStartBlock = switch onBlockConfig.startBlock {
|
|
618
|
-
| Some(startBlock) => startBlock
|
|
619
|
-
| None => fetchState.startBlock
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
if (
|
|
623
|
-
blockNumber >= handlerStartBlock &&
|
|
624
|
-
switch onBlockConfig.endBlock {
|
|
625
|
-
| Some(endBlock) => blockNumber <= endBlock
|
|
626
|
-
| None => true
|
|
627
|
-
} &&
|
|
628
|
-
(blockNumber - handlerStartBlock)->Pervasives.mod(onBlockConfig.interval) === 0
|
|
629
|
-
) {
|
|
630
|
-
mutItems->Array.push(
|
|
631
|
-
Block({
|
|
632
|
-
onBlockConfig,
|
|
633
|
-
blockNumber,
|
|
634
|
-
logIndex: blockItemLogIndex + onBlockConfig.index,
|
|
635
|
-
}),
|
|
636
|
-
)
|
|
637
|
-
newItemsCounter := newItemsCounter.contents + 1
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
latestOnBlockBlockNumber.contents
|
|
657
|
+
appendOnBlockItems(
|
|
658
|
+
~mutItems,
|
|
659
|
+
~onBlockConfigs,
|
|
660
|
+
~indexerStartBlock=fetchState.startBlock,
|
|
661
|
+
~fromBlock=fetchState.latestOnBlockBlockNumber,
|
|
662
|
+
~maxBlockNumber,
|
|
663
|
+
~targetBufferSize=fetchState.targetBufferSize,
|
|
664
|
+
)
|
|
643
665
|
}
|
|
644
666
|
}
|
|
645
667
|
|
|
@@ -952,6 +974,10 @@ let registerDynamicContracts = (
|
|
|
952
974
|
// so that later conflicting registrations are detected, and are persisted
|
|
953
975
|
// to envio_addresses so they can be picked up on restart with updated config.
|
|
954
976
|
let noEventsAddresses: dict<indexingAddress> = Dict.make()
|
|
977
|
+
// Batch-level view of all addresses registered so far (across contracts,
|
|
978
|
+
// including no-events ones), so two contracts registering the same address
|
|
979
|
+
// within one batch conflict the same way as against indexingAddresses.
|
|
980
|
+
let registeringAddresses: dict<indexingAddress> = Dict.make()
|
|
955
981
|
|
|
956
982
|
for itemIdx in 0 to items->Array.length - 1 {
|
|
957
983
|
let item = items->Array.getUnsafe(itemIdx)
|
|
@@ -999,9 +1025,7 @@ let registerDynamicContracts = (
|
|
|
999
1025
|
}
|
|
1000
1026
|
shouldRemove := true
|
|
1001
1027
|
| None =>
|
|
1002
|
-
let
|
|
1003
|
-
registeringContractsByContract->Utils.Dict.getOrInsertEmptyDict(dc.contractName)
|
|
1004
|
-
let shouldUpdate = switch registeringContracts->Utils.Dict.dangerouslyGetNonOption(
|
|
1028
|
+
let shouldUpdate = switch registeringAddresses->Utils.Dict.dangerouslyGetNonOption(
|
|
1005
1029
|
dc.address->Address.toString,
|
|
1006
1030
|
) {
|
|
1007
1031
|
| Some(registeringContract) if registeringContract.contractName != dc.contractName =>
|
|
@@ -1024,7 +1048,10 @@ let registerDynamicContracts = (
|
|
|
1024
1048
|
earliestRegisteringEventBlockNumber.contents,
|
|
1025
1049
|
dcWithStartBlock.effectiveStartBlock,
|
|
1026
1050
|
)
|
|
1027
|
-
|
|
1051
|
+
registeringContractsByContract
|
|
1052
|
+
->Utils.Dict.getOrInsertEmptyDict(dc.contractName)
|
|
1053
|
+
->Dict.set(dc.address->Address.toString, dcWithStartBlock)
|
|
1054
|
+
registeringAddresses->Dict.set(dc.address->Address.toString, dcWithStartBlock)
|
|
1028
1055
|
} else {
|
|
1029
1056
|
shouldRemove := true
|
|
1030
1057
|
}
|
|
@@ -1051,11 +1078,14 @@ let registerDynamicContracts = (
|
|
|
1051
1078
|
}
|
|
1052
1079
|
shouldRemove := true
|
|
1053
1080
|
| None =>
|
|
1054
|
-
switch
|
|
1081
|
+
switch registeringAddresses->Utils.Dict.dangerouslyGetNonOption(
|
|
1055
1082
|
dc.address->Address.toString,
|
|
1056
1083
|
) {
|
|
1057
|
-
| Some(
|
|
1058
|
-
|
|
1084
|
+
| Some(existingContract) =>
|
|
1085
|
+
if existingContract.contractName != dc.contractName {
|
|
1086
|
+
fetchState->warnDifferentContractType(~existingContract, ~dc=dcAsIndexingAddress)
|
|
1087
|
+
}
|
|
1088
|
+
// Otherwise already queued for persistence by an earlier item in this batch.
|
|
1059
1089
|
shouldRemove := true
|
|
1060
1090
|
| None =>
|
|
1061
1091
|
let logger = Logging.createChild(
|
|
@@ -1070,6 +1100,7 @@ let registerDynamicContracts = (
|
|
|
1070
1100
|
// skip partition registration since there's nothing to fetch.
|
|
1071
1101
|
logger->Logging.childWarn(`Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`)
|
|
1072
1102
|
noEventsAddresses->Dict.set(dc.address->Address.toString, dcAsIndexingAddress)
|
|
1103
|
+
registeringAddresses->Dict.set(dc.address->Address.toString, dcAsIndexingAddress)
|
|
1073
1104
|
}
|
|
1074
1105
|
}
|
|
1075
1106
|
}
|
|
@@ -1486,7 +1517,7 @@ let getNextQuery = (
|
|
|
1486
1517
|
}
|
|
1487
1518
|
|
|
1488
1519
|
let hasReadyItem = ({buffer} as fetchState: t) => {
|
|
1489
|
-
switch buffer->
|
|
1520
|
+
switch buffer->Array.get(0) {
|
|
1490
1521
|
| Some(item) => item->Internal.getItemBlockNumber <= fetchState->bufferBlockNumber
|
|
1491
1522
|
| None => false
|
|
1492
1523
|
}
|
|
@@ -1497,7 +1528,7 @@ let getReadyItemsCount = (fetchState: t, ~targetSize: int, ~fromItem) => {
|
|
|
1497
1528
|
let acc = ref(0)
|
|
1498
1529
|
let isFinished = ref(false)
|
|
1499
1530
|
while !isFinished.contents {
|
|
1500
|
-
switch fetchState.buffer->
|
|
1531
|
+
switch fetchState.buffer->Array.get(fromItem + acc.contents) {
|
|
1501
1532
|
| Some(item) =>
|
|
1502
1533
|
let itemBlockNumber = item->Internal.getItemBlockNumber
|
|
1503
1534
|
if itemBlockNumber <= readyBlockNumber.contents {
|
|
@@ -1661,35 +1692,59 @@ let make = (
|
|
|
1661
1692
|
)
|
|
1662
1693
|
}
|
|
1663
1694
|
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1695
|
+
// On resume knownHeight is restored from the DB but the buffer starts empty.
|
|
1696
|
+
// For onBlock-only indexers (e.g. SVM onSlot) there are no partitions to drive
|
|
1697
|
+
// fetching, so without seeding the buffer here getNextQuery would return
|
|
1698
|
+
// NothingToQuery and the indexer would get stuck.
|
|
1699
|
+
let buffer = []
|
|
1700
|
+
let latestOnBlockBlockNumber = if knownHeight > 0 && onBlockConfigs->Utils.Array.notEmpty {
|
|
1701
|
+
let maxBlockNumber = switch optimizedPartitions->OptimizedPartitions.getLatestFullyFetchedBlock {
|
|
1702
|
+
| None => knownHeight
|
|
1703
|
+
| Some(latestFullyFetchedBlock) => latestFullyFetchedBlock.blockNumber
|
|
1704
|
+
}
|
|
1705
|
+
appendOnBlockItems(
|
|
1706
|
+
~mutItems=buffer,
|
|
1707
|
+
~onBlockConfigs,
|
|
1708
|
+
~indexerStartBlock=startBlock,
|
|
1709
|
+
~fromBlock=progressBlockNumber,
|
|
1710
|
+
~maxBlockNumber,
|
|
1711
|
+
~targetBufferSize,
|
|
1712
|
+
)
|
|
1713
|
+
} else {
|
|
1714
|
+
progressBlockNumber
|
|
1675
1715
|
}
|
|
1676
1716
|
|
|
1677
|
-
{
|
|
1717
|
+
let fetchState = {
|
|
1678
1718
|
optimizedPartitions,
|
|
1679
1719
|
contractConfigs,
|
|
1680
1720
|
chainId,
|
|
1681
1721
|
startBlock,
|
|
1682
1722
|
endBlock,
|
|
1683
|
-
latestOnBlockBlockNumber
|
|
1723
|
+
latestOnBlockBlockNumber,
|
|
1684
1724
|
normalSelection,
|
|
1685
1725
|
indexingAddresses,
|
|
1686
1726
|
blockLag,
|
|
1687
1727
|
onBlockConfigs,
|
|
1688
1728
|
targetBufferSize,
|
|
1689
1729
|
knownHeight,
|
|
1690
|
-
buffer
|
|
1730
|
+
buffer,
|
|
1691
1731
|
firstEventBlock,
|
|
1692
1732
|
}
|
|
1733
|
+
|
|
1734
|
+
let numAddresses = indexingAddresses->Utils.Dict.size
|
|
1735
|
+
Prometheus.IndexingAddresses.set(~addressesCount=numAddresses, ~chainId)
|
|
1736
|
+
Prometheus.IndexingPartitions.set(
|
|
1737
|
+
~partitionsCount=optimizedPartitions->OptimizedPartitions.count,
|
|
1738
|
+
~chainId,
|
|
1739
|
+
)
|
|
1740
|
+
Prometheus.IndexingBufferSize.set(~bufferSize=buffer->Array.length, ~chainId)
|
|
1741
|
+
Prometheus.IndexingBufferBlockNumber.set(~blockNumber=fetchState->bufferBlockNumber, ~chainId)
|
|
1742
|
+
switch endBlock {
|
|
1743
|
+
| Some(endBlock) => Prometheus.IndexingEndBlock.set(~endBlock, ~chainId)
|
|
1744
|
+
| None => ()
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
fetchState
|
|
1693
1748
|
}
|
|
1694
1749
|
|
|
1695
1750
|
let bufferSize = ({buffer}: t) => buffer->Array.length
|
|
@@ -1905,7 +1960,7 @@ let isReadyToEnterReorgThreshold = ({endBlock, blockLag, buffer, knownHeight} as
|
|
|
1905
1960
|
|
|
1906
1961
|
let sortForUnorderedBatch = {
|
|
1907
1962
|
let hasFullBatch = ({buffer} as fetchState: t, ~batchSizeTarget) => {
|
|
1908
|
-
switch buffer->
|
|
1963
|
+
switch buffer->Array.get(batchSizeTarget - 1) {
|
|
1909
1964
|
| Some(item) => item->Internal.getItemBlockNumber <= fetchState->bufferBlockNumber
|
|
1910
1965
|
| None => false
|
|
1911
1966
|
}
|
|
@@ -1920,7 +1975,7 @@ let sortForUnorderedBatch = {
|
|
|
1920
1975
|
if totalRange <= 0 {
|
|
1921
1976
|
0.
|
|
1922
1977
|
} else {
|
|
1923
|
-
let progress = switch fetchState.buffer->
|
|
1978
|
+
let progress = switch fetchState.buffer->Array.get(0) {
|
|
1924
1979
|
| Some(item) => item->Internal.getItemBlockNumber - firstEventBlock
|
|
1925
1980
|
| None => fetchState->bufferBlockNumber - firstEventBlock
|
|
1926
1981
|
}
|
|
@@ -1955,7 +2010,7 @@ let sortForUnorderedBatch = {
|
|
|
1955
2010
|
|
|
1956
2011
|
let getProgressBlockNumberAt = ({buffer} as fetchState: t, ~index) => {
|
|
1957
2012
|
let bufferBlockNumber = fetchState->bufferBlockNumber
|
|
1958
|
-
switch buffer->
|
|
2013
|
+
switch buffer->Array.get(index) {
|
|
1959
2014
|
| Some(item) if bufferBlockNumber >= item->Internal.getItemBlockNumber =>
|
|
1960
2015
|
item->Internal.getItemBlockNumber - 1
|
|
1961
2016
|
| _ => bufferBlockNumber
|
package/src/FetchState.res.mjs
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
4
|
import * as Js_math from "@rescript/runtime/lib/es6/Js_math.js";
|
|
5
5
|
import * as Logging from "./Logging.res.mjs";
|
|
6
|
-
import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
|
|
7
6
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
8
7
|
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
9
8
|
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
@@ -271,7 +270,7 @@ function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContrac
|
|
|
271
270
|
}
|
|
272
271
|
newPartitions.sort(ascSortFn);
|
|
273
272
|
let partitionsCount = newPartitions.length;
|
|
274
|
-
let idsInAscOrder =
|
|
273
|
+
let idsInAscOrder = Array(partitionsCount);
|
|
275
274
|
let entities = {};
|
|
276
275
|
for (let idx$2 = 0; idx$2 < partitionsCount; ++idx$2) {
|
|
277
276
|
let p$1 = newPartitions[idx$2];
|
|
@@ -450,6 +449,35 @@ function numAddresses(fetchState) {
|
|
|
450
449
|
return Utils.Dict.size(fetchState.indexingAddresses);
|
|
451
450
|
}
|
|
452
451
|
|
|
452
|
+
function appendOnBlockItems(mutItems, onBlockConfigs, indexerStartBlock, fromBlock, maxBlockNumber, targetBufferSize) {
|
|
453
|
+
let newItemsCounter = 0;
|
|
454
|
+
let latestOnBlockBlockNumber = fromBlock;
|
|
455
|
+
while (latestOnBlockBlockNumber < maxBlockNumber && newItemsCounter <= targetBufferSize) {
|
|
456
|
+
let blockNumber = latestOnBlockBlockNumber + 1 | 0;
|
|
457
|
+
latestOnBlockBlockNumber = blockNumber;
|
|
458
|
+
for (let configIdx = 0, configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx) {
|
|
459
|
+
let onBlockConfig = onBlockConfigs[configIdx];
|
|
460
|
+
let startBlock = onBlockConfig.startBlock;
|
|
461
|
+
let handlerStartBlock = startBlock !== undefined ? startBlock : indexerStartBlock;
|
|
462
|
+
let tmp = false;
|
|
463
|
+
if (blockNumber >= handlerStartBlock) {
|
|
464
|
+
let endBlock = onBlockConfig.endBlock;
|
|
465
|
+
tmp = endBlock !== undefined ? blockNumber <= endBlock : true;
|
|
466
|
+
}
|
|
467
|
+
if (tmp && Primitive_int.mod_(blockNumber - handlerStartBlock | 0, onBlockConfig.interval) === 0) {
|
|
468
|
+
mutItems.push({
|
|
469
|
+
kind: 1,
|
|
470
|
+
onBlockConfig: onBlockConfig,
|
|
471
|
+
blockNumber: blockNumber,
|
|
472
|
+
logIndex: 16777216 + onBlockConfig.index | 0
|
|
473
|
+
});
|
|
474
|
+
newItemsCounter = newItemsCounter + 1 | 0;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
};
|
|
478
|
+
return latestOnBlockBlockNumber;
|
|
479
|
+
}
|
|
480
|
+
|
|
453
481
|
function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt, mutItems, blockLagOpt, knownHeightOpt) {
|
|
454
482
|
let optimizedPartitions = optimizedPartitionsOpt !== undefined ? optimizedPartitionsOpt : fetchState.optimizedPartitions;
|
|
455
483
|
let indexingAddresses = indexingAddressesOpt !== undefined ? indexingAddressesOpt : fetchState.indexingAddresses;
|
|
@@ -460,7 +488,9 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
|
|
|
460
488
|
let latestOnBlockBlockNumber;
|
|
461
489
|
if (onBlockConfigs.length !== 0) {
|
|
462
490
|
let mutItems$1 = mutItemsRef;
|
|
463
|
-
let item =
|
|
491
|
+
let item = (
|
|
492
|
+
mutItems$1 !== undefined ? mutItems$1 : fetchState.buffer
|
|
493
|
+
)[fetchState.targetBufferSize - 1 | 0];
|
|
464
494
|
let maxBlockNumber;
|
|
465
495
|
if (item !== undefined) {
|
|
466
496
|
maxBlockNumber = item.blockNumber;
|
|
@@ -472,32 +502,7 @@ function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt
|
|
|
472
502
|
let mutItems$2 = mutItemsRef;
|
|
473
503
|
let mutItems$3 = mutItems$2 !== undefined ? mutItems$2 : fetchState.buffer.slice();
|
|
474
504
|
mutItemsRef = mutItems$3;
|
|
475
|
-
|
|
476
|
-
let latestOnBlockBlockNumber$1 = fetchState.latestOnBlockBlockNumber;
|
|
477
|
-
while (latestOnBlockBlockNumber$1 < maxBlockNumber && newItemsCounter <= fetchState.targetBufferSize) {
|
|
478
|
-
let blockNumber = latestOnBlockBlockNumber$1 + 1 | 0;
|
|
479
|
-
latestOnBlockBlockNumber$1 = blockNumber;
|
|
480
|
-
for (let configIdx = 0, configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx) {
|
|
481
|
-
let onBlockConfig = onBlockConfigs[configIdx];
|
|
482
|
-
let startBlock = onBlockConfig.startBlock;
|
|
483
|
-
let handlerStartBlock = startBlock !== undefined ? startBlock : fetchState.startBlock;
|
|
484
|
-
let tmp = false;
|
|
485
|
-
if (blockNumber >= handlerStartBlock) {
|
|
486
|
-
let endBlock = onBlockConfig.endBlock;
|
|
487
|
-
tmp = endBlock !== undefined ? blockNumber <= endBlock : true;
|
|
488
|
-
}
|
|
489
|
-
if (tmp && Primitive_int.mod_(blockNumber - handlerStartBlock | 0, onBlockConfig.interval) === 0) {
|
|
490
|
-
mutItems$3.push({
|
|
491
|
-
kind: 1,
|
|
492
|
-
onBlockConfig: onBlockConfig,
|
|
493
|
-
blockNumber: blockNumber,
|
|
494
|
-
logIndex: 16777216 + onBlockConfig.index | 0
|
|
495
|
-
});
|
|
496
|
-
newItemsCounter = newItemsCounter + 1 | 0;
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
};
|
|
500
|
-
latestOnBlockBlockNumber = latestOnBlockBlockNumber$1;
|
|
505
|
+
latestOnBlockBlockNumber = appendOnBlockItems(mutItems$3, onBlockConfigs, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.targetBufferSize);
|
|
501
506
|
} else {
|
|
502
507
|
latestOnBlockBlockNumber = knownHeight;
|
|
503
508
|
}
|
|
@@ -722,6 +727,7 @@ function registerDynamicContracts(fetchState, items) {
|
|
|
722
727
|
let earliestRegisteringEventBlockNumber = Infinity;
|
|
723
728
|
let hasDCWithFilterByAddresses = false;
|
|
724
729
|
let noEventsAddresses = {};
|
|
730
|
+
let registeringAddresses = {};
|
|
725
731
|
for (let itemIdx = 0, itemIdx_finish = items.length; itemIdx < itemIdx_finish; ++itemIdx) {
|
|
726
732
|
let item = items[itemIdx];
|
|
727
733
|
let dcs = item.dcs;
|
|
@@ -757,8 +763,7 @@ function registerDynamicContracts(fetchState, items) {
|
|
|
757
763
|
}
|
|
758
764
|
shouldRemove = true;
|
|
759
765
|
} else {
|
|
760
|
-
let
|
|
761
|
-
let registeringContract = registeringContracts[dc.address];
|
|
766
|
+
let registeringContract = registeringAddresses[dc.address];
|
|
762
767
|
let shouldUpdate;
|
|
763
768
|
if (registeringContract !== undefined) {
|
|
764
769
|
if (registeringContract.contractName !== dc.contractName) {
|
|
@@ -773,7 +778,8 @@ function registerDynamicContracts(fetchState, items) {
|
|
|
773
778
|
}
|
|
774
779
|
if (shouldUpdate) {
|
|
775
780
|
earliestRegisteringEventBlockNumber = Primitive_int.min(earliestRegisteringEventBlockNumber, dcWithStartBlock_effectiveStartBlock);
|
|
776
|
-
|
|
781
|
+
Utils.Dict.getOrInsertEmptyDict(registeringContractsByContract, dc.contractName)[dc.address] = dcWithStartBlock;
|
|
782
|
+
registeringAddresses[dc.address] = dcWithStartBlock;
|
|
777
783
|
} else {
|
|
778
784
|
shouldRemove = true;
|
|
779
785
|
}
|
|
@@ -796,8 +802,11 @@ function registerDynamicContracts(fetchState, items) {
|
|
|
796
802
|
}
|
|
797
803
|
shouldRemove = true;
|
|
798
804
|
} else {
|
|
799
|
-
let
|
|
800
|
-
if (
|
|
805
|
+
let existingContract$2 = registeringAddresses[dc.address];
|
|
806
|
+
if (existingContract$2 !== undefined) {
|
|
807
|
+
if (existingContract$2.contractName !== dc.contractName) {
|
|
808
|
+
warnDifferentContractType(fetchState, existingContract$2, dcAsIndexingAddress);
|
|
809
|
+
}
|
|
801
810
|
shouldRemove = true;
|
|
802
811
|
} else {
|
|
803
812
|
let logger$1 = Logging.createChild({
|
|
@@ -807,6 +816,7 @@ function registerDynamicContracts(fetchState, items) {
|
|
|
807
816
|
});
|
|
808
817
|
Logging.childWarn(logger$1, `Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`);
|
|
809
818
|
noEventsAddresses[dc.address] = dcAsIndexingAddress;
|
|
819
|
+
registeringAddresses[dc.address] = dcAsIndexingAddress;
|
|
810
820
|
}
|
|
811
821
|
}
|
|
812
822
|
}
|
|
@@ -884,8 +894,8 @@ function registerDynamicContracts(fetchState, items) {
|
|
|
884
894
|
}
|
|
885
895
|
}
|
|
886
896
|
}
|
|
887
|
-
let registeringContracts
|
|
888
|
-
Object.assign(newIndexingAddresses, registeringContracts
|
|
897
|
+
let registeringContracts = registeringContractsByContract[contractName];
|
|
898
|
+
Object.assign(newIndexingAddresses, registeringContracts);
|
|
889
899
|
}
|
|
890
900
|
Object.assign(newIndexingAddresses, noEventsAddresses);
|
|
891
901
|
let optimizedPartitions = createPartitionsFromIndexingAddresses(registeringContractsByContract, fetchState.contractConfigs, dynamicContractsRef, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, fetchState.optimizedPartitions.nextPartitionIndex + newPartitions.length | 0, mutExistingPartitions.concat(newPartitions), 0);
|
|
@@ -1175,7 +1185,7 @@ function getNextQuery(fetchState, concurrencyLimit) {
|
|
|
1175
1185
|
}
|
|
1176
1186
|
|
|
1177
1187
|
function hasReadyItem(fetchState) {
|
|
1178
|
-
let item =
|
|
1188
|
+
let item = fetchState.buffer[0];
|
|
1179
1189
|
if (item !== undefined) {
|
|
1180
1190
|
return item.blockNumber <= bufferBlockNumber(fetchState);
|
|
1181
1191
|
} else {
|
|
@@ -1188,7 +1198,7 @@ function getReadyItemsCount(fetchState, targetSize, fromItem) {
|
|
|
1188
1198
|
let acc = 0;
|
|
1189
1199
|
let isFinished = false;
|
|
1190
1200
|
while (!isFinished) {
|
|
1191
|
-
let item =
|
|
1201
|
+
let item = fetchState.buffer[fromItem + acc | 0];
|
|
1192
1202
|
if (item !== undefined) {
|
|
1193
1203
|
let itemBlockNumber = item.blockNumber;
|
|
1194
1204
|
if (itemBlockNumber <= readyBlockNumber) {
|
|
@@ -1299,15 +1309,17 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
|
|
|
1299
1309
|
if (optimizedPartitions.idsInAscOrder.length === 0 && Utils.$$Array.isEmpty(onBlockConfigs)) {
|
|
1300
1310
|
Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + chainId.toString() + `. ` + (`addresses=` + addresses.length.toString() + `, `) + (`eventConfigs=` + eventConfigs.length.toString() + `, `) + (`normalEventConfigs=` + normalEventConfigs.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
|
|
1301
1311
|
}
|
|
1302
|
-
let
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1312
|
+
let buffer = [];
|
|
1313
|
+
let latestOnBlockBlockNumber;
|
|
1314
|
+
if (knownHeight > 0 && Utils.$$Array.notEmpty(onBlockConfigs)) {
|
|
1315
|
+
let id = optimizedPartitions.idsInAscOrder[0];
|
|
1316
|
+
let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined;
|
|
1317
|
+
let maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock.blockNumber : knownHeight;
|
|
1318
|
+
latestOnBlockBlockNumber = appendOnBlockItems(buffer, onBlockConfigs, startBlock, progressBlockNumber, maxBlockNumber, targetBufferSize);
|
|
1319
|
+
} else {
|
|
1320
|
+
latestOnBlockBlockNumber = progressBlockNumber;
|
|
1309
1321
|
}
|
|
1310
|
-
|
|
1322
|
+
let fetchState = {
|
|
1311
1323
|
optimizedPartitions: optimizedPartitions,
|
|
1312
1324
|
startBlock: startBlock,
|
|
1313
1325
|
endBlock: endBlock,
|
|
@@ -1315,14 +1327,23 @@ function make$1(startBlock, endBlock, eventConfigs, addresses, maxAddrInPartitio
|
|
|
1315
1327
|
indexingAddresses: indexingAddresses,
|
|
1316
1328
|
contractConfigs: contractConfigs,
|
|
1317
1329
|
chainId: chainId,
|
|
1318
|
-
latestOnBlockBlockNumber:
|
|
1330
|
+
latestOnBlockBlockNumber: latestOnBlockBlockNumber,
|
|
1319
1331
|
blockLag: blockLag,
|
|
1320
|
-
buffer:
|
|
1332
|
+
buffer: buffer,
|
|
1321
1333
|
targetBufferSize: targetBufferSize,
|
|
1322
1334
|
onBlockConfigs: onBlockConfigs,
|
|
1323
1335
|
knownHeight: knownHeight,
|
|
1324
1336
|
firstEventBlock: firstEventBlock
|
|
1325
1337
|
};
|
|
1338
|
+
let numAddresses = Utils.Dict.size(indexingAddresses);
|
|
1339
|
+
Prometheus.IndexingAddresses.set(numAddresses, chainId);
|
|
1340
|
+
Prometheus.IndexingPartitions.set(optimizedPartitions.idsInAscOrder.length, chainId);
|
|
1341
|
+
Prometheus.IndexingBufferSize.set(buffer.length, chainId);
|
|
1342
|
+
Prometheus.IndexingBufferBlockNumber.set(bufferBlockNumber(fetchState), chainId);
|
|
1343
|
+
if (endBlock !== undefined) {
|
|
1344
|
+
Prometheus.IndexingEndBlock.set(endBlock, chainId);
|
|
1345
|
+
}
|
|
1346
|
+
return fetchState;
|
|
1326
1347
|
}
|
|
1327
1348
|
|
|
1328
1349
|
function bufferSize(param) {
|
|
@@ -1529,7 +1550,7 @@ function isReadyToEnterReorgThreshold(fetchState) {
|
|
|
1529
1550
|
}
|
|
1530
1551
|
|
|
1531
1552
|
function hasFullBatch(fetchState, batchSizeTarget) {
|
|
1532
|
-
let item =
|
|
1553
|
+
let item = fetchState.buffer[batchSizeTarget - 1 | 0];
|
|
1533
1554
|
if (item !== undefined) {
|
|
1534
1555
|
return item.blockNumber <= bufferBlockNumber(fetchState);
|
|
1535
1556
|
} else {
|
|
@@ -1546,7 +1567,7 @@ function getProgressPercentage(fetchState) {
|
|
|
1546
1567
|
if (totalRange <= 0) {
|
|
1547
1568
|
return 0;
|
|
1548
1569
|
}
|
|
1549
|
-
let item =
|
|
1570
|
+
let item = fetchState.buffer[0];
|
|
1550
1571
|
let progress = item !== undefined ? item.blockNumber - firstEventBlock | 0 : bufferBlockNumber(fetchState) - firstEventBlock | 0;
|
|
1551
1572
|
return progress / totalRange;
|
|
1552
1573
|
}
|
|
@@ -1578,7 +1599,7 @@ function sortForUnorderedBatch(fetchStates, batchSizeTarget) {
|
|
|
1578
1599
|
|
|
1579
1600
|
function getProgressBlockNumberAt(fetchState, index) {
|
|
1580
1601
|
let bufferBlockNumber$1 = bufferBlockNumber(fetchState);
|
|
1581
|
-
let item =
|
|
1602
|
+
let item = fetchState.buffer[index];
|
|
1582
1603
|
if (item !== undefined && bufferBlockNumber$1 >= item.blockNumber) {
|
|
1583
1604
|
return item.blockNumber - 1 | 0;
|
|
1584
1605
|
} else {
|
|
@@ -1607,6 +1628,7 @@ export {
|
|
|
1607
1628
|
compareBufferItem,
|
|
1608
1629
|
blockItemLogIndex,
|
|
1609
1630
|
numAddresses,
|
|
1631
|
+
appendOnBlockItems,
|
|
1610
1632
|
updateInternal,
|
|
1611
1633
|
warnDifferentContractType,
|
|
1612
1634
|
addressesByContractNameCount,
|