envio 3.6.1 → 3.7.0-subgraph
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +197 -9
- package/package.json +6 -6
- package/src/Batch.res +64 -25
- package/src/Batch.res.mjs +69 -46
- package/src/ChainFetching.res +13 -17
- package/src/ChainFetching.res.mjs +9 -14
- package/src/ChainState.res +160 -119
- package/src/ChainState.res.mjs +75 -46
- package/src/ChainState.resi +15 -12
- package/src/Config.res +18 -2
- package/src/Config.res.mjs +9 -6
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +11 -0
- package/src/Ecosystem.res +0 -1
- package/src/Envio.res +16 -3
- package/src/EventConfigBuilder.res +155 -40
- package/src/EventConfigBuilder.res.mjs +76 -24
- package/src/EventProcessing.res +10 -14
- package/src/EventProcessing.res.mjs +12 -10
- package/src/EventUtils.res +3 -47
- package/src/FetchState.res +124 -104
- package/src/FetchState.res.mjs +116 -100
- package/src/HandlerLoader.res +20 -8
- package/src/HandlerLoader.res.mjs +11 -2
- package/src/HandlerRegister.res +108 -2
- package/src/HandlerRegister.res.mjs +59 -10
- package/src/HandlerRegister.resi +4 -0
- package/src/InMemoryStore.res +1 -6
- package/src/InMemoryStore.res.mjs +1 -3
- package/src/Internal.res +84 -24
- package/src/Internal.res.mjs +33 -2
- package/src/LazyLoader.res +12 -11
- package/src/LazyLoader.res.mjs +13 -7
- package/src/Main.res +22 -9
- package/src/Main.res.mjs +18 -5
- package/src/MemoryStorage.res +756 -0
- package/src/MemoryStorage.res.mjs +623 -0
- package/src/Persistence.res +4 -2
- package/src/PgStorage.res +24 -8
- package/src/PgStorage.res.mjs +15 -12
- package/src/ReorgDetection.res +0 -222
- package/src/ReorgDetection.res.mjs +0 -163
- package/src/Rollback.res +14 -15
- package/src/Rollback.res.mjs +4 -5
- package/src/SimulateItems.res +2 -2
- package/src/UserContext.res +358 -51
- package/src/UserContext.res.mjs +271 -35
- package/src/Utils.res +3 -0
- package/src/bindings/ClickHouse.res +16 -11
- package/src/bindings/ClickHouse.res.mjs +10 -10
- package/src/bindings/Vitest.res +1 -1
- package/src/sources/BlockStore.res +115 -5
- package/src/sources/BlockStore.res.mjs +25 -0
- package/src/sources/Evm.res +0 -1
- package/src/sources/Evm.res.mjs +0 -1
- package/src/sources/EvmHyperSyncSource.res +19 -84
- package/src/sources/EvmHyperSyncSource.res.mjs +24 -60
- package/src/sources/Fuel.res +24 -4
- package/src/sources/Fuel.res.mjs +23 -3
- package/src/sources/FuelHyperSync.res +8 -3
- package/src/sources/FuelHyperSync.res.mjs +5 -4
- package/src/sources/FuelHyperSync.resi +3 -1
- package/src/sources/FuelHyperSyncClient.res +7 -10
- package/src/sources/FuelHyperSyncSource.res +18 -45
- package/src/sources/FuelHyperSyncSource.res.mjs +18 -35
- package/src/sources/HyperSync.res +49 -229
- package/src/sources/HyperSync.res.mjs +74 -191
- package/src/sources/HyperSync.resi +11 -35
- package/src/sources/HyperSyncClient.res +9 -79
- package/src/sources/HyperSyncClient.res.mjs +2 -6
- package/src/sources/RequestStat.res +5 -0
- package/src/sources/RequestStat.res.mjs +2 -0
- package/src/sources/RpcSource.res +149 -43
- package/src/sources/RpcSource.res.mjs +104 -41
- package/src/sources/SimulateSource.res +8 -5
- package/src/sources/SimulateSource.res.mjs +6 -6
- package/src/sources/Source.res +89 -16
- package/src/sources/Source.res.mjs +50 -1
- package/src/sources/SourceManager.res +255 -50
- package/src/sources/SourceManager.res.mjs +149 -55
- package/src/sources/SourceManager.resi +2 -6
- package/src/sources/Svm.res +0 -7
- package/src/sources/Svm.res.mjs +0 -8
- package/src/sources/SvmHyperSyncClient.res +21 -16
- package/src/sources/SvmHyperSyncClient.res.mjs +3 -4
- package/src/sources/SvmHyperSyncSource.res +25 -117
- package/src/sources/SvmHyperSyncSource.res.mjs +8 -121
- package/src/subgraph/blocks.ts +176 -0
- package/src/subgraph/calls.ts +217 -0
- package/src/subgraph/conformance.ts +99 -0
- package/src/subgraph/division.ts +110 -0
- package/src/subgraph/errors.ts +90 -0
- package/src/subgraph/graph-ts-types/VERSION +2 -0
- package/src/subgraph/graph-ts-types/chain/arweave.d.ts +70 -0
- package/src/subgraph/graph-ts-types/chain/cosmos.d.ts +327 -0
- package/src/subgraph/graph-ts-types/chain/ethereum.d.ts +233 -0
- package/src/subgraph/graph-ts-types/chain/near.d.ts +253 -0
- package/src/subgraph/graph-ts-types/chain/starknet.d.ts +32 -0
- package/src/subgraph/graph-ts-types/common/collections.d.ts +136 -0
- package/src/subgraph/graph-ts-types/common/conversion.d.ts +11 -0
- package/src/subgraph/graph-ts-types/common/datasource.d.ts +30 -0
- package/src/subgraph/graph-ts-types/common/eager-offset.d.ts +0 -0
- package/src/subgraph/graph-ts-types/common/json.d.ts +17 -0
- package/src/subgraph/graph-ts-types/common/numbers.d.ts +120 -0
- package/src/subgraph/graph-ts-types/common/value.d.ts +120 -0
- package/src/subgraph/graph-ts-types/common/yaml.d.ts +90 -0
- package/src/subgraph/graph-ts-types/global/global.d.ts +194 -0
- package/src/subgraph/graph-ts-types/helper-functions.d.ts +22 -0
- package/src/subgraph/graph-ts-types/index.d.ts +102 -0
- package/src/subgraph/graph-ts.ts +1871 -0
- package/src/subgraph/hosts.ts +148 -0
- package/src/subgraph/runtime.ts +845 -0
- package/src/subgraph/scope.ts +63 -0
- package/svm.schema.json +3 -2
package/src/ChainState.res
CHANGED
|
@@ -9,7 +9,7 @@ type t = {
|
|
|
9
9
|
mutable fetchState: FetchState.t,
|
|
10
10
|
// The chain-wide address index, kept in Rust. Not `mutable`: registration and
|
|
11
11
|
// rollback mutate it in place, so the handle is stable across fetchState
|
|
12
|
-
// versions
|
|
12
|
+
// versions - and it's the same handle this chain's source clients hold.
|
|
13
13
|
addressStore: AddressStore.t,
|
|
14
14
|
sourceManager: SourceManager.t,
|
|
15
15
|
chainConfig: Config.chain,
|
|
@@ -18,7 +18,7 @@ type t = {
|
|
|
18
18
|
mutable committedProgressBlockNumber: int,
|
|
19
19
|
// Progress of the batch currently being processed. The buffer is consumed at
|
|
20
20
|
// batch creation (see advanceAfterBatch), so this runs ahead of
|
|
21
|
-
// committedProgressBlockNumber until the batch commits
|
|
21
|
+
// committedProgressBlockNumber until the batch commits - it's the true lower
|
|
22
22
|
// boundary of the remaining buffer's block span.
|
|
23
23
|
mutable processingBlockNumber: int,
|
|
24
24
|
mutable numEventsProcessed: float,
|
|
@@ -35,8 +35,12 @@ type t = {
|
|
|
35
35
|
// Empty in the default cross-chain mode, where every entity's table lives on
|
|
36
36
|
// the indexer instead.
|
|
37
37
|
mutable entities: EntityTables.t,
|
|
38
|
-
mutable reorgDetection: ReorgDetection.t,
|
|
39
38
|
mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
|
|
39
|
+
// Reorg handling knobs, mirrored from the chain/indexer config: hashes are
|
|
40
|
+
// compared for blocks within `maxReorgDepth` of the known height, and a
|
|
41
|
+
// detected reorg either rolls back or is only logged.
|
|
42
|
+
shouldRollbackOnReorg: bool,
|
|
43
|
+
maxReorgDepth: int,
|
|
40
44
|
// Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
|
|
41
45
|
// transactionIndex). Fetch responses merge their page in; entries are pruned
|
|
42
46
|
// as the chain progresses and dropped above the target on rollback.
|
|
@@ -90,9 +94,10 @@ let make = (
|
|
|
90
94
|
~onEventRegistrations=[],
|
|
91
95
|
~addressStore: AddressStore.t,
|
|
92
96
|
~sourceManager: SourceManager.t,
|
|
93
|
-
~reorgDetection: ReorgDetection.t,
|
|
94
97
|
~committedProgressBlockNumber: int,
|
|
95
98
|
~safeCheckpointTracking=None,
|
|
99
|
+
~shouldRollbackOnReorg,
|
|
100
|
+
~maxReorgDepth,
|
|
96
101
|
~numEventsProcessed=0.,
|
|
97
102
|
~timestampCaughtUpToHeadOrEndblock=None,
|
|
98
103
|
~isProgressAtHead=false,
|
|
@@ -119,8 +124,9 @@ let make = (
|
|
|
119
124
|
numEventsProcessed,
|
|
120
125
|
pendingBudget: 0.,
|
|
121
126
|
chainDensity,
|
|
122
|
-
reorgDetection,
|
|
123
127
|
safeCheckpointTracking,
|
|
128
|
+
shouldRollbackOnReorg,
|
|
129
|
+
maxReorgDepth,
|
|
124
130
|
transactionStore,
|
|
125
131
|
blockStore,
|
|
126
132
|
reorgThresholdReadyTolerance,
|
|
@@ -211,14 +217,17 @@ let makeInternal = (
|
|
|
211
217
|
~knownHeight,
|
|
212
218
|
~chainId=chainConfig.id,
|
|
213
219
|
// FIXME: Shouldn't set with full history
|
|
220
|
+
// The resumed depth, not the config one: the pre-threshold lag must match
|
|
221
|
+
// the window reorg detection compares, or a reduced config depth would let
|
|
222
|
+
// fetching enter the stored rollback window without history.
|
|
214
223
|
~blockLag=Pervasives.max(
|
|
215
|
-
!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 :
|
|
224
|
+
!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth,
|
|
216
225
|
chainConfig.blockLag,
|
|
217
226
|
),
|
|
218
227
|
~onBlockRegistrations,
|
|
219
228
|
~firstEventBlock,
|
|
220
229
|
// Fuel and SVM route through the address store like EVM does, so the
|
|
221
|
-
// client-side path works for them
|
|
230
|
+
// client-side path works for them - but their address counts are nowhere
|
|
222
231
|
// near the threshold, so switching would only trade a server-side filter
|
|
223
232
|
// that costs nothing today for an over-fetch. Keep them server-side until a
|
|
224
233
|
// chain actually needs it.
|
|
@@ -293,14 +302,41 @@ let makeInternal = (
|
|
|
293
302
|
]
|
|
294
303
|
}
|
|
295
304
|
| Config.SimulateSourceConfig({items, endBlock}) => [
|
|
296
|
-
SimulateSource.make(
|
|
305
|
+
SimulateSource.make(
|
|
306
|
+
~items,
|
|
307
|
+
~endBlock,
|
|
308
|
+
~chainId,
|
|
309
|
+
~addressStore,
|
|
310
|
+
~ecosystem=config.ecosystem.name,
|
|
311
|
+
),
|
|
297
312
|
]
|
|
298
313
|
// For tests: use ready-to-use sources directly
|
|
299
314
|
| Config.CustomSources(sources) => sources
|
|
300
315
|
}
|
|
301
316
|
|
|
317
|
+
let blockStore = BlockStore.make(
|
|
318
|
+
~ecosystem=config.ecosystem.name,
|
|
319
|
+
~shouldChecksum=!lowercaseAddresses,
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
// Seed the stored reorg checkpoints (hash-only rows) so detection resumes
|
|
323
|
+
// against the hashes scanned before the restart.
|
|
324
|
+
if chainReorgCheckpoints->Utils.Array.notEmpty {
|
|
325
|
+
let seedPage = BlockStore.fromJs(
|
|
326
|
+
chainReorgCheckpoints->Array.map((cp): BlockStore.inputBlock => {
|
|
327
|
+
blockNumber: cp.blockNumber,
|
|
328
|
+
blockHash: cp.blockHash,
|
|
329
|
+
}),
|
|
330
|
+
~ecosystem=config.ecosystem.name,
|
|
331
|
+
~shouldChecksum=!lowercaseAddresses,
|
|
332
|
+
)
|
|
333
|
+
blockStore
|
|
334
|
+
->BlockStore.merge(seedPage, ~fromBlock=0, ~reportOnly=false)
|
|
335
|
+
->ignore
|
|
336
|
+
}
|
|
337
|
+
|
|
302
338
|
// Seed chain density from whatever progress this chain already has (from a
|
|
303
|
-
// resumed DB state, or 0 on a fresh chain)
|
|
339
|
+
// resumed DB state, or 0 on a fresh chain) - refined per-batch afterwards.
|
|
304
340
|
let chainDensity = switch fetchState.firstEventBlock {
|
|
305
341
|
| Some(firstEventBlock) if progressBlockNumber > firstEventBlock && numEventsProcessed > 0. =>
|
|
306
342
|
Some(numEventsProcessed /. (progressBlockNumber - firstEventBlock)->Int.toFloat)
|
|
@@ -313,11 +349,8 @@ let makeInternal = (
|
|
|
313
349
|
~onEventRegistrations,
|
|
314
350
|
~addressStore,
|
|
315
351
|
~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
|
|
316
|
-
~
|
|
317
|
-
|
|
318
|
-
~maxReorgDepth,
|
|
319
|
-
~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
|
|
320
|
-
),
|
|
352
|
+
~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
|
|
353
|
+
~maxReorgDepth,
|
|
321
354
|
~safeCheckpointTracking=SafeCheckpointTracking.make(
|
|
322
355
|
~maxReorgDepth,
|
|
323
356
|
~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
|
|
@@ -332,10 +365,7 @@ let makeInternal = (
|
|
|
332
365
|
~shouldChecksum=!lowercaseAddresses,
|
|
333
366
|
),
|
|
334
367
|
~chainDensity,
|
|
335
|
-
~blockStore
|
|
336
|
-
~ecosystem=config.ecosystem.name,
|
|
337
|
-
~shouldChecksum=!lowercaseAddresses,
|
|
338
|
-
),
|
|
368
|
+
~blockStore,
|
|
339
369
|
~reorgThresholdReadyTolerance=config.reorgThresholdReadyTolerance,
|
|
340
370
|
~logger,
|
|
341
371
|
)
|
|
@@ -415,6 +445,7 @@ let makeFromDbState = (
|
|
|
415
445
|
// --- Read accessors. ---
|
|
416
446
|
|
|
417
447
|
let logger = (cs: t) => cs.logger
|
|
448
|
+
let blockStore = (cs: t) => cs.blockStore
|
|
418
449
|
let entities = (cs: t) => cs.entities
|
|
419
450
|
|
|
420
451
|
// Rollback discards every uncommitted change, so this chain's partition is
|
|
@@ -422,7 +453,20 @@ let entities = (cs: t) => cs.entities
|
|
|
422
453
|
let resetEntities = (cs: t, ~perChainEntities) => cs.entities = EntityTables.make(perChainEntities)
|
|
423
454
|
let sourceManager = (cs: t) => cs.sourceManager
|
|
424
455
|
let chainConfig = (cs: t) => cs.chainConfig
|
|
425
|
-
let
|
|
456
|
+
let shouldRollbackOnReorg = (cs: t) => cs.shouldRollbackOnReorg
|
|
457
|
+
// Reorg-scan reads over the block store, for the rollback flow: the scanned
|
|
458
|
+
// block numbers still inside the reorg threshold, and the highest of them whose
|
|
459
|
+
// re-fetched hash still matches.
|
|
460
|
+
let getReorgThresholdBlockNumbersBelow = (cs: t, ~blockNumber) =>
|
|
461
|
+
cs.blockStore->BlockStore.getHashedBlockNumbers(
|
|
462
|
+
~fromBlock=Pervasives.max(cs.fetchState.knownHeight - cs.maxReorgDepth, 0),
|
|
463
|
+
~belowBlock=blockNumber,
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
let getLatestValidScannedBlock = (cs: t, ~blockStore: BlockStore.t, ~blockNumbers: array<int>) =>
|
|
467
|
+
cs.blockStore
|
|
468
|
+
->BlockStore.latestValidBlockFromStore(blockStore, blockNumbers)
|
|
469
|
+
->Null.toOption
|
|
426
470
|
let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
|
|
427
471
|
let isProgressAtHead = (cs: t) => cs.isProgressAtHead
|
|
428
472
|
let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
|
|
@@ -512,7 +556,7 @@ let fetchCeiling = (cs: t) => {
|
|
|
512
556
|
}
|
|
513
557
|
}
|
|
514
558
|
|
|
515
|
-
// Events/block over the ready part of the buffer
|
|
559
|
+
// Events/block over the ready part of the buffer - a live signal that reacts
|
|
516
560
|
// to what fetching just found, unlike the processing EMA which only moves as
|
|
517
561
|
// batches commit.
|
|
518
562
|
let readyBufferDensity = (cs: t) => {
|
|
@@ -527,7 +571,7 @@ let readyBufferDensity = (cs: t) => {
|
|
|
527
571
|
|
|
528
572
|
// Density used for query sizing: the higher of the processing EMA and the
|
|
529
573
|
// ready-buffer density, so a stale-low EMA can't undersize queries while the
|
|
530
|
-
// buffer proves the range is dense. None means the chain is cold
|
|
574
|
+
// buffer proves the range is dense. None means the chain is cold - no density
|
|
531
575
|
// signal at all.
|
|
532
576
|
let effectiveDensity = (cs: t) =>
|
|
533
577
|
switch (cs.chainDensity, cs->readyBufferDensity) {
|
|
@@ -551,7 +595,7 @@ let targetBlock = (cs: t, ~chainTargetItems: float) => {
|
|
|
551
595
|
| Some(density) if density > 0. =>
|
|
552
596
|
// Decided by comparison so no unbounded value is ever converted to int:
|
|
553
597
|
// at low densities chainTargetItems /. density exceeds the int range, and
|
|
554
|
-
// truncating it wraps negative
|
|
598
|
+
// truncating it wraps negative - the target collapses below the frontier
|
|
555
599
|
// and the chain stops querying. The division only runs when its result is
|
|
556
600
|
// provably below the ceiling-bounded range.
|
|
557
601
|
if density *. (fetchCeiling - bufferBlockNumber)->Int.toFloat <= chainTargetItems {
|
|
@@ -565,7 +609,7 @@ let targetBlock = (cs: t, ~chainTargetItems: float) => {
|
|
|
565
609
|
|
|
566
610
|
// Block range that cross-chain progress alignment maps fractions over: from
|
|
567
611
|
// the chain's startBlock to the last block it can fetch right now. The lower
|
|
568
|
-
// bound is deliberately static
|
|
612
|
+
// bound is deliberately static - anchoring it at firstEventBlock would make
|
|
569
613
|
// two chains sitting at the same block read different progress fractions
|
|
570
614
|
// depending on whether each has discovered its first event yet, so the
|
|
571
615
|
// anchor's line could map below a follower's own frontier and stall it.
|
|
@@ -576,7 +620,7 @@ let progressRange = (cs: t) => {
|
|
|
576
620
|
|
|
577
621
|
// A degenerate range (chain already at or past its last block) maps to 1 so it
|
|
578
622
|
// never constrains the other chains. Clamped at 0 for the initial -1 fetch
|
|
579
|
-
// frontier
|
|
623
|
+
// frontier - the only possible blockNumber below the range's lower bound.
|
|
580
624
|
let progressAtBlock = (cs: t, ~blockNumber) => {
|
|
581
625
|
let (lower, upper) = cs->progressRange
|
|
582
626
|
upper <= lower
|
|
@@ -592,7 +636,7 @@ let blockAtProgress = (cs: t, ~progress) => {
|
|
|
592
636
|
lower + Math.ceil(progress *. (upper - lower)->Int.toFloat)->Float.toInt
|
|
593
637
|
}
|
|
594
638
|
|
|
595
|
-
// The fetch frontier as a fraction of the alignable range
|
|
639
|
+
// The fetch frontier as a fraction of the alignable range - what the
|
|
596
640
|
// cross-chain waterfall aligns other chains against. Based on blocks actually
|
|
597
641
|
// fetched, so it holds up even while this chain's queries are in flight.
|
|
598
642
|
let frontierProgress = (cs: t) =>
|
|
@@ -611,7 +655,7 @@ let getNextQuery = (cs: t, ~chainTargetItems: float, ~maxTargetBlock=?) => {
|
|
|
611
655
|
| None => chainTargetBlock
|
|
612
656
|
}
|
|
613
657
|
// When the target block is clamped (head/endBlock/cross-chain alignment) a
|
|
614
|
-
// known-density chain can't use the whole handed budget
|
|
658
|
+
// known-density chain can't use the whole handed budget - cap the fresh part
|
|
615
659
|
// at what the clamped range actually costs (in-flight reservations stay on
|
|
616
660
|
// top: they're already accounted and shouldn't crowd out new partitions), so
|
|
617
661
|
// the waterfall's leftover flows to the next chain in the same tick instead
|
|
@@ -676,7 +720,7 @@ let isDurablyCaughtUp = (cs: t) => {
|
|
|
676
720
|
}
|
|
677
721
|
|
|
678
722
|
let getHighestBlockBelowThreshold = (cs: t): int => {
|
|
679
|
-
let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.
|
|
723
|
+
let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth
|
|
680
724
|
highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
|
|
681
725
|
}
|
|
682
726
|
|
|
@@ -685,7 +729,7 @@ let isActivelyIndexing = (cs: t) => cs.fetchState->FetchState.isActivelyIndexing
|
|
|
685
729
|
// True once the fetch frontier has reached the head/endBlock for this chain.
|
|
686
730
|
let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
|
|
687
731
|
|
|
688
|
-
// Reached head on a chain with no configured endBlock
|
|
732
|
+
// Reached head on a chain with no configured endBlock - used by auto-exit to
|
|
689
733
|
// detect that no events were found in the start..head range.
|
|
690
734
|
let isAtHeadWithoutEndBlock = (cs: t) =>
|
|
691
735
|
cs.isProgressAtHead && cs.fetchState.endBlock->Option.isNone
|
|
@@ -713,12 +757,7 @@ type blockGroups = {
|
|
|
713
757
|
// same batch independently. Items already arrive in (blockNumber, logIndex)
|
|
714
758
|
// order, so a (blockNumber, transactionIndex) run and a blockNumber-only run
|
|
715
759
|
// each stay adjacent; every item is checked against both, in the same pass.
|
|
716
|
-
|
|
717
|
-
// block inline and has no store.
|
|
718
|
-
let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
|
|
719
|
-
transactionGroups,
|
|
720
|
-
blockGroups,
|
|
721
|
-
) => {
|
|
760
|
+
let groupBatchItems = (items: array<Internal.item>): (transactionGroups, blockGroups) => {
|
|
722
761
|
let txBlockNumbers = []
|
|
723
762
|
let transactionIndices = []
|
|
724
763
|
let transactionMasks = []
|
|
@@ -739,7 +778,7 @@ let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
|
|
|
739
778
|
| Some(_) => () // RPC/simulate/Fuel carry the transaction inline.
|
|
740
779
|
| None =>
|
|
741
780
|
let {transactionIndex} = eventItem
|
|
742
|
-
let mask = eventItem.onEventRegistration.
|
|
781
|
+
let mask = eventItem.onEventRegistration.fieldSelection.transactionMask
|
|
743
782
|
if mask != 0. {
|
|
744
783
|
anyTransactionFieldSelected := true
|
|
745
784
|
}
|
|
@@ -762,27 +801,25 @@ let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
|
|
|
762
801
|
}
|
|
763
802
|
}
|
|
764
803
|
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
blockItemGroups->Array.push([eventItem])
|
|
781
|
-
}
|
|
804
|
+
switch eventItem.payload->Internal.getPayloadBlock->Nullable.toOption {
|
|
805
|
+
| Some(_) => () // RPC/simulate carry the block inline.
|
|
806
|
+
| None =>
|
|
807
|
+
let mask = eventItem.onEventRegistration.fieldSelection.blockMask
|
|
808
|
+
let last = blockItemGroups->Array.length - 1
|
|
809
|
+
if last >= 0 && blockBlockNumbers->Array.getUnsafe(last) == blockNumber {
|
|
810
|
+
blockItemGroups->Array.getUnsafe(last)->Array.push(eventItem)
|
|
811
|
+
blockMasks->Array.setUnsafe(
|
|
812
|
+
last,
|
|
813
|
+
FieldMask.orMask(blockMasks->Array.getUnsafe(last), mask),
|
|
814
|
+
)
|
|
815
|
+
} else {
|
|
816
|
+
blockBlockNumbers->Array.push(blockNumber)
|
|
817
|
+
blockMasks->Array.push(mask)
|
|
818
|
+
blockItemGroups->Array.push([eventItem])
|
|
782
819
|
}
|
|
783
820
|
}
|
|
784
821
|
// onBlock items build their block from the handler's own block number, not
|
|
785
|
-
// from the stores
|
|
822
|
+
// from the stores - which is what lets the sources keep only the blocks and
|
|
786
823
|
// transactions an event item references.
|
|
787
824
|
| Internal.Block(_) => ()
|
|
788
825
|
}
|
|
@@ -801,8 +838,8 @@ let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
|
|
|
801
838
|
}
|
|
802
839
|
|
|
803
840
|
// Materialise a `TransactionStore` against precomputed groups (see
|
|
804
|
-
// `groupBatchItems`). Store-backed items always get a transaction object
|
|
805
|
-
// selected fields, or `{}` when nothing was selected
|
|
841
|
+
// `groupBatchItems`). Store-backed items always get a transaction object - the
|
|
842
|
+
// selected fields, or `{}` when nothing was selected - so `event.transaction`
|
|
806
843
|
// is never `undefined` (matching the inline sources).
|
|
807
844
|
let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGroups) => {
|
|
808
845
|
if g.payloadGroups->Utils.Array.notEmpty {
|
|
@@ -838,45 +875,34 @@ let applyBlockGroups = async (store: BlockStore.t, g: blockGroups) => {
|
|
|
838
875
|
}
|
|
839
876
|
}
|
|
840
877
|
|
|
841
|
-
let includeBlocksForEcosystem = (ecosystem: Ecosystem.name) =>
|
|
842
|
-
switch ecosystem {
|
|
843
|
-
| Evm | Svm => true
|
|
844
|
-
| Fuel => false
|
|
845
|
-
}
|
|
846
|
-
|
|
847
878
|
// Materialise the chain stores' selected transaction and block fields onto a
|
|
848
879
|
// batch's items at batch prep (the persistent-store path). A single pass over
|
|
849
880
|
// `items` (`groupBatchItems`) builds both stores' selection masks before the
|
|
850
881
|
// two independent materialize calls run concurrently.
|
|
851
|
-
let materializeBatchItems = async (cs: t, ~items: array<Internal.item
|
|
852
|
-
let (txGroups, blockGroups) =
|
|
853
|
-
items->groupBatchItems(~includeBlocks=ecosystem->includeBlocksForEcosystem)
|
|
882
|
+
let materializeBatchItems = async (cs: t, ~items: array<Internal.item>) => {
|
|
883
|
+
let (txGroups, blockGroups) = items->groupBatchItems
|
|
854
884
|
let _ = await Promise.all2((
|
|
855
885
|
cs.transactionStore->applyTransactionGroups(txGroups),
|
|
856
886
|
cs.blockStore->applyBlockGroups(blockGroups),
|
|
857
887
|
))
|
|
858
888
|
}
|
|
859
889
|
|
|
860
|
-
// Materialise a fetch-response
|
|
861
|
-
//
|
|
862
|
-
//
|
|
890
|
+
// Materialise a fetch-response's transactions and blocks onto its items before
|
|
891
|
+
// contract-register handlers read them. Transactions come from the response's
|
|
892
|
+
// page (`None` when kept inline); blocks come from the chain store, which the
|
|
893
|
+
// response's page was already merged into by `registerReorgGuard`.
|
|
863
894
|
let materializePageItems = async (
|
|
864
895
|
~items: array<Internal.item>,
|
|
865
896
|
~transactionStore: option<TransactionStore.t>,
|
|
866
|
-
~blockStore:
|
|
867
|
-
~ecosystem,
|
|
897
|
+
~blockStore: BlockStore.t,
|
|
868
898
|
) => {
|
|
869
|
-
let (txGroups, blockGroups) =
|
|
870
|
-
items->groupBatchItems(~includeBlocks=ecosystem->includeBlocksForEcosystem)
|
|
899
|
+
let (txGroups, blockGroups) = items->groupBatchItems
|
|
871
900
|
let _ = await Promise.all2((
|
|
872
901
|
switch transactionStore {
|
|
873
902
|
| Some(store) => store->applyTransactionGroups(txGroups)
|
|
874
903
|
| None => Promise.resolve()
|
|
875
904
|
},
|
|
876
|
-
|
|
877
|
-
| Some(store) => store->applyBlockGroups(blockGroups)
|
|
878
|
-
| None => Promise.resolve()
|
|
879
|
-
},
|
|
905
|
+
blockStore->applyBlockGroups(blockGroups),
|
|
880
906
|
))
|
|
881
907
|
}
|
|
882
908
|
|
|
@@ -885,21 +911,17 @@ let handleQueryResult = (
|
|
|
885
911
|
~query: FetchState.query,
|
|
886
912
|
~newItems,
|
|
887
913
|
~newRegistrations,
|
|
888
|
-
~latestFetchedBlock:
|
|
914
|
+
~latestFetchedBlock: int,
|
|
889
915
|
~knownHeight,
|
|
890
916
|
~transactionStore as txPage: option<TransactionStore.t>,
|
|
891
|
-
~blockStore as blockPage: option<BlockStore.t>,
|
|
892
917
|
) => {
|
|
893
|
-
// Merge this response's
|
|
894
|
-
// its items to the buffer. Inline sources contribute no page
|
|
918
|
+
// Merge this response's transaction page into the chain store in lockstep
|
|
919
|
+
// with appending its items to the buffer. Inline sources contribute no page;
|
|
920
|
+
// the block page was already merged by `registerReorgGuard`.
|
|
895
921
|
switch txPage {
|
|
896
922
|
| Some(page) => cs.transactionStore->TransactionStore.merge(page)
|
|
897
923
|
| None => ()
|
|
898
924
|
}
|
|
899
|
-
switch blockPage {
|
|
900
|
-
| Some(page) => cs.blockStore->BlockStore.merge(page)
|
|
901
|
-
| None => ()
|
|
902
|
-
}
|
|
903
925
|
|
|
904
926
|
let fs = switch newRegistrations {
|
|
905
927
|
| [] => cs.fetchState
|
|
@@ -908,9 +930,9 @@ let handleQueryResult = (
|
|
|
908
930
|
~addressStore=cs.addressStore,
|
|
909
931
|
// This response is applied below, after the addresses land. It was routed
|
|
910
932
|
// before they existed, so whatever it claims has to be inside the
|
|
911
|
-
// catch-up range
|
|
933
|
+
// catch-up range - and an unbounded query can reach past the height that
|
|
912
934
|
// was known when it went out.
|
|
913
|
-
~claimCeiling=Pervasives.max(knownHeight, latestFetchedBlock
|
|
935
|
+
~claimCeiling=Pervasives.max(knownHeight, latestFetchedBlock),
|
|
914
936
|
newRegistrations,
|
|
915
937
|
)
|
|
916
938
|
}
|
|
@@ -924,15 +946,25 @@ let handleQueryResult = (
|
|
|
924
946
|
cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsEst->Int.toFloat)
|
|
925
947
|
}
|
|
926
948
|
|
|
927
|
-
// Run reorg detection against a fetch response
|
|
928
|
-
//
|
|
929
|
-
//
|
|
930
|
-
//
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
cs.
|
|
935
|
-
|
|
949
|
+
// Run reorg detection against a fetch response by merging its block-store page
|
|
950
|
+
// into the chain store: hashes of blocks inside the reorg threshold are
|
|
951
|
+
// compared on the way. On a mismatch with rollback enabled the page is
|
|
952
|
+
// discarded (the stored hashes stay for the rollback comparison); in
|
|
953
|
+
// detect-only mode the page still merges, so the overwritten hash doesn't
|
|
954
|
+
// re-report on every response.
|
|
955
|
+
let registerReorgGuard = (cs: t, ~blockStore, ~knownHeight): ReorgDetection.reorgResult => {
|
|
956
|
+
switch cs.blockStore->BlockStore.merge(
|
|
957
|
+
blockStore,
|
|
958
|
+
~fromBlock=Pervasives.max(knownHeight - cs.maxReorgDepth, 0),
|
|
959
|
+
~reportOnly=!cs.shouldRollbackOnReorg,
|
|
960
|
+
) {
|
|
961
|
+
| Null.Null => NoReorg
|
|
962
|
+
| Null.Value({blockNumber, storedHash, receivedHash}) =>
|
|
963
|
+
ReorgDetected({
|
|
964
|
+
scannedBlock: {blockNumber, blockHash: storedHash},
|
|
965
|
+
receivedBlock: {blockNumber, blockHash: receivedHash},
|
|
966
|
+
})
|
|
967
|
+
}
|
|
936
968
|
}
|
|
937
969
|
|
|
938
970
|
// Prepare for a reorg rollback: restore the events-processed counter to its
|
|
@@ -1008,14 +1040,28 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
|
|
|
1008
1040
|
rollbackTargetBlock: cs.rollbackTargetBlock,
|
|
1009
1041
|
}
|
|
1010
1042
|
|
|
1011
|
-
// Snapshot the inputs a batch build needs from this chain
|
|
1043
|
+
// Snapshot the inputs a batch build needs from this chain, including an
|
|
1044
|
+
// immutable copy of the scanned in-threshold block hashes so the batch never
|
|
1045
|
+
// reads the live store mid-build.
|
|
1012
1046
|
let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1047
|
+
let {blockNumbers, hashes} =
|
|
1048
|
+
cs.blockStore->BlockStore.getHashes(
|
|
1049
|
+
~fromBlock=Pervasives.max(cs.fetchState.knownHeight - cs.maxReorgDepth, 0),
|
|
1050
|
+
~belowBlock=cs.fetchState.knownHeight + 1,
|
|
1051
|
+
)
|
|
1052
|
+
let hashByBlockNumber = Dict.make()
|
|
1053
|
+
blockNumbers->Array.forEachWithIndex((blockNumber, idx) =>
|
|
1054
|
+
hashByBlockNumber->Utils.Dict.setByInt(blockNumber, hashes->Array.getUnsafe(idx))
|
|
1055
|
+
)
|
|
1056
|
+
{
|
|
1057
|
+
fetchState: cs.fetchState,
|
|
1058
|
+
progressBlockNumber: cs.committedProgressBlockNumber,
|
|
1059
|
+
totalEventsProcessed: cs.numEventsProcessed,
|
|
1060
|
+
sourceBlockNumber: cs.fetchState.knownHeight,
|
|
1061
|
+
scannedHashes: {blockNumbers, hashByBlockNumber},
|
|
1062
|
+
shouldRollbackOnReorg: cs.shouldRollbackOnReorg,
|
|
1063
|
+
chainConfig: cs.chainConfig,
|
|
1064
|
+
}
|
|
1019
1065
|
}
|
|
1020
1066
|
|
|
1021
1067
|
// Whether the chain's post-batch fetch frontier is ready to cross into the reorg
|
|
@@ -1057,7 +1103,7 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
|
|
|
1057
1103
|
| Some(chainAfterBatch) => {
|
|
1058
1104
|
// Calculate and set latency metrics. The payload block is materialised or
|
|
1059
1105
|
// inline by processing time; its timestamp may still be absent (e.g. an
|
|
1060
|
-
// SVM slot with no block row)
|
|
1106
|
+
// SVM slot with no block row) - the metric is skipped then.
|
|
1061
1107
|
switch batch
|
|
1062
1108
|
->Batch.findLastEventItem(~chainId)
|
|
1063
1109
|
->Option.flatMap(eventItem =>
|
|
@@ -1088,13 +1134,13 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
|
|
|
1088
1134
|
}
|
|
1089
1135
|
|
|
1090
1136
|
// Chain-wide density update: seed with the batch's own events/block on
|
|
1091
|
-
// the first update, then blend weighted by the batch's block span
|
|
1137
|
+
// the first update, then blend weighted by the batch's block span - a
|
|
1092
1138
|
// few sparse/dense blocks barely nudge the estimate, while a
|
|
1093
1139
|
// window-sized batch replaces it.
|
|
1094
1140
|
let deltaBlocks = chainAfterBatch.progressBlockNumber - cs.committedProgressBlockNumber
|
|
1095
1141
|
if deltaBlocks > 0 {
|
|
1096
1142
|
let deltaEvents = chainAfterBatch.totalEventsProcessed -. cs.numEventsProcessed
|
|
1097
|
-
// Don't seed a density before the first event is seen
|
|
1143
|
+
// Don't seed a density before the first event is seen - a progress-only
|
|
1098
1144
|
// batch would otherwise set it to 0, matching the resume-seed guard.
|
|
1099
1145
|
switch (cs.chainDensity, deltaEvents > 0.) {
|
|
1100
1146
|
| (None, false) => ()
|
|
@@ -1122,7 +1168,12 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
|
|
|
1122
1168
|
cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
|
|
1123
1169
|
// Processed blocks' transactions and blocks are no longer needed.
|
|
1124
1170
|
cs.transactionStore->TransactionStore.prune(chainAfterBatch.progressBlockNumber)
|
|
1125
|
-
|
|
1171
|
+
// Processed blocks' other fields are dropped, but hashes still inside the
|
|
1172
|
+
// reorg threshold stay for reorg detection.
|
|
1173
|
+
cs.blockStore->BlockStore.prune(
|
|
1174
|
+
chainAfterBatch.progressBlockNumber,
|
|
1175
|
+
~keepHashesFrom=cs.fetchState.knownHeight - cs.maxReorgDepth,
|
|
1176
|
+
)
|
|
1126
1177
|
cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated
|
|
1127
1178
|
switch cs.safeCheckpointTracking {
|
|
1128
1179
|
| Some(safeCheckpointTracking) =>
|
|
@@ -1154,10 +1205,10 @@ let markReady = (cs: t, ~readyAt) =>
|
|
|
1154
1205
|
}
|
|
1155
1206
|
|
|
1156
1207
|
// Roll a chain back to a reorg target. With a progress diff, restore fetch/
|
|
1157
|
-
// safe-checkpoint/progress state to `newProgressBlockNumber
|
|
1158
|
-
//
|
|
1159
|
-
//
|
|
1160
|
-
//
|
|
1208
|
+
// safe-checkpoint/progress state to `newProgressBlockNumber`. A reorg chain
|
|
1209
|
+
// with no diff entry still rewinds fetch state + stores to the target -
|
|
1210
|
+
// otherwise the stale block hash stays in the store and re-triggers the same
|
|
1211
|
+
// reorg.
|
|
1161
1212
|
let rollback = (
|
|
1162
1213
|
cs: t,
|
|
1163
1214
|
~newProgressBlockNumber,
|
|
@@ -1175,12 +1226,6 @@ let rollback = (
|
|
|
1175
1226
|
~message="Missing events-processed diff for rolled-back chain",
|
|
1176
1227
|
)
|
|
1177
1228
|
|
|
1178
|
-
if isReorgChain {
|
|
1179
|
-
cs.reorgDetection =
|
|
1180
|
-
cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
|
|
1181
|
-
~blockNumber=rollbackTargetBlockNumber,
|
|
1182
|
-
)
|
|
1183
|
-
}
|
|
1184
1229
|
switch cs.safeCheckpointTracking {
|
|
1185
1230
|
| Some(safeCheckpointTracking) =>
|
|
1186
1231
|
cs.safeCheckpointTracking = Some(
|
|
@@ -1202,10 +1247,6 @@ let rollback = (
|
|
|
1202
1247
|
cs.numEventsProcessed = newTotalEventsProcessed
|
|
1203
1248
|
| None =>
|
|
1204
1249
|
if isReorgChain {
|
|
1205
|
-
cs.reorgDetection =
|
|
1206
|
-
cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
|
|
1207
|
-
~blockNumber=rollbackTargetBlockNumber,
|
|
1208
|
-
)
|
|
1209
1250
|
cs.fetchState =
|
|
1210
1251
|
cs.fetchState->FetchState.rollback(
|
|
1211
1252
|
~addressStore=cs.addressStore,
|