envio 3.6.1 → 3.7.0-subgraph

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/index.d.ts +197 -9
  2. package/package.json +6 -6
  3. package/src/Batch.res +64 -25
  4. package/src/Batch.res.mjs +69 -46
  5. package/src/ChainFetching.res +13 -17
  6. package/src/ChainFetching.res.mjs +9 -14
  7. package/src/ChainState.res +160 -119
  8. package/src/ChainState.res.mjs +75 -46
  9. package/src/ChainState.resi +15 -12
  10. package/src/Config.res +18 -2
  11. package/src/Config.res.mjs +9 -6
  12. package/src/Core.res +20 -0
  13. package/src/Core.res.mjs +11 -0
  14. package/src/Ecosystem.res +0 -1
  15. package/src/Envio.res +16 -3
  16. package/src/EventConfigBuilder.res +155 -40
  17. package/src/EventConfigBuilder.res.mjs +76 -24
  18. package/src/EventProcessing.res +10 -14
  19. package/src/EventProcessing.res.mjs +12 -10
  20. package/src/EventUtils.res +3 -47
  21. package/src/FetchState.res +124 -104
  22. package/src/FetchState.res.mjs +116 -100
  23. package/src/HandlerLoader.res +20 -8
  24. package/src/HandlerLoader.res.mjs +11 -2
  25. package/src/HandlerRegister.res +108 -2
  26. package/src/HandlerRegister.res.mjs +59 -10
  27. package/src/HandlerRegister.resi +4 -0
  28. package/src/InMemoryStore.res +1 -6
  29. package/src/InMemoryStore.res.mjs +1 -3
  30. package/src/Internal.res +84 -24
  31. package/src/Internal.res.mjs +33 -2
  32. package/src/LazyLoader.res +12 -11
  33. package/src/LazyLoader.res.mjs +13 -7
  34. package/src/Main.res +22 -9
  35. package/src/Main.res.mjs +18 -5
  36. package/src/MemoryStorage.res +756 -0
  37. package/src/MemoryStorage.res.mjs +623 -0
  38. package/src/Persistence.res +4 -2
  39. package/src/PgStorage.res +24 -8
  40. package/src/PgStorage.res.mjs +15 -12
  41. package/src/ReorgDetection.res +0 -222
  42. package/src/ReorgDetection.res.mjs +0 -163
  43. package/src/Rollback.res +14 -15
  44. package/src/Rollback.res.mjs +4 -5
  45. package/src/SimulateItems.res +2 -2
  46. package/src/UserContext.res +358 -51
  47. package/src/UserContext.res.mjs +271 -35
  48. package/src/Utils.res +3 -0
  49. package/src/bindings/ClickHouse.res +16 -11
  50. package/src/bindings/ClickHouse.res.mjs +10 -10
  51. package/src/bindings/Vitest.res +1 -1
  52. package/src/sources/BlockStore.res +115 -5
  53. package/src/sources/BlockStore.res.mjs +25 -0
  54. package/src/sources/Evm.res +0 -1
  55. package/src/sources/Evm.res.mjs +0 -1
  56. package/src/sources/EvmHyperSyncSource.res +19 -84
  57. package/src/sources/EvmHyperSyncSource.res.mjs +24 -60
  58. package/src/sources/Fuel.res +24 -4
  59. package/src/sources/Fuel.res.mjs +23 -3
  60. package/src/sources/FuelHyperSync.res +8 -3
  61. package/src/sources/FuelHyperSync.res.mjs +5 -4
  62. package/src/sources/FuelHyperSync.resi +3 -1
  63. package/src/sources/FuelHyperSyncClient.res +7 -10
  64. package/src/sources/FuelHyperSyncSource.res +18 -45
  65. package/src/sources/FuelHyperSyncSource.res.mjs +18 -35
  66. package/src/sources/HyperSync.res +49 -229
  67. package/src/sources/HyperSync.res.mjs +74 -191
  68. package/src/sources/HyperSync.resi +11 -35
  69. package/src/sources/HyperSyncClient.res +9 -79
  70. package/src/sources/HyperSyncClient.res.mjs +2 -6
  71. package/src/sources/RequestStat.res +5 -0
  72. package/src/sources/RequestStat.res.mjs +2 -0
  73. package/src/sources/RpcSource.res +149 -43
  74. package/src/sources/RpcSource.res.mjs +104 -41
  75. package/src/sources/SimulateSource.res +8 -5
  76. package/src/sources/SimulateSource.res.mjs +6 -6
  77. package/src/sources/Source.res +89 -16
  78. package/src/sources/Source.res.mjs +50 -1
  79. package/src/sources/SourceManager.res +255 -50
  80. package/src/sources/SourceManager.res.mjs +149 -55
  81. package/src/sources/SourceManager.resi +2 -6
  82. package/src/sources/Svm.res +0 -7
  83. package/src/sources/Svm.res.mjs +0 -8
  84. package/src/sources/SvmHyperSyncClient.res +21 -16
  85. package/src/sources/SvmHyperSyncClient.res.mjs +3 -4
  86. package/src/sources/SvmHyperSyncSource.res +25 -117
  87. package/src/sources/SvmHyperSyncSource.res.mjs +8 -121
  88. package/src/subgraph/blocks.ts +176 -0
  89. package/src/subgraph/calls.ts +217 -0
  90. package/src/subgraph/conformance.ts +99 -0
  91. package/src/subgraph/division.ts +110 -0
  92. package/src/subgraph/errors.ts +90 -0
  93. package/src/subgraph/graph-ts-types/VERSION +2 -0
  94. package/src/subgraph/graph-ts-types/chain/arweave.d.ts +70 -0
  95. package/src/subgraph/graph-ts-types/chain/cosmos.d.ts +327 -0
  96. package/src/subgraph/graph-ts-types/chain/ethereum.d.ts +233 -0
  97. package/src/subgraph/graph-ts-types/chain/near.d.ts +253 -0
  98. package/src/subgraph/graph-ts-types/chain/starknet.d.ts +32 -0
  99. package/src/subgraph/graph-ts-types/common/collections.d.ts +136 -0
  100. package/src/subgraph/graph-ts-types/common/conversion.d.ts +11 -0
  101. package/src/subgraph/graph-ts-types/common/datasource.d.ts +30 -0
  102. package/src/subgraph/graph-ts-types/common/eager-offset.d.ts +0 -0
  103. package/src/subgraph/graph-ts-types/common/json.d.ts +17 -0
  104. package/src/subgraph/graph-ts-types/common/numbers.d.ts +120 -0
  105. package/src/subgraph/graph-ts-types/common/value.d.ts +120 -0
  106. package/src/subgraph/graph-ts-types/common/yaml.d.ts +90 -0
  107. package/src/subgraph/graph-ts-types/global/global.d.ts +194 -0
  108. package/src/subgraph/graph-ts-types/helper-functions.d.ts +22 -0
  109. package/src/subgraph/graph-ts-types/index.d.ts +102 -0
  110. package/src/subgraph/graph-ts.ts +1871 -0
  111. package/src/subgraph/hosts.ts +148 -0
  112. package/src/subgraph/runtime.ts +845 -0
  113. package/src/subgraph/scope.ts +63 -0
  114. package/svm.schema.json +3 -2
@@ -9,7 +9,7 @@ type t = {
9
9
  mutable fetchState: FetchState.t,
10
10
  // The chain-wide address index, kept in Rust. Not `mutable`: registration and
11
11
  // rollback mutate it in place, so the handle is stable across fetchState
12
- // versions and it's the same handle this chain's source clients hold.
12
+ // versions - and it's the same handle this chain's source clients hold.
13
13
  addressStore: AddressStore.t,
14
14
  sourceManager: SourceManager.t,
15
15
  chainConfig: Config.chain,
@@ -18,7 +18,7 @@ type t = {
18
18
  mutable committedProgressBlockNumber: int,
19
19
  // Progress of the batch currently being processed. The buffer is consumed at
20
20
  // batch creation (see advanceAfterBatch), so this runs ahead of
21
- // committedProgressBlockNumber until the batch commits it's the true lower
21
+ // committedProgressBlockNumber until the batch commits - it's the true lower
22
22
  // boundary of the remaining buffer's block span.
23
23
  mutable processingBlockNumber: int,
24
24
  mutable numEventsProcessed: float,
@@ -35,8 +35,12 @@ type t = {
35
35
  // Empty in the default cross-chain mode, where every entity's table lives on
36
36
  // the indexer instead.
37
37
  mutable entities: EntityTables.t,
38
- mutable reorgDetection: ReorgDetection.t,
39
38
  mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
39
+ // Reorg handling knobs, mirrored from the chain/indexer config: hashes are
40
+ // compared for blocks within `maxReorgDepth` of the known height, and a
41
+ // detected reorg either rolls back or is only logged.
42
+ shouldRollbackOnReorg: bool,
43
+ maxReorgDepth: int,
40
44
  // Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
41
45
  // transactionIndex). Fetch responses merge their page in; entries are pruned
42
46
  // as the chain progresses and dropped above the target on rollback.
@@ -90,9 +94,10 @@ let make = (
90
94
  ~onEventRegistrations=[],
91
95
  ~addressStore: AddressStore.t,
92
96
  ~sourceManager: SourceManager.t,
93
- ~reorgDetection: ReorgDetection.t,
94
97
  ~committedProgressBlockNumber: int,
95
98
  ~safeCheckpointTracking=None,
99
+ ~shouldRollbackOnReorg,
100
+ ~maxReorgDepth,
96
101
  ~numEventsProcessed=0.,
97
102
  ~timestampCaughtUpToHeadOrEndblock=None,
98
103
  ~isProgressAtHead=false,
@@ -119,8 +124,9 @@ let make = (
119
124
  numEventsProcessed,
120
125
  pendingBudget: 0.,
121
126
  chainDensity,
122
- reorgDetection,
123
127
  safeCheckpointTracking,
128
+ shouldRollbackOnReorg,
129
+ maxReorgDepth,
124
130
  transactionStore,
125
131
  blockStore,
126
132
  reorgThresholdReadyTolerance,
@@ -211,14 +217,17 @@ let makeInternal = (
211
217
  ~knownHeight,
212
218
  ~chainId=chainConfig.id,
213
219
  // FIXME: Shouldn't set with full history
220
+ // The resumed depth, not the config one: the pre-threshold lag must match
221
+ // the window reorg detection compares, or a reduced config depth would let
222
+ // fetching enter the stored rollback window without history.
214
223
  ~blockLag=Pervasives.max(
215
- !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth,
224
+ !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth,
216
225
  chainConfig.blockLag,
217
226
  ),
218
227
  ~onBlockRegistrations,
219
228
  ~firstEventBlock,
220
229
  // Fuel and SVM route through the address store like EVM does, so the
221
- // client-side path works for them but their address counts are nowhere
230
+ // client-side path works for them - but their address counts are nowhere
222
231
  // near the threshold, so switching would only trade a server-side filter
223
232
  // that costs nothing today for an over-fetch. Keep them server-side until a
224
233
  // chain actually needs it.
@@ -293,14 +302,41 @@ let makeInternal = (
293
302
  ]
294
303
  }
295
304
  | Config.SimulateSourceConfig({items, endBlock}) => [
296
- SimulateSource.make(~items, ~endBlock, ~chainId, ~addressStore),
305
+ SimulateSource.make(
306
+ ~items,
307
+ ~endBlock,
308
+ ~chainId,
309
+ ~addressStore,
310
+ ~ecosystem=config.ecosystem.name,
311
+ ),
297
312
  ]
298
313
  // For tests: use ready-to-use sources directly
299
314
  | Config.CustomSources(sources) => sources
300
315
  }
301
316
 
317
+ let blockStore = BlockStore.make(
318
+ ~ecosystem=config.ecosystem.name,
319
+ ~shouldChecksum=!lowercaseAddresses,
320
+ )
321
+
322
+ // Seed the stored reorg checkpoints (hash-only rows) so detection resumes
323
+ // against the hashes scanned before the restart.
324
+ if chainReorgCheckpoints->Utils.Array.notEmpty {
325
+ let seedPage = BlockStore.fromJs(
326
+ chainReorgCheckpoints->Array.map((cp): BlockStore.inputBlock => {
327
+ blockNumber: cp.blockNumber,
328
+ blockHash: cp.blockHash,
329
+ }),
330
+ ~ecosystem=config.ecosystem.name,
331
+ ~shouldChecksum=!lowercaseAddresses,
332
+ )
333
+ blockStore
334
+ ->BlockStore.merge(seedPage, ~fromBlock=0, ~reportOnly=false)
335
+ ->ignore
336
+ }
337
+
302
338
  // Seed chain density from whatever progress this chain already has (from a
303
- // resumed DB state, or 0 on a fresh chain) refined per-batch afterwards.
339
+ // resumed DB state, or 0 on a fresh chain) - refined per-batch afterwards.
304
340
  let chainDensity = switch fetchState.firstEventBlock {
305
341
  | Some(firstEventBlock) if progressBlockNumber > firstEventBlock && numEventsProcessed > 0. =>
306
342
  Some(numEventsProcessed /. (progressBlockNumber - firstEventBlock)->Int.toFloat)
@@ -313,11 +349,8 @@ let makeInternal = (
313
349
  ~onEventRegistrations,
314
350
  ~addressStore,
315
351
  ~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
316
- ~reorgDetection=ReorgDetection.make(
317
- ~chainReorgCheckpoints,
318
- ~maxReorgDepth,
319
- ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
320
- ),
352
+ ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
353
+ ~maxReorgDepth,
321
354
  ~safeCheckpointTracking=SafeCheckpointTracking.make(
322
355
  ~maxReorgDepth,
323
356
  ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
@@ -332,10 +365,7 @@ let makeInternal = (
332
365
  ~shouldChecksum=!lowercaseAddresses,
333
366
  ),
334
367
  ~chainDensity,
335
- ~blockStore=BlockStore.make(
336
- ~ecosystem=config.ecosystem.name,
337
- ~shouldChecksum=!lowercaseAddresses,
338
- ),
368
+ ~blockStore,
339
369
  ~reorgThresholdReadyTolerance=config.reorgThresholdReadyTolerance,
340
370
  ~logger,
341
371
  )
@@ -415,6 +445,7 @@ let makeFromDbState = (
415
445
  // --- Read accessors. ---
416
446
 
417
447
  let logger = (cs: t) => cs.logger
448
+ let blockStore = (cs: t) => cs.blockStore
418
449
  let entities = (cs: t) => cs.entities
419
450
 
420
451
  // Rollback discards every uncommitted change, so this chain's partition is
@@ -422,7 +453,20 @@ let entities = (cs: t) => cs.entities
422
453
  let resetEntities = (cs: t, ~perChainEntities) => cs.entities = EntityTables.make(perChainEntities)
423
454
  let sourceManager = (cs: t) => cs.sourceManager
424
455
  let chainConfig = (cs: t) => cs.chainConfig
425
- let reorgDetection = (cs: t) => cs.reorgDetection
456
+ let shouldRollbackOnReorg = (cs: t) => cs.shouldRollbackOnReorg
457
+ // Reorg-scan reads over the block store, for the rollback flow: the scanned
458
+ // block numbers still inside the reorg threshold, and the highest of them whose
459
+ // re-fetched hash still matches.
460
+ let getReorgThresholdBlockNumbersBelow = (cs: t, ~blockNumber) =>
461
+ cs.blockStore->BlockStore.getHashedBlockNumbers(
462
+ ~fromBlock=Pervasives.max(cs.fetchState.knownHeight - cs.maxReorgDepth, 0),
463
+ ~belowBlock=blockNumber,
464
+ )
465
+
466
+ let getLatestValidScannedBlock = (cs: t, ~blockStore: BlockStore.t, ~blockNumbers: array<int>) =>
467
+ cs.blockStore
468
+ ->BlockStore.latestValidBlockFromStore(blockStore, blockNumbers)
469
+ ->Null.toOption
426
470
  let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
427
471
  let isProgressAtHead = (cs: t) => cs.isProgressAtHead
428
472
  let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
@@ -512,7 +556,7 @@ let fetchCeiling = (cs: t) => {
512
556
  }
513
557
  }
514
558
 
515
- // Events/block over the ready part of the buffer a live signal that reacts
559
+ // Events/block over the ready part of the buffer - a live signal that reacts
516
560
  // to what fetching just found, unlike the processing EMA which only moves as
517
561
  // batches commit.
518
562
  let readyBufferDensity = (cs: t) => {
@@ -527,7 +571,7 @@ let readyBufferDensity = (cs: t) => {
527
571
 
528
572
  // Density used for query sizing: the higher of the processing EMA and the
529
573
  // ready-buffer density, so a stale-low EMA can't undersize queries while the
530
- // buffer proves the range is dense. None means the chain is cold no density
574
+ // buffer proves the range is dense. None means the chain is cold - no density
531
575
  // signal at all.
532
576
  let effectiveDensity = (cs: t) =>
533
577
  switch (cs.chainDensity, cs->readyBufferDensity) {
@@ -551,7 +595,7 @@ let targetBlock = (cs: t, ~chainTargetItems: float) => {
551
595
  | Some(density) if density > 0. =>
552
596
  // Decided by comparison so no unbounded value is ever converted to int:
553
597
  // at low densities chainTargetItems /. density exceeds the int range, and
554
- // truncating it wraps negative the target collapses below the frontier
598
+ // truncating it wraps negative - the target collapses below the frontier
555
599
  // and the chain stops querying. The division only runs when its result is
556
600
  // provably below the ceiling-bounded range.
557
601
  if density *. (fetchCeiling - bufferBlockNumber)->Int.toFloat <= chainTargetItems {
@@ -565,7 +609,7 @@ let targetBlock = (cs: t, ~chainTargetItems: float) => {
565
609
 
566
610
  // Block range that cross-chain progress alignment maps fractions over: from
567
611
  // the chain's startBlock to the last block it can fetch right now. The lower
568
- // bound is deliberately static anchoring it at firstEventBlock would make
612
+ // bound is deliberately static - anchoring it at firstEventBlock would make
569
613
  // two chains sitting at the same block read different progress fractions
570
614
  // depending on whether each has discovered its first event yet, so the
571
615
  // anchor's line could map below a follower's own frontier and stall it.
@@ -576,7 +620,7 @@ let progressRange = (cs: t) => {
576
620
 
577
621
  // A degenerate range (chain already at or past its last block) maps to 1 so it
578
622
  // never constrains the other chains. Clamped at 0 for the initial -1 fetch
579
- // frontier the only possible blockNumber below the range's lower bound.
623
+ // frontier - the only possible blockNumber below the range's lower bound.
580
624
  let progressAtBlock = (cs: t, ~blockNumber) => {
581
625
  let (lower, upper) = cs->progressRange
582
626
  upper <= lower
@@ -592,7 +636,7 @@ let blockAtProgress = (cs: t, ~progress) => {
592
636
  lower + Math.ceil(progress *. (upper - lower)->Int.toFloat)->Float.toInt
593
637
  }
594
638
 
595
- // The fetch frontier as a fraction of the alignable range what the
639
+ // The fetch frontier as a fraction of the alignable range - what the
596
640
  // cross-chain waterfall aligns other chains against. Based on blocks actually
597
641
  // fetched, so it holds up even while this chain's queries are in flight.
598
642
  let frontierProgress = (cs: t) =>
@@ -611,7 +655,7 @@ let getNextQuery = (cs: t, ~chainTargetItems: float, ~maxTargetBlock=?) => {
611
655
  | None => chainTargetBlock
612
656
  }
613
657
  // When the target block is clamped (head/endBlock/cross-chain alignment) a
614
- // known-density chain can't use the whole handed budget cap the fresh part
658
+ // known-density chain can't use the whole handed budget - cap the fresh part
615
659
  // at what the clamped range actually costs (in-flight reservations stay on
616
660
  // top: they're already accounted and shouldn't crowd out new partitions), so
617
661
  // the waterfall's leftover flows to the next chain in the same tick instead
@@ -676,7 +720,7 @@ let isDurablyCaughtUp = (cs: t) => {
676
720
  }
677
721
 
678
722
  let getHighestBlockBelowThreshold = (cs: t): int => {
679
- let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.chainConfig.maxReorgDepth
723
+ let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth
680
724
  highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
681
725
  }
682
726
 
@@ -685,7 +729,7 @@ let isActivelyIndexing = (cs: t) => cs.fetchState->FetchState.isActivelyIndexing
685
729
  // True once the fetch frontier has reached the head/endBlock for this chain.
686
730
  let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
687
731
 
688
- // Reached head on a chain with no configured endBlock used by auto-exit to
732
+ // Reached head on a chain with no configured endBlock - used by auto-exit to
689
733
  // detect that no events were found in the start..head range.
690
734
  let isAtHeadWithoutEndBlock = (cs: t) =>
691
735
  cs.isProgressAtHead && cs.fetchState.endBlock->Option.isNone
@@ -713,12 +757,7 @@ type blockGroups = {
713
757
  // same batch independently. Items already arrive in (blockNumber, logIndex)
714
758
  // order, so a (blockNumber, transactionIndex) run and a blockNumber-only run
715
759
  // each stay adjacent; every item is checked against both, in the same pass.
716
- // `includeBlocks` skips the block side entirely for Fuel, which carries the
717
- // block inline and has no store.
718
- let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
719
- transactionGroups,
720
- blockGroups,
721
- ) => {
760
+ let groupBatchItems = (items: array<Internal.item>): (transactionGroups, blockGroups) => {
722
761
  let txBlockNumbers = []
723
762
  let transactionIndices = []
724
763
  let transactionMasks = []
@@ -739,7 +778,7 @@ let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
739
778
  | Some(_) => () // RPC/simulate/Fuel carry the transaction inline.
740
779
  | None =>
741
780
  let {transactionIndex} = eventItem
742
- let mask = eventItem.onEventRegistration.eventConfig.transactionFieldMask
781
+ let mask = eventItem.onEventRegistration.fieldSelection.transactionMask
743
782
  if mask != 0. {
744
783
  anyTransactionFieldSelected := true
745
784
  }
@@ -762,27 +801,25 @@ let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
762
801
  }
763
802
  }
764
803
 
765
- if includeBlocks {
766
- switch eventItem.payload->Internal.getPayloadBlock->Nullable.toOption {
767
- | Some(_) => () // RPC/simulate/Fuel carry the block inline.
768
- | None =>
769
- let mask = eventItem.onEventRegistration.eventConfig.blockFieldMask
770
- let last = blockItemGroups->Array.length - 1
771
- if last >= 0 && blockBlockNumbers->Array.getUnsafe(last) == blockNumber {
772
- blockItemGroups->Array.getUnsafe(last)->Array.push(eventItem)
773
- blockMasks->Array.setUnsafe(
774
- last,
775
- FieldMask.orMask(blockMasks->Array.getUnsafe(last), mask),
776
- )
777
- } else {
778
- blockBlockNumbers->Array.push(blockNumber)
779
- blockMasks->Array.push(mask)
780
- blockItemGroups->Array.push([eventItem])
781
- }
804
+ switch eventItem.payload->Internal.getPayloadBlock->Nullable.toOption {
805
+ | Some(_) => () // RPC/simulate carry the block inline.
806
+ | None =>
807
+ let mask = eventItem.onEventRegistration.fieldSelection.blockMask
808
+ let last = blockItemGroups->Array.length - 1
809
+ if last >= 0 && blockBlockNumbers->Array.getUnsafe(last) == blockNumber {
810
+ blockItemGroups->Array.getUnsafe(last)->Array.push(eventItem)
811
+ blockMasks->Array.setUnsafe(
812
+ last,
813
+ FieldMask.orMask(blockMasks->Array.getUnsafe(last), mask),
814
+ )
815
+ } else {
816
+ blockBlockNumbers->Array.push(blockNumber)
817
+ blockMasks->Array.push(mask)
818
+ blockItemGroups->Array.push([eventItem])
782
819
  }
783
820
  }
784
821
  // onBlock items build their block from the handler's own block number, not
785
- // from the stores which is what lets the sources keep only the blocks and
822
+ // from the stores - which is what lets the sources keep only the blocks and
786
823
  // transactions an event item references.
787
824
  | Internal.Block(_) => ()
788
825
  }
@@ -801,8 +838,8 @@ let groupBatchItems = (items: array<Internal.item>, ~includeBlocks: bool): (
801
838
  }
802
839
 
803
840
  // Materialise a `TransactionStore` against precomputed groups (see
804
- // `groupBatchItems`). Store-backed items always get a transaction object the
805
- // selected fields, or `{}` when nothing was selected so `event.transaction`
841
+ // `groupBatchItems`). Store-backed items always get a transaction object - the
842
+ // selected fields, or `{}` when nothing was selected - so `event.transaction`
806
843
  // is never `undefined` (matching the inline sources).
807
844
  let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGroups) => {
808
845
  if g.payloadGroups->Utils.Array.notEmpty {
@@ -838,45 +875,34 @@ let applyBlockGroups = async (store: BlockStore.t, g: blockGroups) => {
838
875
  }
839
876
  }
840
877
 
841
- let includeBlocksForEcosystem = (ecosystem: Ecosystem.name) =>
842
- switch ecosystem {
843
- | Evm | Svm => true
844
- | Fuel => false
845
- }
846
-
847
878
  // Materialise the chain stores' selected transaction and block fields onto a
848
879
  // batch's items at batch prep (the persistent-store path). A single pass over
849
880
  // `items` (`groupBatchItems`) builds both stores' selection masks before the
850
881
  // two independent materialize calls run concurrently.
851
- let materializeBatchItems = async (cs: t, ~items: array<Internal.item>, ~ecosystem) => {
852
- let (txGroups, blockGroups) =
853
- items->groupBatchItems(~includeBlocks=ecosystem->includeBlocksForEcosystem)
882
+ let materializeBatchItems = async (cs: t, ~items: array<Internal.item>) => {
883
+ let (txGroups, blockGroups) = items->groupBatchItems
854
884
  let _ = await Promise.all2((
855
885
  cs.transactionStore->applyTransactionGroups(txGroups),
856
886
  cs.blockStore->applyBlockGroups(blockGroups),
857
887
  ))
858
888
  }
859
889
 
860
- // Materialise a fetch-response page's transactions and blocks onto its items
861
- // before contract-register handlers read them. `None` pages (RPC/Fuel/Simulate
862
- // keep them inline) are a no-op.
890
+ // Materialise a fetch-response's transactions and blocks onto its items before
891
+ // contract-register handlers read them. Transactions come from the response's
892
+ // page (`None` when kept inline); blocks come from the chain store, which the
893
+ // response's page was already merged into by `registerReorgGuard`.
863
894
  let materializePageItems = async (
864
895
  ~items: array<Internal.item>,
865
896
  ~transactionStore: option<TransactionStore.t>,
866
- ~blockStore: option<BlockStore.t>,
867
- ~ecosystem,
897
+ ~blockStore: BlockStore.t,
868
898
  ) => {
869
- let (txGroups, blockGroups) =
870
- items->groupBatchItems(~includeBlocks=ecosystem->includeBlocksForEcosystem)
899
+ let (txGroups, blockGroups) = items->groupBatchItems
871
900
  let _ = await Promise.all2((
872
901
  switch transactionStore {
873
902
  | Some(store) => store->applyTransactionGroups(txGroups)
874
903
  | None => Promise.resolve()
875
904
  },
876
- switch blockStore {
877
- | Some(store) => store->applyBlockGroups(blockGroups)
878
- | None => Promise.resolve()
879
- },
905
+ blockStore->applyBlockGroups(blockGroups),
880
906
  ))
881
907
  }
882
908
 
@@ -885,21 +911,17 @@ let handleQueryResult = (
885
911
  ~query: FetchState.query,
886
912
  ~newItems,
887
913
  ~newRegistrations,
888
- ~latestFetchedBlock: FetchState.blockNumberAndTimestamp,
914
+ ~latestFetchedBlock: int,
889
915
  ~knownHeight,
890
916
  ~transactionStore as txPage: option<TransactionStore.t>,
891
- ~blockStore as blockPage: option<BlockStore.t>,
892
917
  ) => {
893
- // Merge this response's pages into the chain stores in lockstep with appending
894
- // its items to the buffer. Inline sources contribute no page.
918
+ // Merge this response's transaction page into the chain store in lockstep
919
+ // with appending its items to the buffer. Inline sources contribute no page;
920
+ // the block page was already merged by `registerReorgGuard`.
895
921
  switch txPage {
896
922
  | Some(page) => cs.transactionStore->TransactionStore.merge(page)
897
923
  | None => ()
898
924
  }
899
- switch blockPage {
900
- | Some(page) => cs.blockStore->BlockStore.merge(page)
901
- | None => ()
902
- }
903
925
 
904
926
  let fs = switch newRegistrations {
905
927
  | [] => cs.fetchState
@@ -908,9 +930,9 @@ let handleQueryResult = (
908
930
  ~addressStore=cs.addressStore,
909
931
  // This response is applied below, after the addresses land. It was routed
910
932
  // before they existed, so whatever it claims has to be inside the
911
- // catch-up range and an unbounded query can reach past the height that
933
+ // catch-up range - and an unbounded query can reach past the height that
912
934
  // was known when it went out.
913
- ~claimCeiling=Pervasives.max(knownHeight, latestFetchedBlock.blockNumber),
935
+ ~claimCeiling=Pervasives.max(knownHeight, latestFetchedBlock),
914
936
  newRegistrations,
915
937
  )
916
938
  }
@@ -924,15 +946,25 @@ let handleQueryResult = (
924
946
  cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsEst->Int.toFloat)
925
947
  }
926
948
 
927
- // Run reorg detection against a fetch response and commit the updated guard.
928
- // Returns the result so the caller can decide whether to roll back; on the
929
- // rollback path registerReorgGuard returns the guard unchanged, so committing
930
- // here is a no-op there.
931
- let registerReorgGuard = (cs: t, ~blockHashes, ~knownHeight): ReorgDetection.reorgResult => {
932
- let (updatedReorgDetection, reorgResult) =
933
- cs.reorgDetection->ReorgDetection.registerReorgGuard(~blockHashes, ~knownHeight)
934
- cs.reorgDetection = updatedReorgDetection
935
- reorgResult
949
+ // Run reorg detection against a fetch response by merging its block-store page
950
+ // into the chain store: hashes of blocks inside the reorg threshold are
951
+ // compared on the way. On a mismatch with rollback enabled the page is
952
+ // discarded (the stored hashes stay for the rollback comparison); in
953
+ // detect-only mode the page still merges, so the overwritten hash doesn't
954
+ // re-report on every response.
955
+ let registerReorgGuard = (cs: t, ~blockStore, ~knownHeight): ReorgDetection.reorgResult => {
956
+ switch cs.blockStore->BlockStore.merge(
957
+ blockStore,
958
+ ~fromBlock=Pervasives.max(knownHeight - cs.maxReorgDepth, 0),
959
+ ~reportOnly=!cs.shouldRollbackOnReorg,
960
+ ) {
961
+ | Null.Null => NoReorg
962
+ | Null.Value({blockNumber, storedHash, receivedHash}) =>
963
+ ReorgDetected({
964
+ scannedBlock: {blockNumber, blockHash: storedHash},
965
+ receivedBlock: {blockNumber, blockHash: receivedHash},
966
+ })
967
+ }
936
968
  }
937
969
 
938
970
  // Prepare for a reorg rollback: restore the events-processed counter to its
@@ -1008,14 +1040,28 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
1008
1040
  rollbackTargetBlock: cs.rollbackTargetBlock,
1009
1041
  }
1010
1042
 
1011
- // Snapshot the inputs a batch build needs from this chain.
1043
+ // Snapshot the inputs a batch build needs from this chain, including an
1044
+ // immutable copy of the scanned in-threshold block hashes so the batch never
1045
+ // reads the live store mid-build.
1012
1046
  let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
1013
- fetchState: cs.fetchState,
1014
- progressBlockNumber: cs.committedProgressBlockNumber,
1015
- totalEventsProcessed: cs.numEventsProcessed,
1016
- sourceBlockNumber: cs.fetchState.knownHeight,
1017
- reorgDetection: cs.reorgDetection,
1018
- chainConfig: cs.chainConfig,
1047
+ let {blockNumbers, hashes} =
1048
+ cs.blockStore->BlockStore.getHashes(
1049
+ ~fromBlock=Pervasives.max(cs.fetchState.knownHeight - cs.maxReorgDepth, 0),
1050
+ ~belowBlock=cs.fetchState.knownHeight + 1,
1051
+ )
1052
+ let hashByBlockNumber = Dict.make()
1053
+ blockNumbers->Array.forEachWithIndex((blockNumber, idx) =>
1054
+ hashByBlockNumber->Utils.Dict.setByInt(blockNumber, hashes->Array.getUnsafe(idx))
1055
+ )
1056
+ {
1057
+ fetchState: cs.fetchState,
1058
+ progressBlockNumber: cs.committedProgressBlockNumber,
1059
+ totalEventsProcessed: cs.numEventsProcessed,
1060
+ sourceBlockNumber: cs.fetchState.knownHeight,
1061
+ scannedHashes: {blockNumbers, hashByBlockNumber},
1062
+ shouldRollbackOnReorg: cs.shouldRollbackOnReorg,
1063
+ chainConfig: cs.chainConfig,
1064
+ }
1019
1065
  }
1020
1066
 
1021
1067
  // Whether the chain's post-batch fetch frontier is ready to cross into the reorg
@@ -1057,7 +1103,7 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
1057
1103
  | Some(chainAfterBatch) => {
1058
1104
  // Calculate and set latency metrics. The payload block is materialised or
1059
1105
  // inline by processing time; its timestamp may still be absent (e.g. an
1060
- // SVM slot with no block row) the metric is skipped then.
1106
+ // SVM slot with no block row) - the metric is skipped then.
1061
1107
  switch batch
1062
1108
  ->Batch.findLastEventItem(~chainId)
1063
1109
  ->Option.flatMap(eventItem =>
@@ -1088,13 +1134,13 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
1088
1134
  }
1089
1135
 
1090
1136
  // Chain-wide density update: seed with the batch's own events/block on
1091
- // the first update, then blend weighted by the batch's block span a
1137
+ // the first update, then blend weighted by the batch's block span - a
1092
1138
  // few sparse/dense blocks barely nudge the estimate, while a
1093
1139
  // window-sized batch replaces it.
1094
1140
  let deltaBlocks = chainAfterBatch.progressBlockNumber - cs.committedProgressBlockNumber
1095
1141
  if deltaBlocks > 0 {
1096
1142
  let deltaEvents = chainAfterBatch.totalEventsProcessed -. cs.numEventsProcessed
1097
- // Don't seed a density before the first event is seen a progress-only
1143
+ // Don't seed a density before the first event is seen - a progress-only
1098
1144
  // batch would otherwise set it to 0, matching the resume-seed guard.
1099
1145
  switch (cs.chainDensity, deltaEvents > 0.) {
1100
1146
  | (None, false) => ()
@@ -1122,7 +1168,12 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
1122
1168
  cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
1123
1169
  // Processed blocks' transactions and blocks are no longer needed.
1124
1170
  cs.transactionStore->TransactionStore.prune(chainAfterBatch.progressBlockNumber)
1125
- cs.blockStore->BlockStore.prune(chainAfterBatch.progressBlockNumber)
1171
+ // Processed blocks' other fields are dropped, but hashes still inside the
1172
+ // reorg threshold stay for reorg detection.
1173
+ cs.blockStore->BlockStore.prune(
1174
+ chainAfterBatch.progressBlockNumber,
1175
+ ~keepHashesFrom=cs.fetchState.knownHeight - cs.maxReorgDepth,
1176
+ )
1126
1177
  cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated
1127
1178
  switch cs.safeCheckpointTracking {
1128
1179
  | Some(safeCheckpointTracking) =>
@@ -1154,10 +1205,10 @@ let markReady = (cs: t, ~readyAt) =>
1154
1205
  }
1155
1206
 
1156
1207
  // Roll a chain back to a reorg target. With a progress diff, restore fetch/
1157
- // safe-checkpoint/progress state to `newProgressBlockNumber`; the reorg chain
1158
- // additionally rewinds its reorg-detection guard. A reorg chain with no diff
1159
- // entry still rewinds guard + fetch state to the target otherwise the stale
1160
- // block hash stays in the guard and re-triggers the same reorg.
1208
+ // safe-checkpoint/progress state to `newProgressBlockNumber`. A reorg chain
1209
+ // with no diff entry still rewinds fetch state + stores to the target -
1210
+ // otherwise the stale block hash stays in the store and re-triggers the same
1211
+ // reorg.
1161
1212
  let rollback = (
1162
1213
  cs: t,
1163
1214
  ~newProgressBlockNumber,
@@ -1175,12 +1226,6 @@ let rollback = (
1175
1226
  ~message="Missing events-processed diff for rolled-back chain",
1176
1227
  )
1177
1228
 
1178
- if isReorgChain {
1179
- cs.reorgDetection =
1180
- cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
1181
- ~blockNumber=rollbackTargetBlockNumber,
1182
- )
1183
- }
1184
1229
  switch cs.safeCheckpointTracking {
1185
1230
  | Some(safeCheckpointTracking) =>
1186
1231
  cs.safeCheckpointTracking = Some(
@@ -1202,10 +1247,6 @@ let rollback = (
1202
1247
  cs.numEventsProcessed = newTotalEventsProcessed
1203
1248
  | None =>
1204
1249
  if isReorgChain {
1205
- cs.reorgDetection =
1206
- cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
1207
- ~blockNumber=rollbackTargetBlockNumber,
1208
- )
1209
1250
  cs.fetchState =
1210
1251
  cs.fetchState->FetchState.rollback(
1211
1252
  ~addressStore=cs.addressStore,