envio 3.2.0 → 3.3.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +6 -7
  2. package/src/Batch.res +12 -15
  3. package/src/Batch.res.mjs +4 -5
  4. package/src/BatchProcessing.res +199 -0
  5. package/src/BatchProcessing.res.mjs +125 -0
  6. package/src/ChainFetching.res +375 -0
  7. package/src/ChainFetching.res.mjs +206 -0
  8. package/src/ChainMetadata.res +27 -0
  9. package/src/ChainMetadata.res.mjs +27 -0
  10. package/src/ChainState.res +618 -0
  11. package/src/ChainState.res.mjs +461 -0
  12. package/src/ChainState.resi +85 -0
  13. package/src/Config.res +13 -4
  14. package/src/Config.res.mjs +8 -3
  15. package/src/ContractRegisterContext.res +106 -0
  16. package/src/ContractRegisterContext.res.mjs +76 -0
  17. package/src/Core.res +3 -0
  18. package/src/CrossChainState.res +269 -0
  19. package/src/CrossChainState.res.mjs +197 -0
  20. package/src/CrossChainState.resi +47 -0
  21. package/src/Ecosystem.res +58 -0
  22. package/src/Ecosystem.res.mjs +43 -1
  23. package/src/Env.res +7 -2
  24. package/src/Env.res.mjs +5 -2
  25. package/src/EventConfigBuilder.res +35 -0
  26. package/src/EventConfigBuilder.res.mjs +21 -0
  27. package/src/EventProcessing.res +60 -46
  28. package/src/EventProcessing.res.mjs +33 -32
  29. package/src/EventUtils.res +0 -4
  30. package/src/EventUtils.res.mjs +0 -4
  31. package/src/ExitOnCaughtUp.res +10 -0
  32. package/src/ExitOnCaughtUp.res.mjs +22 -0
  33. package/src/FetchState.res +162 -113
  34. package/src/FetchState.res.mjs +181 -119
  35. package/src/HandlerLoader.res +6 -1
  36. package/src/HandlerLoader.res.mjs +13 -36
  37. package/src/InMemoryStore.res +36 -451
  38. package/src/InMemoryStore.res.mjs +20 -369
  39. package/src/IndexerLoop.res +44 -0
  40. package/src/IndexerLoop.res.mjs +46 -0
  41. package/src/IndexerState.res +544 -0
  42. package/src/IndexerState.res.mjs +547 -0
  43. package/src/IndexerState.resi +132 -0
  44. package/src/Internal.res +44 -7
  45. package/src/LoadLayer.res +20 -18
  46. package/src/LoadLayer.res.mjs +14 -12
  47. package/src/LoadLayer.resi +6 -3
  48. package/src/LogSelection.res +86 -17
  49. package/src/LogSelection.res.mjs +67 -14
  50. package/src/Logging.res +11 -44
  51. package/src/Logging.res.mjs +10 -42
  52. package/src/Main.res +54 -79
  53. package/src/Main.res.mjs +44 -56
  54. package/src/PgStorage.res +8 -71
  55. package/src/PgStorage.res.mjs +3 -47
  56. package/src/Prometheus.res +7 -8
  57. package/src/Prometheus.res.mjs +7 -4
  58. package/src/PruneStaleHistory.res +45 -0
  59. package/src/PruneStaleHistory.res.mjs +46 -0
  60. package/src/RawEvent.res +73 -0
  61. package/src/RawEvent.res.mjs +51 -0
  62. package/src/Rollback.res +204 -0
  63. package/src/Rollback.res.mjs +118 -0
  64. package/src/SimulateItems.res +12 -10
  65. package/src/SimulateItems.res.mjs +1 -1
  66. package/src/TestIndexer.res +34 -27
  67. package/src/TestIndexer.res.mjs +4 -1
  68. package/src/UserContext.res +26 -114
  69. package/src/UserContext.res.mjs +15 -78
  70. package/src/Writing.res +242 -0
  71. package/src/Writing.res.mjs +215 -0
  72. package/src/db/InternalTable.res +14 -27
  73. package/src/sources/Evm.res +40 -1
  74. package/src/sources/Evm.res.mjs +94 -84
  75. package/src/sources/Fuel.res +40 -1
  76. package/src/sources/Fuel.res.mjs +36 -26
  77. package/src/sources/HyperFuel.res +41 -120
  78. package/src/sources/HyperFuel.res.mjs +42 -80
  79. package/src/sources/HyperFuel.resi +1 -24
  80. package/src/sources/HyperFuelClient.res +16 -297
  81. package/src/sources/HyperFuelClient.res.mjs +5 -4
  82. package/src/sources/HyperFuelSource.res +33 -8
  83. package/src/sources/HyperFuelSource.res.mjs +56 -10
  84. package/src/sources/HyperSyncSource.res +7 -5
  85. package/src/sources/HyperSyncSource.res.mjs +3 -3
  86. package/src/sources/RpcSource.res +2 -2
  87. package/src/sources/RpcSource.res.mjs +1 -1
  88. package/src/sources/SourceManager.res +8 -18
  89. package/src/sources/SourceManager.res.mjs +8 -5
  90. package/src/sources/SourceManager.resi +4 -1
  91. package/src/sources/Svm.res +18 -1
  92. package/src/sources/Svm.res.mjs +30 -22
  93. package/src/sources/SvmHyperSyncSource.res +1 -1
  94. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
  95. package/src/tui/Tui.res +38 -36
  96. package/src/tui/Tui.res.mjs +51 -51
  97. package/src/ChainFetcher.res +0 -519
  98. package/src/ChainFetcher.res.mjs +0 -314
  99. package/src/ChainManager.res +0 -358
  100. package/src/ChainManager.res.mjs +0 -291
  101. package/src/Ctx.res +0 -6
  102. package/src/Ctx.res.mjs +0 -2
  103. package/src/GlobalState.res +0 -1056
  104. package/src/GlobalState.res.mjs +0 -1086
  105. package/src/GlobalStateManager.res +0 -57
  106. package/src/GlobalStateManager.res.mjs +0 -68
  107. package/src/GlobalStateManager.resi +0 -7
  108. package/src/Ports.res +0 -5
  109. package/src/Ports.res.mjs +0 -9
  110. package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
  111. package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
  112. package/src/sources/EnvioApiClient.res +0 -15
  113. package/src/sources/EnvioApiClient.res.mjs +0 -24
@@ -0,0 +1,618 @@
1
+ // Per-chain runtime state. `t` is mutated in place through the setters below;
2
+ // the type is opaque in the interface so callers can read fields but can only
3
+ // change them through the sanctioned mutators.
4
+
5
+ type t = {
6
+ logger: Pino.t,
7
+ mutable fetchState: FetchState.t,
8
+ sourceManager: SourceManager.t,
9
+ chainConfig: Config.chain,
10
+ mutable isProgressAtHead: bool,
11
+ mutable timestampCaughtUpToHeadOrEndblock: option<Date.t>,
12
+ mutable committedProgressBlockNumber: int,
13
+ mutable numEventsProcessed: float,
14
+ mutable reorgDetection: ReorgDetection.t,
15
+ mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
16
+ }
17
+
18
+ let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
19
+ let addresses = []
20
+ chainConfig.contracts->Array.forEach(contract => {
21
+ contract.addresses->Array.forEach(address => {
22
+ addresses->Array.push({
23
+ Internal.address,
24
+ contractName: contract.name,
25
+ registrationBlock: -1,
26
+ })
27
+ })
28
+ })
29
+ addresses
30
+ }
31
+
32
+ let make = (
33
+ ~chainConfig: Config.chain,
34
+ ~fetchState: FetchState.t,
35
+ ~sourceManager: SourceManager.t,
36
+ ~reorgDetection: ReorgDetection.t,
37
+ ~committedProgressBlockNumber: int,
38
+ ~safeCheckpointTracking=None,
39
+ ~numEventsProcessed=0.,
40
+ ~timestampCaughtUpToHeadOrEndblock=None,
41
+ ~isProgressAtHead=false,
42
+ ~logger: Pino.t,
43
+ ): t => {
44
+ logger,
45
+ fetchState,
46
+ sourceManager,
47
+ chainConfig,
48
+ isProgressAtHead,
49
+ timestampCaughtUpToHeadOrEndblock,
50
+ committedProgressBlockNumber,
51
+ numEventsProcessed,
52
+ reorgDetection,
53
+ safeCheckpointTracking,
54
+ }
55
+
56
+ let makeInternal = (
57
+ ~chainConfig: Config.chain,
58
+ ~indexingAddresses: array<Internal.indexingAddress>,
59
+ ~startBlock,
60
+ ~endBlock,
61
+ ~firstEventBlock=None,
62
+ ~progressBlockNumber,
63
+ ~config: Config.t,
64
+ ~registrations: HandlerRegister.registrations,
65
+ ~logger,
66
+ ~timestampCaughtUpToHeadOrEndblock,
67
+ ~numEventsProcessed,
68
+ ~isInReorgThreshold,
69
+ ~isRealtime,
70
+ ~reorgCheckpoints: array<Internal.reorgCheckpoint>,
71
+ ~maxReorgDepth,
72
+ ~knownHeight=0,
73
+ ~reducedPollingInterval=?,
74
+ ): t => {
75
+ // We don't need the router itself, but only validation logic,
76
+ // since now event router is created for selection of events
77
+ // and validation doesn't work correctly in routers.
78
+ // Ideally to split it into two different parts.
79
+ let eventRouter = EventRouter.empty()
80
+
81
+ // Aggregate events we want to fetch
82
+ let eventConfigs: array<Internal.eventConfig> = []
83
+
84
+ let notRegisteredEvents = []
85
+
86
+ chainConfig.contracts->Array.forEach(contract => {
87
+ let contractName = contract.name
88
+
89
+ contract.events->Array.forEach(eventConfig => {
90
+ let {isWildcard} = eventConfig
91
+ let hasContractRegister = eventConfig.contractRegister->Option.isSome
92
+
93
+ // Should validate the events
94
+ eventRouter->EventRouter.addOrThrow(
95
+ eventConfig.id,
96
+ (),
97
+ ~contractName,
98
+ ~chain=ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id),
99
+ ~eventName=eventConfig.name,
100
+ ~isWildcard,
101
+ )
102
+
103
+ // Filter out non-preRegistration events on preRegistration phase
104
+ // so we don't care about it in fetch state and workers anymore
105
+ let shouldBeIncluded = if config.enableRawEvents {
106
+ true
107
+ } else {
108
+ let isRegistered = hasContractRegister || eventConfig.handler->Option.isSome
109
+ if !isRegistered {
110
+ notRegisteredEvents->Array.push(eventConfig)
111
+ }
112
+ isRegistered
113
+ }
114
+
115
+ // Check if event has Static([]) filters (from a dynamic where
116
+ // callback returning `false` / SkipAll for this chain).
117
+ // If so, skip it entirely - it should never be fetched
118
+ let shouldSkip = try {
119
+ let getEventFiltersOrThrow = (
120
+ eventConfig->(Utils.magic: Internal.eventConfig => Internal.evmEventConfig)
121
+ ).getEventFiltersOrThrow
122
+
123
+ // Check for non-evm chains
124
+ if (
125
+ getEventFiltersOrThrow->(Utils.magic: (ChainMap.Chain.t => Internal.eventFilters) => bool)
126
+ ) {
127
+ switch getEventFiltersOrThrow(ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)) {
128
+ | Static([]) => true
129
+ | _ => false
130
+ }
131
+ } else {
132
+ false
133
+ }
134
+ } catch {
135
+ // Can throw when filter is invalid
136
+ // Don't skip an event in this case. Let it throw in a better place - source code
137
+ | _ => false
138
+ }
139
+
140
+ if shouldBeIncluded && !shouldSkip {
141
+ eventConfigs->Array.push(eventConfig)
142
+ }
143
+ })
144
+
145
+ switch contract.startBlock {
146
+ | Some(startBlock) if startBlock < chainConfig.startBlock =>
147
+ JsError.throwWithMessage(
148
+ `The start block for contract "${contractName}" is less than the chain start block. This is not supported yet.`,
149
+ )
150
+ | _ => ()
151
+ }
152
+ })
153
+
154
+ if notRegisteredEvents->Utils.Array.notEmpty {
155
+ logger->Logging.childInfo(
156
+ `The event${if notRegisteredEvents->Array.length > 1 {
157
+ "s"
158
+ } else {
159
+ ""
160
+ }} ${notRegisteredEvents
161
+ ->Array.map(eventConfig => `${eventConfig.contractName}.${eventConfig.name}`)
162
+ ->Array.joinUnsafe(", ")} don't have an event handler and skipped for indexing.`,
163
+ )
164
+ }
165
+
166
+ let onBlockConfigs =
167
+ registrations.onBlockByChainId->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->Int.toString)
168
+ switch onBlockConfigs {
169
+ | Some(onBlockConfigs) =>
170
+ // TODO: Move it to the HandlerRegister module
171
+ // so the error is thrown with better stack trace
172
+ onBlockConfigs->Array.forEach(onBlockConfig => {
173
+ if onBlockConfig.startBlock->Option.getOr(startBlock) < startBlock {
174
+ JsError.throwWithMessage(
175
+ `The start block for onBlock handler "${onBlockConfig.name}" is less than the chain start block (${startBlock->Int.toString}). This is not supported yet.`,
176
+ )
177
+ }
178
+ switch endBlock {
179
+ | Some(chainEndBlock) =>
180
+ if onBlockConfig.endBlock->Option.getOr(chainEndBlock) > chainEndBlock {
181
+ JsError.throwWithMessage(
182
+ `The end block for onBlock handler "${onBlockConfig.name}" is greater than the chain end block (${chainEndBlock->Int.toString}). This is not supported yet.`,
183
+ )
184
+ }
185
+ | None => ()
186
+ }
187
+ })
188
+ | None => ()
189
+ }
190
+
191
+ let fetchState = FetchState.make(
192
+ ~maxAddrInPartition=config.maxAddrInPartition,
193
+ ~addresses=indexingAddresses,
194
+ ~progressBlockNumber,
195
+ ~startBlock,
196
+ ~endBlock,
197
+ ~eventConfigs,
198
+ ~maxOnBlockBufferSize=2 * config.batchSize,
199
+ ~knownHeight,
200
+ ~chainId=chainConfig.id,
201
+ // FIXME: Shouldn't set with full history
202
+ ~blockLag=Pervasives.max(
203
+ !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth,
204
+ chainConfig.blockLag,
205
+ ),
206
+ ~onBlockConfigs?,
207
+ ~firstEventBlock,
208
+ )
209
+
210
+ let chainReorgCheckpoints = reorgCheckpoints->Array.filterMap(reorgCheckpoint => {
211
+ if reorgCheckpoint.chainId === chainConfig.id {
212
+ Some(reorgCheckpoint)
213
+ } else {
214
+ None
215
+ }
216
+ })
217
+
218
+ // Create sources lazily here - this is where API token validation happens
219
+ let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
220
+ let lowercaseAddresses = config.lowercaseAddresses
221
+ let sources = switch chainConfig.sourceConfig {
222
+ | Config.EvmSourceConfig({hypersync, rpcs}) =>
223
+ // Build Internal.evmContractConfig from contracts for EvmChain.makeSources
224
+ let evmContracts: array<Internal.evmContractConfig> = chainConfig.contracts->Array.map((
225
+ contract
226
+ ): Internal.evmContractConfig => {
227
+ name: contract.name,
228
+ abi: contract.abi,
229
+ events: contract.events->(
230
+ Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
231
+ ),
232
+ })
233
+ let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
234
+ let syncConfig = rpc.syncConfig
235
+ let ws = rpc.ws
236
+ {
237
+ url: rpc.url,
238
+ sourceFor: rpc.sourceFor,
239
+ ?syncConfig,
240
+ ?ws,
241
+ }
242
+ })
243
+ EvmChain.makeSources(
244
+ ~chain,
245
+ ~contracts=evmContracts,
246
+ ~hyperSync=hypersync,
247
+ ~rpcs=evmRpcs,
248
+ ~lowercaseAddresses,
249
+ )
250
+ | Config.FuelSourceConfig({hypersync}) => [
251
+ HyperFuelSource.make({chain, endpointUrl: hypersync, apiToken: Env.envioApiToken}),
252
+ ]
253
+ | Config.SvmSourceConfig({hypersync, rpc}) =>
254
+ switch (hypersync, rpc) {
255
+ | (None, None) =>
256
+ JsError.throwWithMessage(
257
+ `Chain ${chain->ChainMap.Chain.toChainId->Int.toString} has no SVM data source`,
258
+ )
259
+ | (None, Some(rpc)) => [Svm.makeRPCSource(~chain, ~rpc)]
260
+ | (Some(hypersyncUrl), _) =>
261
+ // HyperSync drives instruction sync. A configured RPC is ignored for now
262
+ // (RPC fallback isn't wired up yet).
263
+ let svmEventConfigs =
264
+ chainConfig.contracts
265
+ ->Array.flatMap(contract => contract.events)
266
+ ->(Utils.magic: array<Internal.eventConfig> => array<Internal.svmInstructionEventConfig>)
267
+ let apiToken = Env.envioApiToken
268
+ [
269
+ SvmHyperSyncSource.make({
270
+ chain,
271
+ endpointUrl: hypersyncUrl,
272
+ apiToken,
273
+ eventConfigs: svmEventConfigs,
274
+ clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
275
+ }),
276
+ ]
277
+ }
278
+ // For tests: use ready-to-use sources directly
279
+ | Config.CustomSources(sources) => sources
280
+ }
281
+
282
+ make(
283
+ ~chainConfig,
284
+ ~fetchState,
285
+ ~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
286
+ ~reorgDetection=ReorgDetection.make(
287
+ ~chainReorgCheckpoints,
288
+ ~maxReorgDepth,
289
+ ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
290
+ ),
291
+ ~safeCheckpointTracking=SafeCheckpointTracking.make(
292
+ ~maxReorgDepth,
293
+ ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
294
+ ~chainReorgCheckpoints,
295
+ ),
296
+ ~committedProgressBlockNumber=progressBlockNumber,
297
+ ~timestampCaughtUpToHeadOrEndblock,
298
+ ~numEventsProcessed,
299
+ ~logger,
300
+ )
301
+ }
302
+
303
+ let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~knownHeight) => {
304
+ let logger = Logging.createChild(~params={"chainId": chainConfig.id})
305
+
306
+ makeInternal(
307
+ ~chainConfig,
308
+ ~config,
309
+ ~registrations,
310
+ ~startBlock=chainConfig.startBlock,
311
+ ~endBlock=chainConfig.endBlock,
312
+ ~reorgCheckpoints=[],
313
+ ~maxReorgDepth=chainConfig.maxReorgDepth,
314
+ ~progressBlockNumber=-1,
315
+ ~timestampCaughtUpToHeadOrEndblock=None,
316
+ ~numEventsProcessed=0.,
317
+ ~logger,
318
+ ~indexingAddresses=configAddresses(chainConfig),
319
+ ~isInReorgThreshold=false,
320
+ ~isRealtime=false,
321
+ ~knownHeight,
322
+ )
323
+ }
324
+
325
+ /**
326
+ * This function allows a chain state to be created from metadata, in particular this is useful for restarting an indexer and making sure it fetches blocks from the same place.
327
+ */
328
+ let makeFromDbState = (
329
+ chainConfig: Config.chain,
330
+ ~resumedChainState: Persistence.initialChainState,
331
+ ~reorgCheckpoints,
332
+ ~isInReorgThreshold,
333
+ ~isRealtime,
334
+ ~config,
335
+ ~registrations,
336
+ ~reducedPollingInterval=?,
337
+ ) => {
338
+ let chainId = chainConfig.id
339
+ let logger = Logging.createChild(~params={"chainId": chainId})
340
+
341
+ Prometheus.ProgressEventsCount.set(~processedCount=resumedChainState.numEventsProcessed, ~chainId)
342
+
343
+ let progressBlockNumber =
344
+ // Can be -1 when not set
345
+ resumedChainState.progressBlockNumber >= 0
346
+ ? resumedChainState.progressBlockNumber
347
+ : resumedChainState.startBlock - 1
348
+
349
+ makeInternal(
350
+ ~indexingAddresses=resumedChainState.indexingAddresses,
351
+ ~chainConfig,
352
+ ~startBlock=resumedChainState.startBlock,
353
+ ~endBlock=resumedChainState.endBlock,
354
+ ~config,
355
+ ~registrations,
356
+ ~reorgCheckpoints,
357
+ ~maxReorgDepth=resumedChainState.maxReorgDepth,
358
+ ~firstEventBlock=resumedChainState.firstEventBlockNumber,
359
+ ~progressBlockNumber,
360
+ ~timestampCaughtUpToHeadOrEndblock=Env.updateSyncTimeOnRestart
361
+ ? None
362
+ : resumedChainState.timestampCaughtUpToHeadOrEndblock,
363
+ ~numEventsProcessed=resumedChainState.numEventsProcessed,
364
+ ~logger,
365
+ ~isInReorgThreshold,
366
+ ~isRealtime,
367
+ ~knownHeight=resumedChainState.sourceBlockNumber,
368
+ ~reducedPollingInterval?,
369
+ )
370
+ }
371
+
372
+ // --- Read accessors. ---
373
+
374
+ let logger = (cs: t) => cs.logger
375
+ let fetchState = (cs: t) => cs.fetchState
376
+ let sourceManager = (cs: t) => cs.sourceManager
377
+ let chainConfig = (cs: t) => cs.chainConfig
378
+ let reorgDetection = (cs: t) => cs.reorgDetection
379
+ let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
380
+ let isProgressAtHead = (cs: t) => cs.isProgressAtHead
381
+ let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
382
+ let numEventsProcessed = (cs: t) => cs.numEventsProcessed
383
+ let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
384
+
385
+ // --- Derived (pure). ---
386
+
387
+ let hasProcessedToEndblock = (cs: t) => {
388
+ let {committedProgressBlockNumber, fetchState} = cs
389
+ switch fetchState.endBlock {
390
+ | Some(endBlock) => committedProgressBlockNumber >= endBlock
391
+ | None => false
392
+ }
393
+ }
394
+
395
+ let getHighestBlockBelowThreshold = (cs: t): int => {
396
+ let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.chainConfig.maxReorgDepth
397
+ highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
398
+ }
399
+
400
+ let isActivelyIndexing = (cs: t) => cs.fetchState->FetchState.isActivelyIndexing
401
+
402
+ let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
403
+
404
+ // True once the fetch frontier has reached the head/endBlock for this chain.
405
+ let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
406
+
407
+ // --- State transitions. The chain state is mutated only through these; each
408
+ // owns a cohesive update so callers don't juggle individual fields. ---
409
+
410
+ // Apply a fetch response: register any new dynamic contracts, append the items
411
+ // to the buffer and advance the known head.
412
+ let handleQueryResult = (
413
+ cs: t,
414
+ ~query: FetchState.query,
415
+ ~newItems,
416
+ ~newItemsWithDcs,
417
+ ~latestFetchedBlock,
418
+ ~knownHeight,
419
+ ) => {
420
+ let fs = switch newItemsWithDcs {
421
+ | [] => cs.fetchState
422
+ | _ => cs.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
423
+ }
424
+
425
+ cs.fetchState =
426
+ fs
427
+ ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
428
+ ->FetchState.updateKnownHeight(~knownHeight)
429
+ }
430
+
431
+ // Run reorg detection against a fetch response and commit the updated guard.
432
+ // Returns the result so the caller can decide whether to roll back; on the
433
+ // rollback path registerReorgGuard returns the guard unchanged, so committing
434
+ // here is a no-op there.
435
+ let registerReorgGuard = (cs: t, ~blockHashes, ~knownHeight): ReorgDetection.reorgResult => {
436
+ let (updatedReorgDetection, reorgResult) =
437
+ cs.reorgDetection->ReorgDetection.registerReorgGuard(~blockHashes, ~knownHeight)
438
+ cs.reorgDetection = updatedReorgDetection
439
+ reorgResult
440
+ }
441
+
442
+ // Prepare for a reorg rollback: restore the events-processed counter to its
443
+ // pre-rollback value when an uncommitted rollback diff is being redone, and drop
444
+ // pending queries bound to the about-to-be-invalidated chain state.
445
+ let prepareReorg = (cs: t, ~eventsProcessedDiff) => {
446
+ switch eventsProcessedDiff {
447
+ | Some(diff) => cs.numEventsProcessed = cs.numEventsProcessed +. diff
448
+ | None => ()
449
+ }
450
+ cs.fetchState = cs.fetchState->FetchState.resetPendingQueries
451
+ }
452
+
453
+ let updateKnownHeight = (cs: t, ~knownHeight) =>
454
+ cs.fetchState = cs.fetchState->FetchState.updateKnownHeight(~knownHeight)
455
+
456
+ // In auto-exit mode, pin the endBlock to the earliest observed event block.
457
+ let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
458
+ switch cs.fetchState.endBlock {
459
+ | None => cs.fetchState = {...cs.fetchState, endBlock: Some(blockNumber)}
460
+ | Some(currentEndBlock) if blockNumber < currentEndBlock =>
461
+ cs.fetchState = {...cs.fetchState, endBlock: Some(blockNumber)}
462
+ | Some(_) => ()
463
+ }
464
+
465
+ // Shrink the fetch buffer by the configured blockLag on entering the reorg threshold.
466
+ let enterReorgThreshold = (cs: t) =>
467
+ cs.fetchState = cs.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
468
+
469
+ // Commit the post-batch fetch frontier for a chain that progressed in the batch,
470
+ // applying blockLag when this batch also crosses into the reorg threshold.
471
+ let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
472
+ switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
473
+ cs.fetchState.chainId,
474
+ ) {
475
+ | Some(chainAfterBatch) =>
476
+ cs.fetchState = enteringReorgThreshold
477
+ ? chainAfterBatch.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
478
+ : chainAfterBatch.fetchState
479
+ | None => ()
480
+ }
481
+
482
+ // Commit a processed batch's progress for this chain (progress block, events
483
+ // processed, head/safe-checkpoint tracking, first event block). Emits the
484
+ // per-chain progress metrics. Readiness is decided by CrossChainState once every
485
+ // chain is caught up (see markReady).
486
+ let applyBatchProgress = (cs: t, ~batch: Batch.t) => {
487
+ let chainId = cs.chainConfig.id
488
+
489
+ switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
490
+ | Some(chainAfterBatch) => {
491
+ if cs.committedProgressBlockNumber !== chainAfterBatch.progressBlockNumber {
492
+ Prometheus.ProgressBlockNumber.set(
493
+ ~blockNumber=chainAfterBatch.progressBlockNumber,
494
+ ~chainId,
495
+ )
496
+ }
497
+ if cs.numEventsProcessed !== chainAfterBatch.totalEventsProcessed {
498
+ Prometheus.ProgressEventsCount.set(
499
+ ~processedCount=chainAfterBatch.totalEventsProcessed,
500
+ ~chainId,
501
+ )
502
+ }
503
+
504
+ // Calculate and set latency metrics
505
+ switch batch->Batch.findLastEventItem(~chainId) {
506
+ | Some(eventItem) => {
507
+ let blockTimestampMs = eventItem.timestamp * 1000
508
+ Prometheus.ProgressLatency.set(
509
+ ~latencyMs=Date.now()->Float.toInt - blockTimestampMs,
510
+ ~chainId,
511
+ )
512
+ }
513
+ | None => ()
514
+ }
515
+
516
+ // Since we process per chain always in order, calculate firstEventBlock
517
+ // once, from the first item in a batch.
518
+ switch cs.fetchState.firstEventBlock {
519
+ | Some(_) => ()
520
+ | None =>
521
+ switch batch->Batch.findFirstEventBlockNumber(~chainId) {
522
+ | Some(_) as firstEventBlock => cs.fetchState = {...cs.fetchState, firstEventBlock}
523
+ | None => ()
524
+ }
525
+ }
526
+
527
+ cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
528
+ cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
529
+ cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated
530
+ switch cs.safeCheckpointTracking {
531
+ | Some(safeCheckpointTracking) =>
532
+ cs.safeCheckpointTracking = Some(
533
+ safeCheckpointTracking->SafeCheckpointTracking.updateOnNewBatch(
534
+ ~sourceBlockNumber=cs.fetchState.knownHeight,
535
+ ~chainId,
536
+ ~batchCheckpointIds=batch.checkpointIds,
537
+ ~batchCheckpointBlockNumbers=batch.checkpointBlockNumbers,
538
+ ~batchCheckpointChainIds=batch.checkpointChainIds,
539
+ ),
540
+ )
541
+ | None => ()
542
+ }
543
+ }
544
+ | None => ()
545
+ }
546
+ }
547
+
548
+ // Mark the chain caught up to head/endblock. Called by CrossChainState only once
549
+ // every chain in the indexer is caught up, so no chain flips to ready while
550
+ // another is still backfilling. Sticky: a chain stays ready once set.
551
+ let markReady = (cs: t) =>
552
+ if !(cs->isReady) {
553
+ cs.timestampCaughtUpToHeadOrEndblock = Date.make()->Some
554
+ Prometheus.ProgressReady.set(~chainId=cs.chainConfig.id)
555
+ }
556
+
557
+ // Roll a chain back to a reorg target. With a progress diff, restore fetch/
558
+ // safe-checkpoint/progress state to `newProgressBlockNumber`; the reorg chain
559
+ // additionally rewinds its reorg-detection guard. A reorg chain with no diff
560
+ // entry still rewinds guard + fetch state to the target — otherwise the stale
561
+ // block hash stays in the guard and re-triggers the same reorg.
562
+ let rollback = (
563
+ cs: t,
564
+ ~newProgressBlockNumber,
565
+ ~eventsProcessedDiff,
566
+ ~rollbackTargetBlockNumber,
567
+ ~isReorgChain,
568
+ ) => {
569
+ let chainId = cs.chainConfig.id
570
+ switch newProgressBlockNumber {
571
+ | Some(newProgressBlockNumber) =>
572
+ let newTotalEventsProcessed =
573
+ cs.numEventsProcessed -.
574
+ // Both dicts are populated together per progress-diff row, so a chain with
575
+ // a progress diff always has an events-processed diff too.
576
+ eventsProcessedDiff->Option.getOrThrow(
577
+ ~message="Missing events-processed diff for rolled-back chain",
578
+ )
579
+
580
+ if cs.committedProgressBlockNumber !== newProgressBlockNumber {
581
+ Prometheus.ProgressBlockNumber.set(~blockNumber=newProgressBlockNumber, ~chainId)
582
+ }
583
+ if cs.numEventsProcessed !== newTotalEventsProcessed {
584
+ Prometheus.ProgressEventsCount.set(~processedCount=newTotalEventsProcessed, ~chainId)
585
+ }
586
+ if isReorgChain {
587
+ cs.reorgDetection =
588
+ cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
589
+ ~blockNumber=rollbackTargetBlockNumber,
590
+ )
591
+ }
592
+ switch cs.safeCheckpointTracking {
593
+ | Some(safeCheckpointTracking) =>
594
+ cs.safeCheckpointTracking = Some(
595
+ safeCheckpointTracking->SafeCheckpointTracking.rollback(
596
+ ~targetBlockNumber=newProgressBlockNumber,
597
+ ),
598
+ )
599
+ | None => ()
600
+ }
601
+ cs.fetchState = cs.fetchState->FetchState.rollback(~targetBlockNumber=newProgressBlockNumber)
602
+ cs.committedProgressBlockNumber = newProgressBlockNumber
603
+ cs.numEventsProcessed = newTotalEventsProcessed
604
+ | None =>
605
+ if isReorgChain {
606
+ cs.reorgDetection =
607
+ cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
608
+ ~blockNumber=rollbackTargetBlockNumber,
609
+ )
610
+ cs.fetchState =
611
+ cs.fetchState->FetchState.rollback(~targetBlockNumber=rollbackTargetBlockNumber)
612
+ cs.committedProgressBlockNumber = Pervasives.min(
613
+ cs.committedProgressBlockNumber,
614
+ rollbackTargetBlockNumber,
615
+ )
616
+ }
617
+ }
618
+ }