envio 3.2.1 → 3.3.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/package.json +6 -7
  2. package/src/Batch.res +12 -15
  3. package/src/Batch.res.mjs +4 -5
  4. package/src/BatchProcessing.res +199 -0
  5. package/src/BatchProcessing.res.mjs +125 -0
  6. package/src/ChainFetching.res +373 -0
  7. package/src/ChainFetching.res.mjs +206 -0
  8. package/src/ChainMetadata.res +27 -0
  9. package/src/ChainMetadata.res.mjs +27 -0
  10. package/src/ChainState.res +643 -0
  11. package/src/ChainState.res.mjs +476 -0
  12. package/src/ChainState.resi +87 -0
  13. package/src/Config.res +13 -4
  14. package/src/Config.res.mjs +8 -3
  15. package/src/ContractRegisterContext.res +106 -0
  16. package/src/ContractRegisterContext.res.mjs +76 -0
  17. package/src/Core.res +3 -0
  18. package/src/CrossChainState.res +294 -0
  19. package/src/CrossChainState.res.mjs +221 -0
  20. package/src/CrossChainState.resi +39 -0
  21. package/src/Ecosystem.res +58 -0
  22. package/src/Ecosystem.res.mjs +43 -1
  23. package/src/Env.res +0 -2
  24. package/src/Env.res.mjs +0 -3
  25. package/src/EventConfigBuilder.res +2 -2
  26. package/src/EventProcessing.res +60 -46
  27. package/src/EventProcessing.res.mjs +33 -32
  28. package/src/EventUtils.res +0 -4
  29. package/src/EventUtils.res.mjs +0 -4
  30. package/src/ExitOnCaughtUp.res +10 -0
  31. package/src/ExitOnCaughtUp.res.mjs +22 -0
  32. package/src/FetchState.res +205 -73
  33. package/src/FetchState.res.mjs +242 -103
  34. package/src/InMemoryStore.res +36 -451
  35. package/src/InMemoryStore.res.mjs +20 -369
  36. package/src/IndexerLoop.res +43 -0
  37. package/src/IndexerLoop.res.mjs +46 -0
  38. package/src/IndexerState.res +536 -0
  39. package/src/IndexerState.res.mjs +543 -0
  40. package/src/IndexerState.resi +128 -0
  41. package/src/Internal.res +29 -8
  42. package/src/LoadLayer.res +20 -18
  43. package/src/LoadLayer.res.mjs +14 -12
  44. package/src/LoadLayer.resi +6 -3
  45. package/src/Logging.res +11 -44
  46. package/src/Logging.res.mjs +10 -42
  47. package/src/Main.res +54 -79
  48. package/src/Main.res.mjs +44 -56
  49. package/src/PgStorage.res +8 -71
  50. package/src/PgStorage.res.mjs +3 -47
  51. package/src/Prometheus.res +1 -13
  52. package/src/Prometheus.res.mjs +84 -96
  53. package/src/PruneStaleHistory.res +45 -0
  54. package/src/PruneStaleHistory.res.mjs +46 -0
  55. package/src/RawEvent.res +73 -0
  56. package/src/RawEvent.res.mjs +51 -0
  57. package/src/Rollback.res +204 -0
  58. package/src/Rollback.res.mjs +118 -0
  59. package/src/SimulateItems.res +12 -10
  60. package/src/SimulateItems.res.mjs +1 -1
  61. package/src/UserContext.res +26 -114
  62. package/src/UserContext.res.mjs +15 -78
  63. package/src/Writing.res +242 -0
  64. package/src/Writing.res.mjs +215 -0
  65. package/src/db/InternalTable.res +14 -27
  66. package/src/sources/Evm.res +40 -1
  67. package/src/sources/Evm.res.mjs +94 -84
  68. package/src/sources/Fuel.res +40 -1
  69. package/src/sources/Fuel.res.mjs +36 -26
  70. package/src/sources/HyperFuel.res +41 -120
  71. package/src/sources/HyperFuel.res.mjs +42 -80
  72. package/src/sources/HyperFuel.resi +1 -24
  73. package/src/sources/HyperFuelClient.res +16 -297
  74. package/src/sources/HyperFuelClient.res.mjs +5 -4
  75. package/src/sources/HyperFuelSource.res +33 -8
  76. package/src/sources/HyperFuelSource.res.mjs +56 -10
  77. package/src/sources/HyperSyncSource.res +5 -3
  78. package/src/sources/HyperSyncSource.res.mjs +1 -1
  79. package/src/sources/RpcSource.res +2 -2
  80. package/src/sources/RpcSource.res.mjs +1 -1
  81. package/src/sources/SourceManager.res +14 -23
  82. package/src/sources/SourceManager.res.mjs +27 -29
  83. package/src/sources/SourceManager.resi +4 -2
  84. package/src/sources/Svm.res +18 -1
  85. package/src/sources/Svm.res.mjs +30 -22
  86. package/src/sources/SvmHyperSyncSource.res +1 -1
  87. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
  88. package/src/tui/Tui.res +38 -36
  89. package/src/tui/Tui.res.mjs +51 -51
  90. package/src/ChainFetcher.res +0 -519
  91. package/src/ChainFetcher.res.mjs +0 -314
  92. package/src/ChainManager.res +0 -358
  93. package/src/ChainManager.res.mjs +0 -291
  94. package/src/Ctx.res +0 -6
  95. package/src/Ctx.res.mjs +0 -2
  96. package/src/GlobalState.res +0 -1056
  97. package/src/GlobalState.res.mjs +0 -1086
  98. package/src/GlobalStateManager.res +0 -57
  99. package/src/GlobalStateManager.res.mjs +0 -68
  100. package/src/GlobalStateManager.resi +0 -7
  101. package/src/Ports.res +0 -5
  102. package/src/Ports.res.mjs +0 -9
  103. package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
  104. package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
  105. package/src/sources/EnvioApiClient.res +0 -15
  106. package/src/sources/EnvioApiClient.res.mjs +0 -24
@@ -0,0 +1,643 @@
1
+ // Per-chain runtime state. `t` is mutated in place through the setters below;
2
+ // the type is opaque in the interface so callers can read fields but can only
3
+ // change them through the sanctioned mutators.
4
+
5
+ type t = {
6
+ logger: Pino.t,
7
+ mutable fetchState: FetchState.t,
8
+ sourceManager: SourceManager.t,
9
+ chainConfig: Config.chain,
10
+ mutable isProgressAtHead: bool,
11
+ mutable timestampCaughtUpToHeadOrEndblock: option<Date.t>,
12
+ mutable committedProgressBlockNumber: int,
13
+ mutable numEventsProcessed: float,
14
+ // Running sum of in-flight queries' estResponseSize, kept here so the
15
+ // scheduler doesn't re-sum pending queries on every tick. Incremented when
16
+ // queries are dispatched, decremented as their responses land.
17
+ mutable pendingBudget: float,
18
+ mutable reorgDetection: ReorgDetection.t,
19
+ mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
20
+ }
21
+
22
+ let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
23
+ let addresses = []
24
+ chainConfig.contracts->Array.forEach(contract => {
25
+ contract.addresses->Array.forEach(address => {
26
+ addresses->Array.push({
27
+ Internal.address,
28
+ contractName: contract.name,
29
+ registrationBlock: -1,
30
+ })
31
+ })
32
+ })
33
+ addresses
34
+ }
35
+
36
+ let make = (
37
+ ~chainConfig: Config.chain,
38
+ ~fetchState: FetchState.t,
39
+ ~sourceManager: SourceManager.t,
40
+ ~reorgDetection: ReorgDetection.t,
41
+ ~committedProgressBlockNumber: int,
42
+ ~safeCheckpointTracking=None,
43
+ ~numEventsProcessed=0.,
44
+ ~timestampCaughtUpToHeadOrEndblock=None,
45
+ ~isProgressAtHead=false,
46
+ ~logger: Pino.t,
47
+ ): t => {
48
+ logger,
49
+ fetchState,
50
+ sourceManager,
51
+ chainConfig,
52
+ isProgressAtHead,
53
+ timestampCaughtUpToHeadOrEndblock,
54
+ committedProgressBlockNumber,
55
+ numEventsProcessed,
56
+ pendingBudget: 0.,
57
+ reorgDetection,
58
+ safeCheckpointTracking,
59
+ }
60
+
61
+ let makeInternal = (
62
+ ~chainConfig: Config.chain,
63
+ ~indexingAddresses: array<Internal.indexingAddress>,
64
+ ~startBlock,
65
+ ~endBlock,
66
+ ~firstEventBlock=None,
67
+ ~progressBlockNumber,
68
+ ~config: Config.t,
69
+ ~registrations: HandlerRegister.registrations,
70
+ ~logger,
71
+ ~timestampCaughtUpToHeadOrEndblock,
72
+ ~numEventsProcessed,
73
+ ~isInReorgThreshold,
74
+ ~isRealtime,
75
+ ~reorgCheckpoints: array<Internal.reorgCheckpoint>,
76
+ ~maxReorgDepth,
77
+ ~knownHeight=0,
78
+ ~reducedPollingInterval=?,
79
+ ): t => {
80
+ // We don't need the router itself, but only validation logic,
81
+ // since now event router is created for selection of events
82
+ // and validation doesn't work correctly in routers.
83
+ // Ideally to split it into two different parts.
84
+ let eventRouter = EventRouter.empty()
85
+
86
+ // Aggregate events we want to fetch
87
+ let eventConfigs: array<Internal.eventConfig> = []
88
+
89
+ let notRegisteredEvents = []
90
+
91
+ chainConfig.contracts->Array.forEach(contract => {
92
+ let contractName = contract.name
93
+
94
+ contract.events->Array.forEach(eventConfig => {
95
+ let {isWildcard} = eventConfig
96
+ let hasContractRegister = eventConfig.contractRegister->Option.isSome
97
+
98
+ // Should validate the events
99
+ eventRouter->EventRouter.addOrThrow(
100
+ eventConfig.id,
101
+ (),
102
+ ~contractName,
103
+ ~chain=ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id),
104
+ ~eventName=eventConfig.name,
105
+ ~isWildcard,
106
+ )
107
+
108
+ // Filter out non-preRegistration events on preRegistration phase
109
+ // so we don't care about it in fetch state and workers anymore
110
+ let shouldBeIncluded = if config.enableRawEvents {
111
+ true
112
+ } else {
113
+ let isRegistered = hasContractRegister || eventConfig.handler->Option.isSome
114
+ if !isRegistered {
115
+ notRegisteredEvents->Array.push(eventConfig)
116
+ }
117
+ isRegistered
118
+ }
119
+
120
+ // Check if event has Static([]) filters (from a dynamic where
121
+ // callback returning `false` / SkipAll for this chain).
122
+ // If so, skip it entirely - it should never be fetched
123
+ let shouldSkip = try {
124
+ let getEventFiltersOrThrow = (
125
+ eventConfig->(Utils.magic: Internal.eventConfig => Internal.evmEventConfig)
126
+ ).getEventFiltersOrThrow
127
+
128
+ // Check for non-evm chains
129
+ if (
130
+ getEventFiltersOrThrow->(Utils.magic: (ChainMap.Chain.t => Internal.eventFilters) => bool)
131
+ ) {
132
+ switch getEventFiltersOrThrow(ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)) {
133
+ | Static([]) => true
134
+ | _ => false
135
+ }
136
+ } else {
137
+ false
138
+ }
139
+ } catch {
140
+ // Can throw when filter is invalid
141
+ // Don't skip an event in this case. Let it throw in a better place - source code
142
+ | _ => false
143
+ }
144
+
145
+ if shouldBeIncluded && !shouldSkip {
146
+ eventConfigs->Array.push(eventConfig)
147
+ }
148
+ })
149
+
150
+ switch contract.startBlock {
151
+ | Some(startBlock) if startBlock < chainConfig.startBlock =>
152
+ JsError.throwWithMessage(
153
+ `The start block for contract "${contractName}" is less than the chain start block. This is not supported yet.`,
154
+ )
155
+ | _ => ()
156
+ }
157
+ })
158
+
159
+ if notRegisteredEvents->Utils.Array.notEmpty {
160
+ logger->Logging.childInfo(
161
+ `The event${if notRegisteredEvents->Array.length > 1 {
162
+ "s"
163
+ } else {
164
+ ""
165
+ }} ${notRegisteredEvents
166
+ ->Array.map(eventConfig => `${eventConfig.contractName}.${eventConfig.name}`)
167
+ ->Array.joinUnsafe(", ")} don't have an event handler and skipped for indexing.`,
168
+ )
169
+ }
170
+
171
+ let onBlockConfigs =
172
+ registrations.onBlockByChainId->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->Int.toString)
173
+ switch onBlockConfigs {
174
+ | Some(onBlockConfigs) =>
175
+ // TODO: Move it to the HandlerRegister module
176
+ // so the error is thrown with better stack trace
177
+ onBlockConfigs->Array.forEach(onBlockConfig => {
178
+ if onBlockConfig.startBlock->Option.getOr(startBlock) < startBlock {
179
+ JsError.throwWithMessage(
180
+ `The start block for onBlock handler "${onBlockConfig.name}" is less than the chain start block (${startBlock->Int.toString}). This is not supported yet.`,
181
+ )
182
+ }
183
+ switch endBlock {
184
+ | Some(chainEndBlock) =>
185
+ if onBlockConfig.endBlock->Option.getOr(chainEndBlock) > chainEndBlock {
186
+ JsError.throwWithMessage(
187
+ `The end block for onBlock handler "${onBlockConfig.name}" is greater than the chain end block (${chainEndBlock->Int.toString}). This is not supported yet.`,
188
+ )
189
+ }
190
+ | None => ()
191
+ }
192
+ })
193
+ | None => ()
194
+ }
195
+
196
+ let fetchState = FetchState.make(
197
+ ~maxAddrInPartition=config.maxAddrInPartition,
198
+ ~addresses=indexingAddresses,
199
+ ~progressBlockNumber,
200
+ ~startBlock,
201
+ ~endBlock,
202
+ ~eventConfigs,
203
+ ~maxOnBlockBufferSize=2 * config.batchSize,
204
+ ~knownHeight,
205
+ ~chainId=chainConfig.id,
206
+ // FIXME: Shouldn't set with full history
207
+ ~blockLag=Pervasives.max(
208
+ !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth,
209
+ chainConfig.blockLag,
210
+ ),
211
+ ~onBlockConfigs?,
212
+ ~firstEventBlock,
213
+ )
214
+
215
+ let chainReorgCheckpoints = reorgCheckpoints->Array.filterMap(reorgCheckpoint => {
216
+ if reorgCheckpoint.chainId === chainConfig.id {
217
+ Some(reorgCheckpoint)
218
+ } else {
219
+ None
220
+ }
221
+ })
222
+
223
+ // Create sources lazily here - this is where API token validation happens
224
+ let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
225
+ let lowercaseAddresses = config.lowercaseAddresses
226
+ let sources = switch chainConfig.sourceConfig {
227
+ | Config.EvmSourceConfig({hypersync, rpcs}) =>
228
+ // Build Internal.evmContractConfig from contracts for EvmChain.makeSources
229
+ let evmContracts: array<Internal.evmContractConfig> = chainConfig.contracts->Array.map((
230
+ contract
231
+ ): Internal.evmContractConfig => {
232
+ name: contract.name,
233
+ abi: contract.abi,
234
+ events: contract.events->(
235
+ Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
236
+ ),
237
+ })
238
+ let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
239
+ let syncConfig = rpc.syncConfig
240
+ let ws = rpc.ws
241
+ {
242
+ url: rpc.url,
243
+ sourceFor: rpc.sourceFor,
244
+ ?syncConfig,
245
+ ?ws,
246
+ }
247
+ })
248
+ EvmChain.makeSources(
249
+ ~chain,
250
+ ~contracts=evmContracts,
251
+ ~hyperSync=hypersync,
252
+ ~rpcs=evmRpcs,
253
+ ~lowercaseAddresses,
254
+ )
255
+ | Config.FuelSourceConfig({hypersync}) => [
256
+ HyperFuelSource.make({chain, endpointUrl: hypersync, apiToken: Env.envioApiToken}),
257
+ ]
258
+ | Config.SvmSourceConfig({hypersync, rpc}) =>
259
+ switch (hypersync, rpc) {
260
+ | (None, None) =>
261
+ JsError.throwWithMessage(
262
+ `Chain ${chain->ChainMap.Chain.toChainId->Int.toString} has no SVM data source`,
263
+ )
264
+ | (None, Some(rpc)) => [Svm.makeRPCSource(~chain, ~rpc)]
265
+ | (Some(hypersyncUrl), _) =>
266
+ // HyperSync drives instruction sync. A configured RPC is ignored for now
267
+ // (RPC fallback isn't wired up yet).
268
+ let svmEventConfigs =
269
+ chainConfig.contracts
270
+ ->Array.flatMap(contract => contract.events)
271
+ ->(Utils.magic: array<Internal.eventConfig> => array<Internal.svmInstructionEventConfig>)
272
+ let apiToken = Env.envioApiToken
273
+ [
274
+ SvmHyperSyncSource.make({
275
+ chain,
276
+ endpointUrl: hypersyncUrl,
277
+ apiToken,
278
+ eventConfigs: svmEventConfigs,
279
+ clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
280
+ }),
281
+ ]
282
+ }
283
+ // For tests: use ready-to-use sources directly
284
+ | Config.CustomSources(sources) => sources
285
+ }
286
+
287
+ make(
288
+ ~chainConfig,
289
+ ~fetchState,
290
+ ~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
291
+ ~reorgDetection=ReorgDetection.make(
292
+ ~chainReorgCheckpoints,
293
+ ~maxReorgDepth,
294
+ ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
295
+ ),
296
+ ~safeCheckpointTracking=SafeCheckpointTracking.make(
297
+ ~maxReorgDepth,
298
+ ~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
299
+ ~chainReorgCheckpoints,
300
+ ),
301
+ ~committedProgressBlockNumber=progressBlockNumber,
302
+ ~timestampCaughtUpToHeadOrEndblock,
303
+ ~numEventsProcessed,
304
+ ~logger,
305
+ )
306
+ }
307
+
308
+ let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~knownHeight) => {
309
+ let logger = Logging.createChild(~params={"chainId": chainConfig.id})
310
+
311
+ makeInternal(
312
+ ~chainConfig,
313
+ ~config,
314
+ ~registrations,
315
+ ~startBlock=chainConfig.startBlock,
316
+ ~endBlock=chainConfig.endBlock,
317
+ ~reorgCheckpoints=[],
318
+ ~maxReorgDepth=chainConfig.maxReorgDepth,
319
+ ~progressBlockNumber=-1,
320
+ ~timestampCaughtUpToHeadOrEndblock=None,
321
+ ~numEventsProcessed=0.,
322
+ ~logger,
323
+ ~indexingAddresses=configAddresses(chainConfig),
324
+ ~isInReorgThreshold=false,
325
+ ~isRealtime=false,
326
+ ~knownHeight,
327
+ )
328
+ }
329
+
330
+ /**
331
+ * This function allows a chain state to be created from metadata, in particular this is useful for restarting an indexer and making sure it fetches blocks from the same place.
332
+ */
333
+ let makeFromDbState = (
334
+ chainConfig: Config.chain,
335
+ ~resumedChainState: Persistence.initialChainState,
336
+ ~reorgCheckpoints,
337
+ ~isInReorgThreshold,
338
+ ~isRealtime,
339
+ ~config,
340
+ ~registrations,
341
+ ~reducedPollingInterval=?,
342
+ ) => {
343
+ let chainId = chainConfig.id
344
+ let logger = Logging.createChild(~params={"chainId": chainId})
345
+
346
+ Prometheus.ProgressEventsCount.set(~processedCount=resumedChainState.numEventsProcessed, ~chainId)
347
+
348
+ let progressBlockNumber =
349
+ // Can be -1 when not set
350
+ resumedChainState.progressBlockNumber >= 0
351
+ ? resumedChainState.progressBlockNumber
352
+ : resumedChainState.startBlock - 1
353
+
354
+ makeInternal(
355
+ ~indexingAddresses=resumedChainState.indexingAddresses,
356
+ ~chainConfig,
357
+ ~startBlock=resumedChainState.startBlock,
358
+ ~endBlock=resumedChainState.endBlock,
359
+ ~config,
360
+ ~registrations,
361
+ ~reorgCheckpoints,
362
+ ~maxReorgDepth=resumedChainState.maxReorgDepth,
363
+ ~firstEventBlock=resumedChainState.firstEventBlockNumber,
364
+ ~progressBlockNumber,
365
+ ~timestampCaughtUpToHeadOrEndblock=Env.updateSyncTimeOnRestart
366
+ ? None
367
+ : resumedChainState.timestampCaughtUpToHeadOrEndblock,
368
+ ~numEventsProcessed=resumedChainState.numEventsProcessed,
369
+ ~logger,
370
+ ~isInReorgThreshold,
371
+ ~isRealtime,
372
+ ~knownHeight=resumedChainState.sourceBlockNumber,
373
+ ~reducedPollingInterval?,
374
+ )
375
+ }
376
+
377
+ // --- Read accessors. ---
378
+
379
+ let logger = (cs: t) => cs.logger
380
+ let fetchState = (cs: t) => cs.fetchState
381
+ let sourceManager = (cs: t) => cs.sourceManager
382
+ let chainConfig = (cs: t) => cs.chainConfig
383
+ let reorgDetection = (cs: t) => cs.reorgDetection
384
+ let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
385
+ let isProgressAtHead = (cs: t) => cs.isProgressAtHead
386
+ let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
387
+ let numEventsProcessed = (cs: t) => cs.numEventsProcessed
388
+ let pendingBudget = (cs: t) => cs.pendingBudget
389
+ let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
390
+
391
+ // Mark queries as in flight and reserve their estimated size against the shared
392
+ // buffer budget in one step, so the counter stays in sync with the pending
393
+ // queries it tracks.
394
+ let startFetchingQueries = (cs: t, ~queries: array<FetchState.query>) => {
395
+ cs.fetchState->FetchState.startFetchingQueries(~queries)
396
+ cs.pendingBudget =
397
+ cs.pendingBudget +. queries->Array.reduce(0., (acc, query) => acc +. query.estResponseSize)
398
+ }
399
+
400
+ // Drop every in-flight query and release their reservations together, keeping
401
+ // pendingBudget coupled to the pending queries it tracks.
402
+ let resetPendingQueries = (cs: t) => {
403
+ cs.fetchState = cs.fetchState->FetchState.resetPendingQueries
404
+ cs.pendingBudget = 0.
405
+ }
406
+
407
+ // --- Derived (pure). ---
408
+
409
+ let hasProcessedToEndblock = (cs: t) => {
410
+ let {committedProgressBlockNumber, fetchState} = cs
411
+ switch fetchState.endBlock {
412
+ | Some(endBlock) => committedProgressBlockNumber >= endBlock
413
+ | None => false
414
+ }
415
+ }
416
+
417
+ let getHighestBlockBelowThreshold = (cs: t): int => {
418
+ let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.chainConfig.maxReorgDepth
419
+ highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
420
+ }
421
+
422
+ let isActivelyIndexing = (cs: t) => cs.fetchState->FetchState.isActivelyIndexing
423
+
424
+ let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
425
+
426
+ // True once the fetch frontier has reached the head/endBlock for this chain.
427
+ let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
428
+
429
+ // --- State transitions. The chain state is mutated only through these; each
430
+ // owns a cohesive update so callers don't juggle individual fields. ---
431
+
432
+ // Apply a fetch response: register any new dynamic contracts, append the items
433
+ // to the buffer and advance the known head.
434
+ let handleQueryResult = (
435
+ cs: t,
436
+ ~query: FetchState.query,
437
+ ~newItems,
438
+ ~newItemsWithDcs,
439
+ ~latestFetchedBlock,
440
+ ~knownHeight,
441
+ ) => {
442
+ let fs = switch newItemsWithDcs {
443
+ | [] => cs.fetchState
444
+ | _ => cs.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
445
+ }
446
+
447
+ cs.fetchState =
448
+ fs
449
+ ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
450
+ ->FetchState.updateKnownHeight(~knownHeight)
451
+
452
+ // The query is no longer in flight, so release its reservation.
453
+ cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.estResponseSize)
454
+ }
455
+
456
+ // Run reorg detection against a fetch response and commit the updated guard.
457
+ // Returns the result so the caller can decide whether to roll back; on the
458
+ // rollback path registerReorgGuard returns the guard unchanged, so committing
459
+ // here is a no-op there.
460
+ let registerReorgGuard = (cs: t, ~blockHashes, ~knownHeight): ReorgDetection.reorgResult => {
461
+ let (updatedReorgDetection, reorgResult) =
462
+ cs.reorgDetection->ReorgDetection.registerReorgGuard(~blockHashes, ~knownHeight)
463
+ cs.reorgDetection = updatedReorgDetection
464
+ reorgResult
465
+ }
466
+
467
+ // Prepare for a reorg rollback: restore the events-processed counter to its
468
+ // pre-rollback value when an uncommitted rollback diff is being redone, and drop
469
+ // pending queries bound to the about-to-be-invalidated chain state.
470
+ let prepareReorg = (cs: t, ~eventsProcessedDiff) => {
471
+ switch eventsProcessedDiff {
472
+ | Some(diff) => cs.numEventsProcessed = cs.numEventsProcessed +. diff
473
+ | None => ()
474
+ }
475
+ cs->resetPendingQueries
476
+ }
477
+
478
+ let updateKnownHeight = (cs: t, ~knownHeight) =>
479
+ cs.fetchState = cs.fetchState->FetchState.updateKnownHeight(~knownHeight)
480
+
481
+ // In auto-exit mode, pin the endBlock to the earliest observed event block.
482
+ let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
483
+ switch cs.fetchState.endBlock {
484
+ | None => cs.fetchState = {...cs.fetchState, endBlock: Some(blockNumber)}
485
+ | Some(currentEndBlock) if blockNumber < currentEndBlock =>
486
+ cs.fetchState = {...cs.fetchState, endBlock: Some(blockNumber)}
487
+ | Some(_) => ()
488
+ }
489
+
490
+ // Shrink the fetch buffer by the configured blockLag on entering the reorg threshold.
491
+ let enterReorgThreshold = (cs: t) =>
492
+ cs.fetchState = cs.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
493
+
494
+ // Commit the post-batch fetch frontier for a chain that progressed in the batch,
495
+ // applying blockLag when this batch also crosses into the reorg threshold.
496
+ let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
497
+ switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
498
+ cs.fetchState.chainId,
499
+ ) {
500
+ | Some(chainAfterBatch) =>
501
+ cs.fetchState = enteringReorgThreshold
502
+ ? chainAfterBatch.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
503
+ : chainAfterBatch.fetchState
504
+ | None => ()
505
+ }
506
+
507
+ // Commit a processed batch's progress for this chain (progress block, events
508
+ // processed, head/safe-checkpoint tracking, first event block). Emits the
509
+ // per-chain progress metrics. Readiness is decided by CrossChainState once every
510
+ // chain is caught up (see markReady).
511
+ let applyBatchProgress = (cs: t, ~batch: Batch.t) => {
512
+ let chainId = cs.chainConfig.id
513
+
514
+ switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
515
+ | Some(chainAfterBatch) => {
516
+ if cs.committedProgressBlockNumber !== chainAfterBatch.progressBlockNumber {
517
+ Prometheus.ProgressBlockNumber.set(
518
+ ~blockNumber=chainAfterBatch.progressBlockNumber,
519
+ ~chainId,
520
+ )
521
+ }
522
+ if cs.numEventsProcessed !== chainAfterBatch.totalEventsProcessed {
523
+ Prometheus.ProgressEventsCount.set(
524
+ ~processedCount=chainAfterBatch.totalEventsProcessed,
525
+ ~chainId,
526
+ )
527
+ }
528
+
529
+ // Calculate and set latency metrics
530
+ switch batch->Batch.findLastEventItem(~chainId) {
531
+ | Some(eventItem) => {
532
+ let blockTimestampMs = eventItem.timestamp * 1000
533
+ Prometheus.ProgressLatency.set(
534
+ ~latencyMs=Date.now()->Float.toInt - blockTimestampMs,
535
+ ~chainId,
536
+ )
537
+ }
538
+ | None => ()
539
+ }
540
+
541
+ // Since we process per chain always in order, calculate firstEventBlock
542
+ // once, from the first item in a batch.
543
+ switch cs.fetchState.firstEventBlock {
544
+ | Some(_) => ()
545
+ | None =>
546
+ switch batch->Batch.findFirstEventBlockNumber(~chainId) {
547
+ | Some(_) as firstEventBlock => cs.fetchState = {...cs.fetchState, firstEventBlock}
548
+ | None => ()
549
+ }
550
+ }
551
+
552
+ cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
553
+ cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
554
+ cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated
555
+ switch cs.safeCheckpointTracking {
556
+ | Some(safeCheckpointTracking) =>
557
+ cs.safeCheckpointTracking = Some(
558
+ safeCheckpointTracking->SafeCheckpointTracking.updateOnNewBatch(
559
+ ~sourceBlockNumber=cs.fetchState.knownHeight,
560
+ ~chainId,
561
+ ~batchCheckpointIds=batch.checkpointIds,
562
+ ~batchCheckpointBlockNumbers=batch.checkpointBlockNumbers,
563
+ ~batchCheckpointChainIds=batch.checkpointChainIds,
564
+ ),
565
+ )
566
+ | None => ()
567
+ }
568
+ }
569
+ | None => ()
570
+ }
571
+ }
572
+
573
+ // Mark the chain caught up to head/endblock. Called by CrossChainState only once
574
+ // every chain in the indexer is caught up, so no chain flips to ready while
575
+ // another is still backfilling. Sticky: a chain stays ready once set.
576
+ let markReady = (cs: t) =>
577
+ if !(cs->isReady) {
578
+ cs.timestampCaughtUpToHeadOrEndblock = Date.make()->Some
579
+ Prometheus.ProgressReady.set(~chainId=cs.chainConfig.id)
580
+ }
581
+
582
+ // Roll a chain back to a reorg target. With a progress diff, restore fetch/
583
+ // safe-checkpoint/progress state to `newProgressBlockNumber`; the reorg chain
584
+ // additionally rewinds its reorg-detection guard. A reorg chain with no diff
585
+ // entry still rewinds guard + fetch state to the target — otherwise the stale
586
+ // block hash stays in the guard and re-triggers the same reorg.
587
+ let rollback = (
588
+ cs: t,
589
+ ~newProgressBlockNumber,
590
+ ~eventsProcessedDiff,
591
+ ~rollbackTargetBlockNumber,
592
+ ~isReorgChain,
593
+ ) => {
594
+ let chainId = cs.chainConfig.id
595
+ switch newProgressBlockNumber {
596
+ | Some(newProgressBlockNumber) =>
597
+ let newTotalEventsProcessed =
598
+ cs.numEventsProcessed -.
599
+ // Both dicts are populated together per progress-diff row, so a chain with
600
+ // a progress diff always has an events-processed diff too.
601
+ eventsProcessedDiff->Option.getOrThrow(
602
+ ~message="Missing events-processed diff for rolled-back chain",
603
+ )
604
+
605
+ if cs.committedProgressBlockNumber !== newProgressBlockNumber {
606
+ Prometheus.ProgressBlockNumber.set(~blockNumber=newProgressBlockNumber, ~chainId)
607
+ }
608
+ if cs.numEventsProcessed !== newTotalEventsProcessed {
609
+ Prometheus.ProgressEventsCount.set(~processedCount=newTotalEventsProcessed, ~chainId)
610
+ }
611
+ if isReorgChain {
612
+ cs.reorgDetection =
613
+ cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
614
+ ~blockNumber=rollbackTargetBlockNumber,
615
+ )
616
+ }
617
+ switch cs.safeCheckpointTracking {
618
+ | Some(safeCheckpointTracking) =>
619
+ cs.safeCheckpointTracking = Some(
620
+ safeCheckpointTracking->SafeCheckpointTracking.rollback(
621
+ ~targetBlockNumber=newProgressBlockNumber,
622
+ ),
623
+ )
624
+ | None => ()
625
+ }
626
+ cs.fetchState = cs.fetchState->FetchState.rollback(~targetBlockNumber=newProgressBlockNumber)
627
+ cs.committedProgressBlockNumber = newProgressBlockNumber
628
+ cs.numEventsProcessed = newTotalEventsProcessed
629
+ | None =>
630
+ if isReorgChain {
631
+ cs.reorgDetection =
632
+ cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
633
+ ~blockNumber=rollbackTargetBlockNumber,
634
+ )
635
+ cs.fetchState =
636
+ cs.fetchState->FetchState.rollback(~targetBlockNumber=rollbackTargetBlockNumber)
637
+ cs.committedProgressBlockNumber = Pervasives.min(
638
+ cs.committedProgressBlockNumber,
639
+ rollbackTargetBlockNumber,
640
+ )
641
+ }
642
+ }
643
+ }