envio 3.2.0 → 3.3.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -7
- package/src/Batch.res +12 -15
- package/src/Batch.res.mjs +4 -5
- package/src/BatchProcessing.res +199 -0
- package/src/BatchProcessing.res.mjs +125 -0
- package/src/ChainFetching.res +375 -0
- package/src/ChainFetching.res.mjs +206 -0
- package/src/ChainMetadata.res +27 -0
- package/src/ChainMetadata.res.mjs +27 -0
- package/src/ChainState.res +618 -0
- package/src/ChainState.res.mjs +461 -0
- package/src/ChainState.resi +85 -0
- package/src/Config.res +13 -4
- package/src/Config.res.mjs +8 -3
- package/src/ContractRegisterContext.res +106 -0
- package/src/ContractRegisterContext.res.mjs +76 -0
- package/src/Core.res +3 -0
- package/src/CrossChainState.res +269 -0
- package/src/CrossChainState.res.mjs +197 -0
- package/src/CrossChainState.resi +47 -0
- package/src/Ecosystem.res +58 -0
- package/src/Ecosystem.res.mjs +43 -1
- package/src/Env.res +7 -2
- package/src/Env.res.mjs +5 -2
- package/src/EventConfigBuilder.res +35 -0
- package/src/EventConfigBuilder.res.mjs +21 -0
- package/src/EventProcessing.res +60 -46
- package/src/EventProcessing.res.mjs +33 -32
- package/src/EventUtils.res +0 -4
- package/src/EventUtils.res.mjs +0 -4
- package/src/ExitOnCaughtUp.res +10 -0
- package/src/ExitOnCaughtUp.res.mjs +22 -0
- package/src/FetchState.res +162 -113
- package/src/FetchState.res.mjs +181 -119
- package/src/HandlerLoader.res +6 -1
- package/src/HandlerLoader.res.mjs +13 -36
- package/src/InMemoryStore.res +36 -451
- package/src/InMemoryStore.res.mjs +20 -369
- package/src/IndexerLoop.res +44 -0
- package/src/IndexerLoop.res.mjs +46 -0
- package/src/IndexerState.res +544 -0
- package/src/IndexerState.res.mjs +547 -0
- package/src/IndexerState.resi +132 -0
- package/src/Internal.res +44 -7
- package/src/LoadLayer.res +20 -18
- package/src/LoadLayer.res.mjs +14 -12
- package/src/LoadLayer.resi +6 -3
- package/src/LogSelection.res +86 -17
- package/src/LogSelection.res.mjs +67 -14
- package/src/Logging.res +11 -44
- package/src/Logging.res.mjs +10 -42
- package/src/Main.res +54 -79
- package/src/Main.res.mjs +44 -56
- package/src/PgStorage.res +8 -71
- package/src/PgStorage.res.mjs +3 -47
- package/src/Prometheus.res +7 -8
- package/src/Prometheus.res.mjs +7 -4
- package/src/PruneStaleHistory.res +45 -0
- package/src/PruneStaleHistory.res.mjs +46 -0
- package/src/RawEvent.res +73 -0
- package/src/RawEvent.res.mjs +51 -0
- package/src/Rollback.res +204 -0
- package/src/Rollback.res.mjs +118 -0
- package/src/SimulateItems.res +12 -10
- package/src/SimulateItems.res.mjs +1 -1
- package/src/TestIndexer.res +34 -27
- package/src/TestIndexer.res.mjs +4 -1
- package/src/UserContext.res +26 -114
- package/src/UserContext.res.mjs +15 -78
- package/src/Writing.res +242 -0
- package/src/Writing.res.mjs +215 -0
- package/src/db/InternalTable.res +14 -27
- package/src/sources/Evm.res +40 -1
- package/src/sources/Evm.res.mjs +94 -84
- package/src/sources/Fuel.res +40 -1
- package/src/sources/Fuel.res.mjs +36 -26
- package/src/sources/HyperFuel.res +41 -120
- package/src/sources/HyperFuel.res.mjs +42 -80
- package/src/sources/HyperFuel.resi +1 -24
- package/src/sources/HyperFuelClient.res +16 -297
- package/src/sources/HyperFuelClient.res.mjs +5 -4
- package/src/sources/HyperFuelSource.res +33 -8
- package/src/sources/HyperFuelSource.res.mjs +56 -10
- package/src/sources/HyperSyncSource.res +7 -5
- package/src/sources/HyperSyncSource.res.mjs +3 -3
- package/src/sources/RpcSource.res +2 -2
- package/src/sources/RpcSource.res.mjs +1 -1
- package/src/sources/SourceManager.res +8 -18
- package/src/sources/SourceManager.res.mjs +8 -5
- package/src/sources/SourceManager.resi +4 -1
- package/src/sources/Svm.res +18 -1
- package/src/sources/Svm.res.mjs +30 -22
- package/src/sources/SvmHyperSyncSource.res +1 -1
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
- package/src/tui/Tui.res +38 -36
- package/src/tui/Tui.res.mjs +51 -51
- package/src/ChainFetcher.res +0 -519
- package/src/ChainFetcher.res.mjs +0 -314
- package/src/ChainManager.res +0 -358
- package/src/ChainManager.res.mjs +0 -291
- package/src/Ctx.res +0 -6
- package/src/Ctx.res.mjs +0 -2
- package/src/GlobalState.res +0 -1056
- package/src/GlobalState.res.mjs +0 -1086
- package/src/GlobalStateManager.res +0 -57
- package/src/GlobalStateManager.res.mjs +0 -68
- package/src/GlobalStateManager.resi +0 -7
- package/src/Ports.res +0 -5
- package/src/Ports.res.mjs +0 -9
- package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
- package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
- package/src/sources/EnvioApiClient.res +0 -15
- package/src/sources/EnvioApiClient.res.mjs +0 -24
|
@@ -0,0 +1,618 @@
|
|
|
1
|
+
// Per-chain runtime state. `t` is mutated in place through the setters below;
|
|
2
|
+
// the type is opaque in the interface so callers can read fields but can only
|
|
3
|
+
// change them through the sanctioned mutators.
|
|
4
|
+
|
|
5
|
+
type t = {
|
|
6
|
+
logger: Pino.t,
|
|
7
|
+
mutable fetchState: FetchState.t,
|
|
8
|
+
sourceManager: SourceManager.t,
|
|
9
|
+
chainConfig: Config.chain,
|
|
10
|
+
mutable isProgressAtHead: bool,
|
|
11
|
+
mutable timestampCaughtUpToHeadOrEndblock: option<Date.t>,
|
|
12
|
+
mutable committedProgressBlockNumber: int,
|
|
13
|
+
mutable numEventsProcessed: float,
|
|
14
|
+
mutable reorgDetection: ReorgDetection.t,
|
|
15
|
+
mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
|
|
19
|
+
let addresses = []
|
|
20
|
+
chainConfig.contracts->Array.forEach(contract => {
|
|
21
|
+
contract.addresses->Array.forEach(address => {
|
|
22
|
+
addresses->Array.push({
|
|
23
|
+
Internal.address,
|
|
24
|
+
contractName: contract.name,
|
|
25
|
+
registrationBlock: -1,
|
|
26
|
+
})
|
|
27
|
+
})
|
|
28
|
+
})
|
|
29
|
+
addresses
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
let make = (
|
|
33
|
+
~chainConfig: Config.chain,
|
|
34
|
+
~fetchState: FetchState.t,
|
|
35
|
+
~sourceManager: SourceManager.t,
|
|
36
|
+
~reorgDetection: ReorgDetection.t,
|
|
37
|
+
~committedProgressBlockNumber: int,
|
|
38
|
+
~safeCheckpointTracking=None,
|
|
39
|
+
~numEventsProcessed=0.,
|
|
40
|
+
~timestampCaughtUpToHeadOrEndblock=None,
|
|
41
|
+
~isProgressAtHead=false,
|
|
42
|
+
~logger: Pino.t,
|
|
43
|
+
): t => {
|
|
44
|
+
logger,
|
|
45
|
+
fetchState,
|
|
46
|
+
sourceManager,
|
|
47
|
+
chainConfig,
|
|
48
|
+
isProgressAtHead,
|
|
49
|
+
timestampCaughtUpToHeadOrEndblock,
|
|
50
|
+
committedProgressBlockNumber,
|
|
51
|
+
numEventsProcessed,
|
|
52
|
+
reorgDetection,
|
|
53
|
+
safeCheckpointTracking,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
let makeInternal = (
|
|
57
|
+
~chainConfig: Config.chain,
|
|
58
|
+
~indexingAddresses: array<Internal.indexingAddress>,
|
|
59
|
+
~startBlock,
|
|
60
|
+
~endBlock,
|
|
61
|
+
~firstEventBlock=None,
|
|
62
|
+
~progressBlockNumber,
|
|
63
|
+
~config: Config.t,
|
|
64
|
+
~registrations: HandlerRegister.registrations,
|
|
65
|
+
~logger,
|
|
66
|
+
~timestampCaughtUpToHeadOrEndblock,
|
|
67
|
+
~numEventsProcessed,
|
|
68
|
+
~isInReorgThreshold,
|
|
69
|
+
~isRealtime,
|
|
70
|
+
~reorgCheckpoints: array<Internal.reorgCheckpoint>,
|
|
71
|
+
~maxReorgDepth,
|
|
72
|
+
~knownHeight=0,
|
|
73
|
+
~reducedPollingInterval=?,
|
|
74
|
+
): t => {
|
|
75
|
+
// We don't need the router itself, but only validation logic,
|
|
76
|
+
// since now event router is created for selection of events
|
|
77
|
+
// and validation doesn't work correctly in routers.
|
|
78
|
+
// Ideally to split it into two different parts.
|
|
79
|
+
let eventRouter = EventRouter.empty()
|
|
80
|
+
|
|
81
|
+
// Aggregate events we want to fetch
|
|
82
|
+
let eventConfigs: array<Internal.eventConfig> = []
|
|
83
|
+
|
|
84
|
+
let notRegisteredEvents = []
|
|
85
|
+
|
|
86
|
+
chainConfig.contracts->Array.forEach(contract => {
|
|
87
|
+
let contractName = contract.name
|
|
88
|
+
|
|
89
|
+
contract.events->Array.forEach(eventConfig => {
|
|
90
|
+
let {isWildcard} = eventConfig
|
|
91
|
+
let hasContractRegister = eventConfig.contractRegister->Option.isSome
|
|
92
|
+
|
|
93
|
+
// Should validate the events
|
|
94
|
+
eventRouter->EventRouter.addOrThrow(
|
|
95
|
+
eventConfig.id,
|
|
96
|
+
(),
|
|
97
|
+
~contractName,
|
|
98
|
+
~chain=ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id),
|
|
99
|
+
~eventName=eventConfig.name,
|
|
100
|
+
~isWildcard,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
// Filter out non-preRegistration events on preRegistration phase
|
|
104
|
+
// so we don't care about it in fetch state and workers anymore
|
|
105
|
+
let shouldBeIncluded = if config.enableRawEvents {
|
|
106
|
+
true
|
|
107
|
+
} else {
|
|
108
|
+
let isRegistered = hasContractRegister || eventConfig.handler->Option.isSome
|
|
109
|
+
if !isRegistered {
|
|
110
|
+
notRegisteredEvents->Array.push(eventConfig)
|
|
111
|
+
}
|
|
112
|
+
isRegistered
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Check if event has Static([]) filters (from a dynamic where
|
|
116
|
+
// callback returning `false` / SkipAll for this chain).
|
|
117
|
+
// If so, skip it entirely - it should never be fetched
|
|
118
|
+
let shouldSkip = try {
|
|
119
|
+
let getEventFiltersOrThrow = (
|
|
120
|
+
eventConfig->(Utils.magic: Internal.eventConfig => Internal.evmEventConfig)
|
|
121
|
+
).getEventFiltersOrThrow
|
|
122
|
+
|
|
123
|
+
// Check for non-evm chains
|
|
124
|
+
if (
|
|
125
|
+
getEventFiltersOrThrow->(Utils.magic: (ChainMap.Chain.t => Internal.eventFilters) => bool)
|
|
126
|
+
) {
|
|
127
|
+
switch getEventFiltersOrThrow(ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)) {
|
|
128
|
+
| Static([]) => true
|
|
129
|
+
| _ => false
|
|
130
|
+
}
|
|
131
|
+
} else {
|
|
132
|
+
false
|
|
133
|
+
}
|
|
134
|
+
} catch {
|
|
135
|
+
// Can throw when filter is invalid
|
|
136
|
+
// Don't skip an event in this case. Let it throw in a better place - source code
|
|
137
|
+
| _ => false
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if shouldBeIncluded && !shouldSkip {
|
|
141
|
+
eventConfigs->Array.push(eventConfig)
|
|
142
|
+
}
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
switch contract.startBlock {
|
|
146
|
+
| Some(startBlock) if startBlock < chainConfig.startBlock =>
|
|
147
|
+
JsError.throwWithMessage(
|
|
148
|
+
`The start block for contract "${contractName}" is less than the chain start block. This is not supported yet.`,
|
|
149
|
+
)
|
|
150
|
+
| _ => ()
|
|
151
|
+
}
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
if notRegisteredEvents->Utils.Array.notEmpty {
|
|
155
|
+
logger->Logging.childInfo(
|
|
156
|
+
`The event${if notRegisteredEvents->Array.length > 1 {
|
|
157
|
+
"s"
|
|
158
|
+
} else {
|
|
159
|
+
""
|
|
160
|
+
}} ${notRegisteredEvents
|
|
161
|
+
->Array.map(eventConfig => `${eventConfig.contractName}.${eventConfig.name}`)
|
|
162
|
+
->Array.joinUnsafe(", ")} don't have an event handler and skipped for indexing.`,
|
|
163
|
+
)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
let onBlockConfigs =
|
|
167
|
+
registrations.onBlockByChainId->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->Int.toString)
|
|
168
|
+
switch onBlockConfigs {
|
|
169
|
+
| Some(onBlockConfigs) =>
|
|
170
|
+
// TODO: Move it to the HandlerRegister module
|
|
171
|
+
// so the error is thrown with better stack trace
|
|
172
|
+
onBlockConfigs->Array.forEach(onBlockConfig => {
|
|
173
|
+
if onBlockConfig.startBlock->Option.getOr(startBlock) < startBlock {
|
|
174
|
+
JsError.throwWithMessage(
|
|
175
|
+
`The start block for onBlock handler "${onBlockConfig.name}" is less than the chain start block (${startBlock->Int.toString}). This is not supported yet.`,
|
|
176
|
+
)
|
|
177
|
+
}
|
|
178
|
+
switch endBlock {
|
|
179
|
+
| Some(chainEndBlock) =>
|
|
180
|
+
if onBlockConfig.endBlock->Option.getOr(chainEndBlock) > chainEndBlock {
|
|
181
|
+
JsError.throwWithMessage(
|
|
182
|
+
`The end block for onBlock handler "${onBlockConfig.name}" is greater than the chain end block (${chainEndBlock->Int.toString}). This is not supported yet.`,
|
|
183
|
+
)
|
|
184
|
+
}
|
|
185
|
+
| None => ()
|
|
186
|
+
}
|
|
187
|
+
})
|
|
188
|
+
| None => ()
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
let fetchState = FetchState.make(
|
|
192
|
+
~maxAddrInPartition=config.maxAddrInPartition,
|
|
193
|
+
~addresses=indexingAddresses,
|
|
194
|
+
~progressBlockNumber,
|
|
195
|
+
~startBlock,
|
|
196
|
+
~endBlock,
|
|
197
|
+
~eventConfigs,
|
|
198
|
+
~maxOnBlockBufferSize=2 * config.batchSize,
|
|
199
|
+
~knownHeight,
|
|
200
|
+
~chainId=chainConfig.id,
|
|
201
|
+
// FIXME: Shouldn't set with full history
|
|
202
|
+
~blockLag=Pervasives.max(
|
|
203
|
+
!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth,
|
|
204
|
+
chainConfig.blockLag,
|
|
205
|
+
),
|
|
206
|
+
~onBlockConfigs?,
|
|
207
|
+
~firstEventBlock,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
let chainReorgCheckpoints = reorgCheckpoints->Array.filterMap(reorgCheckpoint => {
|
|
211
|
+
if reorgCheckpoint.chainId === chainConfig.id {
|
|
212
|
+
Some(reorgCheckpoint)
|
|
213
|
+
} else {
|
|
214
|
+
None
|
|
215
|
+
}
|
|
216
|
+
})
|
|
217
|
+
|
|
218
|
+
// Create sources lazily here - this is where API token validation happens
|
|
219
|
+
let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
|
|
220
|
+
let lowercaseAddresses = config.lowercaseAddresses
|
|
221
|
+
let sources = switch chainConfig.sourceConfig {
|
|
222
|
+
| Config.EvmSourceConfig({hypersync, rpcs}) =>
|
|
223
|
+
// Build Internal.evmContractConfig from contracts for EvmChain.makeSources
|
|
224
|
+
let evmContracts: array<Internal.evmContractConfig> = chainConfig.contracts->Array.map((
|
|
225
|
+
contract
|
|
226
|
+
): Internal.evmContractConfig => {
|
|
227
|
+
name: contract.name,
|
|
228
|
+
abi: contract.abi,
|
|
229
|
+
events: contract.events->(
|
|
230
|
+
Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
|
|
231
|
+
),
|
|
232
|
+
})
|
|
233
|
+
let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
|
|
234
|
+
let syncConfig = rpc.syncConfig
|
|
235
|
+
let ws = rpc.ws
|
|
236
|
+
{
|
|
237
|
+
url: rpc.url,
|
|
238
|
+
sourceFor: rpc.sourceFor,
|
|
239
|
+
?syncConfig,
|
|
240
|
+
?ws,
|
|
241
|
+
}
|
|
242
|
+
})
|
|
243
|
+
EvmChain.makeSources(
|
|
244
|
+
~chain,
|
|
245
|
+
~contracts=evmContracts,
|
|
246
|
+
~hyperSync=hypersync,
|
|
247
|
+
~rpcs=evmRpcs,
|
|
248
|
+
~lowercaseAddresses,
|
|
249
|
+
)
|
|
250
|
+
| Config.FuelSourceConfig({hypersync}) => [
|
|
251
|
+
HyperFuelSource.make({chain, endpointUrl: hypersync, apiToken: Env.envioApiToken}),
|
|
252
|
+
]
|
|
253
|
+
| Config.SvmSourceConfig({hypersync, rpc}) =>
|
|
254
|
+
switch (hypersync, rpc) {
|
|
255
|
+
| (None, None) =>
|
|
256
|
+
JsError.throwWithMessage(
|
|
257
|
+
`Chain ${chain->ChainMap.Chain.toChainId->Int.toString} has no SVM data source`,
|
|
258
|
+
)
|
|
259
|
+
| (None, Some(rpc)) => [Svm.makeRPCSource(~chain, ~rpc)]
|
|
260
|
+
| (Some(hypersyncUrl), _) =>
|
|
261
|
+
// HyperSync drives instruction sync. A configured RPC is ignored for now
|
|
262
|
+
// (RPC fallback isn't wired up yet).
|
|
263
|
+
let svmEventConfigs =
|
|
264
|
+
chainConfig.contracts
|
|
265
|
+
->Array.flatMap(contract => contract.events)
|
|
266
|
+
->(Utils.magic: array<Internal.eventConfig> => array<Internal.svmInstructionEventConfig>)
|
|
267
|
+
let apiToken = Env.envioApiToken
|
|
268
|
+
[
|
|
269
|
+
SvmHyperSyncSource.make({
|
|
270
|
+
chain,
|
|
271
|
+
endpointUrl: hypersyncUrl,
|
|
272
|
+
apiToken,
|
|
273
|
+
eventConfigs: svmEventConfigs,
|
|
274
|
+
clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
|
|
275
|
+
}),
|
|
276
|
+
]
|
|
277
|
+
}
|
|
278
|
+
// For tests: use ready-to-use sources directly
|
|
279
|
+
| Config.CustomSources(sources) => sources
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
make(
|
|
283
|
+
~chainConfig,
|
|
284
|
+
~fetchState,
|
|
285
|
+
~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
|
|
286
|
+
~reorgDetection=ReorgDetection.make(
|
|
287
|
+
~chainReorgCheckpoints,
|
|
288
|
+
~maxReorgDepth,
|
|
289
|
+
~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
|
|
290
|
+
),
|
|
291
|
+
~safeCheckpointTracking=SafeCheckpointTracking.make(
|
|
292
|
+
~maxReorgDepth,
|
|
293
|
+
~shouldRollbackOnReorg=config.shouldRollbackOnReorg,
|
|
294
|
+
~chainReorgCheckpoints,
|
|
295
|
+
),
|
|
296
|
+
~committedProgressBlockNumber=progressBlockNumber,
|
|
297
|
+
~timestampCaughtUpToHeadOrEndblock,
|
|
298
|
+
~numEventsProcessed,
|
|
299
|
+
~logger,
|
|
300
|
+
)
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~knownHeight) => {
|
|
304
|
+
let logger = Logging.createChild(~params={"chainId": chainConfig.id})
|
|
305
|
+
|
|
306
|
+
makeInternal(
|
|
307
|
+
~chainConfig,
|
|
308
|
+
~config,
|
|
309
|
+
~registrations,
|
|
310
|
+
~startBlock=chainConfig.startBlock,
|
|
311
|
+
~endBlock=chainConfig.endBlock,
|
|
312
|
+
~reorgCheckpoints=[],
|
|
313
|
+
~maxReorgDepth=chainConfig.maxReorgDepth,
|
|
314
|
+
~progressBlockNumber=-1,
|
|
315
|
+
~timestampCaughtUpToHeadOrEndblock=None,
|
|
316
|
+
~numEventsProcessed=0.,
|
|
317
|
+
~logger,
|
|
318
|
+
~indexingAddresses=configAddresses(chainConfig),
|
|
319
|
+
~isInReorgThreshold=false,
|
|
320
|
+
~isRealtime=false,
|
|
321
|
+
~knownHeight,
|
|
322
|
+
)
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* This function allows a chain state to be created from metadata, in particular this is useful for restarting an indexer and making sure it fetches blocks from the same place.
|
|
327
|
+
*/
|
|
328
|
+
let makeFromDbState = (
|
|
329
|
+
chainConfig: Config.chain,
|
|
330
|
+
~resumedChainState: Persistence.initialChainState,
|
|
331
|
+
~reorgCheckpoints,
|
|
332
|
+
~isInReorgThreshold,
|
|
333
|
+
~isRealtime,
|
|
334
|
+
~config,
|
|
335
|
+
~registrations,
|
|
336
|
+
~reducedPollingInterval=?,
|
|
337
|
+
) => {
|
|
338
|
+
let chainId = chainConfig.id
|
|
339
|
+
let logger = Logging.createChild(~params={"chainId": chainId})
|
|
340
|
+
|
|
341
|
+
Prometheus.ProgressEventsCount.set(~processedCount=resumedChainState.numEventsProcessed, ~chainId)
|
|
342
|
+
|
|
343
|
+
let progressBlockNumber =
|
|
344
|
+
// Can be -1 when not set
|
|
345
|
+
resumedChainState.progressBlockNumber >= 0
|
|
346
|
+
? resumedChainState.progressBlockNumber
|
|
347
|
+
: resumedChainState.startBlock - 1
|
|
348
|
+
|
|
349
|
+
makeInternal(
|
|
350
|
+
~indexingAddresses=resumedChainState.indexingAddresses,
|
|
351
|
+
~chainConfig,
|
|
352
|
+
~startBlock=resumedChainState.startBlock,
|
|
353
|
+
~endBlock=resumedChainState.endBlock,
|
|
354
|
+
~config,
|
|
355
|
+
~registrations,
|
|
356
|
+
~reorgCheckpoints,
|
|
357
|
+
~maxReorgDepth=resumedChainState.maxReorgDepth,
|
|
358
|
+
~firstEventBlock=resumedChainState.firstEventBlockNumber,
|
|
359
|
+
~progressBlockNumber,
|
|
360
|
+
~timestampCaughtUpToHeadOrEndblock=Env.updateSyncTimeOnRestart
|
|
361
|
+
? None
|
|
362
|
+
: resumedChainState.timestampCaughtUpToHeadOrEndblock,
|
|
363
|
+
~numEventsProcessed=resumedChainState.numEventsProcessed,
|
|
364
|
+
~logger,
|
|
365
|
+
~isInReorgThreshold,
|
|
366
|
+
~isRealtime,
|
|
367
|
+
~knownHeight=resumedChainState.sourceBlockNumber,
|
|
368
|
+
~reducedPollingInterval?,
|
|
369
|
+
)
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// --- Read accessors. ---
|
|
373
|
+
|
|
374
|
+
let logger = (cs: t) => cs.logger
|
|
375
|
+
let fetchState = (cs: t) => cs.fetchState
|
|
376
|
+
let sourceManager = (cs: t) => cs.sourceManager
|
|
377
|
+
let chainConfig = (cs: t) => cs.chainConfig
|
|
378
|
+
let reorgDetection = (cs: t) => cs.reorgDetection
|
|
379
|
+
let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
|
|
380
|
+
let isProgressAtHead = (cs: t) => cs.isProgressAtHead
|
|
381
|
+
let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
|
|
382
|
+
let numEventsProcessed = (cs: t) => cs.numEventsProcessed
|
|
383
|
+
let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock
|
|
384
|
+
|
|
385
|
+
// --- Derived (pure). ---
|
|
386
|
+
|
|
387
|
+
let hasProcessedToEndblock = (cs: t) => {
|
|
388
|
+
let {committedProgressBlockNumber, fetchState} = cs
|
|
389
|
+
switch fetchState.endBlock {
|
|
390
|
+
| Some(endBlock) => committedProgressBlockNumber >= endBlock
|
|
391
|
+
| None => false
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
let getHighestBlockBelowThreshold = (cs: t): int => {
|
|
396
|
+
let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.chainConfig.maxReorgDepth
|
|
397
|
+
highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
let isActivelyIndexing = (cs: t) => cs.fetchState->FetchState.isActivelyIndexing
|
|
401
|
+
|
|
402
|
+
let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
|
|
403
|
+
|
|
404
|
+
// True once the fetch frontier has reached the head/endBlock for this chain.
|
|
405
|
+
let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
|
|
406
|
+
|
|
407
|
+
// --- State transitions. The chain state is mutated only through these; each
|
|
408
|
+
// owns a cohesive update so callers don't juggle individual fields. ---
|
|
409
|
+
|
|
410
|
+
// Apply a fetch response: register any new dynamic contracts, append the items
|
|
411
|
+
// to the buffer and advance the known head.
|
|
412
|
+
let handleQueryResult = (
|
|
413
|
+
cs: t,
|
|
414
|
+
~query: FetchState.query,
|
|
415
|
+
~newItems,
|
|
416
|
+
~newItemsWithDcs,
|
|
417
|
+
~latestFetchedBlock,
|
|
418
|
+
~knownHeight,
|
|
419
|
+
) => {
|
|
420
|
+
let fs = switch newItemsWithDcs {
|
|
421
|
+
| [] => cs.fetchState
|
|
422
|
+
| _ => cs.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
cs.fetchState =
|
|
426
|
+
fs
|
|
427
|
+
->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
|
|
428
|
+
->FetchState.updateKnownHeight(~knownHeight)
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// Run reorg detection against a fetch response and commit the updated guard.
|
|
432
|
+
// Returns the result so the caller can decide whether to roll back; on the
|
|
433
|
+
// rollback path registerReorgGuard returns the guard unchanged, so committing
|
|
434
|
+
// here is a no-op there.
|
|
435
|
+
let registerReorgGuard = (cs: t, ~blockHashes, ~knownHeight): ReorgDetection.reorgResult => {
|
|
436
|
+
let (updatedReorgDetection, reorgResult) =
|
|
437
|
+
cs.reorgDetection->ReorgDetection.registerReorgGuard(~blockHashes, ~knownHeight)
|
|
438
|
+
cs.reorgDetection = updatedReorgDetection
|
|
439
|
+
reorgResult
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
// Prepare for a reorg rollback: restore the events-processed counter to its
|
|
443
|
+
// pre-rollback value when an uncommitted rollback diff is being redone, and drop
|
|
444
|
+
// pending queries bound to the about-to-be-invalidated chain state.
|
|
445
|
+
let prepareReorg = (cs: t, ~eventsProcessedDiff) => {
|
|
446
|
+
switch eventsProcessedDiff {
|
|
447
|
+
| Some(diff) => cs.numEventsProcessed = cs.numEventsProcessed +. diff
|
|
448
|
+
| None => ()
|
|
449
|
+
}
|
|
450
|
+
cs.fetchState = cs.fetchState->FetchState.resetPendingQueries
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
let updateKnownHeight = (cs: t, ~knownHeight) =>
|
|
454
|
+
cs.fetchState = cs.fetchState->FetchState.updateKnownHeight(~knownHeight)
|
|
455
|
+
|
|
456
|
+
// In auto-exit mode, pin the endBlock to the earliest observed event block.
|
|
457
|
+
let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
|
|
458
|
+
switch cs.fetchState.endBlock {
|
|
459
|
+
| None => cs.fetchState = {...cs.fetchState, endBlock: Some(blockNumber)}
|
|
460
|
+
| Some(currentEndBlock) if blockNumber < currentEndBlock =>
|
|
461
|
+
cs.fetchState = {...cs.fetchState, endBlock: Some(blockNumber)}
|
|
462
|
+
| Some(_) => ()
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
// Shrink the fetch buffer by the configured blockLag on entering the reorg threshold.
|
|
466
|
+
let enterReorgThreshold = (cs: t) =>
|
|
467
|
+
cs.fetchState = cs.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
|
|
468
|
+
|
|
469
|
+
// Commit the post-batch fetch frontier for a chain that progressed in the batch,
|
|
470
|
+
// applying blockLag when this batch also crosses into the reorg threshold.
|
|
471
|
+
let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
|
|
472
|
+
switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
473
|
+
cs.fetchState.chainId,
|
|
474
|
+
) {
|
|
475
|
+
| Some(chainAfterBatch) =>
|
|
476
|
+
cs.fetchState = enteringReorgThreshold
|
|
477
|
+
? chainAfterBatch.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
|
|
478
|
+
: chainAfterBatch.fetchState
|
|
479
|
+
| None => ()
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// Commit a processed batch's progress for this chain (progress block, events
|
|
483
|
+
// processed, head/safe-checkpoint tracking, first event block). Emits the
|
|
484
|
+
// per-chain progress metrics. Readiness is decided by CrossChainState once every
|
|
485
|
+
// chain is caught up (see markReady).
|
|
486
|
+
let applyBatchProgress = (cs: t, ~batch: Batch.t) => {
|
|
487
|
+
let chainId = cs.chainConfig.id
|
|
488
|
+
|
|
489
|
+
switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
|
|
490
|
+
| Some(chainAfterBatch) => {
|
|
491
|
+
if cs.committedProgressBlockNumber !== chainAfterBatch.progressBlockNumber {
|
|
492
|
+
Prometheus.ProgressBlockNumber.set(
|
|
493
|
+
~blockNumber=chainAfterBatch.progressBlockNumber,
|
|
494
|
+
~chainId,
|
|
495
|
+
)
|
|
496
|
+
}
|
|
497
|
+
if cs.numEventsProcessed !== chainAfterBatch.totalEventsProcessed {
|
|
498
|
+
Prometheus.ProgressEventsCount.set(
|
|
499
|
+
~processedCount=chainAfterBatch.totalEventsProcessed,
|
|
500
|
+
~chainId,
|
|
501
|
+
)
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// Calculate and set latency metrics
|
|
505
|
+
switch batch->Batch.findLastEventItem(~chainId) {
|
|
506
|
+
| Some(eventItem) => {
|
|
507
|
+
let blockTimestampMs = eventItem.timestamp * 1000
|
|
508
|
+
Prometheus.ProgressLatency.set(
|
|
509
|
+
~latencyMs=Date.now()->Float.toInt - blockTimestampMs,
|
|
510
|
+
~chainId,
|
|
511
|
+
)
|
|
512
|
+
}
|
|
513
|
+
| None => ()
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// Since we process per chain always in order, calculate firstEventBlock
|
|
517
|
+
// once, from the first item in a batch.
|
|
518
|
+
switch cs.fetchState.firstEventBlock {
|
|
519
|
+
| Some(_) => ()
|
|
520
|
+
| None =>
|
|
521
|
+
switch batch->Batch.findFirstEventBlockNumber(~chainId) {
|
|
522
|
+
| Some(_) as firstEventBlock => cs.fetchState = {...cs.fetchState, firstEventBlock}
|
|
523
|
+
| None => ()
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber
|
|
528
|
+
cs.numEventsProcessed = chainAfterBatch.totalEventsProcessed
|
|
529
|
+
cs.isProgressAtHead = cs.isProgressAtHead || chainAfterBatch.isProgressAtHeadWhenBatchCreated
|
|
530
|
+
switch cs.safeCheckpointTracking {
|
|
531
|
+
| Some(safeCheckpointTracking) =>
|
|
532
|
+
cs.safeCheckpointTracking = Some(
|
|
533
|
+
safeCheckpointTracking->SafeCheckpointTracking.updateOnNewBatch(
|
|
534
|
+
~sourceBlockNumber=cs.fetchState.knownHeight,
|
|
535
|
+
~chainId,
|
|
536
|
+
~batchCheckpointIds=batch.checkpointIds,
|
|
537
|
+
~batchCheckpointBlockNumbers=batch.checkpointBlockNumbers,
|
|
538
|
+
~batchCheckpointChainIds=batch.checkpointChainIds,
|
|
539
|
+
),
|
|
540
|
+
)
|
|
541
|
+
| None => ()
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
| None => ()
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// Mark the chain caught up to head/endblock. Called by CrossChainState only once
|
|
549
|
+
// every chain in the indexer is caught up, so no chain flips to ready while
|
|
550
|
+
// another is still backfilling. Sticky: a chain stays ready once set.
|
|
551
|
+
let markReady = (cs: t) =>
|
|
552
|
+
if !(cs->isReady) {
|
|
553
|
+
cs.timestampCaughtUpToHeadOrEndblock = Date.make()->Some
|
|
554
|
+
Prometheus.ProgressReady.set(~chainId=cs.chainConfig.id)
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
// Roll a chain back to a reorg target. With a progress diff, restore fetch/
|
|
558
|
+
// safe-checkpoint/progress state to `newProgressBlockNumber`; the reorg chain
|
|
559
|
+
// additionally rewinds its reorg-detection guard. A reorg chain with no diff
|
|
560
|
+
// entry still rewinds guard + fetch state to the target — otherwise the stale
|
|
561
|
+
// block hash stays in the guard and re-triggers the same reorg.
|
|
562
|
+
let rollback = (
|
|
563
|
+
cs: t,
|
|
564
|
+
~newProgressBlockNumber,
|
|
565
|
+
~eventsProcessedDiff,
|
|
566
|
+
~rollbackTargetBlockNumber,
|
|
567
|
+
~isReorgChain,
|
|
568
|
+
) => {
|
|
569
|
+
let chainId = cs.chainConfig.id
|
|
570
|
+
switch newProgressBlockNumber {
|
|
571
|
+
| Some(newProgressBlockNumber) =>
|
|
572
|
+
let newTotalEventsProcessed =
|
|
573
|
+
cs.numEventsProcessed -.
|
|
574
|
+
// Both dicts are populated together per progress-diff row, so a chain with
|
|
575
|
+
// a progress diff always has an events-processed diff too.
|
|
576
|
+
eventsProcessedDiff->Option.getOrThrow(
|
|
577
|
+
~message="Missing events-processed diff for rolled-back chain",
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
if cs.committedProgressBlockNumber !== newProgressBlockNumber {
|
|
581
|
+
Prometheus.ProgressBlockNumber.set(~blockNumber=newProgressBlockNumber, ~chainId)
|
|
582
|
+
}
|
|
583
|
+
if cs.numEventsProcessed !== newTotalEventsProcessed {
|
|
584
|
+
Prometheus.ProgressEventsCount.set(~processedCount=newTotalEventsProcessed, ~chainId)
|
|
585
|
+
}
|
|
586
|
+
if isReorgChain {
|
|
587
|
+
cs.reorgDetection =
|
|
588
|
+
cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
|
|
589
|
+
~blockNumber=rollbackTargetBlockNumber,
|
|
590
|
+
)
|
|
591
|
+
}
|
|
592
|
+
switch cs.safeCheckpointTracking {
|
|
593
|
+
| Some(safeCheckpointTracking) =>
|
|
594
|
+
cs.safeCheckpointTracking = Some(
|
|
595
|
+
safeCheckpointTracking->SafeCheckpointTracking.rollback(
|
|
596
|
+
~targetBlockNumber=newProgressBlockNumber,
|
|
597
|
+
),
|
|
598
|
+
)
|
|
599
|
+
| None => ()
|
|
600
|
+
}
|
|
601
|
+
cs.fetchState = cs.fetchState->FetchState.rollback(~targetBlockNumber=newProgressBlockNumber)
|
|
602
|
+
cs.committedProgressBlockNumber = newProgressBlockNumber
|
|
603
|
+
cs.numEventsProcessed = newTotalEventsProcessed
|
|
604
|
+
| None =>
|
|
605
|
+
if isReorgChain {
|
|
606
|
+
cs.reorgDetection =
|
|
607
|
+
cs.reorgDetection->ReorgDetection.rollbackToValidBlockNumber(
|
|
608
|
+
~blockNumber=rollbackTargetBlockNumber,
|
|
609
|
+
)
|
|
610
|
+
cs.fetchState =
|
|
611
|
+
cs.fetchState->FetchState.rollback(~targetBlockNumber=rollbackTargetBlockNumber)
|
|
612
|
+
cs.committedProgressBlockNumber = Pervasives.min(
|
|
613
|
+
cs.committedProgressBlockNumber,
|
|
614
|
+
rollbackTargetBlockNumber,
|
|
615
|
+
)
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
}
|