envio 3.5.0-alpha.3 → 3.5.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/Batch.res +8 -8
- package/src/Batch.res.mjs +4 -3
- package/src/BatchProcessing.res +17 -7
- package/src/BatchProcessing.res.mjs +8 -4
- package/src/ChainFetching.res +20 -19
- package/src/ChainFetching.res.mjs +14 -14
- package/src/ChainId.res +58 -0
- package/src/ChainId.res.mjs +49 -0
- package/src/ChainId.resi +27 -0
- package/src/ChainMap.res +12 -26
- package/src/ChainMap.res.mjs +5 -41
- package/src/ChainMap.resi +5 -20
- package/src/ChainMetadata.res +1 -1
- package/src/ChainMetadata.res.mjs +3 -2
- package/src/ChainState.res +35 -20
- package/src/ChainState.res.mjs +34 -11
- package/src/ChainState.resi +2 -0
- package/src/Config.res +41 -24
- package/src/Config.res.mjs +17 -14
- package/src/ContractRegisterContext.res +1 -1
- package/src/ContractRegisterContext.res.mjs +1 -1
- package/src/Core.res +1 -0
- package/src/CrossChainState.res +61 -16
- package/src/CrossChainState.res.mjs +37 -13
- package/src/CrossChainState.resi +3 -2
- package/src/Env.res +0 -3
- package/src/Env.res.mjs +0 -3
- package/src/EventConfigBuilder.res +1 -1
- package/src/EventProcessing.res +4 -4
- package/src/EventProcessing.res.mjs +5 -5
- package/src/FetchState.res +261 -89
- package/src/FetchState.res.mjs +187 -63
- package/src/FinalizeBackfill.res +33 -6
- package/src/FinalizeBackfill.res.mjs +19 -1
- package/src/HandlerRegister.res +8 -8
- package/src/HandlerRegister.res.mjs +5 -4
- package/src/HandlerRegister.resi +1 -1
- package/src/InMemoryStore.res +4 -0
- package/src/InMemoryStore.res.mjs +1 -1
- package/src/IndexerLoop.res +9 -2
- package/src/IndexerLoop.res.mjs +8 -1
- package/src/IndexerState.res +41 -24
- package/src/IndexerState.res.mjs +36 -17
- package/src/IndexerState.resi +9 -7
- package/src/Internal.res +19 -17
- package/src/Internal.res.mjs +18 -9
- package/src/LoadLayer.res +1 -1
- package/src/LoadLayer.res.mjs +2 -1
- package/src/LogSelection.res +2 -2
- package/src/Main.res +7 -11
- package/src/Main.res.mjs +4 -6
- package/src/Metrics.res +6 -6
- package/src/Metrics.res.mjs +4 -3
- package/src/Persistence.res +15 -7
- package/src/PgStorage.res +284 -258
- package/src/PgStorage.res.mjs +181 -208
- package/src/RawEvent.res +1 -2
- package/src/RawEvent.res.mjs +1 -1
- package/src/Rollback.res +18 -20
- package/src/Rollback.res.mjs +9 -9
- package/src/RollbackCommit.res +2 -2
- package/src/RollbackCommit.res.mjs +3 -3
- package/src/SafeCheckpointTracking.res +2 -2
- package/src/SimulateDeadInputTracker.res +17 -21
- package/src/SimulateDeadInputTracker.res.mjs +9 -8
- package/src/SimulateItems.res +8 -10
- package/src/SimulateItems.res.mjs +9 -10
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +3 -2
- package/src/TestIndexer.res +11 -14
- package/src/TestIndexer.res.mjs +14 -10
- package/src/UserContext.res +2 -2
- package/src/UserContext.res.mjs +2 -1
- package/src/Utils.res +0 -4
- package/src/Utils.res.mjs +1 -6
- package/src/bindings/ClickHouse.res +20 -2
- package/src/bindings/ClickHouse.res.mjs +25 -14
- package/src/db/EntityHistory.res +1 -1
- package/src/db/IndexCatalog.res +248 -0
- package/src/db/IndexCatalog.res.mjs +222 -0
- package/src/db/IndexDefinition.res +131 -0
- package/src/db/IndexDefinition.res.mjs +138 -0
- package/src/db/IndexManager.res +145 -0
- package/src/db/IndexManager.res.mjs +130 -0
- package/src/db/InternalTable.res +50 -26
- package/src/db/InternalTable.res.mjs +32 -24
- package/src/db/Table.res +11 -1
- package/src/db/Table.res.mjs +9 -4
- package/src/sources/AddressSet.res +5 -4
- package/src/sources/AddressStore.res +25 -14
- package/src/sources/AddressStore.res.mjs +10 -12
- package/src/sources/Evm.res +2 -2
- package/src/sources/Evm.res.mjs +1 -1
- package/src/sources/EvmChain.res +3 -3
- package/src/sources/EvmChain.res.mjs +3 -3
- package/src/sources/EvmHyperSyncSource.res +7 -7
- package/src/sources/EvmHyperSyncSource.res.mjs +6 -6
- package/src/sources/Fuel.res +2 -2
- package/src/sources/Fuel.res.mjs +1 -1
- package/src/sources/FuelHyperSyncClient.res +3 -0
- package/src/sources/FuelHyperSyncClient.res.mjs +1 -0
- package/src/sources/FuelHyperSyncSource.res +5 -7
- package/src/sources/FuelHyperSyncSource.res.mjs +5 -5
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +5 -0
- package/src/sources/HyperSyncClient.res.mjs +1 -0
- package/src/sources/RpcSource.res +13 -14
- package/src/sources/RpcSource.res.mjs +13 -12
- package/src/sources/SimulateSource.res +42 -15
- package/src/sources/SimulateSource.res.mjs +27 -9
- package/src/sources/Source.res +1 -5
- package/src/sources/SourceManager.res +12 -12
- package/src/sources/SourceManager.res.mjs +11 -10
- package/src/sources/SourceManager.resi +2 -2
- package/src/sources/Svm.res +4 -5
- package/src/sources/Svm.res.mjs +5 -4
- package/src/sources/SvmHyperSyncClient.res +3 -0
- package/src/sources/SvmHyperSyncClient.res.mjs +1 -0
- package/src/sources/SvmHyperSyncSource.res +4 -4
- package/src/sources/SvmHyperSyncSource.res.mjs +3 -3
- package/src/tui/Tui.res +1 -1
- package/src/tui/Tui.res.mjs +2 -1
- package/src/tui/components/CustomHooks.res +5 -4
- package/src/tui/components/CustomHooks.res.mjs +4 -2
- package/src/db/IndexRegistry.res +0 -219
- package/src/db/IndexRegistry.res.mjs +0 -195
package/src/IndexerLoop.res
CHANGED
|
@@ -19,10 +19,10 @@ let start = (state: IndexerState.t) => {
|
|
|
19
19
|
launch(state, () =>
|
|
20
20
|
state
|
|
21
21
|
->IndexerState.crossChainState
|
|
22
|
-
->CrossChainState.checkAndFetch(~dispatchChain=(~
|
|
22
|
+
->CrossChainState.checkAndFetch(~dispatchChain=(~chainId, ~action) =>
|
|
23
23
|
ChainFetching.fetchChain(
|
|
24
24
|
state,
|
|
25
|
-
|
|
25
|
+
chainId,
|
|
26
26
|
~action,
|
|
27
27
|
~stateId=state->IndexerState.epoch,
|
|
28
28
|
~scheduleFetch,
|
|
@@ -38,6 +38,13 @@ let start = (state: IndexerState.t) => {
|
|
|
38
38
|
Rollback.rollback(state, ~scheduleFetch, ~scheduleProcessing, ~scheduleRollback)
|
|
39
39
|
)
|
|
40
40
|
|
|
41
|
+
// Resuming already ready means the FinalizingIndexes phase is behind us and
|
|
42
|
+
// won't run again, so this is the only pass that can restore an index the
|
|
43
|
+
// database lost while the indexer was down.
|
|
44
|
+
if state->IndexerState.isRealtime {
|
|
45
|
+
launch(state, () => FinalizeBackfill.repairSchemaIndexes(state))
|
|
46
|
+
}
|
|
47
|
+
|
|
41
48
|
scheduleFetch()
|
|
42
49
|
scheduleProcessing()
|
|
43
50
|
}
|
package/src/IndexerLoop.res.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import * as IndexerState from "./IndexerState.res.mjs";
|
|
|
5
5
|
import * as ChainFetching from "./ChainFetching.res.mjs";
|
|
6
6
|
import * as BatchProcessing from "./BatchProcessing.res.mjs";
|
|
7
7
|
import * as CrossChainState from "./CrossChainState.res.mjs";
|
|
8
|
+
import * as FinalizeBackfill from "./FinalizeBackfill.res.mjs";
|
|
8
9
|
|
|
9
10
|
function launch(state, work) {
|
|
10
11
|
if (!IndexerState.isStopped(state)) {
|
|
@@ -15,7 +16,7 @@ function launch(state, work) {
|
|
|
15
16
|
|
|
16
17
|
function start(state) {
|
|
17
18
|
let scheduleFetch = () => {
|
|
18
|
-
let work = () => CrossChainState.checkAndFetch(IndexerState.crossChainState(state), (
|
|
19
|
+
let work = () => CrossChainState.checkAndFetch(IndexerState.crossChainState(state), (chainId, action) => ChainFetching.fetchChain(state, chainId, action, IndexerState.epoch(state), scheduleFetch, scheduleProcessing, scheduleRollback));
|
|
19
20
|
if (!IndexerState.isStopped(state)) {
|
|
20
21
|
work();
|
|
21
22
|
return;
|
|
@@ -35,6 +36,12 @@ function start(state) {
|
|
|
35
36
|
return;
|
|
36
37
|
}
|
|
37
38
|
};
|
|
39
|
+
if (IndexerState.isRealtime(state)) {
|
|
40
|
+
let work = () => FinalizeBackfill.repairSchemaIndexes(state);
|
|
41
|
+
if (!IndexerState.isStopped(state)) {
|
|
42
|
+
work();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
38
45
|
scheduleFetch();
|
|
39
46
|
scheduleProcessing();
|
|
40
47
|
}
|
package/src/IndexerState.res
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
type chain = ChainMap.Chain.t
|
|
2
1
|
type rollbackState =
|
|
3
2
|
| NoRollback
|
|
4
|
-
| ReorgDetected({
|
|
3
|
+
| ReorgDetected({chainId: ChainId.t, blockNumber: int})
|
|
5
4
|
| FindingReorgDepth
|
|
6
|
-
| FoundReorgDepth({
|
|
5
|
+
| FoundReorgDepth({chainId: ChainId.t, rollbackTargetBlockNumber: int})
|
|
7
6
|
| RollbackReady({eventsProcessedDiffByChain: dict<float>})
|
|
8
7
|
|
|
9
8
|
module EntityTables = {
|
|
@@ -90,6 +89,10 @@ type t = {
|
|
|
90
89
|
mutable processedBatchesCount: int,
|
|
91
90
|
// The single in-flight write loop, None when idle.
|
|
92
91
|
mutable writeFiber: option<promise<unit>>,
|
|
92
|
+
// The single in-flight finalization, None when none is running. Every path
|
|
93
|
+
// that reaches the FinalizingIndexes phase awaits this one instead of
|
|
94
|
+
// starting a second pass over the same indexes.
|
|
95
|
+
mutable finalizeFiber: option<promise<unit>>,
|
|
93
96
|
// Set once a write throws, to stop the loop. The error itself goes to onError.
|
|
94
97
|
mutable hasFailedWrite: bool,
|
|
95
98
|
// Resolved after every commit so capacity/flush waiters can re-evaluate.
|
|
@@ -191,6 +194,7 @@ let make = (
|
|
|
191
194
|
processedBatches: [],
|
|
192
195
|
processedBatchesCount: 0,
|
|
193
196
|
writeFiber: None,
|
|
197
|
+
finalizeFiber: None,
|
|
194
198
|
hasFailedWrite: false,
|
|
195
199
|
commitWaiters: [],
|
|
196
200
|
chainMeta: Dict.make(),
|
|
@@ -256,27 +260,26 @@ let makeFromDbState = (
|
|
|
256
260
|
false
|
|
257
261
|
} else {
|
|
258
262
|
// Check if any chain is in reorg threshold by comparing progress with sourceBlock - maxReorgDepth.
|
|
259
|
-
initialState.chains->Array.some(
|
|
263
|
+
initialState.chains->Array.some(resumedChainState =>
|
|
260
264
|
isProgressInReorgThreshold(
|
|
261
|
-
~progressBlockNumber=
|
|
262
|
-
~sourceBlockNumber=
|
|
263
|
-
~maxReorgDepth=
|
|
265
|
+
~progressBlockNumber=resumedChainState.progressBlockNumber,
|
|
266
|
+
~sourceBlockNumber=resumedChainState.sourceBlockNumber,
|
|
267
|
+
~maxReorgDepth=resumedChainState.maxReorgDepth,
|
|
264
268
|
)
|
|
265
269
|
)
|
|
266
270
|
}
|
|
267
271
|
|
|
268
|
-
//
|
|
269
|
-
//
|
|
272
|
+
// `ready_at` is durable: a chain that once caught up resumes realtime, and the
|
|
273
|
+
// deferred indexes committed alongside that stamp are not owed again.
|
|
270
274
|
let isRealtime =
|
|
271
|
-
!Env.updateSyncTimeOnRestart &&
|
|
272
275
|
initialState.chains->Array.length > 0 &&
|
|
273
|
-
|
|
276
|
+
initialState.chains->Array.every(c => c.timestampCaughtUpToHeadOrEndblock->Option.isSome)
|
|
274
277
|
|
|
275
278
|
let chainStates = Dict.make()
|
|
276
279
|
initialState.chains->Array.forEach((resumedChainState: Persistence.initialChainState) => {
|
|
277
|
-
let
|
|
278
|
-
let chainConfig = config.chainMap->ChainMap.get(
|
|
279
|
-
chainStates->
|
|
280
|
+
let chainId = Config.getChain(config, ~chainId=resumedChainState.id)
|
|
281
|
+
let chainConfig = config.chainMap->ChainMap.get(chainId)
|
|
282
|
+
chainStates->ChainId.Dict.set(
|
|
280
283
|
resumedChainState.id,
|
|
281
284
|
chainConfig->ChainState.makeFromDbState(
|
|
282
285
|
~resumedChainState,
|
|
@@ -304,6 +307,7 @@ let makeFromDbState = (
|
|
|
304
307
|
~onError,
|
|
305
308
|
~onExit?,
|
|
306
309
|
)
|
|
310
|
+
state.crossChainState->CrossChainState.markCaughtUpOnResume
|
|
307
311
|
initialState.cache->Utils.Dict.forEach(({effectName, count, scope}) => {
|
|
308
312
|
state.effectState->EffectState.setUnregisteredCacheCount(~effectName, ~scope, ~count)
|
|
309
313
|
})
|
|
@@ -358,15 +362,15 @@ let stop = (state: t) => {
|
|
|
358
362
|
state.isStopped = true
|
|
359
363
|
}
|
|
360
364
|
|
|
361
|
-
let getChainState = (state: t, ~
|
|
365
|
+
let getChainState = (state: t, ~chainId: ChainId.t): ChainState.t =>
|
|
362
366
|
switch state.crossChainState
|
|
363
367
|
->CrossChainState.chainStates
|
|
364
|
-
->
|
|
368
|
+
->ChainId.Dict.dangerouslyGetNonOption(chainId) {
|
|
365
369
|
| Some(cs) => cs
|
|
366
370
|
| None =>
|
|
367
|
-
// Should be unreachable
|
|
371
|
+
// Should be unreachable: every configured chain gets a state at startup
|
|
368
372
|
JsError.throwWithMessage(
|
|
369
|
-
"No chain with id " ++
|
|
373
|
+
"No chain with id " ++ chainId->ChainId.toString ++ " found in chain states",
|
|
370
374
|
)
|
|
371
375
|
}
|
|
372
376
|
|
|
@@ -390,18 +394,18 @@ let enterReorgThreshold = (state: t) => state.crossChainState->CrossChainState.e
|
|
|
390
394
|
// ReorgDetected state as one step, so the epoch bump can never be left out. The
|
|
391
395
|
// caller has already mutated the chain states (restored counters, reset pending
|
|
392
396
|
// queries). isResolvingReorg derives from rollbackState.
|
|
393
|
-
let beginReorg = (state: t, ~
|
|
397
|
+
let beginReorg = (state: t, ~chainId, ~blockNumber) => {
|
|
394
398
|
// Settle here, or the rollback that follows would be folded into the stall on
|
|
395
399
|
// the next beginProcessing — time envio_rollback_seconds already counts.
|
|
396
400
|
state->settleStalledOnFetch
|
|
397
401
|
state.epoch = state.epoch + 1
|
|
398
|
-
state.rollbackState = ReorgDetected({
|
|
402
|
+
state.rollbackState = ReorgDetected({chainId, blockNumber})
|
|
399
403
|
}
|
|
400
404
|
|
|
401
405
|
let enterFindingReorgDepth = (state: t) => state.rollbackState = FindingReorgDepth
|
|
402
406
|
|
|
403
|
-
let foundReorgDepth = (state: t, ~
|
|
404
|
-
state.rollbackState = FoundReorgDepth({
|
|
407
|
+
let foundReorgDepth = (state: t, ~chainId, ~rollbackTargetBlockNumber) =>
|
|
408
|
+
state.rollbackState = FoundReorgDepth({chainId, rollbackTargetBlockNumber})
|
|
405
409
|
|
|
406
410
|
// Finish a rollback. The caller has already rolled the chain states back in
|
|
407
411
|
// place; this leaves the diff ready for the next batch to consume.
|
|
@@ -414,7 +418,15 @@ let clearRollback = (state: t) => state.rollbackState = NoRollback
|
|
|
414
418
|
|
|
415
419
|
// Invalidate in-flight fetches/waiters without starting a rollback, eg on the
|
|
416
420
|
// realtime transition where the parked waiter is bound to the pre-realtime source.
|
|
417
|
-
|
|
421
|
+
// Drops the pending queries with it, the way the reorg path does: a response
|
|
422
|
+
// carrying the old epoch is discarded before handleQueryResult can retire its
|
|
423
|
+
// query, and a partition that still holds one never asks for another range.
|
|
424
|
+
let invalidateInflight = (state: t) => {
|
|
425
|
+
state.epoch = state.epoch + 1
|
|
426
|
+
state.crossChainState
|
|
427
|
+
->CrossChainState.chainStates
|
|
428
|
+
->Utils.Dict.forEach(ChainState.resetPendingQueries)
|
|
429
|
+
}
|
|
418
430
|
|
|
419
431
|
let applyBatchProgress = (state: t, ~batch: Batch.t) =>
|
|
420
432
|
state.crossChainState->CrossChainState.applyBatchProgress(
|
|
@@ -461,6 +473,7 @@ let processedCheckpointId = (state: t) => state.processedCheckpointId
|
|
|
461
473
|
let processedBatches = (state: t) => state.processedBatches
|
|
462
474
|
let processedBatchesCount = (state: t) => state.processedBatchesCount
|
|
463
475
|
let writeFiber = (state: t) => state.writeFiber
|
|
476
|
+
let finalizeFiber = (state: t) => state.finalizeFiber
|
|
464
477
|
let hasFailedWrite = (state: t) => state.hasFailedWrite
|
|
465
478
|
let chainMetaDirty = (state: t) => state.chainMetaDirty
|
|
466
479
|
let chainMetaThrottler = (state: t) => state.chainMetaThrottler
|
|
@@ -476,7 +489,8 @@ let isFinalizingIndexes = (state: t) =>
|
|
|
476
489
|
state.crossChainState->CrossChainState.isCaughtUp &&
|
|
477
490
|
!(state.crossChainState->CrossChainState.isRealtime)
|
|
478
491
|
|
|
479
|
-
let
|
|
492
|
+
let markCaughtUpIfSettled = (state: t) =>
|
|
493
|
+
state.crossChainState->CrossChainState.markCaughtUpIfSettled
|
|
480
494
|
|
|
481
495
|
let markReady = (state: t, ~readyAt) => state.crossChainState->CrossChainState.markReady(~readyAt)
|
|
482
496
|
|
|
@@ -806,6 +820,9 @@ let recordWriteFailure = (state: t, exn) => {
|
|
|
806
820
|
let beginWriteFiber = (state: t, fiber) => state.writeFiber = Some(fiber)
|
|
807
821
|
let endWriteFiber = (state: t) => state.writeFiber = None
|
|
808
822
|
|
|
823
|
+
let beginFinalizeFiber = (state: t, fiber) => state.finalizeFiber = Some(fiber)
|
|
824
|
+
let endFinalizeFiber = (state: t) => state.finalizeFiber = None
|
|
825
|
+
|
|
809
826
|
// Resolve and clear everyone waiting on a commit so they can re-evaluate.
|
|
810
827
|
let wakeCommitWaiters = (state: t) => {
|
|
811
828
|
let waiters = state.commitWaiters
|
package/src/IndexerState.res.mjs
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import * as Env from "./Env.res.mjs";
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
5
|
import * as Config from "./Config.res.mjs";
|
|
6
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
6
7
|
import * as Logging from "./Logging.res.mjs";
|
|
7
8
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
8
9
|
import * as Internal from "./Internal.res.mjs";
|
|
@@ -67,6 +68,7 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
67
68
|
processedBatches: [],
|
|
68
69
|
processedBatchesCount: 0,
|
|
69
70
|
writeFiber: undefined,
|
|
71
|
+
finalizeFiber: undefined,
|
|
70
72
|
hasFailedWrite: false,
|
|
71
73
|
commitWaiters: [],
|
|
72
74
|
chainMeta: {},
|
|
@@ -106,24 +108,25 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
|
|
|
106
108
|
let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
|
|
107
109
|
let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
|
|
108
110
|
let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
|
|
109
|
-
let isInReorgThreshold = initialState.cleanRun ? false : initialState.chains.some(
|
|
110
|
-
let progressBlockNumber =
|
|
111
|
-
let sourceBlockNumber =
|
|
112
|
-
let maxReorgDepth =
|
|
111
|
+
let isInReorgThreshold = initialState.cleanRun ? false : initialState.chains.some(resumedChainState => {
|
|
112
|
+
let progressBlockNumber = resumedChainState.progressBlockNumber;
|
|
113
|
+
let sourceBlockNumber = resumedChainState.sourceBlockNumber;
|
|
114
|
+
let maxReorgDepth = resumedChainState.maxReorgDepth;
|
|
113
115
|
if (maxReorgDepth > 0 && sourceBlockNumber > 0) {
|
|
114
116
|
return progressBlockNumber > (sourceBlockNumber - maxReorgDepth | 0);
|
|
115
117
|
} else {
|
|
116
118
|
return false;
|
|
117
119
|
}
|
|
118
120
|
});
|
|
119
|
-
let isRealtime =
|
|
121
|
+
let isRealtime = initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
|
|
120
122
|
let chainStates = {};
|
|
121
123
|
initialState.chains.forEach(resumedChainState => {
|
|
122
|
-
let
|
|
123
|
-
let chainConfig = ChainMap.get(config.chainMap,
|
|
124
|
+
let chainId = Config.getChain(config, resumedChainState.id);
|
|
125
|
+
let chainConfig = ChainMap.get(config.chainMap, chainId);
|
|
124
126
|
chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval);
|
|
125
127
|
});
|
|
126
128
|
let state = make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
|
|
129
|
+
CrossChainState.markCaughtUpOnResume(state.crossChainState);
|
|
127
130
|
Utils.Dict.forEach(initialState.cache, param => {
|
|
128
131
|
let count = param.count;
|
|
129
132
|
EffectState.setUnregisteredCacheCount(state.effectState, param.effectName, param.scope, count);
|
|
@@ -170,12 +173,12 @@ function stop(state) {
|
|
|
170
173
|
state.isStopped = true;
|
|
171
174
|
}
|
|
172
175
|
|
|
173
|
-
function getChainState(state,
|
|
174
|
-
let cs = CrossChainState.chainStates(state.crossChainState)[
|
|
176
|
+
function getChainState(state, chainId) {
|
|
177
|
+
let cs = CrossChainState.chainStates(state.crossChainState)[chainId];
|
|
175
178
|
if (cs !== undefined) {
|
|
176
179
|
return Primitive_option.valFromOption(cs);
|
|
177
180
|
} else {
|
|
178
|
-
return Stdlib_JsError.throwWithMessage("No chain with id " +
|
|
181
|
+
return Stdlib_JsError.throwWithMessage("No chain with id " + ChainId.toString(chainId) + " found in chain states");
|
|
179
182
|
}
|
|
180
183
|
}
|
|
181
184
|
|
|
@@ -191,7 +194,7 @@ function enterReorgThreshold(state) {
|
|
|
191
194
|
CrossChainState.enterReorgThreshold(state.crossChainState);
|
|
192
195
|
}
|
|
193
196
|
|
|
194
|
-
function beginReorg(state,
|
|
197
|
+
function beginReorg(state, chainId, blockNumber) {
|
|
195
198
|
let since = state.processingStalledOnFetchSince;
|
|
196
199
|
if (since !== undefined) {
|
|
197
200
|
state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
|
|
@@ -200,7 +203,7 @@ function beginReorg(state, chain, blockNumber) {
|
|
|
200
203
|
state.epoch = state.epoch + 1 | 0;
|
|
201
204
|
state.rollbackState = {
|
|
202
205
|
TAG: "ReorgDetected",
|
|
203
|
-
|
|
206
|
+
chainId: chainId,
|
|
204
207
|
blockNumber: blockNumber
|
|
205
208
|
};
|
|
206
209
|
}
|
|
@@ -209,10 +212,10 @@ function enterFindingReorgDepth(state) {
|
|
|
209
212
|
state.rollbackState = "FindingReorgDepth";
|
|
210
213
|
}
|
|
211
214
|
|
|
212
|
-
function foundReorgDepth(state,
|
|
215
|
+
function foundReorgDepth(state, chainId, rollbackTargetBlockNumber) {
|
|
213
216
|
state.rollbackState = {
|
|
214
217
|
TAG: "FoundReorgDepth",
|
|
215
|
-
|
|
218
|
+
chainId: chainId,
|
|
216
219
|
rollbackTargetBlockNumber: rollbackTargetBlockNumber
|
|
217
220
|
};
|
|
218
221
|
}
|
|
@@ -230,6 +233,7 @@ function clearRollback(state) {
|
|
|
230
233
|
|
|
231
234
|
function invalidateInflight(state) {
|
|
232
235
|
state.epoch = state.epoch + 1 | 0;
|
|
236
|
+
Utils.Dict.forEach(CrossChainState.chainStates(state.crossChainState), ChainState.resetPendingQueries);
|
|
233
237
|
}
|
|
234
238
|
|
|
235
239
|
function applyBatchProgress(state, batch) {
|
|
@@ -308,6 +312,10 @@ function writeFiber(state) {
|
|
|
308
312
|
return state.writeFiber;
|
|
309
313
|
}
|
|
310
314
|
|
|
315
|
+
function finalizeFiber(state) {
|
|
316
|
+
return state.finalizeFiber;
|
|
317
|
+
}
|
|
318
|
+
|
|
311
319
|
function hasFailedWrite(state) {
|
|
312
320
|
return state.hasFailedWrite;
|
|
313
321
|
}
|
|
@@ -344,8 +352,8 @@ function isFinalizingIndexes(state) {
|
|
|
344
352
|
}
|
|
345
353
|
}
|
|
346
354
|
|
|
347
|
-
function
|
|
348
|
-
CrossChainState.
|
|
355
|
+
function markCaughtUpIfSettled(state) {
|
|
356
|
+
CrossChainState.markCaughtUpIfSettled(state.crossChainState);
|
|
349
357
|
}
|
|
350
358
|
|
|
351
359
|
function markReady(state, readyAt) {
|
|
@@ -692,6 +700,14 @@ function endWriteFiber(state) {
|
|
|
692
700
|
state.writeFiber = undefined;
|
|
693
701
|
}
|
|
694
702
|
|
|
703
|
+
function beginFinalizeFiber(state, fiber) {
|
|
704
|
+
state.finalizeFiber = fiber;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
function endFinalizeFiber(state) {
|
|
708
|
+
state.finalizeFiber = undefined;
|
|
709
|
+
}
|
|
710
|
+
|
|
695
711
|
function wakeCommitWaiters(state) {
|
|
696
712
|
let waiters = state.commitWaiters;
|
|
697
713
|
state.commitWaiters = [];
|
|
@@ -771,6 +787,7 @@ export {
|
|
|
771
787
|
processedBatches,
|
|
772
788
|
processedBatchesCount,
|
|
773
789
|
writeFiber,
|
|
790
|
+
finalizeFiber,
|
|
774
791
|
hasFailedWrite,
|
|
775
792
|
chainMetaDirty,
|
|
776
793
|
chainMetaThrottler,
|
|
@@ -779,7 +796,7 @@ export {
|
|
|
779
796
|
isInReorgThreshold,
|
|
780
797
|
isRealtime,
|
|
781
798
|
isFinalizingIndexes,
|
|
782
|
-
|
|
799
|
+
markCaughtUpIfSettled,
|
|
783
800
|
markReady,
|
|
784
801
|
rollbackState,
|
|
785
802
|
indexerStartTime,
|
|
@@ -810,6 +827,8 @@ export {
|
|
|
810
827
|
recordWriteFailure,
|
|
811
828
|
beginWriteFiber,
|
|
812
829
|
endWriteFiber,
|
|
830
|
+
beginFinalizeFiber,
|
|
831
|
+
endFinalizeFiber,
|
|
813
832
|
wakeCommitWaiters,
|
|
814
833
|
addCommitWaiter,
|
|
815
834
|
stageChainMeta,
|
package/src/IndexerState.resi
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// The indexer state. `t` is opaque: other modules read it through the accessors
|
|
2
2
|
// and change it only through the transitions and setters exposed here.
|
|
3
3
|
|
|
4
|
-
type chain = ChainMap.Chain.t
|
|
5
4
|
|
|
6
5
|
type rollbackState =
|
|
7
6
|
| NoRollback
|
|
8
|
-
| ReorgDetected({
|
|
7
|
+
| ReorgDetected({chainId: ChainId.t, blockNumber: int})
|
|
9
8
|
| FindingReorgDepth
|
|
10
|
-
| FoundReorgDepth({
|
|
9
|
+
| FoundReorgDepth({chainId: ChainId.t, rollbackTargetBlockNumber: int})
|
|
11
10
|
| RollbackReady({eventsProcessedDiffByChain: dict<float>})
|
|
12
11
|
|
|
13
12
|
module EntityTables: {
|
|
@@ -54,11 +53,11 @@ let isStale: (t, ~stateId: int) => bool
|
|
|
54
53
|
let isResolvingReorg: t => bool
|
|
55
54
|
let errorExit: (t, ErrorHandling.t) => unit
|
|
56
55
|
let stop: t => unit
|
|
57
|
-
let getChainState: (t, ~
|
|
56
|
+
let getChainState: (t, ~chainId: ChainId.t) => ChainState.t
|
|
58
57
|
let enterReorgThreshold: t => unit
|
|
59
|
-
let beginReorg: (t, ~
|
|
58
|
+
let beginReorg: (t, ~chainId: ChainId.t, ~blockNumber: int) => unit
|
|
60
59
|
let enterFindingReorgDepth: t => unit
|
|
61
|
-
let foundReorgDepth: (t, ~
|
|
60
|
+
let foundReorgDepth: (t, ~chainId: ChainId.t, ~rollbackTargetBlockNumber: int) => unit
|
|
62
61
|
let completeRollback: (t, ~eventsProcessedDiffByChain: dict<float>) => unit
|
|
63
62
|
let clearRollback: t => unit
|
|
64
63
|
let invalidateInflight: t => unit
|
|
@@ -87,6 +86,7 @@ let processedCheckpointId: t => Internal.checkpointId
|
|
|
87
86
|
let processedBatches: t => array<Batch.t>
|
|
88
87
|
let processedBatchesCount: t => int
|
|
89
88
|
let writeFiber: t => option<promise<unit>>
|
|
89
|
+
let finalizeFiber: t => option<promise<unit>>
|
|
90
90
|
let hasFailedWrite: t => bool
|
|
91
91
|
let chainMetaDirty: t => bool
|
|
92
92
|
let chainMetaThrottler: t => Throttler.t
|
|
@@ -95,7 +95,7 @@ let chainStates: t => dict<ChainState.t>
|
|
|
95
95
|
let isInReorgThreshold: t => bool
|
|
96
96
|
let isRealtime: t => bool
|
|
97
97
|
let isFinalizingIndexes: t => bool
|
|
98
|
-
let
|
|
98
|
+
let markCaughtUpIfSettled: t => unit
|
|
99
99
|
let markReady: (t, ~readyAt: Date.t) => unit
|
|
100
100
|
let rollbackState: t => rollbackState
|
|
101
101
|
let indexerStartTime: t => Date.t
|
|
@@ -145,6 +145,8 @@ let beginRollbackDiff: (
|
|
|
145
145
|
let recordWriteFailure: (t, exn) => unit
|
|
146
146
|
let beginWriteFiber: (t, promise<unit>) => unit
|
|
147
147
|
let endWriteFiber: t => unit
|
|
148
|
+
let beginFinalizeFiber: (t, promise<unit>) => unit
|
|
149
|
+
let endFinalizeFiber: t => unit
|
|
148
150
|
let wakeCommitWaiters: t => unit
|
|
149
151
|
let addCommitWaiter: (t, unit => unit) => unit
|
|
150
152
|
let stageChainMeta: (t, dict<InternalTable.Chains.metaFields>) => unit
|
package/src/Internal.res
CHANGED
|
@@ -301,7 +301,7 @@ type genericEvent<'params, 'block, 'transaction> = {
|
|
|
301
301
|
contractName: string,
|
|
302
302
|
eventName: string,
|
|
303
303
|
params: 'params,
|
|
304
|
-
chainId:
|
|
304
|
+
chainId: ChainId.t,
|
|
305
305
|
srcAddress: Address.t,
|
|
306
306
|
logIndex: int,
|
|
307
307
|
transaction: 'transaction,
|
|
@@ -369,7 +369,7 @@ type entityHandlerContext<'entity> = {
|
|
|
369
369
|
}
|
|
370
370
|
|
|
371
371
|
type chainInfo = {
|
|
372
|
-
id:
|
|
372
|
+
id: ChainId.t,
|
|
373
373
|
// True once every chain has caught up to head/endBlock and entered real-time
|
|
374
374
|
// indexing mode. False while any chain is still backfilling.
|
|
375
375
|
isRealtime: bool,
|
|
@@ -624,7 +624,7 @@ type dcs = array<indexingAddress>
|
|
|
624
624
|
type eventItem = private {
|
|
625
625
|
kind: [#0],
|
|
626
626
|
onEventRegistration: onEventRegistration,
|
|
627
|
-
|
|
627
|
+
chainId: ChainId.t,
|
|
628
628
|
blockNumber: int,
|
|
629
629
|
logIndex: int,
|
|
630
630
|
// Within-block transaction index — the key into the per-chain transaction
|
|
@@ -637,7 +637,7 @@ type eventItem = private {
|
|
|
637
637
|
// `InternalTable`) so the ecosystem's `toRawEvent` can reference it without
|
|
638
638
|
// pulling in `InternalTable`'s dependency on `Config`.
|
|
639
639
|
type rawEvent = {
|
|
640
|
-
chain_id:
|
|
640
|
+
chain_id: ChainId.t,
|
|
641
641
|
event_id: bigint,
|
|
642
642
|
event_name: string,
|
|
643
643
|
contract_name: string,
|
|
@@ -665,7 +665,7 @@ type onBlockRegistration = {
|
|
|
665
665
|
// we want to use the order they are defined for sorting
|
|
666
666
|
index: int,
|
|
667
667
|
name: string,
|
|
668
|
-
chainId:
|
|
668
|
+
chainId: ChainId.t,
|
|
669
669
|
startBlock: option<int>,
|
|
670
670
|
endBlock: option<int>,
|
|
671
671
|
interval: int,
|
|
@@ -677,7 +677,7 @@ type item =
|
|
|
677
677
|
| @as(0)
|
|
678
678
|
Event({
|
|
679
679
|
onEventRegistration: onEventRegistration,
|
|
680
|
-
|
|
680
|
+
chainId: ChainId.t,
|
|
681
681
|
blockNumber: int,
|
|
682
682
|
logIndex: int,
|
|
683
683
|
transactionIndex: int,
|
|
@@ -694,7 +694,7 @@ external getItemLogIndex: item => int = "logIndex"
|
|
|
694
694
|
|
|
695
695
|
let getItemChainId = item =>
|
|
696
696
|
switch item {
|
|
697
|
-
| Event({
|
|
697
|
+
| Event({chainId})
|
|
698
698
|
| Block({onBlockRegistration: {chainId}}) => chainId
|
|
699
699
|
}
|
|
700
700
|
|
|
@@ -796,7 +796,7 @@ type effect = {
|
|
|
796
796
|
@unboxed
|
|
797
797
|
type chainScope =
|
|
798
798
|
| @as("crossChain") CrossChain
|
|
799
|
-
| Chain(
|
|
799
|
+
| Chain(ChainId.t)
|
|
800
800
|
|
|
801
801
|
let cacheTablePrefix = "envio_effect_"
|
|
802
802
|
|
|
@@ -804,27 +804,29 @@ let cacheTablePrefix = "envio_effect_"
|
|
|
804
804
|
// canonical Postgres cache-table name and .envio/cache file path. Everything
|
|
805
805
|
// that needs a cache address goes through here instead of slicing prefixes.
|
|
806
806
|
// CrossChain -> envio_effect_<name> <name>.tsv
|
|
807
|
-
// Chain(1) -> envio_1_effect_<name> 1/<name>.tsv
|
|
808
|
-
// Chain(137) -> envio_137_effect_<name> 137/<name>.tsv
|
|
807
|
+
// Chain(1->ChainId.fromInt) -> envio_1_effect_<name> 1/<name>.tsv
|
|
808
|
+
// Chain(137->ChainId.fromInt) -> envio_137_effect_<name> 137/<name>.tsv
|
|
809
809
|
module EffectCache = {
|
|
810
810
|
let toTableName = (~effectName, ~scope) =>
|
|
811
811
|
switch scope {
|
|
812
812
|
| CrossChain => cacheTablePrefix ++ effectName
|
|
813
|
-
| Chain(chainId) => `envio_${chainId->
|
|
813
|
+
| Chain(chainId) => `envio_${chainId->ChainId.toString}_effect_${effectName}`
|
|
814
814
|
}
|
|
815
815
|
|
|
816
816
|
// "crossChain" or the decimal chain id. Used as the `scope` Prometheus label.
|
|
817
817
|
let scopeToString = scope =>
|
|
818
818
|
switch scope {
|
|
819
819
|
| CrossChain => "crossChain"
|
|
820
|
-
| Chain(chainId) => chainId->
|
|
820
|
+
| Chain(chainId) => chainId->ChainId.toString
|
|
821
821
|
}
|
|
822
822
|
|
|
823
823
|
// Only accepts a canonical decimal chain id ("7", not "007" or "1foo") —
|
|
824
|
-
//
|
|
824
|
+
// the schema's parser follows parseFloat semantics and accepts both.
|
|
825
825
|
let parseChainId = str =>
|
|
826
|
-
switch
|
|
827
|
-
|
|
|
826
|
+
switch try Some(str->ChainId.normalizeOrThrow) catch {
|
|
827
|
+
| _ => None
|
|
828
|
+
} {
|
|
829
|
+
| Some(chainId) if chainId->ChainId.toString === str => Some(chainId)
|
|
828
830
|
| _ => None
|
|
829
831
|
}
|
|
830
832
|
|
|
@@ -864,7 +866,7 @@ module EffectCache = {
|
|
|
864
866
|
let toCachePath = (~effectName, ~scope) =>
|
|
865
867
|
switch scope {
|
|
866
868
|
| CrossChain => effectName ++ ".tsv"
|
|
867
|
-
| Chain(chainId) => `${chainId->
|
|
869
|
+
| Chain(chainId) => `${chainId->ChainId.toString}/${effectName}.tsv`
|
|
868
870
|
}
|
|
869
871
|
}
|
|
870
872
|
|
|
@@ -893,7 +895,7 @@ type reorgCheckpoint = {
|
|
|
893
895
|
@as("id")
|
|
894
896
|
checkpointId: bigint,
|
|
895
897
|
@as("chain_id")
|
|
896
|
-
chainId:
|
|
898
|
+
chainId: ChainId.t,
|
|
897
899
|
@as("block_number")
|
|
898
900
|
blockNumber: int,
|
|
899
901
|
@as("block_hash")
|
package/src/Internal.res.mjs
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import * as Table from "./db/Table.res.mjs";
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
5
|
import * as Address from "./Address.res.mjs";
|
|
6
|
-
import * as
|
|
6
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
7
7
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
8
8
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
9
9
|
|
|
@@ -149,7 +149,7 @@ function dependsOnAddresses(isWildcard, filterByAddresses) {
|
|
|
149
149
|
|
|
150
150
|
function getItemChainId(item) {
|
|
151
151
|
if (item.kind === 0) {
|
|
152
|
-
return item.
|
|
152
|
+
return item.chainId;
|
|
153
153
|
} else {
|
|
154
154
|
return item.onBlockRegistration.chainId;
|
|
155
155
|
}
|
|
@@ -172,7 +172,7 @@ function toTableName(effectName, scope) {
|
|
|
172
172
|
if (scope === "crossChain") {
|
|
173
173
|
return cacheTablePrefix + effectName;
|
|
174
174
|
} else {
|
|
175
|
-
return `envio_` +
|
|
175
|
+
return `envio_` + ChainId.toString(scope) + `_effect_` + effectName;
|
|
176
176
|
}
|
|
177
177
|
}
|
|
178
178
|
|
|
@@ -180,14 +180,23 @@ function scopeToString(scope) {
|
|
|
180
180
|
if (scope === "crossChain") {
|
|
181
181
|
return "crossChain";
|
|
182
182
|
} else {
|
|
183
|
-
return
|
|
183
|
+
return ChainId.toString(scope);
|
|
184
184
|
}
|
|
185
185
|
}
|
|
186
186
|
|
|
187
187
|
function parseChainId(str) {
|
|
188
|
-
let chainId
|
|
189
|
-
|
|
190
|
-
|
|
188
|
+
let chainId;
|
|
189
|
+
try {
|
|
190
|
+
chainId = Primitive_option.some(ChainId.normalizeOrThrow(str));
|
|
191
|
+
} catch (exn) {
|
|
192
|
+
chainId = undefined;
|
|
193
|
+
}
|
|
194
|
+
if (chainId === undefined) {
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
let chainId$1 = Primitive_option.valFromOption(chainId);
|
|
198
|
+
if (ChainId.toString(chainId$1) === str) {
|
|
199
|
+
return Primitive_option.some(chainId$1);
|
|
191
200
|
}
|
|
192
201
|
}
|
|
193
202
|
|
|
@@ -236,7 +245,7 @@ function fromTableName(tableName) {
|
|
|
236
245
|
if (chainId !== undefined) {
|
|
237
246
|
return [
|
|
238
247
|
effectName$1,
|
|
239
|
-
chainId
|
|
248
|
+
Primitive_option.valFromOption(chainId)
|
|
240
249
|
];
|
|
241
250
|
}
|
|
242
251
|
}
|
|
@@ -245,7 +254,7 @@ function toCachePath(effectName, scope) {
|
|
|
245
254
|
if (scope === "crossChain") {
|
|
246
255
|
return effectName + ".tsv";
|
|
247
256
|
} else {
|
|
248
|
-
return
|
|
257
|
+
return ChainId.toString(scope) + `/` + effectName + `.tsv`;
|
|
249
258
|
}
|
|
250
259
|
}
|
|
251
260
|
|
package/src/LoadLayer.res
CHANGED
|
@@ -226,7 +226,7 @@ let loadEffect = (
|
|
|
226
226
|
// the storage-load metric stays stable.
|
|
227
227
|
let key = switch scope {
|
|
228
228
|
| CrossChain => `${effectName}.effect`
|
|
229
|
-
| Chain(chainId) => `${effectName}.effect.${chainId->
|
|
229
|
+
| Chain(chainId) => `${effectName}.effect.${chainId->ChainId.toString}`
|
|
230
230
|
}
|
|
231
231
|
|
|
232
232
|
let load = async (args, ~onError) => {
|
package/src/LoadLayer.res.mjs
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Table from "./db/Table.res.mjs";
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
5
6
|
import * as Logging from "./Logging.res.mjs";
|
|
6
7
|
import * as Ecosystem from "./Ecosystem.res.mjs";
|
|
7
8
|
import * as EffectState from "./EffectState.res.mjs";
|
|
@@ -113,7 +114,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
|
|
|
113
114
|
let table = inMemTable.table;
|
|
114
115
|
let tableName = table.tableName;
|
|
115
116
|
let key;
|
|
116
|
-
key = scope === "crossChain" ? effectName + `.effect` : effectName + `.effect.` +
|
|
117
|
+
key = scope === "crossChain" ? effectName + `.effect` : effectName + `.effect.` + ChainId.toString(scope);
|
|
117
118
|
let load = async (args, onError) => {
|
|
118
119
|
let idsToLoad = args.map(arg => arg.cacheKey);
|
|
119
120
|
let idsFromCache = new Set();
|