envio 3.0.2 → 3.1.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/evm.schema.json +15 -8
- package/fuel.schema.json +19 -12
- package/index.d.ts +0 -2
- package/package.json +6 -7
- package/rescript.json +1 -1
- package/src/Batch.res +4 -214
- package/src/Batch.res.mjs +6 -165
- package/src/ChainFetcher.res +4 -5
- package/src/ChainFetcher.res.mjs +6 -7
- package/src/ChainManager.res +10 -9
- package/src/ChainManager.res.mjs +6 -10
- package/src/Config.res +9 -25
- package/src/Config.res.mjs +17 -27
- package/src/Core.res +7 -0
- package/src/Ctx.res +1 -0
- package/src/Env.res +0 -1
- package/src/Env.res.mjs +0 -3
- package/src/EventConfigBuilder.res +13 -123
- package/src/EventConfigBuilder.res.mjs +6 -73
- package/src/EventProcessing.res +5 -29
- package/src/EventProcessing.res.mjs +11 -20
- package/src/EventUtils.res +0 -27
- package/src/EventUtils.res.mjs +0 -24
- package/src/FetchState.res +1 -11
- package/src/FetchState.res.mjs +2 -16
- package/src/GlobalState.res +23 -37
- package/src/GlobalState.res.mjs +10 -38
- package/src/HandlerLoader.res +6 -5
- package/src/HandlerLoader.res.mjs +27 -9
- package/src/HandlerRegister.res +1 -12
- package/src/HandlerRegister.res.mjs +1 -6
- package/src/HandlerRegister.resi +1 -1
- package/src/Hasura.res +96 -32
- package/src/Hasura.res.mjs +93 -38
- package/src/InMemoryStore.res +181 -45
- package/src/InMemoryStore.res.mjs +143 -40
- package/src/InMemoryTable.res +147 -247
- package/src/InMemoryTable.res.mjs +131 -230
- package/src/Internal.res +10 -34
- package/src/Internal.res.mjs +9 -3
- package/src/LoadLayer.res +5 -5
- package/src/LoadLayer.res.mjs +5 -5
- package/src/Main.res +4 -6
- package/src/Main.res.mjs +26 -15
- package/src/Persistence.res +7 -132
- package/src/Persistence.res.mjs +1 -102
- package/src/PgStorage.res +57 -40
- package/src/PgStorage.res.mjs +60 -34
- package/src/ReorgDetection.res +35 -58
- package/src/ReorgDetection.res.mjs +21 -29
- package/src/SimulateItems.res.mjs +21 -3
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +1 -1
- package/src/TableIndices.res +9 -2
- package/src/TableIndices.res.mjs +7 -1
- package/src/TestIndexer.res +53 -60
- package/src/TestIndexer.res.mjs +77 -63
- package/src/TestIndexerProxyStorage.res +4 -14
- package/src/TestIndexerProxyStorage.res.mjs +1 -5
- package/src/UserContext.res +2 -4
- package/src/UserContext.res.mjs +4 -5
- package/src/Utils.res +0 -2
- package/src/Utils.res.mjs +0 -3
- package/src/bindings/ClickHouse.res +45 -38
- package/src/bindings/ClickHouse.res.mjs +16 -17
- package/src/bindings/Vitest.res +3 -0
- package/src/db/InternalTable.res +59 -18
- package/src/db/InternalTable.res.mjs +82 -51
- package/src/db/Table.res +9 -2
- package/src/db/Table.res.mjs +10 -7
- package/src/sources/EvmChain.res +32 -9
- package/src/sources/EvmChain.res.mjs +31 -4
- package/src/sources/HyperFuelSource.res +14 -57
- package/src/sources/HyperFuelSource.res.mjs +18 -38
- package/src/sources/HyperSync.res +36 -101
- package/src/sources/HyperSync.res.mjs +42 -96
- package/src/sources/HyperSync.resi +4 -22
- package/src/sources/HyperSyncClient.res +67 -245
- package/src/sources/HyperSyncClient.res.mjs +47 -46
- package/src/sources/HyperSyncSource.res +76 -147
- package/src/sources/HyperSyncSource.res.mjs +61 -114
- package/src/sources/RpcSource.res +43 -22
- package/src/sources/RpcSource.res.mjs +50 -35
- package/src/sources/SimulateSource.res +1 -7
- package/src/sources/SimulateSource.res.mjs +1 -7
- package/src/sources/Source.res +8 -1
- package/src/sources/SourceManager.res +9 -0
- package/src/sources/SourceManager.res.mjs +10 -0
- package/src/sources/SourceManager.resi +2 -0
- package/svm.schema.json +11 -4
package/src/FetchState.res
CHANGED
|
@@ -1485,15 +1485,6 @@ let getNextQuery = (
|
|
|
1485
1485
|
}
|
|
1486
1486
|
}
|
|
1487
1487
|
|
|
1488
|
-
let getTimestampAt = (fetchState: t, ~index) => {
|
|
1489
|
-
switch fetchState.buffer->Belt.Array.get(index) {
|
|
1490
|
-
| Some(Event({timestamp})) => timestamp
|
|
1491
|
-
| Some(Block(_)) =>
|
|
1492
|
-
JsError.throwWithMessage("Block handlers are not supported for ordered multichain mode.")
|
|
1493
|
-
| None => (fetchState->bufferBlock).blockTimestamp
|
|
1494
|
-
}
|
|
1495
|
-
}
|
|
1496
|
-
|
|
1497
1488
|
let hasReadyItem = ({buffer} as fetchState: t) => {
|
|
1498
1489
|
switch buffer->Belt.Array.get(0) {
|
|
1499
1490
|
| Some(item) => item->Internal.getItemBlockNumber <= fetchState->bufferBlockNumber
|
|
@@ -1965,8 +1956,7 @@ let sortForUnorderedBatch = {
|
|
|
1965
1956
|
}
|
|
1966
1957
|
}
|
|
1967
1958
|
|
|
1968
|
-
|
|
1969
|
-
let getUnorderedMultichainProgressBlockNumberAt = ({buffer} as fetchState: t, ~index) => {
|
|
1959
|
+
let getProgressBlockNumberAt = ({buffer} as fetchState: t, ~index) => {
|
|
1970
1960
|
let bufferBlockNumber = fetchState->bufferBlockNumber
|
|
1971
1961
|
switch buffer->Belt.Array.get(index) {
|
|
1972
1962
|
| Some(item) if bufferBlockNumber >= item->Internal.getItemBlockNumber =>
|
package/src/FetchState.res.mjs
CHANGED
|
@@ -1174,19 +1174,6 @@ function getNextQuery(fetchState, concurrencyLimit) {
|
|
|
1174
1174
|
};
|
|
1175
1175
|
}
|
|
1176
1176
|
|
|
1177
|
-
function getTimestampAt(fetchState, index) {
|
|
1178
|
-
let match = Belt_Array.get(fetchState.buffer, index);
|
|
1179
|
-
if (match !== undefined) {
|
|
1180
|
-
if (match.kind === 0) {
|
|
1181
|
-
return match.timestamp;
|
|
1182
|
-
} else {
|
|
1183
|
-
return Stdlib_JsError.throwWithMessage("Block handlers are not supported for ordered multichain mode.");
|
|
1184
|
-
}
|
|
1185
|
-
} else {
|
|
1186
|
-
return bufferBlock(fetchState).blockTimestamp;
|
|
1187
|
-
}
|
|
1188
|
-
}
|
|
1189
|
-
|
|
1190
1177
|
function hasReadyItem(fetchState) {
|
|
1191
1178
|
let item = Belt_Array.get(fetchState.buffer, 0);
|
|
1192
1179
|
if (item !== undefined) {
|
|
@@ -1590,7 +1577,7 @@ function sortForUnorderedBatch(fetchStates, batchSizeTarget) {
|
|
|
1590
1577
|
return copied;
|
|
1591
1578
|
}
|
|
1592
1579
|
|
|
1593
|
-
function
|
|
1580
|
+
function getProgressBlockNumberAt(fetchState, index) {
|
|
1594
1581
|
let bufferBlockNumber$1 = bufferBlockNumber(fetchState);
|
|
1595
1582
|
let item = Belt_Array.get(fetchState.buffer, index);
|
|
1596
1583
|
if (item !== undefined && bufferBlockNumber$1 >= item.blockNumber) {
|
|
@@ -1631,7 +1618,6 @@ export {
|
|
|
1631
1618
|
startFetchingQueries,
|
|
1632
1619
|
pushQueriesForRange,
|
|
1633
1620
|
getNextQuery,
|
|
1634
|
-
getTimestampAt,
|
|
1635
1621
|
hasReadyItem,
|
|
1636
1622
|
getReadyItemsCount,
|
|
1637
1623
|
make$1 as make,
|
|
@@ -1642,7 +1628,7 @@ export {
|
|
|
1642
1628
|
isActivelyIndexing,
|
|
1643
1629
|
isReadyToEnterReorgThreshold,
|
|
1644
1630
|
sortForUnorderedBatch,
|
|
1645
|
-
|
|
1631
|
+
getProgressBlockNumberAt,
|
|
1646
1632
|
updateKnownHeight,
|
|
1647
1633
|
}
|
|
1648
1634
|
/* Utils Not a pure module */
|
package/src/GlobalState.res
CHANGED
|
@@ -4,7 +4,7 @@ type rollbackState =
|
|
|
4
4
|
| ReorgDetected({chain: chain, blockNumber: int})
|
|
5
5
|
| FindingReorgDepth
|
|
6
6
|
| FoundReorgDepth({chain: chain, rollbackTargetBlockNumber: int})
|
|
7
|
-
| RollbackReady({
|
|
7
|
+
| RollbackReady({eventsProcessedDiffByChain: dict<float>})
|
|
8
8
|
|
|
9
9
|
module WriteThrottlers = {
|
|
10
10
|
type t = {
|
|
@@ -132,7 +132,6 @@ type action =
|
|
|
132
132
|
| SuccessExit
|
|
133
133
|
| ErrorExit(ErrorHandling.t)
|
|
134
134
|
| SetRollbackState({
|
|
135
|
-
diffInMemoryStore: InMemoryStore.t,
|
|
136
135
|
rollbackedChainManager: ChainManager.t,
|
|
137
136
|
eventsProcessedDiffByChain: dict<float>,
|
|
138
137
|
})
|
|
@@ -331,10 +330,6 @@ let updateProgressedChains = (chainManager: ChainManager.t, ~batch: Batch.t, ~ct
|
|
|
331
330
|
|
|
332
331
|
{
|
|
333
332
|
...chainManager,
|
|
334
|
-
committedCheckpointId: switch batch.checkpointIds->Utils.Array.last {
|
|
335
|
-
| Some(checkpointId) => checkpointId
|
|
336
|
-
| None => chainManager.committedCheckpointId
|
|
337
|
-
},
|
|
338
333
|
chainFetchers,
|
|
339
334
|
isRealtime: chainManager.isRealtime || allChainsReady.contents,
|
|
340
335
|
}
|
|
@@ -350,7 +345,7 @@ let validatePartitionQueryResponse = (
|
|
|
350
345
|
latestFetchedBlockNumber,
|
|
351
346
|
stats,
|
|
352
347
|
knownHeight,
|
|
353
|
-
|
|
348
|
+
blockHashes,
|
|
354
349
|
fromBlockQueried,
|
|
355
350
|
} = response
|
|
356
351
|
|
|
@@ -374,7 +369,7 @@ let validatePartitionQueryResponse = (
|
|
|
374
369
|
)
|
|
375
370
|
|
|
376
371
|
let (updatedReorgDetection, reorgResult: ReorgDetection.reorgResult) =
|
|
377
|
-
chainFetcher.reorgDetection->ReorgDetection.registerReorgGuard(~
|
|
372
|
+
chainFetcher.reorgDetection->ReorgDetection.registerReorgGuard(~blockHashes, ~knownHeight)
|
|
378
373
|
|
|
379
374
|
let updatedChainFetcher = {
|
|
380
375
|
...chainFetcher,
|
|
@@ -764,12 +759,11 @@ let actionReducer = (state: t, action: action) => {
|
|
|
764
759
|
},
|
|
765
760
|
[NextQuery(CheckAllChains)],
|
|
766
761
|
)
|
|
767
|
-
| SetRollbackState({
|
|
762
|
+
| SetRollbackState({rollbackedChainManager, eventsProcessedDiffByChain}) => (
|
|
768
763
|
{
|
|
769
764
|
...state,
|
|
770
765
|
rollbackState: RollbackReady({
|
|
771
|
-
|
|
772
|
-
eventsProcessedDiffByChain,
|
|
766
|
+
eventsProcessedDiffByChain: eventsProcessedDiffByChain,
|
|
773
767
|
}),
|
|
774
768
|
chainManager: rollbackedChainManager,
|
|
775
769
|
},
|
|
@@ -823,14 +817,9 @@ let checkAndFetchForChain = (
|
|
|
823
817
|
let {fetchState} = chainFetcher
|
|
824
818
|
let isRealtime = state.chainManager.isRealtime
|
|
825
819
|
|
|
826
|
-
//
|
|
827
|
-
//
|
|
828
|
-
|
|
829
|
-
let reducedPolling =
|
|
830
|
-
(chainFetcher->ChainFetcher.isReady && !isRealtime) ||
|
|
831
|
-
(state.ctx.config.shouldRollbackOnReorg &&
|
|
832
|
-
!state.chainManager.isInReorgThreshold &&
|
|
833
|
-
fetchState->FetchState.isReadyToEnterReorgThreshold)
|
|
820
|
+
// Only affects the WaitingForNewBlock branch of fetchNext, where
|
|
821
|
+
// there's nothing to fetch. During backfill any such chain is idle.
|
|
822
|
+
let reducedPolling = !isRealtime
|
|
834
823
|
|
|
835
824
|
await chainFetcher.sourceManager->SourceManager.fetchNext(
|
|
836
825
|
~fetchState,
|
|
@@ -951,23 +940,21 @@ let injectedTaskReducer = (
|
|
|
951
940
|
}
|
|
952
941
|
| ProcessEventBatch =>
|
|
953
942
|
if !state.currentlyProcessingBatch && !isPreparingRollback(state) {
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
| RollbackReady({diffInMemoryStore}) => Some(diffInMemoryStore)
|
|
958
|
-
| _ => None
|
|
943
|
+
let isRollbackBatch = switch state.rollbackState {
|
|
944
|
+
| RollbackReady(_) => true
|
|
945
|
+
| _ => false
|
|
959
946
|
}
|
|
960
947
|
|
|
961
948
|
let batch =
|
|
962
949
|
state.chainManager->ChainManager.createBatch(
|
|
950
|
+
~committedCheckpointId=state.ctx.inMemoryStore.committedCheckpointId,
|
|
963
951
|
~batchSizeTarget=state.ctx.config.batchSize,
|
|
964
|
-
~isRollback=
|
|
952
|
+
~isRollback=isRollbackBatch,
|
|
965
953
|
)
|
|
966
954
|
|
|
967
955
|
let progressedChainsById = batch.progressedChainsById
|
|
968
956
|
|
|
969
957
|
let isInReorgThreshold = state.chainManager.isInReorgThreshold
|
|
970
|
-
let shouldSaveHistory = state.ctx.config->Config.shouldSaveHistory(~isInReorgThreshold)
|
|
971
958
|
|
|
972
959
|
let isBelowReorgThreshold =
|
|
973
960
|
!state.chainManager.isInReorgThreshold && state.ctx.config.shouldRollbackOnReorg
|
|
@@ -1007,12 +994,8 @@ let injectedTaskReducer = (
|
|
|
1007
994
|
dispatchAction(StartProcessingBatch)
|
|
1008
995
|
dispatchAction(UpdateQueues({progressedChainsById, shouldEnterReorgThreshold}))
|
|
1009
996
|
|
|
1010
|
-
let inMemoryStore =
|
|
1011
|
-
|
|
1012
|
-
InMemoryStore.make(~entities=state.ctx.persistence.allEntities),
|
|
1013
|
-
)
|
|
1014
|
-
|
|
1015
|
-
inMemoryStore->InMemoryStore.setBatchDcs(~batch, ~shouldSaveHistory)
|
|
997
|
+
let inMemoryStore = state.ctx.inMemoryStore
|
|
998
|
+
inMemoryStore->InMemoryStore.setBatchDcs(~batch)
|
|
1016
999
|
|
|
1017
1000
|
switch await EventProcessing.processEventBatch(
|
|
1018
1001
|
~batch,
|
|
@@ -1051,6 +1034,10 @@ let injectedTaskReducer = (
|
|
|
1051
1034
|
~reorgBlockNumber,
|
|
1052
1035
|
)
|
|
1053
1036
|
|
|
1037
|
+
chainFetcher.sourceManager->SourceManager.onReorg(
|
|
1038
|
+
~rollbackTargetBlock=rollbackTargetBlockNumber,
|
|
1039
|
+
)
|
|
1040
|
+
|
|
1054
1041
|
dispatchAction(FindReorgDepth({chain, rollbackTargetBlockNumber}))
|
|
1055
1042
|
}
|
|
1056
1043
|
// We can come to this case when event batch finished processing
|
|
@@ -1189,10 +1176,10 @@ let injectedTaskReducer = (
|
|
|
1189
1176
|
}
|
|
1190
1177
|
})
|
|
1191
1178
|
|
|
1192
|
-
|
|
1193
|
-
|
|
1179
|
+
let diff = await state.ctx.inMemoryStore->InMemoryStore.prepareRollbackDiff(
|
|
1180
|
+
~persistence=state.ctx.persistence,
|
|
1194
1181
|
~rollbackTargetCheckpointId,
|
|
1195
|
-
~rollbackDiffCheckpointId=state.
|
|
1182
|
+
~rollbackDiffCheckpointId=state.ctx.inMemoryStore.committedCheckpointId->BigInt.add(1n),
|
|
1196
1183
|
)
|
|
1197
1184
|
|
|
1198
1185
|
let chainManager = {
|
|
@@ -1207,7 +1194,7 @@ let injectedTaskReducer = (
|
|
|
1207
1194
|
"upserted": diff["setEntities"],
|
|
1208
1195
|
},
|
|
1209
1196
|
"rollbackedEvents": rollbackedProcessedEvents.contents,
|
|
1210
|
-
"beforeCheckpointId": state.
|
|
1197
|
+
"beforeCheckpointId": state.ctx.inMemoryStore.committedCheckpointId,
|
|
1211
1198
|
"targetCheckpointId": rollbackTargetCheckpointId,
|
|
1212
1199
|
})
|
|
1213
1200
|
Prometheus.RollbackSuccess.increment(
|
|
@@ -1217,7 +1204,6 @@ let injectedTaskReducer = (
|
|
|
1217
1204
|
|
|
1218
1205
|
dispatchAction(
|
|
1219
1206
|
SetRollbackState({
|
|
1220
|
-
diffInMemoryStore: diff["inMemStore"],
|
|
1221
1207
|
rollbackedChainManager: chainManager,
|
|
1222
1208
|
eventsProcessedDiffByChain,
|
|
1223
1209
|
}),
|
package/src/GlobalState.res.mjs
CHANGED
|
@@ -14,7 +14,6 @@ import * as FetchState from "./FetchState.res.mjs";
|
|
|
14
14
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
15
15
|
import * as Belt_Option from "@rescript/runtime/lib/es6/Belt_Option.js";
|
|
16
16
|
import * as LoadManager from "./LoadManager.res.mjs";
|
|
17
|
-
import * as Persistence from "./Persistence.res.mjs";
|
|
18
17
|
import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
|
|
19
18
|
import * as ChainFetcher from "./ChainFetcher.res.mjs";
|
|
20
19
|
import * as ChainManager from "./ChainManager.res.mjs";
|
|
@@ -244,11 +243,8 @@ function updateProgressedChains(chainManager, batch, ctx) {
|
|
|
244
243
|
if (allChainsReady.contents) {
|
|
245
244
|
Prometheus.ProgressReady.setAllReady();
|
|
246
245
|
}
|
|
247
|
-
let checkpointId = Utils.$$Array.last(batch.checkpointIds);
|
|
248
246
|
return {
|
|
249
|
-
committedCheckpointId: checkpointId !== undefined ? checkpointId : chainManager.committedCheckpointId,
|
|
250
247
|
chainFetchers: chainFetchers,
|
|
251
|
-
multichain: chainManager.multichain,
|
|
252
248
|
isInReorgThreshold: chainManager.isInReorgThreshold,
|
|
253
249
|
isRealtime: chainManager.isRealtime || allChainsReady.contents
|
|
254
250
|
};
|
|
@@ -264,7 +260,7 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
|
|
|
264
260
|
Prometheus.SourceHeight.set(SourceManager.getActiveSource(chainFetcher.sourceManager).name, chainFetcher.chainConfig.id, knownHeight);
|
|
265
261
|
}
|
|
266
262
|
Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Belt_Option.getWithDefault(stats["parsing time (s)"], 0), response.parsedQueueItems.length, (response.latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
|
|
267
|
-
let match = ReorgDetection.registerReorgGuard(chainFetcher.reorgDetection, response.
|
|
263
|
+
let match = ReorgDetection.registerReorgGuard(chainFetcher.reorgDetection, response.blockHashes, knownHeight);
|
|
268
264
|
let reorgResult = match[1];
|
|
269
265
|
let updatedChainFetcher_logger = chainFetcher.logger;
|
|
270
266
|
let updatedChainFetcher_fetchState = chainFetcher.fetchState;
|
|
@@ -291,9 +287,7 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
|
|
|
291
287
|
let init = state.chainManager;
|
|
292
288
|
let nextState_ctx = state.ctx;
|
|
293
289
|
let nextState_chainManager = {
|
|
294
|
-
committedCheckpointId: init.committedCheckpointId,
|
|
295
290
|
chainFetchers: ChainMap.set(state.chainManager.chainFetchers, chain, updatedChainFetcher),
|
|
296
|
-
multichain: init.multichain,
|
|
297
291
|
isInReorgThreshold: init.isInReorgThreshold,
|
|
298
292
|
isRealtime: init.isRealtime
|
|
299
293
|
};
|
|
@@ -346,7 +340,6 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
|
|
|
346
340
|
let eventsProcessedDiffByChain = match$1.eventsProcessedDiffByChain;
|
|
347
341
|
let init$1 = state.chainManager;
|
|
348
342
|
chainManager = {
|
|
349
|
-
committedCheckpointId: init$1.committedCheckpointId,
|
|
350
343
|
chainFetchers: ChainMap.mapWithKey(state.chainManager.chainFetchers, (c, chainFetcher) => {
|
|
351
344
|
let eventsProcessedDiff = eventsProcessedDiffByChain[c];
|
|
352
345
|
if (eventsProcessedDiff !== undefined) {
|
|
@@ -366,7 +359,6 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
|
|
|
366
359
|
return chainFetcher;
|
|
367
360
|
}
|
|
368
361
|
}),
|
|
369
|
-
multichain: init$1.multichain,
|
|
370
362
|
isInReorgThreshold: init$1.isInReorgThreshold,
|
|
371
363
|
isRealtime: init$1.isRealtime
|
|
372
364
|
};
|
|
@@ -375,7 +367,6 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
|
|
|
375
367
|
{
|
|
376
368
|
ctx: nextState_ctx,
|
|
377
369
|
chainManager: {
|
|
378
|
-
committedCheckpointId: chainManager.committedCheckpointId,
|
|
379
370
|
chainFetchers: ChainMap.map(chainManager.chainFetchers, chainFetcher => ({
|
|
380
371
|
logger: chainFetcher.logger,
|
|
381
372
|
fetchState: FetchState.resetPendingQueries(chainFetcher.fetchState),
|
|
@@ -388,7 +379,6 @@ function validatePartitionQueryResponse(state, partitionQueryResponse) {
|
|
|
388
379
|
reorgDetection: chainFetcher.reorgDetection,
|
|
389
380
|
safeCheckpointTracking: chainFetcher.safeCheckpointTracking
|
|
390
381
|
})),
|
|
391
|
-
multichain: chainManager.multichain,
|
|
392
382
|
isInReorgThreshold: chainManager.isInReorgThreshold,
|
|
393
383
|
isRealtime: chainManager.isRealtime
|
|
394
384
|
},
|
|
@@ -489,9 +479,7 @@ function submitPartitionQueryResponse(state, newItems, newItemsWithDcs, knownHei
|
|
|
489
479
|
let init$2 = state.chainManager;
|
|
490
480
|
let nextState_ctx = state.ctx;
|
|
491
481
|
let nextState_chainManager = {
|
|
492
|
-
committedCheckpointId: init$2.committedCheckpointId,
|
|
493
482
|
chainFetchers: ChainMap.set(state.chainManager.chainFetchers, chain, updatedChainFetcher$1),
|
|
494
|
-
multichain: init$2.multichain,
|
|
495
483
|
isInReorgThreshold: init$2.isInReorgThreshold,
|
|
496
484
|
isRealtime: init$2.isRealtime
|
|
497
485
|
};
|
|
@@ -569,9 +557,7 @@ function updateChainFetcher(chainFetcherUpdate, state, chain) {
|
|
|
569
557
|
{
|
|
570
558
|
ctx: state.ctx,
|
|
571
559
|
chainManager: {
|
|
572
|
-
committedCheckpointId: init.committedCheckpointId,
|
|
573
560
|
chainFetchers: ChainMap.update(state.chainManager.chainFetchers, chain, chainFetcherUpdate),
|
|
574
|
-
multichain: init.multichain,
|
|
575
561
|
isInReorgThreshold: init.isInReorgThreshold,
|
|
576
562
|
isRealtime: init.isRealtime
|
|
577
563
|
},
|
|
@@ -608,9 +594,7 @@ function onEnterReorgThreshold(state) {
|
|
|
608
594
|
return {
|
|
609
595
|
ctx: state.ctx,
|
|
610
596
|
chainManager: {
|
|
611
|
-
committedCheckpointId: init.committedCheckpointId,
|
|
612
597
|
chainFetchers: chainFetchers,
|
|
613
|
-
multichain: init.multichain,
|
|
614
598
|
isInReorgThreshold: true,
|
|
615
599
|
isRealtime: init.isRealtime
|
|
616
600
|
},
|
|
@@ -712,9 +696,7 @@ function actionReducer(state, action) {
|
|
|
712
696
|
let init = state.chainManager;
|
|
713
697
|
let state_ctx = state.ctx;
|
|
714
698
|
let state_chainManager = {
|
|
715
|
-
committedCheckpointId: init.committedCheckpointId,
|
|
716
699
|
chainFetchers: updatedChainFetchers,
|
|
717
|
-
multichain: init.multichain,
|
|
718
700
|
isInReorgThreshold: init.isInReorgThreshold,
|
|
719
701
|
isRealtime: init.isRealtime
|
|
720
702
|
};
|
|
@@ -853,14 +835,10 @@ function actionReducer(state, action) {
|
|
|
853
835
|
};
|
|
854
836
|
});
|
|
855
837
|
let init$1 = state.chainManager;
|
|
856
|
-
let chainManager_committedCheckpointId = init$1.committedCheckpointId;
|
|
857
|
-
let chainManager_multichain = init$1.multichain;
|
|
858
838
|
let chainManager_isInReorgThreshold = init$1.isInReorgThreshold;
|
|
859
839
|
let chainManager_isRealtime = init$1.isRealtime;
|
|
860
840
|
let chainManager = {
|
|
861
|
-
committedCheckpointId: chainManager_committedCheckpointId,
|
|
862
841
|
chainFetchers: chainFetchers,
|
|
863
|
-
multichain: chainManager_multichain,
|
|
864
842
|
isInReorgThreshold: chainManager_isInReorgThreshold,
|
|
865
843
|
isRealtime: chainManager_isRealtime
|
|
866
844
|
};
|
|
@@ -899,7 +877,6 @@ function actionReducer(state, action) {
|
|
|
899
877
|
currentlyProcessingBatch: state.currentlyProcessingBatch,
|
|
900
878
|
rollbackState: {
|
|
901
879
|
TAG: "RollbackReady",
|
|
902
|
-
diffInMemoryStore: action.diffInMemoryStore,
|
|
903
880
|
eventsProcessedDiffByChain: action.eventsProcessedDiffByChain
|
|
904
881
|
},
|
|
905
882
|
indexerStartTime: state.indexerStartTime,
|
|
@@ -967,7 +944,7 @@ function checkAndFetchForChain(waitForNewBlock, executeQuery, state, dispatchAct
|
|
|
967
944
|
}
|
|
968
945
|
let fetchState = chainFetcher.fetchState;
|
|
969
946
|
let isRealtime = state.chainManager.isRealtime;
|
|
970
|
-
let reducedPolling =
|
|
947
|
+
let reducedPolling = !isRealtime;
|
|
971
948
|
return await SourceManager.fetchNext(chainFetcher.sourceManager, fetchState, async query => {
|
|
972
949
|
try {
|
|
973
950
|
let response = await executeQuery(chainFetcher.sourceManager, query, fetchState.knownHeight, isRealtime);
|
|
@@ -1003,12 +980,11 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
|
|
|
1003
980
|
return;
|
|
1004
981
|
}
|
|
1005
982
|
let match = state.rollbackState;
|
|
1006
|
-
let
|
|
1007
|
-
|
|
1008
|
-
let batch = ChainManager.createBatch(state.chainManager, state.ctx.config.batchSize,
|
|
983
|
+
let isRollbackBatch;
|
|
984
|
+
isRollbackBatch = typeof match !== "object" ? false : match.TAG === "RollbackReady";
|
|
985
|
+
let batch = ChainManager.createBatch(state.chainManager, state.ctx.inMemoryStore.committedCheckpointId, state.ctx.config.batchSize, isRollbackBatch);
|
|
1009
986
|
let progressedChainsById = batch.progressedChainsById;
|
|
1010
987
|
let isInReorgThreshold = state.chainManager.isInReorgThreshold;
|
|
1011
|
-
let shouldSaveHistory = Config.shouldSaveHistory(state.ctx.config, isInReorgThreshold);
|
|
1012
988
|
let isBelowReorgThreshold = !state.chainManager.isInReorgThreshold && state.ctx.config.shouldRollbackOnReorg;
|
|
1013
989
|
let shouldEnterReorgThreshold = isBelowReorgThreshold && ChainMap.values(state.chainManager.chainFetchers).every(chainFetcher => {
|
|
1014
990
|
let chainAfterBatch = progressedChainsById[chainFetcher.fetchState.chainId];
|
|
@@ -1036,8 +1012,8 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
|
|
|
1036
1012
|
progressedChainsById: progressedChainsById,
|
|
1037
1013
|
shouldEnterReorgThreshold: shouldEnterReorgThreshold
|
|
1038
1014
|
});
|
|
1039
|
-
let inMemoryStore =
|
|
1040
|
-
InMemoryStore.setBatchDcs(inMemoryStore, batch
|
|
1015
|
+
let inMemoryStore = state.ctx.inMemoryStore;
|
|
1016
|
+
InMemoryStore.setBatchDcs(inMemoryStore, batch);
|
|
1041
1017
|
let res;
|
|
1042
1018
|
try {
|
|
1043
1019
|
res = await EventProcessing.processEventBatch(batch, inMemoryStore, isInReorgThreshold, state.loadManager, state.ctx, state.chainManager.chainFetchers);
|
|
@@ -1076,6 +1052,7 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
|
|
|
1076
1052
|
let chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain);
|
|
1077
1053
|
dispatchAction("StartFindingReorgDepth");
|
|
1078
1054
|
let rollbackTargetBlockNumber = await getLastKnownValidBlock(chainFetcher, match$1.blockNumber);
|
|
1055
|
+
SourceManager.onReorg(chainFetcher.sourceManager, rollbackTargetBlockNumber);
|
|
1079
1056
|
return dispatchAction({
|
|
1080
1057
|
TAG: "FindReorgDepth",
|
|
1081
1058
|
chain: chain,
|
|
@@ -1152,16 +1129,12 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
|
|
|
1152
1129
|
safeCheckpointTracking: safeCheckpointTracking !== undefined ? SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber) : undefined
|
|
1153
1130
|
};
|
|
1154
1131
|
});
|
|
1155
|
-
let diff$1 = await
|
|
1132
|
+
let diff$1 = await InMemoryStore.prepareRollbackDiff(state.ctx.inMemoryStore, state.ctx.persistence, rollbackTargetCheckpointId, state.ctx.inMemoryStore.committedCheckpointId + 1n);
|
|
1156
1133
|
let init = state.chainManager;
|
|
1157
|
-
let chainManager_committedCheckpointId = init.committedCheckpointId;
|
|
1158
|
-
let chainManager_multichain = init.multichain;
|
|
1159
1134
|
let chainManager_isInReorgThreshold = init.isInReorgThreshold;
|
|
1160
1135
|
let chainManager_isRealtime = init.isRealtime;
|
|
1161
1136
|
let chainManager = {
|
|
1162
|
-
committedCheckpointId: chainManager_committedCheckpointId,
|
|
1163
1137
|
chainFetchers: chainFetchers,
|
|
1164
|
-
multichain: chainManager_multichain,
|
|
1165
1138
|
isInReorgThreshold: chainManager_isInReorgThreshold,
|
|
1166
1139
|
isRealtime: chainManager_isRealtime
|
|
1167
1140
|
};
|
|
@@ -1172,13 +1145,12 @@ function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlo
|
|
|
1172
1145
|
upserted: diff$1.setEntities
|
|
1173
1146
|
},
|
|
1174
1147
|
rollbackedEvents: rollbackedProcessedEvents,
|
|
1175
|
-
beforeCheckpointId: state.
|
|
1148
|
+
beforeCheckpointId: state.ctx.inMemoryStore.committedCheckpointId,
|
|
1176
1149
|
targetCheckpointId: rollbackTargetCheckpointId
|
|
1177
1150
|
});
|
|
1178
1151
|
Prometheus.RollbackSuccess.increment(Hrtime.toSecondsFloat(Hrtime.timeSince(startTime)), rollbackedProcessedEvents);
|
|
1179
1152
|
return dispatchAction({
|
|
1180
1153
|
TAG: "SetRollbackState",
|
|
1181
|
-
diffInMemoryStore: diff$1.inMemStore,
|
|
1182
1154
|
rollbackedChainManager: chainManager,
|
|
1183
1155
|
eventsProcessedDiffByChain: eventsProcessedDiffByChain
|
|
1184
1156
|
});
|
package/src/HandlerLoader.res
CHANGED
|
@@ -114,20 +114,21 @@ let applyRegistrations = (~config: Config.t): Config.t => {
|
|
|
114
114
|
~contractName=ev.contractName,
|
|
115
115
|
~eventName=ev.name,
|
|
116
116
|
)
|
|
117
|
+
let indexedParams = evmEv.paramsMetadata->Array.filter(p => p.indexed)
|
|
117
118
|
let {getEventFiltersOrThrow, filterByAddresses} = LogSelection.parseEventFiltersOrThrow(
|
|
118
119
|
~eventFilters,
|
|
119
120
|
~sighash=evmEv.sighash,
|
|
120
|
-
~params=
|
|
121
|
+
~params=indexedParams->Array.map(p => p.name),
|
|
121
122
|
~contractName=ev.contractName,
|
|
122
123
|
~probeChainId=chain.id,
|
|
123
124
|
~onEventBlockFilterSchema=config.ecosystem.onEventBlockFilterSchema,
|
|
124
|
-
~topic1=?
|
|
125
|
+
~topic1=?indexedParams
|
|
125
126
|
->Array.get(0)
|
|
126
127
|
->Option.map(EventConfigBuilder.buildTopicGetter),
|
|
127
|
-
~topic2=?
|
|
128
|
+
~topic2=?indexedParams
|
|
128
129
|
->Array.get(1)
|
|
129
130
|
->Option.map(EventConfigBuilder.buildTopicGetter),
|
|
130
|
-
~topic3=?
|
|
131
|
+
~topic3=?indexedParams
|
|
131
132
|
->Array.get(2)
|
|
132
133
|
->Option.map(EventConfigBuilder.buildTopicGetter),
|
|
133
134
|
)
|
|
@@ -156,7 +157,7 @@ let applyRegistrations = (~config: Config.t): Config.t => {
|
|
|
156
157
|
// `Config` never reads `HandlerRegister`. The only way to get a config that
|
|
157
158
|
// reflects registration state is through the returned value here.
|
|
158
159
|
let registerAllHandlers = async (~config: Config.t) => {
|
|
159
|
-
HandlerRegister.startRegistration(~ecosystem=config.ecosystem
|
|
160
|
+
HandlerRegister.startRegistration(~ecosystem=config.ecosystem)
|
|
160
161
|
|
|
161
162
|
// Auto-load all .js files from src/handlers directory
|
|
162
163
|
await autoLoadFromSrcHandlers(~handlers=config.handlers)
|
|
@@ -72,7 +72,8 @@ function applyRegistrations(config) {
|
|
|
72
72
|
switch (match) {
|
|
73
73
|
case "evm" :
|
|
74
74
|
let eventFilters = HandlerRegister.getOnEventWhere(ev.contractName, ev.name);
|
|
75
|
-
let
|
|
75
|
+
let indexedParams = ev.paramsMetadata.filter(p => p.indexed);
|
|
76
|
+
let match$1 = LogSelection.parseEventFiltersOrThrow(eventFilters, ev.sighash, indexedParams.map(p => p.name), ev.contractName, chain.id, config.ecosystem.onEventBlockFilterSchema, Stdlib_Option.map(indexedParams[0], EventConfigBuilder.buildTopicGetter), Stdlib_Option.map(indexedParams[1], EventConfigBuilder.buildTopicGetter), Stdlib_Option.map(indexedParams[2], EventConfigBuilder.buildTopicGetter));
|
|
76
77
|
let filterByAddresses = match$1.filterByAddresses;
|
|
77
78
|
return {
|
|
78
79
|
id: ev.id,
|
|
@@ -87,11 +88,11 @@ function applyRegistrations(config) {
|
|
|
87
88
|
simulateParamsSchema: ev.simulateParamsSchema,
|
|
88
89
|
startBlock: ev.startBlock,
|
|
89
90
|
getEventFiltersOrThrow: match$1.getEventFiltersOrThrow,
|
|
90
|
-
convertHyperSyncEventArgs: ev.convertHyperSyncEventArgs,
|
|
91
91
|
selectedBlockFields: ev.selectedBlockFields,
|
|
92
92
|
selectedTransactionFields: ev.selectedTransactionFields,
|
|
93
93
|
sighash: ev.sighash,
|
|
94
|
-
|
|
94
|
+
topicCount: ev.topicCount,
|
|
95
|
+
paramsMetadata: ev.paramsMetadata
|
|
95
96
|
};
|
|
96
97
|
case "fuel" :
|
|
97
98
|
return {
|
|
@@ -117,21 +118,38 @@ function applyRegistrations(config) {
|
|
|
117
118
|
abi: contract.abi,
|
|
118
119
|
addresses: contract.addresses,
|
|
119
120
|
events: newEvents,
|
|
120
|
-
startBlock: contract.startBlock
|
|
121
|
-
eventSignatures: contract.eventSignatures
|
|
121
|
+
startBlock: contract.startBlock
|
|
122
122
|
};
|
|
123
123
|
});
|
|
124
124
|
let newrecord = {...chain};
|
|
125
125
|
newrecord.contracts = newContracts;
|
|
126
126
|
return newrecord;
|
|
127
127
|
});
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
128
|
+
return {
|
|
129
|
+
name: config.name,
|
|
130
|
+
description: config.description,
|
|
131
|
+
handlers: config.handlers,
|
|
132
|
+
contractHandlers: config.contractHandlers,
|
|
133
|
+
shouldRollbackOnReorg: config.shouldRollbackOnReorg,
|
|
134
|
+
shouldSaveFullHistory: config.shouldSaveFullHistory,
|
|
135
|
+
storage: config.storage,
|
|
136
|
+
chainMap: newChainMap,
|
|
137
|
+
defaultChain: config.defaultChain,
|
|
138
|
+
ecosystem: config.ecosystem,
|
|
139
|
+
enableRawEvents: config.enableRawEvents,
|
|
140
|
+
maxAddrInPartition: config.maxAddrInPartition,
|
|
141
|
+
batchSize: config.batchSize,
|
|
142
|
+
lowercaseAddresses: config.lowercaseAddresses,
|
|
143
|
+
isDev: config.isDev,
|
|
144
|
+
userEntitiesByName: config.userEntitiesByName,
|
|
145
|
+
userEntities: config.userEntities,
|
|
146
|
+
allEntities: config.allEntities,
|
|
147
|
+
allEnums: config.allEnums
|
|
148
|
+
};
|
|
131
149
|
}
|
|
132
150
|
|
|
133
151
|
async function registerAllHandlers(config) {
|
|
134
|
-
HandlerRegister.startRegistration(config.ecosystem
|
|
152
|
+
HandlerRegister.startRegistration(config.ecosystem);
|
|
135
153
|
await autoLoadFromSrcHandlers(config.handlers);
|
|
136
154
|
await Promise.all(config.contractHandlers.map(param => registerContractHandlers(param.name, param.handler)));
|
|
137
155
|
let registrations = HandlerRegister.finishRegistration();
|
package/src/HandlerRegister.res
CHANGED
|
@@ -14,7 +14,6 @@ type registrations = {onBlockByChainId: dict<array<Internal.onBlockConfig>>}
|
|
|
14
14
|
|
|
15
15
|
type activeRegistration = {
|
|
16
16
|
ecosystem: Ecosystem.t,
|
|
17
|
-
multichain: Internal.multichain,
|
|
18
17
|
registrations: registrations,
|
|
19
18
|
mutable finished: bool,
|
|
20
19
|
}
|
|
@@ -102,10 +101,9 @@ let withRegistration = (fn: activeRegistration => unit) => {
|
|
|
102
101
|
}
|
|
103
102
|
}
|
|
104
103
|
|
|
105
|
-
let startRegistration = (~ecosystem
|
|
104
|
+
let startRegistration = (~ecosystem) => {
|
|
106
105
|
let r = {
|
|
107
106
|
ecosystem,
|
|
108
|
-
multichain,
|
|
109
107
|
registrations: {
|
|
110
108
|
onBlockByChainId: Dict.make(),
|
|
111
109
|
},
|
|
@@ -163,15 +161,6 @@ let registerOnBlock = (
|
|
|
163
161
|
~handler: Internal.onBlockArgs => promise<unit>,
|
|
164
162
|
) => {
|
|
165
163
|
withRegistration(registration => {
|
|
166
|
-
// We need to get timestamp for ordered multichain mode
|
|
167
|
-
switch registration.multichain {
|
|
168
|
-
| Unordered => ()
|
|
169
|
-
| Ordered =>
|
|
170
|
-
JsError.throwWithMessage(
|
|
171
|
-
"Block Handlers are not supported for ordered multichain mode. Please reach out to the Envio team if you need this feature. Or enable unordered multichain mode by removing `multichain: ordered` from the config.yaml file.",
|
|
172
|
-
)
|
|
173
|
-
}
|
|
174
|
-
|
|
175
164
|
let onBlockByChainId = registration.registrations.onBlockByChainId
|
|
176
165
|
let key = chainId->Belt.Int.toString
|
|
177
166
|
let index =
|
|
@@ -71,10 +71,9 @@ function withRegistration(fn) {
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
-
function startRegistration(ecosystem
|
|
74
|
+
function startRegistration(ecosystem) {
|
|
75
75
|
let r = {
|
|
76
76
|
ecosystem: ecosystem,
|
|
77
|
-
multichain: multichain,
|
|
78
77
|
registrations: {
|
|
79
78
|
onBlockByChainId: {}
|
|
80
79
|
},
|
|
@@ -121,10 +120,6 @@ function throwIfFinishedRegistration(methodName) {
|
|
|
121
120
|
|
|
122
121
|
function registerOnBlock(name, chainId, interval, startBlock, endBlock, handler) {
|
|
123
122
|
withRegistration(registration => {
|
|
124
|
-
let match = registration.multichain;
|
|
125
|
-
if (match === "ordered") {
|
|
126
|
-
Stdlib_JsError.throwWithMessage("Block Handlers are not supported for ordered multichain mode. Please reach out to the Envio team if you need this feature. Or enable unordered multichain mode by removing `multichain: ordered` from the config.yaml file.");
|
|
127
|
-
}
|
|
128
123
|
let onBlockByChainId = registration.registrations.onBlockByChainId;
|
|
129
124
|
let key = String(chainId);
|
|
130
125
|
let index = Belt_Option.mapWithDefault(onBlockByChainId[key], 0, configs => configs.length);
|
package/src/HandlerRegister.resi
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
type registrations = {onBlockByChainId: dict<array<Internal.onBlockConfig>>}
|
|
2
2
|
|
|
3
|
-
let startRegistration: (~ecosystem: Ecosystem.t
|
|
3
|
+
let startRegistration: (~ecosystem: Ecosystem.t) => unit
|
|
4
4
|
let isPendingRegistration: unit => bool
|
|
5
5
|
let finishRegistration: unit => registrations
|
|
6
6
|
let throwIfFinishedRegistration: (~methodName: string) => unit
|