envio 3.5.0-alpha.3 → 3.5.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/Batch.res +8 -8
- package/src/Batch.res.mjs +4 -3
- package/src/BatchProcessing.res +17 -7
- package/src/BatchProcessing.res.mjs +8 -4
- package/src/ChainFetching.res +20 -19
- package/src/ChainFetching.res.mjs +14 -14
- package/src/ChainId.res +58 -0
- package/src/ChainId.res.mjs +49 -0
- package/src/ChainId.resi +27 -0
- package/src/ChainMap.res +12 -26
- package/src/ChainMap.res.mjs +5 -41
- package/src/ChainMap.resi +5 -20
- package/src/ChainMetadata.res +1 -1
- package/src/ChainMetadata.res.mjs +3 -2
- package/src/ChainState.res +35 -20
- package/src/ChainState.res.mjs +34 -11
- package/src/ChainState.resi +2 -0
- package/src/Config.res +41 -24
- package/src/Config.res.mjs +17 -14
- package/src/ContractRegisterContext.res +1 -1
- package/src/ContractRegisterContext.res.mjs +1 -1
- package/src/Core.res +1 -0
- package/src/CrossChainState.res +61 -16
- package/src/CrossChainState.res.mjs +37 -13
- package/src/CrossChainState.resi +3 -2
- package/src/Env.res +0 -3
- package/src/Env.res.mjs +0 -3
- package/src/EventConfigBuilder.res +1 -1
- package/src/EventProcessing.res +4 -4
- package/src/EventProcessing.res.mjs +5 -5
- package/src/FetchState.res +261 -89
- package/src/FetchState.res.mjs +187 -63
- package/src/FinalizeBackfill.res +33 -6
- package/src/FinalizeBackfill.res.mjs +19 -1
- package/src/HandlerRegister.res +8 -8
- package/src/HandlerRegister.res.mjs +5 -4
- package/src/HandlerRegister.resi +1 -1
- package/src/InMemoryStore.res +4 -0
- package/src/InMemoryStore.res.mjs +1 -1
- package/src/IndexerLoop.res +9 -2
- package/src/IndexerLoop.res.mjs +8 -1
- package/src/IndexerState.res +41 -24
- package/src/IndexerState.res.mjs +36 -17
- package/src/IndexerState.resi +9 -7
- package/src/Internal.res +19 -17
- package/src/Internal.res.mjs +18 -9
- package/src/LoadLayer.res +1 -1
- package/src/LoadLayer.res.mjs +2 -1
- package/src/LogSelection.res +2 -2
- package/src/Main.res +7 -11
- package/src/Main.res.mjs +4 -6
- package/src/Metrics.res +6 -6
- package/src/Metrics.res.mjs +4 -3
- package/src/Persistence.res +15 -7
- package/src/PgStorage.res +284 -258
- package/src/PgStorage.res.mjs +181 -208
- package/src/RawEvent.res +1 -2
- package/src/RawEvent.res.mjs +1 -1
- package/src/Rollback.res +18 -20
- package/src/Rollback.res.mjs +9 -9
- package/src/RollbackCommit.res +2 -2
- package/src/RollbackCommit.res.mjs +3 -3
- package/src/SafeCheckpointTracking.res +2 -2
- package/src/SimulateDeadInputTracker.res +17 -21
- package/src/SimulateDeadInputTracker.res.mjs +9 -8
- package/src/SimulateItems.res +8 -10
- package/src/SimulateItems.res.mjs +9 -10
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +3 -2
- package/src/TestIndexer.res +11 -14
- package/src/TestIndexer.res.mjs +14 -10
- package/src/UserContext.res +2 -2
- package/src/UserContext.res.mjs +2 -1
- package/src/Utils.res +0 -4
- package/src/Utils.res.mjs +1 -6
- package/src/bindings/ClickHouse.res +20 -2
- package/src/bindings/ClickHouse.res.mjs +25 -14
- package/src/db/EntityHistory.res +1 -1
- package/src/db/IndexCatalog.res +248 -0
- package/src/db/IndexCatalog.res.mjs +222 -0
- package/src/db/IndexDefinition.res +131 -0
- package/src/db/IndexDefinition.res.mjs +138 -0
- package/src/db/IndexManager.res +145 -0
- package/src/db/IndexManager.res.mjs +130 -0
- package/src/db/InternalTable.res +50 -26
- package/src/db/InternalTable.res.mjs +32 -24
- package/src/db/Table.res +11 -1
- package/src/db/Table.res.mjs +9 -4
- package/src/sources/AddressSet.res +5 -4
- package/src/sources/AddressStore.res +25 -14
- package/src/sources/AddressStore.res.mjs +10 -12
- package/src/sources/Evm.res +2 -2
- package/src/sources/Evm.res.mjs +1 -1
- package/src/sources/EvmChain.res +3 -3
- package/src/sources/EvmChain.res.mjs +3 -3
- package/src/sources/EvmHyperSyncSource.res +7 -7
- package/src/sources/EvmHyperSyncSource.res.mjs +6 -6
- package/src/sources/Fuel.res +2 -2
- package/src/sources/Fuel.res.mjs +1 -1
- package/src/sources/FuelHyperSyncClient.res +3 -0
- package/src/sources/FuelHyperSyncClient.res.mjs +1 -0
- package/src/sources/FuelHyperSyncSource.res +5 -7
- package/src/sources/FuelHyperSyncSource.res.mjs +5 -5
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +5 -0
- package/src/sources/HyperSyncClient.res.mjs +1 -0
- package/src/sources/RpcSource.res +13 -14
- package/src/sources/RpcSource.res.mjs +13 -12
- package/src/sources/SimulateSource.res +42 -15
- package/src/sources/SimulateSource.res.mjs +27 -9
- package/src/sources/Source.res +1 -5
- package/src/sources/SourceManager.res +12 -12
- package/src/sources/SourceManager.res.mjs +11 -10
- package/src/sources/SourceManager.resi +2 -2
- package/src/sources/Svm.res +4 -5
- package/src/sources/Svm.res.mjs +5 -4
- package/src/sources/SvmHyperSyncClient.res +3 -0
- package/src/sources/SvmHyperSyncClient.res.mjs +1 -0
- package/src/sources/SvmHyperSyncSource.res +4 -4
- package/src/sources/SvmHyperSyncSource.res.mjs +3 -3
- package/src/tui/Tui.res +1 -1
- package/src/tui/Tui.res.mjs +2 -1
- package/src/tui/components/CustomHooks.res +5 -4
- package/src/tui/components/CustomHooks.res.mjs +4 -2
- package/src/db/IndexRegistry.res +0 -219
- package/src/db/IndexRegistry.res.mjs +0 -195
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.5.0-
|
|
3
|
+
"version": "3.5.0-rc.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -69,10 +69,10 @@
|
|
|
69
69
|
"tsx": "4.21.0"
|
|
70
70
|
},
|
|
71
71
|
"optionalDependencies": {
|
|
72
|
-
"envio-linux-x64": "3.5.0-
|
|
73
|
-
"envio-linux-x64-musl": "3.5.0-
|
|
74
|
-
"envio-linux-arm64": "3.5.0-
|
|
75
|
-
"envio-darwin-x64": "3.5.0-
|
|
76
|
-
"envio-darwin-arm64": "3.5.0-
|
|
72
|
+
"envio-linux-x64": "3.5.0-rc.0",
|
|
73
|
+
"envio-linux-x64-musl": "3.5.0-rc.0",
|
|
74
|
+
"envio-linux-arm64": "3.5.0-rc.0",
|
|
75
|
+
"envio-darwin-x64": "3.5.0-rc.0",
|
|
76
|
+
"envio-darwin-arm64": "3.5.0-rc.0"
|
|
77
77
|
}
|
|
78
78
|
}
|
package/src/Batch.res
CHANGED
|
@@ -28,7 +28,7 @@ type t = {
|
|
|
28
28
|
isInReorgThreshold: bool,
|
|
29
29
|
// Unnest-like checkpoint fields:
|
|
30
30
|
checkpointIds: array<bigint>,
|
|
31
|
-
checkpointChainIds: array<
|
|
31
|
+
checkpointChainIds: array<ChainId.t>,
|
|
32
32
|
checkpointBlockNumbers: array<int>,
|
|
33
33
|
checkpointBlockHashes: array<Null.t<string>>,
|
|
34
34
|
checkpointEventsProcessed: array<int>,
|
|
@@ -80,14 +80,14 @@ let getProgressedChainsById = {
|
|
|
80
80
|
let fetchState = chainBeforeBatch.fetchState
|
|
81
81
|
|
|
82
82
|
let progressBlockNumberAfterBatch = switch progressBlockNumberPerChain->Utils.Dict.dangerouslyGetNonOption(
|
|
83
|
-
fetchState.chainId->
|
|
83
|
+
fetchState.chainId->ChainId.toString,
|
|
84
84
|
) {
|
|
85
85
|
| Some(progressBlockNumber) => progressBlockNumber
|
|
86
86
|
| None => chainBeforeBatch.progressBlockNumber
|
|
87
87
|
}
|
|
88
88
|
|
|
89
89
|
switch switch batchSizePerChain->Utils.Dict.dangerouslyGetNonOption(
|
|
90
|
-
fetchState.chainId->
|
|
90
|
+
fetchState.chainId->ChainId.toString,
|
|
91
91
|
) {
|
|
92
92
|
| Some(batchSize) =>
|
|
93
93
|
let leftItems = fetchState.buffer->Array.slice(~start=batchSize)
|
|
@@ -107,7 +107,7 @@ let getProgressedChainsById = {
|
|
|
107
107
|
)
|
|
108
108
|
} {
|
|
109
109
|
| Some(progressedChain) =>
|
|
110
|
-
progressedChainsById->
|
|
110
|
+
progressedChainsById->ChainId.Dict.set(
|
|
111
111
|
chainBeforeBatch.fetchState.chainId,
|
|
112
112
|
progressedChain,
|
|
113
113
|
)
|
|
@@ -195,7 +195,7 @@ let prepareBatch = (
|
|
|
195
195
|
)
|
|
196
196
|
let chainBeforeBatch =
|
|
197
197
|
chainsBeforeBatch
|
|
198
|
-
->
|
|
198
|
+
->ChainId.Dict.dangerouslyGetNonOption(fetchState.chainId)
|
|
199
199
|
->Option.getUnsafe
|
|
200
200
|
|
|
201
201
|
let prevBlockNumber = ref(chainBeforeBatch.progressBlockNumber)
|
|
@@ -245,7 +245,7 @@ let prepareBatch = (
|
|
|
245
245
|
}
|
|
246
246
|
|
|
247
247
|
totalBatchSize := totalBatchSize.contents + chainBatchSize
|
|
248
|
-
mutBatchSizePerChain->
|
|
248
|
+
mutBatchSizePerChain->ChainId.Dict.set(fetchState.chainId, chainBatchSize)
|
|
249
249
|
}
|
|
250
250
|
|
|
251
251
|
let progressBlockNumberAfterBatch =
|
|
@@ -265,7 +265,7 @@ let prepareBatch = (
|
|
|
265
265
|
~mutCheckpointEventsProcessed=checkpointEventsProcessed,
|
|
266
266
|
)
|
|
267
267
|
|
|
268
|
-
mutProgressBlockNumberPerChain->
|
|
268
|
+
mutProgressBlockNumberPerChain->ChainId.Dict.set(
|
|
269
269
|
fetchState.chainId,
|
|
270
270
|
progressBlockNumberAfterBatch,
|
|
271
271
|
)
|
|
@@ -325,7 +325,7 @@ let findLastEventItem = (batch: t, ~chainId) => {
|
|
|
325
325
|
switch item {
|
|
326
326
|
| Internal.Event(_) as eventItem => {
|
|
327
327
|
let eventItem = eventItem->Internal.castUnsafeEventItem
|
|
328
|
-
if eventItem.
|
|
328
|
+
if eventItem.chainId === chainId {
|
|
329
329
|
result := Some(eventItem)
|
|
330
330
|
} else {
|
|
331
331
|
idx := idx.contents - 1
|
package/src/Batch.res.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
|
+
import * as ChainId from "./ChainId.res.mjs";
|
|
4
5
|
import * as FetchState from "./FetchState.res.mjs";
|
|
5
6
|
import * as ReorgDetection from "./ReorgDetection.res.mjs";
|
|
6
7
|
|
|
@@ -21,9 +22,9 @@ function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressB
|
|
|
21
22
|
let progressedChainsById = {};
|
|
22
23
|
Object.values(chainsBeforeBatch).forEach(chainBeforeBatch => {
|
|
23
24
|
let fetchState = chainBeforeBatch.fetchState;
|
|
24
|
-
let progressBlockNumber = progressBlockNumberPerChain[fetchState.chainId
|
|
25
|
+
let progressBlockNumber = progressBlockNumberPerChain[ChainId.toString(fetchState.chainId)];
|
|
25
26
|
let progressBlockNumberAfterBatch = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
|
|
26
|
-
let batchSize = batchSizePerChain[fetchState.chainId
|
|
27
|
+
let batchSize = batchSizePerChain[ChainId.toString(fetchState.chainId)];
|
|
27
28
|
let progressedChain;
|
|
28
29
|
if (batchSize !== undefined) {
|
|
29
30
|
let leftItems = fetchState.buffer.slice(batchSize);
|
|
@@ -188,7 +189,7 @@ function findLastEventItem(batch, chainId) {
|
|
|
188
189
|
let result;
|
|
189
190
|
while (idx >= 0 && result === undefined) {
|
|
190
191
|
let item = batch.items[idx];
|
|
191
|
-
if (item.kind === 0 && item.
|
|
192
|
+
if (item.kind === 0 && item.chainId === chainId) {
|
|
192
193
|
result = item;
|
|
193
194
|
} else {
|
|
194
195
|
idx = idx - 1;
|
package/src/BatchProcessing.res
CHANGED
|
@@ -101,16 +101,26 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
101
101
|
scheduleFetch()
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
// Nothing progressed, but a backfill that reached the head and died before
|
|
105
|
+
// finalizing resumes exactly here: it still owes the schema its deferred
|
|
106
|
+
// indexes, and no batch will ever come along to notice.
|
|
107
|
+
state->IndexerState.markCaughtUpIfSettled
|
|
108
|
+
if state->IndexerState.isFinalizingIndexes {
|
|
109
|
+
await FinalizeBackfill.run(state)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Same realtime handoff the progressed-batch path does below. IndexerLoop
|
|
113
|
+
// starts fetching before processing, so by now the chain is already parked
|
|
114
|
+
// on a pre-realtime waiter; without this it stays there, polling the sync
|
|
115
|
+
// source at the backfill interval, until that source reports a new height.
|
|
116
|
+
if !isRealtimeBeforeUpdate && state->IndexerState.isRealtime {
|
|
117
|
+
state->IndexerState.invalidateInflight
|
|
118
|
+
scheduleFetch()
|
|
119
|
+
}
|
|
120
|
+
|
|
104
121
|
// When resuming from persisted state, all events may already be processed.
|
|
105
122
|
if EventProcessing.allChainsEventsProcessedToEndblock(state->IndexerState.chainStates) {
|
|
106
123
|
Logging.info("All chains are caught up to end blocks.")
|
|
107
|
-
// A run that has nothing left to process still owes the schema its
|
|
108
|
-
// deferred indexes — otherwise a resume of a backfill that died right
|
|
109
|
-
// before finalizing would report ready without them.
|
|
110
|
-
state->IndexerState.markCaughtUp
|
|
111
|
-
if state->IndexerState.isFinalizingIndexes {
|
|
112
|
-
await FinalizeBackfill.run(state)
|
|
113
|
-
}
|
|
114
124
|
if !(state->IndexerState.keepProcessAlive) {
|
|
115
125
|
await ExitOnCaughtUp.run(state)
|
|
116
126
|
}
|
|
@@ -37,12 +37,16 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
37
37
|
if (shouldEnterReorgThreshold) {
|
|
38
38
|
scheduleFetch();
|
|
39
39
|
}
|
|
40
|
+
IndexerState.markCaughtUpIfSettled(state);
|
|
41
|
+
if (IndexerState.isFinalizingIndexes(state)) {
|
|
42
|
+
await FinalizeBackfill.run(state);
|
|
43
|
+
}
|
|
44
|
+
if (!isRealtimeBeforeUpdate && IndexerState.isRealtime(state)) {
|
|
45
|
+
IndexerState.invalidateInflight(state);
|
|
46
|
+
scheduleFetch();
|
|
47
|
+
}
|
|
40
48
|
if (EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state))) {
|
|
41
49
|
Logging.info("All chains are caught up to end blocks.");
|
|
42
|
-
IndexerState.markCaughtUp(state);
|
|
43
|
-
if (IndexerState.isFinalizingIndexes(state)) {
|
|
44
|
-
await FinalizeBackfill.run(state);
|
|
45
|
-
}
|
|
46
50
|
if (!IndexerState.keepProcessAlive(state)) {
|
|
47
51
|
return await ExitOnCaughtUp.run(state);
|
|
48
52
|
} else {
|
package/src/ChainFetching.res
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// (state + transitions) and leaf effect modules.
|
|
4
4
|
|
|
5
5
|
type partitionQueryResponse = {
|
|
6
|
-
|
|
6
|
+
chainId: ChainId.t,
|
|
7
7
|
response: Source.blockRangeFetchResponse,
|
|
8
8
|
query: FetchState.query,
|
|
9
9
|
}
|
|
@@ -106,7 +106,7 @@ let runContractRegistersOrThrow = async (
|
|
|
106
106
|
|
|
107
107
|
let rec onQueryResponse = async (
|
|
108
108
|
state: IndexerState.t,
|
|
109
|
-
{
|
|
109
|
+
{chainId, response, query}: partitionQueryResponse,
|
|
110
110
|
~stateId,
|
|
111
111
|
~scheduleFetch,
|
|
112
112
|
~scheduleProcessing,
|
|
@@ -115,7 +115,7 @@ let rec onQueryResponse = async (
|
|
|
115
115
|
if state->IndexerState.isStale(~stateId) {
|
|
116
116
|
()
|
|
117
117
|
} else {
|
|
118
|
-
let chainState = state->IndexerState.getChainState(~
|
|
118
|
+
let chainState = state->IndexerState.getChainState(~chainId)
|
|
119
119
|
let {
|
|
120
120
|
parsedQueueItems,
|
|
121
121
|
transactionStore,
|
|
@@ -142,7 +142,7 @@ let rec onQueryResponse = async (
|
|
|
142
142
|
if numContractRegisterEvents === 0 {
|
|
143
143
|
Logging.trace({
|
|
144
144
|
"msg": "Finished querying",
|
|
145
|
-
"chainId":
|
|
145
|
+
"chainId": chainId,
|
|
146
146
|
"partitionId": query.partitionId,
|
|
147
147
|
"fromBlock": fromBlockQueried,
|
|
148
148
|
"toBlock": latestFetchedBlockNumber,
|
|
@@ -151,7 +151,7 @@ let rec onQueryResponse = async (
|
|
|
151
151
|
} else {
|
|
152
152
|
Logging.trace({
|
|
153
153
|
"msg": "Finished querying",
|
|
154
|
-
"chainId":
|
|
154
|
+
"chainId": chainId,
|
|
155
155
|
"partitionId": query.partitionId,
|
|
156
156
|
"fromBlock": fromBlockQueried,
|
|
157
157
|
"toBlock": latestFetchedBlockNumber,
|
|
@@ -199,20 +199,21 @@ let rec onQueryResponse = async (
|
|
|
199
199
|
cs->ChainState.prepareReorg(
|
|
200
200
|
~eventsProcessedDiff=switch eventsProcessedDiffByChain {
|
|
201
201
|
| Some(byChain) =>
|
|
202
|
-
byChain->
|
|
202
|
+
byChain->ChainId.Dict.dangerouslyGetNonOption((cs->ChainState.chainConfig).id)
|
|
203
203
|
| None => None
|
|
204
204
|
},
|
|
205
205
|
)
|
|
206
206
|
)
|
|
207
|
-
state->IndexerState.beginReorg(~
|
|
207
|
+
state->IndexerState.beginReorg(~chainId, ~blockNumber=reorgDetectedBlockNumber)
|
|
208
208
|
// Advances synchronously to FindingReorgDepth, so a concurrent rollback
|
|
209
209
|
// kick (eg from the processing loop quiescing) collapses into this one.
|
|
210
210
|
scheduleRollback()
|
|
211
211
|
| None =>
|
|
212
212
|
// Over-fetched events (a merged partition returning an address before its
|
|
213
|
-
// effectiveStartBlock,
|
|
214
|
-
//
|
|
215
|
-
//
|
|
213
|
+
// effectiveStartBlock, a wildcard param referencing an address registered
|
|
214
|
+
// after the log's block, or a registration whose own start block is later
|
|
215
|
+
// than its contract's) are already dropped by the source's native gates,
|
|
216
|
+
// so everything here is indexable.
|
|
216
217
|
let newItems = parsedQueueItems
|
|
217
218
|
let itemsWithContractRegister = []
|
|
218
219
|
for idx in 0 to newItems->Array.length - 1 {
|
|
@@ -229,7 +230,7 @@ let rec onQueryResponse = async (
|
|
|
229
230
|
if !(state->IndexerState.isStale(~stateId)) {
|
|
230
231
|
applyQueryResponse(
|
|
231
232
|
state,
|
|
232
|
-
~
|
|
233
|
+
~chainId,
|
|
233
234
|
~newItems,
|
|
234
235
|
~newItemsWithDcs,
|
|
235
236
|
~knownHeight,
|
|
@@ -264,7 +265,7 @@ let rec onQueryResponse = async (
|
|
|
264
265
|
|
|
265
266
|
and applyQueryResponse = (
|
|
266
267
|
state: IndexerState.t,
|
|
267
|
-
~
|
|
268
|
+
~chainId,
|
|
268
269
|
~newItems,
|
|
269
270
|
~newItemsWithDcs,
|
|
270
271
|
~knownHeight,
|
|
@@ -273,7 +274,7 @@ and applyQueryResponse = (
|
|
|
273
274
|
~transactionStore,
|
|
274
275
|
~blockStore,
|
|
275
276
|
) => {
|
|
276
|
-
let chainState = state->IndexerState.getChainState(~
|
|
277
|
+
let chainState = state->IndexerState.getChainState(~chainId)
|
|
277
278
|
let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
|
|
278
279
|
|
|
279
280
|
chainState->ChainState.handleQueryResult(
|
|
@@ -307,7 +308,7 @@ and applyQueryResponse = (
|
|
|
307
308
|
|
|
308
309
|
let finishWaitingForNewBlock = (
|
|
309
310
|
state: IndexerState.t,
|
|
310
|
-
~
|
|
311
|
+
~chainId,
|
|
311
312
|
~knownHeight,
|
|
312
313
|
~stateId,
|
|
313
314
|
~scheduleFetch,
|
|
@@ -316,7 +317,7 @@ let finishWaitingForNewBlock = (
|
|
|
316
317
|
if state->IndexerState.isStale(~stateId) {
|
|
317
318
|
()
|
|
318
319
|
} else {
|
|
319
|
-
let chainState = state->IndexerState.getChainState(~
|
|
320
|
+
let chainState = state->IndexerState.getChainState(~chainId)
|
|
320
321
|
chainState->ChainState.updateKnownHeight(~knownHeight)
|
|
321
322
|
|
|
322
323
|
// No reorg-threshold check here: scheduleProcessing always runs at least one
|
|
@@ -327,14 +328,14 @@ let finishWaitingForNewBlock = (
|
|
|
327
328
|
|
|
328
329
|
let fetchChain = async (
|
|
329
330
|
state: IndexerState.t,
|
|
330
|
-
|
|
331
|
+
chainId,
|
|
331
332
|
~action,
|
|
332
333
|
~stateId,
|
|
333
334
|
~scheduleFetch,
|
|
334
335
|
~scheduleProcessing,
|
|
335
336
|
~scheduleRollback,
|
|
336
337
|
) => {
|
|
337
|
-
let chainState = state->IndexerState.getChainState(~
|
|
338
|
+
let chainState = state->IndexerState.getChainState(~chainId)
|
|
338
339
|
if !(state->IndexerState.isResolvingReorg) && !(state->IndexerState.isStopped) {
|
|
339
340
|
let isRealtime = state->IndexerState.isRealtime
|
|
340
341
|
let sourceManager = chainState->ChainState.sourceManager
|
|
@@ -352,7 +353,7 @@ let fetchChain = async (
|
|
|
352
353
|
~onNewBlock=(~knownHeight) =>
|
|
353
354
|
finishWaitingForNewBlock(
|
|
354
355
|
state,
|
|
355
|
-
~
|
|
356
|
+
~chainId,
|
|
356
357
|
~knownHeight,
|
|
357
358
|
~stateId,
|
|
358
359
|
~scheduleFetch,
|
|
@@ -370,7 +371,7 @@ let fetchChain = async (
|
|
|
370
371
|
)
|
|
371
372
|
await onQueryResponse(
|
|
372
373
|
state,
|
|
373
|
-
{
|
|
374
|
+
{chainId, response, query},
|
|
374
375
|
~stateId,
|
|
375
376
|
~scheduleFetch,
|
|
376
377
|
~scheduleProcessing,
|
|
@@ -71,8 +71,8 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config, tr
|
|
|
71
71
|
return itemsWithDcs;
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
-
function applyQueryResponse(state,
|
|
75
|
-
let chainState = IndexerState.getChainState(state,
|
|
74
|
+
function applyQueryResponse(state, chainId, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, transactionStore, blockStore) {
|
|
75
|
+
let chainState = IndexerState.getChainState(state, chainId);
|
|
76
76
|
let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
|
|
77
77
|
ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, transactionStore, blockStore);
|
|
78
78
|
if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
|
|
@@ -86,11 +86,11 @@ function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight
|
|
|
86
86
|
async function onQueryResponse(state, param, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
|
|
87
87
|
let query = param.query;
|
|
88
88
|
let response = param.response;
|
|
89
|
-
let
|
|
89
|
+
let chainId = param.chainId;
|
|
90
90
|
if (IndexerState.isStale(state, stateId)) {
|
|
91
91
|
return;
|
|
92
92
|
}
|
|
93
|
-
let chainState = IndexerState.getChainState(state,
|
|
93
|
+
let chainState = IndexerState.getChainState(state, chainId);
|
|
94
94
|
let stats = response.stats;
|
|
95
95
|
let latestFetchedBlockTimestamp = response.latestFetchedBlockTimestamp;
|
|
96
96
|
let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
|
|
@@ -110,7 +110,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
110
110
|
if (numContractRegisterEvents === 0) {
|
|
111
111
|
Logging.trace({
|
|
112
112
|
msg: "Finished querying",
|
|
113
|
-
chainId:
|
|
113
|
+
chainId: chainId,
|
|
114
114
|
partitionId: query.partitionId,
|
|
115
115
|
fromBlock: fromBlockQueried,
|
|
116
116
|
toBlock: latestFetchedBlockNumber,
|
|
@@ -119,7 +119,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
119
119
|
} else {
|
|
120
120
|
Logging.trace({
|
|
121
121
|
msg: "Finished querying",
|
|
122
|
-
chainId:
|
|
122
|
+
chainId: chainId,
|
|
123
123
|
partitionId: query.partitionId,
|
|
124
124
|
fromBlock: fromBlockQueried,
|
|
125
125
|
toBlock: latestFetchedBlockNumber,
|
|
@@ -142,7 +142,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
142
142
|
let eventsProcessedDiffByChain;
|
|
143
143
|
eventsProcessedDiffByChain = typeof match !== "object" || match.TAG !== "RollbackReady" ? undefined : match.eventsProcessedDiffByChain;
|
|
144
144
|
Utils.Dict.forEach(IndexerState.chainStates(state), cs => ChainState.prepareReorg(cs, eventsProcessedDiffByChain !== undefined ? eventsProcessedDiffByChain[ChainState.chainConfig(cs).id] : undefined));
|
|
145
|
-
IndexerState.beginReorg(state,
|
|
145
|
+
IndexerState.beginReorg(state, chainId, rollbackWithReorgDetectedBlockNumber);
|
|
146
146
|
return scheduleRollback();
|
|
147
147
|
}
|
|
148
148
|
let itemsWithContractRegister = [];
|
|
@@ -154,7 +154,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
154
154
|
}
|
|
155
155
|
let proceed = newItemsWithDcs => {
|
|
156
156
|
if (!IndexerState.isStale(state, stateId)) {
|
|
157
|
-
applyQueryResponse(state,
|
|
157
|
+
applyQueryResponse(state, chainId, parsedQueueItems, newItemsWithDcs, knownHeight, {
|
|
158
158
|
blockNumber: latestFetchedBlockNumber,
|
|
159
159
|
blockTimestamp: latestFetchedBlockTimestamp
|
|
160
160
|
}, query, transactionStore, blockStore);
|
|
@@ -176,18 +176,18 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
176
176
|
return proceed(newItemsWithDcs);
|
|
177
177
|
}
|
|
178
178
|
|
|
179
|
-
function finishWaitingForNewBlock(state,
|
|
179
|
+
function finishWaitingForNewBlock(state, chainId, knownHeight, stateId, scheduleFetch, scheduleProcessing) {
|
|
180
180
|
if (IndexerState.isStale(state, stateId)) {
|
|
181
181
|
return;
|
|
182
182
|
}
|
|
183
|
-
let chainState = IndexerState.getChainState(state,
|
|
183
|
+
let chainState = IndexerState.getChainState(state, chainId);
|
|
184
184
|
ChainState.updateKnownHeight(chainState, knownHeight);
|
|
185
185
|
scheduleFetch();
|
|
186
186
|
scheduleProcessing();
|
|
187
187
|
}
|
|
188
188
|
|
|
189
|
-
async function fetchChain(state,
|
|
190
|
-
let chainState = IndexerState.getChainState(state,
|
|
189
|
+
async function fetchChain(state, chainId, action, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
|
|
190
|
+
let chainState = IndexerState.getChainState(state, chainId);
|
|
191
191
|
if (IndexerState.isResolvingReorg(state) || IndexerState.isStopped(state)) {
|
|
192
192
|
return;
|
|
193
193
|
}
|
|
@@ -199,7 +199,7 @@ async function fetchChain(state, chain, action, stateId, scheduleFetch, schedule
|
|
|
199
199
|
try {
|
|
200
200
|
let response = await SourceManager.executeQuery(sourceManager, query, ChainState.knownHeight(chainState), isRealtime);
|
|
201
201
|
return await onQueryResponse(state, {
|
|
202
|
-
|
|
202
|
+
chainId: chainId,
|
|
203
203
|
response: response,
|
|
204
204
|
query: query
|
|
205
205
|
}, stateId, scheduleFetch, scheduleProcessing, scheduleRollback);
|
|
@@ -207,7 +207,7 @@ async function fetchChain(state, chain, action, stateId, scheduleFetch, schedule
|
|
|
207
207
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
208
208
|
return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
|
|
209
209
|
}
|
|
210
|
-
}, knownHeight => SourceManager.waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling), knownHeight => finishWaitingForNewBlock(state,
|
|
210
|
+
}, knownHeight => SourceManager.waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling), knownHeight => finishWaitingForNewBlock(state, chainId, knownHeight, stateId, scheduleFetch, scheduleProcessing), action, stateId);
|
|
211
211
|
} catch (raw_exn) {
|
|
212
212
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
213
213
|
return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, IndexerState.unexpectedErrorMsg));
|
package/src/ChainId.res
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Chain ids are identifiers, never arithmetic operands, so the runtime
|
|
2
|
+
// representation is a plain JS number. ReScript's `int` would cap them at
|
|
3
|
+
// 2^31-1, which networks like Tron Shasta (2494104990) already exceed.
|
|
4
|
+
type t = float
|
|
5
|
+
|
|
6
|
+
// Widest scalar the internal chain-id columns need. Resolved by the CLI from
|
|
7
|
+
// the maximum active chain id and carried through the public config, so a
|
|
8
|
+
// resume against a schema built for the other mode is rejected instead of
|
|
9
|
+
// silently truncating ids.
|
|
10
|
+
type mode = | @as("int32") Int32 | @as("int64") Int64
|
|
11
|
+
|
|
12
|
+
let modeSchema = S.enum([Int32, Int64])
|
|
13
|
+
|
|
14
|
+
// Number.MAX_SAFE_INTEGER — above this a chain id can't round-trip through
|
|
15
|
+
// JSON or a JS number at all.
|
|
16
|
+
let maxSafe = 9007199254740991.
|
|
17
|
+
|
|
18
|
+
@scope("Number") @val external isSafeInteger: float => bool = "isSafeInteger"
|
|
19
|
+
|
|
20
|
+
// Escapes for the boundaries that stay `int`: the handler-facing
|
|
21
|
+
// `context.chain.id` / `Envio.effectChain.id`, and int literals in configs and
|
|
22
|
+
// tests. Both are the identity at runtime — an `int` is already a JS number.
|
|
23
|
+
external fromInt: int => t = "%identity"
|
|
24
|
+
external toInt: t => int = "%identity"
|
|
25
|
+
|
|
26
|
+
let toString = (chainId: t) => chainId->Float.toString
|
|
27
|
+
let compare = (a: t, b: t) => a < b ? -1 : a > b ? 1 : 0
|
|
28
|
+
|
|
29
|
+
// PostgreSQL BIGINT and ClickHouse UInt64 columns come back as strings (a
|
|
30
|
+
// JS number can't hold their full range), so the parser accepts both and
|
|
31
|
+
// range-checks the result rather than trusting the driver.
|
|
32
|
+
let schema: S.t<t> = S.float->S.preprocess(s => {
|
|
33
|
+
parser: value => {
|
|
34
|
+
let number = switch value->typeof {
|
|
35
|
+
| #string => value->(Utils.magic: unknown => string)->Float.parseFloat
|
|
36
|
+
| _ => value->(Utils.magic: unknown => float)
|
|
37
|
+
}
|
|
38
|
+
if !isSafeInteger(number) || number < 0. {
|
|
39
|
+
s.fail(
|
|
40
|
+
`Expected a chain id between 0 and ${maxSafe->Float.toString}, received ${value->(
|
|
41
|
+
Utils.magic: unknown => string
|
|
42
|
+
)}`,
|
|
43
|
+
)
|
|
44
|
+
}
|
|
45
|
+
number
|
|
46
|
+
},
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
// Dicts keyed by chain id. JS coerces the number key to its decimal string,
|
|
50
|
+
// which is exactly what `toString` produces, so the two key forms interoperate.
|
|
51
|
+
module Dict = {
|
|
52
|
+
@get_index external dangerouslyGetNonOption: (dict<'a>, t) => option<'a> = ""
|
|
53
|
+
@set_index external set: (dict<'a>, t, 'a) => unit = ""
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Postgres returns BIGINT columns as strings, so raw (schema-less) reads of a
|
|
57
|
+
// chain-id column go through this instead of trusting the driver's type.
|
|
58
|
+
let normalizeOrThrow = (value: 'a): t => value->S.parseOrThrow(schema)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
4
|
+
|
|
5
|
+
let modeSchema = S$RescriptSchema.$$enum([
|
|
6
|
+
"int32",
|
|
7
|
+
"int64"
|
|
8
|
+
]);
|
|
9
|
+
|
|
10
|
+
function toString(chainId) {
|
|
11
|
+
return chainId.toString();
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function compare(a, b) {
|
|
15
|
+
if (a < b) {
|
|
16
|
+
return -1;
|
|
17
|
+
} else if (a > b) {
|
|
18
|
+
return 1;
|
|
19
|
+
} else {
|
|
20
|
+
return 0;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
let schema = S$RescriptSchema.preprocess(S$RescriptSchema.float, s => ({
|
|
25
|
+
p: value => {
|
|
26
|
+
let match = typeof value;
|
|
27
|
+
let number = match === "string" ? parseFloat(value) : value;
|
|
28
|
+
if (!Number.isSafeInteger(number) || number < 0) {
|
|
29
|
+
s.fail(`Expected a chain id between 0 and ` + (9007199254740991).toString() + `, received ` + value, undefined);
|
|
30
|
+
}
|
|
31
|
+
return number;
|
|
32
|
+
}
|
|
33
|
+
}));
|
|
34
|
+
|
|
35
|
+
let Dict = {};
|
|
36
|
+
|
|
37
|
+
function normalizeOrThrow(value) {
|
|
38
|
+
return S$RescriptSchema.parseOrThrow(value, schema);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export {
|
|
42
|
+
modeSchema,
|
|
43
|
+
toString,
|
|
44
|
+
compare,
|
|
45
|
+
schema,
|
|
46
|
+
Dict,
|
|
47
|
+
normalizeOrThrow,
|
|
48
|
+
}
|
|
49
|
+
/* modeSchema Not a pure module */
|
package/src/ChainId.resi
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
type t
|
|
2
|
+
|
|
3
|
+
type mode = | @as("int32") Int32 | @as("int64") Int64
|
|
4
|
+
|
|
5
|
+
let modeSchema: S.t<mode>
|
|
6
|
+
|
|
7
|
+
// Escapes for the boundaries that stay `int`: the handler-facing
|
|
8
|
+
// `context.chain.id` / `Envio.effectChain.id`, and int literals in configs and
|
|
9
|
+
// tests. Both are the identity at runtime — an `int` is already a JS number.
|
|
10
|
+
external fromInt: int => t = "%identity"
|
|
11
|
+
external toInt: t => int = "%identity"
|
|
12
|
+
|
|
13
|
+
let toString: t => string
|
|
14
|
+
let compare: (t, t) => int
|
|
15
|
+
|
|
16
|
+
let schema: S.t<t>
|
|
17
|
+
|
|
18
|
+
// Dicts keyed by chain id. JS coerces the number key to its decimal string,
|
|
19
|
+
// which is exactly what `toString` produces, so the two key forms interoperate.
|
|
20
|
+
module Dict: {
|
|
21
|
+
@get_index external dangerouslyGetNonOption: (dict<'a>, t) => option<'a> = ""
|
|
22
|
+
@set_index external set: (dict<'a>, t, 'a) => unit = ""
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Postgres returns BIGINT columns as strings, so raw (schema-less) reads of a
|
|
26
|
+
// chain-id column go through this instead of trusting the driver's type.
|
|
27
|
+
let normalizeOrThrow: 'a => t
|
package/src/ChainMap.res
CHANGED
|
@@ -1,40 +1,26 @@
|
|
|
1
|
-
module Chain = {
|
|
2
|
-
type t = int
|
|
3
|
-
|
|
4
|
-
external toChainId: t => int = "%identity"
|
|
5
|
-
|
|
6
|
-
let toString = chainId => chainId->Int.toString
|
|
7
|
-
|
|
8
|
-
let makeUnsafe = (~chainId) => chainId
|
|
9
|
-
}
|
|
10
|
-
|
|
11
1
|
module ChainIdCmp = Belt.Id.MakeComparable({
|
|
12
|
-
type t =
|
|
13
|
-
let cmp =
|
|
2
|
+
type t = ChainId.t
|
|
3
|
+
let cmp = ChainId.compare
|
|
14
4
|
})
|
|
15
5
|
|
|
16
6
|
type t<'a> = Belt.Map.t<ChainIdCmp.t, 'a, ChainIdCmp.identity>
|
|
17
7
|
|
|
18
|
-
let fromArrayUnsafe: array<(
|
|
8
|
+
let fromArrayUnsafe: array<(ChainId.t, 'a)> => t<'a> = arr => {
|
|
19
9
|
arr->Belt.Map.fromArray(~id=module(ChainIdCmp))
|
|
20
10
|
}
|
|
21
11
|
|
|
22
|
-
let get: (t<'a>,
|
|
23
|
-
switch Belt.Map.get(self,
|
|
12
|
+
let get: (t<'a>, ChainId.t) => 'a = (self, chainId) =>
|
|
13
|
+
switch Belt.Map.get(self, chainId) {
|
|
24
14
|
| Some(v) => v
|
|
25
15
|
| None =>
|
|
26
|
-
// Should be unreachable, since we validate
|
|
16
|
+
// Should be unreachable, since we validate chain ids when parsing the config.
|
|
27
17
|
// Still throw just in case something went wrong
|
|
28
|
-
JsError.throwWithMessage(
|
|
18
|
+
JsError.throwWithMessage(
|
|
19
|
+
"No chain with id " ++ chainId->ChainId.toString ++ " found in chain map",
|
|
20
|
+
)
|
|
29
21
|
}
|
|
30
22
|
|
|
31
|
-
let set: (t<'a>, Chain.t, 'a) => t<'a> = (map, chain, v) => Belt.Map.set(map, chain, v)
|
|
32
23
|
let values: t<'a> => array<'a> = map => Belt.Map.valuesToArray(map)
|
|
33
|
-
let keys: t<'a> => array<
|
|
34
|
-
let
|
|
35
|
-
let
|
|
36
|
-
let map: (t<'a>, 'a => 'b) => t<'b> = (map, fn) => Belt.Map.map(map, fn)
|
|
37
|
-
let mapWithKey: (t<'a>, (Chain.t, 'a) => 'b) => t<'b> = (map, fn) => Belt.Map.mapWithKey(map, fn)
|
|
38
|
-
let size: t<'a> => int = map => Belt.Map.size(map)
|
|
39
|
-
let update: (t<'a>, Chain.t, 'a => 'a) => t<'a> = (map, chain, updateFn) =>
|
|
40
|
-
Belt.Map.update(map, chain, opt => opt->Option.map(updateFn))
|
|
24
|
+
let keys: t<'a> => array<ChainId.t> = map => Belt.Map.keysToArray(map)
|
|
25
|
+
let has: (t<'a>, ChainId.t) => bool = (map, chainId) => Belt.Map.has(map, chainId)
|
|
26
|
+
let mapWithKey: (t<'a>, (ChainId.t, 'a) => 'b) => t<'b> = (map, fn) => Belt.Map.mapWithKey(map, fn)
|