envio 3.0.2 → 3.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/evm.schema.json +15 -8
- package/fuel.schema.json +19 -12
- package/index.d.ts +0 -2
- package/package.json +6 -7
- package/rescript.json +1 -1
- package/src/Batch.res +4 -214
- package/src/Batch.res.mjs +6 -165
- package/src/ChainFetcher.res +12 -28
- package/src/ChainFetcher.res.mjs +8 -17
- package/src/ChainManager.res +10 -9
- package/src/ChainManager.res.mjs +6 -10
- package/src/Config.res +9 -25
- package/src/Config.res.mjs +17 -27
- package/src/Core.res +7 -0
- package/src/Ctx.res +1 -0
- package/src/Env.res +0 -8
- package/src/Env.res.mjs +0 -6
- package/src/EventConfigBuilder.res +13 -123
- package/src/EventConfigBuilder.res.mjs +6 -73
- package/src/EventProcessing.res +5 -29
- package/src/EventProcessing.res.mjs +11 -20
- package/src/EventUtils.res +0 -27
- package/src/EventUtils.res.mjs +0 -24
- package/src/FetchState.res +2 -15
- package/src/FetchState.res.mjs +3 -18
- package/src/GlobalState.res +26 -39
- package/src/GlobalState.res.mjs +12 -40
- package/src/HandlerLoader.res +6 -5
- package/src/HandlerLoader.res.mjs +27 -9
- package/src/HandlerRegister.res +1 -12
- package/src/HandlerRegister.res.mjs +1 -6
- package/src/HandlerRegister.resi +1 -1
- package/src/Hasura.res +96 -32
- package/src/Hasura.res.mjs +93 -38
- package/src/InMemoryStore.res +205 -45
- package/src/InMemoryStore.res.mjs +157 -40
- package/src/InMemoryTable.res +165 -249
- package/src/InMemoryTable.res.mjs +156 -227
- package/src/Internal.res +10 -34
- package/src/Internal.res.mjs +9 -3
- package/src/LoadLayer.res +5 -5
- package/src/LoadLayer.res.mjs +5 -5
- package/src/LogSelection.res +15 -19
- package/src/LogSelection.res.mjs +5 -6
- package/src/Main.res +4 -6
- package/src/Main.res.mjs +26 -15
- package/src/Persistence.res +7 -132
- package/src/Persistence.res.mjs +1 -102
- package/src/PgStorage.res +57 -40
- package/src/PgStorage.res.mjs +60 -34
- package/src/ReorgDetection.res +35 -58
- package/src/ReorgDetection.res.mjs +21 -29
- package/src/SimulateItems.res.mjs +21 -3
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +1 -1
- package/src/TableIndices.res +9 -2
- package/src/TableIndices.res.mjs +7 -1
- package/src/TestIndexer.res +53 -60
- package/src/TestIndexer.res.mjs +77 -63
- package/src/TestIndexerProxyStorage.res +4 -14
- package/src/TestIndexerProxyStorage.res.mjs +1 -5
- package/src/UserContext.res +2 -4
- package/src/UserContext.res.mjs +4 -5
- package/src/Utils.res +0 -2
- package/src/Utils.res.mjs +0 -3
- package/src/bindings/ClickHouse.res +45 -38
- package/src/bindings/ClickHouse.res.mjs +16 -17
- package/src/bindings/Vitest.res +3 -0
- package/src/db/InternalTable.res +59 -18
- package/src/db/InternalTable.res.mjs +82 -51
- package/src/db/Table.res +9 -2
- package/src/db/Table.res.mjs +10 -7
- package/src/sources/EnvioApiClient.res +15 -0
- package/src/sources/EnvioApiClient.res.mjs +24 -0
- package/src/sources/EvmChain.res +32 -10
- package/src/sources/EvmChain.res.mjs +31 -5
- package/src/sources/HyperFuelSource.res +15 -58
- package/src/sources/HyperFuelSource.res.mjs +20 -39
- package/src/sources/HyperSync.res +54 -100
- package/src/sources/HyperSync.res.mjs +67 -96
- package/src/sources/HyperSync.resi +4 -22
- package/src/sources/HyperSyncClient.res +70 -247
- package/src/sources/HyperSyncClient.res.mjs +47 -46
- package/src/sources/HyperSyncSource.res +94 -166
- package/src/sources/HyperSyncSource.res.mjs +100 -127
- package/src/sources/RpcSource.res +43 -22
- package/src/sources/RpcSource.res.mjs +50 -35
- package/src/sources/SimulateSource.res +1 -7
- package/src/sources/SimulateSource.res.mjs +1 -7
- package/src/sources/Source.res +10 -1
- package/src/sources/Source.res.mjs +3 -0
- package/src/sources/SourceManager.res +177 -8
- package/src/sources/SourceManager.res.mjs +141 -3
- package/src/sources/SourceManager.resi +19 -0
- package/src/tui/Tui.res +44 -6
- package/src/tui/Tui.res.mjs +56 -8
- package/src/tui/components/TuiData.res +3 -0
- package/svm.schema.json +11 -4
- package/src/sources/HyperSyncJsonApi.res +0 -390
- package/src/sources/HyperSyncJsonApi.res.mjs +0 -237
package/README.md
CHANGED
|
@@ -70,7 +70,6 @@ A few things already running in production:
|
|
|
70
70
|
**Multichain indexing**
|
|
71
71
|
- Index EVM, SVM, and Fuel blockchains from a single indexer
|
|
72
72
|
- 70+ EVM chains with native HyperSync support, plus any EVM chain via RPC
|
|
73
|
-
- Unordered multichain mode for maximum throughput across chains
|
|
74
73
|
- Real-time indexing with reorg handling built in
|
|
75
74
|
|
|
76
75
|
**Developer experience**
|
package/evm.schema.json
CHANGED
|
@@ -4,6 +4,10 @@
|
|
|
4
4
|
"description": "Schema for a YAML config for an envio indexer",
|
|
5
5
|
"type": "object",
|
|
6
6
|
"properties": {
|
|
7
|
+
"name": {
|
|
8
|
+
"description": "Name of the project",
|
|
9
|
+
"type": "string"
|
|
10
|
+
},
|
|
7
11
|
"description": {
|
|
8
12
|
"description": "Description of the project",
|
|
9
13
|
"type": [
|
|
@@ -11,10 +15,6 @@
|
|
|
11
15
|
"null"
|
|
12
16
|
]
|
|
13
17
|
},
|
|
14
|
-
"name": {
|
|
15
|
-
"description": "Name of the project",
|
|
16
|
-
"type": "string"
|
|
17
|
-
},
|
|
18
18
|
"schema": {
|
|
19
19
|
"description": "Custom path to schema.graphql file",
|
|
20
20
|
"type": [
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"null"
|
|
68
68
|
],
|
|
69
69
|
"items": {
|
|
70
|
-
"$ref": "#/$defs/
|
|
70
|
+
"$ref": "#/$defs/GlobalContract"
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
73
|
"chains": {
|
|
@@ -189,7 +189,7 @@
|
|
|
189
189
|
"evm"
|
|
190
190
|
]
|
|
191
191
|
},
|
|
192
|
-
"
|
|
192
|
+
"GlobalContract": {
|
|
193
193
|
"type": "object",
|
|
194
194
|
"properties": {
|
|
195
195
|
"name": {
|
|
@@ -356,6 +356,13 @@
|
|
|
356
356
|
"format": "uint64",
|
|
357
357
|
"minimum": 0
|
|
358
358
|
},
|
|
359
|
+
"skip": {
|
|
360
|
+
"description": "Excludes the chain from indexing and migrations. Code generation is unaffected. For testing, prefer using a test framework instead.",
|
|
361
|
+
"type": [
|
|
362
|
+
"boolean",
|
|
363
|
+
"null"
|
|
364
|
+
]
|
|
365
|
+
},
|
|
359
366
|
"rpc": {
|
|
360
367
|
"description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
|
|
361
368
|
"anyOf": [
|
|
@@ -418,7 +425,7 @@
|
|
|
418
425
|
"null"
|
|
419
426
|
],
|
|
420
427
|
"items": {
|
|
421
|
-
"$ref": "#/$defs/
|
|
428
|
+
"$ref": "#/$defs/ChainContract"
|
|
422
429
|
}
|
|
423
430
|
}
|
|
424
431
|
},
|
|
@@ -578,7 +585,7 @@
|
|
|
578
585
|
"url"
|
|
579
586
|
]
|
|
580
587
|
},
|
|
581
|
-
"
|
|
588
|
+
"ChainContract": {
|
|
582
589
|
"type": "object",
|
|
583
590
|
"properties": {
|
|
584
591
|
"name": {
|
package/fuel.schema.json
CHANGED
|
@@ -4,6 +4,10 @@
|
|
|
4
4
|
"description": "Schema for a YAML config for an envio indexer",
|
|
5
5
|
"type": "object",
|
|
6
6
|
"properties": {
|
|
7
|
+
"name": {
|
|
8
|
+
"description": "Name of the project",
|
|
9
|
+
"type": "string"
|
|
10
|
+
},
|
|
7
11
|
"description": {
|
|
8
12
|
"description": "Description of the project",
|
|
9
13
|
"type": [
|
|
@@ -11,10 +15,6 @@
|
|
|
11
15
|
"null"
|
|
12
16
|
]
|
|
13
17
|
},
|
|
14
|
-
"name": {
|
|
15
|
-
"description": "Name of the project",
|
|
16
|
-
"type": "string"
|
|
17
|
-
},
|
|
18
18
|
"schema": {
|
|
19
19
|
"description": "Custom path to schema.graphql file",
|
|
20
20
|
"type": [
|
|
@@ -60,7 +60,7 @@
|
|
|
60
60
|
"null"
|
|
61
61
|
],
|
|
62
62
|
"items": {
|
|
63
|
-
"$ref": "#/$defs/
|
|
63
|
+
"$ref": "#/$defs/GlobalContract"
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
66
|
"chains": {
|
|
@@ -147,7 +147,7 @@
|
|
|
147
147
|
"fuel"
|
|
148
148
|
]
|
|
149
149
|
},
|
|
150
|
-
"
|
|
150
|
+
"GlobalContract": {
|
|
151
151
|
"type": "object",
|
|
152
152
|
"properties": {
|
|
153
153
|
"name": {
|
|
@@ -183,6 +183,10 @@
|
|
|
183
183
|
"EventConfig": {
|
|
184
184
|
"type": "object",
|
|
185
185
|
"properties": {
|
|
186
|
+
"name": {
|
|
187
|
+
"description": "Name of the event in the HyperIndex generated code",
|
|
188
|
+
"type": "string"
|
|
189
|
+
},
|
|
186
190
|
"type": {
|
|
187
191
|
"description": "Explicitly set the event type you want to index. It's derived from the event name and fallbacks to LogData.",
|
|
188
192
|
"anyOf": [
|
|
@@ -194,10 +198,6 @@
|
|
|
194
198
|
}
|
|
195
199
|
]
|
|
196
200
|
},
|
|
197
|
-
"name": {
|
|
198
|
-
"description": "Name of the event in the HyperIndex generated code",
|
|
199
|
-
"type": "string"
|
|
200
|
-
},
|
|
201
201
|
"logId": {
|
|
202
202
|
"description": "An identifier of a logged type from ABI. Used for indexing LogData receipts. The option can be omitted when the event name matches the logged struct/enum name.",
|
|
203
203
|
"type": [
|
|
@@ -230,6 +230,13 @@
|
|
|
230
230
|
"format": "uint64",
|
|
231
231
|
"minimum": 0
|
|
232
232
|
},
|
|
233
|
+
"skip": {
|
|
234
|
+
"description": "Excludes the chain from indexing and migrations. Code generation is unaffected. For testing, prefer using a test framework instead.",
|
|
235
|
+
"type": [
|
|
236
|
+
"boolean",
|
|
237
|
+
"null"
|
|
238
|
+
]
|
|
239
|
+
},
|
|
233
240
|
"start_block": {
|
|
234
241
|
"description": "The block at which the indexer should start ingesting data",
|
|
235
242
|
"type": "integer",
|
|
@@ -281,7 +288,7 @@
|
|
|
281
288
|
"null"
|
|
282
289
|
],
|
|
283
290
|
"items": {
|
|
284
|
-
"$ref": "#/$defs/
|
|
291
|
+
"$ref": "#/$defs/ChainContract"
|
|
285
292
|
}
|
|
286
293
|
}
|
|
287
294
|
},
|
|
@@ -304,7 +311,7 @@
|
|
|
304
311
|
"url"
|
|
305
312
|
]
|
|
306
313
|
},
|
|
307
|
-
"
|
|
314
|
+
"ChainContract": {
|
|
308
315
|
"type": "object",
|
|
309
316
|
"properties": {
|
|
310
317
|
"name": {
|
package/index.d.ts
CHANGED
|
@@ -299,8 +299,6 @@ type IndexerConfig = {
|
|
|
299
299
|
description?: string;
|
|
300
300
|
/** Path to handlers directory for auto-loading (default: "src/handlers"). */
|
|
301
301
|
handlers?: string;
|
|
302
|
-
/** Multichain mode: ordered or unordered (default: "unordered"). */
|
|
303
|
-
multichain?: "ordered" | "unordered";
|
|
304
302
|
/** Target batch size for event processing (default: 5000). */
|
|
305
303
|
fullBatchSize?: number;
|
|
306
304
|
/** Whether to rollback on chain reorg (default: true). */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.1.0-rc.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -43,7 +43,6 @@
|
|
|
43
43
|
"@clickhouse/client": "1.17.0",
|
|
44
44
|
"@elastic/ecs-pino-format": "1.4.0",
|
|
45
45
|
"@envio-dev/hyperfuel-client": "1.2.2",
|
|
46
|
-
"@envio-dev/hypersync-client": "1.3.0",
|
|
47
46
|
"@fuel-ts/crypto": "0.96.1",
|
|
48
47
|
"@fuel-ts/errors": "0.96.1",
|
|
49
48
|
"@fuel-ts/hasher": "0.96.1",
|
|
@@ -71,10 +70,10 @@
|
|
|
71
70
|
"tsx": "4.21.0"
|
|
72
71
|
},
|
|
73
72
|
"optionalDependencies": {
|
|
74
|
-
"envio-linux-x64": "3.0.
|
|
75
|
-
"envio-linux-x64-musl": "3.0.
|
|
76
|
-
"envio-linux-arm64": "3.0.
|
|
77
|
-
"envio-darwin-x64": "3.0.
|
|
78
|
-
"envio-darwin-arm64": "3.0.
|
|
73
|
+
"envio-linux-x64": "3.1.0-rc.1",
|
|
74
|
+
"envio-linux-x64-musl": "3.1.0-rc.1",
|
|
75
|
+
"envio-linux-arm64": "3.1.0-rc.1",
|
|
76
|
+
"envio-darwin-x64": "3.1.0-rc.1",
|
|
77
|
+
"envio-darwin-arm64": "3.1.0-rc.1"
|
|
79
78
|
}
|
|
80
79
|
}
|
package/rescript.json
CHANGED
package/src/Batch.res
CHANGED
|
@@ -31,50 +31,7 @@ type t = {
|
|
|
31
31
|
checkpointEventsProcessed: array<int>,
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
|
|
35
|
-
It either returnes an earliest item among all chains, or None if no chains are actively indexing
|
|
36
|
-
*/
|
|
37
|
-
let getOrderedNextChain = (fetchStates: ChainMap.t<FetchState.t>, ~batchSizePerChain) => {
|
|
38
|
-
let earliestChain: ref<option<FetchState.t>> = ref(None)
|
|
39
|
-
let earliestChainTimestamp = ref(0)
|
|
40
|
-
let chainKeys = fetchStates->ChainMap.keys
|
|
41
|
-
for idx in 0 to chainKeys->Array.length - 1 {
|
|
42
|
-
let chain = chainKeys->Array.getUnsafe(idx)
|
|
43
|
-
let fetchState = fetchStates->ChainMap.get(chain)
|
|
44
|
-
if fetchState->FetchState.isActivelyIndexing {
|
|
45
|
-
let timestamp = fetchState->FetchState.getTimestampAt(
|
|
46
|
-
~index=switch batchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
47
|
-
chain->ChainMap.Chain.toChainId,
|
|
48
|
-
) {
|
|
49
|
-
| Some(batchSize) => batchSize
|
|
50
|
-
| None => 0
|
|
51
|
-
},
|
|
52
|
-
)
|
|
53
|
-
switch earliestChain.contents {
|
|
54
|
-
| Some(earliestChain)
|
|
55
|
-
if timestamp > earliestChainTimestamp.contents ||
|
|
56
|
-
(timestamp === earliestChainTimestamp.contents &&
|
|
57
|
-
chain->ChainMap.Chain.toChainId > earliestChain.chainId) => ()
|
|
58
|
-
| _ => {
|
|
59
|
-
earliestChain := Some(fetchState)
|
|
60
|
-
earliestChainTimestamp := timestamp
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
earliestChain.contents
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// Save overhead of recreating the dict every time
|
|
69
|
-
let immutableEmptyBatchSizePerChain: dict<int> = Dict.make()
|
|
70
|
-
let hasOrderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
|
|
71
|
-
switch fetchStates->getOrderedNextChain(~batchSizePerChain=immutableEmptyBatchSizePerChain) {
|
|
72
|
-
| Some(fetchState) => fetchState->FetchState.hasReadyItem
|
|
73
|
-
| None => false
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
|
|
34
|
+
let hasReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
|
|
78
35
|
fetchStates
|
|
79
36
|
->ChainMap.values
|
|
80
37
|
->Array.some(fetchState => {
|
|
@@ -82,16 +39,6 @@ let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
|
|
|
82
39
|
})
|
|
83
40
|
}
|
|
84
41
|
|
|
85
|
-
let hasMultichainReadyItem = (
|
|
86
|
-
fetchStates: ChainMap.t<FetchState.t>,
|
|
87
|
-
~multichain: Config.multichain,
|
|
88
|
-
) => {
|
|
89
|
-
switch multichain {
|
|
90
|
-
| Ordered => hasOrderedReadyItem(fetchStates)
|
|
91
|
-
| Unordered => hasUnorderedReadyItem(fetchStates)
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
|
|
95
42
|
let getProgressedChainsById = {
|
|
96
43
|
let getChainAfterBatchIfProgressed = (
|
|
97
44
|
~chainBeforeBatch: chainBeforeBatch,
|
|
@@ -214,154 +161,7 @@ let addReorgCheckpoints = (
|
|
|
214
161
|
}
|
|
215
162
|
}
|
|
216
163
|
|
|
217
|
-
let
|
|
218
|
-
~checkpointIdBeforeBatch,
|
|
219
|
-
~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
|
|
220
|
-
~batchSizeTarget,
|
|
221
|
-
) => {
|
|
222
|
-
let totalBatchSize = ref(0)
|
|
223
|
-
let isFinished = ref(false)
|
|
224
|
-
let prevCheckpointId = ref(checkpointIdBeforeBatch)
|
|
225
|
-
let mutBatchSizePerChain = Dict.make()
|
|
226
|
-
let mutProgressBlockNumberPerChain = Dict.make()
|
|
227
|
-
|
|
228
|
-
let fetchStates = chainsBeforeBatch->ChainMap.map(chainBeforeBatch => chainBeforeBatch.fetchState)
|
|
229
|
-
|
|
230
|
-
let items = []
|
|
231
|
-
let checkpointIds = []
|
|
232
|
-
let checkpointChainIds = []
|
|
233
|
-
let checkpointBlockNumbers = []
|
|
234
|
-
let checkpointBlockHashes = []
|
|
235
|
-
let checkpointEventsProcessed = []
|
|
236
|
-
|
|
237
|
-
while totalBatchSize.contents < batchSizeTarget && !isFinished.contents {
|
|
238
|
-
switch fetchStates->getOrderedNextChain(~batchSizePerChain=mutBatchSizePerChain) {
|
|
239
|
-
| Some(fetchState) => {
|
|
240
|
-
let chainBeforeBatch =
|
|
241
|
-
chainsBeforeBatch->ChainMap.get(ChainMap.Chain.makeUnsafe(~chainId=fetchState.chainId))
|
|
242
|
-
let itemsCountBefore = switch mutBatchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
243
|
-
fetchState.chainId,
|
|
244
|
-
) {
|
|
245
|
-
| Some(batchSize) => batchSize
|
|
246
|
-
| None => 0
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
let prevBlockNumber = switch mutProgressBlockNumberPerChain->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
250
|
-
fetchState.chainId,
|
|
251
|
-
) {
|
|
252
|
-
| Some(progressBlockNumber) => progressBlockNumber
|
|
253
|
-
| None => chainBeforeBatch.progressBlockNumber
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
let newItemsCount = fetchState->FetchState.getReadyItemsCount(
|
|
257
|
-
// We should get items only for a single block
|
|
258
|
-
// Since for the ordered mode next block could be after another chain's block
|
|
259
|
-
~targetSize=1,
|
|
260
|
-
~fromItem=itemsCountBefore,
|
|
261
|
-
)
|
|
262
|
-
|
|
263
|
-
if newItemsCount > 0 {
|
|
264
|
-
let item0 = fetchState.buffer->Array.getUnsafe(itemsCountBefore)
|
|
265
|
-
let blockNumber = item0->Internal.getItemBlockNumber
|
|
266
|
-
|
|
267
|
-
prevCheckpointId :=
|
|
268
|
-
addReorgCheckpoints(
|
|
269
|
-
~chainId=fetchState.chainId,
|
|
270
|
-
~reorgDetection=chainBeforeBatch.reorgDetection,
|
|
271
|
-
~prevCheckpointId=prevCheckpointId.contents,
|
|
272
|
-
~fromBlockExclusive=prevBlockNumber,
|
|
273
|
-
~toBlockExclusive=blockNumber,
|
|
274
|
-
~mutCheckpointIds=checkpointIds,
|
|
275
|
-
~mutCheckpointChainIds=checkpointChainIds,
|
|
276
|
-
~mutCheckpointBlockNumbers=checkpointBlockNumbers,
|
|
277
|
-
~mutCheckpointBlockHashes=checkpointBlockHashes,
|
|
278
|
-
~mutCheckpointEventsProcessed=checkpointEventsProcessed,
|
|
279
|
-
)
|
|
280
|
-
|
|
281
|
-
let checkpointId = prevCheckpointId.contents->BigInt.add(1n)
|
|
282
|
-
|
|
283
|
-
items
|
|
284
|
-
->Array.push(item0)
|
|
285
|
-
->ignore
|
|
286
|
-
for idx in 1 to newItemsCount - 1 {
|
|
287
|
-
items
|
|
288
|
-
->Array.push(fetchState.buffer->Belt.Array.getUnsafe(itemsCountBefore + idx))
|
|
289
|
-
->ignore
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
checkpointIds
|
|
293
|
-
->Array.push(checkpointId)
|
|
294
|
-
->ignore
|
|
295
|
-
checkpointChainIds
|
|
296
|
-
->Array.push(fetchState.chainId)
|
|
297
|
-
->ignore
|
|
298
|
-
checkpointBlockNumbers
|
|
299
|
-
->Array.push(blockNumber)
|
|
300
|
-
->ignore
|
|
301
|
-
checkpointBlockHashes
|
|
302
|
-
->Array.push(
|
|
303
|
-
chainBeforeBatch.reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber),
|
|
304
|
-
)
|
|
305
|
-
->ignore
|
|
306
|
-
checkpointEventsProcessed
|
|
307
|
-
->Array.push(newItemsCount)
|
|
308
|
-
->ignore
|
|
309
|
-
|
|
310
|
-
prevCheckpointId := checkpointId
|
|
311
|
-
totalBatchSize := totalBatchSize.contents + newItemsCount
|
|
312
|
-
mutBatchSizePerChain->Utils.Dict.setByInt(
|
|
313
|
-
fetchState.chainId,
|
|
314
|
-
itemsCountBefore + newItemsCount,
|
|
315
|
-
)
|
|
316
|
-
mutProgressBlockNumberPerChain->Utils.Dict.setByInt(fetchState.chainId, blockNumber)
|
|
317
|
-
} else {
|
|
318
|
-
let blockNumberAfterBatch = fetchState->FetchState.bufferBlockNumber
|
|
319
|
-
|
|
320
|
-
prevCheckpointId :=
|
|
321
|
-
addReorgCheckpoints(
|
|
322
|
-
~chainId=fetchState.chainId,
|
|
323
|
-
~reorgDetection=chainBeforeBatch.reorgDetection,
|
|
324
|
-
~prevCheckpointId=prevCheckpointId.contents,
|
|
325
|
-
~fromBlockExclusive=prevBlockNumber,
|
|
326
|
-
~toBlockExclusive=blockNumberAfterBatch + 1, // Make it inclusive
|
|
327
|
-
~mutCheckpointIds=checkpointIds,
|
|
328
|
-
~mutCheckpointChainIds=checkpointChainIds,
|
|
329
|
-
~mutCheckpointBlockNumbers=checkpointBlockNumbers,
|
|
330
|
-
~mutCheckpointBlockHashes=checkpointBlockHashes,
|
|
331
|
-
~mutCheckpointEventsProcessed=checkpointEventsProcessed,
|
|
332
|
-
)
|
|
333
|
-
|
|
334
|
-
// Since the chain was chosen as next
|
|
335
|
-
// the fact that it doesn't have new items means that it reached the buffer block number
|
|
336
|
-
mutProgressBlockNumberPerChain->Utils.Dict.setByInt(
|
|
337
|
-
fetchState.chainId,
|
|
338
|
-
blockNumberAfterBatch,
|
|
339
|
-
)
|
|
340
|
-
isFinished := true
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
| None => isFinished := true
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
{
|
|
349
|
-
totalBatchSize: totalBatchSize.contents,
|
|
350
|
-
items,
|
|
351
|
-
progressedChainsById: getProgressedChainsById(
|
|
352
|
-
~chainsBeforeBatch,
|
|
353
|
-
~batchSizePerChain=mutBatchSizePerChain,
|
|
354
|
-
~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
|
|
355
|
-
),
|
|
356
|
-
checkpointIds,
|
|
357
|
-
checkpointChainIds,
|
|
358
|
-
checkpointBlockNumbers,
|
|
359
|
-
checkpointBlockHashes,
|
|
360
|
-
checkpointEventsProcessed,
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
let prepareUnorderedBatch = (
|
|
164
|
+
let prepareBatch = (
|
|
365
165
|
~checkpointIdBeforeBatch,
|
|
366
166
|
~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
|
|
367
167
|
~batchSizeTarget,
|
|
@@ -454,7 +254,7 @@ let prepareUnorderedBatch = (
|
|
|
454
254
|
}
|
|
455
255
|
|
|
456
256
|
let progressBlockNumberAfterBatch =
|
|
457
|
-
fetchState->FetchState.
|
|
257
|
+
fetchState->FetchState.getProgressBlockNumberAt(~index=chainBatchSize)
|
|
458
258
|
|
|
459
259
|
prevCheckpointId :=
|
|
460
260
|
addReorgCheckpoints(
|
|
@@ -497,19 +297,9 @@ let prepareUnorderedBatch = (
|
|
|
497
297
|
let make = (
|
|
498
298
|
~checkpointIdBeforeBatch,
|
|
499
299
|
~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
|
|
500
|
-
~multichain: Config.multichain,
|
|
501
300
|
~batchSizeTarget,
|
|
502
301
|
) => {
|
|
503
|
-
|
|
504
|
-
switch multichain {
|
|
505
|
-
| Unordered => true
|
|
506
|
-
| Ordered => chainsBeforeBatch->ChainMap.size === 1
|
|
507
|
-
}
|
|
508
|
-
) {
|
|
509
|
-
prepareUnorderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
|
|
510
|
-
} else {
|
|
511
|
-
prepareOrderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
|
|
512
|
-
}
|
|
302
|
+
prepareBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
|
|
513
303
|
}
|
|
514
304
|
|
|
515
305
|
let findFirstEventBlockNumber = (batch: t, ~chainId) => {
|
package/src/Batch.res.mjs
CHANGED
|
@@ -5,38 +5,7 @@ import * as ChainMap from "./ChainMap.res.mjs";
|
|
|
5
5
|
import * as FetchState from "./FetchState.res.mjs";
|
|
6
6
|
import * as ReorgDetection from "./ReorgDetection.res.mjs";
|
|
7
7
|
|
|
8
|
-
function
|
|
9
|
-
let earliestChain;
|
|
10
|
-
let earliestChainTimestamp = 0;
|
|
11
|
-
let chainKeys = ChainMap.keys(fetchStates);
|
|
12
|
-
for (let idx = 0, idx_finish = chainKeys.length - 1; idx <= idx_finish; ++idx) {
|
|
13
|
-
let chain = chainKeys[idx];
|
|
14
|
-
let fetchState = ChainMap.get(fetchStates, chain);
|
|
15
|
-
if (FetchState.isActivelyIndexing(fetchState)) {
|
|
16
|
-
let batchSize = batchSizePerChain[chain];
|
|
17
|
-
let timestamp = FetchState.getTimestampAt(fetchState, batchSize !== undefined ? batchSize : 0);
|
|
18
|
-
let earliestChain$1 = earliestChain;
|
|
19
|
-
if (!(earliestChain$1 !== undefined && (timestamp > earliestChainTimestamp || timestamp === earliestChainTimestamp && chain > earliestChain$1.chainId))) {
|
|
20
|
-
earliestChain = fetchState;
|
|
21
|
-
earliestChainTimestamp = timestamp;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
return earliestChain;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
let immutableEmptyBatchSizePerChain = {};
|
|
29
|
-
|
|
30
|
-
function hasOrderedReadyItem(fetchStates) {
|
|
31
|
-
let fetchState = getOrderedNextChain(fetchStates, immutableEmptyBatchSizePerChain);
|
|
32
|
-
if (fetchState !== undefined) {
|
|
33
|
-
return FetchState.hasReadyItem(fetchState);
|
|
34
|
-
} else {
|
|
35
|
-
return false;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
function hasUnorderedReadyItem(fetchStates) {
|
|
8
|
+
function hasReadyItem(fetchStates) {
|
|
40
9
|
return ChainMap.values(fetchStates).some(fetchState => {
|
|
41
10
|
if (FetchState.isActivelyIndexing(fetchState)) {
|
|
42
11
|
return FetchState.hasReadyItem(fetchState);
|
|
@@ -46,14 +15,6 @@ function hasUnorderedReadyItem(fetchStates) {
|
|
|
46
15
|
});
|
|
47
16
|
}
|
|
48
17
|
|
|
49
|
-
function hasMultichainReadyItem(fetchStates, multichain) {
|
|
50
|
-
if (multichain === "ordered") {
|
|
51
|
-
return hasOrderedReadyItem(fetchStates);
|
|
52
|
-
} else {
|
|
53
|
-
return hasUnorderedReadyItem(fetchStates);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
18
|
function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
|
|
58
19
|
if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
|
|
59
20
|
return {
|
|
@@ -109,114 +70,7 @@ function addReorgCheckpoints(prevCheckpointId, reorgDetection, fromBlockExclusiv
|
|
|
109
70
|
return prevCheckpointId$1;
|
|
110
71
|
}
|
|
111
72
|
|
|
112
|
-
function
|
|
113
|
-
let totalBatchSize = 0;
|
|
114
|
-
let isFinished = false;
|
|
115
|
-
let prevCheckpointId = checkpointIdBeforeBatch;
|
|
116
|
-
let mutBatchSizePerChain = {};
|
|
117
|
-
let mutProgressBlockNumberPerChain = {};
|
|
118
|
-
let fetchStates = ChainMap.map(chainsBeforeBatch, chainBeforeBatch => chainBeforeBatch.fetchState);
|
|
119
|
-
let items = [];
|
|
120
|
-
let checkpointIds = [];
|
|
121
|
-
let checkpointChainIds = [];
|
|
122
|
-
let checkpointBlockNumbers = [];
|
|
123
|
-
let checkpointBlockHashes = [];
|
|
124
|
-
let checkpointEventsProcessed = [];
|
|
125
|
-
while (totalBatchSize < batchSizeTarget && !isFinished) {
|
|
126
|
-
let fetchState = getOrderedNextChain(fetchStates, mutBatchSizePerChain);
|
|
127
|
-
if (fetchState !== undefined) {
|
|
128
|
-
let chainBeforeBatch = ChainMap.get(chainsBeforeBatch, ChainMap.Chain.makeUnsafe(fetchState.chainId));
|
|
129
|
-
let batchSize = mutBatchSizePerChain[fetchState.chainId];
|
|
130
|
-
let itemsCountBefore = batchSize !== undefined ? batchSize : 0;
|
|
131
|
-
let progressBlockNumber = mutProgressBlockNumberPerChain[fetchState.chainId];
|
|
132
|
-
let prevBlockNumber = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
|
|
133
|
-
let newItemsCount = FetchState.getReadyItemsCount(fetchState, 1, itemsCountBefore);
|
|
134
|
-
if (newItemsCount > 0) {
|
|
135
|
-
let item0 = fetchState.buffer[itemsCountBefore];
|
|
136
|
-
let blockNumber = item0.blockNumber;
|
|
137
|
-
let chainId = fetchState.chainId;
|
|
138
|
-
let reorgDetection = chainBeforeBatch.reorgDetection;
|
|
139
|
-
let prevCheckpointId$1 = prevCheckpointId;
|
|
140
|
-
let tmp;
|
|
141
|
-
if (reorgDetection.shouldRollbackOnReorg && !Utils.Dict.isEmpty(reorgDetection.dataByBlockNumber)) {
|
|
142
|
-
let prevCheckpointId$2 = prevCheckpointId$1;
|
|
143
|
-
for (let blockNumber$1 = prevBlockNumber + 1, blockNumber_finish = blockNumber - 1; blockNumber$1 <= blockNumber_finish; ++blockNumber$1) {
|
|
144
|
-
let hash = ReorgDetection.getHashByBlockNumber(reorgDetection, blockNumber$1);
|
|
145
|
-
if (hash !== null) {
|
|
146
|
-
let checkpointId = prevCheckpointId$2 + 1n;
|
|
147
|
-
prevCheckpointId$2 = checkpointId;
|
|
148
|
-
checkpointIds.push(checkpointId);
|
|
149
|
-
checkpointChainIds.push(chainId);
|
|
150
|
-
checkpointBlockNumbers.push(blockNumber$1);
|
|
151
|
-
checkpointBlockHashes.push(hash);
|
|
152
|
-
checkpointEventsProcessed.push(0);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
tmp = prevCheckpointId$2;
|
|
156
|
-
} else {
|
|
157
|
-
tmp = prevCheckpointId$1;
|
|
158
|
-
}
|
|
159
|
-
prevCheckpointId = tmp;
|
|
160
|
-
let checkpointId$1 = prevCheckpointId + 1n;
|
|
161
|
-
items.push(item0);
|
|
162
|
-
for (let idx = 1, idx_finish = newItemsCount - 1; idx <= idx_finish; ++idx) {
|
|
163
|
-
items.push(fetchState.buffer[itemsCountBefore + idx]);
|
|
164
|
-
}
|
|
165
|
-
checkpointIds.push(checkpointId$1);
|
|
166
|
-
checkpointChainIds.push(fetchState.chainId);
|
|
167
|
-
checkpointBlockNumbers.push(blockNumber);
|
|
168
|
-
checkpointBlockHashes.push(ReorgDetection.getHashByBlockNumber(chainBeforeBatch.reorgDetection, blockNumber));
|
|
169
|
-
checkpointEventsProcessed.push(newItemsCount);
|
|
170
|
-
prevCheckpointId = checkpointId$1;
|
|
171
|
-
totalBatchSize = totalBatchSize + newItemsCount;
|
|
172
|
-
mutBatchSizePerChain[fetchState.chainId] = itemsCountBefore + newItemsCount;
|
|
173
|
-
mutProgressBlockNumberPerChain[fetchState.chainId] = blockNumber;
|
|
174
|
-
} else {
|
|
175
|
-
let blockNumberAfterBatch = FetchState.bufferBlockNumber(fetchState);
|
|
176
|
-
let chainId$1 = fetchState.chainId;
|
|
177
|
-
let toBlockExclusive = blockNumberAfterBatch + 1;
|
|
178
|
-
let reorgDetection$1 = chainBeforeBatch.reorgDetection;
|
|
179
|
-
let prevCheckpointId$3 = prevCheckpointId;
|
|
180
|
-
let tmp$1;
|
|
181
|
-
if (reorgDetection$1.shouldRollbackOnReorg && !Utils.Dict.isEmpty(reorgDetection$1.dataByBlockNumber)) {
|
|
182
|
-
let prevCheckpointId$4 = prevCheckpointId$3;
|
|
183
|
-
for (let blockNumber$2 = prevBlockNumber + 1, blockNumber_finish$1 = toBlockExclusive - 1; blockNumber$2 <= blockNumber_finish$1; ++blockNumber$2) {
|
|
184
|
-
let hash$1 = ReorgDetection.getHashByBlockNumber(reorgDetection$1, blockNumber$2);
|
|
185
|
-
if (hash$1 !== null) {
|
|
186
|
-
let checkpointId$2 = prevCheckpointId$4 + 1n;
|
|
187
|
-
prevCheckpointId$4 = checkpointId$2;
|
|
188
|
-
checkpointIds.push(checkpointId$2);
|
|
189
|
-
checkpointChainIds.push(chainId$1);
|
|
190
|
-
checkpointBlockNumbers.push(blockNumber$2);
|
|
191
|
-
checkpointBlockHashes.push(hash$1);
|
|
192
|
-
checkpointEventsProcessed.push(0);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
tmp$1 = prevCheckpointId$4;
|
|
196
|
-
} else {
|
|
197
|
-
tmp$1 = prevCheckpointId$3;
|
|
198
|
-
}
|
|
199
|
-
prevCheckpointId = tmp$1;
|
|
200
|
-
mutProgressBlockNumberPerChain[fetchState.chainId] = blockNumberAfterBatch;
|
|
201
|
-
isFinished = true;
|
|
202
|
-
}
|
|
203
|
-
} else {
|
|
204
|
-
isFinished = true;
|
|
205
|
-
}
|
|
206
|
-
};
|
|
207
|
-
return {
|
|
208
|
-
totalBatchSize: totalBatchSize,
|
|
209
|
-
items: items,
|
|
210
|
-
progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutBatchSizePerChain, mutProgressBlockNumberPerChain),
|
|
211
|
-
checkpointIds: checkpointIds,
|
|
212
|
-
checkpointChainIds: checkpointChainIds,
|
|
213
|
-
checkpointBlockNumbers: checkpointBlockNumbers,
|
|
214
|
-
checkpointBlockHashes: checkpointBlockHashes,
|
|
215
|
-
checkpointEventsProcessed: checkpointEventsProcessed
|
|
216
|
-
};
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
|
|
73
|
+
function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
|
|
220
74
|
let preparedFetchStates = FetchState.sortForUnorderedBatch(ChainMap.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
|
|
221
75
|
let chainIdx = 0;
|
|
222
76
|
let preparedNumber = preparedFetchStates.length;
|
|
@@ -281,7 +135,7 @@ function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batch
|
|
|
281
135
|
totalBatchSize = totalBatchSize + chainBatchSize;
|
|
282
136
|
mutBatchSizePerChain[fetchState.chainId] = chainBatchSize;
|
|
283
137
|
}
|
|
284
|
-
let progressBlockNumberAfterBatch = FetchState.
|
|
138
|
+
let progressBlockNumberAfterBatch = FetchState.getProgressBlockNumberAt(fetchState, chainBatchSize);
|
|
285
139
|
let chainId$1 = fetchState.chainId;
|
|
286
140
|
let toBlockExclusive = progressBlockNumberAfterBatch + 1;
|
|
287
141
|
let fromBlockExclusive$1 = prevBlockNumber;
|
|
@@ -322,15 +176,7 @@ function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batch
|
|
|
322
176
|
};
|
|
323
177
|
}
|
|
324
178
|
|
|
325
|
-
|
|
326
|
-
let tmp;
|
|
327
|
-
tmp = multichain === "ordered" ? ChainMap.size(chainsBeforeBatch) === 1 : true;
|
|
328
|
-
if (tmp) {
|
|
329
|
-
return prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget);
|
|
330
|
-
} else {
|
|
331
|
-
return prepareOrderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget);
|
|
332
|
-
}
|
|
333
|
-
}
|
|
179
|
+
let make = prepareBatch;
|
|
334
180
|
|
|
335
181
|
function findFirstEventBlockNumber(batch, chainId) {
|
|
336
182
|
let idx = 0;
|
|
@@ -362,15 +208,10 @@ function findLastEventItem(batch, chainId) {
|
|
|
362
208
|
}
|
|
363
209
|
|
|
364
210
|
export {
|
|
365
|
-
|
|
366
|
-
immutableEmptyBatchSizePerChain,
|
|
367
|
-
hasOrderedReadyItem,
|
|
368
|
-
hasUnorderedReadyItem,
|
|
369
|
-
hasMultichainReadyItem,
|
|
211
|
+
hasReadyItem,
|
|
370
212
|
getProgressedChainsById,
|
|
371
213
|
addReorgCheckpoints,
|
|
372
|
-
|
|
373
|
-
prepareUnorderedBatch,
|
|
214
|
+
prepareBatch,
|
|
374
215
|
make,
|
|
375
216
|
findFirstEventBlockNumber,
|
|
376
217
|
findLastEventItem,
|