envio 3.0.2-svm-alpha.0 → 3.0.2-svm-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/evm.schema.json +7 -0
- package/fuel.schema.json +7 -0
- package/index.d.ts +12 -2
- package/package.json +6 -6
- package/src/Batch.res +4 -214
- package/src/Batch.res.mjs +6 -165
- package/src/ChainFetcher.res +4 -5
- package/src/ChainFetcher.res.mjs +6 -7
- package/src/ChainManager.res +3 -5
- package/src/ChainManager.res.mjs +4 -6
- package/src/Config.res +102 -32
- package/src/Config.res.mjs +103 -32
- package/src/Core.res +4 -8
- package/src/Env.res +0 -1
- package/src/Env.res.mjs +0 -2
- package/src/Envio.res +9 -0
- package/src/EventConfigBuilder.res +40 -136
- package/src/EventConfigBuilder.res.mjs +16 -80
- package/src/EventUtils.res +0 -18
- package/src/EventUtils.res.mjs +0 -18
- package/src/FetchState.res +1 -11
- package/src/FetchState.res.mjs +2 -16
- package/src/GlobalState.res +3 -8
- package/src/GlobalState.res.mjs +1 -13
- package/src/HandlerLoader.res +6 -5
- package/src/HandlerLoader.res.mjs +27 -9
- package/src/HandlerRegister.res +1 -12
- package/src/HandlerRegister.res.mjs +1 -6
- package/src/HandlerRegister.resi +1 -1
- package/src/Hasura.res +139 -32
- package/src/Hasura.res.mjs +131 -38
- package/src/Internal.res +11 -17
- package/src/Internal.res.mjs +3 -3
- package/src/Main.res +2 -6
- package/src/Main.res.mjs +24 -16
- package/src/Persistence.res +17 -2
- package/src/Persistence.res.mjs +14 -2
- package/src/PgStorage.res +5 -1
- package/src/PgStorage.res.mjs +12 -6
- package/src/SimulateItems.res.mjs +21 -3
- package/src/TestIndexer.res.mjs +23 -4
- package/src/bindings/ClickHouse.res +6 -2
- package/src/bindings/ClickHouse.res.mjs +3 -2
- package/src/bindings/Vitest.res +3 -0
- package/src/db/InternalTable.res.mjs +35 -35
- package/src/db/Table.res +9 -2
- package/src/db/Table.res.mjs +10 -7
- package/src/sources/EvmChain.res +32 -9
- package/src/sources/EvmChain.res.mjs +31 -4
- package/src/sources/HyperSyncClient.res +35 -79
- package/src/sources/HyperSyncClient.res.mjs +35 -46
- package/src/sources/HyperSyncSolanaClient.res +24 -0
- package/src/sources/HyperSyncSolanaSource.res +113 -40
- package/src/sources/HyperSyncSolanaSource.res.mjs +117 -56
- package/src/sources/HyperSyncSource.res +13 -14
- package/src/sources/HyperSyncSource.res.mjs +8 -5
- package/src/sources/RpcSource.res +11 -7
- package/src/sources/RpcSource.res.mjs +6 -5
- package/svm.schema.json +112 -22
package/README.md
CHANGED
|
@@ -70,7 +70,6 @@ A few things already running in production:
|
|
|
70
70
|
**Multichain indexing**
|
|
71
71
|
- Index EVM, SVM, and Fuel blockchains from a single indexer
|
|
72
72
|
- 70+ EVM chains with native HyperSync support, plus any EVM chain via RPC
|
|
73
|
-
- Unordered multichain mode for maximum throughput across chains
|
|
74
73
|
- Real-time indexing with reorg handling built in
|
|
75
74
|
|
|
76
75
|
**Developer experience**
|
package/evm.schema.json
CHANGED
|
@@ -356,6 +356,13 @@
|
|
|
356
356
|
"format": "uint64",
|
|
357
357
|
"minimum": 0
|
|
358
358
|
},
|
|
359
|
+
"skip": {
|
|
360
|
+
"description": "Excludes the chain from indexing and migrations. Code generation is unaffected. For testing, prefer using a test framework instead.",
|
|
361
|
+
"type": [
|
|
362
|
+
"boolean",
|
|
363
|
+
"null"
|
|
364
|
+
]
|
|
365
|
+
},
|
|
359
366
|
"rpc": {
|
|
360
367
|
"description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
|
|
361
368
|
"anyOf": [
|
package/fuel.schema.json
CHANGED
|
@@ -230,6 +230,13 @@
|
|
|
230
230
|
"format": "uint64",
|
|
231
231
|
"minimum": 0
|
|
232
232
|
},
|
|
233
|
+
"skip": {
|
|
234
|
+
"description": "Excludes the chain from indexing and migrations. Code generation is unaffected. For testing, prefer using a test framework instead.",
|
|
235
|
+
"type": [
|
|
236
|
+
"boolean",
|
|
237
|
+
"null"
|
|
238
|
+
]
|
|
239
|
+
},
|
|
233
240
|
"start_block": {
|
|
234
241
|
"description": "The block at which the indexer should start ingesting data",
|
|
235
242
|
"type": "integer",
|
package/index.d.ts
CHANGED
|
@@ -299,8 +299,6 @@ type IndexerConfig = {
|
|
|
299
299
|
description?: string;
|
|
300
300
|
/** Path to handlers directory for auto-loading (default: "src/handlers"). */
|
|
301
301
|
handlers?: string;
|
|
302
|
-
/** Multichain mode: ordered or unordered (default: "unordered"). */
|
|
303
|
-
multichain?: "ordered" | "unordered";
|
|
304
302
|
/** Target batch size for event processing (default: 5000). */
|
|
305
303
|
fullBatchSize?: number;
|
|
306
304
|
/** Whether to rollback on chain reorg (default: true). */
|
|
@@ -1000,6 +998,15 @@ export type SvmInstruction = {
|
|
|
1000
998
|
readonly decoded?: SvmDecodedInstruction;
|
|
1001
999
|
};
|
|
1002
1000
|
|
|
1001
|
+
export type SvmTokenBalance = {
|
|
1002
|
+
readonly account?: string;
|
|
1003
|
+
readonly mint?: string;
|
|
1004
|
+
readonly owner?: string;
|
|
1005
|
+
/** u64 decimal string. Cast with BigInt(...) for arithmetic. */
|
|
1006
|
+
readonly preAmount?: string;
|
|
1007
|
+
readonly postAmount?: string;
|
|
1008
|
+
};
|
|
1009
|
+
|
|
1003
1010
|
/** Parent transaction surfaced when an instruction's
|
|
1004
1011
|
* `include_transaction` flag is `true`. */
|
|
1005
1012
|
export type SvmTransaction = {
|
|
@@ -1013,6 +1020,9 @@ export type SvmTransaction = {
|
|
|
1013
1020
|
readonly accountKeys: readonly string[];
|
|
1014
1021
|
readonly recentBlockhash?: string;
|
|
1015
1022
|
readonly version?: string;
|
|
1023
|
+
/** SPL Token / Token-2022 balance snapshots for this transaction.
|
|
1024
|
+
* Present when `include_token_balances` is `true`. */
|
|
1025
|
+
readonly tokenBalances?: readonly SvmTokenBalance[];
|
|
1016
1026
|
};
|
|
1017
1027
|
|
|
1018
1028
|
export type SvmLog = {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.0.2-svm-alpha.
|
|
3
|
+
"version": "3.0.2-svm-alpha.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -70,10 +70,10 @@
|
|
|
70
70
|
"tsx": "4.21.0"
|
|
71
71
|
},
|
|
72
72
|
"optionalDependencies": {
|
|
73
|
-
"envio-linux-x64": "3.0.2-svm-alpha.
|
|
74
|
-
"envio-linux-x64-musl": "3.0.2-svm-alpha.
|
|
75
|
-
"envio-linux-arm64": "3.0.2-svm-alpha.
|
|
76
|
-
"envio-darwin-x64": "3.0.2-svm-alpha.
|
|
77
|
-
"envio-darwin-arm64": "3.0.2-svm-alpha.
|
|
73
|
+
"envio-linux-x64": "3.0.2-svm-alpha.2",
|
|
74
|
+
"envio-linux-x64-musl": "3.0.2-svm-alpha.2",
|
|
75
|
+
"envio-linux-arm64": "3.0.2-svm-alpha.2",
|
|
76
|
+
"envio-darwin-x64": "3.0.2-svm-alpha.2",
|
|
77
|
+
"envio-darwin-arm64": "3.0.2-svm-alpha.2"
|
|
78
78
|
}
|
|
79
79
|
}
|
package/src/Batch.res
CHANGED
|
@@ -31,50 +31,7 @@ type t = {
|
|
|
31
31
|
checkpointEventsProcessed: array<int>,
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
|
|
35
|
-
It either returnes an earliest item among all chains, or None if no chains are actively indexing
|
|
36
|
-
*/
|
|
37
|
-
let getOrderedNextChain = (fetchStates: ChainMap.t<FetchState.t>, ~batchSizePerChain) => {
|
|
38
|
-
let earliestChain: ref<option<FetchState.t>> = ref(None)
|
|
39
|
-
let earliestChainTimestamp = ref(0)
|
|
40
|
-
let chainKeys = fetchStates->ChainMap.keys
|
|
41
|
-
for idx in 0 to chainKeys->Array.length - 1 {
|
|
42
|
-
let chain = chainKeys->Array.getUnsafe(idx)
|
|
43
|
-
let fetchState = fetchStates->ChainMap.get(chain)
|
|
44
|
-
if fetchState->FetchState.isActivelyIndexing {
|
|
45
|
-
let timestamp = fetchState->FetchState.getTimestampAt(
|
|
46
|
-
~index=switch batchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
47
|
-
chain->ChainMap.Chain.toChainId,
|
|
48
|
-
) {
|
|
49
|
-
| Some(batchSize) => batchSize
|
|
50
|
-
| None => 0
|
|
51
|
-
},
|
|
52
|
-
)
|
|
53
|
-
switch earliestChain.contents {
|
|
54
|
-
| Some(earliestChain)
|
|
55
|
-
if timestamp > earliestChainTimestamp.contents ||
|
|
56
|
-
(timestamp === earliestChainTimestamp.contents &&
|
|
57
|
-
chain->ChainMap.Chain.toChainId > earliestChain.chainId) => ()
|
|
58
|
-
| _ => {
|
|
59
|
-
earliestChain := Some(fetchState)
|
|
60
|
-
earliestChainTimestamp := timestamp
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
earliestChain.contents
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// Save overhead of recreating the dict every time
|
|
69
|
-
let immutableEmptyBatchSizePerChain: dict<int> = Dict.make()
|
|
70
|
-
let hasOrderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
|
|
71
|
-
switch fetchStates->getOrderedNextChain(~batchSizePerChain=immutableEmptyBatchSizePerChain) {
|
|
72
|
-
| Some(fetchState) => fetchState->FetchState.hasReadyItem
|
|
73
|
-
| None => false
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
|
|
34
|
+
let hasReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
|
|
78
35
|
fetchStates
|
|
79
36
|
->ChainMap.values
|
|
80
37
|
->Array.some(fetchState => {
|
|
@@ -82,16 +39,6 @@ let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
|
|
|
82
39
|
})
|
|
83
40
|
}
|
|
84
41
|
|
|
85
|
-
let hasMultichainReadyItem = (
|
|
86
|
-
fetchStates: ChainMap.t<FetchState.t>,
|
|
87
|
-
~multichain: Config.multichain,
|
|
88
|
-
) => {
|
|
89
|
-
switch multichain {
|
|
90
|
-
| Ordered => hasOrderedReadyItem(fetchStates)
|
|
91
|
-
| Unordered => hasUnorderedReadyItem(fetchStates)
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
|
|
95
42
|
let getProgressedChainsById = {
|
|
96
43
|
let getChainAfterBatchIfProgressed = (
|
|
97
44
|
~chainBeforeBatch: chainBeforeBatch,
|
|
@@ -214,154 +161,7 @@ let addReorgCheckpoints = (
|
|
|
214
161
|
}
|
|
215
162
|
}
|
|
216
163
|
|
|
217
|
-
let
|
|
218
|
-
~checkpointIdBeforeBatch,
|
|
219
|
-
~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
|
|
220
|
-
~batchSizeTarget,
|
|
221
|
-
) => {
|
|
222
|
-
let totalBatchSize = ref(0)
|
|
223
|
-
let isFinished = ref(false)
|
|
224
|
-
let prevCheckpointId = ref(checkpointIdBeforeBatch)
|
|
225
|
-
let mutBatchSizePerChain = Dict.make()
|
|
226
|
-
let mutProgressBlockNumberPerChain = Dict.make()
|
|
227
|
-
|
|
228
|
-
let fetchStates = chainsBeforeBatch->ChainMap.map(chainBeforeBatch => chainBeforeBatch.fetchState)
|
|
229
|
-
|
|
230
|
-
let items = []
|
|
231
|
-
let checkpointIds = []
|
|
232
|
-
let checkpointChainIds = []
|
|
233
|
-
let checkpointBlockNumbers = []
|
|
234
|
-
let checkpointBlockHashes = []
|
|
235
|
-
let checkpointEventsProcessed = []
|
|
236
|
-
|
|
237
|
-
while totalBatchSize.contents < batchSizeTarget && !isFinished.contents {
|
|
238
|
-
switch fetchStates->getOrderedNextChain(~batchSizePerChain=mutBatchSizePerChain) {
|
|
239
|
-
| Some(fetchState) => {
|
|
240
|
-
let chainBeforeBatch =
|
|
241
|
-
chainsBeforeBatch->ChainMap.get(ChainMap.Chain.makeUnsafe(~chainId=fetchState.chainId))
|
|
242
|
-
let itemsCountBefore = switch mutBatchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
243
|
-
fetchState.chainId,
|
|
244
|
-
) {
|
|
245
|
-
| Some(batchSize) => batchSize
|
|
246
|
-
| None => 0
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
let prevBlockNumber = switch mutProgressBlockNumberPerChain->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
250
|
-
fetchState.chainId,
|
|
251
|
-
) {
|
|
252
|
-
| Some(progressBlockNumber) => progressBlockNumber
|
|
253
|
-
| None => chainBeforeBatch.progressBlockNumber
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
let newItemsCount = fetchState->FetchState.getReadyItemsCount(
|
|
257
|
-
// We should get items only for a single block
|
|
258
|
-
// Since for the ordered mode next block could be after another chain's block
|
|
259
|
-
~targetSize=1,
|
|
260
|
-
~fromItem=itemsCountBefore,
|
|
261
|
-
)
|
|
262
|
-
|
|
263
|
-
if newItemsCount > 0 {
|
|
264
|
-
let item0 = fetchState.buffer->Array.getUnsafe(itemsCountBefore)
|
|
265
|
-
let blockNumber = item0->Internal.getItemBlockNumber
|
|
266
|
-
|
|
267
|
-
prevCheckpointId :=
|
|
268
|
-
addReorgCheckpoints(
|
|
269
|
-
~chainId=fetchState.chainId,
|
|
270
|
-
~reorgDetection=chainBeforeBatch.reorgDetection,
|
|
271
|
-
~prevCheckpointId=prevCheckpointId.contents,
|
|
272
|
-
~fromBlockExclusive=prevBlockNumber,
|
|
273
|
-
~toBlockExclusive=blockNumber,
|
|
274
|
-
~mutCheckpointIds=checkpointIds,
|
|
275
|
-
~mutCheckpointChainIds=checkpointChainIds,
|
|
276
|
-
~mutCheckpointBlockNumbers=checkpointBlockNumbers,
|
|
277
|
-
~mutCheckpointBlockHashes=checkpointBlockHashes,
|
|
278
|
-
~mutCheckpointEventsProcessed=checkpointEventsProcessed,
|
|
279
|
-
)
|
|
280
|
-
|
|
281
|
-
let checkpointId = prevCheckpointId.contents->BigInt.add(1n)
|
|
282
|
-
|
|
283
|
-
items
|
|
284
|
-
->Array.push(item0)
|
|
285
|
-
->ignore
|
|
286
|
-
for idx in 1 to newItemsCount - 1 {
|
|
287
|
-
items
|
|
288
|
-
->Array.push(fetchState.buffer->Belt.Array.getUnsafe(itemsCountBefore + idx))
|
|
289
|
-
->ignore
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
checkpointIds
|
|
293
|
-
->Array.push(checkpointId)
|
|
294
|
-
->ignore
|
|
295
|
-
checkpointChainIds
|
|
296
|
-
->Array.push(fetchState.chainId)
|
|
297
|
-
->ignore
|
|
298
|
-
checkpointBlockNumbers
|
|
299
|
-
->Array.push(blockNumber)
|
|
300
|
-
->ignore
|
|
301
|
-
checkpointBlockHashes
|
|
302
|
-
->Array.push(
|
|
303
|
-
chainBeforeBatch.reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber),
|
|
304
|
-
)
|
|
305
|
-
->ignore
|
|
306
|
-
checkpointEventsProcessed
|
|
307
|
-
->Array.push(newItemsCount)
|
|
308
|
-
->ignore
|
|
309
|
-
|
|
310
|
-
prevCheckpointId := checkpointId
|
|
311
|
-
totalBatchSize := totalBatchSize.contents + newItemsCount
|
|
312
|
-
mutBatchSizePerChain->Utils.Dict.setByInt(
|
|
313
|
-
fetchState.chainId,
|
|
314
|
-
itemsCountBefore + newItemsCount,
|
|
315
|
-
)
|
|
316
|
-
mutProgressBlockNumberPerChain->Utils.Dict.setByInt(fetchState.chainId, blockNumber)
|
|
317
|
-
} else {
|
|
318
|
-
let blockNumberAfterBatch = fetchState->FetchState.bufferBlockNumber
|
|
319
|
-
|
|
320
|
-
prevCheckpointId :=
|
|
321
|
-
addReorgCheckpoints(
|
|
322
|
-
~chainId=fetchState.chainId,
|
|
323
|
-
~reorgDetection=chainBeforeBatch.reorgDetection,
|
|
324
|
-
~prevCheckpointId=prevCheckpointId.contents,
|
|
325
|
-
~fromBlockExclusive=prevBlockNumber,
|
|
326
|
-
~toBlockExclusive=blockNumberAfterBatch + 1, // Make it inclusive
|
|
327
|
-
~mutCheckpointIds=checkpointIds,
|
|
328
|
-
~mutCheckpointChainIds=checkpointChainIds,
|
|
329
|
-
~mutCheckpointBlockNumbers=checkpointBlockNumbers,
|
|
330
|
-
~mutCheckpointBlockHashes=checkpointBlockHashes,
|
|
331
|
-
~mutCheckpointEventsProcessed=checkpointEventsProcessed,
|
|
332
|
-
)
|
|
333
|
-
|
|
334
|
-
// Since the chain was chosen as next
|
|
335
|
-
// the fact that it doesn't have new items means that it reached the buffer block number
|
|
336
|
-
mutProgressBlockNumberPerChain->Utils.Dict.setByInt(
|
|
337
|
-
fetchState.chainId,
|
|
338
|
-
blockNumberAfterBatch,
|
|
339
|
-
)
|
|
340
|
-
isFinished := true
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
| None => isFinished := true
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
{
|
|
349
|
-
totalBatchSize: totalBatchSize.contents,
|
|
350
|
-
items,
|
|
351
|
-
progressedChainsById: getProgressedChainsById(
|
|
352
|
-
~chainsBeforeBatch,
|
|
353
|
-
~batchSizePerChain=mutBatchSizePerChain,
|
|
354
|
-
~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
|
|
355
|
-
),
|
|
356
|
-
checkpointIds,
|
|
357
|
-
checkpointChainIds,
|
|
358
|
-
checkpointBlockNumbers,
|
|
359
|
-
checkpointBlockHashes,
|
|
360
|
-
checkpointEventsProcessed,
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
let prepareUnorderedBatch = (
|
|
164
|
+
let prepareBatch = (
|
|
365
165
|
~checkpointIdBeforeBatch,
|
|
366
166
|
~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
|
|
367
167
|
~batchSizeTarget,
|
|
@@ -454,7 +254,7 @@ let prepareUnorderedBatch = (
|
|
|
454
254
|
}
|
|
455
255
|
|
|
456
256
|
let progressBlockNumberAfterBatch =
|
|
457
|
-
fetchState->FetchState.
|
|
257
|
+
fetchState->FetchState.getProgressBlockNumberAt(~index=chainBatchSize)
|
|
458
258
|
|
|
459
259
|
prevCheckpointId :=
|
|
460
260
|
addReorgCheckpoints(
|
|
@@ -497,19 +297,9 @@ let prepareUnorderedBatch = (
|
|
|
497
297
|
let make = (
|
|
498
298
|
~checkpointIdBeforeBatch,
|
|
499
299
|
~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
|
|
500
|
-
~multichain: Config.multichain,
|
|
501
300
|
~batchSizeTarget,
|
|
502
301
|
) => {
|
|
503
|
-
|
|
504
|
-
switch multichain {
|
|
505
|
-
| Unordered => true
|
|
506
|
-
| Ordered => chainsBeforeBatch->ChainMap.size === 1
|
|
507
|
-
}
|
|
508
|
-
) {
|
|
509
|
-
prepareUnorderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
|
|
510
|
-
} else {
|
|
511
|
-
prepareOrderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
|
|
512
|
-
}
|
|
302
|
+
prepareBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
|
|
513
303
|
}
|
|
514
304
|
|
|
515
305
|
let findFirstEventBlockNumber = (batch: t, ~chainId) => {
|
package/src/Batch.res.mjs
CHANGED
|
@@ -5,38 +5,7 @@ import * as ChainMap from "./ChainMap.res.mjs";
|
|
|
5
5
|
import * as FetchState from "./FetchState.res.mjs";
|
|
6
6
|
import * as ReorgDetection from "./ReorgDetection.res.mjs";
|
|
7
7
|
|
|
8
|
-
function
|
|
9
|
-
let earliestChain;
|
|
10
|
-
let earliestChainTimestamp = 0;
|
|
11
|
-
let chainKeys = ChainMap.keys(fetchStates);
|
|
12
|
-
for (let idx = 0, idx_finish = chainKeys.length - 1; idx <= idx_finish; ++idx) {
|
|
13
|
-
let chain = chainKeys[idx];
|
|
14
|
-
let fetchState = ChainMap.get(fetchStates, chain);
|
|
15
|
-
if (FetchState.isActivelyIndexing(fetchState)) {
|
|
16
|
-
let batchSize = batchSizePerChain[chain];
|
|
17
|
-
let timestamp = FetchState.getTimestampAt(fetchState, batchSize !== undefined ? batchSize : 0);
|
|
18
|
-
let earliestChain$1 = earliestChain;
|
|
19
|
-
if (!(earliestChain$1 !== undefined && (timestamp > earliestChainTimestamp || timestamp === earliestChainTimestamp && chain > earliestChain$1.chainId))) {
|
|
20
|
-
earliestChain = fetchState;
|
|
21
|
-
earliestChainTimestamp = timestamp;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
return earliestChain;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
let immutableEmptyBatchSizePerChain = {};
|
|
29
|
-
|
|
30
|
-
function hasOrderedReadyItem(fetchStates) {
|
|
31
|
-
let fetchState = getOrderedNextChain(fetchStates, immutableEmptyBatchSizePerChain);
|
|
32
|
-
if (fetchState !== undefined) {
|
|
33
|
-
return FetchState.hasReadyItem(fetchState);
|
|
34
|
-
} else {
|
|
35
|
-
return false;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
function hasUnorderedReadyItem(fetchStates) {
|
|
8
|
+
function hasReadyItem(fetchStates) {
|
|
40
9
|
return ChainMap.values(fetchStates).some(fetchState => {
|
|
41
10
|
if (FetchState.isActivelyIndexing(fetchState)) {
|
|
42
11
|
return FetchState.hasReadyItem(fetchState);
|
|
@@ -46,14 +15,6 @@ function hasUnorderedReadyItem(fetchStates) {
|
|
|
46
15
|
});
|
|
47
16
|
}
|
|
48
17
|
|
|
49
|
-
function hasMultichainReadyItem(fetchStates, multichain) {
|
|
50
|
-
if (multichain === "ordered") {
|
|
51
|
-
return hasOrderedReadyItem(fetchStates);
|
|
52
|
-
} else {
|
|
53
|
-
return hasUnorderedReadyItem(fetchStates);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
18
|
function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
|
|
58
19
|
if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
|
|
59
20
|
return {
|
|
@@ -109,114 +70,7 @@ function addReorgCheckpoints(prevCheckpointId, reorgDetection, fromBlockExclusiv
|
|
|
109
70
|
return prevCheckpointId$1;
|
|
110
71
|
}
|
|
111
72
|
|
|
112
|
-
function
|
|
113
|
-
let totalBatchSize = 0;
|
|
114
|
-
let isFinished = false;
|
|
115
|
-
let prevCheckpointId = checkpointIdBeforeBatch;
|
|
116
|
-
let mutBatchSizePerChain = {};
|
|
117
|
-
let mutProgressBlockNumberPerChain = {};
|
|
118
|
-
let fetchStates = ChainMap.map(chainsBeforeBatch, chainBeforeBatch => chainBeforeBatch.fetchState);
|
|
119
|
-
let items = [];
|
|
120
|
-
let checkpointIds = [];
|
|
121
|
-
let checkpointChainIds = [];
|
|
122
|
-
let checkpointBlockNumbers = [];
|
|
123
|
-
let checkpointBlockHashes = [];
|
|
124
|
-
let checkpointEventsProcessed = [];
|
|
125
|
-
while (totalBatchSize < batchSizeTarget && !isFinished) {
|
|
126
|
-
let fetchState = getOrderedNextChain(fetchStates, mutBatchSizePerChain);
|
|
127
|
-
if (fetchState !== undefined) {
|
|
128
|
-
let chainBeforeBatch = ChainMap.get(chainsBeforeBatch, ChainMap.Chain.makeUnsafe(fetchState.chainId));
|
|
129
|
-
let batchSize = mutBatchSizePerChain[fetchState.chainId];
|
|
130
|
-
let itemsCountBefore = batchSize !== undefined ? batchSize : 0;
|
|
131
|
-
let progressBlockNumber = mutProgressBlockNumberPerChain[fetchState.chainId];
|
|
132
|
-
let prevBlockNumber = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
|
|
133
|
-
let newItemsCount = FetchState.getReadyItemsCount(fetchState, 1, itemsCountBefore);
|
|
134
|
-
if (newItemsCount > 0) {
|
|
135
|
-
let item0 = fetchState.buffer[itemsCountBefore];
|
|
136
|
-
let blockNumber = item0.blockNumber;
|
|
137
|
-
let chainId = fetchState.chainId;
|
|
138
|
-
let reorgDetection = chainBeforeBatch.reorgDetection;
|
|
139
|
-
let prevCheckpointId$1 = prevCheckpointId;
|
|
140
|
-
let tmp;
|
|
141
|
-
if (reorgDetection.shouldRollbackOnReorg && !Utils.Dict.isEmpty(reorgDetection.dataByBlockNumber)) {
|
|
142
|
-
let prevCheckpointId$2 = prevCheckpointId$1;
|
|
143
|
-
for (let blockNumber$1 = prevBlockNumber + 1, blockNumber_finish = blockNumber - 1; blockNumber$1 <= blockNumber_finish; ++blockNumber$1) {
|
|
144
|
-
let hash = ReorgDetection.getHashByBlockNumber(reorgDetection, blockNumber$1);
|
|
145
|
-
if (hash !== null) {
|
|
146
|
-
let checkpointId = prevCheckpointId$2 + 1n;
|
|
147
|
-
prevCheckpointId$2 = checkpointId;
|
|
148
|
-
checkpointIds.push(checkpointId);
|
|
149
|
-
checkpointChainIds.push(chainId);
|
|
150
|
-
checkpointBlockNumbers.push(blockNumber$1);
|
|
151
|
-
checkpointBlockHashes.push(hash);
|
|
152
|
-
checkpointEventsProcessed.push(0);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
tmp = prevCheckpointId$2;
|
|
156
|
-
} else {
|
|
157
|
-
tmp = prevCheckpointId$1;
|
|
158
|
-
}
|
|
159
|
-
prevCheckpointId = tmp;
|
|
160
|
-
let checkpointId$1 = prevCheckpointId + 1n;
|
|
161
|
-
items.push(item0);
|
|
162
|
-
for (let idx = 1, idx_finish = newItemsCount - 1; idx <= idx_finish; ++idx) {
|
|
163
|
-
items.push(fetchState.buffer[itemsCountBefore + idx]);
|
|
164
|
-
}
|
|
165
|
-
checkpointIds.push(checkpointId$1);
|
|
166
|
-
checkpointChainIds.push(fetchState.chainId);
|
|
167
|
-
checkpointBlockNumbers.push(blockNumber);
|
|
168
|
-
checkpointBlockHashes.push(ReorgDetection.getHashByBlockNumber(chainBeforeBatch.reorgDetection, blockNumber));
|
|
169
|
-
checkpointEventsProcessed.push(newItemsCount);
|
|
170
|
-
prevCheckpointId = checkpointId$1;
|
|
171
|
-
totalBatchSize = totalBatchSize + newItemsCount;
|
|
172
|
-
mutBatchSizePerChain[fetchState.chainId] = itemsCountBefore + newItemsCount;
|
|
173
|
-
mutProgressBlockNumberPerChain[fetchState.chainId] = blockNumber;
|
|
174
|
-
} else {
|
|
175
|
-
let blockNumberAfterBatch = FetchState.bufferBlockNumber(fetchState);
|
|
176
|
-
let chainId$1 = fetchState.chainId;
|
|
177
|
-
let toBlockExclusive = blockNumberAfterBatch + 1;
|
|
178
|
-
let reorgDetection$1 = chainBeforeBatch.reorgDetection;
|
|
179
|
-
let prevCheckpointId$3 = prevCheckpointId;
|
|
180
|
-
let tmp$1;
|
|
181
|
-
if (reorgDetection$1.shouldRollbackOnReorg && !Utils.Dict.isEmpty(reorgDetection$1.dataByBlockNumber)) {
|
|
182
|
-
let prevCheckpointId$4 = prevCheckpointId$3;
|
|
183
|
-
for (let blockNumber$2 = prevBlockNumber + 1, blockNumber_finish$1 = toBlockExclusive - 1; blockNumber$2 <= blockNumber_finish$1; ++blockNumber$2) {
|
|
184
|
-
let hash$1 = ReorgDetection.getHashByBlockNumber(reorgDetection$1, blockNumber$2);
|
|
185
|
-
if (hash$1 !== null) {
|
|
186
|
-
let checkpointId$2 = prevCheckpointId$4 + 1n;
|
|
187
|
-
prevCheckpointId$4 = checkpointId$2;
|
|
188
|
-
checkpointIds.push(checkpointId$2);
|
|
189
|
-
checkpointChainIds.push(chainId$1);
|
|
190
|
-
checkpointBlockNumbers.push(blockNumber$2);
|
|
191
|
-
checkpointBlockHashes.push(hash$1);
|
|
192
|
-
checkpointEventsProcessed.push(0);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
tmp$1 = prevCheckpointId$4;
|
|
196
|
-
} else {
|
|
197
|
-
tmp$1 = prevCheckpointId$3;
|
|
198
|
-
}
|
|
199
|
-
prevCheckpointId = tmp$1;
|
|
200
|
-
mutProgressBlockNumberPerChain[fetchState.chainId] = blockNumberAfterBatch;
|
|
201
|
-
isFinished = true;
|
|
202
|
-
}
|
|
203
|
-
} else {
|
|
204
|
-
isFinished = true;
|
|
205
|
-
}
|
|
206
|
-
};
|
|
207
|
-
return {
|
|
208
|
-
totalBatchSize: totalBatchSize,
|
|
209
|
-
items: items,
|
|
210
|
-
progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutBatchSizePerChain, mutProgressBlockNumberPerChain),
|
|
211
|
-
checkpointIds: checkpointIds,
|
|
212
|
-
checkpointChainIds: checkpointChainIds,
|
|
213
|
-
checkpointBlockNumbers: checkpointBlockNumbers,
|
|
214
|
-
checkpointBlockHashes: checkpointBlockHashes,
|
|
215
|
-
checkpointEventsProcessed: checkpointEventsProcessed
|
|
216
|
-
};
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
|
|
73
|
+
function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
|
|
220
74
|
let preparedFetchStates = FetchState.sortForUnorderedBatch(ChainMap.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
|
|
221
75
|
let chainIdx = 0;
|
|
222
76
|
let preparedNumber = preparedFetchStates.length;
|
|
@@ -281,7 +135,7 @@ function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batch
|
|
|
281
135
|
totalBatchSize = totalBatchSize + chainBatchSize;
|
|
282
136
|
mutBatchSizePerChain[fetchState.chainId] = chainBatchSize;
|
|
283
137
|
}
|
|
284
|
-
let progressBlockNumberAfterBatch = FetchState.
|
|
138
|
+
let progressBlockNumberAfterBatch = FetchState.getProgressBlockNumberAt(fetchState, chainBatchSize);
|
|
285
139
|
let chainId$1 = fetchState.chainId;
|
|
286
140
|
let toBlockExclusive = progressBlockNumberAfterBatch + 1;
|
|
287
141
|
let fromBlockExclusive$1 = prevBlockNumber;
|
|
@@ -322,15 +176,7 @@ function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batch
|
|
|
322
176
|
};
|
|
323
177
|
}
|
|
324
178
|
|
|
325
|
-
|
|
326
|
-
let tmp;
|
|
327
|
-
tmp = multichain === "ordered" ? ChainMap.size(chainsBeforeBatch) === 1 : true;
|
|
328
|
-
if (tmp) {
|
|
329
|
-
return prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget);
|
|
330
|
-
} else {
|
|
331
|
-
return prepareOrderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget);
|
|
332
|
-
}
|
|
333
|
-
}
|
|
179
|
+
let make = prepareBatch;
|
|
334
180
|
|
|
335
181
|
function findFirstEventBlockNumber(batch, chainId) {
|
|
336
182
|
let idx = 0;
|
|
@@ -362,15 +208,10 @@ function findLastEventItem(batch, chainId) {
|
|
|
362
208
|
}
|
|
363
209
|
|
|
364
210
|
export {
|
|
365
|
-
|
|
366
|
-
immutableEmptyBatchSizePerChain,
|
|
367
|
-
hasOrderedReadyItem,
|
|
368
|
-
hasUnorderedReadyItem,
|
|
369
|
-
hasMultichainReadyItem,
|
|
211
|
+
hasReadyItem,
|
|
370
212
|
getProgressedChainsById,
|
|
371
213
|
addReorgCheckpoints,
|
|
372
|
-
|
|
373
|
-
prepareUnorderedBatch,
|
|
214
|
+
prepareBatch,
|
|
374
215
|
make,
|
|
375
216
|
findFirstEventBlockNumber,
|
|
376
217
|
findLastEventItem,
|
package/src/ChainFetcher.res
CHANGED
|
@@ -51,6 +51,7 @@ let make = (
|
|
|
51
51
|
~reorgCheckpoints: array<Internal.reorgCheckpoint>,
|
|
52
52
|
~maxReorgDepth,
|
|
53
53
|
~knownHeight=0,
|
|
54
|
+
~reducedPollingInterval=?,
|
|
54
55
|
): t => {
|
|
55
56
|
// We don't need the router itself, but only validation logic,
|
|
56
57
|
// since now event router is created for selection of events
|
|
@@ -210,10 +211,6 @@ let make = (
|
|
|
210
211
|
Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
|
|
211
212
|
),
|
|
212
213
|
})
|
|
213
|
-
// Collect all event signatures from contracts
|
|
214
|
-
let allEventSignatures =
|
|
215
|
-
chainConfig.contracts->Array.flatMap(contract => contract.eventSignatures)
|
|
216
|
-
// Convert rpcs to EvmChain.rpc format
|
|
217
214
|
let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
|
|
218
215
|
let syncConfig = rpc.syncConfig
|
|
219
216
|
let ws = rpc.ws
|
|
@@ -228,7 +225,6 @@ let make = (
|
|
|
228
225
|
~chain,
|
|
229
226
|
~contracts=evmContracts,
|
|
230
227
|
~hyperSync=hypersync,
|
|
231
|
-
~allEventSignatures,
|
|
232
228
|
~rpcs=evmRpcs,
|
|
233
229
|
~lowercaseAddresses,
|
|
234
230
|
)
|
|
@@ -269,6 +265,7 @@ let make = (
|
|
|
269
265
|
~sources,
|
|
270
266
|
~maxPartitionConcurrency=Env.maxPartitionConcurrency,
|
|
271
267
|
~isRealtime,
|
|
268
|
+
~reducedPollingInterval?,
|
|
272
269
|
),
|
|
273
270
|
reorgDetection: ReorgDetection.make(
|
|
274
271
|
~chainReorgCheckpoints,
|
|
@@ -329,6 +326,7 @@ let makeFromDbState = (
|
|
|
329
326
|
~config,
|
|
330
327
|
~registrations,
|
|
331
328
|
~targetBufferSize,
|
|
329
|
+
~reducedPollingInterval=?,
|
|
332
330
|
) => {
|
|
333
331
|
let chainId = chainConfig.id
|
|
334
332
|
let logger = Logging.createChild(~params={"chainId": chainId})
|
|
@@ -361,6 +359,7 @@ let makeFromDbState = (
|
|
|
361
359
|
~isInReorgThreshold,
|
|
362
360
|
~isRealtime,
|
|
363
361
|
~knownHeight=resumedChainState.sourceBlockNumber,
|
|
362
|
+
~reducedPollingInterval?,
|
|
364
363
|
)
|
|
365
364
|
}
|
|
366
365
|
|
package/src/ChainFetcher.res.mjs
CHANGED
|
@@ -38,7 +38,7 @@ function configAddresses(chainConfig) {
|
|
|
38
38
|
return addresses;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt) {
|
|
41
|
+
function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, reducedPollingInterval) {
|
|
42
42
|
let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
|
|
43
43
|
let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
|
|
44
44
|
let eventRouter = EventRouter.empty();
|
|
@@ -114,7 +114,6 @@ function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBl
|
|
|
114
114
|
abi: contract.abi,
|
|
115
115
|
events: contract.events
|
|
116
116
|
}));
|
|
117
|
-
let allEventSignatures = chainConfig.contracts.flatMap(contract => contract.eventSignatures);
|
|
118
117
|
let evmRpcs = sources.rpcs.map(rpc => {
|
|
119
118
|
let syncConfig = rpc.syncConfig;
|
|
120
119
|
let ws = rpc.ws;
|
|
@@ -125,7 +124,7 @@ function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBl
|
|
|
125
124
|
ws: ws
|
|
126
125
|
};
|
|
127
126
|
});
|
|
128
|
-
sources$1 = EvmChain.makeSources(chain, evmContracts, sources.hypersync,
|
|
127
|
+
sources$1 = EvmChain.makeSources(chain, evmContracts, sources.hypersync, evmRpcs, lowercaseAddresses);
|
|
129
128
|
break;
|
|
130
129
|
case "FuelSourceConfig" :
|
|
131
130
|
sources$1 = [HyperFuelSource.make({
|
|
@@ -160,7 +159,7 @@ function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBl
|
|
|
160
159
|
return {
|
|
161
160
|
logger: logger,
|
|
162
161
|
fetchState: fetchState,
|
|
163
|
-
sourceManager: SourceManager.make(sources$1, Env.maxPartitionConcurrency, isRealtime, undefined, undefined, undefined,
|
|
162
|
+
sourceManager: SourceManager.make(sources$1, Env.maxPartitionConcurrency, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined),
|
|
164
163
|
chainConfig: chainConfig,
|
|
165
164
|
isProgressAtHead: false,
|
|
166
165
|
timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
|
|
@@ -175,17 +174,17 @@ function makeFromConfig(chainConfig, config, registrations, targetBufferSize, kn
|
|
|
175
174
|
let logger = Logging.createChild({
|
|
176
175
|
chainId: chainConfig.id
|
|
177
176
|
});
|
|
178
|
-
return make(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight);
|
|
177
|
+
return make(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined);
|
|
179
178
|
}
|
|
180
179
|
|
|
181
|
-
function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrations, targetBufferSize) {
|
|
180
|
+
function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrations, targetBufferSize, reducedPollingInterval) {
|
|
182
181
|
let chainId = chainConfig.id;
|
|
183
182
|
let logger = Logging.createChild({
|
|
184
183
|
chainId: chainId
|
|
185
184
|
});
|
|
186
185
|
Prometheus.ProgressEventsCount.set(resumedChainState.numEventsProcessed, chainId);
|
|
187
186
|
let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
|
|
188
|
-
return make(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber);
|
|
187
|
+
return make(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, reducedPollingInterval);
|
|
189
188
|
}
|
|
190
189
|
|
|
191
190
|
async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
|