envio 3.0.0-alpha.11 → 3.0.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +26 -100
- package/package.json +5 -5
- package/src/Batch.res +2 -0
- package/src/Batch.res.mjs +2 -1
- package/src/ChainFetcher.res +32 -23
- package/src/ChainFetcher.res.mjs +19 -20
- package/src/ChainManager.res +8 -21
- package/src/ChainManager.res.mjs +14 -18
- package/src/Config.res +10 -1
- package/src/Config.res.mjs +9 -4
- package/src/FetchState.res +1160 -622
- package/src/FetchState.res.mjs +1031 -575
- package/src/GlobalState.res +12 -8
- package/src/GlobalState.res.mjs +33 -40
- package/src/PgStorage.res +28 -1
- package/src/PgStorage.res.mjs +31 -1
- package/src/Utils.res +21 -1
- package/src/Utils.res.mjs +30 -4
- package/src/bindings/Ethers.res +20 -1
- package/src/bindings/Ethers.res.mjs +15 -1
- package/src/bindings/WebSocket.res +27 -0
- package/src/bindings/WebSocket.res.mjs +2 -0
- package/src/db/InternalTable.res +5 -3
- package/src/db/InternalTable.res.mjs +6 -4
- package/src/db/Table.res +21 -8
- package/src/db/Table.res.mjs +20 -10
- package/src/sources/EvmChain.res +16 -12
- package/src/sources/EvmChain.res.mjs +13 -10
- package/src/sources/HyperSyncSource.res +15 -4
- package/src/sources/HyperSyncSource.res.mjs +1 -1
- package/src/sources/Rpc.res +46 -21
- package/src/sources/Rpc.res.mjs +43 -18
- package/src/sources/RpcSource.res +127 -40
- package/src/sources/RpcSource.res.mjs +118 -43
- package/src/sources/RpcWebSocketHeightStream.res +158 -0
- package/src/sources/RpcWebSocketHeightStream.res.mjs +169 -0
- package/src/sources/SourceManager.res +2 -10
- package/src/sources/SourceManager.res.mjs +9 -13
package/evm.schema.json
CHANGED
|
@@ -307,17 +307,6 @@
|
|
|
307
307
|
"format": "uint64",
|
|
308
308
|
"minimum": 0
|
|
309
309
|
},
|
|
310
|
-
"rpc_config": {
|
|
311
|
-
"description": "RPC configuration for utilizing as the chain's data-source. Typically optional for chains with HyperSync support, which is highly recommended. HyperSync dramatically enhances performance, providing up to a 1000x speed boost over traditional RPC.",
|
|
312
|
-
"anyOf": [
|
|
313
|
-
{
|
|
314
|
-
"$ref": "#/$defs/RpcConfig"
|
|
315
|
-
},
|
|
316
|
-
{
|
|
317
|
-
"type": "null"
|
|
318
|
-
}
|
|
319
|
-
]
|
|
320
|
-
},
|
|
321
310
|
"rpc": {
|
|
322
311
|
"description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
|
|
323
312
|
"anyOf": [
|
|
@@ -380,91 +369,6 @@
|
|
|
380
369
|
"start_block"
|
|
381
370
|
]
|
|
382
371
|
},
|
|
383
|
-
"RpcConfig": {
|
|
384
|
-
"type": "object",
|
|
385
|
-
"properties": {
|
|
386
|
-
"url": {
|
|
387
|
-
"description": "URL of the RPC endpoint. Can be a single URL or an array of URLs. If multiple URLs are provided, the first one will be used as the primary RPC endpoint and the rest will be used as fallbacks.",
|
|
388
|
-
"anyOf": [
|
|
389
|
-
{
|
|
390
|
-
"type": "string"
|
|
391
|
-
},
|
|
392
|
-
{
|
|
393
|
-
"type": "array",
|
|
394
|
-
"items": {
|
|
395
|
-
"type": "string"
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
]
|
|
399
|
-
},
|
|
400
|
-
"initial_block_interval": {
|
|
401
|
-
"description": "The starting interval in range of blocks per query",
|
|
402
|
-
"type": [
|
|
403
|
-
"integer",
|
|
404
|
-
"null"
|
|
405
|
-
],
|
|
406
|
-
"format": "uint32",
|
|
407
|
-
"minimum": 0
|
|
408
|
-
},
|
|
409
|
-
"backoff_multiplicative": {
|
|
410
|
-
"description": "After an RPC error, how much to scale back the number of blocks requested at once",
|
|
411
|
-
"type": [
|
|
412
|
-
"number",
|
|
413
|
-
"null"
|
|
414
|
-
],
|
|
415
|
-
"format": "double"
|
|
416
|
-
},
|
|
417
|
-
"acceleration_additive": {
|
|
418
|
-
"description": "Without RPC errors or timeouts, how much to increase the number of blocks requested by for the next batch",
|
|
419
|
-
"type": [
|
|
420
|
-
"integer",
|
|
421
|
-
"null"
|
|
422
|
-
],
|
|
423
|
-
"format": "uint32",
|
|
424
|
-
"minimum": 0
|
|
425
|
-
},
|
|
426
|
-
"interval_ceiling": {
|
|
427
|
-
"description": "Do not further increase the block interval past this limit",
|
|
428
|
-
"type": [
|
|
429
|
-
"integer",
|
|
430
|
-
"null"
|
|
431
|
-
],
|
|
432
|
-
"format": "uint32",
|
|
433
|
-
"minimum": 0
|
|
434
|
-
},
|
|
435
|
-
"backoff_millis": {
|
|
436
|
-
"description": "After an error, how long to wait before retrying",
|
|
437
|
-
"type": [
|
|
438
|
-
"integer",
|
|
439
|
-
"null"
|
|
440
|
-
],
|
|
441
|
-
"format": "uint32",
|
|
442
|
-
"minimum": 0
|
|
443
|
-
},
|
|
444
|
-
"fallback_stall_timeout": {
|
|
445
|
-
"description": "If a fallback RPC is provided, the amount of time in ms to wait before kicking off the next provider",
|
|
446
|
-
"type": [
|
|
447
|
-
"integer",
|
|
448
|
-
"null"
|
|
449
|
-
],
|
|
450
|
-
"format": "uint32",
|
|
451
|
-
"minimum": 0
|
|
452
|
-
},
|
|
453
|
-
"query_timeout_millis": {
|
|
454
|
-
"description": "How long to wait before cancelling an RPC request",
|
|
455
|
-
"type": [
|
|
456
|
-
"integer",
|
|
457
|
-
"null"
|
|
458
|
-
],
|
|
459
|
-
"format": "uint32",
|
|
460
|
-
"minimum": 0
|
|
461
|
-
}
|
|
462
|
-
},
|
|
463
|
-
"additionalProperties": false,
|
|
464
|
-
"required": [
|
|
465
|
-
"url"
|
|
466
|
-
]
|
|
467
|
-
},
|
|
468
372
|
"RpcSelection": {
|
|
469
373
|
"anyOf": [
|
|
470
374
|
{
|
|
@@ -489,8 +393,22 @@
|
|
|
489
393
|
"type": "string"
|
|
490
394
|
},
|
|
491
395
|
"for": {
|
|
492
|
-
"description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback.",
|
|
493
|
-
"
|
|
396
|
+
"description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback. If not specified, defaults to \"fallback\" when HyperSync is available for the chain, or \"sync\" otherwise.",
|
|
397
|
+
"anyOf": [
|
|
398
|
+
{
|
|
399
|
+
"$ref": "#/$defs/For"
|
|
400
|
+
},
|
|
401
|
+
{
|
|
402
|
+
"type": "null"
|
|
403
|
+
}
|
|
404
|
+
]
|
|
405
|
+
},
|
|
406
|
+
"ws": {
|
|
407
|
+
"description": "Optional WebSocket endpoint URL (wss:// or ws://) for real-time block header notifications via eth_subscribe(\"newHeads\"). Provides lower latency than HTTP polling for detecting new blocks.",
|
|
408
|
+
"type": [
|
|
409
|
+
"string",
|
|
410
|
+
"null"
|
|
411
|
+
]
|
|
494
412
|
},
|
|
495
413
|
"initial_block_interval": {
|
|
496
414
|
"description": "The starting interval in range of blocks per query",
|
|
@@ -553,12 +471,20 @@
|
|
|
553
471
|
],
|
|
554
472
|
"format": "uint32",
|
|
555
473
|
"minimum": 0
|
|
474
|
+
},
|
|
475
|
+
"polling_interval": {
|
|
476
|
+
"description": "How frequently (in milliseconds) to check for new blocks in realtime. Default is 1000ms. Note: Setting this higher than block time does not reduce RPC usage as every block is still fetched to check for reorgs.",
|
|
477
|
+
"type": [
|
|
478
|
+
"integer",
|
|
479
|
+
"null"
|
|
480
|
+
],
|
|
481
|
+
"format": "uint32",
|
|
482
|
+
"minimum": 0
|
|
556
483
|
}
|
|
557
484
|
},
|
|
558
485
|
"additionalProperties": false,
|
|
559
486
|
"required": [
|
|
560
|
-
"url"
|
|
561
|
-
"for"
|
|
487
|
+
"url"
|
|
562
488
|
]
|
|
563
489
|
},
|
|
564
490
|
"For": {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "v3.0.0-alpha.
|
|
3
|
+
"version": "v3.0.0-alpha.13",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.js",
|
|
@@ -29,10 +29,10 @@
|
|
|
29
29
|
"node": ">=22.0.0"
|
|
30
30
|
},
|
|
31
31
|
"optionalDependencies": {
|
|
32
|
-
"envio-linux-x64": "v3.0.0-alpha.
|
|
33
|
-
"envio-linux-arm64": "v3.0.0-alpha.
|
|
34
|
-
"envio-darwin-x64": "v3.0.0-alpha.
|
|
35
|
-
"envio-darwin-arm64": "v3.0.0-alpha.
|
|
32
|
+
"envio-linux-x64": "v3.0.0-alpha.13",
|
|
33
|
+
"envio-linux-arm64": "v3.0.0-alpha.13",
|
|
34
|
+
"envio-darwin-x64": "v3.0.0-alpha.13",
|
|
35
|
+
"envio-darwin-arm64": "v3.0.0-alpha.13"
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
38
|
"@clickhouse/client": "1.12.1",
|
package/src/Batch.res
CHANGED
|
@@ -6,6 +6,7 @@ open Utils.UnsafeIntOperators
|
|
|
6
6
|
type chainAfterBatch = {
|
|
7
7
|
batchSize: int,
|
|
8
8
|
progressBlockNumber: int,
|
|
9
|
+
sourceBlockNumber: int,
|
|
9
10
|
totalEventsProcessed: int,
|
|
10
11
|
fetchState: FetchState.t,
|
|
11
12
|
isProgressAtHeadWhenBatchCreated: bool,
|
|
@@ -108,6 +109,7 @@ let getProgressedChainsById = {
|
|
|
108
109
|
{
|
|
109
110
|
batchSize,
|
|
110
111
|
progressBlockNumber: progressBlockNumberAfterBatch,
|
|
112
|
+
sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
|
|
111
113
|
totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
|
|
112
114
|
fetchState: fetchStateAfterBatch,
|
|
113
115
|
isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >=
|
package/src/Batch.res.mjs
CHANGED
|
@@ -62,6 +62,7 @@ function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAft
|
|
|
62
62
|
return {
|
|
63
63
|
batchSize: batchSize,
|
|
64
64
|
progressBlockNumber: progressBlockNumberAfterBatch,
|
|
65
|
+
sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
|
|
65
66
|
totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
|
|
66
67
|
fetchState: fetchStateAfterBatch,
|
|
67
68
|
isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber
|
|
@@ -80,7 +81,7 @@ function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressB
|
|
|
80
81
|
var progressedChain;
|
|
81
82
|
if (batchSize !== undefined) {
|
|
82
83
|
var leftItems = fetchState.buffer.slice(batchSize);
|
|
83
|
-
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, undefined,
|
|
84
|
+
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, undefined, leftItems, undefined, undefined), batchSize);
|
|
84
85
|
} else {
|
|
85
86
|
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, 0);
|
|
86
87
|
}
|
package/src/ChainFetcher.res
CHANGED
|
@@ -202,27 +202,29 @@ let make = (
|
|
|
202
202
|
let sources = switch chainConfig.sourceConfig {
|
|
203
203
|
| Config.EvmSourceConfig({hypersync, rpcs}) =>
|
|
204
204
|
// Build Internal.evmContractConfig from contracts for EvmChain.makeSources
|
|
205
|
-
let evmContracts: array<Internal.evmContractConfig> =
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
205
|
+
let evmContracts: array<Internal.evmContractConfig> = chainConfig.contracts->Array.map((
|
|
206
|
+
contract
|
|
207
|
+
): Internal.evmContractConfig => {
|
|
208
|
+
name: contract.name,
|
|
209
|
+
abi: contract.abi,
|
|
210
|
+
events: contract.events->(
|
|
211
|
+
Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
|
|
212
|
+
),
|
|
213
|
+
})
|
|
213
214
|
// Collect all event signatures from contracts
|
|
214
215
|
let allEventSignatures =
|
|
215
216
|
chainConfig.contracts->Array.flatMap(contract => contract.eventSignatures)
|
|
216
217
|
// Convert rpcs to EvmChain.rpc format
|
|
217
|
-
let evmRpcs: array<EvmChain.rpc> =
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
}
|
|
218
|
+
let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
|
|
219
|
+
let syncConfig = rpc.syncConfig
|
|
220
|
+
let ws = rpc.ws
|
|
221
|
+
{
|
|
222
|
+
url: rpc.url,
|
|
223
|
+
sourceFor: rpc.sourceFor,
|
|
224
|
+
?syncConfig,
|
|
225
|
+
?ws,
|
|
226
|
+
}
|
|
227
|
+
})
|
|
226
228
|
EvmChain.makeSources(
|
|
227
229
|
~chain,
|
|
228
230
|
~contracts=evmContracts,
|
|
@@ -264,7 +266,13 @@ let make = (
|
|
|
264
266
|
}
|
|
265
267
|
}
|
|
266
268
|
|
|
267
|
-
let makeFromConfig = (
|
|
269
|
+
let makeFromConfig = (
|
|
270
|
+
chainConfig: Config.chain,
|
|
271
|
+
~config,
|
|
272
|
+
~registrations,
|
|
273
|
+
~targetBufferSize,
|
|
274
|
+
~knownHeight,
|
|
275
|
+
) => {
|
|
268
276
|
let logger = Logging.createChild(~params={"chainId": chainConfig.id})
|
|
269
277
|
|
|
270
278
|
make(
|
|
@@ -284,6 +292,7 @@ let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~targe
|
|
|
284
292
|
~logger,
|
|
285
293
|
~dynamicContracts=[],
|
|
286
294
|
~isInReorgThreshold=false,
|
|
295
|
+
~knownHeight,
|
|
287
296
|
)
|
|
288
297
|
}
|
|
289
298
|
|
|
@@ -446,12 +455,12 @@ let handleQueryResult = (
|
|
|
446
455
|
| _ => chainFetcher.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
|
|
447
456
|
}
|
|
448
457
|
|
|
449
|
-
|
|
450
|
-
->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
|
|
451
|
-
->Result.map(fs => {
|
|
458
|
+
{
|
|
452
459
|
...chainFetcher,
|
|
453
|
-
fetchState: fs
|
|
454
|
-
|
|
460
|
+
fetchState: fs
|
|
461
|
+
->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
|
|
462
|
+
->FetchState.updateKnownHeight(~knownHeight),
|
|
463
|
+
}
|
|
455
464
|
}
|
|
456
465
|
|
|
457
466
|
/**
|
package/src/ChainFetcher.res.mjs
CHANGED
|
@@ -13,7 +13,6 @@ import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
|
|
|
13
13
|
import * as FetchState from "./FetchState.res.mjs";
|
|
14
14
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
15
15
|
import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
|
|
16
|
-
import * as Belt_Result from "rescript/lib/es6/belt_Result.js";
|
|
17
16
|
import * as Caml_option from "rescript/lib/es6/caml_option.js";
|
|
18
17
|
import * as EventRouter from "./sources/EventRouter.res.mjs";
|
|
19
18
|
import * as UserContext from "./UserContext.res.mjs";
|
|
@@ -125,10 +124,12 @@ function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlo
|
|
|
125
124
|
}));
|
|
126
125
|
var evmRpcs = Belt_Array.map(sources.rpcs, (function (rpc) {
|
|
127
126
|
var syncConfig = rpc.syncConfig;
|
|
127
|
+
var ws = rpc.ws;
|
|
128
128
|
return {
|
|
129
129
|
url: rpc.url,
|
|
130
130
|
sourceFor: rpc.sourceFor,
|
|
131
|
-
syncConfig: syncConfig
|
|
131
|
+
syncConfig: syncConfig,
|
|
132
|
+
ws: ws
|
|
132
133
|
};
|
|
133
134
|
}));
|
|
134
135
|
sources$1 = EvmChain.makeSources(chain, evmContracts, sources.hypersync, allEventSignatures, evmRpcs, lowercaseAddresses);
|
|
@@ -163,11 +164,11 @@ function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlo
|
|
|
163
164
|
};
|
|
164
165
|
}
|
|
165
166
|
|
|
166
|
-
function makeFromConfig(chainConfig, config, registrations, targetBufferSize) {
|
|
167
|
+
function makeFromConfig(chainConfig, config, registrations, targetBufferSize, knownHeight) {
|
|
167
168
|
var logger = Logging.createChild({
|
|
168
169
|
chainId: chainConfig.id
|
|
169
170
|
});
|
|
170
|
-
return make(chainConfig, [], chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, 0, false, [], chainConfig.maxReorgDepth,
|
|
171
|
+
return make(chainConfig, [], chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, 0, false, [], chainConfig.maxReorgDepth, knownHeight);
|
|
171
172
|
}
|
|
172
173
|
|
|
173
174
|
async function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, config, registrations, targetBufferSize) {
|
|
@@ -254,22 +255,20 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, chain, con
|
|
|
254
255
|
|
|
255
256
|
function handleQueryResult(chainFetcher, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight) {
|
|
256
257
|
var fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(chainFetcher.fetchState, newItemsWithDcs) : chainFetcher.fetchState;
|
|
257
|
-
return
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
};
|
|
272
|
-
}));
|
|
258
|
+
return {
|
|
259
|
+
logger: chainFetcher.logger,
|
|
260
|
+
fetchState: FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight),
|
|
261
|
+
sourceManager: chainFetcher.sourceManager,
|
|
262
|
+
chainConfig: chainFetcher.chainConfig,
|
|
263
|
+
isProgressAtHead: chainFetcher.isProgressAtHead,
|
|
264
|
+
timestampCaughtUpToHeadOrEndblock: chainFetcher.timestampCaughtUpToHeadOrEndblock,
|
|
265
|
+
committedProgressBlockNumber: chainFetcher.committedProgressBlockNumber,
|
|
266
|
+
firstEventBlockNumber: chainFetcher.firstEventBlockNumber,
|
|
267
|
+
numEventsProcessed: chainFetcher.numEventsProcessed,
|
|
268
|
+
numBatchesFetched: chainFetcher.numBatchesFetched,
|
|
269
|
+
reorgDetection: chainFetcher.reorgDetection,
|
|
270
|
+
safeCheckpointTracking: chainFetcher.safeCheckpointTracking
|
|
271
|
+
};
|
|
273
272
|
}
|
|
274
273
|
|
|
275
274
|
function hasProcessedToEndblock(self) {
|
package/src/ChainManager.res
CHANGED
|
@@ -16,29 +16,17 @@ let isProgressInReorgThreshold = (~progressBlockNumber, ~sourceBlockNumber, ~max
|
|
|
16
16
|
progressBlockNumber > sourceBlockNumber - maxReorgDepth
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
let calculateTargetBufferSize = (~activeChainsCount
|
|
20
|
-
let targetBatchesInBuffer = 3
|
|
19
|
+
let calculateTargetBufferSize = (~activeChainsCount) => {
|
|
21
20
|
switch Env.targetBufferSize {
|
|
22
21
|
| Some(size) => size
|
|
23
22
|
| None =>
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
~config,
|
|
32
|
-
)
|
|
33
|
-
let chainFetchers =
|
|
34
|
-
config.chainMap->ChainMap.map(
|
|
35
|
-
ChainFetcher.makeFromConfig(_, ~config, ~registrations, ~targetBufferSize),
|
|
36
|
-
)
|
|
37
|
-
{
|
|
38
|
-
committedCheckpointId: 0.,
|
|
39
|
-
chainFetchers,
|
|
40
|
-
multichain: config.multichain,
|
|
41
|
-
isInReorgThreshold: false,
|
|
23
|
+
switch activeChainsCount {
|
|
24
|
+
| 1 => 50_000
|
|
25
|
+
| 2 => 30_000
|
|
26
|
+
| 3 => 20_000
|
|
27
|
+
| 4 => 15_000
|
|
28
|
+
| _ => 10_000
|
|
29
|
+
}
|
|
42
30
|
}
|
|
43
31
|
}
|
|
44
32
|
|
|
@@ -62,7 +50,6 @@ let makeFromDbState = async (
|
|
|
62
50
|
|
|
63
51
|
let targetBufferSize = calculateTargetBufferSize(
|
|
64
52
|
~activeChainsCount=initialState.chains->Array.length,
|
|
65
|
-
~config,
|
|
66
53
|
)
|
|
67
54
|
Prometheus.ProcessingMaxBatchSize.set(~maxBatchSize=config.batchSize)
|
|
68
55
|
Prometheus.IndexingTargetBufferSize.set(~targetBufferSize)
|
package/src/ChainManager.res.mjs
CHANGED
|
@@ -18,32 +18,29 @@ function isProgressInReorgThreshold(progressBlockNumber, sourceBlockNumber, maxR
|
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
function calculateTargetBufferSize(activeChainsCount
|
|
21
|
+
function calculateTargetBufferSize(activeChainsCount) {
|
|
22
22
|
if (Env.targetBufferSize !== undefined) {
|
|
23
23
|
return Env.targetBufferSize;
|
|
24
|
-
} else {
|
|
25
|
-
return Math.imul(config.batchSize, activeChainsCount > 3 ? 1 : 3);
|
|
26
24
|
}
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
};
|
|
25
|
+
switch (activeChainsCount) {
|
|
26
|
+
case 1 :
|
|
27
|
+
return 50000;
|
|
28
|
+
case 2 :
|
|
29
|
+
return 30000;
|
|
30
|
+
case 3 :
|
|
31
|
+
return 20000;
|
|
32
|
+
case 4 :
|
|
33
|
+
return 15000;
|
|
34
|
+
default:
|
|
35
|
+
return 10000;
|
|
36
|
+
}
|
|
40
37
|
}
|
|
41
38
|
|
|
42
39
|
async function makeFromDbState(initialState, config, registrations) {
|
|
43
40
|
var isInReorgThreshold = initialState.cleanRun ? false : Belt_Array.some(initialState.chains, (function (chain) {
|
|
44
41
|
return isProgressInReorgThreshold(chain.progressBlockNumber, chain.sourceBlockNumber, chain.maxReorgDepth);
|
|
45
42
|
}));
|
|
46
|
-
var targetBufferSize = calculateTargetBufferSize(initialState.chains.length
|
|
43
|
+
var targetBufferSize = calculateTargetBufferSize(initialState.chains.length);
|
|
47
44
|
Prometheus.ProcessingMaxBatchSize.set(config.batchSize);
|
|
48
45
|
Prometheus.IndexingTargetBufferSize.set(targetBufferSize);
|
|
49
46
|
Prometheus.ReorgThreshold.set(isInReorgThreshold);
|
|
@@ -137,7 +134,6 @@ function getSafeCheckpointId(chainManager) {
|
|
|
137
134
|
export {
|
|
138
135
|
isProgressInReorgThreshold ,
|
|
139
136
|
calculateTargetBufferSize ,
|
|
140
|
-
makeFromConfig ,
|
|
141
137
|
makeFromDbState ,
|
|
142
138
|
getChainFetcher ,
|
|
143
139
|
setChainFetcher ,
|
package/src/Config.res
CHANGED
|
@@ -8,6 +8,7 @@ type sourceSyncOptions = {
|
|
|
8
8
|
backoffMillis?: int,
|
|
9
9
|
queryTimeoutMillis?: int,
|
|
10
10
|
fallbackStallTimeout?: int,
|
|
11
|
+
pollingInterval?: int,
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
type contract = {
|
|
@@ -38,6 +39,7 @@ type evmRpcConfig = {
|
|
|
38
39
|
url: string,
|
|
39
40
|
sourceFor: Source.sourceFor,
|
|
40
41
|
syncConfig: option<sourceSyncOptions>,
|
|
42
|
+
ws: option<string>,
|
|
41
43
|
}
|
|
42
44
|
|
|
43
45
|
type sourceConfig =
|
|
@@ -65,6 +67,7 @@ type sourceSync = {
|
|
|
65
67
|
backoffMillis: int,
|
|
66
68
|
queryTimeoutMillis: int,
|
|
67
69
|
fallbackStallTimeout: int,
|
|
70
|
+
pollingInterval: int,
|
|
68
71
|
}
|
|
69
72
|
|
|
70
73
|
type multichain = | @as("ordered") Ordered | @as("unordered") Unordered
|
|
@@ -102,6 +105,7 @@ let rpcConfigSchema = S.schema(s =>
|
|
|
102
105
|
{
|
|
103
106
|
"url": s.matches(S.string),
|
|
104
107
|
"for": s.matches(rpcSourceForSchema),
|
|
108
|
+
"ws": s.matches(S.option(S.string)),
|
|
105
109
|
"initialBlockInterval": s.matches(S.option(S.int)),
|
|
106
110
|
"backoffMultiplicative": s.matches(S.option(S.float)),
|
|
107
111
|
"accelerationAdditive": s.matches(S.option(S.int)),
|
|
@@ -109,6 +113,7 @@ let rpcConfigSchema = S.schema(s =>
|
|
|
109
113
|
"backoffMillis": s.matches(S.option(S.int)),
|
|
110
114
|
"fallbackStallTimeout": s.matches(S.option(S.int)),
|
|
111
115
|
"queryTimeoutMillis": s.matches(S.option(S.int)),
|
|
116
|
+
"pollingInterval": s.matches(S.option(S.int)),
|
|
112
117
|
}
|
|
113
118
|
)
|
|
114
119
|
|
|
@@ -323,6 +328,7 @@ let fromPublic = (
|
|
|
323
328
|
let backoffMillis = rpcConfig["backoffMillis"]
|
|
324
329
|
let queryTimeoutMillis = rpcConfig["queryTimeoutMillis"]
|
|
325
330
|
let fallbackStallTimeout = rpcConfig["fallbackStallTimeout"]
|
|
331
|
+
let pollingInterval = rpcConfig["pollingInterval"]
|
|
326
332
|
let hasSyncConfig =
|
|
327
333
|
initialBlockInterval->Option.isSome ||
|
|
328
334
|
backoffMultiplicative->Option.isSome ||
|
|
@@ -330,7 +336,8 @@ let fromPublic = (
|
|
|
330
336
|
intervalCeiling->Option.isSome ||
|
|
331
337
|
backoffMillis->Option.isSome ||
|
|
332
338
|
queryTimeoutMillis->Option.isSome ||
|
|
333
|
-
fallbackStallTimeout->Option.isSome
|
|
339
|
+
fallbackStallTimeout->Option.isSome ||
|
|
340
|
+
pollingInterval->Option.isSome
|
|
334
341
|
let syncConfig: option<sourceSyncOptions> = if hasSyncConfig {
|
|
335
342
|
Some({
|
|
336
343
|
?initialBlockInterval,
|
|
@@ -340,6 +347,7 @@ let fromPublic = (
|
|
|
340
347
|
?backoffMillis,
|
|
341
348
|
?queryTimeoutMillis,
|
|
342
349
|
?fallbackStallTimeout,
|
|
350
|
+
?pollingInterval,
|
|
343
351
|
})
|
|
344
352
|
} else {
|
|
345
353
|
None
|
|
@@ -348,6 +356,7 @@ let fromPublic = (
|
|
|
348
356
|
url: rpcConfig["url"],
|
|
349
357
|
sourceFor: parseRpcSourceFor(rpcConfig["for"]),
|
|
350
358
|
syncConfig,
|
|
359
|
+
ws: rpcConfig["ws"],
|
|
351
360
|
}
|
|
352
361
|
})
|
|
353
362
|
EvmSourceConfig({hypersync: publicChainConfig["hypersync"], rpcs})
|
package/src/Config.res.mjs
CHANGED
|
@@ -24,13 +24,15 @@ var rpcConfigSchema = S$RescriptSchema.schema(function (s) {
|
|
|
24
24
|
return {
|
|
25
25
|
url: s.m(S$RescriptSchema.string),
|
|
26
26
|
for: s.m(rpcSourceForSchema),
|
|
27
|
+
ws: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
|
|
27
28
|
initialBlockInterval: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
|
|
28
29
|
backoffMultiplicative: s.m(S$RescriptSchema.option(S$RescriptSchema.$$float)),
|
|
29
30
|
accelerationAdditive: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
|
|
30
31
|
intervalCeiling: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
|
|
31
32
|
backoffMillis: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
|
|
32
33
|
fallbackStallTimeout: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
|
|
33
|
-
queryTimeoutMillis: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int))
|
|
34
|
+
queryTimeoutMillis: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
|
|
35
|
+
pollingInterval: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int))
|
|
34
36
|
};
|
|
35
37
|
});
|
|
36
38
|
|
|
@@ -235,7 +237,8 @@ function fromPublic(publicConfigJson, codegenChainsOpt, maxAddrInPartitionOpt, u
|
|
|
235
237
|
var backoffMillis = rpcConfig.backoffMillis;
|
|
236
238
|
var queryTimeoutMillis = rpcConfig.queryTimeoutMillis;
|
|
237
239
|
var fallbackStallTimeout = rpcConfig.fallbackStallTimeout;
|
|
238
|
-
var
|
|
240
|
+
var pollingInterval = rpcConfig.pollingInterval;
|
|
241
|
+
var hasSyncConfig = Belt_Option.isSome(initialBlockInterval) || Belt_Option.isSome(backoffMultiplicative) || Belt_Option.isSome(accelerationAdditive) || Belt_Option.isSome(intervalCeiling) || Belt_Option.isSome(backoffMillis) || Belt_Option.isSome(queryTimeoutMillis) || Belt_Option.isSome(fallbackStallTimeout) || Belt_Option.isSome(pollingInterval);
|
|
239
242
|
var syncConfig = hasSyncConfig ? ({
|
|
240
243
|
initialBlockInterval: initialBlockInterval,
|
|
241
244
|
backoffMultiplicative: backoffMultiplicative,
|
|
@@ -243,12 +246,14 @@ function fromPublic(publicConfigJson, codegenChainsOpt, maxAddrInPartitionOpt, u
|
|
|
243
246
|
intervalCeiling: intervalCeiling,
|
|
244
247
|
backoffMillis: backoffMillis,
|
|
245
248
|
queryTimeoutMillis: queryTimeoutMillis,
|
|
246
|
-
fallbackStallTimeout: fallbackStallTimeout
|
|
249
|
+
fallbackStallTimeout: fallbackStallTimeout,
|
|
250
|
+
pollingInterval: pollingInterval
|
|
247
251
|
}) : undefined;
|
|
248
252
|
return {
|
|
249
253
|
url: rpcConfig.url,
|
|
250
254
|
sourceFor: parseRpcSourceFor(rpcConfig.for),
|
|
251
|
-
syncConfig: syncConfig
|
|
255
|
+
syncConfig: syncConfig,
|
|
256
|
+
ws: rpcConfig.ws
|
|
252
257
|
};
|
|
253
258
|
}));
|
|
254
259
|
sourceConfig = {
|