envio 3.0.0-alpha.5 → 3.0.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +91 -15
- package/package.json +11 -10
- package/src/Benchmark.res +1 -1
- package/src/Benchmark.res.mjs +1 -1
- package/src/ChainFetcher.res +3 -1
- package/src/ChainFetcher.res.mjs +5 -4
- package/src/ChainManager.res +15 -12
- package/src/ChainManager.res.mjs +13 -2
- package/src/Config.res +22 -0
- package/src/Config.res.mjs +9 -1
- package/src/Env.res +22 -13
- package/src/Env.res.mjs +19 -18
- package/src/HandlerLoader.res +89 -0
- package/src/HandlerLoader.res.mjs +79 -0
- package/src/Logging.res +11 -4
- package/src/Logging.res.mjs +14 -6
- package/src/Main.res +63 -54
- package/src/Main.res.mjs +67 -44
- package/src/Persistence.res +1 -2
- package/src/PgStorage.res +5 -33
- package/src/PgStorage.res.mjs +6 -27
- package/src/TestIndexer.res +476 -0
- package/src/TestIndexer.res.mjs +369 -0
- package/src/TestIndexerProxyStorage.res +210 -0
- package/src/TestIndexerProxyStorage.res.mjs +157 -0
- package/src/bindings/NodeJs.res +44 -3
- package/src/bindings/NodeJs.res.mjs +11 -3
- package/src/bindings/Pino.res +19 -5
- package/src/bindings/Pino.res.mjs +10 -4
- package/src/bindings/Vitest.res +134 -0
- package/src/bindings/Vitest.res.mjs +9 -0
- package/src/db/InternalTable.res +2 -0
- package/src/db/InternalTable.res.mjs +1 -1
- package/src/sources/HyperSyncSource.res +7 -1
- package/src/sources/HyperSyncSource.res.mjs +7 -6
package/index.d.ts
CHANGED
|
@@ -15,6 +15,9 @@ export type {
|
|
|
15
15
|
import type { Address } from "./src/Types.ts";
|
|
16
16
|
export type { EffectCaller, Address } from "./src/Types.ts";
|
|
17
17
|
|
|
18
|
+
/** Utility type to expand/flatten complex types for better IDE display. */
|
|
19
|
+
export type Prettify<T> = { [K in keyof T]: T[K] } & {};
|
|
20
|
+
|
|
18
21
|
import type {
|
|
19
22
|
effect as Effect,
|
|
20
23
|
effectArgs as EffectArgs,
|
|
@@ -162,9 +165,10 @@ export interface Global {}
|
|
|
162
165
|
|
|
163
166
|
/**
|
|
164
167
|
* Shape of the indexer configuration.
|
|
165
|
-
*
|
|
168
|
+
* Will be used internally for defineConfig.
|
|
169
|
+
* Currently should match the internal.config.json structure.
|
|
166
170
|
*/
|
|
167
|
-
|
|
171
|
+
type IndexerConfig = {
|
|
168
172
|
/** The indexer name. */
|
|
169
173
|
name: string;
|
|
170
174
|
/** The indexer description. */
|
|
@@ -336,6 +340,19 @@ type SvmChain<Id extends number = number> = {
|
|
|
336
340
|
|
|
337
341
|
// ============== Indexer Type ==============
|
|
338
342
|
|
|
343
|
+
/** Minimal type constraint for IndexerFromConfig to allow usage without full IndexerConfig. */
|
|
344
|
+
type IndexerConfigTypes = {
|
|
345
|
+
evm?: {
|
|
346
|
+
chains: Record<string, { id: number }>;
|
|
347
|
+
contracts?: Record<string, {}>;
|
|
348
|
+
};
|
|
349
|
+
fuel?: {
|
|
350
|
+
chains: Record<string, { id: number }>;
|
|
351
|
+
contracts?: Record<string, {}>;
|
|
352
|
+
};
|
|
353
|
+
svm?: { chains: Record<string, { id: number }> };
|
|
354
|
+
};
|
|
355
|
+
|
|
339
356
|
// Helper: Check if ecosystem is configured in a given config
|
|
340
357
|
type HasEvm<Config> = "evm" extends keyof Config ? true : false;
|
|
341
358
|
type HasFuel<Config> = "fuel" extends keyof Config ? true : false;
|
|
@@ -351,7 +368,7 @@ type EcosystemTuple<Config> = [
|
|
|
351
368
|
type EcosystemCount<Config> = EcosystemTuple<Config>["length"];
|
|
352
369
|
|
|
353
370
|
// EVM ecosystem type
|
|
354
|
-
type EvmEcosystem<Config extends
|
|
371
|
+
type EvmEcosystem<Config extends IndexerConfigTypes> =
|
|
355
372
|
"evm" extends keyof Config
|
|
356
373
|
? Config["evm"] extends {
|
|
357
374
|
chains: infer Chains;
|
|
@@ -379,7 +396,7 @@ type EvmEcosystem<Config extends IndexerConfig /*= GlobalIndexerConfig*/> =
|
|
|
379
396
|
: never;
|
|
380
397
|
|
|
381
398
|
// Fuel ecosystem type
|
|
382
|
-
type FuelEcosystem<Config extends
|
|
399
|
+
type FuelEcosystem<Config extends IndexerConfigTypes> =
|
|
383
400
|
"fuel" extends keyof Config
|
|
384
401
|
? Config["fuel"] extends {
|
|
385
402
|
chains: infer Chains;
|
|
@@ -407,7 +424,7 @@ type FuelEcosystem<Config extends IndexerConfig /*= GlobalIndexerConfig*/> =
|
|
|
407
424
|
: never;
|
|
408
425
|
|
|
409
426
|
// SVM ecosystem type
|
|
410
|
-
type SvmEcosystem<Config extends
|
|
427
|
+
type SvmEcosystem<Config extends IndexerConfigTypes> =
|
|
411
428
|
"svm" extends keyof Config
|
|
412
429
|
? Config["svm"] extends { chains: infer Chains }
|
|
413
430
|
? Chains extends Record<string, { id: number }>
|
|
@@ -426,7 +443,7 @@ type SvmEcosystem<Config extends IndexerConfig /*= GlobalIndexerConfig*/> =
|
|
|
426
443
|
: never;
|
|
427
444
|
|
|
428
445
|
// Single ecosystem chains (flattened at root level)
|
|
429
|
-
type SingleEcosystemChains<Config extends
|
|
446
|
+
type SingleEcosystemChains<Config extends IndexerConfigTypes> =
|
|
430
447
|
HasEvm<Config> extends true
|
|
431
448
|
? EvmEcosystem<Config>
|
|
432
449
|
: HasFuel<Config> extends true
|
|
@@ -436,7 +453,7 @@ type SingleEcosystemChains<Config extends IndexerConfig> =
|
|
|
436
453
|
: {};
|
|
437
454
|
|
|
438
455
|
// Multi-ecosystem chains (namespaced by ecosystem)
|
|
439
|
-
type MultiEcosystemChains<Config extends
|
|
456
|
+
type MultiEcosystemChains<Config extends IndexerConfigTypes> =
|
|
440
457
|
(HasEvm<Config> extends true
|
|
441
458
|
? {
|
|
442
459
|
/** EVM ecosystem configuration. */
|
|
@@ -461,11 +478,70 @@ type MultiEcosystemChains<Config extends IndexerConfig> =
|
|
|
461
478
|
* - Single ecosystem: chains are at the root level.
|
|
462
479
|
* - Multiple ecosystems: chains are namespaced by ecosystem (evm, fuel, svm).
|
|
463
480
|
*/
|
|
464
|
-
export type IndexerFromConfig<Config extends
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
481
|
+
export type IndexerFromConfig<Config extends IndexerConfigTypes> = Prettify<
|
|
482
|
+
{
|
|
483
|
+
/** The indexer name from config.yaml. */
|
|
484
|
+
readonly name: string;
|
|
485
|
+
/** The indexer description from config.yaml. */
|
|
486
|
+
readonly description: string | undefined;
|
|
487
|
+
} & (EcosystemCount<Config> extends 1
|
|
488
|
+
? SingleEcosystemChains<Config>
|
|
489
|
+
: MultiEcosystemChains<Config>)
|
|
490
|
+
>;
|
|
491
|
+
|
|
492
|
+
// ============== Test Indexer Types ==============
|
|
493
|
+
|
|
494
|
+
/** Configuration for a single chain in the test indexer. */
|
|
495
|
+
export type TestIndexerChainConfig = {
|
|
496
|
+
/** The block number to start processing from. */
|
|
497
|
+
startBlock: number;
|
|
498
|
+
/** The block number to stop processing at. */
|
|
499
|
+
endBlock: number;
|
|
500
|
+
};
|
|
501
|
+
|
|
502
|
+
/** Progress returned after processing blocks with the test indexer. */
|
|
503
|
+
export type TestIndexerProcessResult = {
|
|
504
|
+
/** Changes happened during the processing. */
|
|
505
|
+
changes: unknown[];
|
|
506
|
+
};
|
|
507
|
+
|
|
508
|
+
// Helper to extract chain IDs from config for test indexer
|
|
509
|
+
type TestIndexerChainIds<Config extends IndexerConfigTypes> =
|
|
510
|
+
HasEvm<Config> extends true
|
|
511
|
+
? Config["evm"] extends { chains: infer Chains }
|
|
512
|
+
? Chains extends Record<string, { id: number }>
|
|
513
|
+
? Chains[keyof Chains]["id"]
|
|
514
|
+
: never
|
|
515
|
+
: never
|
|
516
|
+
: HasFuel<Config> extends true
|
|
517
|
+
? Config["fuel"] extends { chains: infer Chains }
|
|
518
|
+
? Chains extends Record<string, { id: number }>
|
|
519
|
+
? Chains[keyof Chains]["id"]
|
|
520
|
+
: never
|
|
521
|
+
: never
|
|
522
|
+
: HasSvm<Config> extends true
|
|
523
|
+
? Config["svm"] extends { chains: infer Chains }
|
|
524
|
+
? Chains extends Record<string, { id: number }>
|
|
525
|
+
? Chains[keyof Chains]["id"]
|
|
526
|
+
: never
|
|
527
|
+
: never
|
|
528
|
+
: never;
|
|
529
|
+
|
|
530
|
+
/** Process configuration for the test indexer, with chains keyed by chain ID. */
|
|
531
|
+
export type TestIndexerProcessConfig<Config extends IndexerConfigTypes> = {
|
|
532
|
+
/** Chain configurations keyed by chain ID. Each chain specifies start and end blocks. */
|
|
533
|
+
chains: {
|
|
534
|
+
[K in TestIndexerChainIds<Config>]?: TestIndexerChainConfig;
|
|
535
|
+
};
|
|
536
|
+
};
|
|
537
|
+
|
|
538
|
+
/**
|
|
539
|
+
* Test indexer type resolved from config.
|
|
540
|
+
* Allows running the indexer for specific block ranges and inspecting results.
|
|
541
|
+
*/
|
|
542
|
+
export type TestIndexerFromConfig<Config extends IndexerConfigTypes> = {
|
|
543
|
+
/** Process blocks for the specified chains and return progress with checkpoints and changes. */
|
|
544
|
+
process: (
|
|
545
|
+
config: Prettify<TestIndexerProcessConfig<Config>>
|
|
546
|
+
) => Promise<TestIndexerProcessResult>;
|
|
547
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "v3.0.0-alpha.
|
|
3
|
+
"version": "v3.0.0-alpha.7",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.js",
|
|
@@ -29,10 +29,10 @@
|
|
|
29
29
|
"node": ">=22.0.0"
|
|
30
30
|
},
|
|
31
31
|
"optionalDependencies": {
|
|
32
|
-
"envio-linux-x64": "v3.0.0-alpha.
|
|
33
|
-
"envio-linux-arm64": "v3.0.0-alpha.
|
|
34
|
-
"envio-darwin-x64": "v3.0.0-alpha.
|
|
35
|
-
"envio-darwin-arm64": "v3.0.0-alpha.
|
|
32
|
+
"envio-linux-x64": "v3.0.0-alpha.7",
|
|
33
|
+
"envio-linux-arm64": "v3.0.0-alpha.7",
|
|
34
|
+
"envio-darwin-x64": "v3.0.0-alpha.7",
|
|
35
|
+
"envio-darwin-arm64": "v3.0.0-alpha.7"
|
|
36
36
|
},
|
|
37
37
|
"dependencies": {
|
|
38
38
|
"@clickhouse/client": "1.12.1",
|
|
@@ -41,9 +41,11 @@
|
|
|
41
41
|
"@envio-dev/hypersync-client": "0.7.0",
|
|
42
42
|
"bignumber.js": "9.1.2",
|
|
43
43
|
"eventsource": "4.1.0",
|
|
44
|
-
"
|
|
45
|
-
"pino
|
|
44
|
+
"express": "4.19.2",
|
|
45
|
+
"pino": "10.1.0",
|
|
46
|
+
"pino-pretty": "13.1.3",
|
|
46
47
|
"prom-client": "15.0.0",
|
|
48
|
+
"yargs": "17.7.2",
|
|
47
49
|
"rescript": "11.1.3",
|
|
48
50
|
"rescript-schema": "9.3.4",
|
|
49
51
|
"viem": "2.21.0",
|
|
@@ -54,9 +56,8 @@
|
|
|
54
56
|
"ink": "6.5.1",
|
|
55
57
|
"ink-big-text": "2.0.0",
|
|
56
58
|
"ink-spinner": "5.0.0",
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"postgres": "3.4.1"
|
|
59
|
+
"postgres": "3.4.1",
|
|
60
|
+
"tsx": "4.21.0"
|
|
60
61
|
},
|
|
61
62
|
"files": [
|
|
62
63
|
"bin.js",
|
package/src/Benchmark.res
CHANGED
|
@@ -216,7 +216,7 @@ let throttler = Throttler.make(
|
|
|
216
216
|
~logger=Logging.createChild(~params={"context": "Benchmarking framework"}),
|
|
217
217
|
)
|
|
218
218
|
let cacheFileName = "BenchmarkCache.json"
|
|
219
|
-
let cacheFilePath = NodeJs.Path.join(NodeJs.Path.
|
|
219
|
+
let cacheFilePath = NodeJs.Path.join(NodeJs.Path.getDirname(NodeJs.ImportMeta.importMeta), cacheFileName)
|
|
220
220
|
|
|
221
221
|
let saveToCacheFile = if (
|
|
222
222
|
Env.Benchmark.saveDataStrategy->Env.Benchmark.SaveDataStrategy.shouldSaveJsonFile
|
package/src/Benchmark.res.mjs
CHANGED
|
@@ -243,7 +243,7 @@ var throttler = Throttler.make(Env.ThrottleWrites.jsonFileBenchmarkIntervalMilli
|
|
|
243
243
|
|
|
244
244
|
var cacheFileName = "BenchmarkCache.json";
|
|
245
245
|
|
|
246
|
-
var cacheFilePath = Path.join(NodeJs.Path
|
|
246
|
+
var cacheFilePath = Path.join(NodeJs.Path.getDirname(import.meta), cacheFileName);
|
|
247
247
|
|
|
248
248
|
var saveToCacheFile = Env.Benchmark.SaveDataStrategy.shouldSaveJsonFile(Env.Benchmark.saveDataStrategy) ? (function (data) {
|
|
249
249
|
var write = function () {
|
package/src/ChainFetcher.res
CHANGED
|
@@ -40,6 +40,7 @@ let make = (
|
|
|
40
40
|
~isInReorgThreshold,
|
|
41
41
|
~reorgCheckpoints: array<Internal.reorgCheckpoint>,
|
|
42
42
|
~maxReorgDepth,
|
|
43
|
+
~knownHeight=0,
|
|
43
44
|
): t => {
|
|
44
45
|
// We don't need the router itself, but only validation logic,
|
|
45
46
|
// since now event router is created for selection of events
|
|
@@ -177,7 +178,7 @@ let make = (
|
|
|
177
178
|
~endBlock,
|
|
178
179
|
~eventConfigs,
|
|
179
180
|
~targetBufferSize,
|
|
180
|
-
~knownHeight
|
|
181
|
+
~knownHeight,
|
|
181
182
|
~chainId=chainConfig.id,
|
|
182
183
|
// FIXME: Shouldn't set with full history
|
|
183
184
|
~blockLag=Pervasives.max(
|
|
@@ -287,6 +288,7 @@ let makeFromDbState = async (
|
|
|
287
288
|
~logger,
|
|
288
289
|
~targetBufferSize,
|
|
289
290
|
~isInReorgThreshold,
|
|
291
|
+
~knownHeight=resumedChainState.sourceBlockNumber,
|
|
290
292
|
)
|
|
291
293
|
}
|
|
292
294
|
|
package/src/ChainFetcher.res.mjs
CHANGED
|
@@ -21,7 +21,8 @@ import * as ReorgDetection from "./ReorgDetection.res.mjs";
|
|
|
21
21
|
import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
|
|
22
22
|
import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
|
|
23
23
|
|
|
24
|
-
function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlockNumber, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, numBatchesFetched, isInReorgThreshold, reorgCheckpoints, maxReorgDepth) {
|
|
24
|
+
function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlockNumber, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, numBatchesFetched, isInReorgThreshold, reorgCheckpoints, maxReorgDepth, knownHeightOpt) {
|
|
25
|
+
var knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
|
|
25
26
|
var eventRouter = EventRouter.empty();
|
|
26
27
|
var contracts = [];
|
|
27
28
|
var eventConfigs = [];
|
|
@@ -96,7 +97,7 @@ function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlo
|
|
|
96
97
|
|
|
97
98
|
}));
|
|
98
99
|
}
|
|
99
|
-
var fetchState = FetchState.make(startBlock, endBlock, eventConfigs, contracts, config.maxAddrInPartition, chainConfig.id, targetBufferSize,
|
|
100
|
+
var fetchState = FetchState.make(startBlock, endBlock, eventConfigs, contracts, config.maxAddrInPartition, chainConfig.id, targetBufferSize, knownHeight, progressBlockNumber, onBlockConfigs, Caml.int_max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, Belt_Option.getWithDefault(Env.indexingBlockLag, 0)));
|
|
100
101
|
var chainReorgCheckpoints = Belt_Array.keepMapU(reorgCheckpoints, (function (reorgCheckpoint) {
|
|
101
102
|
if (reorgCheckpoint.chain_id === chainConfig.id) {
|
|
102
103
|
return reorgCheckpoint;
|
|
@@ -123,7 +124,7 @@ function makeFromConfig(chainConfig, config, registrations, targetBufferSize) {
|
|
|
123
124
|
var logger = Logging.createChild({
|
|
124
125
|
chainId: chainConfig.id
|
|
125
126
|
});
|
|
126
|
-
return make(chainConfig, [], chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, 0, false, [], chainConfig.maxReorgDepth);
|
|
127
|
+
return make(chainConfig, [], chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, 0, false, [], chainConfig.maxReorgDepth, undefined);
|
|
127
128
|
}
|
|
128
129
|
|
|
129
130
|
async function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, config, registrations, targetBufferSize) {
|
|
@@ -133,7 +134,7 @@ async function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints,
|
|
|
133
134
|
});
|
|
134
135
|
Prometheus.ProgressEventsCount.set(resumedChainState.numEventsProcessed, chainId);
|
|
135
136
|
var progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
|
|
136
|
-
return make(chainConfig, resumedChainState.dynamicContracts, resumedChainState.startBlock, resumedChainState.endBlock, resumedChainState.firstEventBlockNumber, progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, 0, isInReorgThreshold, reorgCheckpoints, resumedChainState.maxReorgDepth);
|
|
137
|
+
return make(chainConfig, resumedChainState.dynamicContracts, resumedChainState.startBlock, resumedChainState.endBlock, resumedChainState.firstEventBlockNumber, progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, 0, isInReorgThreshold, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber);
|
|
137
138
|
}
|
|
138
139
|
|
|
139
140
|
function getContractStartBlock(config, chain, contractName) {
|
package/src/ChainManager.res
CHANGED
|
@@ -7,6 +7,13 @@ type t = {
|
|
|
7
7
|
isInReorgThreshold: bool,
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
+
// Check if progress is past the reorg threshold (safe block).
|
|
11
|
+
// A chain is in reorg threshold when progressBlockNumber > sourceBlockNumber - maxReorgDepth.
|
|
12
|
+
// This matches the logic in InternalTable.Checkpoints.makeGetReorgCheckpointsQuery.
|
|
13
|
+
let isProgressInReorgThreshold = (~progressBlockNumber, ~sourceBlockNumber, ~maxReorgDepth) => {
|
|
14
|
+
maxReorgDepth > 0 && progressBlockNumber > sourceBlockNumber - maxReorgDepth
|
|
15
|
+
}
|
|
16
|
+
|
|
10
17
|
let calculateTargetBufferSize = (~activeChainsCount, ~config: Config.t) => {
|
|
11
18
|
let targetBatchesInBuffer = 3
|
|
12
19
|
switch Env.targetBufferSize {
|
|
@@ -37,22 +44,18 @@ let makeFromDbState = async (
|
|
|
37
44
|
~initialState: Persistence.initialState,
|
|
38
45
|
~config: Config.t,
|
|
39
46
|
~registrations,
|
|
40
|
-
~persistence: Persistence.t,
|
|
41
47
|
): t => {
|
|
42
48
|
let isInReorgThreshold = if initialState.cleanRun {
|
|
43
49
|
false
|
|
44
50
|
} else {
|
|
45
|
-
//
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
//If we have started saving history, continue to save history
|
|
54
|
-
//as regardless of whether we are still in a reorg threshold
|
|
55
|
-
hasStartedSavingHistory
|
|
51
|
+
// Check if any chain is in reorg threshold by comparing progress with sourceBlock - maxReorgDepth.
|
|
52
|
+
initialState.chains->Array.some(chain =>
|
|
53
|
+
isProgressInReorgThreshold(
|
|
54
|
+
~progressBlockNumber=chain.progressBlockNumber,
|
|
55
|
+
~sourceBlockNumber=chain.sourceBlockNumber,
|
|
56
|
+
~maxReorgDepth=chain.maxReorgDepth,
|
|
57
|
+
)
|
|
58
|
+
)
|
|
56
59
|
}
|
|
57
60
|
|
|
58
61
|
let targetBufferSize = calculateTargetBufferSize(
|
package/src/ChainManager.res.mjs
CHANGED
|
@@ -10,6 +10,14 @@ import * as Prometheus from "./Prometheus.res.mjs";
|
|
|
10
10
|
import * as ChainFetcher from "./ChainFetcher.res.mjs";
|
|
11
11
|
import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
|
|
12
12
|
|
|
13
|
+
function isProgressInReorgThreshold(progressBlockNumber, sourceBlockNumber, maxReorgDepth) {
|
|
14
|
+
if (maxReorgDepth > 0) {
|
|
15
|
+
return progressBlockNumber > (sourceBlockNumber - maxReorgDepth | 0);
|
|
16
|
+
} else {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
13
21
|
function calculateTargetBufferSize(activeChainsCount, config) {
|
|
14
22
|
if (Env.targetBufferSize !== undefined) {
|
|
15
23
|
return Env.targetBufferSize;
|
|
@@ -31,8 +39,10 @@ function makeFromConfig(config, registrations) {
|
|
|
31
39
|
};
|
|
32
40
|
}
|
|
33
41
|
|
|
34
|
-
async function makeFromDbState(initialState, config, registrations
|
|
35
|
-
var isInReorgThreshold = initialState.cleanRun ? false :
|
|
42
|
+
async function makeFromDbState(initialState, config, registrations) {
|
|
43
|
+
var isInReorgThreshold = initialState.cleanRun ? false : Belt_Array.some(initialState.chains, (function (chain) {
|
|
44
|
+
return isProgressInReorgThreshold(chain.progressBlockNumber, chain.sourceBlockNumber, chain.maxReorgDepth);
|
|
45
|
+
}));
|
|
36
46
|
var targetBufferSize = calculateTargetBufferSize(initialState.chains.length, config);
|
|
37
47
|
Prometheus.ProcessingMaxBatchSize.set(config.batchSize);
|
|
38
48
|
Prometheus.IndexingTargetBufferSize.set(targetBufferSize);
|
|
@@ -125,6 +135,7 @@ function getSafeCheckpointId(chainManager) {
|
|
|
125
135
|
}
|
|
126
136
|
|
|
127
137
|
export {
|
|
138
|
+
isProgressInReorgThreshold ,
|
|
128
139
|
calculateTargetBufferSize ,
|
|
129
140
|
makeFromConfig ,
|
|
130
141
|
makeFromDbState ,
|
package/src/Config.res
CHANGED
|
@@ -53,10 +53,16 @@ type sourceSync = {
|
|
|
53
53
|
|
|
54
54
|
type multichain = | @as("ordered") Ordered | @as("unordered") Unordered
|
|
55
55
|
|
|
56
|
+
type contractHandler = {
|
|
57
|
+
name: string,
|
|
58
|
+
handler: option<string>,
|
|
59
|
+
}
|
|
60
|
+
|
|
56
61
|
type t = {
|
|
57
62
|
name: string,
|
|
58
63
|
description: option<string>,
|
|
59
64
|
handlers: string,
|
|
65
|
+
contractHandlers: array<contractHandler>,
|
|
60
66
|
shouldRollbackOnReorg: bool,
|
|
61
67
|
shouldSaveFullHistory: bool,
|
|
62
68
|
multichain: multichain,
|
|
@@ -83,6 +89,7 @@ let publicConfigChainSchema = S.schema(s =>
|
|
|
83
89
|
let contractConfigSchema = S.schema(s =>
|
|
84
90
|
{
|
|
85
91
|
"abi": s.matches(S.json(~validate=false)),
|
|
92
|
+
"handler": s.matches(S.option(S.string)),
|
|
86
93
|
}
|
|
87
94
|
)
|
|
88
95
|
|
|
@@ -293,10 +300,25 @@ let fromPublic = (
|
|
|
293
300
|
})
|
|
294
301
|
->Js.Dict.fromArray
|
|
295
302
|
|
|
303
|
+
// Extract contract handlers from the public config
|
|
304
|
+
let contractHandlers = switch publicContractsConfig {
|
|
305
|
+
| Some(contractsDict) =>
|
|
306
|
+
contractsDict
|
|
307
|
+
->Js.Dict.entries
|
|
308
|
+
->Js.Array2.map(((contractName, contractConfig)) => {
|
|
309
|
+
{
|
|
310
|
+
name: contractName,
|
|
311
|
+
handler: contractConfig["handler"],
|
|
312
|
+
}
|
|
313
|
+
})
|
|
314
|
+
| None => []
|
|
315
|
+
}
|
|
316
|
+
|
|
296
317
|
{
|
|
297
318
|
name: publicConfig["name"],
|
|
298
319
|
description: publicConfig["description"],
|
|
299
320
|
handlers: publicConfig["handlers"]->Option.getWithDefault("src/handlers"),
|
|
321
|
+
contractHandlers,
|
|
300
322
|
shouldRollbackOnReorg: publicConfig["rollbackOnReorg"]->Option.getWithDefault(true),
|
|
301
323
|
shouldSaveFullHistory: publicConfig["saveFullHistory"]->Option.getWithDefault(false),
|
|
302
324
|
multichain: publicConfig["multichain"]->Option.getWithDefault(Unordered),
|
package/src/Config.res.mjs
CHANGED
|
@@ -25,7 +25,8 @@ var publicConfigChainSchema = S$RescriptSchema.schema(function (s) {
|
|
|
25
25
|
|
|
26
26
|
var contractConfigSchema = S$RescriptSchema.schema(function (s) {
|
|
27
27
|
return {
|
|
28
|
-
abi: s.m(S$RescriptSchema.json(false))
|
|
28
|
+
abi: s.m(S$RescriptSchema.json(false)),
|
|
29
|
+
handler: s.m(S$RescriptSchema.option(S$RescriptSchema.string))
|
|
29
30
|
};
|
|
30
31
|
});
|
|
31
32
|
|
|
@@ -246,10 +247,17 @@ function fromPublic(publicConfigJson, codegenChainsOpt, maxAddrInPartitionOpt, u
|
|
|
246
247
|
entityConfig
|
|
247
248
|
];
|
|
248
249
|
}));
|
|
250
|
+
var contractHandlers = publicContractsConfig !== undefined ? Js_dict.entries(publicContractsConfig).map(function (param) {
|
|
251
|
+
return {
|
|
252
|
+
name: param[0],
|
|
253
|
+
handler: param[1].handler
|
|
254
|
+
};
|
|
255
|
+
}) : [];
|
|
249
256
|
return {
|
|
250
257
|
name: publicConfig.name,
|
|
251
258
|
description: publicConfig.description,
|
|
252
259
|
handlers: Belt_Option.getWithDefault(publicConfig.handlers, "src/handlers"),
|
|
260
|
+
contractHandlers: contractHandlers,
|
|
253
261
|
shouldRollbackOnReorg: Belt_Option.getWithDefault(publicConfig.rollbackOnReorg, true),
|
|
254
262
|
shouldSaveFullHistory: Belt_Option.getWithDefault(publicConfig.saveFullHistory, false),
|
|
255
263
|
multichain: Belt_Option.getWithDefault(publicConfig.multichain, "unordered"),
|
package/src/Env.res
CHANGED
|
@@ -1,17 +1,8 @@
|
|
|
1
|
-
|
|
2
1
|
// Loads the .env from the root working directory
|
|
3
2
|
%%raw(`import 'dotenv/config'`)
|
|
4
3
|
|
|
5
|
-
%%private(
|
|
6
|
-
let envSafe = EnvSafe.make()
|
|
4
|
+
%%private(let envSafe = EnvSafe.make())
|
|
7
5
|
|
|
8
|
-
let getLogLevelConfig = (name, ~default): Pino.logLevel =>
|
|
9
|
-
envSafe->EnvSafe.get(
|
|
10
|
-
name,
|
|
11
|
-
S.enum([#trace, #debug, #info, #warn, #error, #fatal, #udebug, #uinfo, #uwarn, #uerror]),
|
|
12
|
-
~fallback=default,
|
|
13
|
-
)
|
|
14
|
-
)
|
|
15
6
|
// resets the timestampCaughtUpToHeadOrEndblock after a restart when true
|
|
16
7
|
let updateSyncTimeOnRestart =
|
|
17
8
|
envSafe->EnvSafe.get("UPDATE_SYNC_TIME_ON_RESTART", S.bool, ~fallback=true)
|
|
@@ -34,9 +25,22 @@ let serverPort =
|
|
|
34
25
|
|
|
35
26
|
let tuiOffEnvVar = envSafe->EnvSafe.get("TUI_OFF", S.bool, ~fallback=false)
|
|
36
27
|
|
|
28
|
+
let logLevelSchema = S.enum([
|
|
29
|
+
#trace,
|
|
30
|
+
#debug,
|
|
31
|
+
#info,
|
|
32
|
+
#warn,
|
|
33
|
+
#error,
|
|
34
|
+
#fatal,
|
|
35
|
+
#udebug,
|
|
36
|
+
#uinfo,
|
|
37
|
+
#uwarn,
|
|
38
|
+
#uerror,
|
|
39
|
+
#silent,
|
|
40
|
+
])
|
|
37
41
|
let logFilePath = envSafe->EnvSafe.get("LOG_FILE", S.string, ~fallback="logs/envio.log")
|
|
38
|
-
let userLogLevel =
|
|
39
|
-
let defaultFileLogLevel =
|
|
42
|
+
let userLogLevel = envSafe->EnvSafe.get("LOG_LEVEL", S.option(logLevelSchema))
|
|
43
|
+
let defaultFileLogLevel = envSafe->EnvSafe.get("FILE_LOG_LEVEL", logLevelSchema, ~fallback=#trace)
|
|
40
44
|
|
|
41
45
|
let prodEnvioAppUrl = "https://envio.dev"
|
|
42
46
|
let envioAppUrl = envSafe->EnvSafe.get("ENVIO_APP", S.string, ~fallback=prodEnvioAppUrl)
|
|
@@ -121,7 +125,12 @@ let logStrategy =
|
|
|
121
125
|
)
|
|
122
126
|
|
|
123
127
|
Logging.setLogger(
|
|
124
|
-
Logging.makeLogger(
|
|
128
|
+
Logging.makeLogger(
|
|
129
|
+
~logStrategy,
|
|
130
|
+
~logFilePath,
|
|
131
|
+
~defaultFileLogLevel,
|
|
132
|
+
~userLogLevel=userLogLevel->Belt.Option.getWithDefault(#info),
|
|
133
|
+
),
|
|
125
134
|
)
|
|
126
135
|
|
|
127
136
|
module Db = {
|
package/src/Env.res.mjs
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import * as EnvSafe from "rescript-envsafe/src/EnvSafe.res.mjs";
|
|
4
4
|
import * as Logging from "./Logging.res.mjs";
|
|
5
5
|
import * as Postgres from "./bindings/Postgres.res.mjs";
|
|
6
|
+
import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
|
|
6
7
|
import * as Caml_option from "rescript/lib/es6/caml_option.js";
|
|
7
8
|
import * as HyperSyncClient from "./sources/HyperSyncClient.res.mjs";
|
|
8
9
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
@@ -12,21 +13,6 @@ import 'dotenv/config'
|
|
|
12
13
|
|
|
13
14
|
var envSafe = EnvSafe.make(undefined);
|
|
14
15
|
|
|
15
|
-
function getLogLevelConfig(name, $$default) {
|
|
16
|
-
return EnvSafe.get(envSafe, name, S$RescriptSchema.$$enum([
|
|
17
|
-
"trace",
|
|
18
|
-
"debug",
|
|
19
|
-
"info",
|
|
20
|
-
"warn",
|
|
21
|
-
"error",
|
|
22
|
-
"fatal",
|
|
23
|
-
"udebug",
|
|
24
|
-
"uinfo",
|
|
25
|
-
"uwarn",
|
|
26
|
-
"uerror"
|
|
27
|
-
]), undefined, $$default, undefined, undefined);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
16
|
var updateSyncTimeOnRestart = EnvSafe.get(envSafe, "UPDATE_SYNC_TIME_ON_RESTART", S$RescriptSchema.bool, undefined, true, undefined, undefined);
|
|
31
17
|
|
|
32
18
|
var targetBufferSize = EnvSafe.get(envSafe, "ENVIO_INDEXING_MAX_BUFFER_SIZE", S$RescriptSchema.option(S$RescriptSchema.$$int), undefined, undefined, undefined, undefined);
|
|
@@ -41,11 +27,25 @@ var serverPort = EnvSafe.get(envSafe, "ENVIO_INDEXER_PORT", S$RescriptSchema.por
|
|
|
41
27
|
|
|
42
28
|
var tuiOffEnvVar = EnvSafe.get(envSafe, "TUI_OFF", S$RescriptSchema.bool, undefined, false, undefined, undefined);
|
|
43
29
|
|
|
30
|
+
var logLevelSchema = S$RescriptSchema.$$enum([
|
|
31
|
+
"trace",
|
|
32
|
+
"debug",
|
|
33
|
+
"info",
|
|
34
|
+
"warn",
|
|
35
|
+
"error",
|
|
36
|
+
"fatal",
|
|
37
|
+
"udebug",
|
|
38
|
+
"uinfo",
|
|
39
|
+
"uwarn",
|
|
40
|
+
"uerror",
|
|
41
|
+
"silent"
|
|
42
|
+
]);
|
|
43
|
+
|
|
44
44
|
var logFilePath = EnvSafe.get(envSafe, "LOG_FILE", S$RescriptSchema.string, undefined, "logs/envio.log", undefined, undefined);
|
|
45
45
|
|
|
46
|
-
var userLogLevel =
|
|
46
|
+
var userLogLevel = EnvSafe.get(envSafe, "LOG_LEVEL", S$RescriptSchema.option(logLevelSchema), undefined, undefined, undefined, undefined);
|
|
47
47
|
|
|
48
|
-
var defaultFileLogLevel =
|
|
48
|
+
var defaultFileLogLevel = EnvSafe.get(envSafe, "FILE_LOG_LEVEL", logLevelSchema, undefined, "trace", undefined, undefined);
|
|
49
49
|
|
|
50
50
|
var prodEnvioAppUrl = "https://envio.dev";
|
|
51
51
|
|
|
@@ -123,7 +123,7 @@ var logStrategy = EnvSafe.get(envSafe, "LOG_STRATEGY", S$RescriptSchema.$$enum([
|
|
|
123
123
|
"both-prettyconsole"
|
|
124
124
|
]), undefined, "console-pretty", undefined, undefined);
|
|
125
125
|
|
|
126
|
-
Logging.setLogger(Logging.makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel));
|
|
126
|
+
Logging.setLogger(Logging.makeLogger(logStrategy, logFilePath, defaultFileLogLevel, Belt_Option.getWithDefault(userLogLevel, "info")));
|
|
127
127
|
|
|
128
128
|
var host = EnvSafe.get(envSafe, "ENVIO_PG_HOST", S$RescriptSchema.string, undefined, undefined, "localhost", undefined);
|
|
129
129
|
|
|
@@ -249,6 +249,7 @@ export {
|
|
|
249
249
|
indexingBlockLag ,
|
|
250
250
|
serverPort ,
|
|
251
251
|
tuiOffEnvVar ,
|
|
252
|
+
logLevelSchema ,
|
|
252
253
|
logFilePath ,
|
|
253
254
|
userLogLevel ,
|
|
254
255
|
defaultFileLogLevel ,
|