envio 3.3.0-alpha.4 → 3.3.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +10 -0
- package/package.json +7 -7
- package/src/Batch.res.mjs +1 -1
- package/src/BatchProcessing.res +5 -0
- package/src/BatchProcessing.res.mjs +6 -0
- package/src/ChainState.res +41 -7
- package/src/ChainState.res.mjs +22 -18
- package/src/ChainState.resi +3 -1
- package/src/Config.res +3 -0
- package/src/Config.res.mjs +3 -1
- package/src/Core.res +0 -3
- package/src/ExitOnCaughtUp.res +10 -2
- package/src/ExitOnCaughtUp.res.mjs +10 -1
- package/src/FetchState.res +63 -133
- package/src/FetchState.res.mjs +107 -166
- package/src/IndexerState.res +4 -0
- package/src/IndexerState.res.mjs +8 -1
- package/src/IndexerState.resi +1 -0
- package/src/IndexingAddresses.res +105 -0
- package/src/IndexingAddresses.res.mjs +100 -0
- package/src/IndexingAddresses.resi +32 -0
- package/src/Main.res +4 -15
- package/src/Main.res.mjs +6 -14
- package/src/Metrics.res +33 -0
- package/src/Metrics.res.mjs +39 -0
- package/src/Prometheus.res +2 -20
- package/src/Prometheus.res.mjs +83 -95
- package/src/SimulateDeadInputTracker.res +85 -0
- package/src/SimulateDeadInputTracker.res.mjs +73 -0
- package/src/SimulateDeadInputTracker.resi +12 -0
- package/src/SimulateItems.res +23 -42
- package/src/SimulateItems.res.mjs +12 -30
- package/src/TestIndexer.res +3 -27
- package/src/TestIndexer.res.mjs +2 -9
- package/src/bindings/Viem.res +0 -41
- package/src/bindings/Viem.res.mjs +1 -43
- package/src/sources/Evm.res +7 -36
- package/src/sources/Evm.res.mjs +4 -36
- package/src/sources/EvmChain.res +4 -2
- package/src/sources/EvmChain.res.mjs +3 -2
- package/src/sources/EvmRpcClient.res +36 -5
- package/src/sources/EvmRpcClient.res.mjs +7 -4
- package/src/sources/HyperSyncClient.res +0 -35
- package/src/sources/HyperSyncClient.res.mjs +1 -8
- package/src/sources/Rpc.res +15 -47
- package/src/sources/Rpc.res.mjs +25 -56
- package/src/sources/RpcSource.res +289 -204
- package/src/sources/RpcSource.res.mjs +293 -303
- package/src/sources/SimulateSource.res +1 -0
- package/src/sources/SimulateSource.res.mjs +2 -1
- package/src/sources/Source.res +4 -0
- package/src/sources/Svm.res +7 -15
- package/src/sources/Svm.res.mjs +4 -15
- package/src/sources/TransactionStore.res +14 -2
- package/src/sources/TransactionStore.res.mjs +10 -2
package/evm.schema.json
CHANGED
|
@@ -548,6 +548,16 @@
|
|
|
548
548
|
"null"
|
|
549
549
|
]
|
|
550
550
|
},
|
|
551
|
+
"headers": {
|
|
552
|
+
"description": "Optional HTTP headers sent with every request to this RPC endpoint, e.g. an Authorization bearer token for gated endpoints. Values support ${ENV_VAR} interpolation.",
|
|
553
|
+
"type": [
|
|
554
|
+
"object",
|
|
555
|
+
"null"
|
|
556
|
+
],
|
|
557
|
+
"additionalProperties": {
|
|
558
|
+
"type": "string"
|
|
559
|
+
}
|
|
560
|
+
},
|
|
551
561
|
"initial_block_interval": {
|
|
552
562
|
"description": "The starting interval in range of blocks per query",
|
|
553
563
|
"type": [
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.3.0-alpha.
|
|
3
|
+
"version": "3.3.0-alpha.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"yargs": "17.7.2",
|
|
58
58
|
"@rescript/runtime": "12.2.0",
|
|
59
59
|
"rescript-schema": "9.5.1",
|
|
60
|
-
"viem": "2.
|
|
60
|
+
"viem": "2.54.0",
|
|
61
61
|
"dotenv": "16.4.5",
|
|
62
62
|
"date-fns": "3.3.1",
|
|
63
63
|
"@rescript/react": "0.14.1",
|
|
@@ -69,10 +69,10 @@
|
|
|
69
69
|
"tsx": "4.21.0"
|
|
70
70
|
},
|
|
71
71
|
"optionalDependencies": {
|
|
72
|
-
"envio-linux-x64": "3.3.0-alpha.
|
|
73
|
-
"envio-linux-x64-musl": "3.3.0-alpha.
|
|
74
|
-
"envio-linux-arm64": "3.3.0-alpha.
|
|
75
|
-
"envio-darwin-x64": "3.3.0-alpha.
|
|
76
|
-
"envio-darwin-arm64": "3.3.0-alpha.
|
|
72
|
+
"envio-linux-x64": "3.3.0-alpha.6",
|
|
73
|
+
"envio-linux-x64-musl": "3.3.0-alpha.6",
|
|
74
|
+
"envio-linux-arm64": "3.3.0-alpha.6",
|
|
75
|
+
"envio-darwin-x64": "3.3.0-alpha.6",
|
|
76
|
+
"envio-darwin-arm64": "3.3.0-alpha.6"
|
|
77
77
|
}
|
|
78
78
|
}
|
package/src/Batch.res.mjs
CHANGED
|
@@ -27,7 +27,7 @@ function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressB
|
|
|
27
27
|
let progressedChain;
|
|
28
28
|
if (batchSize !== undefined) {
|
|
29
29
|
let leftItems = fetchState.buffer.slice(batchSize);
|
|
30
|
-
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined,
|
|
30
|
+
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, leftItems, undefined, undefined), batchSize);
|
|
31
31
|
} else {
|
|
32
32
|
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, 0);
|
|
33
33
|
}
|
package/src/BatchProcessing.res
CHANGED
|
@@ -127,6 +127,11 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
127
127
|
| Ok() =>
|
|
128
128
|
state->IndexerState.recordProcessedBatch
|
|
129
129
|
|
|
130
|
+
switch state->IndexerState.simulateDeadInputTracker {
|
|
131
|
+
| Some(tracker) => tracker->SimulateDeadInputTracker.recordProcessed(~batch)
|
|
132
|
+
| None => ()
|
|
133
|
+
}
|
|
134
|
+
|
|
130
135
|
if state->IndexerState.isResolvingReorg {
|
|
131
136
|
// A reorg landed while this batch was processing. Apply its progress so
|
|
132
137
|
// the rollback diff is computed against up-to-date chain progress, but
|
|
@@ -10,8 +10,10 @@ import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
|
10
10
|
import * as InMemoryStore from "./InMemoryStore.res.mjs";
|
|
11
11
|
import * as ExitOnCaughtUp from "./ExitOnCaughtUp.res.mjs";
|
|
12
12
|
import * as EventProcessing from "./EventProcessing.res.mjs";
|
|
13
|
+
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
13
14
|
import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
|
|
14
15
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
16
|
+
import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
|
|
15
17
|
|
|
16
18
|
function yieldTick() {
|
|
17
19
|
return new Promise((resolve, param) => {
|
|
@@ -55,6 +57,10 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
55
57
|
return IndexerState.errorExit(state, errHandler._0);
|
|
56
58
|
}
|
|
57
59
|
IndexerState.recordProcessedBatch(state);
|
|
60
|
+
let tracker = IndexerState.simulateDeadInputTracker(state);
|
|
61
|
+
if (tracker !== undefined) {
|
|
62
|
+
SimulateDeadInputTracker.recordProcessed(Primitive_option.valFromOption(tracker), batch);
|
|
63
|
+
}
|
|
58
64
|
if (IndexerState.isResolvingReorg(state)) {
|
|
59
65
|
return IndexerState.applyBatchProgress(state, batch);
|
|
60
66
|
}
|
package/src/ChainState.res
CHANGED
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
type t = {
|
|
6
6
|
logger: Pino.t,
|
|
7
7
|
mutable fetchState: FetchState.t,
|
|
8
|
+
// The chain-wide address index. Not `mutable`: the dict is mutated in place by
|
|
9
|
+
// register/rollback, so the reference is stable across fetchState versions.
|
|
10
|
+
indexingAddresses: IndexingAddresses.t,
|
|
8
11
|
sourceManager: SourceManager.t,
|
|
9
12
|
chainConfig: Config.chain,
|
|
10
13
|
mutable isProgressAtHead: bool,
|
|
@@ -58,6 +61,7 @@ let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddres
|
|
|
58
61
|
let make = (
|
|
59
62
|
~chainConfig: Config.chain,
|
|
60
63
|
~fetchState: FetchState.t,
|
|
64
|
+
~indexingAddresses: IndexingAddresses.t,
|
|
61
65
|
~sourceManager: SourceManager.t,
|
|
62
66
|
~reorgDetection: ReorgDetection.t,
|
|
63
67
|
~committedProgressBlockNumber: int,
|
|
@@ -65,10 +69,12 @@ let make = (
|
|
|
65
69
|
~numEventsProcessed=0.,
|
|
66
70
|
~timestampCaughtUpToHeadOrEndblock=None,
|
|
67
71
|
~isProgressAtHead=false,
|
|
72
|
+
~transactionStore=TransactionStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
|
|
68
73
|
~logger: Pino.t,
|
|
69
74
|
): t => {
|
|
70
75
|
logger,
|
|
71
76
|
fetchState,
|
|
77
|
+
indexingAddresses,
|
|
72
78
|
sourceManager,
|
|
73
79
|
chainConfig,
|
|
74
80
|
isProgressAtHead,
|
|
@@ -78,7 +84,7 @@ let make = (
|
|
|
78
84
|
pendingBudget: 0.,
|
|
79
85
|
reorgDetection,
|
|
80
86
|
safeCheckpointTracking,
|
|
81
|
-
transactionStore
|
|
87
|
+
transactionStore,
|
|
82
88
|
}
|
|
83
89
|
|
|
84
90
|
let makeInternal = (
|
|
@@ -216,8 +222,12 @@ let makeInternal = (
|
|
|
216
222
|
| None => ()
|
|
217
223
|
}
|
|
218
224
|
|
|
225
|
+
let contractConfigs = IndexingAddresses.makeContractConfigs(~eventConfigs)
|
|
226
|
+
let indexingAddressIndex = IndexingAddresses.make(~contractConfigs, ~addresses=indexingAddresses)
|
|
227
|
+
|
|
219
228
|
let fetchState = FetchState.make(
|
|
220
229
|
~maxAddrInPartition=config.maxAddrInPartition,
|
|
230
|
+
~contractConfigs,
|
|
221
231
|
~addresses=indexingAddresses,
|
|
222
232
|
~progressBlockNumber,
|
|
223
233
|
~startBlock,
|
|
@@ -261,11 +271,13 @@ let makeInternal = (
|
|
|
261
271
|
let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
|
|
262
272
|
let syncConfig = rpc.syncConfig
|
|
263
273
|
let ws = rpc.ws
|
|
274
|
+
let headers = rpc.headers
|
|
264
275
|
{
|
|
265
276
|
url: rpc.url,
|
|
266
277
|
sourceFor: rpc.sourceFor,
|
|
267
278
|
?syncConfig,
|
|
268
279
|
?ws,
|
|
280
|
+
?headers,
|
|
269
281
|
}
|
|
270
282
|
})
|
|
271
283
|
EvmChain.makeSources(
|
|
@@ -310,6 +322,7 @@ let makeInternal = (
|
|
|
310
322
|
make(
|
|
311
323
|
~chainConfig,
|
|
312
324
|
~fetchState,
|
|
325
|
+
~indexingAddresses=indexingAddressIndex,
|
|
313
326
|
~sourceManager=SourceManager.make(~sources, ~isRealtime, ~reducedPollingInterval?),
|
|
314
327
|
~reorgDetection=ReorgDetection.make(
|
|
315
328
|
~chainReorgCheckpoints,
|
|
@@ -324,6 +337,10 @@ let makeInternal = (
|
|
|
324
337
|
~committedProgressBlockNumber=progressBlockNumber,
|
|
325
338
|
~timestampCaughtUpToHeadOrEndblock,
|
|
326
339
|
~numEventsProcessed,
|
|
340
|
+
~transactionStore=TransactionStore.make(
|
|
341
|
+
~ecosystem=config.ecosystem.name,
|
|
342
|
+
~shouldChecksum=!lowercaseAddresses,
|
|
343
|
+
),
|
|
327
344
|
~logger,
|
|
328
345
|
)
|
|
329
346
|
}
|
|
@@ -413,7 +430,8 @@ let timestampCaughtUpToHeadOrEndblock = (cs: t) => cs.timestampCaughtUpToHeadOrE
|
|
|
413
430
|
// Fetch-frontier reads. The FetchState is owned here; callers go through these
|
|
414
431
|
// rather than reaching into it.
|
|
415
432
|
let knownHeight = (cs: t) => cs.fetchState.knownHeight
|
|
416
|
-
let
|
|
433
|
+
let contractAddresses = (cs: t, ~contractName) =>
|
|
434
|
+
cs.indexingAddresses->IndexingAddresses.getContractAddresses(~contractName)
|
|
417
435
|
let bufferSize = (cs: t) => cs.fetchState->FetchState.bufferSize
|
|
418
436
|
let bufferReadyCount = (cs: t) => cs.fetchState->FetchState.bufferReadyCount
|
|
419
437
|
let getProgressPercentage = (cs: t) => cs.fetchState->FetchState.getProgressPercentage
|
|
@@ -527,12 +545,21 @@ let handleQueryResult = (
|
|
|
527
545
|
|
|
528
546
|
let fs = switch newItemsWithDcs {
|
|
529
547
|
| [] => cs.fetchState
|
|
530
|
-
| _ =>
|
|
548
|
+
| _ =>
|
|
549
|
+
cs.fetchState->FetchState.registerDynamicContracts(
|
|
550
|
+
~indexingAddresses=cs.indexingAddresses,
|
|
551
|
+
newItemsWithDcs,
|
|
552
|
+
)
|
|
531
553
|
}
|
|
532
554
|
|
|
533
555
|
cs.fetchState =
|
|
534
556
|
fs
|
|
535
|
-
->FetchState.handleQueryResult(
|
|
557
|
+
->FetchState.handleQueryResult(
|
|
558
|
+
~indexingAddresses=cs.indexingAddresses,
|
|
559
|
+
~query,
|
|
560
|
+
~latestFetchedBlock,
|
|
561
|
+
~newItems,
|
|
562
|
+
)
|
|
536
563
|
->FetchState.updateKnownHeight(~knownHeight)
|
|
537
564
|
|
|
538
565
|
// The query is no longer in flight, so release its reservation.
|
|
@@ -602,7 +629,7 @@ let toChainData = (cs: t): chainData => {
|
|
|
602
629
|
numBatchesFetched: 0,
|
|
603
630
|
startBlock: cs.fetchState.startBlock,
|
|
604
631
|
endBlock: cs.fetchState.endBlock,
|
|
605
|
-
numAddresses: cs.
|
|
632
|
+
numAddresses: cs.indexingAddresses->IndexingAddresses.size,
|
|
606
633
|
}
|
|
607
634
|
|
|
608
635
|
// Snapshot the inputs a batch build needs from this chain.
|
|
@@ -761,7 +788,11 @@ let rollback = (
|
|
|
761
788
|
)
|
|
762
789
|
| None => ()
|
|
763
790
|
}
|
|
764
|
-
cs.fetchState =
|
|
791
|
+
cs.fetchState =
|
|
792
|
+
cs.fetchState->FetchState.rollback(
|
|
793
|
+
~indexingAddresses=cs.indexingAddresses,
|
|
794
|
+
~targetBlockNumber=newProgressBlockNumber,
|
|
795
|
+
)
|
|
765
796
|
cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
|
|
766
797
|
cs.committedProgressBlockNumber = newProgressBlockNumber
|
|
767
798
|
cs.numEventsProcessed = newTotalEventsProcessed
|
|
@@ -772,7 +803,10 @@ let rollback = (
|
|
|
772
803
|
~blockNumber=rollbackTargetBlockNumber,
|
|
773
804
|
)
|
|
774
805
|
cs.fetchState =
|
|
775
|
-
cs.fetchState->FetchState.rollback(
|
|
806
|
+
cs.fetchState->FetchState.rollback(
|
|
807
|
+
~indexingAddresses=cs.indexingAddresses,
|
|
808
|
+
~targetBlockNumber=rollbackTargetBlockNumber,
|
|
809
|
+
)
|
|
776
810
|
cs.transactionStore->TransactionStore.rollback(rollbackTargetBlockNumber)
|
|
777
811
|
cs.committedProgressBlockNumber = Pervasives.min(
|
|
778
812
|
cs.committedProgressBlockNumber,
|
package/src/ChainState.res.mjs
CHANGED
|
@@ -21,6 +21,7 @@ import * as HyperFuelSource from "./sources/HyperFuelSource.res.mjs";
|
|
|
21
21
|
import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
|
|
22
22
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
23
23
|
import * as TransactionStore from "./sources/TransactionStore.res.mjs";
|
|
24
|
+
import * as IndexingAddresses from "./IndexingAddresses.res.mjs";
|
|
24
25
|
import * as SvmHyperSyncSource from "./sources/SvmHyperSyncSource.res.mjs";
|
|
25
26
|
import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
|
|
26
27
|
|
|
@@ -38,14 +39,16 @@ function configAddresses(chainConfig) {
|
|
|
38
39
|
return addresses;
|
|
39
40
|
}
|
|
40
41
|
|
|
41
|
-
function make(chainConfig, fetchState, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, logger) {
|
|
42
|
+
function make(chainConfig, fetchState, indexingAddresses, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, logger) {
|
|
42
43
|
let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
|
|
43
44
|
let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
|
|
44
45
|
let timestampCaughtUpToHeadOrEndblock = timestampCaughtUpToHeadOrEndblockOpt !== undefined ? Primitive_option.valFromOption(timestampCaughtUpToHeadOrEndblockOpt) : undefined;
|
|
45
46
|
let isProgressAtHead = isProgressAtHeadOpt !== undefined ? isProgressAtHeadOpt : false;
|
|
47
|
+
let transactionStore = transactionStoreOpt !== undefined ? Primitive_option.valFromOption(transactionStoreOpt) : TransactionStore.make("evm", false);
|
|
46
48
|
return {
|
|
47
49
|
logger: logger,
|
|
48
50
|
fetchState: fetchState,
|
|
51
|
+
indexingAddresses: indexingAddresses,
|
|
49
52
|
sourceManager: sourceManager,
|
|
50
53
|
chainConfig: chainConfig,
|
|
51
54
|
isProgressAtHead: isProgressAtHead,
|
|
@@ -55,7 +58,7 @@ function make(chainConfig, fetchState, sourceManager, reorgDetection, committedP
|
|
|
55
58
|
pendingBudget: 0,
|
|
56
59
|
reorgDetection: reorgDetection,
|
|
57
60
|
safeCheckpointTracking: safeCheckpointTracking,
|
|
58
|
-
transactionStore:
|
|
61
|
+
transactionStore: transactionStore
|
|
59
62
|
};
|
|
60
63
|
}
|
|
61
64
|
|
|
@@ -118,7 +121,9 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
118
121
|
}
|
|
119
122
|
});
|
|
120
123
|
}
|
|
121
|
-
let
|
|
124
|
+
let contractConfigs = IndexingAddresses.makeContractConfigs(eventConfigs);
|
|
125
|
+
let indexingAddressIndex = IndexingAddresses.make(contractConfigs, indexingAddresses);
|
|
126
|
+
let fetchState = FetchState.make(startBlock, endBlock, eventConfigs, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockConfigs, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
|
|
122
127
|
let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
|
|
123
128
|
if (reorgCheckpoint.chain_id === chainConfig.id) {
|
|
124
129
|
return reorgCheckpoint;
|
|
@@ -138,11 +143,13 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
138
143
|
let evmRpcs = sources.rpcs.map(rpc => {
|
|
139
144
|
let syncConfig = rpc.syncConfig;
|
|
140
145
|
let ws = rpc.ws;
|
|
146
|
+
let headers = rpc.headers;
|
|
141
147
|
return {
|
|
142
148
|
url: rpc.url,
|
|
143
149
|
sourceFor: rpc.sourceFor,
|
|
144
150
|
syncConfig: syncConfig,
|
|
145
|
-
ws: ws
|
|
151
|
+
ws: ws,
|
|
152
|
+
headers: headers
|
|
146
153
|
};
|
|
147
154
|
});
|
|
148
155
|
sources$1 = EvmChain.makeSources(chain, evmContracts, sources.hypersync, evmRpcs, lowercaseAddresses);
|
|
@@ -174,7 +181,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
174
181
|
sources$1 = sources._0;
|
|
175
182
|
break;
|
|
176
183
|
}
|
|
177
|
-
return make(chainConfig, fetchState, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, logger);
|
|
184
|
+
return make(chainConfig, fetchState, indexingAddressIndex, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), logger);
|
|
178
185
|
}
|
|
179
186
|
|
|
180
187
|
function makeFromConfig(chainConfig, config, registrations, knownHeight) {
|
|
@@ -238,8 +245,8 @@ function knownHeight(cs) {
|
|
|
238
245
|
return cs.fetchState.knownHeight;
|
|
239
246
|
}
|
|
240
247
|
|
|
241
|
-
function
|
|
242
|
-
return cs.
|
|
248
|
+
function contractAddresses(cs, contractName) {
|
|
249
|
+
return IndexingAddresses.getContractAddresses(cs.indexingAddresses, contractName);
|
|
243
250
|
}
|
|
244
251
|
|
|
245
252
|
function bufferSize(cs) {
|
|
@@ -339,8 +346,8 @@ function handleQueryResult(cs, query, newItems, newItemsWithDcs, latestFetchedBl
|
|
|
339
346
|
if (page !== undefined) {
|
|
340
347
|
cs.transactionStore.merge(Primitive_option.valFromOption(page));
|
|
341
348
|
}
|
|
342
|
-
let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, newItemsWithDcs) : cs.fetchState;
|
|
343
|
-
cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
|
|
349
|
+
let fs = newItemsWithDcs.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.indexingAddresses, newItemsWithDcs) : cs.fetchState;
|
|
350
|
+
cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, cs.indexingAddresses, query, latestFetchedBlock, newItems), knownHeight);
|
|
344
351
|
cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.estResponseSize);
|
|
345
352
|
}
|
|
346
353
|
|
|
@@ -374,7 +381,6 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
|
|
|
374
381
|
startBlock: init.startBlock,
|
|
375
382
|
endBlock: blockNumber,
|
|
376
383
|
normalSelection: init.normalSelection,
|
|
377
|
-
indexingAddresses: init.indexingAddresses,
|
|
378
384
|
contractConfigs: init.contractConfigs,
|
|
379
385
|
chainId: init.chainId,
|
|
380
386
|
latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
|
|
@@ -393,7 +399,6 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
|
|
|
393
399
|
startBlock: init$1.startBlock,
|
|
394
400
|
endBlock: blockNumber,
|
|
395
401
|
normalSelection: init$1.normalSelection,
|
|
396
|
-
indexingAddresses: init$1.indexingAddresses,
|
|
397
402
|
contractConfigs: init$1.contractConfigs,
|
|
398
403
|
chainId: init$1.chainId,
|
|
399
404
|
latestOnBlockBlockNumber: init$1.latestOnBlockBlockNumber,
|
|
@@ -407,7 +412,7 @@ function setEndBlockToFirstEvent(cs, blockNumber) {
|
|
|
407
412
|
}
|
|
408
413
|
|
|
409
414
|
function enterReorgThreshold(cs) {
|
|
410
|
-
cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined,
|
|
415
|
+
cs.fetchState = FetchState.updateInternal(cs.fetchState, undefined, undefined, cs.chainConfig.blockLag, undefined);
|
|
411
416
|
}
|
|
412
417
|
|
|
413
418
|
function toChainMetadata(cs) {
|
|
@@ -432,7 +437,7 @@ function toChainData(cs) {
|
|
|
432
437
|
numBatchesFetched: 0,
|
|
433
438
|
startBlock: cs.fetchState.startBlock,
|
|
434
439
|
endBlock: cs.fetchState.endBlock,
|
|
435
|
-
numAddresses:
|
|
440
|
+
numAddresses: IndexingAddresses.size(cs.indexingAddresses)
|
|
436
441
|
};
|
|
437
442
|
}
|
|
438
443
|
|
|
@@ -455,7 +460,7 @@ function isReadyToEnterReorgThresholdAfterBatch(cs, batch) {
|
|
|
455
460
|
function advanceAfterBatch(cs, batch, enteringReorgThreshold) {
|
|
456
461
|
let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
|
|
457
462
|
if (chainAfterBatch !== undefined) {
|
|
458
|
-
cs.fetchState = enteringReorgThreshold ? FetchState.updateInternal(chainAfterBatch.fetchState, undefined, undefined,
|
|
463
|
+
cs.fetchState = enteringReorgThreshold ? FetchState.updateInternal(chainAfterBatch.fetchState, undefined, undefined, cs.chainConfig.blockLag, undefined) : chainAfterBatch.fetchState;
|
|
459
464
|
return;
|
|
460
465
|
}
|
|
461
466
|
}
|
|
@@ -487,7 +492,6 @@ function applyBatchProgress(cs, batch) {
|
|
|
487
492
|
startBlock: init.startBlock,
|
|
488
493
|
endBlock: init.endBlock,
|
|
489
494
|
normalSelection: init.normalSelection,
|
|
490
|
-
indexingAddresses: init.indexingAddresses,
|
|
491
495
|
contractConfigs: init.contractConfigs,
|
|
492
496
|
chainId: init.chainId,
|
|
493
497
|
latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
|
|
@@ -523,7 +527,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
|
|
|
523
527
|
if (newProgressBlockNumber === undefined) {
|
|
524
528
|
if (isReorgChain) {
|
|
525
529
|
cs.reorgDetection = ReorgDetection.rollbackToValidBlockNumber(cs.reorgDetection, rollbackTargetBlockNumber);
|
|
526
|
-
cs.fetchState = FetchState.rollback(cs.fetchState, rollbackTargetBlockNumber);
|
|
530
|
+
cs.fetchState = FetchState.rollback(cs.fetchState, cs.indexingAddresses, rollbackTargetBlockNumber);
|
|
527
531
|
cs.transactionStore.rollback(rollbackTargetBlockNumber);
|
|
528
532
|
cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
|
|
529
533
|
return;
|
|
@@ -545,7 +549,7 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
|
|
|
545
549
|
if (safeCheckpointTracking !== undefined) {
|
|
546
550
|
cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
|
|
547
551
|
}
|
|
548
|
-
cs.fetchState = FetchState.rollback(cs.fetchState, newProgressBlockNumber);
|
|
552
|
+
cs.fetchState = FetchState.rollback(cs.fetchState, cs.indexingAddresses, newProgressBlockNumber);
|
|
549
553
|
cs.transactionStore.rollback(newProgressBlockNumber);
|
|
550
554
|
cs.committedProgressBlockNumber = newProgressBlockNumber;
|
|
551
555
|
cs.numEventsProcessed = newTotalEventsProcessed;
|
|
@@ -568,7 +572,7 @@ export {
|
|
|
568
572
|
startFetchingQueries,
|
|
569
573
|
timestampCaughtUpToHeadOrEndblock,
|
|
570
574
|
knownHeight,
|
|
571
|
-
|
|
575
|
+
contractAddresses,
|
|
572
576
|
bufferSize,
|
|
573
577
|
bufferReadyCount,
|
|
574
578
|
getProgressPercentage,
|
package/src/ChainState.resi
CHANGED
|
@@ -8,6 +8,7 @@ let configAddresses: Config.chain => array<Internal.indexingAddress>
|
|
|
8
8
|
let make: (
|
|
9
9
|
~chainConfig: Config.chain,
|
|
10
10
|
~fetchState: FetchState.t,
|
|
11
|
+
~indexingAddresses: IndexingAddresses.t,
|
|
11
12
|
~sourceManager: SourceManager.t,
|
|
12
13
|
~reorgDetection: ReorgDetection.t,
|
|
13
14
|
~committedProgressBlockNumber: int,
|
|
@@ -15,6 +16,7 @@ let make: (
|
|
|
15
16
|
~numEventsProcessed: float=?,
|
|
16
17
|
~timestampCaughtUpToHeadOrEndblock: option<Date.t>=?,
|
|
17
18
|
~isProgressAtHead: bool=?,
|
|
19
|
+
~transactionStore: TransactionStore.t=?,
|
|
18
20
|
~logger: Pino.t,
|
|
19
21
|
) => t
|
|
20
22
|
|
|
@@ -51,7 +53,7 @@ let timestampCaughtUpToHeadOrEndblock: t => option<Date.t>
|
|
|
51
53
|
|
|
52
54
|
// Fetch-frontier reads.
|
|
53
55
|
let knownHeight: t => int
|
|
54
|
-
let
|
|
56
|
+
let contractAddresses: (t, ~contractName: string) => array<Address.t>
|
|
55
57
|
let bufferSize: t => int
|
|
56
58
|
let bufferReadyCount: t => int
|
|
57
59
|
let getProgressPercentage: t => float
|
package/src/Config.res
CHANGED
|
@@ -23,6 +23,7 @@ type evmRpcConfig = {
|
|
|
23
23
|
sourceFor: Source.sourceFor,
|
|
24
24
|
syncConfig: option<sourceSyncOptions>,
|
|
25
25
|
ws: option<string>,
|
|
26
|
+
headers: option<dict<string>>,
|
|
26
27
|
}
|
|
27
28
|
|
|
28
29
|
type sourceConfig =
|
|
@@ -156,6 +157,7 @@ let rpcConfigSchema = S.schema(s =>
|
|
|
156
157
|
"url": s.matches(S.string),
|
|
157
158
|
"for": s.matches(rpcSourceForSchema),
|
|
158
159
|
"ws": s.matches(S.option(S.string)),
|
|
160
|
+
"headers": s.matches(S.option(S.dict(S.string))),
|
|
159
161
|
"initialBlockInterval": s.matches(S.option(S.int)),
|
|
160
162
|
"backoffMultiplicative": s.matches(S.option(S.float)),
|
|
161
163
|
"accelerationAdditive": s.matches(S.option(S.int)),
|
|
@@ -907,6 +909,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
907
909
|
sourceFor: parseRpcSourceFor(rpcConfig["for"]),
|
|
908
910
|
syncConfig,
|
|
909
911
|
ws: rpcConfig["ws"],
|
|
912
|
+
headers: rpcConfig["headers"],
|
|
910
913
|
}
|
|
911
914
|
})
|
|
912
915
|
EvmSourceConfig({hypersync: publicChainConfig["hypersync"], rpcs})
|
package/src/Config.res.mjs
CHANGED
|
@@ -81,6 +81,7 @@ let rpcConfigSchema = S$RescriptSchema.schema(s => ({
|
|
|
81
81
|
url: s.m(S$RescriptSchema.string),
|
|
82
82
|
for: s.m(rpcSourceForSchema),
|
|
83
83
|
ws: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
|
|
84
|
+
headers: s.m(S$RescriptSchema.option(S$RescriptSchema.dict(S$RescriptSchema.string))),
|
|
84
85
|
initialBlockInterval: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
|
|
85
86
|
backoffMultiplicative: s.m(S$RescriptSchema.option(S$RescriptSchema.float)),
|
|
86
87
|
accelerationAdditive: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
|
|
@@ -606,7 +607,8 @@ function fromPublic(publicConfigJson) {
|
|
|
606
607
|
url: rpcConfig.url,
|
|
607
608
|
sourceFor: parseRpcSourceFor(rpcConfig.for),
|
|
608
609
|
syncConfig: syncConfig,
|
|
609
|
-
ws: rpcConfig.ws
|
|
610
|
+
ws: rpcConfig.ws,
|
|
611
|
+
headers: rpcConfig.headers
|
|
610
612
|
};
|
|
611
613
|
});
|
|
612
614
|
sourceConfig = {
|
package/src/Core.res
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
// tighter `Null.t` captures the exact boundary shape.
|
|
7
7
|
type evmHypersyncClientCtor
|
|
8
8
|
type evmRpcClientCtor
|
|
9
|
-
type evmDecoderCtor
|
|
10
9
|
type svmHypersyncClientCtor
|
|
11
10
|
type hyperfuelClientCtor
|
|
12
11
|
type transactionStoreCtor
|
|
@@ -18,8 +17,6 @@ type addon = {
|
|
|
18
17
|
evmHypersyncClient: evmHypersyncClientCtor,
|
|
19
18
|
@as("EvmRpcClient")
|
|
20
19
|
evmRpcClient: evmRpcClientCtor,
|
|
21
|
-
@as("EvmDecoder")
|
|
22
|
-
evmDecoder: evmDecoderCtor,
|
|
23
20
|
@as("SvmHypersyncClient")
|
|
24
21
|
svmHypersyncClient: svmHypersyncClientCtor,
|
|
25
22
|
@as("HyperfuelClient")
|
package/src/ExitOnCaughtUp.res
CHANGED
|
@@ -4,7 +4,15 @@ let run = async (state: IndexerState.t) => {
|
|
|
4
4
|
ChainMetadata.stage(state)
|
|
5
5
|
await state->Writing.flush
|
|
6
6
|
if !(state->IndexerState.isStopped) && !(state->IndexerState.isResolvingReorg) {
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
// A simulate run fails here when a provided item never reached a handler —
|
|
8
|
+
// dead test code. The tracker is None (a no-op) off the simulate path.
|
|
9
|
+
switch state
|
|
10
|
+
->IndexerState.simulateDeadInputTracker
|
|
11
|
+
->Option.flatMap(SimulateDeadInputTracker.failureMessage) {
|
|
12
|
+
| None =>
|
|
13
|
+
Logging.info("Exiting with success")
|
|
14
|
+
NodeJs.process->NodeJs.exitWithCode(Success)
|
|
15
|
+
| Some(message) => state->IndexerState.errorExit(ErrorHandling.make(Utils.Error.make(message)))
|
|
16
|
+
}
|
|
9
17
|
}
|
|
10
18
|
}
|
|
@@ -5,11 +5,20 @@ import * as Writing from "./Writing.res.mjs";
|
|
|
5
5
|
import * as Process from "process";
|
|
6
6
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
7
7
|
import * as ChainMetadata from "./ChainMetadata.res.mjs";
|
|
8
|
+
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
9
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
10
|
+
import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
|
|
8
11
|
|
|
9
12
|
async function run(state) {
|
|
10
13
|
ChainMetadata.stage(state);
|
|
11
14
|
await Writing.flush(state);
|
|
12
|
-
if (
|
|
15
|
+
if (IndexerState.isStopped(state) || IndexerState.isResolvingReorg(state)) {
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
let message = Stdlib_Option.flatMap(IndexerState.simulateDeadInputTracker(state), SimulateDeadInputTracker.failureMessage);
|
|
19
|
+
if (message !== undefined) {
|
|
20
|
+
return IndexerState.errorExit(state, ErrorHandling.make(new Error(message), undefined, undefined));
|
|
21
|
+
} else {
|
|
13
22
|
Logging.info("Exiting with success");
|
|
14
23
|
Process.exit(0);
|
|
15
24
|
return;
|