envio 3.12.1 → 3.13.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/BatchProcessing.res +9 -9
- package/src/BatchProcessing.res.mjs +5 -5
- package/src/Bin.res +24 -17
- package/src/Bin.res.mjs +5 -0
- package/src/ChainFetching.res +24 -10
- package/src/ChainFetching.res.mjs +8 -3
- package/src/ChainState.res +43 -0
- package/src/ChainState.res.mjs +43 -0
- package/src/ChainState.resi +2 -0
- package/src/Config.res +35 -0
- package/src/Config.res.mjs +39 -0
- package/src/Core.res +4 -0
- package/src/Core.res.mjs +4 -0
- package/src/CrossChainState.res +60 -3
- package/src/CrossChainState.res.mjs +38 -4
- package/src/CrossChainState.resi +10 -1
- package/src/Env.res +4 -0
- package/src/IndexerLoop.res +2 -0
- package/src/IndexerLoop.res.mjs +1 -0
- package/src/IndexerState.res +41 -1
- package/src/IndexerState.res.mjs +43 -4
- package/src/IndexerState.resi +10 -0
- package/src/Logging.res +38 -6
- package/src/Logging.res.mjs +32 -5
- package/src/Main.res +131 -268
- package/src/Main.res.mjs +28 -152
- package/src/Metrics.res +263 -102
- package/src/Metrics.res.mjs +227 -48
- package/src/Persistence.res +27 -2
- package/src/Persistence.res.mjs +9 -2
- package/src/PgStorage.res +17 -9
- package/src/PgStorage.res.mjs +11 -8
- package/src/Server.res +181 -0
- package/src/Server.res.mjs +143 -0
- package/src/Supervisor.res +415 -0
- package/src/Supervisor.res.mjs +325 -0
- package/src/TestIndexer.res.mjs +1 -1
- package/src/Worker.res +95 -0
- package/src/Worker.res.mjs +80 -0
- package/src/bindings/NodeJs.res +41 -0
- package/src/db/InternalTable.res +8 -1
- package/src/db/InternalTable.res.mjs +5 -1
- package/src/tui/Tui.res +24 -0
- package/src/tui/Tui.res.mjs +18 -0
- package/src/tui/components/SyncETA.res +12 -6
- package/src/tui/components/SyncETA.res.mjs +12 -8
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.13.0-alpha.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -68,10 +68,10 @@
|
|
|
68
68
|
"tsx": "4.21.0"
|
|
69
69
|
},
|
|
70
70
|
"optionalDependencies": {
|
|
71
|
-
"envio-linux-x64": "3.
|
|
72
|
-
"envio-linux-x64-musl": "3.
|
|
73
|
-
"envio-linux-arm64": "3.
|
|
74
|
-
"envio-darwin-x64": "3.
|
|
75
|
-
"envio-darwin-arm64": "3.
|
|
71
|
+
"envio-linux-x64": "3.13.0-alpha.0",
|
|
72
|
+
"envio-linux-x64-musl": "3.13.0-alpha.0",
|
|
73
|
+
"envio-linux-arm64": "3.13.0-alpha.0",
|
|
74
|
+
"envio-darwin-x64": "3.13.0-alpha.0",
|
|
75
|
+
"envio-darwin-arm64": "3.13.0-alpha.0"
|
|
76
76
|
}
|
|
77
77
|
}
|
package/src/BatchProcessing.res
CHANGED
|
@@ -76,11 +76,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
76
76
|
let isBelowReorgThreshold =
|
|
77
77
|
!isInReorgThresholdBeforeUpdate && (state->IndexerState.config).shouldRollbackOnReorg
|
|
78
78
|
let shouldEnterReorgThreshold =
|
|
79
|
-
isBelowReorgThreshold &&
|
|
80
|
-
state
|
|
81
|
-
->IndexerState.chainStates
|
|
82
|
-
->Dict.valuesToArray
|
|
83
|
-
->Array.every(cs => cs->ChainState.isReadyToEnterReorgThresholdAfterBatch(~batch))
|
|
79
|
+
isBelowReorgThreshold && state->IndexerState.isReadyToEnterReorgThreshold(~batch)
|
|
84
80
|
|
|
85
81
|
if shouldEnterReorgThreshold {
|
|
86
82
|
IndexerState.enterReorgThreshold(state)
|
|
@@ -95,7 +91,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
95
91
|
// finalizing resumes exactly here: it still owes the schema its deferred
|
|
96
92
|
// indexes, and no batch will ever come along to notice.
|
|
97
93
|
state->IndexerState.markCaughtUpIfSettled
|
|
98
|
-
if state->IndexerState.
|
|
94
|
+
if state->IndexerState.shouldFinalizeIndexes {
|
|
99
95
|
await FinalizeBackfill.run(state)
|
|
100
96
|
}
|
|
101
97
|
|
|
@@ -111,7 +107,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
111
107
|
// When resuming from persisted state, all events may already be processed.
|
|
112
108
|
if EventProcessing.allChainsEventsProcessedToEndblock(state->IndexerState.chainStates) {
|
|
113
109
|
Logging.info("All chains are caught up to end blocks.")
|
|
114
|
-
if !(state->IndexerState.keepProcessAlive) {
|
|
110
|
+
if !(state->IndexerState.keepProcessAlive) && !(state->IndexerState.isHoldingRealtime) {
|
|
115
111
|
await ExitOnCaughtUp.run(state)
|
|
116
112
|
}
|
|
117
113
|
}
|
|
@@ -166,7 +162,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
166
162
|
// Backfilling → FinalizingIndexes → Ready. Awaiting here holds the
|
|
167
163
|
// processing loop for the whole finalize, which is what pauses
|
|
168
164
|
// processing while the indexes are built.
|
|
169
|
-
if state->IndexerState.
|
|
165
|
+
if state->IndexerState.shouldFinalizeIndexes {
|
|
170
166
|
await FinalizeBackfill.run(state)
|
|
171
167
|
}
|
|
172
168
|
|
|
@@ -186,7 +182,11 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
|
|
|
186
182
|
Logging.info("All chains are caught up to end blocks.")
|
|
187
183
|
}
|
|
188
184
|
|
|
189
|
-
if
|
|
185
|
+
if (
|
|
186
|
+
allCaughtUp &&
|
|
187
|
+
!(state->IndexerState.keepProcessAlive) &&
|
|
188
|
+
!(state->IndexerState.isHoldingRealtime)
|
|
189
|
+
) {
|
|
190
190
|
await ExitOnCaughtUp.run(state)
|
|
191
191
|
} else if (
|
|
192
192
|
// In auto-exit mode, error if all chains reached head with no events found
|
|
@@ -26,7 +26,7 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
26
26
|
let batch = IndexerState.createBatch(state, IndexerState.config(state).batchSize);
|
|
27
27
|
let progressedChainsById = batch.progressedChainsById;
|
|
28
28
|
let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
|
|
29
|
-
let shouldEnterReorgThreshold = isBelowReorgThreshold &&
|
|
29
|
+
let shouldEnterReorgThreshold = isBelowReorgThreshold && IndexerState.isReadyToEnterReorgThreshold(state, batch);
|
|
30
30
|
if (shouldEnterReorgThreshold) {
|
|
31
31
|
IndexerState.enterReorgThreshold(state);
|
|
32
32
|
}
|
|
@@ -35,7 +35,7 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
35
35
|
scheduleFetch();
|
|
36
36
|
}
|
|
37
37
|
IndexerState.markCaughtUpIfSettled(state);
|
|
38
|
-
if (IndexerState.
|
|
38
|
+
if (IndexerState.shouldFinalizeIndexes(state)) {
|
|
39
39
|
await FinalizeBackfill.run(state);
|
|
40
40
|
}
|
|
41
41
|
if (!isRealtimeBeforeUpdate && IndexerState.isRealtime(state)) {
|
|
@@ -44,7 +44,7 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
44
44
|
}
|
|
45
45
|
if (EventProcessing.allChainsEventsProcessedToEndblock(IndexerState.chainStates(state))) {
|
|
46
46
|
Logging.info("All chains are caught up to end blocks.");
|
|
47
|
-
if (!IndexerState.keepProcessAlive(state)) {
|
|
47
|
+
if (!IndexerState.keepProcessAlive(state) && !IndexerState.isHoldingRealtime(state)) {
|
|
48
48
|
return await ExitOnCaughtUp.run(state);
|
|
49
49
|
} else {
|
|
50
50
|
return;
|
|
@@ -70,7 +70,7 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
70
70
|
}
|
|
71
71
|
IndexerState.clearRollback(state);
|
|
72
72
|
IndexerState.applyBatchProgress(state, batch);
|
|
73
|
-
if (IndexerState.
|
|
73
|
+
if (IndexerState.shouldFinalizeIndexes(state)) {
|
|
74
74
|
await FinalizeBackfill.run(state);
|
|
75
75
|
}
|
|
76
76
|
if (!isRealtimeBeforeUpdate && IndexerState.isRealtime(state)) {
|
|
@@ -81,7 +81,7 @@ async function processNextBatch(state, scheduleFetch) {
|
|
|
81
81
|
if (allCaughtUp) {
|
|
82
82
|
Logging.info("All chains are caught up to end blocks.");
|
|
83
83
|
}
|
|
84
|
-
if (allCaughtUp && !IndexerState.keepProcessAlive(state)) {
|
|
84
|
+
if (allCaughtUp && !IndexerState.keepProcessAlive(state) && !IndexerState.isHoldingRealtime(state)) {
|
|
85
85
|
return await ExitOnCaughtUp.run(state);
|
|
86
86
|
} else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(ChainState.isAtHeadWithoutEndBlock)) {
|
|
87
87
|
return IndexerState.errorExit(state, ErrorHandling.make(new Error("No events found between startBlock and chain head. Cannot auto-detect endBlock."), undefined, undefined));
|
package/src/Bin.res
CHANGED
|
@@ -51,23 +51,30 @@ let applyEnv = (env: dict<JSON.t>) =>
|
|
|
51
51
|
|
|
52
52
|
let run = async args => {
|
|
53
53
|
try {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
54
|
+
if Worker.isEnabled {
|
|
55
|
+
Worker.bindToSupervisor()
|
|
56
|
+
// Its working directory, its environment and the chains it drives all came
|
|
57
|
+
// with the fork, so a worker starts the same way every other process does.
|
|
58
|
+
await Main.start()
|
|
59
|
+
} else {
|
|
60
|
+
switch (await Core.runCli(args))->Null.toOption {
|
|
61
|
+
// Rust-only command (codegen / init / stop / docker / metrics / help /
|
|
62
|
+
// version / scripts) — nothing for JS to do, exit cleanly.
|
|
63
|
+
| None => ()
|
|
64
|
+
| Some(json) =>
|
|
65
|
+
switch decodeCommand(json->JSON.parseOrThrow) {
|
|
66
|
+
| Start({reset, cwd, env, config}) =>
|
|
67
|
+
Config.prime(config)
|
|
68
|
+
processChdir(cwd)
|
|
69
|
+
applyEnv(env)
|
|
70
|
+
await Main.start(~reset)
|
|
71
|
+
| Migrate({reset, config}) =>
|
|
72
|
+
Config.prime(config)
|
|
73
|
+
await Main.migrate(~reset)
|
|
74
|
+
| DropSchema({config}) =>
|
|
75
|
+
Config.prime(config)
|
|
76
|
+
await Main.dropSchema()
|
|
77
|
+
}
|
|
71
78
|
}
|
|
72
79
|
}
|
|
73
80
|
} catch {
|
package/src/Bin.res.mjs
CHANGED
|
@@ -4,6 +4,7 @@ import * as Core from "./Core.res.mjs";
|
|
|
4
4
|
import * as Main from "./Main.res.mjs";
|
|
5
5
|
import * as Utils from "./Utils.res.mjs";
|
|
6
6
|
import * as Config from "./Config.res.mjs";
|
|
7
|
+
import * as Worker from "./Worker.res.mjs";
|
|
7
8
|
import * as Logging from "./Logging.res.mjs";
|
|
8
9
|
import * as Process from "process";
|
|
9
10
|
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
@@ -57,6 +58,10 @@ function applyEnv(env) {
|
|
|
57
58
|
|
|
58
59
|
async function run(args) {
|
|
59
60
|
try {
|
|
61
|
+
if (Worker.isEnabled) {
|
|
62
|
+
Worker.bindToSupervisor();
|
|
63
|
+
return await Main.start(undefined, undefined, undefined, undefined, undefined);
|
|
64
|
+
}
|
|
60
65
|
let json = await Core.runCli(args);
|
|
61
66
|
if (json === null) {
|
|
62
67
|
return;
|
package/src/ChainFetching.res
CHANGED
|
@@ -262,7 +262,6 @@ and applyQueryResponse = (
|
|
|
262
262
|
~transactionStore,
|
|
263
263
|
) => {
|
|
264
264
|
let chainState = state->IndexerState.getChainState(~chainId)
|
|
265
|
-
let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
|
|
266
265
|
|
|
267
266
|
chainState->ChainState.handleQueryResult(
|
|
268
267
|
~query,
|
|
@@ -280,15 +279,30 @@ and applyQueryResponse = (
|
|
|
280
279
|
)
|
|
281
280
|
}
|
|
282
281
|
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
)
|
|
291
|
-
|
|
282
|
+
// Report the milestone this response brought the fetch frontier to, once.
|
|
283
|
+
// The chain reaches it again every time the head moves and it catches up, so
|
|
284
|
+
// what keeps the line off the log is the chain having already reported it,
|
|
285
|
+
// not the transition — which re-arms on every advance.
|
|
286
|
+
switch chainState->ChainState.isFetchingAtHead
|
|
287
|
+
? chainState->ChainState.takeFetchedTo
|
|
288
|
+
: None {
|
|
289
|
+
| None => ()
|
|
290
|
+
| Some((target, block)) =>
|
|
291
|
+
// What the chain is waiting on, which is not itself. A held process is
|
|
292
|
+
// waiting on chains it doesn't drive, so it says so even when it drives
|
|
293
|
+
// only one — how the run is split is not the reader's problem.
|
|
294
|
+
let waitingOn = if (
|
|
295
|
+
state->IndexerState.isHoldingRealtime ||
|
|
296
|
+
state->IndexerState.chainStates->Dict.keysToArray->Array.length > 1
|
|
297
|
+
) {
|
|
298
|
+
" Waiting for other chains."
|
|
299
|
+
} else {
|
|
300
|
+
""
|
|
301
|
+
}
|
|
302
|
+
chainState->ChainState.logger->Logging.childInfo({
|
|
303
|
+
"msg": `Fetched to ${target}.${waitingOn}`,
|
|
304
|
+
"block": block,
|
|
305
|
+
})
|
|
292
306
|
}
|
|
293
307
|
}
|
|
294
308
|
|
|
@@ -65,14 +65,19 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config, ch
|
|
|
65
65
|
|
|
66
66
|
function applyQueryResponse(state, chainId, newItems, newRegistrations, knownHeight, latestFetchedBlock, query, transactionStore) {
|
|
67
67
|
let chainState = IndexerState.getChainState(state, chainId);
|
|
68
|
-
let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
|
|
69
68
|
ChainState.handleQueryResult(chainState, query, newItems, newRegistrations, latestFetchedBlock, knownHeight, transactionStore);
|
|
70
69
|
if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
|
|
71
70
|
ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
|
|
72
71
|
}
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
let match = ChainState.isFetchingAtHead(chainState) ? ChainState.takeFetchedTo(chainState) : undefined;
|
|
73
|
+
if (match === undefined) {
|
|
74
|
+
return;
|
|
75
75
|
}
|
|
76
|
+
let waitingOn = IndexerState.isHoldingRealtime(state) || Object.keys(IndexerState.chainStates(state)).length > 1 ? " Waiting for other chains." : "";
|
|
77
|
+
Logging.childInfo(ChainState.logger(chainState), {
|
|
78
|
+
msg: `Fetched to ` + match[0] + `.` + waitingOn,
|
|
79
|
+
block: match[1]
|
|
80
|
+
});
|
|
76
81
|
}
|
|
77
82
|
|
|
78
83
|
async function onQueryResponse(state, param, stateId, scheduleFetch, scheduleProcessing, scheduleRollback) {
|
package/src/ChainState.res
CHANGED
|
@@ -58,6 +58,9 @@ type t = {
|
|
|
58
58
|
mutable blockRangeFetchCount: float,
|
|
59
59
|
mutable blockRangeFetchedEvents: float,
|
|
60
60
|
mutable blockRangeFetchedBlocks: float,
|
|
61
|
+
// What this chain last reported reaching. The head moves, so a chain reaches
|
|
62
|
+
// it again on every catch-up; only a new milestone is worth a line.
|
|
63
|
+
mutable reportedFetchedTo: option<string>,
|
|
61
64
|
mutable reorgCount: int,
|
|
62
65
|
mutable reorgDetectedBlock: option<int>,
|
|
63
66
|
mutable rollbackTargetBlock: option<int>,
|
|
@@ -171,6 +174,7 @@ let make = (
|
|
|
171
174
|
blockRangeFetchCount: 0.,
|
|
172
175
|
blockRangeFetchedEvents: 0.,
|
|
173
176
|
blockRangeFetchedBlocks: 0.,
|
|
177
|
+
reportedFetchedTo: None,
|
|
174
178
|
reorgCount: 0,
|
|
175
179
|
reorgDetectedBlock: None,
|
|
176
180
|
rollbackTargetBlock: None,
|
|
@@ -642,6 +646,37 @@ let dispatch = (
|
|
|
642
646
|
|
|
643
647
|
// --- Derived (pure). ---
|
|
644
648
|
|
|
649
|
+
%%private(
|
|
650
|
+
// Where the fetch frontier has just landed. Below the reorg threshold a chain
|
|
651
|
+
// can only fetch the finalized range, so reaching the end of it is not
|
|
652
|
+
// reaching the head — the rest opens up once the indexer enters the threshold.
|
|
653
|
+
let fetchedTo = (cs: t) =>
|
|
654
|
+
switch cs.fetchState.endBlock {
|
|
655
|
+
| Some(endBlock) if endBlock <= cs.fetchState.knownHeight - cs.fetchState.blockLag => (
|
|
656
|
+
"the end block",
|
|
657
|
+
endBlock,
|
|
658
|
+
)
|
|
659
|
+
| _ =>
|
|
660
|
+
cs.isInReorgThreshold
|
|
661
|
+
? ("the chain head", cs.fetchState.knownHeight)
|
|
662
|
+
: ("the safe block", cs.fetchState.knownHeight - cs.fetchState.blockLag)
|
|
663
|
+
}
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
// What this chain has just reached, the first time it reaches it. `None` once
|
|
667
|
+
// it has been reported: a chain catches up to a moving head over and over, and
|
|
668
|
+
// the milestone is the same one every time. A new one — the head past the
|
|
669
|
+
// threshold, an end block — is a line of its own.
|
|
670
|
+
let takeFetchedTo = (cs: t) => {
|
|
671
|
+
let (target, block) = cs->fetchedTo
|
|
672
|
+
if cs.reportedFetchedTo == Some(target) {
|
|
673
|
+
None
|
|
674
|
+
} else {
|
|
675
|
+
cs.reportedFetchedTo = Some(target)
|
|
676
|
+
Some((target, block))
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
|
|
645
680
|
let hasProcessedToEndblock = (cs: t) => {
|
|
646
681
|
let {committedProgressBlockNumber, fetchState} = cs
|
|
647
682
|
switch fetchState.endBlock {
|
|
@@ -1051,6 +1086,14 @@ let toChainBeforeBatch = (cs: t, ~isRealtime): Batch.chainBeforeBatch => {
|
|
|
1051
1086
|
|
|
1052
1087
|
// Whether the chain's post-batch fetch frontier is ready to cross into the reorg
|
|
1053
1088
|
// threshold, using the batch's progressed frontier when this chain advanced.
|
|
1089
|
+
// The same question asked of where the chain stands now rather than of where a
|
|
1090
|
+
// batch would leave it. Entering the threshold is what lifts the pre-threshold
|
|
1091
|
+
// lag, so a chain waiting to enter it has fetched as far as it can.
|
|
1092
|
+
let isReadyToEnterReorgThreshold = (cs: t) =>
|
|
1093
|
+
cs.fetchState->FetchState.isReadyToEnterReorgThreshold(
|
|
1094
|
+
~tolerance=cs.reorgThresholdReadyTolerance,
|
|
1095
|
+
)
|
|
1096
|
+
|
|
1054
1097
|
let isReadyToEnterReorgThresholdAfterBatch = (cs: t, ~batch: Batch.t) => {
|
|
1055
1098
|
let fetchState = switch batch.progressedChainsById->ChainId.Dict.dangerouslyGetNonOption(
|
|
1056
1099
|
cs.fetchState.chainId,
|
package/src/ChainState.res.mjs
CHANGED
|
@@ -21,6 +21,7 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
|
21
21
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
22
22
|
import * as ContractMapping from "./ContractMapping.res.mjs";
|
|
23
23
|
import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
|
|
24
|
+
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
24
25
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
25
26
|
import * as TransactionStore from "./sources/TransactionStore.res.mjs";
|
|
26
27
|
import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
|
|
@@ -96,6 +97,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
|
|
|
96
97
|
blockRangeFetchCount: 0,
|
|
97
98
|
blockRangeFetchedEvents: 0,
|
|
98
99
|
blockRangeFetchedBlocks: 0,
|
|
100
|
+
reportedFetchedTo: undefined,
|
|
99
101
|
reorgCount: 0,
|
|
100
102
|
reorgDetectedBlock: undefined,
|
|
101
103
|
rollbackTargetBlock: undefined,
|
|
@@ -398,6 +400,41 @@ function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId
|
|
|
398
400
|
return SourceManager.dispatch(cs.sourceManager, cs.fetchState, executeQuery, waitForNewBlock, onNewBlock, action, stateId);
|
|
399
401
|
}
|
|
400
402
|
|
|
403
|
+
function fetchedTo(cs) {
|
|
404
|
+
let endBlock = cs.fetchState.endBlock;
|
|
405
|
+
if (endBlock !== undefined && endBlock <= (cs.fetchState.knownHeight - cs.fetchState.blockLag | 0)) {
|
|
406
|
+
return [
|
|
407
|
+
"the end block",
|
|
408
|
+
endBlock
|
|
409
|
+
];
|
|
410
|
+
}
|
|
411
|
+
if (cs.isInReorgThreshold) {
|
|
412
|
+
return [
|
|
413
|
+
"the chain head",
|
|
414
|
+
cs.fetchState.knownHeight
|
|
415
|
+
];
|
|
416
|
+
} else {
|
|
417
|
+
return [
|
|
418
|
+
"the safe block",
|
|
419
|
+
cs.fetchState.knownHeight - cs.fetchState.blockLag | 0
|
|
420
|
+
];
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function takeFetchedTo(cs) {
|
|
425
|
+
let match = fetchedTo(cs);
|
|
426
|
+
let target = match[0];
|
|
427
|
+
if (Primitive_object.equal(cs.reportedFetchedTo, target)) {
|
|
428
|
+
return;
|
|
429
|
+
} else {
|
|
430
|
+
cs.reportedFetchedTo = target;
|
|
431
|
+
return [
|
|
432
|
+
target,
|
|
433
|
+
match[1]
|
|
434
|
+
];
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
401
438
|
function hasProcessedToEndblock(cs) {
|
|
402
439
|
let fetchState = cs.fetchState;
|
|
403
440
|
let committedProgressBlockNumber = cs.committedProgressBlockNumber;
|
|
@@ -741,6 +778,10 @@ function toChainBeforeBatch(cs, isRealtime) {
|
|
|
741
778
|
};
|
|
742
779
|
}
|
|
743
780
|
|
|
781
|
+
function isReadyToEnterReorgThreshold(cs) {
|
|
782
|
+
return FetchState.isReadyToEnterReorgThreshold(cs.reorgThresholdReadyTolerance, cs.fetchState);
|
|
783
|
+
}
|
|
784
|
+
|
|
744
785
|
function isReadyToEnterReorgThresholdAfterBatch(cs, batch) {
|
|
745
786
|
let chainAfterBatch = batch.progressedChainsById[cs.fetchState.chainId];
|
|
746
787
|
let fetchState = chainAfterBatch !== undefined ? chainAfterBatch.fetchState : cs.fetchState;
|
|
@@ -921,7 +962,9 @@ export {
|
|
|
921
962
|
toMetrics,
|
|
922
963
|
toChainMetadata,
|
|
923
964
|
toChainBeforeBatch,
|
|
965
|
+
isReadyToEnterReorgThreshold,
|
|
924
966
|
isReadyToEnterReorgThresholdAfterBatch,
|
|
967
|
+
takeFetchedTo,
|
|
925
968
|
hasProcessedToEndblock,
|
|
926
969
|
isDurablyCaughtUp,
|
|
927
970
|
getHighestBlockBelowThreshold,
|
package/src/ChainState.resi
CHANGED
|
@@ -120,9 +120,11 @@ let dispatch: (
|
|
|
120
120
|
let toMetrics: t => Metrics.chainMetrics
|
|
121
121
|
let toChainMetadata: t => InternalTable.Chains.metaFields
|
|
122
122
|
let toChainBeforeBatch: (t, ~isRealtime: bool) => Batch.chainBeforeBatch
|
|
123
|
+
let isReadyToEnterReorgThreshold: t => bool
|
|
123
124
|
let isReadyToEnterReorgThresholdAfterBatch: (t, ~batch: Batch.t) => bool
|
|
124
125
|
|
|
125
126
|
// Derived (pure).
|
|
127
|
+
let takeFetchedTo: t => option<(string, int)>
|
|
126
128
|
let hasProcessedToEndblock: t => bool
|
|
127
129
|
let isDurablyCaughtUp: t => bool
|
|
128
130
|
let getHighestBlockBelowThreshold: t => int
|
package/src/Config.res
CHANGED
|
@@ -625,6 +625,22 @@ let getChain = (config, ~chainId) =>
|
|
|
625
625
|
"No chain with id " ++ chainId->ChainId.toString ++ " found in config.yaml",
|
|
626
626
|
)
|
|
627
627
|
|
|
628
|
+
// Whether every entity belongs to exactly one chain. Only then is a unit of
|
|
629
|
+
// this indexer's work attributable to a chain at all, which is what lets a run
|
|
630
|
+
// be split across processes.
|
|
631
|
+
let isPerChain = (config: t) => !(config.userEntities->Array.some(entity => entity.crossChain))
|
|
632
|
+
|
|
633
|
+
// What every line this process logs is attributed to. A process driving one
|
|
634
|
+
// of the schema's chains while siblings drive the rest names it, on the lines
|
|
635
|
+
// that had no chain in hand. One driving several has no single owner to name,
|
|
636
|
+
// and its chain-scoped lines already carry theirs.
|
|
637
|
+
let logContext = (config: t): option<dict<JSON.t>> =>
|
|
638
|
+
switch (config.isolated, config.chainMap->ChainMap.keys) {
|
|
639
|
+
| (true, [chainId]) =>
|
|
640
|
+
Some(Dict.fromArray([("chainId", chainId->S.reverseConvertToJsonOrThrow(ChainId.schema))]))
|
|
641
|
+
| _ => None
|
|
642
|
+
}
|
|
643
|
+
|
|
628
644
|
// Narrows a config to the chains one `envio start --chain` process drives.
|
|
629
645
|
// `contractMapping` is deliberately left whole: its ids are what the migration
|
|
630
646
|
// that created the schema stored, and one rebuilt from a subset would hand the
|
|
@@ -1221,6 +1237,21 @@ let prime = (json: JSON.t): unit => {
|
|
|
1221
1237
|
cached := None
|
|
1222
1238
|
}
|
|
1223
1239
|
|
|
1240
|
+
// Narrows a public config to the chains one process drives. The supervisor
|
|
1241
|
+
// plans the split; each worker applies the plan to the config it parsed itself.
|
|
1242
|
+
let withIsolatedChains = (json: JSON.t, ~chainIds) =>
|
|
1243
|
+
switch json->JSON.Decode.object {
|
|
1244
|
+
| Some(fields) => {
|
|
1245
|
+
let narrowed = fields->Dict.copy
|
|
1246
|
+
narrowed->Dict.set(
|
|
1247
|
+
"isolatedChains",
|
|
1248
|
+
chainIds->S.reverseConvertToJsonOrThrow(S.array(ChainId.schema)),
|
|
1249
|
+
)
|
|
1250
|
+
JSON.Object(narrowed)
|
|
1251
|
+
}
|
|
1252
|
+
| None => JsError.throwWithMessage("Invalid indexer config: not an object")
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1224
1255
|
let getPublicConfigJson = () =>
|
|
1225
1256
|
switch primedJson.contents {
|
|
1226
1257
|
| Some(json) => json
|
|
@@ -1266,6 +1297,10 @@ let stripSensitiveData = (json: JSON.t): JSON.t => {
|
|
|
1266
1297
|
cloned
|
|
1267
1298
|
}
|
|
1268
1299
|
|
|
1300
|
+
// What the storage layer records as the config this schema was built from,
|
|
1301
|
+
// and checks a resuming run against.
|
|
1302
|
+
let envioInfo = () => getPublicConfigJson()->stripSensitiveData
|
|
1303
|
+
|
|
1269
1304
|
// Postgres jsonb doesn't preserve key order, so canonicalize with sorted
|
|
1270
1305
|
// keys before string-comparing.
|
|
1271
1306
|
let rec canonicalJson = (json: JSON.t): JSON.t =>
|
package/src/Config.res.mjs
CHANGED
|
@@ -13,6 +13,7 @@ import * as Logging from "./Logging.res.mjs";
|
|
|
13
13
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
14
14
|
import * as Internal from "./Internal.res.mjs";
|
|
15
15
|
import * as BigDecimal from "./bindings/BigDecimal.res.mjs";
|
|
16
|
+
import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
|
|
16
17
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
17
18
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
18
19
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
@@ -470,6 +471,26 @@ function getChain(config, chainId) {
|
|
|
470
471
|
}
|
|
471
472
|
}
|
|
472
473
|
|
|
474
|
+
function isPerChain(config) {
|
|
475
|
+
return !config.userEntities.some(entity => entity.crossChain);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function logContext(config) {
|
|
479
|
+
let match = config.isolated;
|
|
480
|
+
let match$1 = ChainMap.keys(config.chainMap);
|
|
481
|
+
if (!match) {
|
|
482
|
+
return;
|
|
483
|
+
}
|
|
484
|
+
if (match$1.length !== 1) {
|
|
485
|
+
return;
|
|
486
|
+
}
|
|
487
|
+
let chainId = match$1[0];
|
|
488
|
+
return Object.fromEntries([[
|
|
489
|
+
"chainId",
|
|
490
|
+
S$RescriptSchema.reverseConvertToJsonOrThrow(chainId, ChainId.schema)
|
|
491
|
+
]]);
|
|
492
|
+
}
|
|
493
|
+
|
|
473
494
|
function isolate(config, chainIds) {
|
|
474
495
|
let shared = config.userEntities.filter(entityConfig => entityConfig.crossChain);
|
|
475
496
|
if (shared.length !== 0) {
|
|
@@ -907,6 +928,16 @@ function prime(json) {
|
|
|
907
928
|
cached.contents = undefined;
|
|
908
929
|
}
|
|
909
930
|
|
|
931
|
+
function withIsolatedChains(json, chainIds) {
|
|
932
|
+
let fields = Stdlib_JSON.Decode.object(json);
|
|
933
|
+
if (fields === undefined) {
|
|
934
|
+
return Stdlib_JsError.throwWithMessage("Invalid indexer config: not an object");
|
|
935
|
+
}
|
|
936
|
+
let narrowed = Object.assign({}, fields);
|
|
937
|
+
narrowed["isolatedChains"] = S$RescriptSchema.reverseConvertToJsonOrThrow(chainIds, S$RescriptSchema.array(ChainId.schema));
|
|
938
|
+
return narrowed;
|
|
939
|
+
}
|
|
940
|
+
|
|
910
941
|
function getPublicConfigJson() {
|
|
911
942
|
let json = primedJson.contents;
|
|
912
943
|
if (json !== undefined) {
|
|
@@ -950,6 +981,10 @@ function stripSensitiveData(json) {
|
|
|
950
981
|
return cloned;
|
|
951
982
|
}
|
|
952
983
|
|
|
984
|
+
function envioInfo() {
|
|
985
|
+
return stripSensitiveData(getPublicConfigJson());
|
|
986
|
+
}
|
|
987
|
+
|
|
953
988
|
function canonicalJson(json) {
|
|
954
989
|
if (Array.isArray(json)) {
|
|
955
990
|
return json.map(canonicalJson);
|
|
@@ -1172,14 +1207,18 @@ export {
|
|
|
1172
1207
|
publicConfigSchema,
|
|
1173
1208
|
contractMappingOf,
|
|
1174
1209
|
getChain,
|
|
1210
|
+
isPerChain,
|
|
1211
|
+
logContext,
|
|
1175
1212
|
isolate,
|
|
1176
1213
|
fromPublic,
|
|
1177
1214
|
normalizeUserAddress,
|
|
1178
1215
|
normalizeSimulateAddress,
|
|
1179
1216
|
getEventConfig,
|
|
1180
1217
|
prime,
|
|
1218
|
+
withIsolatedChains,
|
|
1181
1219
|
getPublicConfigJson,
|
|
1182
1220
|
stripSensitiveData,
|
|
1221
|
+
envioInfo,
|
|
1183
1222
|
canonicalJson,
|
|
1184
1223
|
diffPaths,
|
|
1185
1224
|
throwIfIncompatible,
|
package/src/Core.res
CHANGED
|
@@ -166,6 +166,10 @@ let loadDevAddon: ({..}, string) => addon = %raw(`function(req, envioDir) {
|
|
|
166
166
|
fs.copyFileSync(srcPath, nodePath);
|
|
167
167
|
}
|
|
168
168
|
|
|
169
|
+
// Forked workers inherit this, so only the first process in a run pays for
|
|
170
|
+
// the cargo build (and they don't contend over the cargo lock).
|
|
171
|
+
process.env.ENVIO_DEV_ADDON = nodePath;
|
|
172
|
+
|
|
169
173
|
return req(nodePath);
|
|
170
174
|
}`)
|
|
171
175
|
|
package/src/Core.res.mjs
CHANGED
|
@@ -88,6 +88,10 @@ let loadDevAddon = (function(req, envioDir) {
|
|
|
88
88
|
fs.copyFileSync(srcPath, nodePath);
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
+
// Forked workers inherit this, so only the first process in a run pays for
|
|
92
|
+
// the cargo build (and they don't contend over the cargo lock).
|
|
93
|
+
process.env.ENVIO_DEV_ADDON = nodePath;
|
|
94
|
+
|
|
91
95
|
return req(nodePath);
|
|
92
96
|
});
|
|
93
97
|
|