envio 3.13.0-alpha.0 → 3.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -8
- package/src/BatchProcessing.res +4 -5
- package/src/BatchProcessing.res.mjs +4 -11
- package/src/ChainFetching.res +18 -38
- package/src/ChainFetching.res.mjs +14 -20
- package/src/ChainState.res +113 -77
- package/src/ChainState.res.mjs +90 -73
- package/src/ChainState.resi +14 -5
- package/src/Config.res +14 -17
- package/src/Config.res.mjs +5 -20
- package/src/CrossChainState.res +68 -20
- package/src/CrossChainState.res.mjs +48 -5
- package/src/CrossChainState.resi +1 -0
- package/src/Env.res +2 -2
- package/src/ErrorHandling.res +15 -4
- package/src/ErrorHandling.res.mjs +16 -5
- package/src/FinalizeBackfill.res +21 -9
- package/src/FinalizeBackfill.res.mjs +17 -4
- package/src/HandlerRegister.res +0 -37
- package/src/HandlerRegister.res.mjs +0 -18
- package/src/IndexerState.res +2 -0
- package/src/IndexerState.res.mjs +5 -0
- package/src/IndexerState.resi +1 -0
- package/src/Logging.res +6 -38
- package/src/Logging.res.mjs +5 -32
- package/src/Main.res +6 -9
- package/src/Main.res.mjs +3 -5
- package/src/Metrics.res +9 -4
- package/src/Metrics.res.mjs +3 -3
- package/src/Persistence.res +6 -3
- package/src/Persistence.res.mjs +5 -3
- package/src/PgStorage.res +2 -2
- package/src/PgStorage.res.mjs +2 -2
- package/src/Rollback.res +3 -3
- package/src/Server.res +0 -3
- package/src/SimulateItems.res +1 -1
- package/src/SimulateItems.res.mjs +1 -1
- package/src/Supervisor.res +236 -109
- package/src/Supervisor.res.mjs +185 -55
- package/src/TestIndexer.res +6 -3
- package/src/TestIndexer.res.mjs +3 -3
- package/src/Worker.res +6 -0
- package/src/Worker.res.mjs +2 -1
- package/src/bindings/NodeJs.res +25 -16
- package/src/bindings/NodeJs.res.mjs +8 -1
- package/src/sources/AddressSet.res +5 -3
- package/src/sources/AddressStore.res +19 -11
- package/src/sources/AddressStore.res.mjs +2 -7
- package/src/sources/BlockStore.res +10 -9
- package/src/sources/BlockStore.res.mjs +4 -4
- package/src/sources/ChainSources.res +4 -0
- package/src/sources/ChainSources.res.mjs +2 -2
- package/src/sources/EvmChain.res +4 -0
- package/src/sources/EvmChain.res.mjs +3 -1
- package/src/sources/EvmEventItem.res +56 -0
- package/src/sources/EvmEventItem.res.mjs +32 -0
- package/src/sources/EvmHyperSyncSource.res +7 -37
- package/src/sources/EvmHyperSyncSource.res.mjs +8 -29
- package/src/sources/EvmRpcClient.res +41 -59
- package/src/sources/EvmRpcClient.res.mjs +4 -54
- package/src/sources/EvmRpcWs.res +4 -1
- package/src/sources/EvmRpcWs.res.mjs +6 -2
- package/src/sources/FuelHyperSyncSource.res +6 -4
- package/src/sources/FuelHyperSyncSource.res.mjs +7 -9
- package/src/sources/HeightFeed.res +7 -1
- package/src/sources/HeightFeed.res.mjs +9 -7
- package/src/sources/HyperSyncClient.res +1 -14
- package/src/sources/RequestStat.res +7 -0
- package/src/sources/RequestStat.res.mjs +14 -1
- package/src/sources/RpcSource.res +100 -1043
- package/src/sources/RpcSource.res.mjs +81 -1056
- package/src/sources/SimulateSource.res +2 -5
- package/src/sources/SimulateSource.res.mjs +2 -3
- package/src/sources/Source.res +34 -12
- package/src/sources/Source.res.mjs +11 -0
- package/src/sources/SourceManager.res +111 -31
- package/src/sources/SourceManager.res.mjs +65 -31
- package/src/sources/SourceManager.resi +4 -0
- package/src/sources/StartBlockResolver.res +8 -5
- package/src/sources/StartBlockResolver.res.mjs +4 -2
- package/src/sources/SvmHyperSyncSource.res +4 -2
- package/src/sources/SvmHyperSyncSource.res.mjs +4 -6
- package/src/sources/TransactionStore.res +9 -6
- package/src/sources/TransactionStore.res.mjs +2 -2
- package/src/tui/Tui.res +4 -18
- package/src/tui/Tui.res.mjs +3 -6
- package/src/LazyLoader.res +0 -135
- package/src/LazyLoader.res.mjs +0 -118
- package/src/bindings/SDSL.res +0 -12
- package/src/bindings/SDSL.res.mjs +0 -9
- package/src/bindings/Yargs.res +0 -8
- package/src/bindings/Yargs.res.mjs +0 -2
- package/src/sources/Rpc.res +0 -185
- package/src/sources/Rpc.res.mjs +0 -183
|
@@ -125,9 +125,13 @@ function isReadyToEnterReorgThreshold(crossChainState, batch) {
|
|
|
125
125
|
}
|
|
126
126
|
|
|
127
127
|
function enterReorgThreshold(crossChainState) {
|
|
128
|
-
Logging.info("Reorg threshold reached");
|
|
129
128
|
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
130
|
-
|
|
129
|
+
let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
|
|
130
|
+
let liftsCeiling = ChainState.reorgThresholdLiftsCeiling(cs);
|
|
131
|
+
ChainState.enterReorgThreshold(cs);
|
|
132
|
+
if (liftsCeiling) {
|
|
133
|
+
Logging.childInfo(ChainState.logger(cs), ChainState.reorgThresholdEntryMessage);
|
|
134
|
+
}
|
|
131
135
|
}
|
|
132
136
|
}
|
|
133
137
|
|
|
@@ -137,7 +141,7 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
|
|
|
137
141
|
for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
|
|
138
142
|
let cs = crossChainState.chainStates[chainIds[i]];
|
|
139
143
|
ChainState.applyBatchProgress(cs, batch, blockTimestampName);
|
|
140
|
-
if (!
|
|
144
|
+
if (!ChainState.hasCaughtUp(cs)) {
|
|
141
145
|
everyChainCaughtUp = false;
|
|
142
146
|
}
|
|
143
147
|
}
|
|
@@ -165,7 +169,7 @@ function markCaughtUpOnResume(crossChainState) {
|
|
|
165
169
|
let everyChainCaughtUp = crossChainState.chainIds.length !== 0;
|
|
166
170
|
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
167
171
|
let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
|
|
168
|
-
if (!ChainState.
|
|
172
|
+
if (!ChainState.hasCaughtUp(cs)) {
|
|
169
173
|
everyChainCaughtUp = false;
|
|
170
174
|
}
|
|
171
175
|
}
|
|
@@ -177,11 +181,49 @@ function markCaughtUpOnResume(crossChainState) {
|
|
|
177
181
|
|
|
178
182
|
function markReady(crossChainState, readyAt) {
|
|
179
183
|
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
180
|
-
|
|
184
|
+
let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
|
|
185
|
+
let wasReady = ChainState.isReady(cs);
|
|
186
|
+
ChainState.markReady(cs, readyAt);
|
|
187
|
+
if (!wasReady) {
|
|
188
|
+
Logging.childInfo(ChainState.logger(cs), "Ready. Fully indexed for queries.");
|
|
189
|
+
}
|
|
181
190
|
}
|
|
182
191
|
crossChainState.isRealtime = true;
|
|
183
192
|
}
|
|
184
193
|
|
|
194
|
+
function isWaitingOnOthers(crossChainState, cs) {
|
|
195
|
+
if (crossChainState.holdRealtime) {
|
|
196
|
+
return true;
|
|
197
|
+
} else {
|
|
198
|
+
return Object.values(crossChainState.chainStates).some(other => {
|
|
199
|
+
if (other !== cs) {
|
|
200
|
+
return !ChainState.hasCaughtUp(other);
|
|
201
|
+
} else {
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function reportFinished(crossChainState) {
|
|
209
|
+
Object.values(crossChainState.chainStates).forEach(cs => {
|
|
210
|
+
let match = ChainState.takeFinished(cs);
|
|
211
|
+
if (match !== undefined) {
|
|
212
|
+
if (match.TAG === "EndBlock") {
|
|
213
|
+
return Logging.childInfo(ChainState.logger(cs), {
|
|
214
|
+
msg: "Indexed to the end block. This chain is done.",
|
|
215
|
+
block: match._0
|
|
216
|
+
});
|
|
217
|
+
} else {
|
|
218
|
+
return Logging.childInfo(ChainState.logger(cs), {
|
|
219
|
+
msg: isWaitingOnOthers(crossChainState, cs) ? "Finished backfill. Waiting for the other chains." : "Finished backfill.",
|
|
220
|
+
block: match._0
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
185
227
|
function priorityOrder(crossChainState) {
|
|
186
228
|
return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.frontierProgress(a), ChainState.frontierProgress(b)));
|
|
187
229
|
}
|
|
@@ -296,6 +338,7 @@ export {
|
|
|
296
338
|
markCaughtUpIfSettled,
|
|
297
339
|
markCaughtUpOnResume,
|
|
298
340
|
markReady,
|
|
341
|
+
reportFinished,
|
|
299
342
|
priorityOrder,
|
|
300
343
|
checkAndFetch,
|
|
301
344
|
}
|
package/src/CrossChainState.resi
CHANGED
|
@@ -40,6 +40,7 @@ let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => uni
|
|
|
40
40
|
let markCaughtUpIfSettled: t => unit
|
|
41
41
|
let markCaughtUpOnResume: t => unit
|
|
42
42
|
let markReady: (t, ~readyAt: Date.t) => unit
|
|
43
|
+
let reportFinished: t => unit
|
|
43
44
|
|
|
44
45
|
// Fetch control.
|
|
45
46
|
let priorityOrder: t => array<ChainState.t>
|
package/src/Env.res
CHANGED
|
@@ -129,8 +129,8 @@ module Db = {
|
|
|
129
129
|
)
|
|
130
130
|
// The budget for the whole run, not for one process: a run that splits across
|
|
131
131
|
// workers divides it among them, and each caps its own pool to its share.
|
|
132
|
-
//
|
|
133
|
-
//
|
|
132
|
+
// Splitting takes two workers and a worker takes two connections, so a budget
|
|
133
|
+
// under 4 — the default among them — keeps the run in one process.
|
|
134
134
|
let maxConnections = envSafe->EnvSafe.get("ENVIO_PG_MAX_CONNECTIONS", S.int, ~fallback=2)
|
|
135
135
|
}
|
|
136
136
|
|
package/src/ErrorHandling.res
CHANGED
|
@@ -11,14 +11,25 @@ let log = (self: t) => {
|
|
|
11
11
|
}
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
// The exception to hand to whoever is outside the indexer. A failure raised
|
|
15
|
+
// from a message alone carries no exception of its own, so rethrowing or
|
|
16
|
+
// rejecting with `exn` would hand over a bare `null` and the reason would live
|
|
17
|
+
// only in the logs. Fall back to the message, which is the whole of what such a
|
|
18
|
+
// failure knows.
|
|
19
|
+
let toExn = (self: t) =>
|
|
20
|
+
switch self.exn->(Utils.magic: exn => Nullable.t<exn>)->Nullable.toOption {
|
|
21
|
+
| Some(exn) => exn->Utils.prettifyExn
|
|
22
|
+
| None => Utils.Error.make(self.msg->Option.getOr("Indexer has failed with an unexpected error"))
|
|
23
|
+
}
|
|
24
|
+
|
|
14
25
|
let raiseExn = (self: t) => {
|
|
15
|
-
self
|
|
26
|
+
self->toExn->throw
|
|
16
27
|
}
|
|
17
28
|
|
|
18
29
|
let mkLogAndRaise = (~logger=?, ~msg=?, exn) => {
|
|
19
|
-
let
|
|
20
|
-
|
|
21
|
-
|
|
30
|
+
let self = exn->Utils.prettifyExn->make(~logger?, ~msg?)
|
|
31
|
+
self->log
|
|
32
|
+
self->raiseExn
|
|
22
33
|
}
|
|
23
34
|
|
|
24
35
|
let unwrapLogAndRaise = (~logger=?, ~msg=?, result) => {
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Utils from "./Utils.res.mjs";
|
|
4
4
|
import * as Logging from "./Logging.res.mjs";
|
|
5
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
5
6
|
|
|
6
7
|
function make(exn, loggerOpt, msg) {
|
|
7
8
|
let logger = loggerOpt !== undefined ? loggerOpt : Logging.getLogger();
|
|
@@ -23,14 +24,23 @@ function log(self) {
|
|
|
23
24
|
}
|
|
24
25
|
}
|
|
25
26
|
|
|
27
|
+
function toExn(self) {
|
|
28
|
+
let exn = self.exn;
|
|
29
|
+
if (exn == null) {
|
|
30
|
+
return new Error(Stdlib_Option.getOr(self.msg, "Indexer has failed with an unexpected error"));
|
|
31
|
+
} else {
|
|
32
|
+
return Utils.prettifyExn(exn);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
26
36
|
function raiseExn(self) {
|
|
27
|
-
throw
|
|
37
|
+
throw toExn(self);
|
|
28
38
|
}
|
|
29
39
|
|
|
30
40
|
function mkLogAndRaise(logger, msg, exn) {
|
|
31
|
-
let
|
|
32
|
-
log(
|
|
33
|
-
throw
|
|
41
|
+
let self = make(Utils.prettifyExn(exn), logger, msg);
|
|
42
|
+
log(self);
|
|
43
|
+
throw toExn(self);
|
|
34
44
|
}
|
|
35
45
|
|
|
36
46
|
function unwrapLogAndRaise(logger, msg, result) {
|
|
@@ -43,12 +53,13 @@ function unwrapLogAndRaise(logger, msg, result) {
|
|
|
43
53
|
|
|
44
54
|
function logAndRaise(self) {
|
|
45
55
|
log(self);
|
|
46
|
-
throw
|
|
56
|
+
throw toExn(self);
|
|
47
57
|
}
|
|
48
58
|
|
|
49
59
|
export {
|
|
50
60
|
make,
|
|
51
61
|
log,
|
|
62
|
+
toExn,
|
|
52
63
|
raiseExn,
|
|
53
64
|
mkLogAndRaise,
|
|
54
65
|
unwrapLogAndRaise,
|
package/src/FinalizeBackfill.res
CHANGED
|
@@ -11,9 +11,25 @@
|
|
|
11
11
|
// `envio start --chain` process is indexing and never waits on one.
|
|
12
12
|
|
|
13
13
|
let runOnce = async (state: IndexerState.t) => {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
let chainIds = state->IndexerState.crossChainState->CrossChainState.chainIds
|
|
15
|
+
|
|
16
|
+
// Whatever brought the process here — a batch, a tick that progressed
|
|
17
|
+
// nothing, a supervisor's release — its chains say where they finished
|
|
18
|
+
// before it says what it does about that. Said here rather than left to each
|
|
19
|
+
// caller to order: a chain that has already spoken says nothing again.
|
|
20
|
+
state->IndexerState.reportFinished
|
|
21
|
+
|
|
22
|
+
// Said by the process rather than by each of its chains: the indexes are one
|
|
23
|
+
// build over the tables, and the pause is the whole process's. A chain has
|
|
24
|
+
// already said it caught up, and says it is ready once this commits. The
|
|
25
|
+
// chains are named because the pause is theirs, and a split run has a process
|
|
26
|
+
// saying this for each part of it — one of them, for a process driving a
|
|
27
|
+
// single chain.
|
|
28
|
+
let msg = "Building database indexes. Indexing is paused until they are ready, which can take a while on a large database."
|
|
29
|
+
switch chainIds {
|
|
30
|
+
| [chainId] => Logging.info({"msg": msg, "chainId": chainId})
|
|
31
|
+
| chainIds => Logging.info({"msg": msg, "chainIds": chainIds})
|
|
32
|
+
}
|
|
17
33
|
|
|
18
34
|
await Writing.flush(state)
|
|
19
35
|
|
|
@@ -24,16 +40,12 @@ let runOnce = async (state: IndexerState.t) => {
|
|
|
24
40
|
let storage = persistence->Persistence.getInitializedStorageOrThrow
|
|
25
41
|
let readyAt = Date.make()
|
|
26
42
|
|
|
27
|
-
await storage.finalizeBackfill(
|
|
28
|
-
~entities=persistence.allEntities,
|
|
29
|
-
~chainIds=state->IndexerState.crossChainState->CrossChainState.chainIds,
|
|
30
|
-
~readyAt,
|
|
31
|
-
)
|
|
43
|
+
await storage.finalizeBackfill(~entities=persistence.allEntities, ~chainIds, ~readyAt)
|
|
32
44
|
|
|
33
45
|
// Only after the commit: in-memory readiness must never run ahead of the
|
|
34
46
|
// `ready_at` a restart would read back.
|
|
47
|
+
// Says so per chain, which is the grain `ready_at` is committed at.
|
|
35
48
|
state->IndexerState.markReady(~readyAt)
|
|
36
|
-
Logging.info("The indexer is ready. Switching to realtime indexing.")
|
|
37
49
|
}
|
|
38
50
|
}
|
|
39
51
|
|
|
@@ -7,7 +7,21 @@ import * as IndexerState from "./IndexerState.res.mjs";
|
|
|
7
7
|
import * as CrossChainState from "./CrossChainState.res.mjs";
|
|
8
8
|
|
|
9
9
|
async function runOnce(state) {
|
|
10
|
-
|
|
10
|
+
let chainIds = CrossChainState.chainIds(IndexerState.crossChainState(state));
|
|
11
|
+
IndexerState.reportFinished(state);
|
|
12
|
+
let msg = "Building database indexes. Indexing is paused until they are ready, which can take a while on a large database.";
|
|
13
|
+
if (chainIds.length !== 1) {
|
|
14
|
+
Logging.info({
|
|
15
|
+
msg: msg,
|
|
16
|
+
chainIds: chainIds
|
|
17
|
+
});
|
|
18
|
+
} else {
|
|
19
|
+
let chainId = chainIds[0];
|
|
20
|
+
Logging.info({
|
|
21
|
+
msg: msg,
|
|
22
|
+
chainId: chainId
|
|
23
|
+
});
|
|
24
|
+
}
|
|
11
25
|
await Writing.flush(state);
|
|
12
26
|
if (IndexerState.hasFailedWrite(state)) {
|
|
13
27
|
return;
|
|
@@ -15,9 +29,8 @@ async function runOnce(state) {
|
|
|
15
29
|
let persistence = IndexerState.persistence(state);
|
|
16
30
|
let storage = Persistence.getInitializedStorageOrThrow(persistence);
|
|
17
31
|
let readyAt = new Date();
|
|
18
|
-
await storage.finalizeBackfill(persistence.allEntities,
|
|
19
|
-
IndexerState.markReady(state, readyAt);
|
|
20
|
-
return Logging.info("The indexer is ready. Switching to realtime indexing.");
|
|
32
|
+
await storage.finalizeBackfill(persistence.allEntities, chainIds, readyAt);
|
|
33
|
+
return IndexerState.markReady(state, readyAt);
|
|
21
34
|
}
|
|
22
35
|
|
|
23
36
|
function run(state) {
|
package/src/HandlerRegister.res
CHANGED
|
@@ -493,41 +493,6 @@ let getSimulateOnEventRegistrations = (
|
|
|
493
493
|
}
|
|
494
494
|
}
|
|
495
495
|
|
|
496
|
-
// An RPC source can only deliver the fields it knows how to parse; the rest are
|
|
497
|
-
// silently skipped at materialisation. Every RPC on the chain counts, whatever
|
|
498
|
-
// it's for — a fallback or realtime source runs the same parsers as a sync one,
|
|
499
|
-
// so a field it can't deliver would go missing for whichever blocks it served.
|
|
500
|
-
// The selection can come from either `config.yaml` or the inline `fields`
|
|
501
|
-
// option, so the message names neither. Runs on the registrations a chain
|
|
502
|
-
// actually keeps, so a handler whose `where` opts out of this chain isn't held
|
|
503
|
-
// to its limits.
|
|
504
|
-
//
|
|
505
|
-
// The `config.yaml` half of this is also rejected at codegen, by the
|
|
506
|
-
// `RpcTransactionField` subenum in `system_config.rs`. That one reports every
|
|
507
|
-
// offending field at once and before the project builds; this one is the only
|
|
508
|
-
// check an inline selection reaches. `RpcFieldSelection_test.res` pins the two
|
|
509
|
-
// to the same field set.
|
|
510
|
-
let validateRpcFieldSelection = (
|
|
511
|
-
chainConfig: Config.chain,
|
|
512
|
-
registrations: array<Internal.onEventRegistration>,
|
|
513
|
-
) => {
|
|
514
|
-
let hasRpc = switch chainConfig.sourceConfig {
|
|
515
|
-
| EvmSourceConfig({rpcs}) => !(rpcs->Utils.Array.isEmpty)
|
|
516
|
-
| _ => false
|
|
517
|
-
}
|
|
518
|
-
if hasRpc {
|
|
519
|
-
registrations->Array.forEach(reg =>
|
|
520
|
-
reg.fieldSelection.transactionFields->Utils.Set.forEach(name =>
|
|
521
|
-
if !RpcSource.isRpcTransactionField(name) {
|
|
522
|
-
JsError.throwWithMessage(
|
|
523
|
-
`The "${name}" transaction field selected for the "${reg.eventConfig.name}" event on contract "${reg.eventConfig.contractName}" is unavailable for indexing via RPC. Remove it from the field selection, or remove chain ${chainConfig.id->ChainId.toString}'s RPC source — even an RPC the chain only falls back to has to deliver the selection.`,
|
|
524
|
-
)
|
|
525
|
-
}
|
|
526
|
-
)
|
|
527
|
-
)
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
|
|
531
496
|
let finishRegistration = (~config: Config.t): registrationsByChainId => {
|
|
532
497
|
switch getActiveRegistration() {
|
|
533
498
|
| Some(r) => {
|
|
@@ -614,8 +579,6 @@ let finishRegistration = (~config: Config.t): registrationsByChainId => {
|
|
|
614
579
|
}
|
|
615
580
|
})
|
|
616
581
|
|
|
617
|
-
validateRpcFieldSelection(chainConfig, onEventRegistrations)
|
|
618
|
-
|
|
619
582
|
registrationsByChainId->Dict.set(
|
|
620
583
|
key,
|
|
621
584
|
{
|
|
@@ -5,7 +5,6 @@ import * as ChainId from "./ChainId.res.mjs";
|
|
|
5
5
|
import * as Logging from "./Logging.res.mjs";
|
|
6
6
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
7
7
|
import * as Internal from "./Internal.res.mjs";
|
|
8
|
-
import * as RpcSource from "./sources/RpcSource.res.mjs";
|
|
9
8
|
import * as EnvioGlobal from "./EnvioGlobal.res.mjs";
|
|
10
9
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
11
10
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
@@ -327,22 +326,6 @@ function getSimulateOnEventRegistrations(config, chainId, eventConfig) {
|
|
|
327
326
|
}
|
|
328
327
|
}
|
|
329
328
|
|
|
330
|
-
function validateRpcFieldSelection(chainConfig, registrations) {
|
|
331
|
-
let match = chainConfig.sourceConfig;
|
|
332
|
-
let hasRpc;
|
|
333
|
-
hasRpc = match.TAG === "EvmSourceConfig" ? !Utils.$$Array.isEmpty(match.rpcs) : false;
|
|
334
|
-
if (hasRpc) {
|
|
335
|
-
registrations.forEach(reg => {
|
|
336
|
-
reg.fieldSelection.transactionFields.forEach(name => {
|
|
337
|
-
if (!RpcSource.isRpcTransactionField(name)) {
|
|
338
|
-
return Stdlib_JsError.throwWithMessage(`The "` + name + `" transaction field selected for the "` + reg.eventConfig.name + `" event on contract "` + reg.eventConfig.contractName + `" is unavailable for indexing via RPC. Remove it from the field selection, or remove chain ` + ChainId.toString(chainConfig.id) + `'s RPC source — even an RPC the chain only falls back to has to deliver the selection.`);
|
|
339
|
-
}
|
|
340
|
-
});
|
|
341
|
-
});
|
|
342
|
-
return;
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
|
|
346
329
|
function finishRegistration(config) {
|
|
347
330
|
let r = EnvioGlobal.value.activeRegistration;
|
|
348
331
|
if (r === undefined) {
|
|
@@ -389,7 +372,6 @@ function finishRegistration(config) {
|
|
|
389
372
|
let newrecord = {...reg};
|
|
390
373
|
onEventRegistrations.push((newrecord.index = onEventRegistrations.length, newrecord));
|
|
391
374
|
});
|
|
392
|
-
validateRpcFieldSelection(chainConfig, onEventRegistrations);
|
|
393
375
|
let chainRegs = r.registrationsByChainId[key];
|
|
394
376
|
registrationsByChainId[key] = {
|
|
395
377
|
onEventRegistrations: onEventRegistrations,
|
package/src/IndexerState.res
CHANGED
|
@@ -544,6 +544,8 @@ let releaseRealtime = (state: t) => {
|
|
|
544
544
|
|
|
545
545
|
let markReady = (state: t, ~readyAt) => state.crossChainState->CrossChainState.markReady(~readyAt)
|
|
546
546
|
|
|
547
|
+
let reportFinished = (state: t) => state.crossChainState->CrossChainState.reportFinished
|
|
548
|
+
|
|
547
549
|
let rollbackState = (state: t) => state.rollbackState
|
|
548
550
|
let indexerStartTime = (state: t) => state.indexerStartTime
|
|
549
551
|
let loadManager = (state: t) => state.loadManager
|
package/src/IndexerState.res.mjs
CHANGED
|
@@ -390,6 +390,10 @@ function markReady(state, readyAt) {
|
|
|
390
390
|
CrossChainState.markReady(state.crossChainState, readyAt);
|
|
391
391
|
}
|
|
392
392
|
|
|
393
|
+
function reportFinished(state) {
|
|
394
|
+
CrossChainState.reportFinished(state.crossChainState);
|
|
395
|
+
}
|
|
396
|
+
|
|
393
397
|
function rollbackState(state) {
|
|
394
398
|
return state.rollbackState;
|
|
395
399
|
}
|
|
@@ -884,6 +888,7 @@ export {
|
|
|
884
888
|
hasArrivedAtHead,
|
|
885
889
|
releaseRealtime,
|
|
886
890
|
markReady,
|
|
891
|
+
reportFinished,
|
|
887
892
|
rollbackState,
|
|
888
893
|
indexerStartTime,
|
|
889
894
|
loadManager,
|
package/src/IndexerState.resi
CHANGED
|
@@ -104,6 +104,7 @@ let hasArrivedAtHead: t => bool
|
|
|
104
104
|
// The supervisor's go-ahead for a process driving part of a split run.
|
|
105
105
|
let releaseRealtime: t => unit
|
|
106
106
|
let markReady: (t, ~readyAt: Date.t) => unit
|
|
107
|
+
let reportFinished: t => unit
|
|
107
108
|
let rollbackState: t => rollbackState
|
|
108
109
|
let indexerStartTime: t => Date.t
|
|
109
110
|
let loadManager: t => LoadManager.t
|
package/src/Logging.res
CHANGED
|
@@ -26,30 +26,6 @@ let logLevels = [
|
|
|
26
26
|
|
|
27
27
|
%%private(let logger = ref(None))
|
|
28
28
|
|
|
29
|
-
// Fields every line this process logs carries. Merged into each line rather
|
|
30
|
-
// than bound to a child logger: pino writes a child's bindings and the line's
|
|
31
|
-
// own fields side by side, so a line that names the same key would carry it
|
|
32
|
-
// twice. A fresh object per line, since pino merges the line's fields into
|
|
33
|
-
// whatever this returns.
|
|
34
|
-
%%private(let context: ref<dict<JSON.t>> = ref(Dict.make()))
|
|
35
|
-
%%private(let mixin = () => JSON.Object(context.contents->Dict.copy))
|
|
36
|
-
|
|
37
|
-
// A child logger that binds a field the process already carries would have pino
|
|
38
|
-
// write it twice: a child's bindings and the context are concatenated into the
|
|
39
|
-
// line, not merged. The context is the one place a line names it, which it can
|
|
40
|
-
// be because a process only takes one when its every line is about that chain.
|
|
41
|
-
%%private(
|
|
42
|
-
let withoutContext = (params: 'a) =>
|
|
43
|
-
switch context.contents->Dict.keysToArray {
|
|
44
|
-
| [] => params
|
|
45
|
-
| keys => {
|
|
46
|
-
let narrowed = params->(Utils.magic: 'a => dict<JSON.t>)->Dict.copy
|
|
47
|
-
keys->Array.forEach(key => narrowed->Dict.delete(key))
|
|
48
|
-
narrowed->(Utils.magic: dict<JSON.t> => 'a)
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
)
|
|
52
|
-
|
|
53
29
|
let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLevel) => {
|
|
54
30
|
// Currently unused - useful if using multiple transports.
|
|
55
31
|
// let pinoRaw = {"target": "pino/file", "level": Config.userLogLevel}
|
|
@@ -80,19 +56,17 @@ let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLeve
|
|
|
80
56
|
...Pino.ECS.make(),
|
|
81
57
|
customLevels: logLevels,
|
|
82
58
|
base,
|
|
83
|
-
mixin,
|
|
84
59
|
},
|
|
85
60
|
Transport.make(pinoFile),
|
|
86
61
|
)
|
|
87
62
|
| EcsConsoleMultistream =>
|
|
88
|
-
makeMultiStreamLogger(~logFile=None, ~options=Some({...Pino.ECS.make(), base
|
|
63
|
+
makeMultiStreamLogger(~logFile=None, ~options=Some({...Pino.ECS.make(), base}))
|
|
89
64
|
| EcsConsole =>
|
|
90
65
|
make({
|
|
91
66
|
...Pino.ECS.make(),
|
|
92
67
|
level: userLogLevel,
|
|
93
68
|
customLevels: logLevels,
|
|
94
69
|
base,
|
|
95
|
-
mixin,
|
|
96
70
|
})
|
|
97
71
|
| FileOnly =>
|
|
98
72
|
makeWithOptionsAndTransport(
|
|
@@ -100,13 +74,12 @@ let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLeve
|
|
|
100
74
|
customLevels: logLevels,
|
|
101
75
|
level: defaultFileLogLevel,
|
|
102
76
|
base,
|
|
103
|
-
mixin,
|
|
104
77
|
},
|
|
105
78
|
Transport.make(pinoFile),
|
|
106
79
|
)
|
|
107
|
-
| ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=Some({base
|
|
108
|
-
| ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=Some({base
|
|
109
|
-
| Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=Some({base
|
|
80
|
+
| ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=Some({base: base}))
|
|
81
|
+
| ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=Some({base: base}))
|
|
82
|
+
| Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=Some({base: base}))
|
|
110
83
|
}
|
|
111
84
|
}
|
|
112
85
|
|
|
@@ -176,15 +149,10 @@ let childFatal = (logger, params: 'a) => {
|
|
|
176
149
|
}
|
|
177
150
|
|
|
178
151
|
let createChild = (~params: 'a) => {
|
|
179
|
-
getLogger()->child(params->
|
|
152
|
+
getLogger()->child(params->createChildParams)
|
|
180
153
|
}
|
|
181
|
-
|
|
182
|
-
// What belongs on every line is the run's to decide; the logger only carries
|
|
183
|
-
// what it is handed. A line that names one of these fields itself wins.
|
|
184
|
-
let setContext = (fields: dict<JSON.t>) => context := fields
|
|
185
|
-
|
|
186
154
|
let createChildFrom = (~logger: t, ~params: 'a) => {
|
|
187
|
-
logger->child(params->
|
|
155
|
+
logger->child(params->createChildParams)
|
|
188
156
|
}
|
|
189
157
|
|
|
190
158
|
@inline
|
package/src/Logging.res.mjs
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
import * as Pino from "./bindings/Pino.res.mjs";
|
|
4
4
|
import * as Pino$1 from "pino";
|
|
5
5
|
import * as Utils from "./Utils.res.mjs";
|
|
6
|
-
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
7
6
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
8
7
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
9
8
|
|
|
@@ -54,24 +53,6 @@ let logger = {
|
|
|
54
53
|
contents: undefined
|
|
55
54
|
};
|
|
56
55
|
|
|
57
|
-
let context = {
|
|
58
|
-
contents: {}
|
|
59
|
-
};
|
|
60
|
-
|
|
61
|
-
function mixin() {
|
|
62
|
-
return Object.assign({}, context.contents);
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function withoutContext(params) {
|
|
66
|
-
let keys = Object.keys(context.contents);
|
|
67
|
-
if (keys.length === 0) {
|
|
68
|
-
return params;
|
|
69
|
-
}
|
|
70
|
-
let narrowed = Object.assign({}, params);
|
|
71
|
-
keys.forEach(key => Stdlib_Dict.$$delete(narrowed, key));
|
|
72
|
-
return narrowed;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
56
|
function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel) {
|
|
76
57
|
let pinoFile_target = "pino/file";
|
|
77
58
|
let pinoFile_options = Primitive_option.some(Pino.Transport.makeTransportOptions({
|
|
@@ -90,18 +71,17 @@ function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel)
|
|
|
90
71
|
switch (logStrategy) {
|
|
91
72
|
case "ecs-file" :
|
|
92
73
|
let newrecord = {...Pino.ECS.make()};
|
|
93
|
-
return Pino$1.pino((newrecord.base = base, newrecord.
|
|
74
|
+
return Pino$1.pino((newrecord.base = base, newrecord.customLevels = logLevels, newrecord), Pino$1.transport(pinoFile));
|
|
94
75
|
case "ecs-console" :
|
|
95
76
|
let newrecord$1 = {...Pino.ECS.make()};
|
|
96
|
-
return Pino$1.pino((newrecord$1.base = base, newrecord$1.
|
|
77
|
+
return Pino$1.pino((newrecord$1.base = base, newrecord$1.customLevels = logLevels, newrecord$1.level = userLogLevel, newrecord$1));
|
|
97
78
|
case "ecs-console-multistream" :
|
|
98
79
|
let newrecord$2 = {...Pino.ECS.make()};
|
|
99
|
-
return makeMultiStreamLogger(undefined, (newrecord$2.base = base, newrecord$2
|
|
80
|
+
return makeMultiStreamLogger(undefined, (newrecord$2.base = base, newrecord$2));
|
|
100
81
|
case "file-only" :
|
|
101
82
|
return Pino$1.pino({
|
|
102
83
|
level: defaultFileLogLevel,
|
|
103
84
|
customLevels: logLevels,
|
|
104
|
-
mixin: mixin,
|
|
105
85
|
base: base
|
|
106
86
|
}, Pino$1.transport(pinoFile));
|
|
107
87
|
case "console-raw" :
|
|
@@ -109,12 +89,10 @@ function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel)
|
|
|
109
89
|
break;
|
|
110
90
|
case "both-prettyconsole" :
|
|
111
91
|
return makeMultiStreamLogger(logFilePath, {
|
|
112
|
-
mixin: mixin,
|
|
113
92
|
base: base
|
|
114
93
|
});
|
|
115
94
|
}
|
|
116
95
|
return makeMultiStreamLogger(undefined, {
|
|
117
|
-
mixin: mixin,
|
|
118
96
|
base: base
|
|
119
97
|
});
|
|
120
98
|
}
|
|
@@ -193,15 +171,11 @@ function childFatal(logger, params) {
|
|
|
193
171
|
}
|
|
194
172
|
|
|
195
173
|
function createChild(params) {
|
|
196
|
-
return getLogger().child(Pino.createChildParams(
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
function setContext(fields) {
|
|
200
|
-
context.contents = fields;
|
|
174
|
+
return getLogger().child(Pino.createChildParams(params));
|
|
201
175
|
}
|
|
202
176
|
|
|
203
177
|
function createChildFrom(logger, params) {
|
|
204
|
-
return logger.child(Pino.createChildParams(
|
|
178
|
+
return logger.child(Pino.createChildParams(params));
|
|
205
179
|
}
|
|
206
180
|
|
|
207
181
|
function logAtLevel(logger, level, message, params) {
|
|
@@ -272,7 +246,6 @@ export {
|
|
|
272
246
|
childErrorWithExn,
|
|
273
247
|
childFatal,
|
|
274
248
|
createChild,
|
|
275
|
-
setContext,
|
|
276
249
|
createChildFrom,
|
|
277
250
|
logAtLevel,
|
|
278
251
|
noopLogger,
|
package/src/Main.res
CHANGED
|
@@ -453,9 +453,6 @@ exception FatalError(exn)
|
|
|
453
453
|
) => {
|
|
454
454
|
// A worker reports to its supervisor, which draws for the whole run.
|
|
455
455
|
let shouldUseTui = Tui.shouldUse(~suppressed=isTest || Worker.isEnabled)
|
|
456
|
-
// In per-chain mode every line this process writes belongs to the chains it
|
|
457
|
-
// drives, whether or not a supervisor split the run across processes.
|
|
458
|
-
config->Config.logContext->Option.forEach(Logging.setContext)
|
|
459
456
|
// isDevelopmentMode controls whether the indexer stays alive after all
|
|
460
457
|
// chains finish (keepProcessAlive) and whether the console API is exposed.
|
|
461
458
|
// Set by `envio dev` via the public config's `isDev` field; `envio start`
|
|
@@ -561,12 +558,12 @@ let start = async (
|
|
|
561
558
|
~exitAfterFirstEventBlock=false,
|
|
562
559
|
~patchConfig: option<(Config.t, HandlerRegister.registrationsByChainId) => Config.t>=?,
|
|
563
560
|
) => {
|
|
564
|
-
// A worker parses the
|
|
565
|
-
//
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
Worker.config->Option.forEach(({chainIds}) =>
|
|
569
|
-
Config.prime(Config.getPublicConfigJson()->Config.
|
|
561
|
+
// A worker parses the project's files itself rather than being handed the
|
|
562
|
+
// config: a public config carries every contract's ABI, which is far more
|
|
563
|
+
// than a spawn environment should. What the command decided rides along
|
|
564
|
+
// instead, and re-applying it is what makes the two configs the same one.
|
|
565
|
+
Worker.config->Option.forEach(({chainIds, isDev}) =>
|
|
566
|
+
Config.prime(Config.getPublicConfigJson()->Config.withCommandFields(~chainIds, ~isDev))
|
|
570
567
|
)
|
|
571
568
|
let config = Config.load()
|
|
572
569
|
switch isTest ? None : Supervisor.planForRun(~config) {
|
package/src/Main.res.mjs
CHANGED
|
@@ -6,7 +6,6 @@ import * as Config from "./Config.res.mjs";
|
|
|
6
6
|
import * as Server from "./Server.res.mjs";
|
|
7
7
|
import * as Worker from "./Worker.res.mjs";
|
|
8
8
|
import * as ChainId from "./ChainId.res.mjs";
|
|
9
|
-
import * as Logging from "./Logging.res.mjs";
|
|
10
9
|
import * as Metrics from "./Metrics.res.mjs";
|
|
11
10
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
12
11
|
import * as PgStorage from "./PgStorage.res.mjs";
|
|
@@ -352,7 +351,7 @@ function getGlobalIndexer() {
|
|
|
352
351
|
async function migrate(reset) {
|
|
353
352
|
let config = Config.load();
|
|
354
353
|
let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
|
|
355
|
-
await Persistence.init(persistence, ChainMap.values(config.chainMap), config.contractMapping, Config.envioInfo(), "envio local db-migrate setup", undefined, reset, config.lowercaseAddresses, undefined, "Once");
|
|
354
|
+
await Persistence.init(persistence, ChainMap.values(config.chainMap), config.contractMapping, Config.envioInfo(), "envio local db-migrate setup", undefined, reset, config.lowercaseAddresses, undefined, undefined, "Once");
|
|
356
355
|
return await persistence.storage.close();
|
|
357
356
|
}
|
|
358
357
|
|
|
@@ -369,8 +368,7 @@ async function startIndexer(config, persistence, resetOpt, isTestOpt, exitAfterF
|
|
|
369
368
|
let reset = resetOpt !== undefined ? resetOpt : false;
|
|
370
369
|
let isTest = isTestOpt !== undefined ? isTestOpt : false;
|
|
371
370
|
let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
|
|
372
|
-
let shouldUseTui = Tui.shouldUse(isTest || Worker.isEnabled);
|
|
373
|
-
Stdlib_Option.forEach(Config.logContext(config), Logging.setContext);
|
|
371
|
+
let shouldUseTui = Tui.shouldUse(isTest || Worker.isEnabled, undefined);
|
|
374
372
|
let isDevelopmentMode = !isTest && config.isDev;
|
|
375
373
|
let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(config, undefined);
|
|
376
374
|
EnvioGlobal.value.persistence = Primitive_option.some(persistence$1);
|
|
@@ -418,7 +416,7 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
418
416
|
let reset = resetOpt !== undefined ? resetOpt : false;
|
|
419
417
|
let isTest = isTestOpt !== undefined ? isTestOpt : false;
|
|
420
418
|
let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
|
|
421
|
-
Stdlib_Option.forEach(Worker.config, param => Config.prime(Config.
|
|
419
|
+
Stdlib_Option.forEach(Worker.config, param => Config.prime(Config.withCommandFields(Config.getPublicConfigJson(), param.chainIds, param.isDev)));
|
|
422
420
|
let config = Config.load();
|
|
423
421
|
let workers = isTest ? undefined : Supervisor.planForRun(config, undefined);
|
|
424
422
|
if (workers !== undefined) {
|