envio 3.13.0-alpha.0 → 3.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/package.json +6 -8
  2. package/src/BatchProcessing.res +4 -5
  3. package/src/BatchProcessing.res.mjs +4 -11
  4. package/src/ChainFetching.res +18 -38
  5. package/src/ChainFetching.res.mjs +14 -20
  6. package/src/ChainState.res +113 -77
  7. package/src/ChainState.res.mjs +90 -73
  8. package/src/ChainState.resi +14 -5
  9. package/src/Config.res +14 -17
  10. package/src/Config.res.mjs +5 -20
  11. package/src/CrossChainState.res +68 -20
  12. package/src/CrossChainState.res.mjs +48 -5
  13. package/src/CrossChainState.resi +1 -0
  14. package/src/Env.res +2 -2
  15. package/src/ErrorHandling.res +15 -4
  16. package/src/ErrorHandling.res.mjs +16 -5
  17. package/src/FinalizeBackfill.res +21 -9
  18. package/src/FinalizeBackfill.res.mjs +17 -4
  19. package/src/HandlerRegister.res +0 -37
  20. package/src/HandlerRegister.res.mjs +0 -18
  21. package/src/IndexerState.res +2 -0
  22. package/src/IndexerState.res.mjs +5 -0
  23. package/src/IndexerState.resi +1 -0
  24. package/src/Logging.res +6 -38
  25. package/src/Logging.res.mjs +5 -32
  26. package/src/Main.res +6 -9
  27. package/src/Main.res.mjs +3 -5
  28. package/src/Metrics.res +9 -4
  29. package/src/Metrics.res.mjs +3 -3
  30. package/src/Persistence.res +6 -3
  31. package/src/Persistence.res.mjs +5 -3
  32. package/src/PgStorage.res +2 -2
  33. package/src/PgStorage.res.mjs +2 -2
  34. package/src/Rollback.res +3 -3
  35. package/src/Server.res +0 -3
  36. package/src/SimulateItems.res +1 -1
  37. package/src/SimulateItems.res.mjs +1 -1
  38. package/src/Supervisor.res +236 -109
  39. package/src/Supervisor.res.mjs +185 -55
  40. package/src/TestIndexer.res +6 -3
  41. package/src/TestIndexer.res.mjs +3 -3
  42. package/src/Worker.res +6 -0
  43. package/src/Worker.res.mjs +2 -1
  44. package/src/bindings/NodeJs.res +25 -16
  45. package/src/bindings/NodeJs.res.mjs +8 -1
  46. package/src/sources/AddressSet.res +5 -3
  47. package/src/sources/AddressStore.res +19 -11
  48. package/src/sources/AddressStore.res.mjs +2 -7
  49. package/src/sources/BlockStore.res +10 -9
  50. package/src/sources/BlockStore.res.mjs +4 -4
  51. package/src/sources/ChainSources.res +4 -0
  52. package/src/sources/ChainSources.res.mjs +2 -2
  53. package/src/sources/EvmChain.res +4 -0
  54. package/src/sources/EvmChain.res.mjs +3 -1
  55. package/src/sources/EvmEventItem.res +56 -0
  56. package/src/sources/EvmEventItem.res.mjs +32 -0
  57. package/src/sources/EvmHyperSyncSource.res +7 -37
  58. package/src/sources/EvmHyperSyncSource.res.mjs +8 -29
  59. package/src/sources/EvmRpcClient.res +41 -59
  60. package/src/sources/EvmRpcClient.res.mjs +4 -54
  61. package/src/sources/EvmRpcWs.res +4 -1
  62. package/src/sources/EvmRpcWs.res.mjs +6 -2
  63. package/src/sources/FuelHyperSyncSource.res +6 -4
  64. package/src/sources/FuelHyperSyncSource.res.mjs +7 -9
  65. package/src/sources/HeightFeed.res +7 -1
  66. package/src/sources/HeightFeed.res.mjs +9 -7
  67. package/src/sources/HyperSyncClient.res +1 -14
  68. package/src/sources/RequestStat.res +7 -0
  69. package/src/sources/RequestStat.res.mjs +14 -1
  70. package/src/sources/RpcSource.res +100 -1043
  71. package/src/sources/RpcSource.res.mjs +81 -1056
  72. package/src/sources/SimulateSource.res +2 -5
  73. package/src/sources/SimulateSource.res.mjs +2 -3
  74. package/src/sources/Source.res +34 -12
  75. package/src/sources/Source.res.mjs +11 -0
  76. package/src/sources/SourceManager.res +111 -31
  77. package/src/sources/SourceManager.res.mjs +65 -31
  78. package/src/sources/SourceManager.resi +4 -0
  79. package/src/sources/StartBlockResolver.res +8 -5
  80. package/src/sources/StartBlockResolver.res.mjs +4 -2
  81. package/src/sources/SvmHyperSyncSource.res +4 -2
  82. package/src/sources/SvmHyperSyncSource.res.mjs +4 -6
  83. package/src/sources/TransactionStore.res +9 -6
  84. package/src/sources/TransactionStore.res.mjs +2 -2
  85. package/src/tui/Tui.res +4 -18
  86. package/src/tui/Tui.res.mjs +3 -6
  87. package/src/LazyLoader.res +0 -135
  88. package/src/LazyLoader.res.mjs +0 -118
  89. package/src/bindings/SDSL.res +0 -12
  90. package/src/bindings/SDSL.res.mjs +0 -9
  91. package/src/bindings/Yargs.res +0 -8
  92. package/src/bindings/Yargs.res.mjs +0 -2
  93. package/src/sources/Rpc.res +0 -185
  94. package/src/sources/Rpc.res.mjs +0 -183
@@ -125,9 +125,13 @@ function isReadyToEnterReorgThreshold(crossChainState, batch) {
125
125
  }
126
126
 
127
127
  function enterReorgThreshold(crossChainState) {
128
- Logging.info("Reorg threshold reached");
129
128
  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
130
- ChainState.enterReorgThreshold(crossChainState.chainStates[crossChainState.chainIds[i]]);
129
+ let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
130
+ let liftsCeiling = ChainState.reorgThresholdLiftsCeiling(cs);
131
+ ChainState.enterReorgThreshold(cs);
132
+ if (liftsCeiling) {
133
+ Logging.childInfo(ChainState.logger(cs), ChainState.reorgThresholdEntryMessage);
134
+ }
131
135
  }
132
136
  }
133
137
 
@@ -137,7 +141,7 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
137
141
  for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
138
142
  let cs = crossChainState.chainStates[chainIds[i]];
139
143
  ChainState.applyBatchProgress(cs, batch, blockTimestampName);
140
- if (!(ChainState.hasProcessedToEndblock(cs) || ChainState.isProgressAtHead(cs))) {
144
+ if (!ChainState.hasCaughtUp(cs)) {
141
145
  everyChainCaughtUp = false;
142
146
  }
143
147
  }
@@ -165,7 +169,7 @@ function markCaughtUpOnResume(crossChainState) {
165
169
  let everyChainCaughtUp = crossChainState.chainIds.length !== 0;
166
170
  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
167
171
  let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
168
- if (!ChainState.isDurablyCaughtUp(cs)) {
172
+ if (!ChainState.hasCaughtUp(cs)) {
169
173
  everyChainCaughtUp = false;
170
174
  }
171
175
  }
@@ -177,11 +181,49 @@ function markCaughtUpOnResume(crossChainState) {
177
181
 
178
182
  function markReady(crossChainState, readyAt) {
179
183
  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
180
- ChainState.markReady(crossChainState.chainStates[crossChainState.chainIds[i]], readyAt);
184
+ let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
185
+ let wasReady = ChainState.isReady(cs);
186
+ ChainState.markReady(cs, readyAt);
187
+ if (!wasReady) {
188
+ Logging.childInfo(ChainState.logger(cs), "Ready. Fully indexed for queries.");
189
+ }
181
190
  }
182
191
  crossChainState.isRealtime = true;
183
192
  }
184
193
 
194
+ function isWaitingOnOthers(crossChainState, cs) {
195
+ if (crossChainState.holdRealtime) {
196
+ return true;
197
+ } else {
198
+ return Object.values(crossChainState.chainStates).some(other => {
199
+ if (other !== cs) {
200
+ return !ChainState.hasCaughtUp(other);
201
+ } else {
202
+ return false;
203
+ }
204
+ });
205
+ }
206
+ }
207
+
208
+ function reportFinished(crossChainState) {
209
+ Object.values(crossChainState.chainStates).forEach(cs => {
210
+ let match = ChainState.takeFinished(cs);
211
+ if (match !== undefined) {
212
+ if (match.TAG === "EndBlock") {
213
+ return Logging.childInfo(ChainState.logger(cs), {
214
+ msg: "Indexed to the end block. This chain is done.",
215
+ block: match._0
216
+ });
217
+ } else {
218
+ return Logging.childInfo(ChainState.logger(cs), {
219
+ msg: isWaitingOnOthers(crossChainState, cs) ? "Finished backfill. Waiting for the other chains." : "Finished backfill.",
220
+ block: match._0
221
+ });
222
+ }
223
+ }
224
+ });
225
+ }
226
+
185
227
  function priorityOrder(crossChainState) {
186
228
  return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.frontierProgress(a), ChainState.frontierProgress(b)));
187
229
  }
@@ -296,6 +338,7 @@ export {
296
338
  markCaughtUpIfSettled,
297
339
  markCaughtUpOnResume,
298
340
  markReady,
341
+ reportFinished,
299
342
  priorityOrder,
300
343
  checkAndFetch,
301
344
  }
@@ -40,6 +40,7 @@ let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => uni
40
40
  let markCaughtUpIfSettled: t => unit
41
41
  let markCaughtUpOnResume: t => unit
42
42
  let markReady: (t, ~readyAt: Date.t) => unit
43
+ let reportFinished: t => unit
43
44
 
44
45
  // Fetch control.
45
46
  let priorityOrder: t => array<ChainState.t>
package/src/Env.res CHANGED
@@ -129,8 +129,8 @@ module Db = {
129
129
  )
130
130
  // The budget for the whole run, not for one process: a run that splits across
131
131
  // workers divides it among them, and each caps its own pool to its share.
132
- // The default buys a single worker, so a run splits only once the operator
133
- // raises the budget it may spend.
132
+ // Splitting takes two workers and a worker takes two connections, so a budget
133
+ // under 4 — the default among them — keeps the run in one process.
134
134
  let maxConnections = envSafe->EnvSafe.get("ENVIO_PG_MAX_CONNECTIONS", S.int, ~fallback=2)
135
135
  }
136
136
 
@@ -11,14 +11,25 @@ let log = (self: t) => {
11
11
  }
12
12
  }
13
13
 
14
+ // The exception to hand to whoever is outside the indexer. A failure raised
15
+ // from a message alone carries no exception of its own, so rethrowing or
16
+ // rejecting with `exn` would hand over a bare `null` and the reason would live
17
+ // only in the logs. Fall back to the message, which is the whole of what such a
18
+ // failure knows.
19
+ let toExn = (self: t) =>
20
+ switch self.exn->(Utils.magic: exn => Nullable.t<exn>)->Nullable.toOption {
21
+ | Some(exn) => exn->Utils.prettifyExn
22
+ | None => Utils.Error.make(self.msg->Option.getOr("Indexer has failed with an unexpected error"))
23
+ }
24
+
14
25
  let raiseExn = (self: t) => {
15
- self.exn->Utils.prettifyExn->throw
26
+ self->toExn->throw
16
27
  }
17
28
 
18
29
  let mkLogAndRaise = (~logger=?, ~msg=?, exn) => {
19
- let exn = exn->Utils.prettifyExn
20
- exn->make(~logger?, ~msg?)->log
21
- exn->throw
30
+ let self = exn->Utils.prettifyExn->make(~logger?, ~msg?)
31
+ self->log
32
+ self->raiseExn
22
33
  }
23
34
 
24
35
  let unwrapLogAndRaise = (~logger=?, ~msg=?, result) => {
@@ -2,6 +2,7 @@
2
2
 
3
3
  import * as Utils from "./Utils.res.mjs";
4
4
  import * as Logging from "./Logging.res.mjs";
5
+ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
5
6
 
6
7
  function make(exn, loggerOpt, msg) {
7
8
  let logger = loggerOpt !== undefined ? loggerOpt : Logging.getLogger();
@@ -23,14 +24,23 @@ function log(self) {
23
24
  }
24
25
  }
25
26
 
27
+ function toExn(self) {
28
+ let exn = self.exn;
29
+ if (exn == null) {
30
+ return new Error(Stdlib_Option.getOr(self.msg, "Indexer has failed with an unexpected error"));
31
+ } else {
32
+ return Utils.prettifyExn(exn);
33
+ }
34
+ }
35
+
26
36
  function raiseExn(self) {
27
- throw Utils.prettifyExn(self.exn);
37
+ throw toExn(self);
28
38
  }
29
39
 
30
40
  function mkLogAndRaise(logger, msg, exn) {
31
- let exn$1 = Utils.prettifyExn(exn);
32
- log(make(exn$1, logger, msg));
33
- throw exn$1;
41
+ let self = make(Utils.prettifyExn(exn), logger, msg);
42
+ log(self);
43
+ throw toExn(self);
34
44
  }
35
45
 
36
46
  function unwrapLogAndRaise(logger, msg, result) {
@@ -43,12 +53,13 @@ function unwrapLogAndRaise(logger, msg, result) {
43
53
 
44
54
  function logAndRaise(self) {
45
55
  log(self);
46
- throw Utils.prettifyExn(self.exn);
56
+ throw toExn(self);
47
57
  }
48
58
 
49
59
  export {
50
60
  make,
51
61
  log,
62
+ toExn,
52
63
  raiseExn,
53
64
  mkLogAndRaise,
54
65
  unwrapLogAndRaise,
@@ -11,9 +11,25 @@
11
11
  // `envio start --chain` process is indexing and never waits on one.
12
12
 
13
13
  let runOnce = async (state: IndexerState.t) => {
14
- Logging.info(
15
- "All chains are caught up. Finalizing the indexer before switching to realtime: flushing pending writes, then creating the indexes the schema promises.",
16
- )
14
+ let chainIds = state->IndexerState.crossChainState->CrossChainState.chainIds
15
+
16
+ // Whatever brought the process here — a batch, a tick that progressed
17
+ // nothing, a supervisor's release — its chains say where they finished
18
+ // before it says what it does about that. Said here rather than left to each
19
+ // caller to order: a chain that has already spoken says nothing again.
20
+ state->IndexerState.reportFinished
21
+
22
+ // Said by the process rather than by each of its chains: the indexes are one
23
+ // build over the tables, and the pause is the whole process's. A chain has
24
+ // already said it caught up, and says it is ready once this commits. The
25
+ // chains are named because the pause is theirs, and a split run has a process
26
+ // saying this for each part of it — one of them, for a process driving a
27
+ // single chain.
28
+ let msg = "Building database indexes. Indexing is paused until they are ready, which can take a while on a large database."
29
+ switch chainIds {
30
+ | [chainId] => Logging.info({"msg": msg, "chainId": chainId})
31
+ | chainIds => Logging.info({"msg": msg, "chainIds": chainIds})
32
+ }
17
33
 
18
34
  await Writing.flush(state)
19
35
 
@@ -24,16 +40,12 @@ let runOnce = async (state: IndexerState.t) => {
24
40
  let storage = persistence->Persistence.getInitializedStorageOrThrow
25
41
  let readyAt = Date.make()
26
42
 
27
- await storage.finalizeBackfill(
28
- ~entities=persistence.allEntities,
29
- ~chainIds=state->IndexerState.crossChainState->CrossChainState.chainIds,
30
- ~readyAt,
31
- )
43
+ await storage.finalizeBackfill(~entities=persistence.allEntities, ~chainIds, ~readyAt)
32
44
 
33
45
  // Only after the commit: in-memory readiness must never run ahead of the
34
46
  // `ready_at` a restart would read back.
47
+ // Says so per chain, which is the grain `ready_at` is committed at.
35
48
  state->IndexerState.markReady(~readyAt)
36
- Logging.info("The indexer is ready. Switching to realtime indexing.")
37
49
  }
38
50
  }
39
51
 
@@ -7,7 +7,21 @@ import * as IndexerState from "./IndexerState.res.mjs";
7
7
  import * as CrossChainState from "./CrossChainState.res.mjs";
8
8
 
9
9
  async function runOnce(state) {
10
- Logging.info("All chains are caught up. Finalizing the indexer before switching to realtime: flushing pending writes, then creating the indexes the schema promises.");
10
+ let chainIds = CrossChainState.chainIds(IndexerState.crossChainState(state));
11
+ IndexerState.reportFinished(state);
12
+ let msg = "Building database indexes. Indexing is paused until they are ready, which can take a while on a large database.";
13
+ if (chainIds.length !== 1) {
14
+ Logging.info({
15
+ msg: msg,
16
+ chainIds: chainIds
17
+ });
18
+ } else {
19
+ let chainId = chainIds[0];
20
+ Logging.info({
21
+ msg: msg,
22
+ chainId: chainId
23
+ });
24
+ }
11
25
  await Writing.flush(state);
12
26
  if (IndexerState.hasFailedWrite(state)) {
13
27
  return;
@@ -15,9 +29,8 @@ async function runOnce(state) {
15
29
  let persistence = IndexerState.persistence(state);
16
30
  let storage = Persistence.getInitializedStorageOrThrow(persistence);
17
31
  let readyAt = new Date();
18
- await storage.finalizeBackfill(persistence.allEntities, CrossChainState.chainIds(IndexerState.crossChainState(state)), readyAt);
19
- IndexerState.markReady(state, readyAt);
20
- return Logging.info("The indexer is ready. Switching to realtime indexing.");
32
+ await storage.finalizeBackfill(persistence.allEntities, chainIds, readyAt);
33
+ return IndexerState.markReady(state, readyAt);
21
34
  }
22
35
 
23
36
  function run(state) {
@@ -493,41 +493,6 @@ let getSimulateOnEventRegistrations = (
493
493
  }
494
494
  }
495
495
 
496
- // An RPC source can only deliver the fields it knows how to parse; the rest are
497
- // silently skipped at materialisation. Every RPC on the chain counts, whatever
498
- // it's for — a fallback or realtime source runs the same parsers as a sync one,
499
- // so a field it can't deliver would go missing for whichever blocks it served.
500
- // The selection can come from either `config.yaml` or the inline `fields`
501
- // option, so the message names neither. Runs on the registrations a chain
502
- // actually keeps, so a handler whose `where` opts out of this chain isn't held
503
- // to its limits.
504
- //
505
- // The `config.yaml` half of this is also rejected at codegen, by the
506
- // `RpcTransactionField` subenum in `system_config.rs`. That one reports every
507
- // offending field at once and before the project builds; this one is the only
508
- // check an inline selection reaches. `RpcFieldSelection_test.res` pins the two
509
- // to the same field set.
510
- let validateRpcFieldSelection = (
511
- chainConfig: Config.chain,
512
- registrations: array<Internal.onEventRegistration>,
513
- ) => {
514
- let hasRpc = switch chainConfig.sourceConfig {
515
- | EvmSourceConfig({rpcs}) => !(rpcs->Utils.Array.isEmpty)
516
- | _ => false
517
- }
518
- if hasRpc {
519
- registrations->Array.forEach(reg =>
520
- reg.fieldSelection.transactionFields->Utils.Set.forEach(name =>
521
- if !RpcSource.isRpcTransactionField(name) {
522
- JsError.throwWithMessage(
523
- `The "${name}" transaction field selected for the "${reg.eventConfig.name}" event on contract "${reg.eventConfig.contractName}" is unavailable for indexing via RPC. Remove it from the field selection, or remove chain ${chainConfig.id->ChainId.toString}'s RPC source — even an RPC the chain only falls back to has to deliver the selection.`,
524
- )
525
- }
526
- )
527
- )
528
- }
529
- }
530
-
531
496
  let finishRegistration = (~config: Config.t): registrationsByChainId => {
532
497
  switch getActiveRegistration() {
533
498
  | Some(r) => {
@@ -614,8 +579,6 @@ let finishRegistration = (~config: Config.t): registrationsByChainId => {
614
579
  }
615
580
  })
616
581
 
617
- validateRpcFieldSelection(chainConfig, onEventRegistrations)
618
-
619
582
  registrationsByChainId->Dict.set(
620
583
  key,
621
584
  {
@@ -5,7 +5,6 @@ import * as ChainId from "./ChainId.res.mjs";
5
5
  import * as Logging from "./Logging.res.mjs";
6
6
  import * as ChainMap from "./ChainMap.res.mjs";
7
7
  import * as Internal from "./Internal.res.mjs";
8
- import * as RpcSource from "./sources/RpcSource.res.mjs";
9
8
  import * as EnvioGlobal from "./EnvioGlobal.res.mjs";
10
9
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
11
10
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
@@ -327,22 +326,6 @@ function getSimulateOnEventRegistrations(config, chainId, eventConfig) {
327
326
  }
328
327
  }
329
328
 
330
- function validateRpcFieldSelection(chainConfig, registrations) {
331
- let match = chainConfig.sourceConfig;
332
- let hasRpc;
333
- hasRpc = match.TAG === "EvmSourceConfig" ? !Utils.$$Array.isEmpty(match.rpcs) : false;
334
- if (hasRpc) {
335
- registrations.forEach(reg => {
336
- reg.fieldSelection.transactionFields.forEach(name => {
337
- if (!RpcSource.isRpcTransactionField(name)) {
338
- return Stdlib_JsError.throwWithMessage(`The "` + name + `" transaction field selected for the "` + reg.eventConfig.name + `" event on contract "` + reg.eventConfig.contractName + `" is unavailable for indexing via RPC. Remove it from the field selection, or remove chain ` + ChainId.toString(chainConfig.id) + `'s RPC source — even an RPC the chain only falls back to has to deliver the selection.`);
339
- }
340
- });
341
- });
342
- return;
343
- }
344
- }
345
-
346
329
  function finishRegistration(config) {
347
330
  let r = EnvioGlobal.value.activeRegistration;
348
331
  if (r === undefined) {
@@ -389,7 +372,6 @@ function finishRegistration(config) {
389
372
  let newrecord = {...reg};
390
373
  onEventRegistrations.push((newrecord.index = onEventRegistrations.length, newrecord));
391
374
  });
392
- validateRpcFieldSelection(chainConfig, onEventRegistrations);
393
375
  let chainRegs = r.registrationsByChainId[key];
394
376
  registrationsByChainId[key] = {
395
377
  onEventRegistrations: onEventRegistrations,
@@ -544,6 +544,8 @@ let releaseRealtime = (state: t) => {
544
544
 
545
545
  let markReady = (state: t, ~readyAt) => state.crossChainState->CrossChainState.markReady(~readyAt)
546
546
 
547
+ let reportFinished = (state: t) => state.crossChainState->CrossChainState.reportFinished
548
+
547
549
  let rollbackState = (state: t) => state.rollbackState
548
550
  let indexerStartTime = (state: t) => state.indexerStartTime
549
551
  let loadManager = (state: t) => state.loadManager
@@ -390,6 +390,10 @@ function markReady(state, readyAt) {
390
390
  CrossChainState.markReady(state.crossChainState, readyAt);
391
391
  }
392
392
 
393
+ function reportFinished(state) {
394
+ CrossChainState.reportFinished(state.crossChainState);
395
+ }
396
+
393
397
  function rollbackState(state) {
394
398
  return state.rollbackState;
395
399
  }
@@ -884,6 +888,7 @@ export {
884
888
  hasArrivedAtHead,
885
889
  releaseRealtime,
886
890
  markReady,
891
+ reportFinished,
887
892
  rollbackState,
888
893
  indexerStartTime,
889
894
  loadManager,
@@ -104,6 +104,7 @@ let hasArrivedAtHead: t => bool
104
104
  // The supervisor's go-ahead for a process driving part of a split run.
105
105
  let releaseRealtime: t => unit
106
106
  let markReady: (t, ~readyAt: Date.t) => unit
107
+ let reportFinished: t => unit
107
108
  let rollbackState: t => rollbackState
108
109
  let indexerStartTime: t => Date.t
109
110
  let loadManager: t => LoadManager.t
package/src/Logging.res CHANGED
@@ -26,30 +26,6 @@ let logLevels = [
26
26
 
27
27
  %%private(let logger = ref(None))
28
28
 
29
- // Fields every line this process logs carries. Merged into each line rather
30
- // than bound to a child logger: pino writes a child's bindings and the line's
31
- // own fields side by side, so a line that names the same key would carry it
32
- // twice. A fresh object per line, since pino merges the line's fields into
33
- // whatever this returns.
34
- %%private(let context: ref<dict<JSON.t>> = ref(Dict.make()))
35
- %%private(let mixin = () => JSON.Object(context.contents->Dict.copy))
36
-
37
- // A child logger that binds a field the process already carries would have pino
38
- // write it twice: a child's bindings and the context are concatenated into the
39
- // line, not merged. The context is the one place a line names it, which it can
40
- // be because a process only takes one when its every line is about that chain.
41
- %%private(
42
- let withoutContext = (params: 'a) =>
43
- switch context.contents->Dict.keysToArray {
44
- | [] => params
45
- | keys => {
46
- let narrowed = params->(Utils.magic: 'a => dict<JSON.t>)->Dict.copy
47
- keys->Array.forEach(key => narrowed->Dict.delete(key))
48
- narrowed->(Utils.magic: dict<JSON.t> => 'a)
49
- }
50
- }
51
- )
52
-
53
29
  let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLevel) => {
54
30
  // Currently unused - useful if using multiple transports.
55
31
  // let pinoRaw = {"target": "pino/file", "level": Config.userLogLevel}
@@ -80,19 +56,17 @@ let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLeve
80
56
  ...Pino.ECS.make(),
81
57
  customLevels: logLevels,
82
58
  base,
83
- mixin,
84
59
  },
85
60
  Transport.make(pinoFile),
86
61
  )
87
62
  | EcsConsoleMultistream =>
88
- makeMultiStreamLogger(~logFile=None, ~options=Some({...Pino.ECS.make(), base, mixin}))
63
+ makeMultiStreamLogger(~logFile=None, ~options=Some({...Pino.ECS.make(), base}))
89
64
  | EcsConsole =>
90
65
  make({
91
66
  ...Pino.ECS.make(),
92
67
  level: userLogLevel,
93
68
  customLevels: logLevels,
94
69
  base,
95
- mixin,
96
70
  })
97
71
  | FileOnly =>
98
72
  makeWithOptionsAndTransport(
@@ -100,13 +74,12 @@ let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLeve
100
74
  customLevels: logLevels,
101
75
  level: defaultFileLogLevel,
102
76
  base,
103
- mixin,
104
77
  },
105
78
  Transport.make(pinoFile),
106
79
  )
107
- | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=Some({base, mixin}))
108
- | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=Some({base, mixin}))
109
- | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=Some({base, mixin}))
80
+ | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=Some({base: base}))
81
+ | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=Some({base: base}))
82
+ | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=Some({base: base}))
110
83
  }
111
84
  }
112
85
 
@@ -176,15 +149,10 @@ let childFatal = (logger, params: 'a) => {
176
149
  }
177
150
 
178
151
  let createChild = (~params: 'a) => {
179
- getLogger()->child(params->withoutContext->createChildParams)
152
+ getLogger()->child(params->createChildParams)
180
153
  }
181
-
182
- // What belongs on every line is the run's to decide; the logger only carries
183
- // what it is handed. A line that names one of these fields itself wins.
184
- let setContext = (fields: dict<JSON.t>) => context := fields
185
-
186
154
  let createChildFrom = (~logger: t, ~params: 'a) => {
187
- logger->child(params->withoutContext->createChildParams)
155
+ logger->child(params->createChildParams)
188
156
  }
189
157
 
190
158
  @inline
@@ -3,7 +3,6 @@
3
3
  import * as Pino from "./bindings/Pino.res.mjs";
4
4
  import * as Pino$1 from "pino";
5
5
  import * as Utils from "./Utils.res.mjs";
6
- import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
7
6
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
8
7
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
9
8
 
@@ -54,24 +53,6 @@ let logger = {
54
53
  contents: undefined
55
54
  };
56
55
 
57
- let context = {
58
- contents: {}
59
- };
60
-
61
- function mixin() {
62
- return Object.assign({}, context.contents);
63
- }
64
-
65
- function withoutContext(params) {
66
- let keys = Object.keys(context.contents);
67
- if (keys.length === 0) {
68
- return params;
69
- }
70
- let narrowed = Object.assign({}, params);
71
- keys.forEach(key => Stdlib_Dict.$$delete(narrowed, key));
72
- return narrowed;
73
- }
74
-
75
56
  function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel) {
76
57
  let pinoFile_target = "pino/file";
77
58
  let pinoFile_options = Primitive_option.some(Pino.Transport.makeTransportOptions({
@@ -90,18 +71,17 @@ function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel)
90
71
  switch (logStrategy) {
91
72
  case "ecs-file" :
92
73
  let newrecord = {...Pino.ECS.make()};
93
- return Pino$1.pino((newrecord.base = base, newrecord.mixin = mixin, newrecord.customLevels = logLevels, newrecord), Pino$1.transport(pinoFile));
74
+ return Pino$1.pino((newrecord.base = base, newrecord.customLevels = logLevels, newrecord), Pino$1.transport(pinoFile));
94
75
  case "ecs-console" :
95
76
  let newrecord$1 = {...Pino.ECS.make()};
96
- return Pino$1.pino((newrecord$1.base = base, newrecord$1.mixin = mixin, newrecord$1.customLevels = logLevels, newrecord$1.level = userLogLevel, newrecord$1));
77
+ return Pino$1.pino((newrecord$1.base = base, newrecord$1.customLevels = logLevels, newrecord$1.level = userLogLevel, newrecord$1));
97
78
  case "ecs-console-multistream" :
98
79
  let newrecord$2 = {...Pino.ECS.make()};
99
- return makeMultiStreamLogger(undefined, (newrecord$2.base = base, newrecord$2.mixin = mixin, newrecord$2));
80
+ return makeMultiStreamLogger(undefined, (newrecord$2.base = base, newrecord$2));
100
81
  case "file-only" :
101
82
  return Pino$1.pino({
102
83
  level: defaultFileLogLevel,
103
84
  customLevels: logLevels,
104
- mixin: mixin,
105
85
  base: base
106
86
  }, Pino$1.transport(pinoFile));
107
87
  case "console-raw" :
@@ -109,12 +89,10 @@ function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel)
109
89
  break;
110
90
  case "both-prettyconsole" :
111
91
  return makeMultiStreamLogger(logFilePath, {
112
- mixin: mixin,
113
92
  base: base
114
93
  });
115
94
  }
116
95
  return makeMultiStreamLogger(undefined, {
117
- mixin: mixin,
118
96
  base: base
119
97
  });
120
98
  }
@@ -193,15 +171,11 @@ function childFatal(logger, params) {
193
171
  }
194
172
 
195
173
  function createChild(params) {
196
- return getLogger().child(Pino.createChildParams(withoutContext(params)));
197
- }
198
-
199
- function setContext(fields) {
200
- context.contents = fields;
174
+ return getLogger().child(Pino.createChildParams(params));
201
175
  }
202
176
 
203
177
  function createChildFrom(logger, params) {
204
- return logger.child(Pino.createChildParams(withoutContext(params)));
178
+ return logger.child(Pino.createChildParams(params));
205
179
  }
206
180
 
207
181
  function logAtLevel(logger, level, message, params) {
@@ -272,7 +246,6 @@ export {
272
246
  childErrorWithExn,
273
247
  childFatal,
274
248
  createChild,
275
- setContext,
276
249
  createChildFrom,
277
250
  logAtLevel,
278
251
  noopLogger,
package/src/Main.res CHANGED
@@ -453,9 +453,6 @@ exception FatalError(exn)
453
453
  ) => {
454
454
  // A worker reports to its supervisor, which draws for the whole run.
455
455
  let shouldUseTui = Tui.shouldUse(~suppressed=isTest || Worker.isEnabled)
456
- // In per-chain mode every line this process writes belongs to the chains it
457
- // drives, whether or not a supervisor split the run across processes.
458
- config->Config.logContext->Option.forEach(Logging.setContext)
459
456
  // isDevelopmentMode controls whether the indexer stays alive after all
460
457
  // chains finish (keepProcessAlive) and whether the console API is exposed.
461
458
  // Set by `envio dev` via the public config's `isDev` field; `envio start`
@@ -561,12 +558,12 @@ let start = async (
561
558
  ~exitAfterFirstEventBlock=false,
562
559
  ~patchConfig: option<(Config.t, HandlerRegister.registrationsByChainId) => Config.t>=?,
563
560
  ) => {
564
- // A worker parses the same config its supervisor did and narrows it to the
565
- // chains it was handed, rather than being told what to index: the storage it
566
- // resumes refuses a config that disagrees with the one the run was created
567
- // from, which is a stronger guarantee than a handover could give.
568
- Worker.config->Option.forEach(({chainIds}) =>
569
- Config.prime(Config.getPublicConfigJson()->Config.withIsolatedChains(~chainIds))
561
+ // A worker parses the project's files itself rather than being handed the
562
+ // config: a public config carries every contract's ABI, which is far more
563
+ // than a spawn environment should. What the command decided rides along
564
+ // instead, and re-applying it is what makes the two configs the same one.
565
+ Worker.config->Option.forEach(({chainIds, isDev}) =>
566
+ Config.prime(Config.getPublicConfigJson()->Config.withCommandFields(~chainIds, ~isDev))
570
567
  )
571
568
  let config = Config.load()
572
569
  switch isTest ? None : Supervisor.planForRun(~config) {
package/src/Main.res.mjs CHANGED
@@ -6,7 +6,6 @@ import * as Config from "./Config.res.mjs";
6
6
  import * as Server from "./Server.res.mjs";
7
7
  import * as Worker from "./Worker.res.mjs";
8
8
  import * as ChainId from "./ChainId.res.mjs";
9
- import * as Logging from "./Logging.res.mjs";
10
9
  import * as Metrics from "./Metrics.res.mjs";
11
10
  import * as ChainMap from "./ChainMap.res.mjs";
12
11
  import * as PgStorage from "./PgStorage.res.mjs";
@@ -352,7 +351,7 @@ function getGlobalIndexer() {
352
351
  async function migrate(reset) {
353
352
  let config = Config.load();
354
353
  let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
355
- await Persistence.init(persistence, ChainMap.values(config.chainMap), config.contractMapping, Config.envioInfo(), "envio local db-migrate setup", undefined, reset, config.lowercaseAddresses, undefined, "Once");
354
+ await Persistence.init(persistence, ChainMap.values(config.chainMap), config.contractMapping, Config.envioInfo(), "envio local db-migrate setup", undefined, reset, config.lowercaseAddresses, undefined, undefined, "Once");
356
355
  return await persistence.storage.close();
357
356
  }
358
357
 
@@ -369,8 +368,7 @@ async function startIndexer(config, persistence, resetOpt, isTestOpt, exitAfterF
369
368
  let reset = resetOpt !== undefined ? resetOpt : false;
370
369
  let isTest = isTestOpt !== undefined ? isTestOpt : false;
371
370
  let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
372
- let shouldUseTui = Tui.shouldUse(isTest || Worker.isEnabled);
373
- Stdlib_Option.forEach(Config.logContext(config), Logging.setContext);
371
+ let shouldUseTui = Tui.shouldUse(isTest || Worker.isEnabled, undefined);
374
372
  let isDevelopmentMode = !isTest && config.isDev;
375
373
  let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(config, undefined);
376
374
  EnvioGlobal.value.persistence = Primitive_option.some(persistence$1);
@@ -418,7 +416,7 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
418
416
  let reset = resetOpt !== undefined ? resetOpt : false;
419
417
  let isTest = isTestOpt !== undefined ? isTestOpt : false;
420
418
  let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
421
- Stdlib_Option.forEach(Worker.config, param => Config.prime(Config.withIsolatedChains(Config.getPublicConfigJson(), param.chainIds)));
419
+ Stdlib_Option.forEach(Worker.config, param => Config.prime(Config.withCommandFields(Config.getPublicConfigJson(), param.chainIds, param.isDev)));
422
420
  let config = Config.load();
423
421
  let workers = isTest ? undefined : Supervisor.planForRun(config, undefined);
424
422
  if (workers !== undefined) {