envio 3.12.0 → 3.13.0-alpha.0

package/src/Logging.res.mjs

@@ -3,6 +3,7 @@
 import * as Pino from "./bindings/Pino.res.mjs";
 import * as Pino$1 from "pino";
 import * as Utils from "./Utils.res.mjs";
+import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 
@@ -53,6 +54,24 @@ let logger = {
   contents: undefined
 };
 
+let context = {
+  contents: {}
+};
+
+function mixin() {
+  return Object.assign({}, context.contents);
+}
+
+function withoutContext(params) {
+  let keys = Object.keys(context.contents);
+  if (keys.length === 0) {
+    return params;
+  }
+  let narrowed = Object.assign({}, params);
+  keys.forEach(key => Stdlib_Dict.$$delete(narrowed, key));
+  return narrowed;
+}
+
 function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel) {
   let pinoFile_target = "pino/file";
   let pinoFile_options = Primitive_option.some(Pino.Transport.makeTransportOptions({
@@ -71,17 +90,18 @@ function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel)
   switch (logStrategy) {
     case "ecs-file" :
       let newrecord = {...Pino.ECS.make()};
-      return Pino$1.pino((newrecord.base = base, newrecord.customLevels = logLevels, newrecord), Pino$1.transport(pinoFile));
+      return Pino$1.pino((newrecord.base = base, newrecord.mixin = mixin, newrecord.customLevels = logLevels, newrecord), Pino$1.transport(pinoFile));
     case "ecs-console" :
       let newrecord$1 = {...Pino.ECS.make()};
-      return Pino$1.pino((newrecord$1.base = base, newrecord$1.customLevels = logLevels, newrecord$1.level = userLogLevel, newrecord$1));
+      return Pino$1.pino((newrecord$1.base = base, newrecord$1.mixin = mixin, newrecord$1.customLevels = logLevels, newrecord$1.level = userLogLevel, newrecord$1));
     case "ecs-console-multistream" :
       let newrecord$2 = {...Pino.ECS.make()};
-      return makeMultiStreamLogger(undefined, (newrecord$2.base = base, newrecord$2));
+      return makeMultiStreamLogger(undefined, (newrecord$2.base = base, newrecord$2.mixin = mixin, newrecord$2));
     case "file-only" :
       return Pino$1.pino({
         level: defaultFileLogLevel,
         customLevels: logLevels,
+        mixin: mixin,
         base: base
       }, Pino$1.transport(pinoFile));
     case "console-raw" :
@@ -89,10 +109,12 @@ function makeLogger(logStrategy, logFilePath, defaultFileLogLevel, userLogLevel)
       break;
     case "both-prettyconsole" :
       return makeMultiStreamLogger(logFilePath, {
+        mixin: mixin,
         base: base
       });
   }
   return makeMultiStreamLogger(undefined, {
+    mixin: mixin,
     base: base
   });
 }
@@ -171,11 +193,15 @@ function childFatal(logger, params) {
 }
 
 function createChild(params) {
-  return getLogger().child(Pino.createChildParams(params));
+  return getLogger().child(Pino.createChildParams(withoutContext(params)));
+}
+
+function setContext(fields) {
+  context.contents = fields;
 }
 
 function createChildFrom(logger, params) {
-  return logger.child(Pino.createChildParams(params));
+  return logger.child(Pino.createChildParams(withoutContext(params)));
 }
 
 function logAtLevel(logger, level, message, params) {
@@ -246,6 +272,7 @@ export {
   childErrorWithExn,
   childFatal,
   createChild,
+  setContext,
   createChildFrom,
   logAtLevel,
   noopLogger,

package/src/Main.res

@@ -1,80 +1,3 @@
-// The public console/state chain shape. Kept to exactly this field set for
-// backward compatibility with consumers like RACE — new metric fields stay off
-// the HTTP response.
-type chainData = {
-  chainId: ChainId.t,
-  poweredByHyperSync: bool,
-  firstEventBlockNumber: option<int>,
-  latestProcessedBlock: option<int>,
-  timestampCaughtUpToHeadOrEndblock: option<Date.t>,
-  numEventsProcessed: float,
-  latestFetchedBlockNumber: int,
-  // Need this for API backwards compatibility
-  @as("currentBlockHeight")
-  knownHeight: int,
-  numBatchesFetched: int,
-  startBlock: int,
-  endBlock: option<int>,
-  numAddresses: int,
-}
-@tag("status")
-type state =
-  | @as("disabled") Disabled({})
-  | @as("initializing") Initializing({})
-  | @as("active")
-  Active({
-      envioVersion: string,
-      chains: array<chainData>,
-      indexerStartTime: Date.t,
-      isPreRegisteringDynamicContracts: bool,
-      rollbackOnReorg: bool,
-    })
-
-let toChainData = (m: Metrics.chainMetrics): chainData => {
-  chainId: m.chainId,
-  poweredByHyperSync: m.poweredByHyperSync,
-  firstEventBlockNumber: m.firstEventBlockNumber,
-  latestProcessedBlock: m.latestProcessedBlock,
-  timestampCaughtUpToHeadOrEndblock: m.timestampCaughtUpToHeadOrEndblock,
-  numEventsProcessed: m.numEventsProcessed,
-  latestFetchedBlockNumber: m.latestFetchedBlockNumber,
-  knownHeight: m.knownHeight,
-  numBatchesFetched: m.numBatchesFetched,
-  startBlock: m.startBlock,
-  endBlock: m.endBlock,
-  numAddresses: m.numAddresses,
-}
-
-let chainDataSchema = S.schema((s): chainData => {
-  chainId: s.matches(ChainId.schema),
-  poweredByHyperSync: s.matches(S.bool),
-  firstEventBlockNumber: s.matches(S.option(S.int)),
-  latestProcessedBlock: s.matches(S.option(S.int)),
-  timestampCaughtUpToHeadOrEndblock: s.matches(S.option(S.datetime(S.string))),
-  numEventsProcessed: s.matches(S.float),
-  latestFetchedBlockNumber: s.matches(S.int),
-  knownHeight: s.matches(S.int),
-  numBatchesFetched: s.matches(S.int),
-  startBlock: s.matches(S.int),
-  endBlock: s.matches(S.option(S.int)),
-  numAddresses: s.matches(S.int),
-})
-let stateSchema = S.union([
-  S.literal(Disabled({})),
-  S.literal(Initializing({})),
-  S.schema(s => Active({
-    envioVersion: s.matches(S.string),
-    chains: s.matches(S.array(chainDataSchema)),
-    indexerStartTime: s.matches(S.datetime(S.string)),
-    // Keep the field, since Dev Console expects it to be present
-    isPreRegisteringDynamicContracts: false,
-    rollbackOnReorg: s.matches(S.bool),
-  })),
-])
-
-// Runtime state lives in the process-wide `EnvioGlobal` record (shared
-// across duplicate envio module instances); the slots are opaque there, so
-// cast them to the real types here.
 let getIndexerState = () =>
   EnvioGlobal.value.indexerState->(Utils.magic: option<unknown> => option<IndexerState.t>)
 let setIndexerState = (state: IndexerState.t) =>
@@ -488,111 +411,8 @@ let getGlobalIndexer = (): 'indexer => {
   Utils.Proxy.make(Utils.Object.createNullObject(), traps)->(Utils.magic: {..} => 'indexer)
 }
 
-let startServer = (
-  ~getMetrics: unit => option<Metrics.t>,
-  ~envioVersion: string,
-  ~persistence: Persistence.t,
-  ~isDevelopmentMode: bool,
-) => {
-  open Express
-
-  let app = make()
-
-  let consoleCorsMiddleware = (req, res, next) => {
-    switch req.headers->Dict.get("origin") {
-    | Some(origin) if origin === Env.prodEnvioAppUrl || origin === Env.envioAppUrl =>
-      res->setHeader("Access-Control-Allow-Origin", origin)
-    | _ => ()
-    }
-
-    res->setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
-    res->setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
-
-    if req.method === Rest.Options {
-      res->sendStatus(200)
-    } else {
-      next()
-    }
-  }
-  app->useFor("/console", consoleCorsMiddleware)
-  app->useFor("/metrics", consoleCorsMiddleware)
-  app->useFor("/metrics/runtime", consoleCorsMiddleware)
-
-  app->get("/healthz", (_req, res) => {
-    // this is the machine readable port used in kubernetes to check the health of this service.
-    //   aditional health information could be added in the future (info about errors, back-offs, etc).
-    res->sendStatus(200)
-  })
-
-  app->get("/console/state", (_req, res) => {
-    let state = if !isDevelopmentMode {
-      Disabled({})
-    } else {
-      switch getMetrics() {
-      | None => Initializing({})
-      | Some(metrics) =>
-        Active({
-          envioVersion,
-          chains: metrics.chains->Array.map(toChainData),
-          indexerStartTime: metrics.startTime,
-          isPreRegisteringDynamicContracts: false,
-          rollbackOnReorg: metrics.rollbackEnabled,
-        })
-      }
-    }
-
-    res->json(state->S.reverseConvertToJsonOrThrow(stateSchema))
-  })
-
-  app->post("/console/syncCache", (_req, res) => {
-    if isDevelopmentMode {
-      (persistence->Persistence.getInitializedStorageOrThrow).dumpEffectCache()
-      ->Promise.thenResolve(_ => res->json(Boolean(true)))
-      ->Promise.ignore
-    } else {
-      res->json(Boolean(false))
-    }
-  })
-
-  Metrics.startRuntimeCollectors()
-
-  app->get("/metrics", (_req, res) => {
-    res->set("Content-Type", Metrics.contentType)
-    let _ = res->endWithData(Metrics.collect(~metrics=getMetrics()))
-  })
-
-  app->get("/metrics/runtime", (_req, res) => {
-    res->set("Content-Type", Metrics.contentType)
-    let _ = res->endWithData(Metrics.collectRuntime())
-  })
-
-  let server = app->listen(Env.serverPort)
-  server->Express.onError(err => {
-    let code = (err->(Utils.magic: JsExn.t => {..}))["code"]
-    if code === "EADDRINUSE" {
-      Logging.error(
-        `Port ${Env.serverPort->Int.toString} is already in use. To fix this either:` ++
-        `\n  1. Kill the process using the port: lsof -ti :${Env.serverPort->Int.toString} | xargs kill -9` ++ `\n  2. Use a different port by setting the ENVIO_INDEXER_PORT environment variable: ENVIO_INDEXER_PORT=9899 envio start`,
-      )
-    } else {
-      Logging.errorWithExn(err, "Failed to start indexer server")
-    }
-    NodeJs.process->NodeJs.exitWithCode(Failure)
-  })
-}
-
-type args = {@as("tui-off") tuiOff?: bool}
-
-type process
-@val external process: process = "process"
-@get external argv: process => 'a = "argv"
-
-type mainArgs = Yargs.parsedArgs<args>
-
 // The RPC-stripped public config that the storage layer persists in
 // `envio_info` (on initialize) and validates against (on resume).
-let getEnvioInfo = () => Config.getPublicConfigJson()->Config.stripSensitiveData
-
 let migrate = async (~reset) => {
   let config = Config.load()
   let persistence = PgStorage.makePersistenceFromConfig(~config)
@@ -600,7 +420,7 @@ let migrate = async (~reset) => {
     ~reset,
     ~chainConfigs=config.chainMap->ChainMap.values,
     ~contractMapping=config.contractMapping,
-    ~envioInfo=getEnvioInfo(),
+    ~envioInfo=Config.envioInfo(),
     ~resetCommand="envio local db-migrate setup",
     ~runCommand=None,
     ~lowercaseAddresses=config.lowercaseAddresses,
@@ -622,6 +442,118 @@ let dropSchema = async () => {
 // context, so callers should act on it (exit / re-throw) without logging again.
 exception FatalError(exn)
 
+%%private(
+  let startIndexer = async (
+    ~config: Config.t,
+    ~persistence: option<Persistence.t>=?,
+    ~reset=false,
+    ~isTest=false,
+    ~exitAfterFirstEventBlock=false,
+    ~patchConfig: option<(Config.t, HandlerRegister.registrationsByChainId) => Config.t>=?,
+  ) => {
+    // A worker reports to its supervisor, which draws for the whole run.
+    let shouldUseTui = Tui.shouldUse(~suppressed=isTest || Worker.isEnabled)
+    // In per-chain mode every line this process writes belongs to the chains it
+    // drives, whether or not a supervisor split the run across processes.
+    config->Config.logContext->Option.forEach(Logging.setContext)
+    // isDevelopmentMode controls whether the indexer stays alive after all
+    // chains finish (keepProcessAlive) and whether the console API is exposed.
+    // Set by `envio dev` via the public config's `isDev` field; `envio start`
+    // leaves it false so the process exits cleanly when indexing completes.
+    let isDevelopmentMode = !isTest && config.isDev
+    // Initialized first so the exported indexer value contains state from the
+    // database when handler files are loaded (they may access the indexer at
+    // module top level).
+    let persistence = switch persistence {
+    | Some(p) => p
+    | None => PgStorage.makePersistenceFromConfig(~config)
+    }
+    setGlobalPersistence(persistence)
+    await persistence->Persistence.initForRun(
+      ~config,
+      ~reset,
+      ~isDevelopmentMode,
+      ~requireInitialized=config.isolated,
+    )
+
+    // Loads user handler files, which register handler/contractRegister/where
+    // state into the global `HandlerRegister` registry as a side effect; this
+    // returns that state resolved into per-chain registrations. `config` itself
+    // is never mutated by registration — it holds only event definitions.
+    let registrationsByChainId = await HandlerLoader.registerAllHandlers(~config)
+    let config = if isTest {
+      {...config, shouldRollbackOnReorg: false}
+    } else {
+      config
+    }
+
+    let config = switch patchConfig {
+    | Some(patchConfig) => patchConfig(config, registrationsByChainId)
+    | None => config
+    }
+    // The single fatal-error handler, invoked once via IndexerState.errorExit.
+    // It logs the failure once (with chain context) and rejects the run wrapped in
+    // `FatalError` so callers know it's already logged — `Bin.res` just exits, the
+    // test worker unwraps and re-throws it to the parent thread. `runUntilFatalError`
+    // only ever rejects: on a clean run it stays pending and the process exits via
+    // ExitOnCaughtUp / when the indexer loop drains.
+    let onErrorReject = ref(None)
+    let runUntilFatalError: promise<unit> = Promise.make((_resolve, reject) =>
+      onErrorReject := Some(reject)
+    )
+    // `onErrorReject` is filled synchronously by `Promise.make` above, before the
+    // indexer can run and call `onError`, so it's always present here.
+    let onError = (errHandler: ErrorHandling.t) => {
+      errHandler->ErrorHandling.log
+      (onErrorReject.contents->Option.getUnsafe)(FatalError(errHandler.exn->Utils.prettifyExn))
+    }
+    let envioVersion = Utils.EnvioPackage.value.version
+
+    let getMetrics = () => getIndexerState()->Option.map(IndexerState.toMetrics)
+    let dumpEffectCache = () =>
+      (persistence->Persistence.getInitializedStorageOrThrow).dumpEffectCache()
+
+    // A worker reports through its supervisor, which owns the one server and the
+    // one display the run has.
+    if !isTest && !Worker.isEnabled {
+      Metrics.startRuntimeCollectors()
+      Server.startServer(
+        ~onSyncCache=() => dumpEffectCache()->Promise.thenResolve(ignore),
+        ~collectRuntime=Metrics.collectRuntime,
+        ~isDevelopmentMode,
+        ~envioVersion,
+        ~getMetrics,
+      )
+    }
+
+    let state = IndexerState.makeFromDbState(
+      ~config,
+      ~persistence,
+      ~initialState=persistence->Persistence.getInitializedState,
+      ~registrationsByChainId,
+      ~isDevelopmentMode,
+      ~shouldUseTui,
+      ~exitAfterFirstEventBlock,
+      ~holdRealtime=Worker.config->Option.mapOr(false, worker => worker.holdRealtime),
+      ~onError,
+    )
+    if shouldUseTui {
+      let _rerender = Tui.start(~config, ~getMetrics=() => state->IndexerState.toMetrics)
+    }
+    Worker.bindRun(
+      ~getMetrics=() => state->IndexerState.toMetrics,
+      ~onReleaseRealtime=() => state->IndexerState.releaseRealtime,
+    )
+    setIndexerState(state)
+    state->IndexerLoop.start
+    await runUntilFatalError
+  }
+)
+
+// Starts this process's part of a run: the group's supervisor when the budget
+// and the schema afford splitting the chains across processes, and the indexer
+// itself otherwise. A worker is already one process's part, so it never splits
+// again — `planForRun` refuses an isolated config.
 let start = async (
   ~persistence: option<Persistence.t>=?,
   ~reset=false,
@@ -629,93 +561,24 @@ let start = async (
   ~exitAfterFirstEventBlock=false,
   ~patchConfig: option<(Config.t, HandlerRegister.registrationsByChainId) => Config.t>=?,
 ) => {
-  let mainArgs: mainArgs = process->argv->Yargs.hideBin->Yargs.yargs->Yargs.argv
-  let explicitTui = switch mainArgs.tuiOff {
-  | Some(off) => Some(!off)
-  | None => Env.tuiEnvVar
-  }
-  let shouldUseTui = switch (isTest, explicitTui) {
-  | (true, _) => false
-  | (_, Some(tui)) => tui
-  | (_, None) => !Envio.isNonInteractive()
-  }
-  // Initialize persistence first so the exported indexer value contains state from the database
-  // when handler files are loaded (they may access the indexer at module top level).
-  let config = Config.load()
-  // isDevelopmentMode controls whether the indexer stays alive after all
-  // chains finish (keepProcessAlive) and whether the console API is exposed.
-  // Set by `envio dev` via the public config's `isDev` field; `envio start`
-  // leaves it false so the process exits cleanly when indexing completes.
-  let isDevelopmentMode = !isTest && config.isDev
-  let persistence = switch persistence {
-  | Some(p) => p
-  | None => PgStorage.makePersistenceFromConfig(~config)
-  }
-  setGlobalPersistence(persistence)
-  await persistence->Persistence.init(
-    ~reset,
-    ~chainConfigs=config.chainMap->ChainMap.values,
-    ~contractMapping=config.contractMapping,
-    ~envioInfo=getEnvioInfo(),
-    ~resetCommand=isDevelopmentMode ? "envio dev -r" : "envio start -r",
-    ~runCommand=Some(isDevelopmentMode ? "envio dev" : "envio start"),
-    ~lowercaseAddresses=config.lowercaseAddresses,
-    ~requireInitialized=config.isolated,
+  // A worker parses the same config its supervisor did and narrows it to the
+  // chains it was handed, rather than being told what to index: the storage it
+  // resumes refuses a config that disagrees with the one the run was created
+  // from, which is a stronger guarantee than a handover could give.
+  Worker.config->Option.forEach(({chainIds}) =>
+    Config.prime(Config.getPublicConfigJson()->Config.withIsolatedChains(~chainIds))
   )
-
-  // Loads user handler files, which register handler/contractRegister/where
-  // state into the global `HandlerRegister` registry as a side effect; this
-  // returns that state resolved into per-chain registrations. `config` itself
-  // is never mutated by registration — it holds only event definitions.
-  let registrationsByChainId = await HandlerLoader.registerAllHandlers(~config)
-  let config = if isTest {
-    {...config, shouldRollbackOnReorg: false}
-  } else {
-    config
-  }
-
-  let config = switch patchConfig {
-  | Some(patchConfig) => patchConfig(config, registrationsByChainId)
-  | None => config
-  }
-  // The single fatal-error handler, invoked once via IndexerState.errorExit.
-  // It logs the failure once (with chain context) and rejects the run wrapped in
-  // `FatalError` so callers know it's already logged — `Bin.res` just exits, the
-  // test worker unwraps and re-throws it to the parent thread. `runUntilFatalError`
-  // only ever rejects: on a clean run it stays pending and the process exits via
-  // ExitOnCaughtUp / when the indexer loop drains.
-  let onErrorReject = ref(None)
-  let runUntilFatalError: promise<unit> = Promise.make((_resolve, reject) =>
-    onErrorReject := Some(reject)
-  )
-  // `onErrorReject` is filled synchronously by `Promise.make` above, before the
-  // indexer can run and call `onError`, so it's always present here.
-  let onError = (errHandler: ErrorHandling.t) => {
-    errHandler->ErrorHandling.log
-    (onErrorReject.contents->Option.getUnsafe)(FatalError(errHandler.exn->Utils.prettifyExn))
-  }
-  let envioVersion = Utils.EnvioPackage.value.version
-
-  let getMetrics = () => getIndexerState()->Option.map(IndexerState.toMetrics)
-
-  if !isTest {
-    startServer(~persistence, ~isDevelopmentMode, ~envioVersion, ~getMetrics)
-  }
-
-  let state = IndexerState.makeFromDbState(
-    ~config,
-    ~persistence,
-    ~initialState=persistence->Persistence.getInitializedState,
-    ~registrationsByChainId,
-    ~isDevelopmentMode,
-    ~shouldUseTui,
-    ~exitAfterFirstEventBlock,
-    ~onError,
-  )
-  if shouldUseTui {
-    let _rerender = Tui.start(~config, ~getMetrics=() => state->IndexerState.toMetrics)
+  let config = Config.load()
+  switch isTest ? None : Supervisor.planForRun(~config) {
+  | Some(workers) => await Supervisor.run(~config, ~workers, ~reset)
+  | None =>
+    await startIndexer(
+      ~config,
+      ~persistence?,
+      ~reset,
+      ~isTest,
+      ~exitAfterFirstEventBlock,
+      ~patchConfig?,
+    )
   }
-  setIndexerState(state)
-  state->IndexerLoop.start
-  await runUntilFatalError
 }