envio 3.13.0 → 3.14.0-subgraph

package/src/tui/InitApi.res

@@ -0,0 +1,85 @@
+type ecosystem = | @as("evm") Evm | @as("fuel") Fuel | @as("svm") Svm
+type body = {
+  envioVersion: string,
+  envioApiToken: option<string>,
+  ecosystem: ecosystem,
+  hyperSyncNetworks: array<ChainId.t>,
+  rpcNetworks: array<ChainId.t>,
+}
+
+let bodySchema = S.object(s => {
+  envioVersion: s.field("envioVersion", S.string),
+  envioApiToken: s.field("envioApiToken", S.option(S.string)),
+  ecosystem: s.field("ecosystem", S.enum([Evm, Fuel, Svm])),
+  hyperSyncNetworks: s.field("hyperSyncNetworks", S.array(ChainId.schema)),
+  rpcNetworks: s.field("rpcNetworks", S.array(ChainId.schema)),
+})
+
+let makeBody = (~envioVersion, ~envioApiToken, ~config: Config.t) => {
+  let hyperSyncNetworks = []
+  let rpcNetworks = []
+  config.chainMap
+  ->ChainMap.values
+  ->Array.forEach(({sourceConfig, id}) => {
+    let usesHyperSync = switch sourceConfig {
+    | Config.EvmSourceConfig({hypersync: Some(_)}) => true
+    | Config.EvmSourceConfig({hypersync: None}) => false
+    | Config.FuelSourceConfig(_) => true // Fuel always uses HyperFuel
+    | Config.SvmSourceConfig(_) => false
+    | Config.SimulateSourceConfig(_) => false
+    | Config.CustomSources(sources) => sources->Array.some(s => s.poweredByHyperSync)
+    }
+    switch usesHyperSync {
+    | true => hyperSyncNetworks
+    | false => rpcNetworks
+    }
+    ->Array.push(id)
+    ->ignore
+  })
+
+  {
+    envioVersion,
+    envioApiToken,
+    ecosystem: (config.ecosystem.name :> ecosystem),
+    hyperSyncNetworks,
+    rpcNetworks,
+  }
+}
+
+type messageColor =
+  | @as("primary") Primary
+  | @as("secondary") Secondary
+  | @as("info") Info
+  | @as("danger") Danger
+  | @as("success") Success
+  | @as("white") White
+  | @as("gray") Gray
+
+type message = {
+  color: messageColor,
+  content: string,
+}
+
+let messageSchema = S.object(s => {
+  color: s.field("color", S.enum([Primary, Secondary, Info, Danger, Success, White, Gray])),
+  content: s.field("content", S.string),
+})
+
+let client = Rest.client(Env.envioAppUrl ++ "/api")
+
+let route = Rest.route(() => {
+  method: Post,
+  path: "/hyperindex/init",
+  input: s => s.body(bodySchema),
+  responses: [s => s.field("messages", S.array(messageSchema))],
+})
+
+let getMessages = async (~config) => {
+  let envioVersion = Utils.EnvioPackage.value.version
+  let body = makeBody(~envioVersion, ~envioApiToken=Env.envioApiToken, ~config)
+
+  switch await route->Rest.fetch(body, ~client) {
+  | exception exn => Error(exn)
+  | messages => Ok(messages)
+  }
+}

package/src/tui/{components/CustomHooks.res.mjs → InitApi.res.mjs}

@@ -1,12 +1,10 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
-import * as Env from "../../Env.res.mjs";
-import * as Rest from "../../vendored/Rest.res.mjs";
-import * as Utils from "../../Utils.res.mjs";
-import * as React from "react";
-import * as ChainId from "../../ChainId.res.mjs";
-import * as Logging from "../../Logging.res.mjs";
-import * as ChainMap from "../../ChainMap.res.mjs";
+import * as Env from "../Env.res.mjs";
+import * as Rest from "../vendored/Rest.res.mjs";
+import * as Utils from "../Utils.res.mjs";
+import * as ChainId from "../ChainId.res.mjs";
+import * as ChainMap from "../ChainMap.res.mjs";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
@@ -56,25 +54,6 @@ function makeBody(envioVersion, envioApiToken, config) {
   };
 }
 
-function toTheme(color) {
-  switch (color) {
-    case "primary" :
-      return "#9860E5";
-    case "secondary" :
-      return "#FFBB2F";
-    case "info" :
-      return "#6CBFEE";
-    case "danger" :
-      return "#FF8269";
-    case "success" :
-      return "#3B8C3D";
-    case "white" :
-      return "white";
-    case "gray" :
-      return "gray";
-  }
-}
-
 let messageSchema = S$RescriptSchema.object(s => ({
   color: s.f("color", S$RescriptSchema.$$enum([
     "primary",
@@ -118,44 +97,12 @@ async function getMessages(config) {
   };
 }
 
-let InitApi = {
-  bodySchema: bodySchema,
-  makeBody: makeBody,
-  toTheme: toTheme,
-  messageSchema: messageSchema,
-  client: client,
-  route: route,
-  getMessages: getMessages
-};
-
-function useMessages(config) {
-  let match = React.useState(() => "Loading");
-  let setRequest = match[1];
-  React.useEffect(() => {
-    getMessages(config).then(res => {
-      if (res.TAG === "Ok") {
-        let data = res._0;
-        return setRequest(param => ({
-          TAG: "Data",
-          _0: data
-        }));
-      }
-      let e = res._0;
-      Logging.error({
-        msg: "Failed to load messages from envio server",
-        err: Utils.prettifyExn(e)
-      });
-      setRequest(param => ({
-        TAG: "Err",
-        _0: e
-      }));
-    });
-  }, []);
-  return match[0];
-}
-
 export {
-  InitApi,
-  useMessages,
+  bodySchema,
+  makeBody,
+  messageSchema,
+  client,
+  route,
+  getMessages,
 }
 /* bodySchema Not a pure module */

package/src/tui/Tui.res

@@ -1,252 +1,38 @@
-open Ink
-
-module ChainLine = {
-  @react.component
-  let make = (
-    ~chainId,
-    ~maxChainIdLength,
-    ~stdoutColumns: int,
-    ~progressBlock: int,
-    ~bufferBlock: int,
-    ~toBlock: int,
-    ~startBlock,
-    ~endBlock,
-    ~poweredByHyperSync,
-    ~eventsProcessed,
-    ~blockUnit: string,
-  ) => {
-    let chainsWidth = Pervasives.min(stdoutColumns - 2, 60)
-    let headerWidth = maxChainIdLength + 10 // 10 for additional text
-
-    let progressBlockStr = progressBlock->TuiData.formatLocaleString
-    let toBlockStr = toBlock->TuiData.formatLocaleString
-    let eventsStr = eventsProcessed->TuiData.formatFloatLocaleString
-
-    let endLabel = ` (End ${blockUnit})`
-    let blocksText =
-      `${blockUnit}s: ${progressBlockStr} / ${toBlockStr}` ++
-      (endBlock->Option.isSome ? endLabel : "") ++ `  `
-    let eventsText = `Events: ${eventsStr}`
-
-    let fitsSameLine = blocksText->String.length + eventsText->String.length <= chainsWidth
-
-    <Box flexDirection={Column}>
-      <Box flexDirection=Row width=Num(chainsWidth)>
-        <Box width={Num(headerWidth)}>
-          <Text> {"Chain: "->React.string} </Text>
-          <Text bold=true> {chainId->React.string} </Text>
-          <Text> {" "->React.string} </Text>
-          {poweredByHyperSync ? <Text color=Secondary> {"⚡"->React.string} </Text> : React.null}
-        </Box>
-        <BufferedProgressBar
-          barWidth={chainsWidth - headerWidth}
-          loaded={progressBlock - startBlock}
-          buffered={bufferBlock - startBlock}
-          outOf={toBlock - startBlock}
-          loadingColor={Secondary}
-        />
-      </Box>
-      <Box flexDirection={Row}>
-        <Text color={Gray}> {blocksText->React.string} </Text>
-        {fitsSameLine ? <Text color={Gray}> {eventsText->React.string} </Text> : React.null}
-      </Box>
-      {fitsSameLine
-        ? React.null
-        : <Box flexDirection={Row}>
-            <Text color={Gray}> {eventsText->String.trim->React.string} </Text>
-          </Box>}
-      <Newline />
-    </Box>
-  }
+type t
+
+type info = {
+  ecosystem: string,
+  version: string,
+  projectDir: string,
+  startTime: float,
+  graphqlUrl: string,
+  graphqlPassword: option<string>,
+  devConsoleUrl: option<string>,
+  clickhouseUrl: option<string>,
 }
 
-module EventsPerSecond = {
-  type sample = {time: float, events: float}
-
-  let windowMs = 60_000.
-
-  let computeEps = (samples: array<sample>) => {
-    let len = samples->Array.length
-    switch (samples->Array.get(0), samples->Array.get(len - 1)) {
-    | (Some(first), Some(last)) if last.time > first.time =>
-      Some((last.events -. first.events) /. ((last.time -. first.time) /. 1000.))
-    | _ => None
-    }
-  }
-
-  let use = (~totalEventsProcessed: float, ~tick: int) => {
-    let (samples, setSamples) = React.useState((): array<sample> => [])
-
-    React.useEffect1(() => {
-      let now = Date.now()
-      let cutoff = now -. windowMs
-      setSamples(prev => {
-        let kept = prev->Array.filter(s => s.time >= cutoff)
-        kept->Array.concat([{time: now, events: totalEventsProcessed}])
-      })
-      None
-    }, [tick])
-
-    computeEps(samples)
-  }
-}
-
-module TotalEventsProcessed = {
-  @react.component
-  let make = (~totalEventsProcessed, ~eventsPerSecond: option<float>) => {
-    <Text>
-      <Text bold=true> {"Total Events: "->React.string} </Text>
-      <Text color={Secondary}>
-        {`${totalEventsProcessed->TuiData.formatFloatLocaleString}`->React.string}
-      </Text>
-      {switch eventsPerSecond {
-      | Some(eps) =>
-        <Text color={Gray}>
-          {` (${Math.round(eps)->TuiData.formatFloatLocaleString} events/sec)`->React.string}
-        </Text>
-      | None => React.null
-      }}
-    </Text>
-  }
+type chain = {
+  chainId: string,
+  poweredByHyperSync: bool,
+  startBlock: int,
+  endBlock: option<int>,
+  firstEventBlockNumber: option<int>,
+  progressBlockNumber: int,
+  latestFetchedBlockNumber: int,
+  knownHeight: int,
+  sourceBlockNumber: int,
+  timestampCaughtUpToHeadOrEndblock: option<float>,
+  numEventsProcessed: float,
+  rateLimitTimeMs: float,
+  rateLimitResetInMs: option<float>,
+  processedToEndblock: bool,
 }
 
-module App = {
-  @react.component
-  let make = (~config: Config.t, ~getMetrics) => {
-    let stdoutColumns = Hooks.useStdoutColumns()
-    // Metrics are rebuilt from state mutated in place — passing the same value
-    // to useState would bail out via Object.is and skip the re-render. Tick a
-    // counter instead and read metrics freshly on every render.
-    let (tick, setTick) = React.useState(() => 0)
-    let metrics: Metrics.t = getMetrics()
-
-    React.useEffect(() => {
-      let intervalId = setInterval(() => {
-        setTick(t => t + 1)
-      }, 500)
-
-      Some(
-        () => {
-          clearInterval(intervalId)
-        },
-      )
-    }, [getMetrics])
-
-    let blockUnit = switch config.ecosystem.name {
-    | Svm => "Slot"
-    | Evm | Fuel => "Block"
-    }
-    let chains = metrics.chains->Array.map(m => m->TuiData.fromChainMetrics(~blockUnit))
-
-    let totalEventsProcessed = chains->Array.reduce(0., (acc, chain) => {
-      acc +. chain.eventsProcessed
-    })
-    let maxChainIdLength = chains->Array.reduce(0, (acc, chain) => {
-      let chainIdLength = chain.chainId->String.length
-      if chainIdLength > acc {
-        chainIdLength
-      } else {
-        acc
-      }
-    })
-    let eventsPerSecond = EventsPerSecond.use(~totalEventsProcessed, ~tick)
-
-    <Box flexDirection={Column}>
-      <BigText
-        text="envio"
-        colors=[Secondary, Primary]
-        font={chains->Array.length > 5 ? Tiny : Block}
-        space=false
-      />
-      <Newline />
-      {chains
-      ->Array.mapWithIndex((chainData, i) => {
-        <ChainLine
-          key={i->Int.toString}
-          chainId={chainData.chainId}
-          maxChainIdLength={maxChainIdLength}
-          progressBlock={chainData.progressBlock}
-          bufferBlock={chainData.bufferBlock}
-          toBlock={chainData.toBlock}
-          startBlock={chainData.startBlock}
-          endBlock={chainData.endBlock}
-          stdoutColumns={stdoutColumns}
-          poweredByHyperSync={chainData.poweredByHyperSync}
-          eventsProcessed={chainData.eventsProcessed}
-          blockUnit={chainData.blockUnit}
-        />
-      })
-      ->React.array}
-      <TotalEventsProcessed
-        totalEventsProcessed
-        eventsPerSecond={SyncETA.isIndexerFullySynced(chains) ? None : eventsPerSecond}
-      />
-      <SyncETA chains indexerStartTime={metrics.startTime} />
-      {
-        let maxRateLimitTimeMs =
-          chains->Array.reduce(0., (acc, chain) => Pervasives.max(acc, chain.rateLimitTimeMs))
-        let maxResetInMs =
-          chains->Array.reduce(0.0, (acc, chain) =>
-            Pervasives.max(acc, chain.rateLimitResetInMs->Option.getOr(0.0))
-          )
-        maxRateLimitTimeMs > 1000.
-          ? {
-              let rateLimitSecs = Math.round(maxRateLimitTimeMs /. 1000.)
-              let activeSuffix = if maxResetInMs > 0.0 {
-                let resetSecs = Pervasives.max(1.0, Math.ceil(maxResetInMs /. 1000.))
-                ` (⏳ ${resetSecs->TuiData.formatFloatLocaleString}s until reset)`
-              } else {
-                ""
-              }
-              <Box flexDirection={Column}>
-                <Newline />
-                <Text color={Danger}>
-                  {`Backfill ${rateLimitSecs->TuiData.formatFloatLocaleString}s slower due to your plan's rate limit${activeSuffix}`->React.string}
-                </Text>
-                <Text color={Danger}>
-                  <Text color={Danger}> {"Upgrade at "->React.string} </Text>
-                  <Text color={Danger} underline=true>
-                    {"https://envio.dev/app/api-tokens"->React.string}
-                  </Text>
-                  <Text color={Danger}> {" for higher rate limits."->React.string} </Text>
-                </Text>
-              </Box>
-            }
-          : React.null
-      }
-      <Newline />
-      <Box flexDirection={Row}>
-        <Text> {"GraphQL: "->React.string} </Text>
-        <Text color={Info} underline=true> {Env.Hasura.url->React.string} </Text>
-        {
-          let defaultPassword = "testing"
-          if Env.Hasura.secret == defaultPassword {
-            <Text color={Gray}> {` (password: ${defaultPassword})`->React.string} </Text>
-          } else {
-            React.null
-          }
-        }
-      </Box>
-      {if config.isDev {
-        <Box flexDirection={Row}>
-          <Text> {"Dev Console: "->React.string} </Text>
-          <Text color={Info} underline=true> {`${Env.envioAppUrl}/console`->React.string} </Text>
-        </Box>
-      } else {
-        React.null
-      }}
-      {switch (config.storage.clickhouse, Env.ClickHouse.host()) {
-      | (true, Some(host)) =>
-        <Box flexDirection={Row}>
-          <Text> {"ClickHouse: "->React.string} </Text>
-          <Text color={Info} underline=true> {`${host}/play`->React.string} </Text>
-        </Box>
-      | _ => React.null
-      }}
-      <Messages config />
-    </Box>
-  }
-}
+@send external make: (Core.tuiCtor, info) => t = "start"
+@send external update: (t, array<chain>) => unit = "update"
+@send external setMessages: (t, Null.t<array<InitApi.message>>) => unit = "setMessages"
+@send external print: (t, string) => bool = "print"
+@send external stop: t => unit = "stop"
 
 // Whether this process draws the progress display: `ENVIO_TUI` first, then
 // whether anything is watching. A supervisor asks the same question its
@@ -258,9 +44,116 @@ let shouldUse = (~suppressed=false, ~explicitTui=Env.tuiEnvVar) =>
   | (_, None) => !Envio.isNonInteractive()
   }
 
-let start = (~config, ~getMetrics) => {
-  let {rerender} = render(<App config getMetrics />)
-  () => {
-    rerender(<App config getMetrics />)
+let toChain = (m: Metrics.chainMetrics): chain => {
+  chainId: m.chainId->ChainId.toString,
+  poweredByHyperSync: m.poweredByHyperSync,
+  startBlock: m.startBlock,
+  endBlock: m.endBlock,
+  firstEventBlockNumber: m.firstEventBlockNumber,
+  progressBlockNumber: m.progressBlockNumber,
+  latestFetchedBlockNumber: m.latestFetchedBlockNumber,
+  knownHeight: m.knownHeight,
+  sourceBlockNumber: m.sourceBlockNumber,
+  timestampCaughtUpToHeadOrEndblock: m.timestampCaughtUpToHeadOrEndblock->Option.map(Date.getTime),
+  numEventsProcessed: m.numEventsProcessed,
+  rateLimitTimeMs: m.rateLimitTimeMs,
+  rateLimitResetInMs: m.rateLimitResetInMs,
+  processedToEndblock: m->Metrics.hasProcessedToEndblock,
+}
+
+type consoleClass
+type writableClass
+@module("node:console") external consoleClass: consoleClass = "Console"
+@module("node:stream") external writableClass: writableClass = "Writable"
+
+// Sends everything written through `console` above the display, the way the
+// terminal would have shown it, and returns the function that restores it.
+// Routed through a `Console` of our own so every method formats as Node's
+// would; stderr stays where it is when it isn't the terminal, and once the
+// display is gone, output goes back to where it came from.
+let redirectConsole: (
+  consoleClass,
+  writableClass,
+  string => bool,
+) => unit => unit = %raw(`(Console, Writable, print) => {
+  let original = [];
+  const restore = () => original.forEach(([name, method]) => { console[name] = method; });
+  const toDisplay = (stream) => new Writable({
+    write(chunk, _encoding, callback) {
+      // Each call ends its output with a newline, which printing a line adds.
+      if (!print(chunk.toString().replace(/\n$/, ""))) {
+        restore();
+        stream.write(chunk);
+      }
+      callback();
+    },
+  });
+  const redirected = new Console({
+    stdout: toDisplay(process.stdout),
+    stderr: process.stderr.isTTY ? toDisplay(process.stderr) : process.stderr,
+    colorMode: process.stdout.hasColors(),
+  });
+  const methods = Object.keys(console).filter((name) => typeof redirected[name] === "function");
+  original = methods.map((name) => [name, console[name]]);
+  methods.forEach((name) => { console[name] = redirected[name].bind(redirected); });
+  return restore;
+}`)
+
+let start = (~config: Config.t, ~getMetrics: unit => Metrics.t) => {
+  let metrics = getMetrics()
+  let info = {
+    ecosystem: (config.ecosystem.name :> string),
+    version: Utils.EnvioPackage.value.version,
+    projectDir: NodeJs.Process.cwd(),
+    startTime: metrics.startTime->Date.getTime,
+    graphqlUrl: Env.Hasura.url,
+    graphqlPassword: Env.Hasura.secret === "testing" ? Some("testing") : None,
+    devConsoleUrl: config.isDev ? Some(`${Env.envioAppUrl}/console`) : None,
+    clickhouseUrl: switch (config.storage.clickhouse, Env.ClickHouse.host()) {
+    | (true, Some(host)) => Some(`${host}/play`)
+    | _ => None
+    },
+  }
+  switch Core.getAddon().tui->make(info) {
+  | exception exn =>
+    Logging.warn({
+      "msg": "Failed to start the TUI. Continuing without it.",
+      "err": exn->Utils.prettifyExn,
+    })
+  | tui =>
+    let restoreConsole = redirectConsole(consoleClass, writableClass, text => tui->print(text))
+    let update = () => tui->update(getMetrics().chains->Array.map(toChain))
+    update()
+    let _ = setInterval(update, 500)
+    let finish = () => {
+      tui->stop
+      restoreConsole()
+    }
+    NodeJs.Process.onExit(finish)
+    // A signal nothing else listens for ends the process without an `exit`
+    // event, so the display finishes first and the signal is raised again
+    // once its default action is back.
+    ["SIGINT", "SIGTERM", "SIGHUP"]->Array.forEach(signal => {
+      let rec onSignal = () =>
+        if NodeJs.Process.listenerCount(signal) === 1 {
+          finish()
+          NodeJs.Process.offSignal(signal, onSignal)
+          NodeJs.Process.kill(NodeJs.Process.pid, signal)->ignore
+        }
+      NodeJs.Process.onSignal(signal, onSignal)
+    })
+    InitApi.getMessages(~config)
+    ->Promise.thenResolve(result =>
+      switch result {
+      | Ok(messages) => tui->setMessages(Null.Value(messages))
+      | Error(exn) =>
+        Logging.error({
+          "msg": "Failed to load messages from envio server",
+          "err": exn->Utils.prettifyExn,
+        })
+        tui->setMessages(Null.Null)
+      }
+    )
+    ->ignore
   }
 }