envio 3.0.0-alpha.2 → 3.0.0-alpha.20

package/src/sources/SourceManager.resi

@@ -1,10 +1,21 @@
 type t
 
+type sourceRole = Primary | Secondary
+
+let getSourceRole: (
+  ~sourceFor: Source.sourceFor,
+  ~isLive: bool,
+  ~hasLive: bool,
+) => option<sourceRole>
+
 let make: (
   ~sources: array<Source.t>,
   ~maxPartitionConcurrency: int,
-  ~newBlockFallbackStallTimeout: int=?,
+  ~isLive: bool,
+  ~newBlockStallTimeout: int=?,
+  ~newBlockStallTimeoutLive: int=?,
   ~stalledPollingInterval: int=?,
+  ~recoveryTimeout: float=?,
   ~getHeightRetryInterval: (~retry: int) => int=?,
 ) => t
 
@@ -13,19 +24,19 @@ let getActiveSource: t => Source.t
 let fetchNext: (
   t,
   ~fetchState: FetchState.t,
-  ~currentBlockHeight: int,
   ~executeQuery: FetchState.query => promise<unit>,
-  ~waitForNewBlock: (~currentBlockHeight: int) => promise<int>,
-  ~onNewBlock: (~currentBlockHeight: int) => unit,
+  ~waitForNewBlock: (~knownHeight: int) => promise<int>,
+  ~onNewBlock: (~knownHeight: int) => unit,
   ~stateId: int,
 ) => promise<unit>
 
-let waitForNewBlock: (t, ~currentBlockHeight: int) => promise<int>
+let waitForNewBlock: (t, ~knownHeight: int, ~isLive: bool) => promise<int>
 
 let executeQuery: (
   t,
   ~query: FetchState.query,
-  ~currentBlockHeight: int,
+  ~knownHeight: int,
+  ~isLive: bool,
 ) => promise<Source.blockRangeFetchResponse>
 
 let makeGetHeightRetryInterval: (

package/src/sources/Svm.res

@@ -0,0 +1,81 @@
+@get external getNumber: Internal.eventBlock => int = "height"
+@get external getTimestamp: Internal.eventBlock => int = "time"
+@get external getId: Internal.eventBlock => string = "hash"
+
+let cleanUpRawEventFieldsInPlace: Js.Json.t => unit = %raw(`fields => {
+    delete fields.hash
+    delete fields.height
+    delete fields.time
+  }`)
+
+let ecosystem: Ecosystem.t = {
+  name: Svm,
+  blockFields: ["slot"],
+  transactionFields: [],
+  blockNumberName: "height",
+  blockTimestampName: "time",
+  blockHashName: "hash",
+  getNumber,
+  getTimestamp,
+  getId,
+  cleanUpRawEventFieldsInPlace,
+}
+
+module GetFinalizedSlot = {
+  let route = Rpc.makeRpcRoute(
+    "getSlot",
+    S.tuple(s => {
+      s.tag(0, {"commitment": "finalized"})
+      ()
+    }),
+    S.int,
+  )
+}
+
+let makeRPCSource = (~chain, ~rpc: string): Source.t => {
+  let client = Rest.client(rpc)
+  let chainId = chain->ChainMap.Chain.toChainId
+
+  let urlHost = switch Utils.Url.getHostFromUrl(rpc) {
+  | None =>
+    Js.Exn.raiseError(
+      `The RPC url for chain ${chainId->Belt.Int.toString} is in incorrect format. The RPC url needs to start with either http:// or https://`,
+    )
+  | Some(host) => host
+  }
+  let name = `RPC (${urlHost})`
+
+  {
+    name,
+    sourceFor: Sync,
+    chain,
+    poweredByHyperSync: false,
+    pollingInterval: 10_000,
+    getBlockHashes: (~blockNumbers as _, ~logger as _) =>
+      Js.Exn.raiseError("Svm does not support getting block hashes"),
+    getHeightOrThrow: async () => {
+      let timerRef = Hrtime.makeTimer()
+      let height = await GetFinalizedSlot.route->Rest.fetch((), ~client)
+      let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat
+      Prometheus.SourceRequestCount.increment(~sourceName=name, ~chainId, ~method="getSlot")
+      Prometheus.SourceRequestCount.addSeconds(
+        ~sourceName=name,
+        ~chainId,
+        ~method="getSlot",
+        ~seconds,
+      )
+      height
+    },
+    getItemsOrThrow: (
+      ~fromBlock as _,
+      ~toBlock as _,
+      ~addressesByContractName as _,
+      ~indexingContracts as _,
+      ~knownHeight as _,
+      ~partitionId as _,
+      ~selection as _,
+      ~retry as _,
+      ~logger as _,
+    ) => Js.Exn.raiseError("Svm does not support getting items"),
+  }
+}

package/src/sources/Svm.res.mjs

@@ -0,0 +1,90 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Rpc from "./Rpc.res.mjs";
+import * as Rest from "../vendored/Rest.res.mjs";
+import * as Utils from "../Utils.res.mjs";
+import * as Hrtime from "../bindings/Hrtime.res.mjs";
+import * as Js_exn from "rescript/lib/es6/js_exn.js";
+import * as Prometheus from "../Prometheus.res.mjs";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+
+var cleanUpRawEventFieldsInPlace = (fields => {
+    delete fields.hash
+    delete fields.height
+    delete fields.time
+  });
+
+var ecosystem_blockFields = ["slot"];
+
+var ecosystem_transactionFields = [];
+
+function ecosystem_getNumber(prim) {
+  return prim.height;
+}
+
+function ecosystem_getTimestamp(prim) {
+  return prim.time;
+}
+
+function ecosystem_getId(prim) {
+  return prim.hash;
+}
+
+var ecosystem = {
+  name: "svm",
+  blockFields: ecosystem_blockFields,
+  transactionFields: ecosystem_transactionFields,
+  blockNumberName: "height",
+  blockTimestampName: "time",
+  blockHashName: "hash",
+  getNumber: ecosystem_getNumber,
+  getTimestamp: ecosystem_getTimestamp,
+  getId: ecosystem_getId,
+  cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace
+};
+
+var route = Rpc.makeRpcRoute("getSlot", S$RescriptSchema.tuple(function (s) {
+          s.tag(0, {
+                commitment: "finalized"
+              });
+        }), S$RescriptSchema.$$int);
+
+var GetFinalizedSlot = {
+  route: route
+};
+
+function makeRPCSource(chain, rpc) {
+  var client = Rest.client(rpc, undefined);
+  var host = Utils.Url.getHostFromUrl(rpc);
+  var urlHost = host !== undefined ? host : Js_exn.raiseError("The RPC url for chain " + String(chain) + " is in incorrect format. The RPC url needs to start with either http:// or https://");
+  var name = "RPC (" + urlHost + ")";
+  return {
+          name: name,
+          sourceFor: "Sync",
+          chain: chain,
+          poweredByHyperSync: false,
+          pollingInterval: 10000,
+          getBlockHashes: (function (param, param$1) {
+              return Js_exn.raiseError("Svm does not support getting block hashes");
+            }),
+          getHeightOrThrow: (async function () {
+              var timerRef = Hrtime.makeTimer();
+              var height = await Rest.$$fetch(route, undefined, client);
+              var seconds = Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef));
+              Prometheus.SourceRequestCount.increment(name, chain, "getSlot");
+              Prometheus.SourceRequestCount.addSeconds(name, chain, "getSlot", seconds);
+              return height;
+            }),
+          getItemsOrThrow: (function (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) {
+              return Js_exn.raiseError("Svm does not support getting items");
+            })
+        };
+}
+
+export {
+  cleanUpRawEventFieldsInPlace ,
+  ecosystem ,
+  GetFinalizedSlot ,
+  makeRPCSource ,
+}
+/* route Not a pure module */

package/src/tui/Tui.res

@@ -0,0 +1,247 @@
+open Ink
+open Belt
+
+module ChainLine = {
+  @react.component
+  let make = (
+    ~chainId,
+    ~maxChainIdLength,
+    ~stdoutColumns: int,
+    ~progressBlock,
+    ~bufferBlock,
+    ~sourceBlock,
+    ~startBlock,
+    ~endBlock,
+    ~poweredByHyperSync,
+    ~eventsProcessed,
+  ) => {
+    let chainsWidth = Pervasives.min(stdoutColumns - 2, 60)
+    let headerWidth = maxChainIdLength + 10 // 10 for additional text
+
+    switch (progressBlock, bufferBlock, sourceBlock) {
+    | (Some(progressBlock), Some(bufferBlock), Some(sourceBlock)) =>
+      let toBlock = switch endBlock {
+      | Some(endBlock) => Pervasives.min(sourceBlock, endBlock)
+      | None => sourceBlock
+      }
+      let progressBlockStr = progressBlock->TuiData.formatLocaleString
+      let toBlockStr = toBlock->TuiData.formatLocaleString
+      let eventsStr = eventsProcessed->TuiData.formatFloatLocaleString
+
+      let blocksText =
+        `Blocks: ${progressBlockStr} / ${toBlockStr}` ++
+        (endBlock->Option.isSome ? " (End Block)" : "") ++ `  `
+      let eventsText = `Events: ${eventsStr}`
+
+      let fitsSameLine = blocksText->String.length + eventsText->String.length <= chainsWidth
+
+      <Box flexDirection={Column}>
+        <Box flexDirection=Row width=Num(chainsWidth)>
+          <Box width={Num(headerWidth)}>
+            <Text> {"Chain: "->React.string} </Text>
+            <Text bold=true> {chainId->React.string} </Text>
+            <Text> {" "->React.string} </Text>
+            {poweredByHyperSync ? <Text color=Secondary> {"⚡"->React.string} </Text> : React.null}
+          </Box>
+          <BufferedProgressBar
+            barWidth={chainsWidth - headerWidth}
+            loaded={progressBlock - startBlock}
+            buffered={bufferBlock - startBlock}
+            outOf={toBlock - startBlock}
+            loadingColor={Secondary}
+          />
+        </Box>
+        <Box flexDirection={Row}>
+          <Text color={Gray}> {blocksText->React.string} </Text>
+          {fitsSameLine ? <Text color={Gray}> {eventsText->React.string} </Text> : React.null}
+        </Box>
+        {fitsSameLine
+          ? React.null
+          : <Box flexDirection={Row}>
+              <Text color={Gray}> {eventsText->String.trim->React.string} </Text>
+            </Box>}
+        <Newline />
+      </Box>
+    | (_, _, _) =>
+      <>
+        <Box flexDirection=Row width=Num(chainsWidth)>
+          <Box width={Num(headerWidth)}>
+            <Text> {"Chain: "->React.string} </Text>
+            <Text bold=true> {chainId->React.string} </Text>
+            <Text> {" "->React.string} </Text>
+            {poweredByHyperSync ? <Text color=Secondary> {"⚡"->React.string} </Text> : React.null}
+          </Box>
+          <Text> {"Loading progress..."->React.string} </Text>
+        </Box>
+        <Newline />
+      </>
+    }
+  }
+}
+
+module TotalEventsProcessed = {
+  @react.component
+  let make = (~totalEventsProcessed) => {
+    let label = "Total Events: "
+    <Text>
+      <Text bold=true> {label->React.string} </Text>
+      <Text color={Secondary}>
+        {`${totalEventsProcessed->TuiData.formatFloatLocaleString}`->React.string}
+      </Text>
+    </Text>
+  }
+}
+
+module App = {
+  @react.component
+  let make = (~getState) => {
+    let stdoutColumns = Hooks.useStdoutColumns()
+    let (state: GlobalState.t, setState) = React.useState(() => getState())
+
+    // useEffect to refresh state every 500ms
+    React.useEffect(() => {
+      let intervalId = Js.Global.setInterval(() => {
+        setState(_ => getState())
+      }, 500)
+
+      Some(
+        () => {
+          Js.Global.clearInterval(intervalId)
+        },
+      )
+    }, [getState])
+
+    let chains =
+      state.chainManager.chainFetchers
+      ->ChainMap.values
+      ->Array.map(cf => {
+        let {numEventsProcessed, fetchState} = cf
+        let latestFetchedBlockNumber = Pervasives.max(fetchState->FetchState.bufferBlockNumber, 0)
+        let hasProcessedToEndblock = cf->ChainFetcher.hasProcessedToEndblock
+        let knownHeight =
+          cf->ChainFetcher.hasProcessedToEndblock
+            ? cf.fetchState.endBlock->Option.getWithDefault(cf.fetchState.knownHeight)
+            : cf.fetchState.knownHeight
+
+        let firstEventBlock = cf.fetchState.firstEventBlock
+        let progress: TuiData.progress = if hasProcessedToEndblock {
+          // If the endblock has been reached then set the progress to synced.
+          // if there's chains that have no events in the block range start->end,
+          // it's possible there are no events in that block  range (ie firstEventBlock = None)
+          // This ensures TUI still displays synced in this case
+          Synced({
+            firstEventBlockNumber: firstEventBlock->Option.getWithDefault(0),
+            latestProcessedBlock: cf.committedProgressBlockNumber,
+            timestampCaughtUpToHeadOrEndblock: cf.timestampCaughtUpToHeadOrEndblock->Option.getWithDefault(
+              Js.Date.now()->Js.Date.fromFloat,
+            ),
+            numEventsProcessed,
+          })
+        } else {
+          switch (firstEventBlock, cf.timestampCaughtUpToHeadOrEndblock) {
+          | (Some(firstEventBlockNumber), Some(timestampCaughtUpToHeadOrEndblock)) =>
+            Synced({
+              firstEventBlockNumber,
+              latestProcessedBlock: cf.committedProgressBlockNumber,
+              timestampCaughtUpToHeadOrEndblock,
+              numEventsProcessed,
+            })
+          | (Some(firstEventBlockNumber), None) =>
+            Syncing({
+              firstEventBlockNumber,
+              latestProcessedBlock: cf.committedProgressBlockNumber,
+              numEventsProcessed,
+            })
+          | (None, _) => SearchingForEvents
+          }
+        }
+
+        (
+          {
+            progress,
+            knownHeight,
+            latestFetchedBlockNumber,
+            eventsProcessed: numEventsProcessed,
+            chainId: cf.chainConfig.id->Int.toString,
+            progressBlock: cf.committedProgressBlockNumber < cf.fetchState.startBlock
+              ? Some(cf.fetchState.startBlock)
+              : Some(cf.committedProgressBlockNumber),
+            bufferBlock: Some(latestFetchedBlockNumber),
+            sourceBlock: Some(cf.fetchState.knownHeight),
+            firstEventBlockNumber: cf.fetchState.firstEventBlock,
+            startBlock: cf.fetchState.startBlock,
+            endBlock: cf.fetchState.endBlock,
+            poweredByHyperSync: (
+              cf.sourceManager->SourceManager.getActiveSource
+            ).poweredByHyperSync,
+          }: TuiData.chain
+        )
+      })
+
+    let totalEventsProcessed = chains->Array.reduce(0., (acc, chain) => {
+      acc +. chain.eventsProcessed
+    })
+    let maxChainIdLength = chains->Array.reduce(0, (acc, chain) => {
+      let chainIdLength = chain.chainId->String.length
+      if chainIdLength > acc {
+        chainIdLength
+      } else {
+        acc
+      }
+    })
+
+    <Box flexDirection={Column}>
+      <BigText
+        text="envio"
+        colors=[Secondary, Primary]
+        font={chains->Array.length > 5 ? Tiny : Block}
+        space=false
+      />
+      <Newline />
+      {chains
+      ->Array.mapWithIndex((i, chainData) => {
+        <ChainLine
+          key={i->Int.toString}
+          chainId={chainData.chainId}
+          maxChainIdLength={maxChainIdLength}
+          progressBlock={chainData.progressBlock}
+          bufferBlock={chainData.bufferBlock}
+          sourceBlock={chainData.sourceBlock}
+          startBlock={chainData.startBlock}
+          endBlock={chainData.endBlock}
+          stdoutColumns={stdoutColumns}
+          poweredByHyperSync={chainData.poweredByHyperSync}
+          eventsProcessed={chainData.eventsProcessed}
+        />
+      })
+      ->React.array}
+      <TotalEventsProcessed totalEventsProcessed />
+      <SyncETA chains indexerStartTime=state.indexerStartTime />
+      <Newline />
+      <Box flexDirection={Row}>
+        <Text> {"GraphQL: "->React.string} </Text>
+        <Text color={Info} underline=true> {Env.Hasura.url->React.string} </Text>
+        {
+          let defaultPassword = "testing"
+          if Env.Hasura.secret == defaultPassword {
+            <Text color={Gray}> {` (password: ${defaultPassword})`->React.string} </Text>
+          } else {
+            React.null
+          }
+        }
+      </Box>
+      <Box flexDirection={Row}>
+        <Text> {"Dev Console: "->React.string} </Text>
+        <Text color={Info} underline=true> {`${Env.envioAppUrl}/console`->React.string} </Text>
+      </Box>
+      <Messages config=state.ctx.config />
+    </Box>
+  }
+}
+
+let start = (~getState) => {
+  let {rerender} = render(<App getState />)
+  () => {
+    rerender(<App getState />)
+  }
+}