envio 3.0.0-alpha.1 → 3.0.0-alpha.2

package/evm.schema.json

@@ -50,18 +50,22 @@
         "$ref": "#/$defs/GlobalContract_for_ContractConfig"
       }
     },
-    "networks": {
-      "description": "Configuration of the blockchain networks that the project is deployed on.",
+    "chains": {
+      "description": "Configuration of the blockchain chains that the project is deployed on.",
       "type": "array",
       "items": {
         "$ref": "#/$defs/Network"
       }
     },
-    "unordered_multichain_mode": {
-      "description": "A flag to indicate if the indexer should use a single queue for all chains or a queue per chain (default: false)",
-      "type": [
-        "boolean",
-        "null"
+    "multichain": {
+      "description": "Multichain mode: 'ordered' processes events across chains in order, 'unordered' processes chain events in order, but non-deterministically relatively to other chains (default: unordered)",
+      "anyOf": [
+        {
+          "$ref": "#/$defs/Multichain"
+        },
+        {
+          "type": "null"
+        }
       ]
     },
     "event_decoder": {
@@ -107,13 +111,6 @@
         "null"
       ]
     },
-    "preload_handlers": {
-      "description": "Makes handlers run twice to enable preload optimisations. Removes handlerWithLoader API, since it's not needed. (recommended, default: false)",
-      "type": [
-        "boolean",
-        "null"
-      ]
-    },
     "address_format": {
       "description": "Address format for Ethereum addresses: 'checksum' or 'lowercase' (default: checksum)",
       "anyOf": [
@@ -136,7 +133,7 @@
   "additionalProperties": false,
   "required": [
     "name",
-    "networks"
+    "chains"
   ],
   "$defs": {
     "EcosystemTag": {
@@ -675,6 +672,13 @@
         }
       ]
     },
+    "Multichain": {
+      "type": "string",
+      "enum": [
+        "ordered",
+        "unordered"
+      ]
+    },
     "EventDecoder": {
       "type": "string",
       "enum": [

package/fuel.schema.json

@@ -43,8 +43,8 @@
         "$ref": "#/$defs/GlobalContract_for_ContractConfig"
       }
     },
-    "networks": {
-      "description": "Configuration of the blockchain networks that the project is deployed on.",
+    "chains": {
+      "description": "Configuration of the blockchain chains that the project is deployed on.",
       "type": "array",
       "items": {
         "$ref": "#/$defs/Network"
@@ -57,13 +57,6 @@
         "null"
       ]
     },
-    "preload_handlers": {
-      "description": "Makes handlers run twice to enable preload optimisations. Removes handlerWithLoader API, since it's not needed. (recommended, default: false)",
-      "type": [
-        "boolean",
-        "null"
-      ]
-    },
     "handlers": {
       "description": "Optional relative path to handlers directory for auto-loading. Defaults to 'src/handlers' if not specified.",
       "type": [
@@ -76,7 +69,7 @@
   "required": [
     "name",
     "ecosystem",
-    "networks"
+    "chains"
   ],
   "$defs": {
     "EcosystemTag": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v3.0.0-alpha.1",
+  "version": "v3.0.0-alpha.2",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
@@ -29,23 +29,26 @@
     "node": ">=22.0.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "v3.0.0-alpha.1",
-    "envio-linux-arm64": "v3.0.0-alpha.1",
-    "envio-darwin-x64": "v3.0.0-alpha.1",
-    "envio-darwin-arm64": "v3.0.0-alpha.1"
+    "envio-linux-x64": "v3.0.0-alpha.2",
+    "envio-linux-arm64": "v3.0.0-alpha.2",
+    "envio-darwin-x64": "v3.0.0-alpha.2",
+    "envio-darwin-arm64": "v3.0.0-alpha.2"
   },
   "dependencies": {
     "@clickhouse/client": "1.12.1",
-    "@envio-dev/hypersync-client": "0.7.0",
+    "@elastic/ecs-pino-format": "1.4.0",
     "@envio-dev/hyperfuel-client": "1.2.2",
-    "rescript": "11.1.3",
-    "rescript-schema": "9.3.4",
-    "viem": "2.21.0",
+    "@envio-dev/hypersync-client": "0.7.0",
     "bignumber.js": "9.1.2",
+    "eventsource": "4.1.0",
     "pino": "8.16.1",
     "pino-pretty": "10.2.3",
-    "@elastic/ecs-pino-format": "1.4.0",
-    "prom-client": "15.0.0"
+    "prom-client": "15.0.0",
+    "rescript": "11.1.3",
+    "rescript-schema": "9.3.4",
+    "viem": "2.21.0",
+    "rescript-envsafe": "5.0.0",
+    "dotenv": "16.4.5"
   },
   "files": [
     "bin.js",

package/rescript.json

@@ -19,6 +19,6 @@
     "generatedFileExtension": ".gen.ts",
     "moduleResolution": "node16"
   },
-  "bs-dependencies": ["rescript-schema"],
+  "bs-dependencies": ["rescript-schema", "rescript-envsafe"],
   "bsc-flags": ["-open RescriptSchema"]
 }

package/src/Config.res

@@ -45,9 +45,8 @@ type t = {
   multichain: multichain,
   chainMap: ChainMap.t<chain>,
   defaultChain: option<chain>,
-  platform: Platform.t,
+  ecosystem: Ecosystem.t,
   enableRawEvents: bool,
-  preloadHandlers: bool,
   maxAddrInPartition: int,
   batchSize: int,
   lowercaseAddresses: bool,
@@ -59,8 +58,7 @@ let make = (
   ~shouldSaveFullHistory=false,
   ~chains: array<chain>=[],
   ~enableRawEvents=false,
-  ~preloadHandlers=false,
-  ~ecosystem: Platform.name=Platform.Evm,
+  ~ecosystem: Ecosystem.name=Ecosystem.Evm,
   ~batchSize=5000,
   ~lowercaseAddresses=false,
   ~multichain=Unordered,
@@ -90,7 +88,7 @@ let make = (
     })
   })
 
-  let platform = Platform.fromName(ecosystem)
+  let ecosystem = Ecosystem.fromName(ecosystem)
 
   {
     shouldRollbackOnReorg,
@@ -99,9 +97,8 @@ let make = (
     chainMap,
     defaultChain: chains->Array.get(0),
     enableRawEvents,
-    platform,
+    ecosystem,
     maxAddrInPartition,
-    preloadHandlers,
     batchSize,
     lowercaseAddresses,
     addContractNameToContractNameMapping,

package/src/Config.res.mjs

@@ -3,15 +3,14 @@
 import * as Utils from "./Utils.res.mjs";
 import * as Js_exn from "rescript/lib/es6/js_exn.js";
 import * as ChainMap from "./ChainMap.res.mjs";
-import * as Platform from "./Platform.res.mjs";
+import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
 
-function make(shouldRollbackOnReorgOpt, shouldSaveFullHistoryOpt, chainsOpt, enableRawEventsOpt, preloadHandlersOpt, ecosystemOpt, batchSizeOpt, lowercaseAddressesOpt, multichainOpt, shouldUseHypersyncClientDecoderOpt, maxAddrInPartitionOpt) {
+function make(shouldRollbackOnReorgOpt, shouldSaveFullHistoryOpt, chainsOpt, enableRawEventsOpt, ecosystemOpt, batchSizeOpt, lowercaseAddressesOpt, multichainOpt, shouldUseHypersyncClientDecoderOpt, maxAddrInPartitionOpt) {
   var shouldRollbackOnReorg = shouldRollbackOnReorgOpt !== undefined ? shouldRollbackOnReorgOpt : true;
   var shouldSaveFullHistory = shouldSaveFullHistoryOpt !== undefined ? shouldSaveFullHistoryOpt : false;
   var chains = chainsOpt !== undefined ? chainsOpt : [];
   var enableRawEvents = enableRawEventsOpt !== undefined ? enableRawEventsOpt : false;
-  var preloadHandlers = preloadHandlersOpt !== undefined ? preloadHandlersOpt : false;
   var ecosystem = ecosystemOpt !== undefined ? ecosystemOpt : "evm";
   var batchSize = batchSizeOpt !== undefined ? batchSizeOpt : 5000;
   var lowercaseAddresses = lowercaseAddressesOpt !== undefined ? lowercaseAddressesOpt : false;
@@ -34,16 +33,15 @@ function make(shouldRollbackOnReorgOpt, shouldSaveFullHistoryOpt, chainsOpt, ena
                   addContractNameToContractNameMapping[addKey] = contract.name;
                 }));
         }));
-  var platform = Platform.fromName(ecosystem);
+  var ecosystem$1 = Ecosystem.fromName(ecosystem);
   return {
           shouldRollbackOnReorg: shouldRollbackOnReorg,
           shouldSaveFullHistory: shouldSaveFullHistory,
           multichain: multichain,
           chainMap: chainMap,
           defaultChain: Belt_Array.get(chains, 0),
-          platform: platform,
+          ecosystem: ecosystem$1,
           enableRawEvents: enableRawEvents,
-          preloadHandlers: preloadHandlers,
           maxAddrInPartition: maxAddrInPartition,
           batchSize: batchSize,
           lowercaseAddresses: lowercaseAddresses,

package/src/{Platform.res → Ecosystem.res}

@@ -132,9 +132,9 @@ let fromName = (name: name): t => {
   }
 }
 
-// Create a block event object for block handlers based on platform
-let makeBlockEvent = (~blockNumber: int, platform: t): Internal.blockEvent => {
+// Create a block event object for block handlers based on ecosystem
+let makeBlockEvent = (~blockNumber: int, ecosystem: t): Internal.blockEvent => {
   let blockEvent = Js.Dict.empty()
-  blockEvent->Js.Dict.set(platform.blockNumberName, blockNumber->Utils.magic)
+  blockEvent->Js.Dict.set(ecosystem.blockNumberName, blockNumber->Utils.magic)
   blockEvent->Utils.magic
 }

package/src/{Platform.res.mjs → Ecosystem.res.mjs}

@@ -153,9 +153,9 @@ function fromName(name) {
   }
 }
 
-function makeBlockEvent(blockNumber, platform) {
+function makeBlockEvent(blockNumber, ecosystem) {
   var blockEvent = {};
-  blockEvent[platform.blockNumberName] = blockNumber;
+  blockEvent[ecosystem.blockNumberName] = blockNumber;
   return blockEvent;
 }
 

package/src/Env.res

@@ -0,0 +1,244 @@
+
+// Loads the .env from the root working directory
+%%raw(`import 'dotenv/config'`)
+
+%%private(
+  let envSafe = EnvSafe.make()
+
+  let getLogLevelConfig = (name, ~default): Pino.logLevel =>
+    envSafe->EnvSafe.get(
+      name,
+      S.enum([#trace, #debug, #info, #warn, #error, #fatal, #udebug, #uinfo, #uwarn, #uerror]),
+      ~fallback=default,
+    )
+)
+// resets the timestampCaughtUpToHeadOrEndblock after a restart when true
+let updateSyncTimeOnRestart =
+  envSafe->EnvSafe.get("UPDATE_SYNC_TIME_ON_RESTART", S.bool, ~fallback=true)
+let batchSize = envSafe->EnvSafe.get("MAX_BATCH_SIZE", S.option(S.int))
+let targetBufferSize = envSafe->EnvSafe.get("ENVIO_INDEXING_MAX_BUFFER_SIZE", S.option(S.int))
+let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fallback=5_000)
+let maxPartitionConcurrency =
+  envSafe->EnvSafe.get("ENVIO_MAX_PARTITION_CONCURRENCY", S.int, ~fallback=10)
+let indexingBlockLag = envSafe->EnvSafe.get("ENVIO_INDEXING_BLOCK_LAG", S.option(S.int))
+
+// FIXME: This broke HS grafana dashboard. Should investigate it later. Maybe we should use :: as a default value?
+// We want to be able to set it to 0.0.0.0
+// to allow to passthrough the port from a Docker container
+// let serverHost = envSafe->EnvSafe.get("ENVIO_INDEXER_HOST", S.string, ~fallback="localhost")
+let serverPort =
+  envSafe->EnvSafe.get(
+    "ENVIO_INDEXER_PORT",
+    S.int->S.port,
+    ~fallback=envSafe->EnvSafe.get("METRICS_PORT", S.int->S.port, ~fallback=9898),
+  )
+
+let tuiOffEnvVar = envSafe->EnvSafe.get("TUI_OFF", S.bool, ~fallback=false)
+
+let logFilePath = envSafe->EnvSafe.get("LOG_FILE", S.string, ~fallback="logs/envio.log")
+let userLogLevel = getLogLevelConfig("LOG_LEVEL", ~default=#info)
+let defaultFileLogLevel = getLogLevelConfig("FILE_LOG_LEVEL", ~default=#trace)
+
+let prodEnvioAppUrl = "https://envio.dev"
+let envioAppUrl = envSafe->EnvSafe.get("ENVIO_APP", S.string, ~fallback=prodEnvioAppUrl)
+let envioApiToken = envSafe->EnvSafe.get("ENVIO_API_TOKEN", S.option(S.string))
+let hyperSyncClientTimeoutMillis =
+  envSafe->EnvSafe.get("ENVIO_HYPERSYNC_CLIENT_TIMEOUT_MILLIS", S.int, ~fallback=120_000)
+
+/** 
+This is the number of retries that the binary client makes before rejecting the promise with an error 
+Default is 0 so that the indexer can handle retries internally
+*/
+let hyperSyncClientMaxRetries =
+  envSafe->EnvSafe.get("ENVIO_HYPERSYNC_CLIENT_MAX_RETRIES", S.int, ~fallback=0)
+
+let hypersyncClientSerializationFormat =
+  envSafe->EnvSafe.get(
+    "ENVIO_HYPERSYNC_CLIENT_SERIALIZATION_FORMAT",
+    HyperSyncClient.serializationFormatSchema,
+    ~fallback=CapnProto,
+  )
+
+let hypersyncClientEnableQueryCaching =
+  envSafe->EnvSafe.get("ENVIO_HYPERSYNC_CLIENT_ENABLE_QUERY_CACHING", S.bool, ~fallback=true)
+
+module Benchmark = {
+  module SaveDataStrategy: {
+    type t
+    let schema: S.t<t>
+    let default: t
+    let shouldSaveJsonFile: t => bool
+    let shouldSavePrometheus: t => bool
+    let shouldSaveData: t => bool
+  } = {
+    @unboxed
+    type t = Bool(bool) | @as("json-file") JsonFile | @as("prometheus") Prometheus
+
+    let schema = S.enum([Bool(true), Bool(false), JsonFile, Prometheus])
+    let default = Bool(false)
+
+    let shouldSaveJsonFile = self =>
+      switch self {
+      | JsonFile | Bool(true) => true
+      | _ => false
+      }
+
+    let shouldSavePrometheus = _ => true
+
+    let shouldSaveData = self => self->shouldSavePrometheus || self->shouldSaveJsonFile
+  }
+
+  let saveDataStrategy =
+    envSafe->EnvSafe.get(
+      "ENVIO_SAVE_BENCHMARK_DATA",
+      SaveDataStrategy.schema,
+      ~fallback=SaveDataStrategy.default,
+    )
+
+  let shouldSaveData = saveDataStrategy->SaveDataStrategy.shouldSaveData
+
+  /**
+  StdDev involves saving sum of squares of data points, which could get very large.
+
+  Currently only do this for local runs on json-file and not prometheus.
+  */
+  let shouldSaveStdDev =
+    saveDataStrategy->SaveDataStrategy.shouldSaveJsonFile
+}
+
+let logStrategy =
+  envSafe->EnvSafe.get(
+    "LOG_STRATEGY",
+    S.enum([
+      Logging.EcsFile,
+      EcsConsole,
+      EcsConsoleMultistream,
+      FileOnly,
+      ConsoleRaw,
+      ConsolePretty,
+      Both,
+    ]),
+    ~fallback=ConsolePretty,
+  )
+
+Logging.setLogger(
+  Logging.makeLogger(~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLevel),
+)
+
+module Db = {
+  let host = envSafe->EnvSafe.get("ENVIO_PG_HOST", S.string, ~devFallback="localhost")
+  let port = envSafe->EnvSafe.get("ENVIO_PG_PORT", S.int->S.port, ~devFallback=5433)
+  let user = envSafe->EnvSafe.get("ENVIO_PG_USER", S.string, ~devFallback="postgres")
+  let password = envSafe->EnvSafe.get(
+    "ENVIO_PG_PASSWORD",
+    S.string,
+    ~fallback={
+      envSafe->EnvSafe.get("ENVIO_POSTGRES_PASSWORD", S.string, ~fallback="testing")
+    },
+  )
+  let database = envSafe->EnvSafe.get("ENVIO_PG_DATABASE", S.string, ~devFallback="envio-dev")
+  let publicSchema = envSafe->EnvSafe.get("ENVIO_PG_PUBLIC_SCHEMA", S.string, ~fallback="public")
+  let ssl = envSafe->EnvSafe.get(
+    "ENVIO_PG_SSL_MODE",
+    Postgres.sslOptionsSchema,
+    //this is a dev fallback option for local deployments, shouldn't run in the prod env
+    //the SSL modes should be provided as string otherwise as 'require' | 'allow' | 'prefer' | 'verify-full'
+    ~devFallback=Bool(false),
+  )
+}
+
+module ClickHouseSink = {
+  let host = envSafe->EnvSafe.get("ENVIO_CLICKHOUSE_SINK_HOST", S.option(S.string))
+  let database = envSafe->EnvSafe.get("ENVIO_CLICKHOUSE_SINK_DATABASE", S.option(S.string))
+  let username = switch host {
+  | None => ""
+  | Some(_) => envSafe->EnvSafe.get("ENVIO_CLICKHOUSE_SINK_USERNAME", S.string)
+  }
+  let password = switch host {
+  | None => ""
+  | Some(_) => envSafe->EnvSafe.get("ENVIO_CLICKHOUSE_SINK_PASSWORD", S.string)
+  }
+}
+
+module Hasura = {
+  // Disable it on HS indexer run, since we don't have Hasura credentials anyways
+  // Also, it might be useful for some users who don't care about Hasura
+  let enabled = envSafe->EnvSafe.get("ENVIO_HASURA", S.bool, ~fallback=true)
+
+  let responseLimit = switch envSafe->EnvSafe.get("ENVIO_HASURA_RESPONSE_LIMIT", S.option(S.int)) {
+  | Some(_) as s => s
+  | None => envSafe->EnvSafe.get("HASURA_RESPONSE_LIMIT", S.option(S.int))
+  }
+
+  let graphqlEndpoint =
+    envSafe->EnvSafe.get(
+      "HASURA_GRAPHQL_ENDPOINT",
+      S.string,
+      ~devFallback="http://localhost:8080/v1/metadata",
+    )
+
+  let url = graphqlEndpoint->Js.String2.slice(~from=0, ~to_=-("/v1/metadata"->Js.String2.length))
+
+  let role = envSafe->EnvSafe.get("HASURA_GRAPHQL_ROLE", S.string, ~devFallback="admin")
+
+  let secret = envSafe->EnvSafe.get("HASURA_GRAPHQL_ADMIN_SECRET", S.string, ~devFallback="testing")
+
+  let aggregateEntities = envSafe->EnvSafe.get(
+    "ENVIO_HASURA_PUBLIC_AGGREGATE",
+    S.union([
+      S.array(S.string),
+      // Temporary workaround: Hosted Service can't use commas in env vars for multiple entities.
+      // Will be removed once comma support is added — don't rely on this.
+      S.string->S.transform(s => {
+        parser: string =>
+          switch string->Js.String2.split("&") {
+          | []
+          | [_] =>
+            s.fail(`Provide an array of entities in the JSON format.`)
+          | entities => entities
+          },
+      }),
+    ]),
+    ~fallback=[],
+  )
+}
+
+module Configurable = {
+  module SyncConfig = {
+    let initialBlockInterval =
+      envSafe->EnvSafe.get("ENVIO_RPC_INITIAL_BLOCK_INTERVAL", S.option(S.int))
+    let backoffMultiplicative =
+      envSafe->EnvSafe.get("ENVIO_RPC_BACKOFF_MULTIPLICATIVE", S.option(S.float))
+    let accelerationAdditive =
+      envSafe->EnvSafe.get("ENVIO_RPC_ACCELERATION_ADDITIVE", S.option(S.int))
+    let intervalCeiling = envSafe->EnvSafe.get("ENVIO_RPC_INTERVAL_CEILING", S.option(S.int))
+  }
+}
+
+module ThrottleWrites = {
+  let chainMetadataIntervalMillis =
+    envSafe->EnvSafe.get("ENVIO_THROTTLE_CHAIN_METADATA_INTERVAL_MILLIS", S.int, ~devFallback=500)
+  let pruneStaleDataIntervalMillis =
+    envSafe->EnvSafe.get(
+      "ENVIO_THROTTLE_PRUNE_STALE_DATA_INTERVAL_MILLIS",
+      S.int,
+      ~devFallback=30_000,
+    )
+
+  let liveMetricsBenchmarkIntervalMillis =
+    envSafe->EnvSafe.get(
+      "ENVIO_THROTTLE_LIVE_METRICS_BENCHMARK_INTERVAL_MILLIS",
+      S.int,
+      ~devFallback=1_000,
+    )
+
+  let jsonFileBenchmarkIntervalMillis =
+    envSafe->EnvSafe.get(
+      "ENVIO_THROTTLE_JSON_FILE_BENCHMARK_INTERVAL_MILLIS",
+      S.int,
+      ~devFallback=500,
+    )
+}
+
+// You need to close the envSafe after you're done with it so that it immediately tells you about your  misconfigured environment on startup.
+envSafe->EnvSafe.close