envio 2.17.0 → 2.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.17.0",
+  "version": "v2.18.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "types": "index.d.ts",
@@ -24,15 +24,15 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.17.0",
-    "envio-linux-arm64": "v2.17.0",
-    "envio-darwin-x64": "v2.17.0",
-    "envio-darwin-arm64": "v2.17.0"
+    "envio-linux-x64": "v2.18.0",
+    "envio-linux-arm64": "v2.18.0",
+    "envio-darwin-x64": "v2.18.0",
+    "envio-darwin-arm64": "v2.18.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.3",
     "rescript": "11.1.3",
-    "rescript-schema": "9.1.0",
+    "rescript-schema": "9.3.0",
     "viem": "2.21.0"
   },
   "files": [

package/src/ContractAddressingMap.res

@@ -0,0 +1,115 @@
+type contractName = string
+
+// Currently this mapping append only, so we don't need to worry about
+// protecting static addresses from de-registration.
+
+type mapping = {
+  nameByAddress: dict<contractName>,
+  addressesByName: dict<Belt.Set.String.t>,
+}
+
+exception AddressRegisteredForMultipleContracts({address: Address.t, names: array<contractName>})
+
+let addAddress = (map: mapping, ~name: string, ~address: Address.t) => {
+  switch map.nameByAddress->Utils.Dict.dangerouslyGetNonOption(address->Address.toString) {
+  | Some(currentName) if currentName != name =>
+    let logger = Logging.createChild(
+      ~params={
+        "address": address->Address.toString,
+        "existingContract": currentName,
+        "newContract": name,
+      },
+    )
+    AddressRegisteredForMultipleContracts({
+      address,
+      names: [currentName, name],
+    })->ErrorHandling.mkLogAndRaise(~msg="Address registered for multiple contracts", ~logger)
+  | _ => ()
+  }
+  map.nameByAddress->Js.Dict.set(address->Address.toString, name)
+
+  let oldAddresses =
+    map.addressesByName
+    ->Utils.Dict.dangerouslyGetNonOption(name)
+    ->Belt.Option.getWithDefault(Belt.Set.String.empty)
+  let newAddresses = oldAddresses->Belt.Set.String.add(address->Address.toString)
+  map.addressesByName->Js.Dict.set(name, newAddresses)
+}
+
+let getAddresses = (map: mapping, name: string) => {
+  map.addressesByName->Utils.Dict.dangerouslyGetNonOption(name)
+}
+
+let getName = (map: mapping, address: string) => {
+  map.nameByAddress->Utils.Dict.dangerouslyGetNonOption(address)
+}
+
+let make = () => {
+  nameByAddress: Js.Dict.empty(),
+  addressesByName: Js.Dict.empty(),
+}
+
+let getContractNameFromAddress = (mapping, ~contractAddress: Address.t): option<contractName> => {
+  mapping->getName(contractAddress->Address.toString)
+}
+
+let stringsToAddresses: array<string> => array<Address.t> = Utils.magic
+let keyValStringToAddress: array<(string, string)> => array<(Address.t, string)> = Utils.magic
+
+let getAddressesFromContractName = (mapping, ~contractName) => {
+  switch mapping->getAddresses(contractName) {
+  | Some(addresses) => addresses
+  | None => Belt.Set.String.empty
+  }
+  ->Belt.Set.String.toArray
+  ->stringsToAddresses
+}
+
+let getAllAddresses = (mapping: mapping) => {
+  mapping.nameByAddress->Js.Dict.keys->stringsToAddresses
+}
+
+let copy = (mapping: mapping) => {
+  {
+    nameByAddress: mapping.nameByAddress->Utils.Dict.shallowCopy,
+    // Since Belt.Set.String.t is immutable, we can simply do shallow copy here
+    addressesByName: mapping.addressesByName->Utils.Dict.shallowCopy,
+  }
+}
+
+let mergeInPlace = (map, ~target) => {
+  map.nameByAddress
+  ->Js.Dict.keys
+  ->Belt.Array.forEach(addr => {
+    let name = map.nameByAddress->Js.Dict.unsafeGet(addr)
+    target->addAddress(~address=addr->Address.unsafeFromString, ~name)
+  })
+}
+
+let fromArray = (nameAddrTuples: array<(Address.t, string)>) => {
+  let m = make()
+  nameAddrTuples->Belt.Array.forEach(((address, name)) => m->addAddress(~name, ~address))
+  m
+}
+
+/**
+Creates a new mapping from the previous without the addresses passed in as "addressesToRemove"
+*/
+let removeAddresses = (mapping: mapping, ~addressesToRemove: array<Address.t>) => {
+  switch addressesToRemove {
+  | [] => mapping
+  | _ =>
+    mapping.nameByAddress
+    ->Js.Dict.entries
+    ->Belt.Array.keep(((addr, _name)) => {
+      let shouldRemove = addressesToRemove->Utils.Array.includes(addr->Utils.magic)
+      !shouldRemove
+    })
+    ->keyValStringToAddress
+    ->fromArray
+  }
+}
+
+let addressCount = (mapping: mapping) => mapping.nameByAddress->Js.Dict.keys->Belt.Array.length
+
+let isEmpty = (mapping: mapping) => mapping->addressCount == 0

package/src/ErrorHandling.res

@@ -0,0 +1,61 @@
+type t = {logger: Pino.t, exn: exn, msg: option<string>}
+
+let make = (exn, ~logger=Logging.getLogger(), ~msg=?) => {
+  {logger, msg, exn}
+}
+
+let log = (self: t) => {
+  switch self {
+  | {exn, msg: Some(msg), logger} =>
+    logger->Logging.childErrorWithExn(exn->Internal.prettifyExn, msg)
+  | {exn, msg: None, logger} => logger->Logging.childError(exn->Internal.prettifyExn)
+  }
+}
+
+let raiseExn = (self: t) => {
+  self.exn->Internal.prettifyExn->raise
+}
+
+let mkLogAndRaise = (~logger=?, ~msg=?, exn) => {
+  let exn = exn->Internal.prettifyExn
+  exn->make(~logger?, ~msg?)->log
+  exn->raise
+}
+
+let unwrapLogAndRaise = (~logger=?, ~msg=?, result) => {
+  switch result {
+  | Ok(v) => v
+  | Error(exn) => exn->mkLogAndRaise(~logger?, ~msg?)
+  }
+}
+
+let logAndRaise = self => {
+  self->log
+  self->raiseExn
+}
+
+/**
+An environment to manage control flow propogating results 
+with Error that contain ErrorHandling.t in async
+contexts and avoid nested switch statements on awaited promises
+Similar to rust result propogation
+*/
+module ResultPropogateEnv = {
+  exception ErrorHandlingEarlyReturn(t)
+
+  type resultWithErrorHandle<'a> = result<'a, t>
+  type asyncBody<'a> = unit => promise<resultWithErrorHandle<'a>>
+
+  let runAsyncEnv = async (body: asyncBody<'a>) => {
+    switch await body() {
+    | exception ErrorHandlingEarlyReturn(e) => Error(e)
+    | endReturn => endReturn
+    }
+  }
+
+  let propogate = (res: resultWithErrorHandle<'a>) =>
+    switch res {
+    | Ok(v) => v
+    | Error(e) => raise(ErrorHandlingEarlyReturn(e))
+    }
+}

package/src/EventUtils.res

@@ -0,0 +1,87 @@
+type multiChainEventIndex = {
+  timestamp: int,
+  chainId: int,
+  blockNumber: int,
+  logIndex: int,
+}
+
+//Comparator used when ordering multichain events
+let getEventComparator = (multiChainEventIndex: multiChainEventIndex) => {
+  let {timestamp, chainId, blockNumber, logIndex} = multiChainEventIndex
+  (timestamp, chainId, blockNumber, logIndex)
+}
+
+let getEventComparatorFromQueueItem = (
+  {chain, timestamp, blockNumber, logIndex}: Internal.eventItem,
+) => {
+  let chainId = chain->ChainMap.Chain.toChainId
+  (timestamp, chainId, blockNumber, logIndex)
+}
+
+//Function used to determine if one event is earlier than another
+let isEarlierEvent = (event1: multiChainEventIndex, event2: multiChainEventIndex) => {
+  event1->getEventComparator < event2->getEventComparator
+}
+
+type eventIndex = {
+  blockNumber: int,
+  logIndex: int,
+}
+
+// takes blockNumber, logIndex and packs them into a number with
+//32 bits, 16 bits and 16 bits respectively
+let packEventIndex = (~blockNumber, ~logIndex) => {
+  let blockNumber = blockNumber->BigInt.fromInt
+  let logIndex = logIndex->BigInt.fromInt
+  let blockNumber = BigInt.Bitwise.shift_left(blockNumber, 16->BigInt.fromInt)
+
+  blockNumber->BigInt.Bitwise.logor(logIndex)
+}
+
+//Currently not used but keeping in utils
+//using @live flag for dead code analyser
+@live
+let packMultiChainEventIndex = (~timestamp, ~chainId, ~blockNumber, ~logIndex) => {
+  let timestamp = timestamp->BigInt.fromInt
+  let chainId = chainId->BigInt.fromInt
+  let blockNumber = blockNumber->BigInt.fromInt
+  let logIndex = logIndex->BigInt.fromInt
+
+  let timestamp = BigInt.Bitwise.shift_left(timestamp, 48->BigInt.fromInt)
+  let chainId = BigInt.Bitwise.shift_left(chainId, 16->BigInt.fromInt)
+  let blockNumber = BigInt.Bitwise.shift_left(blockNumber, 16->BigInt.fromInt)
+
+  timestamp
+  ->BigInt.Bitwise.logor(chainId)
+  ->BigInt.Bitwise.logor(blockNumber)
+  ->BigInt.Bitwise.logor(logIndex)
+}
+
+//Currently not used but keeping in utils
+//using @live flag for dead code analyser
+@live
+let unpackEventIndex = (packedEventIndex: bigint) => {
+  let blockNumber = packedEventIndex->BigInt.Bitwise.shift_right(16->BigInt.fromInt)
+  let logIndexMask = 65535->BigInt.fromInt
+  let logIndex = packedEventIndex->BigInt.Bitwise.logand(logIndexMask)
+  {
+    blockNumber: blockNumber->BigInt.toString->Belt.Int.fromString->Belt.Option.getUnsafe,
+    logIndex: logIndex->BigInt.toString->Belt.Int.fromString->Belt.Option.getUnsafe,
+  }
+}
+
+//takes an eventIndex record and returnts a packed event index
+//used in TS tests
+@live
+let packEventIndexFromRecord = (eventIndex: eventIndex) => {
+  packEventIndex(~blockNumber=eventIndex.blockNumber, ~logIndex=eventIndex.logIndex)
+}
+
+//Returns unique string id for an event using its chain id combined with event id
+//Used in IO for the key in the in mem rawEvents table
+let getEventIdKeyString = (~chainId: int, ~eventId: string) => {
+  let chainIdStr = chainId->Belt.Int.toString
+  let key = chainIdStr ++ "_" ++ eventId
+
+  key
+}

package/src/Internal.res

@@ -134,6 +134,8 @@ type eventItem = {
   //be reprocessed after it has loaded dynamic contracts
   //This gets set to true and does not try and reload events
   hasRegisteredDynamicContracts?: bool,
+  // Reuse logger object for event
+  mutable loggerCache?: Pino.t,
 }
 
 @genType

package/src/LoadManager.res

@@ -0,0 +1,174 @@
+open Belt
+
+module Call = {
+  type input
+  type output
+  type t = {
+    input: input,
+    resolve: output => unit,
+    reject: exn => unit,
+    mutable promise: promise<output>,
+    mutable isLoading: bool,
+  }
+}
+
+module Group = {
+  type t = {
+    // Unique calls by input as a key
+    calls: dict<Call.t>,
+    load: array<Call.input> => promise<unit>,
+    getUnsafeInMemory: string => Call.output,
+    hasInMemory: string => bool,
+  }
+}
+
+type t = {
+  // Batches of different operations by operation key
+  // Can be: Load by id, load by index, effect
+  groups: dict<Group.t>,
+  mutable isCollecting: bool,
+}
+
+let make = () => {
+  groups: Js.Dict.empty(),
+  isCollecting: false,
+}
+
+let schedule = async loadManager => {
+  // For the first schedule, wait for a microtask first
+  // to collect all calls before the next await
+  // If the loadManager is already collecting,
+  // then we do nothing. The call will be automatically
+  // handled when the promise below resolves
+  loadManager.isCollecting = true
+  await Promise.resolve()
+  loadManager.isCollecting = false
+
+  let groups = loadManager.groups
+  groups
+  ->Js.Dict.keys
+  ->Utils.Array.forEachAsync(async key => {
+    let group = groups->Js.Dict.unsafeGet(key)
+    let calls = group.calls
+
+    let inputsToLoad = []
+    let currentInputKeys = []
+    calls
+    ->Js.Dict.keys
+    ->Js.Array2.forEach(inputKey => {
+      let call = calls->Js.Dict.unsafeGet(inputKey)
+      if !call.isLoading {
+        call.isLoading = true
+        currentInputKeys->Js.Array2.push(inputKey)->ignore
+        if group.hasInMemory(inputKey)->not {
+          inputsToLoad->Js.Array2.push(call.input)->ignore
+        }
+      }
+    })
+
+    if inputsToLoad->Utils.Array.isEmpty->not {
+      try {
+        await group.load(inputsToLoad)
+      } catch {
+      | exn => {
+          let exn = exn->Internal.prettifyExn
+          currentInputKeys->Array.forEach(inputKey => {
+            let call = calls->Js.Dict.unsafeGet(inputKey)
+            call.reject(exn)
+          })
+        }
+      }
+    }
+
+    if currentInputKeys->Utils.Array.isEmpty->not {
+      currentInputKeys->Js.Array2.forEach(inputKey => {
+        let call = calls->Js.Dict.unsafeGet(inputKey)
+        calls->Utils.Dict.deleteInPlace(inputKey)
+        call.resolve(group.getUnsafeInMemory(inputKey))
+      })
+
+      // Clean up executed batch to reset
+      // provided load function which
+      // might have an outdated function context
+      let latestGroup = groups->Js.Dict.unsafeGet(key)
+      if latestGroup.calls->Js.Dict.keys->Utils.Array.isEmpty {
+        groups->Utils.Dict.deleteInPlace(key)
+      }
+    }
+  })
+}
+
+let noopHasher = input => input->(Utils.magic: 'input => string)
+
+let call = (
+  loadManager,
+  ~input,
+  ~key,
+  ~load,
+  ~hasher,
+  ~shouldGroup,
+  ~hasInMemory,
+  ~getUnsafeInMemory,
+) => {
+  // This is a micro-optimization to avoid a function call
+  let inputKey = hasher === noopHasher ? input->(Utils.magic: 'input => string) : hasher(input)
+
+  // We group external calls by operation to:
+  // 1. Reduce the IO by allowing batch requests
+  // 2. By allowing parallel processing of events
+  //    and make awaits run at the same time
+  //
+  // In the handlers it's not as important to group
+  // calls, because usually we run a single handler at a time
+  // So have a quick exit when an entity is already in memory
+  //
+  // But since we're going to parallelize handlers per chain,
+  // keep the grouping logic when the data needs to be loaded
+  // It has a small additional runtime cost, but might reduce IO time
+  if !shouldGroup && hasInMemory(inputKey) {
+    getUnsafeInMemory(inputKey)->Promise.resolve
+  } else {
+    let group = switch loadManager.groups->Utils.Dict.dangerouslyGetNonOption(key) {
+    | Some(group) => group
+    | None => {
+        let g: Group.t = {
+          calls: Js.Dict.empty(),
+          load: load->(
+            Utils.magic: (array<'input> => promise<unit>) => array<Call.input> => promise<unit>
+          ),
+          getUnsafeInMemory: getUnsafeInMemory->(
+            Utils.magic: (string => 'output) => string => Call.output
+          ),
+          hasInMemory: hasInMemory->(Utils.magic: (string => bool) => string => bool),
+        }
+        loadManager.groups->Js.Dict.set(key, g)
+        g
+      }
+    }
+
+    switch group.calls->Utils.Dict.dangerouslyGetNonOption(inputKey) {
+    | Some(c) => c.promise
+    | None => {
+        let promise = Promise.make((resolve, reject) => {
+          let call: Call.t = {
+            input: input->(Utils.magic: 'input => Call.input),
+            resolve,
+            reject,
+            promise: %raw(`null`),
+            isLoading: false,
+          }
+          group.calls->Js.Dict.set(inputKey, call)
+        })
+
+        // Don't use ref since it'll allocate an object to store .contents
+        (group.calls->Js.Dict.unsafeGet(inputKey)).promise = promise
+
+        if !loadManager.isCollecting {
+          let _: promise<unit> = loadManager->schedule
+        }
+
+        promise
+      }
+    }->(Utils.magic: promise<Call.output> => promise<'output>)
+  }
+}

package/src/Logging.res

@@ -0,0 +1,179 @@
+open Pino
+
+type logStrategyType =
+  | @as("ecs-file") EcsFile
+  | @as("ecs-console") EcsConsole
+  | @as("ecs-console-multistream") EcsConsoleMultistream
+  | @as("file-only") FileOnly
+  | @as("console-raw") ConsoleRaw
+  | @as("console-pretty") ConsolePretty
+  | @as("both-prettyconsole") Both
+
+let logLevels = [
+  // custom levels
+  ("udebug", 32),
+  ("uinfo", 34),
+  ("uwarn", 36),
+  ("uerror", 38),
+  // Default levels
+  ("trace", 10),
+  ("debug", 20),
+  ("info", 30),
+  ("warn", 40),
+  ("error", 50),
+  ("fatal", 60),
+]->Js.Dict.fromArray
+
+%%private(let logger = ref(None))
+
+let setLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLevel) => {
+  // Currently unused - useful if using multiple transports.
+  // let pinoRaw = {"target": "pino/file", "level": Config.userLogLevel}
+  let pinoFile: Transport.transportTarget = {
+    target: "pino/file",
+    options: {
+      "destination": logFilePath,
+      "append": true,
+      "mkdir": true,
+    }->Transport.makeTransportOptions,
+    level: defaultFileLogLevel,
+  }
+
+  let makeMultiStreamLogger =
+    MultiStreamLogger.make(~userLogLevel, ~defaultFileLogLevel, ~customLevels=logLevels, ...)
+
+  logger :=
+    Some(
+      switch logStrategy {
+      | EcsFile =>
+        makeWithOptionsAndTransport(
+          {
+            ...Pino.ECS.make(),
+            customLevels: logLevels,
+          },
+          Transport.make(pinoFile),
+        )
+      | EcsConsoleMultistream =>
+        makeMultiStreamLogger(~logFile=None, ~options=Some(Pino.ECS.make()))
+      | EcsConsole =>
+        make({
+          ...Pino.ECS.make(),
+          level: userLogLevel,
+          customLevels: logLevels,
+        })
+      | FileOnly =>
+        makeWithOptionsAndTransport(
+          {
+            customLevels: logLevels,
+            level: defaultFileLogLevel,
+          },
+          Transport.make(pinoFile),
+        )
+      | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=None)
+      | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=None)
+      | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=None)
+      },
+    )
+}
+
+let getLogger = () => {
+  switch logger.contents {
+  | Some(logger) => logger
+  | None => Js.Exn.raiseError("Unreachable code. Logger not initialized")
+  }
+}
+
+let setLogLevel = (level: Pino.logLevel) => {
+  getLogger()->setLevel(level)
+}
+
+let trace = message => {
+  getLogger().trace(message->createPinoMessage)
+}
+
+let debug = message => {
+  getLogger().debug(message->createPinoMessage)
+}
+
+let info = message => {
+  getLogger().info(message->createPinoMessage)
+}
+
+let warn = message => {
+  getLogger().warn(message->createPinoMessage)
+}
+
+let error = message => {
+  getLogger().error(message->createPinoMessage)
+}
+let errorWithExn = (error, message) => {
+  getLogger()->Pino.errorExn(message->createPinoMessageWithError(error))
+}
+
+let fatal = message => {
+  getLogger().fatal(message->createPinoMessage)
+}
+
+let childTrace = (logger, params: 'a) => {
+  logger.trace(params->createPinoMessage)
+}
+let childDebug = (logger, params: 'a) => {
+  logger.debug(params->createPinoMessage)
+}
+let childInfo = (logger, params: 'a) => {
+  logger.info(params->createPinoMessage)
+}
+let childWarn = (logger, params: 'a) => {
+  logger.warn(params->createPinoMessage)
+}
+let childError = (logger, params: 'a) => {
+  logger.error(params->createPinoMessage)
+}
+let childErrorWithExn = (logger, error, params: 'a) => {
+  logger->Pino.errorExn(params->createPinoMessageWithError(error))
+}
+
+let childFatal = (logger, params: 'a) => {
+  logger.fatal(params->createPinoMessage)
+}
+
+let createChild = (~params: 'a) => {
+  getLogger()->child(params->createChildParams)
+}
+let createChildFrom = (~logger: t, ~params: 'a) => {
+  logger->child(params->createChildParams)
+}
+
+let getEventLogger = (eventItem: Internal.eventItem) => {
+  switch eventItem.loggerCache {
+  | Some(l) => l
+  | None => {
+      let l = getLogger()->child(
+        {
+          "context": `Event '${eventItem.eventConfig.name}' for contract '${eventItem.eventConfig.contractName}'`,
+          "chainId": eventItem.chain->ChainMap.Chain.toChainId,
+          "block": eventItem.blockNumber,
+          "logIndex": eventItem.logIndex,
+        }->createChildParams,
+      )
+      eventItem.loggerCache = Some(l)
+      l
+    }
+  }
+}
+
+let getUserLogger = {
+  @inline
+  let log = (eventItem, level: Pino.logLevelUser, message: string, ~params) => {
+    (eventItem->getEventLogger->Utils.magic->Js.Dict.unsafeGet((level :> string)))(params, message)
+  }
+
+  (eventItem): Envio.logger => {
+    info: (message: string, ~params=?) => eventItem->log(#uinfo, message, ~params),
+    debug: (message: string, ~params=?) => eventItem->log(#udebug, message, ~params),
+    warn: (message: string, ~params=?) => eventItem->log(#uwarn, message, ~params),
+    error: (message: string, ~params=?) => eventItem->log(#uerror, message, ~params),
+    errorWithExn: (message: string, exn) =>
+      eventItem->log(#uerror, message, ~params={"err": exn->Internal.prettifyExn}),
+  }
+}