envio 2.16.1 → 2.17.1

package/package.json

@@ -1,8 +1,9 @@
 {
   "name": "envio",
-  "version": "v2.16.1",
+  "version": "v2.17.1",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
+  "types": "index.d.ts",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/enviodev/hyperindex.git"
@@ -23,10 +24,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.16.1",
-    "envio-linux-arm64": "v2.16.1",
-    "envio-darwin-x64": "v2.16.1",
-    "envio-darwin-arm64": "v2.16.1"
+    "envio-linux-x64": "v2.17.1",
+    "envio-linux-arm64": "v2.17.1",
+    "envio-darwin-x64": "v2.17.1",
+    "envio-darwin-arm64": "v2.17.1"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.3",

package/src/Address.gen.ts

@@ -3,6 +3,6 @@
 /* eslint-disable */
 /* tslint:disable */
 
-import type {Address as $$t} from './bindings/OpaqueTypes.ts';
+import type {Address as $$t} from './Types.ts';
 
 export type t = $$t;

package/src/Address.res

@@ -1,4 +1,4 @@
-@genType.import(("./bindings/OpaqueTypes.ts", "Address"))
+@genType.import(("./Types.ts", "Address"))
 type t
 
 let schema = S.string->S.setName("Address")->(Utils.magic: S.t<string> => S.t<t>)

package/src/ContractAddressingMap.res

@@ -0,0 +1,115 @@
+type contractName = string
+
+// Currently this mapping append only, so we don't need to worry about
+// protecting static addresses from de-registration.
+
+type mapping = {
+  nameByAddress: dict<contractName>,
+  addressesByName: dict<Belt.Set.String.t>,
+}
+
+exception AddressRegisteredForMultipleContracts({address: Address.t, names: array<contractName>})
+
+let addAddress = (map: mapping, ~name: string, ~address: Address.t) => {
+  switch map.nameByAddress->Utils.Dict.dangerouslyGetNonOption(address->Address.toString) {
+  | Some(currentName) if currentName != name =>
+    let logger = Logging.createChild(
+      ~params={
+        "address": address->Address.toString,
+        "existingContract": currentName,
+        "newContract": name,
+      },
+    )
+    AddressRegisteredForMultipleContracts({
+      address,
+      names: [currentName, name],
+    })->ErrorHandling.mkLogAndRaise(~msg="Address registered for multiple contracts", ~logger)
+  | _ => ()
+  }
+  map.nameByAddress->Js.Dict.set(address->Address.toString, name)
+
+  let oldAddresses =
+    map.addressesByName
+    ->Utils.Dict.dangerouslyGetNonOption(name)
+    ->Belt.Option.getWithDefault(Belt.Set.String.empty)
+  let newAddresses = oldAddresses->Belt.Set.String.add(address->Address.toString)
+  map.addressesByName->Js.Dict.set(name, newAddresses)
+}
+
+let getAddresses = (map: mapping, name: string) => {
+  map.addressesByName->Utils.Dict.dangerouslyGetNonOption(name)
+}
+
+let getName = (map: mapping, address: string) => {
+  map.nameByAddress->Utils.Dict.dangerouslyGetNonOption(address)
+}
+
+let make = () => {
+  nameByAddress: Js.Dict.empty(),
+  addressesByName: Js.Dict.empty(),
+}
+
+let getContractNameFromAddress = (mapping, ~contractAddress: Address.t): option<contractName> => {
+  mapping->getName(contractAddress->Address.toString)
+}
+
+let stringsToAddresses: array<string> => array<Address.t> = Utils.magic
+let keyValStringToAddress: array<(string, string)> => array<(Address.t, string)> = Utils.magic
+
+let getAddressesFromContractName = (mapping, ~contractName) => {
+  switch mapping->getAddresses(contractName) {
+  | Some(addresses) => addresses
+  | None => Belt.Set.String.empty
+  }
+  ->Belt.Set.String.toArray
+  ->stringsToAddresses
+}
+
+let getAllAddresses = (mapping: mapping) => {
+  mapping.nameByAddress->Js.Dict.keys->stringsToAddresses
+}
+
+let copy = (mapping: mapping) => {
+  {
+    nameByAddress: mapping.nameByAddress->Utils.Dict.shallowCopy,
+    // Since Belt.Set.String.t is immutable, we can simply do shallow copy here
+    addressesByName: mapping.addressesByName->Utils.Dict.shallowCopy,
+  }
+}
+
+let mergeInPlace = (map, ~target) => {
+  map.nameByAddress
+  ->Js.Dict.keys
+  ->Belt.Array.forEach(addr => {
+    let name = map.nameByAddress->Js.Dict.unsafeGet(addr)
+    target->addAddress(~address=addr->Address.unsafeFromString, ~name)
+  })
+}
+
+let fromArray = (nameAddrTuples: array<(Address.t, string)>) => {
+  let m = make()
+  nameAddrTuples->Belt.Array.forEach(((address, name)) => m->addAddress(~name, ~address))
+  m
+}
+
+/**
+Creates a new mapping from the previous without the addresses passed in as "addressesToRemove"
+*/
+let removeAddresses = (mapping: mapping, ~addressesToRemove: array<Address.t>) => {
+  switch addressesToRemove {
+  | [] => mapping
+  | _ =>
+    mapping.nameByAddress
+    ->Js.Dict.entries
+    ->Belt.Array.keep(((addr, _name)) => {
+      let shouldRemove = addressesToRemove->Utils.Array.includes(addr->Utils.magic)
+      !shouldRemove
+    })
+    ->keyValStringToAddress
+    ->fromArray
+  }
+}
+
+let addressCount = (mapping: mapping) => mapping.nameByAddress->Js.Dict.keys->Belt.Array.length
+
+let isEmpty = (mapping: mapping) => mapping->addressCount == 0

package/src/Envio.gen.ts

@@ -0,0 +1,8 @@
+/* TypeScript file generated from Envio.res by genType. */
+
+/* eslint-disable */
+/* tslint:disable */
+
+import type {Logger as $$logger} from './Types.ts';
+
+export type logger = $$logger;

package/src/Envio.res

@@ -0,0 +1,12 @@
+// The file with public API.
+// Should be an entry point after we get rid of the generated project.
+// Don't forget to keep index.d.ts in sync with this file.
+
+@genType.import(("./Types.ts", "Logger"))
+type logger = {
+  debug: 'params. (string, ~params: {..} as 'params=?) => unit,
+  info: 'params. (string, ~params: {..} as 'params=?) => unit,
+  warn: 'params. (string, ~params: {..} as 'params=?) => unit,
+  error: 'params. (string, ~params: {..} as 'params=?) => unit,
+  errorWithExn: (string, exn) => unit,
+}

package/src/ErrorHandling.res

@@ -0,0 +1,61 @@
+type t = {logger: Pino.t, exn: exn, msg: option<string>}
+
+let make = (exn, ~logger=Logging.getLogger(), ~msg=?) => {
+  {logger, msg, exn}
+}
+
+let log = (self: t) => {
+  switch self {
+  | {exn, msg: Some(msg), logger} =>
+    logger->Logging.childErrorWithExn(exn->Internal.prettifyExn, msg)
+  | {exn, msg: None, logger} => logger->Logging.childError(exn->Internal.prettifyExn)
+  }
+}
+
+let raiseExn = (self: t) => {
+  self.exn->Internal.prettifyExn->raise
+}
+
+let mkLogAndRaise = (~logger=?, ~msg=?, exn) => {
+  let exn = exn->Internal.prettifyExn
+  exn->make(~logger?, ~msg?)->log
+  exn->raise
+}
+
+let unwrapLogAndRaise = (~logger=?, ~msg=?, result) => {
+  switch result {
+  | Ok(v) => v
+  | Error(exn) => exn->mkLogAndRaise(~logger?, ~msg?)
+  }
+}
+
+let logAndRaise = self => {
+  self->log
+  self->raiseExn
+}
+
+/**
+An environment to manage control flow propogating results 
+with Error that contain ErrorHandling.t in async
+contexts and avoid nested switch statements on awaited promises
+Similar to rust result propogation
+*/
+module ResultPropogateEnv = {
+  exception ErrorHandlingEarlyReturn(t)
+
+  type resultWithErrorHandle<'a> = result<'a, t>
+  type asyncBody<'a> = unit => promise<resultWithErrorHandle<'a>>
+
+  let runAsyncEnv = async (body: asyncBody<'a>) => {
+    switch await body() {
+    | exception ErrorHandlingEarlyReturn(e) => Error(e)
+    | endReturn => endReturn
+    }
+  }
+
+  let propogate = (res: resultWithErrorHandle<'a>) =>
+    switch res {
+    | Ok(v) => v
+    | Error(e) => raise(ErrorHandlingEarlyReturn(e))
+    }
+}

package/src/EventUtils.res

@@ -0,0 +1,87 @@
+type multiChainEventIndex = {
+  timestamp: int,
+  chainId: int,
+  blockNumber: int,
+  logIndex: int,
+}
+
+//Comparator used when ordering multichain events
+let getEventComparator = (multiChainEventIndex: multiChainEventIndex) => {
+  let {timestamp, chainId, blockNumber, logIndex} = multiChainEventIndex
+  (timestamp, chainId, blockNumber, logIndex)
+}
+
+let getEventComparatorFromQueueItem = (
+  {chain, timestamp, blockNumber, logIndex}: Internal.eventItem,
+) => {
+  let chainId = chain->ChainMap.Chain.toChainId
+  (timestamp, chainId, blockNumber, logIndex)
+}
+
+//Function used to determine if one event is earlier than another
+let isEarlierEvent = (event1: multiChainEventIndex, event2: multiChainEventIndex) => {
+  event1->getEventComparator < event2->getEventComparator
+}
+
+type eventIndex = {
+  blockNumber: int,
+  logIndex: int,
+}
+
+// takes blockNumber, logIndex and packs them into a number with
+//32 bits, 16 bits and 16 bits respectively
+let packEventIndex = (~blockNumber, ~logIndex) => {
+  let blockNumber = blockNumber->BigInt.fromInt
+  let logIndex = logIndex->BigInt.fromInt
+  let blockNumber = BigInt.Bitwise.shift_left(blockNumber, 16->BigInt.fromInt)
+
+  blockNumber->BigInt.Bitwise.logor(logIndex)
+}
+
+//Currently not used but keeping in utils
+//using @live flag for dead code analyser
+@live
+let packMultiChainEventIndex = (~timestamp, ~chainId, ~blockNumber, ~logIndex) => {
+  let timestamp = timestamp->BigInt.fromInt
+  let chainId = chainId->BigInt.fromInt
+  let blockNumber = blockNumber->BigInt.fromInt
+  let logIndex = logIndex->BigInt.fromInt
+
+  let timestamp = BigInt.Bitwise.shift_left(timestamp, 48->BigInt.fromInt)
+  let chainId = BigInt.Bitwise.shift_left(chainId, 16->BigInt.fromInt)
+  let blockNumber = BigInt.Bitwise.shift_left(blockNumber, 16->BigInt.fromInt)
+
+  timestamp
+  ->BigInt.Bitwise.logor(chainId)
+  ->BigInt.Bitwise.logor(blockNumber)
+  ->BigInt.Bitwise.logor(logIndex)
+}
+
+//Currently not used but keeping in utils
+//using @live flag for dead code analyser
+@live
+let unpackEventIndex = (packedEventIndex: bigint) => {
+  let blockNumber = packedEventIndex->BigInt.Bitwise.shift_right(16->BigInt.fromInt)
+  let logIndexMask = 65535->BigInt.fromInt
+  let logIndex = packedEventIndex->BigInt.Bitwise.logand(logIndexMask)
+  {
+    blockNumber: blockNumber->BigInt.toString->Belt.Int.fromString->Belt.Option.getUnsafe,
+    logIndex: logIndex->BigInt.toString->Belt.Int.fromString->Belt.Option.getUnsafe,
+  }
+}
+
+//takes an eventIndex record and returnts a packed event index
+//used in TS tests
+@live
+let packEventIndexFromRecord = (eventIndex: eventIndex) => {
+  packEventIndex(~blockNumber=eventIndex.blockNumber, ~logIndex=eventIndex.logIndex)
+}
+
+//Returns unique string id for an event using its chain id combined with event id
+//Used in IO for the key in the in mem rawEvents table
+let getEventIdKeyString = (~chainId: int, ~eventId: string) => {
+  let chainIdStr = chainId->Belt.Int.toString
+  let key = chainIdStr ++ "_" ++ eventId
+
+  key
+}

package/src/Internal.gen.ts

@@ -3,7 +3,7 @@
 /* eslint-disable */
 /* tslint:disable */
 
-import type {invalid as $$noEventFilters} from './bindings/OpaqueTypes.ts';
+import type {Invalid as $$noEventFilters} from './Types.ts';
 
 import type {t as Address_t} from './Address.gen';
 

package/src/Internal.res

@@ -134,6 +134,8 @@ type eventItem = {
   //be reprocessed after it has loaded dynamic contracts
   //This gets set to true and does not try and reload events
   hasRegisteredDynamicContracts?: bool,
+  // Reuse logger object for event
+  mutable loggerCache?: Pino.t,
 }
 
 @genType
@@ -159,5 +161,12 @@ let fuelTransferParamsSchema = S.schema(s => {
 
 type entity = private {id: string}
 
-@genType.import(("./bindings/OpaqueTypes.ts", "invalid"))
+@genType.import(("./Types.ts", "Invalid"))
 type noEventFilters
+
+let prettifyExn = exn => {
+  switch exn->Js.Exn.anyToExnInternal {
+  | Js.Exn.Error(e) => e->(Utils.magic: Js.Exn.t => exn)
+  | exn => exn
+  }
+}

package/src/LoadManager.res

@@ -0,0 +1,174 @@
+open Belt
+
+module Call = {
+  type input
+  type output
+  type t = {
+    input: input,
+    resolve: output => unit,
+    reject: exn => unit,
+    mutable promise: promise<output>,
+    mutable isLoading: bool,
+  }
+}
+
+module Group = {
+  type t = {
+    // Unique calls by input as a key
+    calls: dict<Call.t>,
+    load: array<Call.input> => promise<unit>,
+    getUnsafeInMemory: string => Call.output,
+    hasInMemory: string => bool,
+  }
+}
+
+type t = {
+  // Batches of different operations by operation key
+  // Can be: Load by id, load by index, effect
+  groups: dict<Group.t>,
+  mutable isCollecting: bool,
+}
+
+let make = () => {
+  groups: Js.Dict.empty(),
+  isCollecting: false,
+}
+
+let schedule = async loadManager => {
+  // For the first schedule, wait for a microtask first
+  // to collect all calls before the next await
+  // If the loadManager is already collecting,
+  // then we do nothing. The call will be automatically
+  // handled when the promise below resolves
+  loadManager.isCollecting = true
+  await Promise.resolve()
+  loadManager.isCollecting = false
+
+  let groups = loadManager.groups
+  groups
+  ->Js.Dict.keys
+  ->Utils.Array.forEachAsync(async key => {
+    let group = groups->Js.Dict.unsafeGet(key)
+    let calls = group.calls
+
+    let inputsToLoad = []
+    let currentInputKeys = []
+    calls
+    ->Js.Dict.keys
+    ->Js.Array2.forEach(inputKey => {
+      let call = calls->Js.Dict.unsafeGet(inputKey)
+      if !call.isLoading {
+        call.isLoading = true
+        currentInputKeys->Js.Array2.push(inputKey)->ignore
+        if group.hasInMemory(inputKey)->not {
+          inputsToLoad->Js.Array2.push(call.input)->ignore
+        }
+      }
+    })
+
+    if inputsToLoad->Utils.Array.isEmpty->not {
+      try {
+        await group.load(inputsToLoad)
+      } catch {
+      | exn => {
+          let exn = exn->Internal.prettifyExn
+          currentInputKeys->Array.forEach(inputKey => {
+            let call = calls->Js.Dict.unsafeGet(inputKey)
+            call.reject(exn)
+          })
+        }
+      }
+    }
+
+    if currentInputKeys->Utils.Array.isEmpty->not {
+      currentInputKeys->Js.Array2.forEach(inputKey => {
+        let call = calls->Js.Dict.unsafeGet(inputKey)
+        calls->Utils.Dict.deleteInPlace(inputKey)
+        call.resolve(group.getUnsafeInMemory(inputKey))
+      })
+
+      // Clean up executed batch to reset
+      // provided load function which
+      // might have an outdated function context
+      let latestGroup = groups->Js.Dict.unsafeGet(key)
+      if latestGroup.calls->Js.Dict.keys->Utils.Array.isEmpty {
+        groups->Utils.Dict.deleteInPlace(key)
+      }
+    }
+  })
+}
+
+let noopHasher = input => input->(Utils.magic: 'input => string)
+
+let call = (
+  loadManager,
+  ~input,
+  ~key,
+  ~load,
+  ~hasher,
+  ~shouldGroup,
+  ~hasInMemory,
+  ~getUnsafeInMemory,
+) => {
+  // This is a micro-optimization to avoid a function call
+  let inputKey = hasher === noopHasher ? input->(Utils.magic: 'input => string) : hasher(input)
+
+  // We group external calls by operation to:
+  // 1. Reduce the IO by allowing batch requests
+  // 2. By allowing parallel processing of events
+  //    and make awaits run at the same time
+  //
+  // In the handlers it's not as important to group
+  // calls, because usually we run a single handler at a time
+  // So have a quick exit when an entity is already in memory
+  //
+  // But since we're going to parallelize handlers per chain,
+  // keep the grouping logic when the data needs to be loaded
+  // It has a small additional runtime cost, but might reduce IO time
+  if !shouldGroup && hasInMemory(inputKey) {
+    getUnsafeInMemory(inputKey)->Promise.resolve
+  } else {
+    let group = switch loadManager.groups->Utils.Dict.dangerouslyGetNonOption(key) {
+    | Some(group) => group
+    | None => {
+        let g: Group.t = {
+          calls: Js.Dict.empty(),
+          load: load->(
+            Utils.magic: (array<'input> => promise<unit>) => array<Call.input> => promise<unit>
+          ),
+          getUnsafeInMemory: getUnsafeInMemory->(
+            Utils.magic: (string => 'output) => string => Call.output
+          ),
+          hasInMemory: hasInMemory->(Utils.magic: (string => bool) => string => bool),
+        }
+        loadManager.groups->Js.Dict.set(key, g)
+        g
+      }
+    }
+
+    switch group.calls->Utils.Dict.dangerouslyGetNonOption(inputKey) {
+    | Some(c) => c.promise
+    | None => {
+        let promise = Promise.make((resolve, reject) => {
+          let call: Call.t = {
+            input: input->(Utils.magic: 'input => Call.input),
+            resolve,
+            reject,
+            promise: %raw(`null`),
+            isLoading: false,
+          }
+          group.calls->Js.Dict.set(inputKey, call)
+        })
+
+        // Don't use ref since it'll allocate an object to store .contents
+        (group.calls->Js.Dict.unsafeGet(inputKey)).promise = promise
+
+        if !loadManager.isCollecting {
+          let _: promise<unit> = loadManager->schedule
+        }
+
+        promise
+      }
+    }->(Utils.magic: promise<Call.output> => promise<'output>)
+  }
+}