envio 2.9.0 → 2.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.9.0",
+  "version": "v2.10.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "repository": {
@@ -23,10 +23,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.9.0",
-    "envio-linux-arm64": "v2.9.0",
-    "envio-darwin-x64": "v2.9.0",
-    "envio-darwin-arm64": "v2.9.0"
+    "envio-linux-x64": "v2.10.0",
+    "envio-linux-arm64": "v2.10.0",
+    "envio-darwin-x64": "v2.10.0",
+    "envio-darwin-arm64": "v2.10.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.2",

package/src/LazyLoader.res

@@ -0,0 +1,134 @@
+exception LoaderTimeout(string)
+
+type rec asyncMap<'key, 'value> = {
+  // The number of loaded results to keep cached. (For block loading this should be our maximum block interval)
+  _cacheSize: int,
+  // The maximum number of results we can try to load simultaneously.
+  _loaderPoolSize: int,
+  // How long to wait before retrying failures
+  // TODO: Add randomized exponential back-off (outdated)
+  _retryDelayMillis: int,
+  // How long to wait before cancelling a load request
+  _timeoutMillis: int,
+  // The promises we return to callers. We satisfy them asynchronously.
+  externalPromises: Utils.Map.t<'key, promise<'value>>,
+  // The handled used to populate the external promises once we have loaded their data.
+  resolvers: Utils.Map.t<'key, 'value => unit>,
+  // The keys currently being loaded
+  inProgress: Utils.Set.t<'key>,
+  // Keys  for items that we have not started loading yet.
+  loaderQueue: SDSL.Queue.t<'key>,
+  // Keys for items that have been loaded already. Used to evict the oldest keys from cache.
+  loadedKeys: SDSL.Queue.t<'key>,
+  // The function used to load the result.
+  loaderFn: 'key => promise<'value>,
+  // Callback on load error
+  onError: option<(asyncMap<'key, 'value>, ~exn: exn) => unit>,
+}
+
+let make = (
+  ~loaderFn,
+  ~onError=?,
+  ~cacheSize: int=10_000,
+  ~loaderPoolSize: int=10,
+  ~retryDelayMillis=5_000,
+  ~timeoutMillis=300_000,
+) => // After 5 minutes (unclear what is best to do here - crash or just keep printing the error)
+{
+  _cacheSize: cacheSize,
+  _loaderPoolSize: loaderPoolSize,
+  _retryDelayMillis: retryDelayMillis,
+  _timeoutMillis: timeoutMillis,
+  externalPromises: Utils.Map.make(),
+  resolvers: Utils.Map.make(),
+  inProgress: Utils.Set.make(),
+  loaderQueue: SDSL.Queue.make(),
+  loadedKeys: SDSL.Queue.make(),
+  loaderFn,
+  onError,
+}
+
+let deleteKey: (dict<'value>, string) => unit = (_obj, _k) => %raw(`delete _obj[_k]`)
+
+// If something takes longer than this to load, reject the promise and try again
+let timeoutAfter = timeoutMillis =>
+  Utils.delay(timeoutMillis)->Promise.then(() =>
+    Promise.reject(
+      LoaderTimeout(`Query took longer than ${Belt.Int.toString(timeoutMillis / 1000)} seconds`),
+    )
+  )
+
+let rec loadNext = async (am: asyncMap<'key, 'value>, k: 'key) => {
+  // Track that we are loading it now
+  let _ = am.inProgress->Utils.Set.add(k)
+
+  let awaitTaskPromiseAndLoadNextWithTimeout = async () => {
+    let val = await Promise.race([am.loaderFn(k), timeoutAfter(am._timeoutMillis)])
+    // Resolve the external promise
+    am.resolvers
+    ->Utils.Map.get(k)
+    ->Belt.Option.forEach(r => {
+      let _ = am.resolvers->Utils.Map.delete(k)
+      r(val)
+    })
+
+    // Track that it is no longer in progress
+    let _ = am.inProgress->Utils.Set.delete(k)
+
+    // Track that we've loaded this key
+    let loadedKeysNumber = am.loadedKeys->SDSL.Queue.push(k)
+
+    // Delete the oldest key if the cache is overly full
+    if loadedKeysNumber > am._cacheSize {
+      switch am.loadedKeys->SDSL.Queue.pop {
+      | None => ()
+      | Some(old) =>
+        let _ = am.externalPromises->Utils.Map.delete(old)
+      }
+    }
+
+    // Load the next one, if there is anything in the queue
+    switch am.loaderQueue->SDSL.Queue.pop {
+    | None => ()
+    | Some(next) => await loadNext(am, next)
+    }
+  }
+
+  await (
+    switch await awaitTaskPromiseAndLoadNextWithTimeout() {
+    | _ => Promise.resolve()
+    | exception err =>
+      switch am.onError {
+      | None => ()
+      | Some(onError) => onError(am, ~exn=err)
+      }
+      await Utils.delay(am._retryDelayMillis)
+      awaitTaskPromiseAndLoadNextWithTimeout()
+    }
+  )
+}
+
+let get = (am: asyncMap<'key, 'value>, k: 'key): promise<'value> => {
+  switch am.externalPromises->Utils.Map.get(k) {
+  | Some(x) => x
+  | None => {
+      // Create a promise to deliver the eventual value asynchronously
+      let promise = Promise.make((resolve, _) => {
+        // Expose the resolver externally, so that we can run it from the loader.
+        let _ = am.resolvers->Utils.Map.set(k, resolve)
+      })
+      // Cache the promise to de-duplicate requests
+      let _ = am.externalPromises->Utils.Map.set(k, promise)
+
+      //   Do we have a free loader in the pool?
+      if am.inProgress->Utils.Set.size < am._loaderPoolSize {
+        loadNext(am, k)->ignore
+      } else {
+        // Queue the loader
+        let _ = am.loaderQueue->SDSL.Queue.push(k)
+      }
+
+      promise
+    }
+  }
+}

package/src/TopicFilter.res

@@ -0,0 +1,27 @@
+let toTwosComplement = (num: bigint, ~bytesLen: int) => {
+  let maxValue = 1n->BigInt.Bitwise.shift_left(BigInt.fromInt(bytesLen * 8))
+  let mask = maxValue->BigInt.sub(1n)
+  num->BigInt.add(maxValue)->BigInt.Bitwise.logand(mask)
+}
+
+let fromSignedBigInt = val => {
+  let bytesLen = 32
+  let val = val >= 0n ? val : val->toTwosComplement(~bytesLen)
+  val->Viem.bigintToHex(~options={size: bytesLen})
+}
+
+type hex = EvmTypes.Hex.t
+//bytes currently does not work with genType and we also currently generate bytes as a string type
+type bytesHex = string
+let keccak256 = Viem.keccak256
+let bytesToHex = Viem.bytesToHex
+let concat = Viem.concat
+let castToHexUnsafe: 'a => hex = val => val->Utils.magic
+let fromBigInt: bigint => hex = val => val->Viem.bigintToHex(~options={size: 32})
+let fromDynamicString: string => hex = val => val->(Utils.magic: string => hex)->keccak256
+let fromString: string => hex = val => val->Viem.stringToHex(~options={size: 32})
+let fromAddress: Address.t => hex = addr => addr->(Utils.magic: Address.t => hex)->Viem.pad
+let fromDynamicBytes: bytesHex => hex = bytes => bytes->(Utils.magic: bytesHex => hex)->keccak256
+let fromBytes: bytesHex => hex = bytes =>
+  bytes->(Utils.magic: bytesHex => bytes)->Viem.bytesToHex(~options={size: 32})
+let fromBool: bool => hex = b => b->Viem.boolToHex(~options={size: 32})

package/src/Utils.res

@@ -1,5 +1,12 @@
 external magic: 'a => 'b = "%identity"
 
+let delay = milliseconds =>
+  Js.Promise2.make((~resolve, ~reject as _) => {
+    let _interval = Js.Global.setTimeout(_ => {
+      resolve()
+    }, milliseconds)
+  })
+
 module Option = {
   let mapNone = (opt: option<'a>, val: 'b): option<'b> => {
     switch opt {
@@ -224,6 +231,13 @@ external queueMicrotask: (unit => unit) => unit = "queueMicrotask"
 module Schema = {
   let enum = items => S.union(items->Belt.Array.mapU(S.literal))
 
+  // A hot fix after we use the version where it's supported
+  // https://github.com/DZakh/rescript-schema/blob/v8.4.0/docs/rescript-usage.md#removetypevalidation
+  let removeTypeValidationInPlace = schema => {
+    // The variables input is guaranteed to be an object, so we reset the rescript-schema type filter here
+    (schema->Obj.magic)["f"] = ()
+  }
+
   let getNonOptionalFieldNames = schema => {
     let acc = []
     switch schema->S.classify {
@@ -362,3 +376,15 @@ module WeakMap = {
   @send external has: (t<'k, 'v>, 'k) => bool = "has"
   @send external set: (t<'k, 'v>, 'k, 'v) => t<'k, 'v> = "set"
 }
+
+module Map = {
+  type t<'k, 'v> = Js.Map.t<'k, 'v>
+
+  @new external make: unit => t<'k, 'v> = "Map"
+
+  @send external get: (t<'k, 'v>, 'k) => option<'v> = "get"
+  @send external unsafeGet: (t<'k, 'v>, 'k) => 'v = "get"
+  @send external has: (t<'k, 'v>, 'k) => bool = "has"
+  @send external set: (t<'k, 'v>, 'k, 'v) => t<'k, 'v> = "set"
+  @send external delete: (t<'k, 'v>, 'k) => bool = "delete"
+}

package/src/bindings/BigInt.res

@@ -48,10 +48,22 @@ let schema =
   S.string
   ->S.setName("BigInt")
   ->S.transform(s => {
-    parser: (. string) =>
+    parser: string =>
       switch string->fromString {
       | Some(bigInt) => bigInt
-      | None => s.fail(. "The string is not valid BigInt")
+      | None => s.fail("The string is not valid BigInt")
       },
-    serializer: (. bigint) => bigint->toString,
+    serializer: bigint => bigint->toString,
   })
+
+let nativeSchema: S.t<bigint> = S.custom("BigInt", s => {
+  {
+    parser: unknown => {
+      if Js.typeof(unknown) !== "bigint" {
+        s.fail("Expected bigint")
+      } else {
+        unknown->Obj.magic
+      }
+    },
+  }
+})

package/src/bindings/Ethers.gen.ts

@@ -0,0 +1,14 @@
+/* TypeScript file generated from Ethers.res by genType. */
+
+/* eslint-disable */
+/* tslint:disable */
+
+const EthersJS = require('./Ethers.bs.js');
+
+import type {t as Address_t} from '../../src/Address.gen';
+
+export const Addresses_mockAddresses: Address_t[] = EthersJS.Addresses.mockAddresses as any;
+
+export const Addresses_defaultAddress: Address_t = EthersJS.Addresses.defaultAddress as any;
+
+export const Addresses: { mockAddresses: Address_t[]; defaultAddress: Address_t } = EthersJS.Addresses as any;

package/src/bindings/Ethers.res

@@ -0,0 +1,259 @@
+type abi = EvmTypes.Abi.t
+
+let makeAbi = (abi: Js.Json.t): abi => abi->Utils.magic
+
+@deprecated("Use Address.t instead. The type will be removed in v3")
+type ethAddress = Address.t
+@deprecated("Use Address.Evm.fromStringOrThrow instead. The function will be removed in v3")
+let getAddressFromStringUnsafe = Address.Evm.fromStringOrThrow
+@deprecated("Use Address.toString instead. The function will be removed in v3")
+let ethAddressToString = Address.toString
+@deprecated("Use Address.schema instead. The function will be removed in v3")
+let ethAddressSchema = Address.schema
+
+type txHash = string
+
+module Constants = {
+  @module("ethers") @scope("ethers") external zeroHash: string = "ZeroHash"
+  @module("ethers") @scope("ethers") external zeroAddress: Address.t = "ZeroAddress"
+}
+
+module Addresses = {
+  @genType
+  let mockAddresses = [
+    "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266",
+    "0x70997970C51812dc3A010C7d01b50e0d17dc79C8",
+    "0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC",
+    "0x90F79bf6EB2c4f870365E785982E1f101E93b906",
+    "0x15d34AAf54267DB7D7c367839AAf71A00a2C6A65",
+    "0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc",
+    "0x976EA74026E726554dB657fA54763abd0C3a0aa9",
+    "0x14dC79964da2C08b23698B3D3cc7Ca32193d9955",
+    "0x23618e81E3f5cdF7f54C3d65f7FBc0aBf5B21E8f",
+    "0xa0Ee7A142d267C1f36714E4a8F75612F20a79720",
+    "0xBcd4042DE499D14e55001CcbB24a551F3b954096",
+    "0x71bE63f3384f5fb98995898A86B02Fb2426c5788",
+    "0xFABB0ac9d68B0B445fB7357272Ff202C5651694a",
+    "0x1CBd3b2770909D4e10f157cABC84C7264073C9Ec",
+    "0xdF3e18d64BC6A983f673Ab319CCaE4f1a57C7097",
+    "0xcd3B766CCDd6AE721141F452C550Ca635964ce71",
+    "0x2546BcD3c84621e976D8185a91A922aE77ECEc30",
+    "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E",
+    "0xdD2FD4581271e230360230F9337D5c0430Bf44C0",
+    "0x8626f6940E2eb28930eFb4CeF49B2d1F2C9C1199",
+  ]->Belt.Array.map(getAddressFromStringUnsafe)
+  @genType
+  let defaultAddress =
+    mockAddresses[0]
+}
+
+module BlockTag = {
+  type t
+
+  type semanticTag = [#latest | #earliest | #pending]
+  type hexString = string
+  type blockNumber = int
+
+  type blockTagVariant = Latest | Earliest | Pending | HexString(string) | BlockNumber(int)
+
+  let blockTagFromSemantic = (semanticTag: semanticTag): t => semanticTag->Utils.magic
+  let blockTagFromBlockNumber = (blockNumber: blockNumber): t => blockNumber->Utils.magic
+  let blockTagFromHexString = (hexString: hexString): t => hexString->Utils.magic
+
+  let blockTagFromVariant = variant =>
+    switch variant {
+    | Latest => #latest->blockTagFromSemantic
+    | Earliest => #earliest->blockTagFromSemantic
+    | Pending => #pending->blockTagFromSemantic
+    | HexString(str) => str->blockTagFromHexString
+    | BlockNumber(num) => num->blockTagFromBlockNumber
+    }
+}
+
+module EventFilter = {
+  type topic = EvmTypes.Hex.t
+  type t = {
+    address: Address.t,
+    topics: array<topic>,
+  }
+}
+module Filter = {
+  type t
+
+  //This can be used as a filter but should not assume all filters  are the same type
+  //address could be an array of addresses like in combined filter
+  type filterRecord = {
+    address: Address.t,
+    topics: array<EventFilter.topic>,
+    fromBlock: BlockTag.t,
+    toBlock: BlockTag.t,
+  }
+
+  let filterFromRecord = (filterRecord: filterRecord): t => filterRecord->Utils.magic
+}
+
+module CombinedFilter = {
+  type combinedFilterRecord = {
+    address: array<Address.t>,
+    //The second element of the tuple is the
+    topics: array<array<EventFilter.topic>>,
+    fromBlock: BlockTag.t,
+    toBlock: BlockTag.t,
+  }
+
+  let combinedFilterToFilter = (combinedFilter: combinedFilterRecord): Filter.t =>
+    combinedFilter->Utils.magic
+}
+
+type log = {
+  blockNumber: int,
+  blockHash: string,
+  removed: option<bool>,
+  //Note: this is the index of the log in the transaction and should be used whenever we use "logIndex"
+  address: Address.t,
+  data: string,
+  topics: array<EventFilter.topic>,
+  transactionHash: txHash,
+  transactionIndex: int,
+  //Note: this logIndex is the index of the log in the block, not the transaction
+  @as("index") logIndex: int,
+}
+
+type transaction
+
+type minimumParseableLogData = {topics: array<EventFilter.topic>, data: string}
+
+//Can safely convert from log to minimumParseableLogData since it contains
+//both data points required
+let logToMinimumParseableLogData: log => minimumParseableLogData = Utils.magic
+
+type logDescription<'a> = {
+  args: 'a,
+  name: string,
+  signature: string,
+  topic: string,
+}
+
+module Network = {
+  type t
+
+  @module("ethers") @new
+  external make: (~name: string, ~chainId: int) => t = "Network"
+
+  @module("ethers") @scope("Network")
+  external fromChainId: (~chainId: int) => t = "from"
+}
+
+module JsonRpcProvider = {
+  type t
+
+  type rpcOptions = {
+    staticNetwork?: Network.t,
+    // Options for FallbackProvider
+    /**
+     *  The amount of time to wait before kicking off the next provider.
+     *
+     *  Any providers that have not responded can still respond and be
+     *  counted, but this ensures new providers start.
+     *  Default: 400ms
+     */
+    stallTimeout?: int,
+    /**
+     *  The priority. Lower priority providers are dispatched first.
+     *  Default: 1
+     */
+    priority?: int,
+    /**
+     *  The amount of weight a provider is given against the quorum.
+     *  Default: 1
+     */
+    weight?: int,
+  }
+
+  type fallbackProviderOptions = {
+    // How many providers must agree on a value before reporting
+    // back the response
+    // Note: Default the half of the providers weight, so we need to set it to accept result from the first rpc
+    quorum?: int,
+  }
+
+  @module("ethers") @scope("ethers") @new
+  external makeWithOptions: (~rpcUrl: string, ~network: Network.t, ~options: rpcOptions) => t =
+    "JsonRpcProvider"
+
+  @module("ethers") @scope("ethers") @new
+  external makeFallbackProvider: (
+    ~providers: array<t>,
+    ~network: Network.t,
+    ~options: fallbackProviderOptions,
+  ) => t = "FallbackProvider"
+
+  let makeStatic = (~rpcUrl: string, ~network: Network.t, ~priority=?, ~stallTimeout=?): t => {
+    makeWithOptions(~rpcUrl, ~network, ~options={staticNetwork: network, ?priority, ?stallTimeout})
+  }
+
+  let make = (~rpcUrls: array<string>, ~chainId: int, ~fallbackStallTimeout): t => {
+    let network = Network.fromChainId(~chainId)
+    switch rpcUrls {
+    | [rpcUrl] => makeStatic(~rpcUrl, ~network)
+    | rpcUrls =>
+      makeFallbackProvider(
+        ~providers=rpcUrls->Js.Array2.mapi((rpcUrl, index) =>
+          makeStatic(~rpcUrl, ~network, ~priority=index, ~stallTimeout=fallbackStallTimeout)
+        ),
+        ~network,
+        ~options={
+          quorum: 1,
+        },
+      )
+    }
+  }
+
+  @send
+  external getLogs: (t, ~filter: Filter.t) => promise<array<log>> = "getLogs"
+
+  @send
+  external getTransaction: (t, ~transactionHash: string) => promise<transaction> = "getTransaction"
+
+  let makeGetTransactionFields = (~getTransactionByHash) => async (log: log): promise<unknown> => {
+    let transaction = await getTransactionByHash(log.transactionHash)
+    // Mutating should be fine, since the transaction isn't used anywhere else outside the function
+    let fields: {..} = transaction->Obj.magic
+
+    // Make it compatible with HyperSync transaction fields
+    fields["transactionIndex"] = log.transactionIndex
+    fields["input"] = fields["data"]
+
+    fields->Obj.magic
+  }
+
+  type listenerEvent = [#block]
+  @send external onEventListener: (t, listenerEvent, int => unit) => unit = "on"
+
+  @send external offAllEventListeners: (t, listenerEvent) => unit = "off"
+
+  let onBlock = (t, callback: int => unit) => t->onEventListener(#block, callback)
+
+  let removeOnBlockEventListener = t => t->offAllEventListeners(#block)
+
+  @send
+  external getBlockNumber: t => promise<int> = "getBlockNumber"
+
+  type block = {
+    _difficulty: bigint,
+    difficulty: int,
+    extraData: Address.t,
+    gasLimit: bigint,
+    gasUsed: bigint,
+    hash: string,
+    miner: Address.t,
+    nonce: int,
+    number: int,
+    parentHash: Address.t,
+    timestamp: int,
+    transactions: array<Address.t>,
+  }
+
+  @send
+  external getBlock: (t, int) => promise<Js.nullable<block>> = "getBlock"
+}

package/src/bindings/SDSL.res

@@ -0,0 +1,12 @@
+module Queue = {
+  type t<'a>
+
+  @module("js-sdsl") @new external make: unit => t<'a> = "Queue"
+
+  type containerSize = int
+  @send external size: t<'a> => containerSize = "size"
+  @send external push: (t<'a>, 'a) => containerSize = "push"
+  @send external pop: t<'a> => option<'a> = "pop"
+  //Returns the front item without popping it
+  @send external front: t<'a> => option<'a> = "front"
+}