envio 2.9.1 → 2.11.0

package/evm.schema.json

@@ -83,7 +83,7 @@
       ]
     },
     "field_selection": {
-      "description": "An object representing additional fields to add to the event passed to handlers.",
+      "description": "Select the block and transaction fields to include in all events globally",
       "anyOf": [
         {
           "$ref": "#/$defs/FieldSelection"
@@ -159,6 +159,17 @@
             "string",
             "null"
           ]
+        },
+        "field_selection": {
+          "description": "Select the block and transaction fields to include in the specific event",
+          "anyOf": [
+            {
+              "$ref": "#/$defs/FieldSelection"
+            },
+            {
+              "type": "null"
+            }
+          ]
         }
       },
       "additionalProperties": false,
@@ -166,6 +177,97 @@
         "event"
       ]
     },
+    "FieldSelection": {
+      "type": "object",
+      "properties": {
+        "transaction_fields": {
+          "description": "The transaction fields to include in the event, or in all events if applied globally",
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "$ref": "#/$defs/TransactionField"
+          }
+        },
+        "block_fields": {
+          "description": "The block fields to include in the event, or in all events if applied globally",
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "$ref": "#/$defs/BlockField"
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "TransactionField": {
+      "type": "string",
+      "enum": [
+        "transactionIndex",
+        "hash",
+        "from",
+        "to",
+        "gas",
+        "gasPrice",
+        "maxPriorityFeePerGas",
+        "maxFeePerGas",
+        "cumulativeGasUsed",
+        "effectiveGasPrice",
+        "gasUsed",
+        "input",
+        "nonce",
+        "value",
+        "v",
+        "r",
+        "s",
+        "contractAddress",
+        "logsBloom",
+        "root",
+        "status",
+        "yParity",
+        "chainId",
+        "maxFeePerBlobGas",
+        "blobVersionedHashes",
+        "kind",
+        "l1Fee",
+        "l1GasPrice",
+        "l1GasUsed",
+        "l1FeeScalar",
+        "gasUsedForL1"
+      ]
+    },
+    "BlockField": {
+      "type": "string",
+      "enum": [
+        "parentHash",
+        "nonce",
+        "sha3Uncles",
+        "logsBloom",
+        "transactionsRoot",
+        "stateRoot",
+        "receiptsRoot",
+        "miner",
+        "difficulty",
+        "totalDifficulty",
+        "extraData",
+        "size",
+        "gasLimit",
+        "gasUsed",
+        "uncles",
+        "baseFeePerGas",
+        "blobGasUsed",
+        "excessBlobGas",
+        "parentBeaconBlockRoot",
+        "withdrawalsRoot",
+        "l1BlockNumber",
+        "sendCount",
+        "sendRoot",
+        "mixHash"
+      ]
+    },
     "Network": {
       "type": "object",
       "properties": {
@@ -405,97 +507,6 @@
         "viem",
         "hypersync-client"
       ]
-    },
-    "FieldSelection": {
-      "type": "object",
-      "properties": {
-        "transaction_fields": {
-          "description": "Fields of a transaction to add to the event passed to handlers",
-          "type": [
-            "array",
-            "null"
-          ],
-          "items": {
-            "$ref": "#/$defs/TransactionField"
-          }
-        },
-        "block_fields": {
-          "description": "Fields of a block to add to the event passed to handlers",
-          "type": [
-            "array",
-            "null"
-          ],
-          "items": {
-            "$ref": "#/$defs/BlockField"
-          }
-        }
-      },
-      "additionalProperties": false
-    },
-    "TransactionField": {
-      "type": "string",
-      "enum": [
-        "transactionIndex",
-        "hash",
-        "from",
-        "to",
-        "gas",
-        "gasPrice",
-        "maxPriorityFeePerGas",
-        "maxFeePerGas",
-        "cumulativeGasUsed",
-        "effectiveGasPrice",
-        "gasUsed",
-        "input",
-        "nonce",
-        "value",
-        "v",
-        "r",
-        "s",
-        "contractAddress",
-        "logsBloom",
-        "root",
-        "status",
-        "yParity",
-        "chainId",
-        "maxFeePerBlobGas",
-        "blobVersionedHashes",
-        "kind",
-        "l1Fee",
-        "l1GasPrice",
-        "l1GasUsed",
-        "l1FeeScalar",
-        "gasUsedForL1"
-      ]
-    },
-    "BlockField": {
-      "type": "string",
-      "enum": [
-        "parentHash",
-        "nonce",
-        "sha3Uncles",
-        "logsBloom",
-        "transactionsRoot",
-        "stateRoot",
-        "receiptsRoot",
-        "miner",
-        "difficulty",
-        "totalDifficulty",
-        "extraData",
-        "size",
-        "gasLimit",
-        "gasUsed",
-        "uncles",
-        "baseFeePerGas",
-        "blobGasUsed",
-        "excessBlobGas",
-        "parentBeaconBlockRoot",
-        "withdrawalsRoot",
-        "l1BlockNumber",
-        "sendCount",
-        "sendRoot",
-        "mixHash"
-      ]
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.9.1",
+  "version": "v2.11.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "repository": {
@@ -23,10 +23,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.9.1",
-    "envio-linux-arm64": "v2.9.1",
-    "envio-darwin-x64": "v2.9.1",
-    "envio-darwin-arm64": "v2.9.1"
+    "envio-linux-x64": "v2.11.0",
+    "envio-linux-arm64": "v2.11.0",
+    "envio-darwin-x64": "v2.11.0",
+    "envio-darwin-arm64": "v2.11.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.2",

package/src/Internal.gen.ts

@@ -0,0 +1,47 @@
+/* TypeScript file generated from Internal.res by genType. */
+
+/* eslint-disable */
+/* tslint:disable */
+
+import type {t as Address_t} from './Address.gen';
+
+export type genericEvent<params,block,transaction> = {
+  readonly params: params; 
+  readonly chainId: number; 
+  readonly srcAddress: Address_t; 
+  readonly logIndex: number; 
+  readonly transaction: transaction; 
+  readonly block: block
+};
+
+export type genericLoaderArgs<event,context> = { readonly event: event; readonly context: context };
+
+export type genericLoader<args,loaderReturn> = (_1:args) => Promise<loaderReturn>;
+
+export type genericContractRegisterArgs<event,context> = { readonly event: event; readonly context: context };
+
+export type genericContractRegister<args> = (_1:args) => void;
+
+export type genericHandlerArgs<event,context,loaderReturn> = {
+  readonly event: event; 
+  readonly context: context; 
+  readonly loaderReturn: loaderReturn
+};
+
+export type genericHandler<args> = (_1:args) => Promise<void>;
+
+export type genericHandlerWithLoader<loader,handler,eventFilters> = {
+  readonly loader: loader; 
+  readonly handler: handler; 
+  readonly wildcard?: boolean; 
+  readonly eventFilters?: eventFilters; 
+  readonly preRegisterDynamicContracts?: boolean
+};
+
+export type fuelSupplyParams = { readonly subId: string; readonly amount: bigint };
+
+export type fuelTransferParams = {
+  readonly to: Address_t; 
+  readonly assetId: string; 
+  readonly amount: bigint
+};

package/src/Internal.res

@@ -0,0 +1,124 @@
+type eventParams
+type eventBlock
+type eventTransaction
+
+@genType
+type genericEvent<'params, 'block, 'transaction> = {
+  params: 'params,
+  chainId: int,
+  srcAddress: Address.t,
+  logIndex: int,
+  transaction: 'transaction,
+  block: 'block,
+}
+
+type event = genericEvent<eventParams, eventBlock, eventTransaction>
+
+external fromGenericEvent: genericEvent<'a, 'b, 'c> => event = "%identity"
+
+type loaderReturn
+
+@genType
+type genericLoaderArgs<'event, 'context> = {
+  event: 'event,
+  context: 'context,
+}
+@genType
+type genericLoader<'args, 'loaderReturn> = 'args => promise<'loaderReturn>
+
+type loaderContext
+type loaderArgs = genericLoaderArgs<event, loaderContext>
+type loader = genericLoader<loaderArgs, loaderReturn>
+
+@genType
+type genericContractRegisterArgs<'event, 'context> = {
+  event: 'event,
+  context: 'context,
+}
+@genType
+type genericContractRegister<'args> = 'args => unit
+
+type contractRegisterContext
+type contractRegisterArgs = genericContractRegisterArgs<event, contractRegisterContext>
+type contractRegister = genericContractRegister<contractRegisterArgs>
+
+@genType
+type genericHandlerArgs<'event, 'context, 'loaderReturn> = {
+  event: 'event,
+  context: 'context,
+  loaderReturn: 'loaderReturn,
+}
+@genType
+type genericHandler<'args> = 'args => promise<unit>
+
+type handlerContext
+type handlerArgs = genericHandlerArgs<event, handlerContext, loaderReturn>
+type handler = genericHandler<handlerArgs>
+
+@genType
+type genericHandlerWithLoader<'loader, 'handler, 'eventFilters> = {
+  loader: 'loader,
+  handler: 'handler,
+  wildcard?: bool,
+  eventFilters?: 'eventFilters,
+  preRegisterDynamicContracts?: bool,
+}
+
+type eventItem = {
+  eventName: string,
+  contractName: string,
+  loader: option<loader>,
+  handler: option<handler>,
+  contractRegister: option<contractRegister>,
+  timestamp: int,
+  chain: ChainMap.Chain.t,
+  blockNumber: int,
+  logIndex: int,
+  event: event,
+  paramsRawEventSchema: S.schema<eventParams>,
+  //Default to false, if an event needs to
+  //be reprocessed after it has loaded dynamic contracts
+  //This gets set to true and does not try and reload events
+  hasRegisteredDynamicContracts?: bool,
+}
+
+type fuelEventKind =
+  | LogData({logId: string, decode: string => eventParams})
+  | Mint
+  | Burn
+  | Transfer
+  | Call
+type fuelEventConfig = {
+  name: string,
+  kind: fuelEventKind,
+  isWildcard: bool,
+  loader: option<loader>,
+  handler: option<handler>,
+  contractRegister: option<contractRegister>,
+  paramsRawEventSchema: S.schema<eventParams>,
+}
+type fuelContractConfig = {
+  name: string,
+  events: array<fuelEventConfig>,
+}
+
+@genType
+type fuelSupplyParams = {
+  subId: string,
+  amount: bigint,
+}
+let fuelSupplyParamsSchema = S.schema(s => {
+  subId: s.matches(S.string),
+  amount: s.matches(BigInt.schema),
+})
+@genType
+type fuelTransferParams = {
+  to: Address.t,
+  assetId: string,
+  amount: bigint,
+}
+let fuelTransferParamsSchema = S.schema(s => {
+  to: s.matches(Address.schema),
+  assetId: s.matches(S.string),
+  amount: s.matches(BigInt.schema),
+})

package/src/LazyLoader.res

@@ -0,0 +1,134 @@
+exception LoaderTimeout(string)
+
+type rec asyncMap<'key, 'value> = {
+  // The number of loaded results to keep cached. (For block loading this should be our maximum block interval)
+  _cacheSize: int,
+  // The maximum number of results we can try to load simultaneously.
+  _loaderPoolSize: int,
+  // How long to wait before retrying failures
+  // TODO: Add randomized exponential back-off (outdated)
+  _retryDelayMillis: int,
+  // How long to wait before cancelling a load request
+  _timeoutMillis: int,
+  // The promises we return to callers. We satisfy them asynchronously.
+  externalPromises: Utils.Map.t<'key, promise<'value>>,
+  // The handled used to populate the external promises once we have loaded their data.
+  resolvers: Utils.Map.t<'key, 'value => unit>,
+  // The keys currently being loaded
+  inProgress: Utils.Set.t<'key>,
+  // Keys  for items that we have not started loading yet.
+  loaderQueue: SDSL.Queue.t<'key>,
+  // Keys for items that have been loaded already. Used to evict the oldest keys from cache.
+  loadedKeys: SDSL.Queue.t<'key>,
+  // The function used to load the result.
+  loaderFn: 'key => promise<'value>,
+  // Callback on load error
+  onError: option<(asyncMap<'key, 'value>, ~exn: exn) => unit>,
+}
+
+let make = (
+  ~loaderFn,
+  ~onError=?,
+  ~cacheSize: int=10_000,
+  ~loaderPoolSize: int=10,
+  ~retryDelayMillis=5_000,
+  ~timeoutMillis=300_000,
+) => // After 5 minutes (unclear what is best to do here - crash or just keep printing the error)
+{
+  _cacheSize: cacheSize,
+  _loaderPoolSize: loaderPoolSize,
+  _retryDelayMillis: retryDelayMillis,
+  _timeoutMillis: timeoutMillis,
+  externalPromises: Utils.Map.make(),
+  resolvers: Utils.Map.make(),
+  inProgress: Utils.Set.make(),
+  loaderQueue: SDSL.Queue.make(),
+  loadedKeys: SDSL.Queue.make(),
+  loaderFn,
+  onError,
+}
+
+let deleteKey: (dict<'value>, string) => unit = (_obj, _k) => %raw(`delete _obj[_k]`)
+
+// If something takes longer than this to load, reject the promise and try again
+let timeoutAfter = timeoutMillis =>
+  Utils.delay(timeoutMillis)->Promise.then(() =>
+    Promise.reject(
+      LoaderTimeout(`Query took longer than ${Belt.Int.toString(timeoutMillis / 1000)} seconds`),
+    )
+  )
+
+let rec loadNext = async (am: asyncMap<'key, 'value>, k: 'key) => {
+  // Track that we are loading it now
+  let _ = am.inProgress->Utils.Set.add(k)
+
+  let awaitTaskPromiseAndLoadNextWithTimeout = async () => {
+    let val = await Promise.race([am.loaderFn(k), timeoutAfter(am._timeoutMillis)])
+    // Resolve the external promise
+    am.resolvers
+    ->Utils.Map.get(k)
+    ->Belt.Option.forEach(r => {
+      let _ = am.resolvers->Utils.Map.delete(k)
+      r(val)
+    })
+
+    // Track that it is no longer in progress
+    let _ = am.inProgress->Utils.Set.delete(k)
+
+    // Track that we've loaded this key
+    let loadedKeysNumber = am.loadedKeys->SDSL.Queue.push(k)
+
+    // Delete the oldest key if the cache is overly full
+    if loadedKeysNumber > am._cacheSize {
+      switch am.loadedKeys->SDSL.Queue.pop {
+      | None => ()
+      | Some(old) =>
+        let _ = am.externalPromises->Utils.Map.delete(old)
+      }
+    }
+
+    // Load the next one, if there is anything in the queue
+    switch am.loaderQueue->SDSL.Queue.pop {
+    | None => ()
+    | Some(next) => await loadNext(am, next)
+    }
+  }
+
+  await (
+    switch await awaitTaskPromiseAndLoadNextWithTimeout() {
+    | _ => Promise.resolve()
+    | exception err =>
+      switch am.onError {
+      | None => ()
+      | Some(onError) => onError(am, ~exn=err)
+      }
+      await Utils.delay(am._retryDelayMillis)
+      awaitTaskPromiseAndLoadNextWithTimeout()
+    }
+  )
+}
+
+let get = (am: asyncMap<'key, 'value>, k: 'key): promise<'value> => {
+  switch am.externalPromises->Utils.Map.get(k) {
+  | Some(x) => x
+  | None => {
+      // Create a promise to deliver the eventual value asynchronously
+      let promise = Promise.make((resolve, _) => {
+        // Expose the resolver externally, so that we can run it from the loader.
+        let _ = am.resolvers->Utils.Map.set(k, resolve)
+      })
+      // Cache the promise to de-duplicate requests
+      let _ = am.externalPromises->Utils.Map.set(k, promise)
+
+      //   Do we have a free loader in the pool?
+      if am.inProgress->Utils.Set.size < am._loaderPoolSize {
+        loadNext(am, k)->ignore
+      } else {
+        // Queue the loader
+        let _ = am.loaderQueue->SDSL.Queue.push(k)
+      }
+
+      promise
+    }
+  }
+}