envio 2.31.1 → 2.32.0

package/index.d.ts

@@ -4,6 +4,8 @@ export type {
   effectContext as EffectContext,
   effectArgs as EffectArgs,
   effectOptions as EffectOptions,
+  rateLimitDuration as RateLimitDuration,
+  rateLimit as RateLimit,
   blockEvent as BlockEvent,
   onBlockArgs as OnBlockArgs,
   onBlockOptions as OnBlockOptions,
@@ -13,6 +15,7 @@ export type { EffectCaller } from "./src/Types.ts";
 import type {
   effect as Effect,
   effectArgs as EffectArgs,
+  rateLimit as RateLimit,
 } from "./src/Envio.gen.ts";
 
 import { schema as bigDecimalSchema } from "./src/bindings/BigDecimal.gen.ts";
@@ -73,6 +76,33 @@ type Flatten<T> = T extends object
 // })
 // The behaviour is inspired by Sury code:
 // https://github.com/DZakh/sury/blob/551f8ee32c1af95320936d00c086e5fb337f59fa/packages/sury/src/S.d.ts#L344C1-L355C50
+export function createEffect<
+  IS,
+  OS,
+  I = UnknownToOutput<IS>,
+  O = UnknownToOutput<OS>,
+  // A hack to enforce that the inferred return type
+  // matches the output schema type
+  R extends O = O
+>(
+  options: {
+    /** The name of the effect. Used for logging and debugging. */
+    readonly name: string;
+    /** The input schema of the effect. */
+    readonly input: IS;
+    /** The output schema of the effect. */
+    readonly output: OS;
+    /** Rate limit for the effect. Set to false to disable or provide {calls: number, per: "second" | "minute"} to enable. */
+    readonly rateLimit: RateLimit;
+    /** Whether the effect should be cached. */
+    readonly cache?: boolean;
+  },
+  handler: (args: EffectArgs<I>) => Promise<R>
+): Effect<I, O>;
+
+/**
+ * @deprecated Use createEffect instead. The only difference is that rateLimit option becomes required. Set it to false to keep the same behaviour.
+ */
 export function experimental_createEffect<
   IS,
   OS,
@@ -89,6 +119,8 @@ export function experimental_createEffect<
     readonly input: IS;
     /** The output schema of the effect. */
     readonly output: OS;
+    /** Rate limit for the effect. Set to false to disable or provide {calls: number, per: "second" | "minute"} to enable. */
+    readonly rateLimit?: RateLimit;
     /** Whether the effect should be cached. */
     readonly cache?: boolean;
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.31.1",
+  "version": "v2.32.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.31.1",
-    "envio-linux-arm64": "v2.31.1",
-    "envio-darwin-x64": "v2.31.1",
-    "envio-darwin-arm64": "v2.31.1"
+    "envio-linux-x64": "v2.32.0",
+    "envio-linux-arm64": "v2.32.0",
+    "envio-darwin-x64": "v2.32.0",
+    "envio-darwin-arm64": "v2.32.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.6",

package/src/Batch.res

@@ -84,7 +84,7 @@ let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
 
 let hasMultichainReadyItem = (
   fetchStates: ChainMap.t<FetchState.t>,
-  ~multichain: InternalConfig.multichain,
+  ~multichain: Config.multichain,
 ) => {
   switch multichain {
   | Ordered => hasOrderedReadyItem(fetchStates)
@@ -496,7 +496,7 @@ let prepareUnorderedBatch = (
 let make = (
   ~checkpointIdBeforeBatch,
   ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
-  ~multichain: InternalConfig.multichain,
+  ~multichain: Config.multichain,
   ~batchSizeTarget,
 ) => {
   if (
@@ -528,3 +528,23 @@ let findFirstEventBlockNumber = (batch: t, ~chainId) => {
   }
   result.contents
 }
+
+let findLastEventItem = (batch: t, ~chainId) => {
+  let idx = ref(batch.items->Array.length - 1)
+  let result = ref(None)
+  while idx.contents >= 0 && result.contents === None {
+    let item = batch.items->Array.getUnsafe(idx.contents)
+    switch item {
+    | Internal.Event(_) as eventItem => {
+        let eventItem = eventItem->Internal.castUnsafeEventItem
+        if eventItem.chain->ChainMap.Chain.toChainId === chainId {
+          result := Some(eventItem)
+        } else {
+          idx := idx.contents - 1
+        }
+      }
+    | Internal.Block(_) => idx := idx.contents - 1
+    }
+  }
+  result.contents
+}

package/src/Batch.res.js

@@ -361,6 +361,20 @@ function findFirstEventBlockNumber(batch, chainId) {
   return result;
 }
 
+function findLastEventItem(batch, chainId) {
+  var idx = batch.items.length - 1;
+  var result;
+  while(idx >= 0 && result === undefined) {
+    var item = batch.items[idx];
+    if (item.kind === 0 && item.chain === chainId) {
+      result = item;
+    } else {
+      idx = idx - 1;
+    }
+  };
+  return result;
+}
+
 exports.getOrderedNextChain = getOrderedNextChain;
 exports.immutableEmptyBatchSizePerChain = immutableEmptyBatchSizePerChain;
 exports.hasOrderedReadyItem = hasOrderedReadyItem;
@@ -372,4 +386,5 @@ exports.prepareOrderedBatch = prepareOrderedBatch;
 exports.prepareUnorderedBatch = prepareUnorderedBatch;
 exports.make = make;
 exports.findFirstEventBlockNumber = findFirstEventBlockNumber;
+exports.findLastEventItem = findLastEventItem;
 /* Utils Not a pure module */

package/src/Config.res

@@ -0,0 +1,124 @@
+open Belt
+
+type ecosystem = | @as("evm") Evm | @as("fuel") Fuel
+
+type sourceSyncOptions = {
+  initialBlockInterval?: int,
+  backoffMultiplicative?: float,
+  accelerationAdditive?: int,
+  intervalCeiling?: int,
+  backoffMillis?: int,
+  queryTimeoutMillis?: int,
+  fallbackStallTimeout?: int,
+}
+
+type contract = {
+  name: string,
+  abi: EvmTypes.Abi.t,
+  addresses: array<Address.t>,
+  events: array<Internal.eventConfig>,
+  startBlock: option<int>,
+}
+
+type chain = {
+  id: int,
+  startBlock: int,
+  endBlock?: int,
+  maxReorgDepth: int,
+  contracts: array<contract>,
+  sources: array<Source.t>,
+}
+
+type sourceSync = {
+  initialBlockInterval: int,
+  backoffMultiplicative: float,
+  accelerationAdditive: int,
+  intervalCeiling: int,
+  backoffMillis: int,
+  queryTimeoutMillis: int,
+  fallbackStallTimeout: int,
+}
+
+type multichain = | @as("ordered") Ordered | @as("unordered") Unordered
+
+type t = {
+  shouldRollbackOnReorg: bool,
+  shouldSaveFullHistory: bool,
+  multichain: multichain,
+  chainMap: ChainMap.t<chain>,
+  defaultChain: option<chain>,
+  ecosystem: ecosystem,
+  enableRawEvents: bool,
+  preloadHandlers: bool,
+  maxAddrInPartition: int,
+  batchSize: int,
+  lowercaseAddresses: bool,
+  addContractNameToContractNameMapping: dict<string>,
+}
+
+let make = (
+  ~shouldRollbackOnReorg=true,
+  ~shouldSaveFullHistory=false,
+  ~chains: array<chain>=[],
+  ~enableRawEvents=false,
+  ~preloadHandlers=false,
+  ~ecosystem=Evm,
+  ~batchSize=5000,
+  ~lowercaseAddresses=false,
+  ~multichain=Unordered,
+  ~shouldUseHypersyncClientDecoder=true,
+  ~maxAddrInPartition=5000,
+) => {
+  // Validate that lowercase addresses is not used with viem decoder
+  if lowercaseAddresses && !shouldUseHypersyncClientDecoder {
+    Js.Exn.raiseError(
+      "lowercase addresses is not supported when event_decoder is 'viem'. Please set event_decoder to 'hypersync-client' or change address_format to 'checksum'.",
+    )
+  }
+
+  let chainMap =
+    chains
+    ->Js.Array2.map(n => {
+      (ChainMap.Chain.makeUnsafe(~chainId=n.id), n)
+    })
+    ->ChainMap.fromArrayUnsafe
+
+  // Build the contract name mapping for efficient lookup
+  let addContractNameToContractNameMapping = Js.Dict.empty()
+  chains->Array.forEach(chainConfig => {
+    chainConfig.contracts->Array.forEach(contract => {
+      let addKey = "add" ++ contract.name->Utils.String.capitalize
+      addContractNameToContractNameMapping->Js.Dict.set(addKey, contract.name)
+    })
+  })
+
+  {
+    shouldRollbackOnReorg,
+    shouldSaveFullHistory,
+    multichain,
+    chainMap,
+    defaultChain: chains->Array.get(0),
+    enableRawEvents,
+    ecosystem,
+    maxAddrInPartition,
+    preloadHandlers,
+    batchSize,
+    lowercaseAddresses,
+    addContractNameToContractNameMapping,
+  }
+}
+
+let shouldSaveHistory = (config, ~isInReorgThreshold) =>
+  config.shouldSaveFullHistory || (config.shouldRollbackOnReorg && isInReorgThreshold)
+
+let shouldPruneHistory = (config, ~isInReorgThreshold) =>
+  !config.shouldSaveFullHistory && (config.shouldRollbackOnReorg && isInReorgThreshold)
+
+let getChain = (config, ~chainId) => {
+  let chain = ChainMap.Chain.makeUnsafe(~chainId)
+  config.chainMap->ChainMap.has(chain)
+    ? chain
+    : Js.Exn.raiseError(
+        "No chain with id " ++ chain->ChainMap.Chain.toString ++ " found in config.yaml",
+      )
+}

package/src/Config.res.js

@@ -0,0 +1,84 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+var Utils = require("./Utils.res.js");
+var Js_exn = require("rescript/lib/js/js_exn.js");
+var ChainMap = require("./ChainMap.res.js");
+var Belt_Array = require("rescript/lib/js/belt_Array.js");
+
+function make(shouldRollbackOnReorgOpt, shouldSaveFullHistoryOpt, chainsOpt, enableRawEventsOpt, preloadHandlersOpt, ecosystemOpt, batchSizeOpt, lowercaseAddressesOpt, multichainOpt, shouldUseHypersyncClientDecoderOpt, maxAddrInPartitionOpt) {
+  var shouldRollbackOnReorg = shouldRollbackOnReorgOpt !== undefined ? shouldRollbackOnReorgOpt : true;
+  var shouldSaveFullHistory = shouldSaveFullHistoryOpt !== undefined ? shouldSaveFullHistoryOpt : false;
+  var chains = chainsOpt !== undefined ? chainsOpt : [];
+  var enableRawEvents = enableRawEventsOpt !== undefined ? enableRawEventsOpt : false;
+  var preloadHandlers = preloadHandlersOpt !== undefined ? preloadHandlersOpt : false;
+  var ecosystem = ecosystemOpt !== undefined ? ecosystemOpt : "evm";
+  var batchSize = batchSizeOpt !== undefined ? batchSizeOpt : 5000;
+  var lowercaseAddresses = lowercaseAddressesOpt !== undefined ? lowercaseAddressesOpt : false;
+  var multichain = multichainOpt !== undefined ? multichainOpt : "unordered";
+  var shouldUseHypersyncClientDecoder = shouldUseHypersyncClientDecoderOpt !== undefined ? shouldUseHypersyncClientDecoderOpt : true;
+  var maxAddrInPartition = maxAddrInPartitionOpt !== undefined ? maxAddrInPartitionOpt : 5000;
+  if (lowercaseAddresses && !shouldUseHypersyncClientDecoder) {
+    Js_exn.raiseError("lowercase addresses is not supported when event_decoder is 'viem'. Please set event_decoder to 'hypersync-client' or change address_format to 'checksum'.");
+  }
+  var chainMap = ChainMap.fromArrayUnsafe(chains.map(function (n) {
+            return [
+                    ChainMap.Chain.makeUnsafe(n.id),
+                    n
+                  ];
+          }));
+  var addContractNameToContractNameMapping = {};
+  Belt_Array.forEach(chains, (function (chainConfig) {
+          Belt_Array.forEach(chainConfig.contracts, (function (contract) {
+                  var addKey = "add" + Utils.$$String.capitalize(contract.name);
+                  addContractNameToContractNameMapping[addKey] = contract.name;
+                }));
+        }));
+  return {
+          shouldRollbackOnReorg: shouldRollbackOnReorg,
+          shouldSaveFullHistory: shouldSaveFullHistory,
+          multichain: multichain,
+          chainMap: chainMap,
+          defaultChain: Belt_Array.get(chains, 0),
+          ecosystem: ecosystem,
+          enableRawEvents: enableRawEvents,
+          preloadHandlers: preloadHandlers,
+          maxAddrInPartition: maxAddrInPartition,
+          batchSize: batchSize,
+          lowercaseAddresses: lowercaseAddresses,
+          addContractNameToContractNameMapping: addContractNameToContractNameMapping
+        };
+}
+
+function shouldSaveHistory(config, isInReorgThreshold) {
+  if (config.shouldSaveFullHistory) {
+    return true;
+  } else if (config.shouldRollbackOnReorg) {
+    return isInReorgThreshold;
+  } else {
+    return false;
+  }
+}
+
+function shouldPruneHistory(config, isInReorgThreshold) {
+  if (!config.shouldSaveFullHistory && config.shouldRollbackOnReorg) {
+    return isInReorgThreshold;
+  } else {
+    return false;
+  }
+}
+
+function getChain(config, chainId) {
+  var chain = ChainMap.Chain.makeUnsafe(chainId);
+  if (ChainMap.has(config.chainMap, chain)) {
+    return chain;
+  } else {
+    return Js_exn.raiseError("No chain with id " + ChainMap.Chain.toString(chain) + " found in config.yaml");
+  }
+}
+
+exports.make = make;
+exports.shouldSaveHistory = shouldSaveHistory;
+exports.shouldPruneHistory = shouldPruneHistory;
+exports.getChain = getChain;
+/* Utils Not a pure module */

package/src/Envio.gen.ts

@@ -29,6 +29,25 @@ export type logger = $$logger;
 
 export type effect<input,output> = $$effect<input,output>;
 
+export type rateLimitDuration = "second" | "minute" | number;
+
+export type rateLimit = 
+    false
+  | { readonly calls: number; readonly per: rateLimitDuration };
+
+export type experimental_effectOptions<input,output> = {
+  /** The name of the effect. Used for logging and debugging. */
+  readonly name: string; 
+  /** The input schema of the effect. */
+  readonly input: RescriptSchema_S_t<input>; 
+  /** The output schema of the effect. */
+  readonly output: RescriptSchema_S_t<output>; 
+  /** Rate limit for the effect. Set to false to disable or provide {calls: number, per: "second" | "minute"} to enable. */
+  readonly rateLimit?: rateLimit; 
+  /** Whether the effect should be cached. */
+  readonly cache?: boolean
+};
+
 export type effectOptions<input,output> = {
   /** The name of the effect. Used for logging and debugging. */
   readonly name: string; 
@@ -36,6 +55,8 @@ export type effectOptions<input,output> = {
   readonly input: RescriptSchema_S_t<input>; 
   /** The output schema of the effect. */
   readonly output: RescriptSchema_S_t<output>; 
+  /** Rate limit for the effect. Set to false to disable or provide {calls: number, per: "second" | "minute"} to enable. */
+  readonly rateLimit: rateLimit; 
   /** Whether the effect should be cached. */
   readonly cache?: boolean
 };

package/src/Envio.res

@@ -32,6 +32,28 @@ type logger = {
 @@warning("-30") // Duplicated type names (input)
 @genType.import(("./Types.ts", "Effect"))
 type rec effect<'input, 'output>
+@genType @unboxed
+and rateLimitDuration =
+  | @as("second") Second
+  | @as("minute") Minute
+  | Milliseconds(int)
+@genType @unboxed
+and rateLimit =
+  | @as(false) Disable
+  | Enable({calls: int, per: rateLimitDuration})
+@genType
+and experimental_effectOptions<'input, 'output> = {
+  /** The name of the effect. Used for logging and debugging. */
+  name: string,
+  /** The input schema of the effect. */
+  input: S.t<'input>,
+  /** The output schema of the effect. */
+  output: S.t<'output>,
+  /** Rate limit for the effect. Set to false to disable or provide {calls: number, per: "second" | "minute"} to enable. */
+  rateLimit?: rateLimit,
+  /** Whether the effect should be cached. */
+  cache?: bool,
+}
 @genType
 and effectOptions<'input, 'output> = {
   /** The name of the effect. Used for logging and debugging. */
@@ -40,6 +62,8 @@ and effectOptions<'input, 'output> = {
   input: S.t<'input>,
   /** The output schema of the effect. */
   output: S.t<'output>,
+  /** Rate limit for the effect. Set to false to disable or provide {calls: number, per: "second" | "minute"} to enable. */
+  rateLimit: rateLimit,
   /** Whether the effect should be cached. */
   cache?: bool,
 }
@@ -47,6 +71,7 @@ and effectOptions<'input, 'output> = {
 and effectContext = {
   log: logger,
   effect: 'input 'output. (effect<'input, 'output>, 'input) => promise<'output>,
+  mutable cache: bool,
 }
 @genType
 and effectArgs<'input> = {
@@ -55,12 +80,23 @@ and effectArgs<'input> = {
 }
 @@warning("+30")
 
-let experimental_createEffect = (
+let durationToMs = (duration: rateLimitDuration) =>
+  switch duration {
+  | Second => 1000
+  | Minute => 60000
+  | Milliseconds(ms) => ms
+  }
+
+let createEffect = (
   options: effectOptions<'input, 'output>,
   handler: effectArgs<'input> => promise<'output>,
 ) => {
   let outputSchema =
     S.schema(_ => options.output)->(Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>)
+  let itemSchema = S.schema((s): Internal.effectCacheItem => {
+    id: s.matches(S.string),
+    output: s.matches(outputSchema),
+  })
   {
     name: options.name,
     handler: handler->(
@@ -68,7 +104,8 @@ let experimental_createEffect = (
         Internal.effectOutput,
       >
     ),
-    callsCount: 0,
+    activeCallsCount: 0,
+    prevCallStartTimerRef: %raw(`null`),
     // This is the way to make the createEffect API
     // work without the need for users to call S.schema themselves,
     // but simply pass the desired object/tuple/etc.
@@ -77,20 +114,48 @@ let experimental_createEffect = (
       Utils.magic: S.t<S.t<'input>> => S.t<Internal.effectInput>
     ),
     output: outputSchema,
-    cache: switch options.cache {
-    | Some(true) =>
-      let itemSchema = S.schema((s): Internal.effectCacheItem => {
-        id: s.matches(S.string),
-        output: s.matches(outputSchema),
-      })
+    storageMeta: {
+      table: Internal.makeCacheTable(~effectName=options.name),
+      outputSchema,
+      itemSchema,
+    },
+    defaultShouldCache: switch options.cache {
+    | Some(true) => true
+    | _ => false
+    },
+    rateLimit: switch options.rateLimit {
+    | Disable => None
+    | Enable({calls, per}) =>
       Some({
-        table: Internal.makeCacheTable(~effectName=options.name),
-        outputSchema,
-        itemSchema,
+        callsPerDuration: calls,
+        durationMs: per->durationToMs,
+        availableCalls: calls,
+        windowStartTime: Js.Date.now(),
+        queueCount: 0,
+        nextWindowPromise: None,
       })
-    | None
-    | Some(false) =>
-      None
     },
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)
 }
+
+@deprecated(
+  "Use createEffect instead. The only difference is that rateLimit option becomes required. Set it to false to keep the same behaviour."
+)
+let experimental_createEffect = (
+  options: experimental_effectOptions<'input, 'output>,
+  handler: effectArgs<'input> => promise<'output>,
+) => {
+  createEffect(
+    {
+      name: options.name,
+      input: options.input,
+      output: options.output,
+      rateLimit: switch options.rateLimit {
+      | Some(rateLimit) => rateLimit
+      | None => Disable
+      },
+      cache: ?options.cache,
+    },
+    handler,
+  )
+}

package/src/Envio.res.js

@@ -4,38 +4,75 @@
 var Internal = require("./Internal.res.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
-function experimental_createEffect(options, handler) {
+function durationToMs(duration) {
+  if (typeof duration !== "number") {
+    if (duration === "second") {
+      return 1000;
+    } else {
+      return 60000;
+    }
+  } else {
+    return duration;
+  }
+}
+
+function createEffect(options, handler) {
   var outputSchema = S$RescriptSchema.schema(function (param) {
         return options.output;
       });
+  var itemSchema = S$RescriptSchema.schema(function (s) {
+        return {
+                id: s.m(S$RescriptSchema.string),
+                output: s.m(outputSchema)
+              };
+      });
   var match = options.cache;
+  var match$1 = options.rateLimit;
   var tmp;
-  if (match !== undefined && match) {
-    var itemSchema = S$RescriptSchema.schema(function (s) {
-          return {
-                  id: s.m(S$RescriptSchema.string),
-                  output: s.m(outputSchema)
-                };
-        });
+  if (typeof match$1 !== "object") {
+    tmp = undefined;
+  } else {
+    var calls = match$1.calls;
     tmp = {
-      itemSchema: itemSchema,
-      outputSchema: outputSchema,
-      table: Internal.makeCacheTable(options.name)
+      callsPerDuration: calls,
+      durationMs: durationToMs(match$1.per),
+      availableCalls: calls,
+      windowStartTime: Date.now(),
+      queueCount: 0,
+      nextWindowPromise: undefined
     };
-  } else {
-    tmp = undefined;
   }
   return {
           name: options.name,
           handler: handler,
-          cache: tmp,
+          storageMeta: {
+            itemSchema: itemSchema,
+            outputSchema: outputSchema,
+            table: Internal.makeCacheTable(options.name)
+          },
+          defaultShouldCache: match !== undefined && match ? true : false,
           output: outputSchema,
           input: S$RescriptSchema.schema(function (param) {
                 return options.input;
               }),
-          callsCount: 0
+          activeCallsCount: 0,
+          prevCallStartTimerRef: null,
+          rateLimit: tmp
         };
 }
 
+function experimental_createEffect(options, handler) {
+  var rateLimit = options.rateLimit;
+  return createEffect({
+              name: options.name,
+              input: options.input,
+              output: options.output,
+              rateLimit: rateLimit !== undefined ? rateLimit : false,
+              cache: options.cache
+            }, handler);
+}
+
+exports.durationToMs = durationToMs;
+exports.createEffect = createEffect;
 exports.experimental_createEffect = experimental_createEffect;
 /* Internal Not a pure module */

package/src/EventRegister.res

@@ -4,8 +4,8 @@ type registrations = {
 }
 
 type activeRegistration = {
-  ecosystem: InternalConfig.ecosystem,
-  multichain: InternalConfig.multichain,
+  ecosystem: Config.ecosystem,
+  multichain: Config.multichain,
   preloadHandlers: bool,
   registrations: registrations,
   mutable finished: bool,

package/src/EventRegister.resi

@@ -4,8 +4,8 @@ type registrations = {
 }
 
 let startRegistration: (
-  ~ecosystem: InternalConfig.ecosystem,
-  ~multichain: InternalConfig.multichain,
+  ~ecosystem: Config.ecosystem,
+  ~multichain: Config.multichain,
   ~preloadHandlers: bool,
 ) => unit
 let finishRegistration: unit => registrations