envio 2.29.1 → 2.30.0-rc.0

package/evm.schema.json

@@ -113,6 +113,17 @@
         "boolean",
         "null"
       ]
+    },
+    "address_format": {
+      "description": "Address format for Ethereum addresses: 'checksum' or 'lowercase' (default: checksum)",
+      "anyOf": [
+        {
+          "$ref": "#/$defs/AddressFormat"
+        },
+        {
+          "type": "null"
+        }
+      ]
     }
   },
   "additionalProperties": false,
@@ -653,6 +664,13 @@
         "viem",
         "hypersync-client"
       ]
+    },
+    "AddressFormat": {
+      "type": "string",
+      "enum": [
+        "checksum",
+        "lowercase"
+      ]
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.29.1",
+  "version": "v2.30.0-rc.0",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.29.1",
-    "envio-linux-arm64": "v2.29.1",
-    "envio-darwin-x64": "v2.29.1",
-    "envio-darwin-arm64": "v2.29.1"
+    "envio-linux-x64": "v2.30.0-rc.0",
+    "envio-linux-arm64": "v2.30.0-rc.0",
+    "envio-darwin-x64": "v2.30.0-rc.0",
+    "envio-darwin-arm64": "v2.30.0-rc.0"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.6",

package/src/Address.res

@@ -10,6 +10,29 @@ external unsafeFromString: string => t = "%identity"
 module Evm = {
   @module("viem")
   external fromStringOrThrow: string => t = "getAddress"
+
+  // NOTE: the function is named to be overshadowed by the one below, so that we don't have to import viem in the handler code
+  @module("viem")
+  external fromStringLowercaseOrThrow: string => bool = "isAddress"
+
+  // Reassign since the function might be used in the handler code
+  // and we don't want to have a "viem" import there. It's needed to keep "viem" a dependency
+  // of generated code instead of adding it to the indexer project dependencies.
+  // Also, we want a custom error message, which is searchable in our codebase.
+  // Validate that the string is a proper address but return a lowercased value
+  let fromStringLowercaseOrThrow = string => {
+    if fromStringLowercaseOrThrow(string) {
+      unsafeFromString(string->Js.String2.toLowerCase)
+    } else {
+      Js.Exn.raiseError(
+        `Address "${string}" is invalid. Expected a 20-byte hex string starting with 0x.`,
+      )
+    }
+  }
+
+  let fromAddressLowercaseOrThrow = address =>
+    address->toString->Js.String2.toLowerCase->(Utils.magic: string => t)
+
   // Reassign since the function might be used in the handler code
   // and we don't want to have a "viem" import there. It's needed to keep "viem" a dependency
   // of generated code instead of adding it to the indexer project dependencies.

package/src/Address.res.js

@@ -7,6 +7,18 @@ var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 
 var schema = S$RescriptSchema.setName(S$RescriptSchema.string, "Address");
 
+function fromStringLowercaseOrThrow(string) {
+  if (Viem.isAddress(string)) {
+    return string.toLowerCase();
+  } else {
+    return Js_exn.raiseError("Address \"" + string + "\" is invalid. Expected a 20-byte hex string starting with 0x.");
+  }
+}
+
+function fromAddressLowercaseOrThrow(address) {
+  return address.toLowerCase();
+}
+
 function fromStringOrThrow(string) {
   try {
     return Viem.getAddress(string);
@@ -21,6 +33,8 @@ function fromAddressOrThrow(address) {
 }
 
 var Evm = {
+  fromStringLowercaseOrThrow: fromStringLowercaseOrThrow,
+  fromAddressLowercaseOrThrow: fromAddressLowercaseOrThrow,
   fromStringOrThrow: fromStringOrThrow,
   fromAddressOrThrow: fromAddressOrThrow
 };

package/src/Batch.res

@@ -2,146 +2,159 @@ type progressedChain = {
   chainId: int,
   batchSize: int,
   progressBlockNumber: int,
-  progressNextBlockLogIndex: option<int>,
+  isProgressAtHead: bool,
   totalEventsProcessed: int,
 }
 
 type t = {
   items: array<Internal.item>,
   progressedChains: array<progressedChain>,
-  fetchStates: ChainMap.t<FetchState.t>,
+  updatedFetchStates: ChainMap.t<FetchState.t>,
   dcsToStoreByChainId: dict<array<FetchState.indexingContract>>,
-}
-
-type multiChainEventComparitor = {
-  chain: ChainMap.Chain.t,
-  earliestEvent: FetchState.queueItem,
-}
-
-let getQueueItemComparitor = (earliestQueueItem: FetchState.queueItem, ~chain) => {
-  switch earliestQueueItem {
-  | Item({item}) => item->EventUtils.getOrderedBatchItemComparator
-  | NoItem({latestFetchedBlock: {blockTimestamp, blockNumber}}) => (
-      blockTimestamp,
-      chain->ChainMap.Chain.toChainId,
-      blockNumber,
-      0,
-    )
-  }
-}
-
-let isQueueItemEarlier = (a: multiChainEventComparitor, b: multiChainEventComparitor): bool => {
-  a.earliestEvent->getQueueItemComparitor(~chain=a.chain) <
-    b.earliestEvent->getQueueItemComparitor(~chain=b.chain)
+  creationTimeMs: int,
 }
 
 /**
  It either returnes an earliest item among all chains, or None if no chains are actively indexing
  */
-let getOrderedNextItem = (fetchStates: ChainMap.t<FetchState.t>): option<
-  multiChainEventComparitor,
-> => {
-  fetchStates
-  ->ChainMap.entries
-  ->Belt.Array.reduce(None, (accum, (chain, fetchState)) => {
-    // If the fetch state has reached the end block we don't need to consider it
+let getOrderedNextChain = (fetchStates: ChainMap.t<FetchState.t>, ~batchSizePerChain) => {
+  let earliestChain: ref<option<FetchState.t>> = ref(None)
+  let earliestChainTimestamp = ref(0)
+  let chainKeys = fetchStates->ChainMap.keys
+  for idx in 0 to chainKeys->Array.length - 1 {
+    let chain = chainKeys->Array.get(idx)
+    let fetchState = fetchStates->ChainMap.get(chain)
     if fetchState->FetchState.isActivelyIndexing {
-      let earliestEvent = fetchState->FetchState.getEarliestEvent
-      let current: multiChainEventComparitor = {chain, earliestEvent}
-      switch accum {
-      | Some(previous) if isQueueItemEarlier(previous, current) => accum
-      | _ => Some(current)
+      let timestamp = fetchState->FetchState.getTimestampAt(
+        ~index=switch batchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
+          chain->ChainMap.Chain.toChainId,
+        ) {
+        | Some(batchSize) => batchSize
+        | None => 0
+        },
+      )
+      switch earliestChain.contents {
+      | Some(earliestChain)
+        if timestamp > earliestChainTimestamp.contents ||
+          (timestamp === earliestChainTimestamp.contents &&
+            chain->ChainMap.Chain.toChainId > earliestChain.chainId) => ()
+      | _ => {
+          earliestChain := Some(fetchState)
+          earliestChainTimestamp := timestamp
+        }
       }
-    } else {
-      accum
     }
-  })
+  }
+  earliestChain.contents
+}
+
+// Save overhead of recreating the dict every time
+let immutableEmptyBatchSizePerChain: dict<int> = Js.Dict.empty()
+let hasOrderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
+  switch fetchStates->getOrderedNextChain(~batchSizePerChain=immutableEmptyBatchSizePerChain) {
+  | Some(fetchState) => fetchState->FetchState.hasReadyItem
+  | None => false
+  }
 }
 
-let hasUnorderedNextItem = (fetchStates: ChainMap.t<FetchState.t>) => {
+let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
   fetchStates
   ->ChainMap.values
   ->Js.Array2.some(fetchState => {
-    fetchState->FetchState.isActivelyIndexing &&
-      switch fetchState->FetchState.getEarliestEvent {
-      | Item(_) => true
-      | NoItem(_) => false
-      }
+    fetchState->FetchState.isActivelyIndexing && fetchState->FetchState.hasReadyItem
   })
 }
 
-let popOrderedBatchItems = (
-  ~maxBatchSize,
+let hasMultichainReadyItem = (
+  fetchStates: ChainMap.t<FetchState.t>,
+  ~multichain: InternalConfig.multichain,
+) => {
+  switch multichain {
+  | Ordered => hasOrderedReadyItem(fetchStates)
+  | Unordered => hasUnorderedReadyItem(fetchStates)
+  }
+}
+
+let prepareOrderedBatch = (
+  ~batchSizeTarget,
   ~fetchStates: ChainMap.t<FetchState.t>,
-  ~sizePerChain: dict<int>,
+  ~mutBatchSizePerChain: dict<int>,
 ) => {
+  let batchSize = ref(0)
+  let isFinished = ref(false)
   let items = []
 
-  let rec loop = () =>
-    if items->Array.length < maxBatchSize {
-      switch fetchStates->getOrderedNextItem {
-      | Some({earliestEvent, chain}) =>
-        switch earliestEvent {
-        | NoItem(_) => ()
-        | Item({item, popItemOffQueue}) => {
-            popItemOffQueue()
-            items->Js.Array2.push(item)->ignore
-            sizePerChain->Utils.Dict.incrementByInt(chain->ChainMap.Chain.toChainId)
-            loop()
+  while batchSize.contents < batchSizeTarget && !isFinished.contents {
+    switch fetchStates->getOrderedNextChain(~batchSizePerChain=mutBatchSizePerChain) {
+    | Some(fetchState) => {
+        let itemsCountBefore = switch mutBatchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
+          fetchState.chainId,
+        ) {
+        | Some(batchSize) => batchSize
+        | None => 0
+        }
+        let newItemsCount =
+          fetchState->FetchState.getReadyItemsCount(
+            ~targetSize=batchSizeTarget - batchSize.contents,
+            ~fromItem=itemsCountBefore,
+          )
+
+        if newItemsCount > 0 {
+          for idx in itemsCountBefore to itemsCountBefore + newItemsCount - 1 {
+            items->Js.Array2.push(fetchState.buffer->Belt.Array.getUnsafe(idx))->ignore
           }
+          batchSize := batchSize.contents + newItemsCount
+          mutBatchSizePerChain->Utils.Dict.setByInt(
+            fetchState.chainId,
+            itemsCountBefore + newItemsCount,
+          )
+        } else {
+          isFinished := true
         }
-      | _ => ()
       }
+
+    | None => isFinished := true
     }
-  loop()
+  }
 
   items
 }
 
-let popUnorderedBatchItems = (
-  ~maxBatchSize,
+let prepareUnorderedBatch = (
+  ~batchSizeTarget,
   ~fetchStates: ChainMap.t<FetchState.t>,
-  ~sizePerChain: dict<int>,
+  ~mutBatchSizePerChain: dict<int>,
 ) => {
-  let items = []
-
   let preparedFetchStates =
     fetchStates
     ->ChainMap.values
-    ->FetchState.filterAndSortForUnorderedBatch(~maxBatchSize)
+    ->FetchState.filterAndSortForUnorderedBatch(~batchSizeTarget)
 
-  let idx = ref(0)
+  let chainIdx = ref(0)
   let preparedNumber = preparedFetchStates->Array.length
   let batchSize = ref(0)
 
+  let items = []
+
   // Accumulate items for all actively indexing chains
   // the way to group as many items from a single chain as possible
   // This way the loaders optimisations will hit more often
-  while batchSize.contents < maxBatchSize && idx.contents < preparedNumber {
-    let fetchState = preparedFetchStates->Js.Array2.unsafe_get(idx.contents)
-    let batchSizeBeforeTheChain = batchSize.contents
-
-    let rec loop = () =>
-      if batchSize.contents < maxBatchSize {
-        let earliestEvent = fetchState->FetchState.getEarliestEvent
-        switch earliestEvent {
-        | NoItem(_) => ()
-        | Item({item, popItemOffQueue}) => {
-            popItemOffQueue()
-            items->Js.Array2.push(item)->ignore
-            batchSize := batchSize.contents + 1
-            loop()
-          }
-        }
-      }
-    loop()
-
-    let chainBatchSize = batchSize.contents - batchSizeBeforeTheChain
+  while batchSize.contents < batchSizeTarget && chainIdx.contents < preparedNumber {
+    let fetchState = preparedFetchStates->Js.Array2.unsafe_get(chainIdx.contents)
+    let chainBatchSize =
+      fetchState->FetchState.getReadyItemsCount(
+        ~targetSize=batchSizeTarget - batchSize.contents,
+        ~fromItem=0,
+      )
     if chainBatchSize > 0 {
-      sizePerChain->Utils.Dict.setByInt(fetchState.chainId, chainBatchSize)
+      for idx in 0 to chainBatchSize - 1 {
+        items->Js.Array2.push(fetchState.buffer->Belt.Array.getUnsafe(idx))->ignore
+      }
+      batchSize := batchSize.contents + chainBatchSize
+      mutBatchSizePerChain->Utils.Dict.setByInt(fetchState.chainId, chainBatchSize)
     }
 
-    idx := idx.contents + 1
+    chainIdx := chainIdx.contents + 1
   }
 
   items

package/src/Batch.res.js

@@ -1,133 +1,113 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
-var Utils = require("./Utils.res.js");
-var Caml_obj = require("rescript/lib/js/caml_obj.js");
 var ChainMap = require("./ChainMap.res.js");
-var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var EventUtils = require("./EventUtils.res.js");
+var Caml_array = require("rescript/lib/js/caml_array.js");
 var FetchState = require("./FetchState.res.js");
 
-function getQueueItemComparitor(earliestQueueItem, chain) {
-  if (earliestQueueItem.TAG === "Item") {
-    return EventUtils.getOrderedBatchItemComparator(earliestQueueItem._0.item);
+function getOrderedNextChain(fetchStates, batchSizePerChain) {
+  var earliestChain;
+  var earliestChainTimestamp = 0;
+  var chainKeys = ChainMap.keys(fetchStates);
+  for(var idx = 0 ,idx_finish = chainKeys.length; idx < idx_finish; ++idx){
+    var chain = Caml_array.get(chainKeys, idx);
+    var fetchState = ChainMap.get(fetchStates, chain);
+    if (FetchState.isActivelyIndexing(fetchState)) {
+      var batchSize = batchSizePerChain[chain];
+      var timestamp = FetchState.getTimestampAt(fetchState, batchSize !== undefined ? batchSize : 0);
+      var earliestChain$1 = earliestChain;
+      if (!(earliestChain$1 !== undefined && (timestamp > earliestChainTimestamp || timestamp === earliestChainTimestamp && chain > earliestChain$1.chainId))) {
+        earliestChain = fetchState;
+        earliestChainTimestamp = timestamp;
+      }
+      
+    }
+    
   }
-  var match = earliestQueueItem.latestFetchedBlock;
-  return [
-          match.blockTimestamp,
-          chain,
-          match.blockNumber,
-          0
-        ];
+  return earliestChain;
 }
 
-function isQueueItemEarlier(a, b) {
-  return Caml_obj.lessthan(getQueueItemComparitor(a.earliestEvent, a.chain), getQueueItemComparitor(b.earliestEvent, b.chain));
-}
+var immutableEmptyBatchSizePerChain = {};
 
-function getOrderedNextItem(fetchStates) {
-  return Belt_Array.reduce(ChainMap.entries(fetchStates), undefined, (function (accum, param) {
-                var fetchState = param[1];
-                if (!FetchState.isActivelyIndexing(fetchState)) {
-                  return accum;
-                }
-                var earliestEvent = FetchState.getEarliestEvent(fetchState);
-                var current_chain = param[0];
-                var current = {
-                  chain: current_chain,
-                  earliestEvent: earliestEvent
-                };
-                if (accum !== undefined && isQueueItemEarlier(accum, current)) {
-                  return accum;
-                } else {
-                  return current;
-                }
-              }));
+function hasOrderedReadyItem(fetchStates) {
+  var fetchState = getOrderedNextChain(fetchStates, immutableEmptyBatchSizePerChain);
+  if (fetchState !== undefined) {
+    return FetchState.hasReadyItem(fetchState);
+  } else {
+    return false;
+  }
 }
 
-function hasUnorderedNextItem(fetchStates) {
+function hasUnorderedReadyItem(fetchStates) {
   return ChainMap.values(fetchStates).some(function (fetchState) {
-              if (!FetchState.isActivelyIndexing(fetchState)) {
-                return false;
-              }
-              var match = FetchState.getEarliestEvent(fetchState);
-              if (match.TAG === "Item") {
-                return true;
+              if (FetchState.isActivelyIndexing(fetchState)) {
+                return FetchState.hasReadyItem(fetchState);
               } else {
                 return false;
               }
             });
 }
 
-function popOrderedBatchItems(maxBatchSize, fetchStates, sizePerChain) {
+function hasMultichainReadyItem(fetchStates, multichain) {
+  if (multichain === "ordered") {
+    return hasOrderedReadyItem(fetchStates);
+  } else {
+    return hasUnorderedReadyItem(fetchStates);
+  }
+}
+
+function prepareOrderedBatch(batchSizeTarget, fetchStates, mutBatchSizePerChain) {
+  var batchSize = 0;
+  var isFinished = false;
   var items = [];
-  var loop = function () {
-    while(true) {
-      if (items.length >= maxBatchSize) {
-        return ;
-      }
-      var match = getOrderedNextItem(fetchStates);
-      if (match === undefined) {
-        return ;
-      }
-      var earliestEvent = match.earliestEvent;
-      if (earliestEvent.TAG !== "Item") {
-        return ;
+  while(batchSize < batchSizeTarget && !isFinished) {
+    var fetchState = getOrderedNextChain(fetchStates, mutBatchSizePerChain);
+    if (fetchState !== undefined) {
+      var batchSize$1 = mutBatchSizePerChain[fetchState.chainId];
+      var itemsCountBefore = batchSize$1 !== undefined ? batchSize$1 : 0;
+      var newItemsCount = FetchState.getReadyItemsCount(fetchState, batchSizeTarget - batchSize | 0, itemsCountBefore);
+      if (newItemsCount > 0) {
+        for(var idx = itemsCountBefore ,idx_finish = itemsCountBefore + newItemsCount | 0; idx < idx_finish; ++idx){
+          items.push(fetchState.buffer[idx]);
+        }
+        batchSize = batchSize + newItemsCount | 0;
+        mutBatchSizePerChain[fetchState.chainId] = itemsCountBefore + newItemsCount | 0;
+      } else {
+        isFinished = true;
       }
-      var match$1 = earliestEvent._0;
-      match$1.popItemOffQueue();
-      items.push(match$1.item);
-      Utils.Dict.incrementByInt(sizePerChain, match.chain);
-      continue ;
-    };
+    } else {
+      isFinished = true;
+    }
   };
-  loop();
   return items;
 }
 
-function popUnorderedBatchItems(maxBatchSize, fetchStates, sizePerChain) {
-  var items = [];
-  var preparedFetchStates = FetchState.filterAndSortForUnorderedBatch(ChainMap.values(fetchStates), maxBatchSize);
-  var idx = 0;
+function prepareUnorderedBatch(batchSizeTarget, fetchStates, mutBatchSizePerChain) {
+  var preparedFetchStates = FetchState.filterAndSortForUnorderedBatch(ChainMap.values(fetchStates), batchSizeTarget);
+  var chainIdx = 0;
   var preparedNumber = preparedFetchStates.length;
-  var batchSize = {
-    contents: 0
-  };
-  while(batchSize.contents < maxBatchSize && idx < preparedNumber) {
-    var fetchState = preparedFetchStates[idx];
-    var batchSizeBeforeTheChain = batchSize.contents;
-    var loop = (function(fetchState){
-    return function loop() {
-      while(true) {
-        if (batchSize.contents >= maxBatchSize) {
-          return ;
-        }
-        var earliestEvent = FetchState.getEarliestEvent(fetchState);
-        if (earliestEvent.TAG !== "Item") {
-          return ;
-        }
-        var match = earliestEvent._0;
-        match.popItemOffQueue();
-        items.push(match.item);
-        batchSize.contents = batchSize.contents + 1 | 0;
-        continue ;
-      };
-    }
-    }(fetchState));
-    loop();
-    var chainBatchSize = batchSize.contents - batchSizeBeforeTheChain | 0;
+  var batchSize = 0;
+  var items = [];
+  while(batchSize < batchSizeTarget && chainIdx < preparedNumber) {
+    var fetchState = preparedFetchStates[chainIdx];
+    var chainBatchSize = FetchState.getReadyItemsCount(fetchState, batchSizeTarget - batchSize | 0, 0);
     if (chainBatchSize > 0) {
-      sizePerChain[fetchState.chainId] = chainBatchSize;
+      for(var idx = 0; idx < chainBatchSize; ++idx){
+        items.push(fetchState.buffer[idx]);
+      }
+      batchSize = batchSize + chainBatchSize | 0;
+      mutBatchSizePerChain[fetchState.chainId] = chainBatchSize;
     }
-    idx = idx + 1 | 0;
+    chainIdx = chainIdx + 1 | 0;
   };
   return items;
 }
 
-exports.getQueueItemComparitor = getQueueItemComparitor;
-exports.isQueueItemEarlier = isQueueItemEarlier;
-exports.getOrderedNextItem = getOrderedNextItem;
-exports.hasUnorderedNextItem = hasUnorderedNextItem;
-exports.popOrderedBatchItems = popOrderedBatchItems;
-exports.popUnorderedBatchItems = popUnorderedBatchItems;
-/* Utils Not a pure module */
+exports.getOrderedNextChain = getOrderedNextChain;
+exports.immutableEmptyBatchSizePerChain = immutableEmptyBatchSizePerChain;
+exports.hasOrderedReadyItem = hasOrderedReadyItem;
+exports.hasUnorderedReadyItem = hasUnorderedReadyItem;
+exports.hasMultichainReadyItem = hasMultichainReadyItem;
+exports.prepareOrderedBatch = prepareOrderedBatch;
+exports.prepareUnorderedBatch = prepareUnorderedBatch;
+/* ChainMap Not a pure module */