envio 3.3.0-alpha.6 → 3.3.0-alpha.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.3.0-alpha.6",
+  "version": "3.3.0-alpha.7",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.3.0-alpha.6",
-    "envio-linux-x64-musl": "3.3.0-alpha.6",
-    "envio-linux-arm64": "3.3.0-alpha.6",
-    "envio-darwin-x64": "3.3.0-alpha.6",
-    "envio-darwin-arm64": "3.3.0-alpha.6"
+    "envio-linux-x64": "3.3.0-alpha.7",
+    "envio-linux-x64-musl": "3.3.0-alpha.7",
+    "envio-linux-arm64": "3.3.0-alpha.7",
+    "envio-darwin-x64": "3.3.0-alpha.7",
+    "envio-darwin-arm64": "3.3.0-alpha.7"
   }
 }

package/src/Address.res

@@ -11,8 +11,11 @@ module Evm = {
   external fromStringOrThrow: string => t = "getAddress"
 
   // NOTE: the function is named to be overshadowed by the one below, so that we don't have to import viem in the handler code
+  // strict:false checks only the 20-byte hex shape, not EIP-55 checksum casing —
+  // since we're about to lowercase it anyway, an all-uppercase or wrongly-cased
+  // input address is just as valid as one that's already lowercase.
   @module("viem")
-  external fromStringLowercaseOrThrow: string => bool = "isAddress"
+  external fromStringLowercaseOrThrow: (string, {"strict": bool}) => bool = "isAddress"
 
   // Reassign since the function might be used in the handler code
   // and we don't want to have a "viem" import there. It's needed to keep "viem" a dependency
@@ -20,7 +23,7 @@ module Evm = {
   // Also, we want a custom error message, which is searchable in our codebase.
   // Validate that the string is a proper address but return a lowercased value
   let fromStringLowercaseOrThrow = string => {
-    if fromStringLowercaseOrThrow(string) {
+    if fromStringLowercaseOrThrow(string, {"strict": false}) {
       unsafeFromString(string->String.toLowerCase)
     } else {
       JsError.throwWithMessage(

package/src/Address.res.mjs

@@ -7,7 +7,9 @@ import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 let schema = S$RescriptSchema.setName(S$RescriptSchema.string, "Address");
 
 function fromStringLowercaseOrThrow(string) {
-  if (Viem.isAddress(string)) {
+  if (Viem.isAddress(string, {
+      strict: false
+    })) {
     return string.toLowerCase();
   } else {
     return Stdlib_JsError.throwWithMessage(`Address "` + string + `" is invalid. Expected a 20-byte hex string starting with 0x.`);

package/src/ChainFetching.res

@@ -140,6 +140,20 @@ let rec onQueryResponse = async (
       ~blockRangeSize=latestFetchedBlockNumber - fromBlockQueried + 1,
     )
 
+    let numContractRegisterEvents = parsedQueueItems->Array.reduce(0, (count, item) => {
+      let eventItem = item->Internal.castUnsafeEventItem
+      eventItem.eventConfig.contractRegister !== None ? count + 1 : count
+    })
+    Logging.trace({
+      "msg": "Finished querying",
+      "chainId": chain->ChainMap.Chain.toChainId,
+      "partitionId": query.partitionId,
+      "fromBlock": fromBlockQueried,
+      "toBlock": latestFetchedBlockNumber,
+      "numEvents": parsedQueueItems->Array.length,
+      "numContractRegisterEvents": numContractRegisterEvents,
+    })
+
     let reorgResult = chainState->ChainState.registerReorgGuard(~blockHashes, ~knownHeight)
 
     let rollbackWithReorgDetectedBlockNumber = switch reorgResult {
@@ -334,15 +348,6 @@ let fetchChain = async (
     // there's nothing to fetch. During backfill any such chain is idle.
     let reducedPolling = !isRealtime
 
-    // Accumulated across all queries dispatched in this tick so their per-query
-    // results collapse into a single completion log instead of one per query.
-    let dispatchedCount = switch action {
-    | FetchState.Ready(queries) => queries->Array.length
-    | WaitingForNewBlock | NothingToQuery => 0
-    }
-    let fetchedByPartition = Dict.make()
-    let fetchedCount = ref(0)
-
     // Owns its error boundary: launch doesn't catch, so any failure here (the
     // query, response handling, or dispatch itself) must stop the indexer.
     try {
@@ -368,22 +373,6 @@ let fetchChain = async (
               ~knownHeight=chainState->ChainState.knownHeight,
               ~isRealtime,
             )
-            fetchedCount := fetchedCount.contents + 1
-            fetchedByPartition->Dict.set(
-              query.partitionId,
-              {
-                "fromBlock": response.fromBlockQueried,
-                "toBlock": response.latestFetchedBlockNumber,
-                "numEvents": response.parsedQueueItems->Array.length,
-              },
-            )
-            if dispatchedCount > 0 && fetchedCount.contents === dispatchedCount {
-              Logging.trace({
-                "msg": "Finished querying",
-                "chainId": chain->ChainMap.Chain.toChainId,
-                "partitions": fetchedByPartition,
-              })
-            }
             await onQueryResponse(
               state,
               {chain, response, query},

package/src/ChainFetching.res.mjs

@@ -6,6 +6,7 @@ import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
@@ -94,13 +95,30 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let stats = response.stats;
   let latestFetchedBlockTimestamp = response.latestFetchedBlockTimestamp;
   let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
+  let fromBlockQueried = response.fromBlockQueried;
   let transactionStore = response.transactionStore;
   let parsedQueueItems = response.parsedQueueItems;
   let knownHeight = response.knownHeight;
   if (knownHeight > ChainState.knownHeight(chainState)) {
     Prometheus.SourceHeight.set(SourceManager.getActiveSource(ChainState.sourceManager(chainState)).name, ChainState.chainConfig(chainState).id, knownHeight);
   }
-  Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
+  Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - fromBlockQueried | 0) + 1 | 0);
+  let numContractRegisterEvents = Stdlib_Array.reduce(parsedQueueItems, 0, (count, item) => {
+    if (item.eventConfig.contractRegister !== undefined) {
+      return count + 1 | 0;
+    } else {
+      return count;
+    }
+  });
+  Logging.trace({
+    msg: "Finished querying",
+    chainId: chain,
+    partitionId: query.partitionId,
+    fromBlock: fromBlockQueried,
+    toBlock: latestFetchedBlockNumber,
+    numEvents: parsedQueueItems.length,
+    numContractRegisterEvents: numContractRegisterEvents
+  });
   let reorgResult = ChainState.registerReorgGuard(chainState, response.blockHashes, knownHeight);
   let rollbackWithReorgDetectedBlockNumber;
   if (typeof reorgResult !== "object") {
@@ -176,29 +194,10 @@ async function fetchChain(state, chain, action, stateId, scheduleFetch, schedule
   let isRealtime = IndexerState.isRealtime(state);
   let sourceManager = ChainState.sourceManager(chainState);
   let reducedPolling = !isRealtime;
-  let dispatchedCount;
-  dispatchedCount = typeof action !== "object" ? 0 : action._0.length;
-  let fetchedByPartition = {};
-  let fetchedCount = {
-    contents: 0
-  };
   try {
     return await ChainState.dispatch(chainState, async query => {
       try {
         let response = await SourceManager.executeQuery(sourceManager, query, ChainState.knownHeight(chainState), isRealtime);
-        fetchedCount.contents = fetchedCount.contents + 1 | 0;
-        fetchedByPartition[query.partitionId] = {
-          fromBlock: response.fromBlockQueried,
-          toBlock: response.latestFetchedBlockNumber,
-          numEvents: response.parsedQueueItems.length
-        };
-        if (dispatchedCount > 0 && fetchedCount.contents === dispatchedCount) {
-          Logging.trace({
-            msg: "Finished querying",
-            chainId: chain,
-            partitions: fetchedByPartition
-          });
-        }
         return await onQueryResponse(state, {
           chain: chain,
           response: response,

package/src/ChainState.res

@@ -457,10 +457,10 @@ let resetPendingQueries = (cs: t) => {
   cs.pendingBudget = 0.
 }
 
-// Propose the chain's candidate queries against the shared buffer budget,
-// accounting for this chain's own in-flight reservations.
-let getNextQuery = (cs: t, ~budget) =>
-  cs.fetchState->FetchState.getNextQuery(~budget, ~chainPendingBudget=cs.pendingBudget)
+// Propose the chain's candidate queries against their natural ceiling
+// (head/endBlock/mergeBlock). CrossChainState admits them against the shared
+// buffer budget afterwards.
+let getNextQuery = (cs: t) => cs.fetchState->FetchState.getNextQuery
 
 // Run a fetch tick for this chain against its sources, feeding the owned fetch
 // frontier to the source manager.

package/src/ChainState.res.mjs

@@ -282,8 +282,8 @@ function startFetchingQueries(cs, queries) {
   cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.estResponseSize);
 }
 
-function getNextQuery(cs, budget) {
-  return FetchState.getNextQuery(cs.fetchState, budget, cs.pendingBudget);
+function getNextQuery(cs) {
+  return FetchState.getNextQuery(cs.fetchState);
 }
 
 function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {

package/src/ChainState.resi

@@ -62,7 +62,7 @@ let hasReadyItem: t => bool
 let isReadyToEnterReorgThreshold: t => bool
 
 // Fetch control.
-let getNextQuery: (t, ~budget: int) => FetchState.nextQuery
+let getNextQuery: t => FetchState.nextQuery
 let dispatch: (
   t,
   ~executeQuery: FetchState.query => promise<unit>,

package/src/Config.res

@@ -1018,6 +1018,50 @@ let fromPublic = (publicConfigJson: JSON.t) => {
   }
 }
 
+// Canonicalize a user-provided address to the configured casing so it matches
+// addresses parsed from config.yaml during routing. HyperSync/RPC data arrives
+// already canonical; only user-land input (simulate srcAddress, contractRegister
+// add) can carry arbitrary casing and needs this before comparison.
+let normalizeUserAddress = (config: t, address: Address.t): Address.t =>
+  switch config.ecosystem.name {
+  | Ecosystem.Evm =>
+    if config.lowercaseAddresses {
+      address->Address.Evm.fromAddressLowercaseOrThrow
+    } else {
+      address->Address.Evm.fromAddressOrThrow
+    }
+  // TODO: Ideally we should do the same for other ecosystems
+  | Ecosystem.Fuel | Ecosystem.Svm => address
+  }
+
+// Relaxed counterpart to normalizeUserAddress, used only for simulate srcAddress.
+// Tests commonly use short placeholder strings like "0xfoo" as a stand-in
+// identity (e.g. for a wildcard event whose srcAddress doesn't need to resolve
+// to any real contract), so only the "0x" prefix is enforced here. Under
+// address_format: checksum, a real address is checksummed even if the input
+// casing doesn't match (getAddress doesn't require the input to already be
+// checksummed) — a placeholder that isn't valid hex falls back unchanged.
+let normalizeSimulateAddress = (config: t, address: Address.t): Address.t =>
+  switch config.ecosystem.name {
+  | Ecosystem.Evm =>
+    let str = address->Address.toString
+    if !(str->String.startsWith("0x")) {
+      JsError.throwWithMessage(
+        `simulate: srcAddress "${str}" is invalid. Expected a string starting with "0x".`,
+      )
+    }
+    if config.lowercaseAddresses {
+      address->Address.Evm.fromAddressLowercaseOrThrow
+    } else {
+      try {
+        address->Address.Evm.fromAddressOrThrow
+      } catch {
+      | _ => address
+      }
+    }
+  | Ecosystem.Fuel | Ecosystem.Svm => address
+  }
+
 // Look up an event config by (contract, event) name. When `chainId` is given,
 // returns that chain's per-chain event config (matters for where-callback
 // probe detection, which runs with the chain's real id). Without `chainId`,

package/src/Config.res.mjs

@@ -706,6 +706,42 @@ function fromPublic(publicConfigJson) {
   };
 }
 
+function normalizeUserAddress(config, address) {
+  let match = config.ecosystem.name;
+  switch (match) {
+    case "evm" :
+      if (config.lowercaseAddresses) {
+        return Address.Evm.fromAddressLowercaseOrThrow(address);
+      } else {
+        return Address.Evm.fromAddressOrThrow(address);
+      }
+    case "fuel" :
+    case "svm" :
+      return address;
+  }
+}
+
+function normalizeSimulateAddress(config, address) {
+  let match = config.ecosystem.name;
+  switch (match) {
+    case "evm" :
+      if (!address.startsWith("0x")) {
+        Stdlib_JsError.throwWithMessage(`simulate: srcAddress "` + address + `" is invalid. Expected a string starting with "0x".`);
+      }
+      if (config.lowercaseAddresses) {
+        return Address.Evm.fromAddressLowercaseOrThrow(address);
+      }
+      try {
+        return Address.Evm.fromAddressOrThrow(address);
+      } catch (exn) {
+        return address;
+      }
+    case "fuel" :
+    case "svm" :
+      return address;
+  }
+}
+
 function getEventConfig(config, contractName, eventName, chainId) {
   let chains;
   if (chainId !== undefined) {
@@ -1012,6 +1048,8 @@ export {
   publicConfigStorageSchema,
   publicConfigSchema,
   fromPublic,
+  normalizeUserAddress,
+  normalizeSimulateAddress,
   getEventConfig,
   shouldSaveHistory,
   shouldPruneHistory,

package/src/ContractRegisterContext.res

@@ -22,18 +22,8 @@ let makeAddFunction = (~params: contractRegisterParams, ~contractName: string):
         ~logger=Ecosystem.getItemLogger(params.item, ~ecosystem=params.config.ecosystem),
       )
     }
-    let validatedAddress = if params.config.ecosystem.name === Evm {
-      // The value is passed from the user-land,
-      // so we need to validate and checksum/lowercase the address.
-      if params.config.lowercaseAddresses {
-        contractAddress->Address.Evm.fromAddressLowercaseOrThrow
-      } else {
-        contractAddress->Address.Evm.fromAddressOrThrow
-      }
-    } else {
-      // TODO: Ideally we should do the same for other ecosystems
-      contractAddress
-    }
+    // The value is passed from user-land, so validate and checksum/lowercase it.
+    let validatedAddress = params.config->Config.normalizeUserAddress(contractAddress)
 
     params.onRegister(~item=params.item, ~contractAddress=validatedAddress, ~contractName)
   }

package/src/ContractRegisterContext.res.mjs

@@ -1,6 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
-import * as Address from "./Address.res.mjs";
+import * as Config from "./Config.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as EntityFilter from "./db/EntityFilter.res.mjs";
@@ -12,9 +12,7 @@ function makeAddFunction(params, contractName) {
     if (params.isResolved) {
       ErrorHandling.mkLogAndRaise(Ecosystem.getItemLogger(params.item, params.config.ecosystem), undefined, new Error(`Impossible to access context.chain after the contract register is resolved. Make sure you didn't miss an await in the handler.`));
     }
-    let validatedAddress = params.config.ecosystem.name === "evm" ? (
-        params.config.lowercaseAddresses ? Address.Evm.fromAddressLowercaseOrThrow(contractAddress) : Address.Evm.fromAddressOrThrow(contractAddress)
-      ) : contractAddress;
+    let validatedAddress = Config.normalizeUserAddress(params.config, contractAddress);
     params.onRegister(params.item, validatedAddress, contractName);
   };
 }
@@ -73,4 +71,4 @@ export {
   getContractRegisterContext,
   getContractRegisterArgs,
 }
-/* Address Not a pure module */
+/* Config Not a pure module */

package/src/CrossChainState.res

@@ -19,7 +19,7 @@ type t = {
 let calculateTargetBufferSize = () =>
   switch Env.targetBufferSize {
   | Some(size) => size
-  | None => 100_000
+  | None => 50_000
   }
 
 let make = (
@@ -198,7 +198,8 @@ let compareByProgress = (a: FetchState.query, b: FetchState.query) =>
 
 // Dispatch a fetch tick across the whole indexer from one shared pool of
 // ~targetBufferSize ready events. Every chain proposes its candidate queries
-// (each carrying an estimated response size) against the full free budget; the
+// (each carrying an estimated response size) up to its own natural ceiling
+// (head/endBlock/mergeBlock), unconstrained by the shared budget; the
 // candidates are then pooled, ordered by chain progress (furthest-behind first),
 // and admitted until the budget is consumed. So the budget is split per query
 // across chains rather than per chain — a chain that can only use a little
@@ -223,7 +224,7 @@ let checkAndFetch = async (
   for i in 0 to chainIds->Array.length - 1 {
     let chainId = chainIds->Array.getUnsafe(i)
     let cs = crossChainState->getChainState(chainId)
-    switch cs->ChainState.getNextQuery(~budget=remaining) {
+    switch cs->ChainState.getNextQuery {
     | (WaitingForNewBlock | NothingToQuery) as action =>
       actionByChain->Utils.Dict.setByInt(chainId, action)
     | Ready(queries) =>
@@ -261,6 +262,7 @@ let checkAndFetch = async (
         {
           "fromBlock": query.fromBlock,
           "targetBlock": query.toBlock,
+          "targetEvents": query.estResponseSize->Math.round->Float.toInt,
         },
       )
     )

package/src/CrossChainState.res.mjs

@@ -17,13 +17,13 @@ function calculateTargetBufferSize() {
   if (Env.targetBufferSize !== undefined) {
     return Env.targetBufferSize;
   } else {
-    return 100000;
+    return 50000;
   }
 }
 
 function make(chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt) {
   let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : (
-      Env.targetBufferSize !== undefined ? Env.targetBufferSize : 100000
+      Env.targetBufferSize !== undefined ? Env.targetBufferSize : 50000
     );
   let crossChainState = {
     chainStates: chainStates,
@@ -147,7 +147,7 @@ async function checkAndFetch(crossChainState, dispatchChain) {
   for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
     let chainId = chainIds[i];
     let cs = crossChainState.chainStates[chainId];
-    let action = ChainState.getNextQuery(cs, remaining);
+    let action = ChainState.getNextQuery(cs);
     if (typeof action !== "object") {
       actionByChain[chainId] = action;
     } else {
@@ -174,7 +174,8 @@ async function checkAndFetch(crossChainState, dispatchChain) {
     queries.forEach(query => {
       partitions[query.partitionId] = {
         fromBlock: query.fromBlock,
-        targetBlock: query.toBlock
+        targetBlock: query.toBlock,
+        targetEvents: Math.round(query.estResponseSize) | 0
       };
     });
     Logging.trace({