envio 3.3.0-alpha.6 → 3.3.0-alpha.8

package/index.d.ts

@@ -979,13 +979,24 @@ export type SvmInstructionParams = {
   readonly extraAccounts: readonly string[];
 };
 
-/** Block context for a matched instruction. */
+/** Permissive fallback shape for an instruction's `block`. The generated
+ * per-instruction type narrows this to `slot`/`hash` (always present) and
+ * `time` (always present but possibly `undefined`), plus the selected
+ * `field_selection.block_fields`. */
 export type SvmInstructionBlock = {
   /** Slot this instruction's block was matched in. */
   readonly slot: number;
-  readonly time: number;
-  /** Always empty for now — reserved for the future reorg-guard route. */
+  /** Unix block time (seconds). Absent when HyperSync/Solana doesn't report a
+   * block time for this slot. */
+  readonly time?: number;
+  /** Block hash. */
   readonly hash: string;
+  /** Block height. Select via `field_selection.block_fields`. */
+  readonly height?: number;
+  /** Parent slot. Select via `field_selection.block_fields`. */
+  readonly parentSlot?: number;
+  /** Parent block hash. Select via `field_selection.block_fields`. */
+  readonly parentHash?: string;
 };
 
 export type SvmTokenBalance = {
@@ -1016,6 +1027,7 @@ export type SvmLog = {
 export type SvmInstruction<
   Params extends SvmInstructionParams = SvmInstructionParams,
   Tx = SvmTransaction,
+  Block = SvmInstructionBlock,
 > = {
   /** Program name as declared under `programs[].name` in `config.yaml`. */
   readonly programName: string;
@@ -1041,7 +1053,12 @@ export type SvmInstruction<
   /** Present when the instruction's `include_logs` is `true`; only logs
    * scoped to this exact instruction (matching `instruction_address`). */
   readonly logs?: readonly SvmLog[];
-  readonly block: SvmInstructionBlock;
+  /** The block this instruction's slot belongs to. Carries `slot`/`hash`
+   * (always present) and `time` (always present but possibly `undefined`),
+   * plus the fields selected via this instruction's
+   * `field_selection.block_fields`; unselected fields are typed as
+   * `FieldNotSelected<...>`. */
+  readonly block: Block;
 };
 
 /** Arguments passed to handlers registered via `indexer.onInstruction`. */
@@ -1315,7 +1332,8 @@ type SvmEcosystem<Config extends IndexerConfigTypes = GlobalConfig> =
                       Config,
                       SvmInstruction<
                         SvmParamsFromProgramTable<Programs[P][I]>,
-                        Programs[P][I]["transaction"]
+                        Programs[P][I]["transaction"],
+                        Programs[P][I]["block"]
                       >
                     >,
                   ) => Promise<void>,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.3.0-alpha.6",
+  "version": "3.3.0-alpha.8",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.3.0-alpha.6",
-    "envio-linux-x64-musl": "3.3.0-alpha.6",
-    "envio-linux-arm64": "3.3.0-alpha.6",
-    "envio-darwin-x64": "3.3.0-alpha.6",
-    "envio-darwin-arm64": "3.3.0-alpha.6"
+    "envio-linux-x64": "3.3.0-alpha.8",
+    "envio-linux-x64-musl": "3.3.0-alpha.8",
+    "envio-linux-arm64": "3.3.0-alpha.8",
+    "envio-darwin-x64": "3.3.0-alpha.8",
+    "envio-darwin-arm64": "3.3.0-alpha.8"
   }
 }

package/src/Address.res

@@ -11,8 +11,11 @@ module Evm = {
   external fromStringOrThrow: string => t = "getAddress"
 
   // NOTE: the function is named to be overshadowed by the one below, so that we don't have to import viem in the handler code
+  // strict:false checks only the 20-byte hex shape, not EIP-55 checksum casing —
+  // since we're about to lowercase it anyway, an all-uppercase or wrongly-cased
+  // input address is just as valid as one that's already lowercase.
   @module("viem")
-  external fromStringLowercaseOrThrow: string => bool = "isAddress"
+  external fromStringLowercaseOrThrow: (string, {"strict": bool}) => bool = "isAddress"
 
   // Reassign since the function might be used in the handler code
   // and we don't want to have a "viem" import there. It's needed to keep "viem" a dependency
@@ -20,7 +23,7 @@ module Evm = {
   // Also, we want a custom error message, which is searchable in our codebase.
   // Validate that the string is a proper address but return a lowercased value
   let fromStringLowercaseOrThrow = string => {
-    if fromStringLowercaseOrThrow(string) {
+    if fromStringLowercaseOrThrow(string, {"strict": false}) {
       unsafeFromString(string->String.toLowerCase)
     } else {
       JsError.throwWithMessage(

package/src/Address.res.mjs

@@ -7,7 +7,9 @@ import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 let schema = S$RescriptSchema.setName(S$RescriptSchema.string, "Address");
 
 function fromStringLowercaseOrThrow(string) {
-  if (Viem.isAddress(string)) {
+  if (Viem.isAddress(string, {
+      strict: false
+    })) {
     return string.toLowerCase();
   } else {
     return Stdlib_JsError.throwWithMessage(`Address "` + string + `" is invalid. Expected a 20-byte hex string starting with 0x.`);

package/src/Api.res

@@ -9,7 +9,7 @@ let createTestIndexer: unit => unknown = () => {
       NodeJs.Path.getDirname(NodeJs.ImportMeta.importMeta),
       "TestIndexerWorker.res.mjs",
     )->NodeJs.Path.toString
-  TestIndexer.makeCreateTestIndexer(~config=Config.loadWithoutRegistrations(), ~workerPath)()->(
+  TestIndexer.makeCreateTestIndexer(~config=Config.load(), ~workerPath)()->(
     Utils.magic: TestIndexer.t<'a> => unknown
   )
 }

package/src/Api.res.mjs

@@ -10,7 +10,7 @@ let indexer = Main.getGlobalIndexer();
 
 function createTestIndexer() {
   let workerPath = Path.join(NodeJs.Path.getDirname(import.meta), "TestIndexerWorker.res.mjs");
-  return TestIndexer.makeCreateTestIndexer(Config.loadWithoutRegistrations(), workerPath)();
+  return TestIndexer.makeCreateTestIndexer(Config.load(), workerPath)();
 }
 
 export {

package/src/ChainFetching.res

@@ -11,12 +11,18 @@ type partitionQueryResponse = {
 let runContractRegistersOrThrow = async (
   ~itemsWithContractRegister: array<Internal.item>,
   ~config: Config.t,
-  ~page: option<TransactionStore.t>,
+  ~transactionStore: option<TransactionStore.t>,
+  ~blockStore: option<BlockStore.t>,
 ) => {
-  // contractRegister handlers can read event.transaction, so materialise the
-  // selected fields onto the payloads before running them. All items belong to
-  // the chain being fetched, hence its single page store and mask.
-  await ChainState.materializePageItems(~items=itemsWithContractRegister, ~page)
+  // contractRegister handlers can read event.transaction and event.block, so
+  // materialise the selected fields onto the payloads before running them. All
+  // items belong to the chain being fetched, hence its single page stores.
+  await ChainState.materializePageItems(
+    ~items=itemsWithContractRegister,
+    ~transactionStore,
+    ~blockStore,
+    ~ecosystem=config.ecosystem.name,
+  )
 
   let itemsWithDcs = []
 
@@ -44,8 +50,8 @@ let runContractRegistersOrThrow = async (
     let item = itemsWithContractRegister->Array.getUnsafe(idx)
     let eventItem = item->Internal.castUnsafeEventItem
     let contractRegister = switch eventItem {
-    | {eventConfig: {contractRegister: Some(contractRegister)}} => contractRegister
-    | {eventConfig: {contractRegister: None, name: eventName}} =>
+    | {onEventRegistration: {contractRegister: Some(contractRegister)}} => contractRegister
+    | {onEventRegistration: {contractRegister: None, eventConfig: {name: eventName}}} =>
       // Unexpected case, since we should pass only events with contract register to this function
       JsError.throwWithMessage("Contract register is not set for event " ++ eventName)
     }
@@ -113,6 +119,7 @@ let rec onQueryResponse = async (
     let {
       parsedQueueItems,
       transactionStore,
+      blockStore,
       latestFetchedBlockNumber,
       latestFetchedBlockTimestamp,
       stats,
@@ -140,6 +147,20 @@ let rec onQueryResponse = async (
       ~blockRangeSize=latestFetchedBlockNumber - fromBlockQueried + 1,
     )
 
+    let numContractRegisterEvents = parsedQueueItems->Array.reduce(0, (count, item) => {
+      let eventItem = item->Internal.castUnsafeEventItem
+      eventItem.onEventRegistration.contractRegister !== None ? count + 1 : count
+    })
+    Logging.trace({
+      "msg": "Finished querying",
+      "chainId": chain->ChainMap.Chain.toChainId,
+      "partitionId": query.partitionId,
+      "fromBlock": fromBlockQueried,
+      "toBlock": latestFetchedBlockNumber,
+      "numEvents": parsedQueueItems->Array.length,
+      "numContractRegisterEvents": numContractRegisterEvents,
+    })
+
     let reorgResult = chainState->ChainState.registerReorgGuard(~blockHashes, ~knownHeight)
 
     let rollbackWithReorgDetectedBlockNumber = switch reorgResult {
@@ -196,7 +217,7 @@ let rec onQueryResponse = async (
       for idx in 0 to parsedQueueItems->Array.length - 1 {
         let item = parsedQueueItems->Array.getUnsafe(idx)
         let eventItem = item->Internal.castUnsafeEventItem
-        if eventItem.eventConfig.contractRegister !== None {
+        if eventItem.onEventRegistration.contractRegister !== None {
           itemsWithContractRegister->Array.push(item)
         }
         // TODO: Don't really need to keep it in the queue
@@ -220,6 +241,7 @@ let rec onQueryResponse = async (
             },
             ~query,
             ~transactionStore,
+            ~blockStore,
           )
           ChainMetadata.stage(state)
           scheduleFetch()
@@ -232,7 +254,8 @@ let rec onQueryResponse = async (
         switch await runContractRegistersOrThrow(
           ~itemsWithContractRegister,
           ~config=state->IndexerState.config,
-          ~page=transactionStore,
+          ~transactionStore,
+          ~blockStore,
         ) {
         | exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
         | newItemsWithDcs => proceed(~newItemsWithDcs)
@@ -250,6 +273,7 @@ and applyQueryResponse = (
   ~latestFetchedBlock,
   ~query,
   ~transactionStore,
+  ~blockStore,
 ) => {
   let chainState = state->IndexerState.getChainState(~chain)
   let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
@@ -261,6 +285,7 @@ and applyQueryResponse = (
     ~newItemsWithDcs,
     ~knownHeight,
     ~transactionStore,
+    ~blockStore,
   )
 
   // In auto-exit mode, set endBlock to the first event's block when events arrive.
@@ -334,15 +359,6 @@ let fetchChain = async (
     // there's nothing to fetch. During backfill any such chain is idle.
     let reducedPolling = !isRealtime
 
-    // Accumulated across all queries dispatched in this tick so their per-query
-    // results collapse into a single completion log instead of one per query.
-    let dispatchedCount = switch action {
-    | FetchState.Ready(queries) => queries->Array.length
-    | WaitingForNewBlock | NothingToQuery => 0
-    }
-    let fetchedByPartition = Dict.make()
-    let fetchedCount = ref(0)
-
     // Owns its error boundary: launch doesn't catch, so any failure here (the
     // query, response handling, or dispatch itself) must stop the indexer.
     try {
@@ -368,22 +384,6 @@ let fetchChain = async (
               ~knownHeight=chainState->ChainState.knownHeight,
               ~isRealtime,
             )
-            fetchedCount := fetchedCount.contents + 1
-            fetchedByPartition->Dict.set(
-              query.partitionId,
-              {
-                "fromBlock": response.fromBlockQueried,
-                "toBlock": response.latestFetchedBlockNumber,
-                "numEvents": response.parsedQueueItems->Array.length,
-              },
-            )
-            if dispatchedCount > 0 && fetchedCount.contents === dispatchedCount {
-              Logging.trace({
-                "msg": "Finished querying",
-                "chainId": chain->ChainMap.Chain.toChainId,
-                "partitions": fetchedByPartition,
-              })
-            }
             await onQueryResponse(
               state,
               {chain, response, query},

package/src/ChainFetching.res.mjs

@@ -6,6 +6,7 @@ import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ChainMetadata from "./ChainMetadata.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as SourceManager from "./sources/SourceManager.res.mjs";
@@ -16,8 +17,8 @@ import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
 
-async function runContractRegistersOrThrow(itemsWithContractRegister, config, page) {
-  await ChainState.materializePageItems(itemsWithContractRegister, page);
+async function runContractRegistersOrThrow(itemsWithContractRegister, config, transactionStore, blockStore) {
+  await ChainState.materializePageItems(itemsWithContractRegister, transactionStore, blockStore, config.ecosystem.name);
   let itemsWithDcs = [];
   let onRegister = (item, contractAddress, contractName) => {
     let dc_registrationBlock = item.blockNumber;
@@ -37,9 +38,9 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config, pa
   let promises = [];
   for (let idx = 0, idx_finish = itemsWithContractRegister.length; idx < idx_finish; ++idx) {
     let item = itemsWithContractRegister[idx];
-    let match = item.eventConfig;
+    let match = item.onEventRegistration;
     let contractRegister = match.contractRegister;
-    let contractRegister$1 = contractRegister !== undefined ? contractRegister : Stdlib_JsError.throwWithMessage("Contract register is not set for event " + match.name);
+    let contractRegister$1 = contractRegister !== undefined ? contractRegister : Stdlib_JsError.throwWithMessage("Contract register is not set for event " + match.eventConfig.name);
     let errorMessage = "Event contractRegister failed, please fix the error to keep the indexer running smoothly";
     try {
       let params = {
@@ -71,10 +72,10 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config, pa
   return itemsWithDcs;
 }
 
-function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, transactionStore) {
+function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, transactionStore, blockStore) {
   let chainState = IndexerState.getChainState(state, chain);
   let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
-  ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, transactionStore);
+  ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, transactionStore, blockStore);
   if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
     ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
   }
@@ -94,13 +95,31 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let stats = response.stats;
   let latestFetchedBlockTimestamp = response.latestFetchedBlockTimestamp;
   let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
+  let fromBlockQueried = response.fromBlockQueried;
+  let blockStore = response.blockStore;
   let transactionStore = response.transactionStore;
   let parsedQueueItems = response.parsedQueueItems;
   let knownHeight = response.knownHeight;
   if (knownHeight > ChainState.knownHeight(chainState)) {
     Prometheus.SourceHeight.set(SourceManager.getActiveSource(ChainState.sourceManager(chainState)).name, ChainState.chainConfig(chainState).id, knownHeight);
   }
-  Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
+  Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - fromBlockQueried | 0) + 1 | 0);
+  let numContractRegisterEvents = Stdlib_Array.reduce(parsedQueueItems, 0, (count, item) => {
+    if (item.onEventRegistration.contractRegister !== undefined) {
+      return count + 1 | 0;
+    } else {
+      return count;
+    }
+  });
+  Logging.trace({
+    msg: "Finished querying",
+    chainId: chain,
+    partitionId: query.partitionId,
+    fromBlock: fromBlockQueried,
+    toBlock: latestFetchedBlockNumber,
+    numEvents: parsedQueueItems.length,
+    numContractRegisterEvents: numContractRegisterEvents
+  });
   let reorgResult = ChainState.registerReorgGuard(chainState, response.blockHashes, knownHeight);
   let rollbackWithReorgDetectedBlockNumber;
   if (typeof reorgResult !== "object") {
@@ -124,7 +143,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   let newItems = [];
   for (let idx = 0, idx_finish = parsedQueueItems.length; idx < idx_finish; ++idx) {
     let item = parsedQueueItems[idx];
-    if (item.eventConfig.contractRegister !== undefined) {
+    if (item.onEventRegistration.contractRegister !== undefined) {
       itemsWithContractRegister.push(item);
     }
     newItems.push(item);
@@ -134,7 +153,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
       applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, {
         blockNumber: latestFetchedBlockNumber,
         blockTimestamp: latestFetchedBlockTimestamp
-      }, query, transactionStore);
+      }, query, transactionStore, blockStore);
       ChainMetadata.stage(state);
       scheduleFetch();
       return scheduleProcessing();
@@ -145,7 +164,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
   }
   let newItemsWithDcs;
   try {
-    newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), transactionStore);
+    newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), transactionStore, blockStore);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
@@ -176,29 +195,10 @@ async function fetchChain(state, chain, action, stateId, scheduleFetch, schedule
   let isRealtime = IndexerState.isRealtime(state);
   let sourceManager = ChainState.sourceManager(chainState);
   let reducedPolling = !isRealtime;
-  let dispatchedCount;
-  dispatchedCount = typeof action !== "object" ? 0 : action._0.length;
-  let fetchedByPartition = {};
-  let fetchedCount = {
-    contents: 0
-  };
   try {
     return await ChainState.dispatch(chainState, async query => {
       try {
         let response = await SourceManager.executeQuery(sourceManager, query, ChainState.knownHeight(chainState), isRealtime);
-        fetchedCount.contents = fetchedCount.contents + 1 | 0;
-        fetchedByPartition[query.partitionId] = {
-          fromBlock: response.fromBlockQueried,
-          toBlock: response.latestFetchedBlockNumber,
-          numEvents: response.parsedQueueItems.length
-        };
-        if (dispatchedCount > 0 && fetchedCount.contents === dispatchedCount) {
-          Logging.trace({
-            msg: "Finished querying",
-            chainId: chain,
-            partitions: fetchedByPartition
-          });
-        }
         return await onQueryResponse(state, {
           chain: chain,
           response: response,