envio 3.6.1 → 3.7.0

package/src/sources/Fuel.res

@@ -1,5 +1,7 @@
 // Fuel's concrete item payload. Erased to `Internal.eventPayload` on the item
-// and recovered here via `toPayload`.
+// and recovered here via `toPayload`. The block lives raw in the per-chain
+// store and is written onto the payload at batch prep (like HyperSync EVM);
+// simulate builds it inline.
 type payload = {
   contractName: string,
   eventName: string,
@@ -8,11 +10,24 @@ type payload = {
   srcAddress: Address.t,
   logIndex: int,
   transaction: Internal.eventTransaction,
-  block: Internal.eventBlock,
+  block?: Internal.eventBlock,
 }
 external fromPayload: payload => Internal.eventPayload = "%identity"
 external toPayload: Internal.eventPayload => payload = "%identity"
 
+// Ordered block field names. The index of each is the field code shared with
+// the Rust store (`FuelBlockField`) — keep this order in sync.
+let blockFields = ["height", "id", "time"]
+
+// Fuel has no per-event block field selection: every event materialises the
+// full (height, time, id) trio, matching what the source always queries.
+let eventBlockFieldMask = BlockStore.makeMaskFn(blockFields)
+
+// Fuel keeps the transaction inline on the event payload rather than in the
+// per-chain store, so there is nothing to select and no field code to hold.
+let transactionFields = []
+let eventTransactionFieldMask = TransactionStore.makeMaskFn(transactionFields)
+
 let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
     delete fields.id
     delete fields.height
@@ -24,7 +39,6 @@ let make = (~logger: Pino.t): Ecosystem.t => {
   blockNumberName: "height",
   blockTimestampName: "time",
   blockHashName: "id",
-  cleanUpRawEventFieldsInPlace,
   onBlockMethodName: "onBlock",
   // Fuel filter shape: `{block: {height: {_gte?, _lte?, _every?}}}`.
   // Inner range chunk parsed by `blockRangeSchema` in `Main.res`.
@@ -56,7 +70,13 @@ let make = (~logger: Pino.t): Ecosystem.t => {
     ),
   toRawEvent: eventItem => {
     let payload = eventItem.payload->toPayload
-    let header = payload.block->(Utils.magic: Internal.eventBlock => {"id": string, "time": int})
+    // Store-backed payloads get `block` written at batch prep and simulate
+    // carries it from the start, so it's present by the time a raw event is built.
+    let header = switch payload.block {
+    | Some(block) => block->(Utils.magic: Internal.eventBlock => {"id": string, "time": int})
+    | None =>
+      JsError.throwWithMessage("Unexpected case: The event block is missing for a raw event")
+    }
     eventItem->RawEvent.make(
       ~block=payload.block,
       ~transaction=payload.transaction,

package/src/sources/Fuel.res.mjs

@@ -2,7 +2,23 @@
 
 import * as Logging from "../Logging.res.mjs";
 import * as RawEvent from "../RawEvent.res.mjs";
+import * as BlockStore from "./BlockStore.res.mjs";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as TransactionStore from "./TransactionStore.res.mjs";
+
+let blockFields = [
+  "height",
+  "id",
+  "time"
+];
+
+let eventBlockFieldMask = BlockStore.makeMaskFn(blockFields);
+
+let transactionFields = [];
+
+let eventTransactionFieldMask = TransactionStore.makeMaskFn(transactionFields);
 
 let cleanUpRawEventFieldsInPlace = (fields => {
     delete fields.id
@@ -16,7 +32,6 @@ function make(logger) {
     blockNumberName: "height",
     blockTimestampName: "time",
     blockHashName: "id",
-    cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
     onBlockMethodName: "onBlock",
     onBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("height", S$RescriptSchema.unknown))))),
     onEventBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.strict(S$RescriptSchema.object(s2 => s2.f("height", S$RescriptSchema.unknown)))))),
@@ -32,14 +47,19 @@ function make(logger) {
     }),
     toRawEvent: eventItem => {
       let payload = eventItem.payload;
-      let header = payload.block;
+      let block = payload.block;
+      let header = block !== undefined ? Primitive_option.valFromOption(block) : Stdlib_JsError.throwWithMessage("Unexpected case: The event block is missing for a raw event");
       return RawEvent.make(eventItem, payload.block, payload.transaction, payload.params, payload.srcAddress, header.id, header.time, cleanUpRawEventFieldsInPlace);
     }
   };
 }
 
 export {
+  blockFields,
+  eventBlockFieldMask,
+  transactionFields,
+  eventTransactionFieldMask,
   cleanUpRawEventFieldsInPlace,
   make,
 }
-/* Logging Not a pure module */
+/* eventBlockFieldMask Not a pure module */

package/src/sources/FuelHyperSync.res

@@ -1,9 +1,11 @@
 type logsQueryPage = {
   items: array<FuelHyperSyncClient.EventItems.item>,
   // Blocks referenced by `items`, one per height.
-  blocks: array<FuelHyperSyncClient.EventItems.block>,
   nextBlock: int,
   archiveHeight: int,
+  // Page store owning this page's raw blocks (built on the Rust side); its ids
+  // drive reorg detection when merged into the chain store.
+  blockStore: BlockStore.t,
 }
 
 module GetLogs = {
@@ -53,7 +55,10 @@ module GetLogs = {
       clientFilteredContracts,
     }
 
-    let res = switch await client->FuelHyperSyncClient.getEventItems(query, addressSet) {
+    let (res, blockStore) = switch await client->FuelHyperSyncClient.getEventItems(
+      query,
+      addressSet,
+    ) {
     | res => res
     | exception exn =>
       switch exn->extractMissingParams {
@@ -67,9 +72,9 @@ module GetLogs = {
     }
     {
       items: res.items,
-      blocks: res.blocks,
       nextBlock: res.nextBlock,
       archiveHeight: res.archiveHeight->Option.getOr(0), // TODO: FIXME: Shouldn't have a default here
+      blockStore,
     }
   }
 }

package/src/sources/FuelHyperSync.res.mjs

@@ -36,9 +36,9 @@ async function query(client, fromBlock, toBlock, registrationIndexes, addressSet
     registrationIndexes: registrationIndexes,
     clientFilteredContracts: clientFilteredContracts
   };
-  let res;
+  let match;
   try {
-    res = await client.getEventItems(query$1, addressSet);
+    match = await client.getEventItems(query$1, addressSet);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     let missingParams = extractMissingParams(exn);
@@ -54,6 +54,7 @@ async function query(client, fromBlock, toBlock, registrationIndexes, addressSet
     }
     throw exn;
   }
+  let res = match[0];
   if (res.nextBlock <= fromBlock) {
     throw {
       RE_EXN_ID: $$Error,
@@ -63,9 +64,9 @@ async function query(client, fromBlock, toBlock, registrationIndexes, addressSet
   }
   return {
     items: res.items,
-    blocks: res.blocks,
     nextBlock: res.nextBlock,
-    archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0)
+    archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0),
+    blockStore: match[1]
   };
 }
 

package/src/sources/FuelHyperSync.resi

@@ -1,8 +1,10 @@
 type logsQueryPage = {
   items: array<FuelHyperSyncClient.EventItems.item>,
-  blocks: array<FuelHyperSyncClient.EventItems.block>,
   nextBlock: int,
   archiveHeight: int,
+  // Page store owning this page's raw blocks (built on the Rust side); its ids
+  // drive reorg detection when merged into the chain store.
+  blockStore: BlockStore.t,
 }
 
 module GetLogs: {

package/src/sources/FuelHyperSyncClient.res

@@ -87,17 +87,9 @@ module EventItems = {
     to?: string,
   }
 
-  type block = {
-    id: string,
-    height: int,
-    time: int,
-  }
-
   type response = {
     archiveHeight: option<int>,
     nextBlock: int,
-    // One block per height; items reference them by `blockHeight`.
-    blocks: array<block>,
     items: array<item>,
   }
 }
@@ -121,9 +113,14 @@ let make = (cfg: cfg, ~eventRegistrations, ~addressStore) => {
   )
 }
 
+// Returns the routed items/blocks plus the page's block store (Fuel blocks
+// keyed by height); the store's ids drive reorg detection on merge.
 @send
-external getEventItems: (t, EventItems.query, AddressSet.t) => promise<EventItems.response> =
-  "getEventItems"
+external getEventItems: (
+  t,
+  EventItems.query,
+  AddressSet.t,
+) => promise<(EventItems.response, BlockStore.t)> = "getEventItems"
 
 @send
 external getHeight: t => promise<int> = "getHeight"

package/src/sources/FuelHyperSyncSource.res

@@ -25,7 +25,9 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
 
   let client = switch FuelHyperSyncClient.make(
     {url: endpointUrl, apiToken},
-    ~eventRegistrations=FuelHyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations),
+    ~eventRegistrations=FuelHyperSyncClient.Registration.fromOnEventRegistrations(
+      onEventRegistrations,
+    ),
     ~addressStore,
   ) {
   | client => client
@@ -57,29 +59,19 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       ~addressSet,
       ~clientFilteredContracts=selection.clientFilteredContracts,
     ) catch {
-    | FuelHyperSync.GetLogs.Error(error) =>
+    | FuelHyperSync.GetLogs.Error(WrongInstance) =>
+      throw(Source.SourceBehindHead({blockNumber: fromBlock, requestStats: []}))
+    | FuelHyperSync.GetLogs.Error(UnexpectedMissingParams({missingParams})) =>
       throw(
         Source.GetItemsError(
           Source.FailedGettingItems({
             exn: %raw(`null`),
             attemptedToBlock: toBlock->Option.getOr(knownHeight),
-            retry: switch error {
-            | WrongInstance =>
-              let backoffMillis = switch retry {
-              | 0 => 100
-              | _ => 500 * retry
-              }
-              WithBackoff({
-                message: `Block #${fromBlock->Int.toString} not found in FuelHyperSync. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${backoffMillis->Int.toString}ms.`,
-                backoffMillis,
-              })
-            | UnexpectedMissingParams({missingParams}) =>
-              ImpossibleForTheQuery({
-                message: `Source returned invalid data with missing required fields: ${missingParams->Array.joinUnsafe(
-                    ", ",
-                  )}`,
-              })
-            },
+            retry: ImpossibleForTheQuery({
+              message: `Source returned invalid data with missing required fields: ${missingParams->Array.joinUnsafe(
+                  ", ",
+                )}`,
+            }),
           }),
         ),
       )
@@ -114,12 +106,6 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
 
     let parsingTimeRef = Performance.now()
 
-    // Blocks are returned once per height; items reference them by blockHeight.
-    let blocksByHeight = Utils.Map.make()
-    pageUnsafe.blocks->Array.forEach(block => {
-      blocksByHeight->Utils.Map.set(block.height, block)->ignore
-    })
-
     let parsedQueueItems = pageUnsafe.items->Array.map(item => {
       // Routing happened in Rust; the item references its registration by
       // chain-scoped index.
@@ -128,8 +114,6 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
         onEventRegistration.eventConfig->(
           Utils.magic: Internal.eventConfig => Internal.fuelEventConfig
         )
-      // Presence of every routed item's block is validated in Rust.
-      let block = blocksByHeight->Utils.Map.unsafeGet(item.blockHeight)
 
       let params = switch eventConfig.kind {
       | LogData({decode}) =>
@@ -178,12 +162,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
         payload: {
           contractName: eventConfig.contractName,
           eventName: eventConfig.name,
-          chainId: chainId,
+          chainId,
           params,
           transaction: {
             "id": item.txId,
           }->Obj.magic, // TODO: Obj.magic needed until the field selection types are not configurable for Fuel and Evm separately
-          block: block->Obj.magic,
+          // `block` is omitted; it's materialised from the block store onto the
+          // payload at batch prep (like EVM/SVM).
           srcAddress: item.srcAddress,
           logIndex: item.receiptIndex,
         }->Fuel.fromPayload,
@@ -192,18 +177,6 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
 
     let parsingTimeElapsed = parsingTimeRef->Performance.secondsSince
 
-    // Fuel never rolls back on reorg, so block hashes here are purely informational
-    // for detect-only logging via ReorgDetection.
-    let blockHashes = pageUnsafe.blocks->Array.map(block => {
-      ReorgDetection.blockNumber: block.height,
-      blockHash: block.id,
-    })
-
-    let latestFetchedBlockTimestamp = switch blocksByHeight->Utils.Map.get(heighestBlockQueried) {
-    | Some(block) => block.time
-    | None => 0
-    }
-
     let totalTimeElapsed = totalTimeRef->Performance.secondsSince
 
     let stats = {
@@ -213,15 +186,15 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     }
 
     {
-      latestFetchedBlockTimestamp,
       parsedQueueItems,
-      // Fuel keeps transaction and block on the payload; no store pages.
+      // Fuel keeps transaction and block inline on the payload; the block store
+      // carries only the (height, id) rows that drive reorg detection (Fuel
+      // never rolls back, so it's detect-only logging).
       transactionStore: None,
-      blockStore: None,
+      blockStore: pageUnsafe.blockStore,
       latestFetchedBlockNumber: heighestBlockQueried,
       stats,
       knownHeight,
-      blockHashes,
       fromBlockQueried: fromBlock,
       requestStats,
     }

package/src/sources/FuelHyperSyncSource.res.mjs

@@ -38,22 +38,16 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     let pageUnsafe;
     try {
       pageUnsafe = await FuelHyperSync.GetLogs.query(client, fromBlock, toBlock, selection.onEventRegistrations.map(reg => reg.index), addressSet, selection.clientFilteredContracts);
-    } catch (raw_error) {
-      let error = Primitive_exceptions.internalToException(raw_error);
-      if (error.RE_EXN_ID === FuelHyperSync.GetLogs.$$Error) {
-        let error$1 = error._1;
-        let tmp;
-        if (typeof error$1 !== "object") {
-          let backoffMillis = retry !== 0 ? 500 * retry | 0 : 100;
-          tmp = {
-            TAG: "WithBackoff",
-            message: `Block #` + fromBlock.toString() + ` not found in FuelHyperSync. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ` + backoffMillis.toString() + `ms.`,
-            backoffMillis: backoffMillis
-          };
-        } else {
-          tmp = {
-            TAG: "ImpossibleForTheQuery",
-            message: `Source returned invalid data with missing required fields: ` + error$1.missingParams.join(", ")
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      if (exn.RE_EXN_ID === FuelHyperSync.GetLogs.$$Error) {
+        let match = exn._1;
+        if (typeof match !== "object") {
+          throw {
+            RE_EXN_ID: Source.SourceBehindHead,
+            blockNumber: fromBlock,
+            requestStats: [],
+            Error: new Error()
           };
         }
         throw {
@@ -62,7 +56,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
             TAG: "FailedGettingItems",
             exn: null,
             attemptedToBlock: Stdlib_Option.getOr(toBlock, knownHeight),
-            retry: tmp
+            retry: {
+              TAG: "ImpossibleForTheQuery",
+              message: `Source returned invalid data with missing required fields: ` + match.missingParams.join(", ")
+            }
           },
           Error: new Error()
         };
@@ -71,7 +68,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
         RE_EXN_ID: Source.GetItemsError,
         _1: {
           TAG: "FailedGettingItems",
-          exn: error,
+          exn: exn,
           attemptedToBlock: Stdlib_Option.getOr(toBlock, knownHeight),
           retry: {
             TAG: "WithBackoff",
@@ -90,14 +87,9 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     let knownHeight$1 = pageUnsafe.archiveHeight;
     let heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
     let parsingTimeRef = Performance.now();
-    let blocksByHeight = new Map();
-    pageUnsafe.blocks.forEach(block => {
-      blocksByHeight.set(block.height, block);
-    });
     let parsedQueueItems = pageUnsafe.items.map(item => {
       let onEventRegistration = onEventRegistrations[item.onEventRegistrationIndex];
       let eventConfig = onEventRegistration.eventConfig;
-      let block = blocksByHeight.get(item.blockHeight);
       let match = eventConfig.kind;
       let params;
       let exit = 0;
@@ -158,18 +150,11 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
           logIndex: item.receiptIndex,
           transaction: {
             id: item.txId
-          },
-          block: block
+          }
         }
       };
     });
     let parsingTimeElapsed = Performance.secondsSince(parsingTimeRef);
-    let blockHashes = pageUnsafe.blocks.map(block => ({
-      blockHash: block.id,
-      blockNumber: block.height
-    }));
-    let block = blocksByHeight.get(heighestBlockQueried);
-    let latestFetchedBlockTimestamp = block !== undefined ? block.time : 0;
     let totalTimeElapsed = Performance.secondsSince(totalTimeRef);
     let stats_parsing$unknowntime$unknown$lpars$rpar = parsingTimeElapsed;
     let stats_page$unknownfetch$unknowntime$unknown$lpars$rpar = pageFetchTime;
@@ -180,13 +165,11 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     };
     return {
       knownHeight: knownHeight$1,
-      blockHashes: blockHashes,
       parsedQueueItems: parsedQueueItems,
       transactionStore: undefined,
-      blockStore: undefined,
+      blockStore: pageUnsafe.blockStore,
       fromBlockQueried: fromBlock,
       latestFetchedBlockNumber: heighestBlockQueried,
-      latestFetchedBlockTimestamp: latestFetchedBlockTimestamp,
       stats: stats,
       requestStats: requestStats
     };
@@ -249,4 +232,4 @@ export {
   isUnauthorizedError,
   make,
 }
-/* Logging Not a pure module */
+/* Source Not a pure module */