envio 3.6.1 → 3.7.0-subgraph

package/src/sources/FuelHyperSync.res.mjs

@@ -36,9 +36,9 @@ async function query(client, fromBlock, toBlock, registrationIndexes, addressSet
     registrationIndexes: registrationIndexes,
     clientFilteredContracts: clientFilteredContracts
   };
-  let res;
+  let match;
   try {
-    res = await client.getEventItems(query$1, addressSet);
+    match = await client.getEventItems(query$1, addressSet);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     let missingParams = extractMissingParams(exn);
@@ -54,6 +54,7 @@ async function query(client, fromBlock, toBlock, registrationIndexes, addressSet
     }
     throw exn;
   }
+  let res = match[0];
   if (res.nextBlock <= fromBlock) {
     throw {
       RE_EXN_ID: $$Error,
@@ -63,9 +64,9 @@ async function query(client, fromBlock, toBlock, registrationIndexes, addressSet
   }
   return {
     items: res.items,
-    blocks: res.blocks,
     nextBlock: res.nextBlock,
-    archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0)
+    archiveHeight: Stdlib_Option.getOr(res.archiveHeight, 0),
+    blockStore: match[1]
   };
 }
 

package/src/sources/FuelHyperSync.resi

@@ -1,8 +1,10 @@
 type logsQueryPage = {
   items: array<FuelHyperSyncClient.EventItems.item>,
-  blocks: array<FuelHyperSyncClient.EventItems.block>,
   nextBlock: int,
   archiveHeight: int,
+  // Page store owning this page's raw blocks (built on the Rust side); its ids
+  // drive reorg detection when merged into the chain store.
+  blockStore: BlockStore.t,
 }
 
 module GetLogs: {

package/src/sources/FuelHyperSyncClient.res

@@ -87,17 +87,9 @@ module EventItems = {
     to?: string,
   }
 
-  type block = {
-    id: string,
-    height: int,
-    time: int,
-  }
-
   type response = {
     archiveHeight: option<int>,
     nextBlock: int,
-    // One block per height; items reference them by `blockHeight`.
-    blocks: array<block>,
     items: array<item>,
   }
 }
@@ -121,9 +113,14 @@ let make = (cfg: cfg, ~eventRegistrations, ~addressStore) => {
   )
 }
 
+// Returns the routed items/blocks plus the page's block store (Fuel blocks
+// keyed by height); the store's ids drive reorg detection on merge.
 @send
-external getEventItems: (t, EventItems.query, AddressSet.t) => promise<EventItems.response> =
-  "getEventItems"
+external getEventItems: (
+  t,
+  EventItems.query,
+  AddressSet.t,
+) => promise<(EventItems.response, BlockStore.t)> = "getEventItems"
 
 @send
 external getHeight: t => promise<int> = "getHeight"

package/src/sources/FuelHyperSyncSource.res

@@ -25,7 +25,9 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
 
   let client = switch FuelHyperSyncClient.make(
     {url: endpointUrl, apiToken},
-    ~eventRegistrations=FuelHyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations),
+    ~eventRegistrations=FuelHyperSyncClient.Registration.fromOnEventRegistrations(
+      onEventRegistrations,
+    ),
     ~addressStore,
   ) {
   | client => client
@@ -57,29 +59,19 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       ~addressSet,
       ~clientFilteredContracts=selection.clientFilteredContracts,
     ) catch {
-    | FuelHyperSync.GetLogs.Error(error) =>
+    | FuelHyperSync.GetLogs.Error(WrongInstance) =>
+      throw(Source.SourceBehindHead({blockNumber: fromBlock, requestStats: []}))
+    | FuelHyperSync.GetLogs.Error(UnexpectedMissingParams({missingParams})) =>
       throw(
         Source.GetItemsError(
           Source.FailedGettingItems({
             exn: %raw(`null`),
             attemptedToBlock: toBlock->Option.getOr(knownHeight),
-            retry: switch error {
-            | WrongInstance =>
-              let backoffMillis = switch retry {
-              | 0 => 100
-              | _ => 500 * retry
-              }
-              WithBackoff({
-                message: `Block #${fromBlock->Int.toString} not found in FuelHyperSync. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${backoffMillis->Int.toString}ms.`,
-                backoffMillis,
-              })
-            | UnexpectedMissingParams({missingParams}) =>
-              ImpossibleForTheQuery({
-                message: `Source returned invalid data with missing required fields: ${missingParams->Array.joinUnsafe(
-                    ", ",
-                  )}`,
-              })
-            },
+            retry: ImpossibleForTheQuery({
+              message: `Source returned invalid data with missing required fields: ${missingParams->Array.joinUnsafe(
+                  ", ",
+                )}`,
+            }),
           }),
         ),
       )
@@ -114,12 +106,6 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
 
     let parsingTimeRef = Performance.now()
 
-    // Blocks are returned once per height; items reference them by blockHeight.
-    let blocksByHeight = Utils.Map.make()
-    pageUnsafe.blocks->Array.forEach(block => {
-      blocksByHeight->Utils.Map.set(block.height, block)->ignore
-    })
-
     let parsedQueueItems = pageUnsafe.items->Array.map(item => {
       // Routing happened in Rust; the item references its registration by
       // chain-scoped index.
@@ -128,8 +114,6 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
         onEventRegistration.eventConfig->(
           Utils.magic: Internal.eventConfig => Internal.fuelEventConfig
         )
-      // Presence of every routed item's block is validated in Rust.
-      let block = blocksByHeight->Utils.Map.unsafeGet(item.blockHeight)
 
       let params = switch eventConfig.kind {
       | LogData({decode}) =>
@@ -178,12 +162,13 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
         payload: {
           contractName: eventConfig.contractName,
           eventName: eventConfig.name,
-          chainId: chainId,
+          chainId,
           params,
           transaction: {
             "id": item.txId,
           }->Obj.magic, // TODO: Obj.magic needed until the field selection types are not configurable for Fuel and Evm separately
-          block: block->Obj.magic,
+          // `block` is omitted; it's materialised from the block store onto the
+          // payload at batch prep (like EVM/SVM).
           srcAddress: item.srcAddress,
           logIndex: item.receiptIndex,
         }->Fuel.fromPayload,
@@ -192,18 +177,6 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
 
     let parsingTimeElapsed = parsingTimeRef->Performance.secondsSince
 
-    // Fuel never rolls back on reorg, so block hashes here are purely informational
-    // for detect-only logging via ReorgDetection.
-    let blockHashes = pageUnsafe.blocks->Array.map(block => {
-      ReorgDetection.blockNumber: block.height,
-      blockHash: block.id,
-    })
-
-    let latestFetchedBlockTimestamp = switch blocksByHeight->Utils.Map.get(heighestBlockQueried) {
-    | Some(block) => block.time
-    | None => 0
-    }
-
     let totalTimeElapsed = totalTimeRef->Performance.secondsSince
 
     let stats = {
@@ -213,15 +186,15 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     }
 
     {
-      latestFetchedBlockTimestamp,
       parsedQueueItems,
-      // Fuel keeps transaction and block on the payload; no store pages.
+      // Fuel keeps transaction and block inline on the payload; the block store
+      // carries only the (height, id) rows that drive reorg detection (Fuel
+      // never rolls back, so it's detect-only logging).
       transactionStore: None,
-      blockStore: None,
+      blockStore: pageUnsafe.blockStore,
       latestFetchedBlockNumber: heighestBlockQueried,
       stats,
       knownHeight,
-      blockHashes,
       fromBlockQueried: fromBlock,
       requestStats,
     }

package/src/sources/FuelHyperSyncSource.res.mjs

@@ -38,22 +38,16 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     let pageUnsafe;
     try {
       pageUnsafe = await FuelHyperSync.GetLogs.query(client, fromBlock, toBlock, selection.onEventRegistrations.map(reg => reg.index), addressSet, selection.clientFilteredContracts);
-    } catch (raw_error) {
-      let error = Primitive_exceptions.internalToException(raw_error);
-      if (error.RE_EXN_ID === FuelHyperSync.GetLogs.$$Error) {
-        let error$1 = error._1;
-        let tmp;
-        if (typeof error$1 !== "object") {
-          let backoffMillis = retry !== 0 ? 500 * retry | 0 : 100;
-          tmp = {
-            TAG: "WithBackoff",
-            message: `Block #` + fromBlock.toString() + ` not found in FuelHyperSync. HyperFuel has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ` + backoffMillis.toString() + `ms.`,
-            backoffMillis: backoffMillis
-          };
-        } else {
-          tmp = {
-            TAG: "ImpossibleForTheQuery",
-            message: `Source returned invalid data with missing required fields: ` + error$1.missingParams.join(", ")
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      if (exn.RE_EXN_ID === FuelHyperSync.GetLogs.$$Error) {
+        let match = exn._1;
+        if (typeof match !== "object") {
+          throw {
+            RE_EXN_ID: Source.SourceBehindHead,
+            blockNumber: fromBlock,
+            requestStats: [],
+            Error: new Error()
           };
         }
         throw {
@@ -62,7 +56,10 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
             TAG: "FailedGettingItems",
             exn: null,
             attemptedToBlock: Stdlib_Option.getOr(toBlock, knownHeight),
-            retry: tmp
+            retry: {
+              TAG: "ImpossibleForTheQuery",
+              message: `Source returned invalid data with missing required fields: ` + match.missingParams.join(", ")
+            }
           },
           Error: new Error()
         };
@@ -71,7 +68,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
         RE_EXN_ID: Source.GetItemsError,
         _1: {
           TAG: "FailedGettingItems",
-          exn: error,
+          exn: exn,
           attemptedToBlock: Stdlib_Option.getOr(toBlock, knownHeight),
           retry: {
             TAG: "WithBackoff",
@@ -90,14 +87,9 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     let knownHeight$1 = pageUnsafe.archiveHeight;
     let heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
     let parsingTimeRef = Performance.now();
-    let blocksByHeight = new Map();
-    pageUnsafe.blocks.forEach(block => {
-      blocksByHeight.set(block.height, block);
-    });
     let parsedQueueItems = pageUnsafe.items.map(item => {
       let onEventRegistration = onEventRegistrations[item.onEventRegistrationIndex];
       let eventConfig = onEventRegistration.eventConfig;
-      let block = blocksByHeight.get(item.blockHeight);
       let match = eventConfig.kind;
       let params;
       let exit = 0;
@@ -158,18 +150,11 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
           logIndex: item.receiptIndex,
           transaction: {
             id: item.txId
-          },
-          block: block
+          }
         }
       };
     });
     let parsingTimeElapsed = Performance.secondsSince(parsingTimeRef);
-    let blockHashes = pageUnsafe.blocks.map(block => ({
-      blockHash: block.id,
-      blockNumber: block.height
-    }));
-    let block = blocksByHeight.get(heighestBlockQueried);
-    let latestFetchedBlockTimestamp = block !== undefined ? block.time : 0;
     let totalTimeElapsed = Performance.secondsSince(totalTimeRef);
     let stats_parsing$unknowntime$unknown$lpars$rpar = parsingTimeElapsed;
     let stats_page$unknownfetch$unknowntime$unknown$lpars$rpar = pageFetchTime;
@@ -180,13 +165,11 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     };
     return {
       knownHeight: knownHeight$1,
-      blockHashes: blockHashes,
       parsedQueueItems: parsedQueueItems,
       transactionStore: undefined,
-      blockStore: undefined,
+      blockStore: pageUnsafe.blockStore,
       fromBlockQueried: fromBlock,
       latestFetchedBlockNumber: heighestBlockQueried,
-      latestFetchedBlockTimestamp: latestFetchedBlockTimestamp,
       stats: stats,
       requestStats: requestStats
     };
@@ -249,4 +232,4 @@ export {
   isUnauthorizedError,
   make,
 }
-/* Logging Not a pure module */
+/* Source Not a pure module */

package/src/sources/HyperSync.res

@@ -1,59 +1,65 @@
-let reraisIfRateLimited = exn =>
-  switch exn->JsExn.anyToExnInternal {
-  | JsExn(e) =>
-    switch e->JsExn.message {
-    | Some(msg) if msg->String.startsWith("RATE_LIMITED:") =>
-      let resetMs =
-        msg
-        ->String.slice(~start=13, ~end=msg->String.length)
-        ->Int.fromString
-        ->Option.getOr(1000)
-      throw(Source.RateLimited({resetMs: resetMs}))
-    | _ => ()
+// A napi error carries nothing but a message, so the native clients signal the
+// recoverable conditions SourceManager knows how to retry as a `PREFIX:<int>`
+// marker. Keep in sync with `request_stats.rs`.
+let rateLimitedPrefix = "RATE_LIMITED:"
+let behindHeadPrefix = "SOURCE_BEHIND_HEAD:"
+
+let markerValue = (msg, ~prefix) =>
+  msg->String.slice(~start=prefix->String.length, ~end=msg->String.length)->Int.fromString
+
+let mapNativeFailure = (failure: Source.nativeRequestFailure) => {
+  switch failure.message {
+  | Some(msg) if msg->String.startsWith(rateLimitedPrefix) =>
+    Source.RateLimited({
+      resetMs: msg->markerValue(~prefix=rateLimitedPrefix)->Option.getOr(1000),
+      requestStats: failure.requestStats,
+    })
+  | Some(msg) if msg->String.startsWith(behindHeadPrefix) =>
+    Source.SourceBehindHead({
+      blockNumber: msg->markerValue(~prefix=behindHeadPrefix)->Option.getOr(0),
+      requestStats: failure.requestStats,
+    })
+  | _ => failure.cause
+  }
+}
+
+let mapNativeFailureExn = exn => exn->Source.unpackNativeRequestFailure->mapNativeFailure
+
+// Every HyperSync client paginates block hashes in Rust and returns the page
+// store with the timings of the requests it took. A failure arrives as the
+// native envelope, which carries those same timings, so the caller records them
+// either way.
+let makeGetBlockHashes = (
+  ~query: (~blockNumbers: array<int>) => promise<(BlockStore.t, array<Source.requestStat>)>,
+) =>
+  async (~blockNumbers, ~logger as _) => {
+    let (result, requestStats) = try {
+      let (blockStore, requestStats) = await query(~blockNumbers)
+      (Ok(blockStore), requestStats)
+    } catch {
+    | exn =>
+      let failure = exn->Source.unpackNativeRequestFailure
+      (Error(failure->mapNativeFailure), failure.requestStats)
     }
+    {Source.result, requestStats}
+  }
+
+let reraiseIfRecoverable = exn =>
+  switch exn->mapNativeFailureExn {
+  | (Source.RateLimited(_) | Source.SourceBehindHead(_)) as exn => throw(exn)
   | _ => ()
   }
 
 type logsQueryPage = {
   items: array<HyperSyncClient.EventItems.item>,
-  // Headers for every returned block, deduplicated by block number.
-  blocks: array<HyperSyncClient.EventItems.blockHeader>,
   nextBlock: int,
   archiveHeight: int,
-  rollbackGuard: option<HyperSyncClient.ResponseTypes.rollbackGuard>,
   // Page store owning this page's raw transactions.
   transactionStore: TransactionStore.t,
   // Page store owning this page's raw blocks.
   blockStore: BlockStore.t,
 }
 
-type missingParams = {
-  queryName: string,
-  missingParams: array<string>,
-}
-type queryError = UnexpectedMissingParams(missingParams)
-
-exception HyperSyncQueryError(queryError)
-
-let queryErrorToExn = queryError => {
-  HyperSyncQueryError(queryError)
-}
-
-let queryErrorToMsq = (e: queryError): string => {
-  switch e {
-  | UnexpectedMissingParams({queryName, missingParams}) =>
-    `${queryName} query failed due to unexpected missing params on response:
-      ${missingParams->Array.joinUnsafe(", ")}`
-  }
-}
-
-type queryResponse<'a> = result<'a, queryError>
-let mapExn = (queryResponse: queryResponse<'a>) =>
-  switch queryResponse {
-  | Ok(v) => Ok(v)
-  | Error(err) => err->queryErrorToExn->Error
-  }
-
 module GetLogs = {
   type error =
     | UnexpectedMissingParams({missingParams: array<string>})
@@ -109,7 +115,7 @@ module GetLogs = {
     ) {
     | res => res
     | exception exn =>
-      reraisIfRateLimited(exn)
+      reraiseIfRecoverable(exn)
       switch extractMissingParams(exn) {
       | Some(missingParams) => throw(Error(UnexpectedMissingParams({missingParams: missingParams})))
       | None => throw(exn)
@@ -122,196 +128,10 @@ module GetLogs = {
 
     {
       items: res.items,
-      blocks: res.blocks,
       nextBlock: res.nextBlock,
       archiveHeight: res.archiveHeight->Option.getOr(0), //Archive Height is only None if height is 0
-      rollbackGuard: res.rollbackGuard,
       transactionStore,
       blockStore,
     }
   }
 }
-
-module BlockData = {
-  let makeRequestBody = (~fromBlock, ~toBlock): HyperSyncClient.QueryTypes.query => {
-    fromBlock,
-    toBlockExclusive: toBlock + 1,
-    fieldSelection: {
-      block: [Number, Hash, Timestamp],
-    },
-    includeAllBlocks: true,
-  }
-
-  let convertResponse = (res: HyperSyncClient.queryResponse): queryResponse<
-    array<ReorgDetection.blockDataWithTimestamp>,
-  > => {
-    res.data.blocks
-    ->Array.map(block => {
-      switch block {
-      | {number: blockNumber, hash: blockHash, timestamp: blockTimestamp} =>
-        Ok(
-          (
-            {
-              blockTimestamp,
-              blockNumber,
-              blockHash,
-            }: ReorgDetection.blockDataWithTimestamp
-          ),
-        )
-      | _ =>
-        let missingParams =
-          [
-            block.number->Utils.Option.mapNone("block.number"),
-            block.timestamp->Utils.Option.mapNone("block.timestamp"),
-            block.hash->Utils.Option.mapNone("block.hash"),
-          ]->Array.filterMap(p => p)
-
-        Error(
-          UnexpectedMissingParams({
-            queryName: "query block data HyperSync",
-            missingParams,
-          }),
-        )
-      }
-    })
-    ->Utils.Array.transposeResults
-  }
-
-  let rec queryBlockData = async (
-    ~client: HyperSyncClient.t,
-    ~fromBlock,
-    ~toBlock,
-    ~sourceName,
-    ~chainId,
-    ~logger,
-    ~requestStats: array<Source.requestStat>,
-  ): queryResponse<array<ReorgDetection.blockDataWithTimestamp>> => {
-    let body = makeRequestBody(~fromBlock, ~toBlock)
-
-    let logger = Logging.createChildFrom(
-      ~logger,
-      ~params={
-        "logType": "HyperSync get block hash query",
-        "fromBlock": fromBlock,
-        "toBlock": toBlock,
-      },
-    )
-
-    let timerRef = Performance.now()
-    let maybeSuccessfulRes = switch await client.get(~query=body) {
-    | exception exn =>
-      reraisIfRateLimited(exn)
-      None
-    | res if res.nextBlock <= fromBlock => None
-    | res => Some(res)
-    }
-    requestStats
-    ->Array.push({Source.method: "getBlockHashes", seconds: timerRef->Performance.secondsSince})
-    ->ignore
-
-    // If the block is not found, retry the query. This can occur since replicas of hypersync might not have caught up yet
-    switch maybeSuccessfulRes {
-    | None => {
-        let delayMilliseconds = 100
-        logger->Logging.childInfo(
-          `Block #${fromBlock->Int.toString} not found in HyperSync. HyperSync has multiple instances and it's possible that they drift independently slightly from the head. Indexing should continue correctly after retrying the query in ${delayMilliseconds->Int.toString}ms.`,
-        )
-        await Time.resolvePromiseAfterDelay(~delayMilliseconds)
-        await queryBlockData(
-          ~client,
-          ~fromBlock,
-          ~toBlock,
-          ~sourceName,
-          ~chainId,
-          ~logger,
-          ~requestStats,
-        )
-      }
-    | Some(res) =>
-      switch res->convertResponse {
-      | Error(_) as err => err
-      | Ok(datas) if res.nextBlock <= toBlock => {
-          let restRes = await queryBlockData(
-            ~client,
-            ~fromBlock=res.nextBlock,
-            ~toBlock,
-            ~sourceName,
-            ~chainId,
-            ~logger,
-            ~requestStats,
-          )
-          restRes->Result.map(rest => datas->Array.concat(rest))
-        }
-      | Ok(_) as ok => ok
-      }
-    }
-  }
-
-  let queryBlockDataMulti = async (
-    ~client: HyperSyncClient.t,
-    ~blockNumbers,
-    ~sourceName,
-    ~chainId,
-    ~logger,
-  ): (queryResponse<array<ReorgDetection.blockDataWithTimestamp>>, array<Source.requestStat>) => {
-    let requestStats = []
-    let result = switch blockNumbers->Array.get(0) {
-    | None => Ok([])
-    | Some(firstBlock) => {
-        let fromBlock = ref(firstBlock)
-        let toBlock = ref(firstBlock)
-        let set = Utils.Set.make()
-        for idx in 0 to blockNumbers->Array.length - 1 {
-          let blockNumber = blockNumbers->Array.getUnsafe(idx)
-          if blockNumber < fromBlock.contents {
-            fromBlock := blockNumber
-          }
-          if blockNumber > toBlock.contents {
-            toBlock := blockNumber
-          }
-          set->Utils.Set.add(blockNumber)->ignore
-        }
-        if toBlock.contents - fromBlock.contents > 1000 {
-          JsError.throwWithMessage(
-            `Invalid block data request. Range of block numbers is too large. Max range is 1000. Requested range: ${fromBlock.contents->Int.toString}-${toBlock.contents->Int.toString}`,
-          )
-        }
-        let res = await queryBlockData(
-          ~fromBlock=fromBlock.contents,
-          ~toBlock=toBlock.contents,
-          ~client,
-          ~sourceName,
-          ~chainId,
-          ~logger,
-          ~requestStats,
-        )
-        let filtered = res->Result.map(datas => {
-          datas->Array.filter(data => set->Utils.Set.delete(data.blockNumber))
-        })
-        if set->Utils.Set.size > 0 {
-          JsError.throwWithMessage(
-            `Invalid response. Failed to get block data for block numbers: ${set
-              ->Utils.Set.toArray
-              ->Array.joinUnsafe(", ")}`,
-          )
-        }
-        filtered
-      }
-    }
-    (result, requestStats)
-  }
-}
-
-let queryBlockData = (~client, ~blockNumber, ~sourceName, ~chainId, ~logger) => {
-  let requestStats = []
-  BlockData.queryBlockData(
-    ~client,
-    ~fromBlock=blockNumber,
-    ~toBlock=blockNumber,
-    ~sourceName,
-    ~chainId,
-    ~logger,
-    ~requestStats,
-  )->Promise.thenResolve(res => (res->Result.map(res => res->Array.get(0)), requestStats))
-}
-let queryBlockDataMulti = BlockData.queryBlockDataMulti