envio 3.6.1 → 3.7.0

package/src/sources/RpcSource.res.mjs

@@ -9,6 +9,7 @@ import * as Address from "../Address.res.mjs";
 import * as ChainId from "../ChainId.res.mjs";
 import * as Logging from "../Logging.res.mjs";
 import * as Internal from "../Internal.res.mjs";
+import * as BlockStore from "./BlockStore.res.mjs";
 import * as LazyLoader from "../LazyLoader.res.mjs";
 import * as Performance from "../bindings/Performance.res.mjs";
 import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
@@ -579,6 +580,16 @@ let fieldRegistryLowercase = makeFieldRegistry(lowercaseAddressSchema);
 
 let fieldRegistryChecksum = makeFieldRegistry(checksumAddressSchema);
 
+function isRpcTransactionField(name) {
+  switch (name) {
+    case "hash" :
+    case "transactionIndex" :
+      return true;
+    default:
+      return Stdlib_Option.isSome(fieldRegistryChecksum[name]);
+  }
+}
+
 function parseFieldsFromJson(mutTransactionAcc, fields, json) {
   fields.forEach(def => {
     let raw = json[def.jsonKey];
@@ -596,6 +607,22 @@ function parseFieldsFromJson(mutTransactionAcc, fields, json) {
   });
 }
 
+let effectiveGasPriceFallbackDef = {
+  location: "effectiveGasPrice",
+  jsonKey: "gasPrice",
+  schema: Rpc.hexBigintSchema,
+  source: "TransactionOnly"
+};
+
+function fillEffectiveGasPriceOrThrow(mutTransactionAcc, txJson) {
+  let gasPrice = txJson["gasPrice"];
+  if (gasPrice !== undefined) {
+    return parseFieldsFromJson(mutTransactionAcc, [effectiveGasPriceFallbackDef], txJson);
+  } else {
+    return Stdlib_JsError.throwWithMessage(`Neither "effectiveGasPrice" nor "gasPrice" is present in the RPC response for the transaction. Remove "effectiveGasPrice" from the field selection, or index this chain via HyperSync.`);
+  }
+}
+
 function makeThrowingGetEventTransaction(getTransactionJson, getReceiptJson, lowercaseAddresses) {
   let fieldRegistry = lowercaseAddresses ? fieldRegistryLowercase : fieldRegistryChecksum;
   let fnsCache = new WeakMap();
@@ -611,6 +638,9 @@ function makeThrowingGetEventTransaction(getTransactionJson, getReceiptJson, low
       let hasHash = {
         contents: false
       };
+      let hasEffectiveGasPrice = {
+        contents: false
+      };
       let txFields = [];
       let receiptFields = [];
       let bothFields = [];
@@ -627,16 +657,22 @@ function makeThrowingGetEventTransaction(getTransactionJson, getReceiptJson, low
             if (def === undefined) {
               return;
             }
-            let match = def.source;
+            let def$1 = fieldName === "effectiveGasPrice" ? (hasEffectiveGasPrice.contents = true, {
+                location: def.location,
+                jsonKey: def.jsonKey,
+                schema: S$RescriptSchema.nullable(def.schema),
+                source: def.source
+              }) : def;
+            let match = def$1.source;
             switch (match) {
               case "TransactionOnly" :
-                txFields.push(def);
+                txFields.push(def$1);
                 return;
               case "ReceiptOnly" :
-                receiptFields.push(def);
+                receiptFields.push(def$1);
                 return;
               case "Both" :
-                bothFields.push(def);
+                bothFields.push(def$1);
                 return;
             }
         }
@@ -713,8 +749,20 @@ function makeThrowingGetEventTransaction(getTransactionJson, getReceiptJson, low
                 if (receiptJson !== undefined) {
                   parseFieldsFromJson(mutTransactionAcc, receiptFields, receiptJson);
                 }
-                return mutTransactionAcc;
-              });
+                if (hasEffectiveGasPrice.contents && Stdlib_Option.isNone(mutTransactionAcc["effectiveGasPrice"])) {
+                  if (txJson !== undefined) {
+                    fillEffectiveGasPriceOrThrow(mutTransactionAcc, txJson);
+                    return Promise.resolve(mutTransactionAcc);
+                  } else {
+                    return getTransactionJson(log.transactionHash).then(json => {
+                      fillEffectiveGasPriceOrThrow(mutTransactionAcc, json);
+                      return mutTransactionAcc;
+                    });
+                  }
+                } else {
+                  return Promise.resolve(mutTransactionAcc);
+                }
+              }).then(mutTransactionAcc => mutTransactionAcc);
             }
         );
       fnsCache.set(selectedTransactionFields, fn$1);
@@ -799,6 +847,11 @@ function make(param) {
   let receiptLoader = {
     contents: makeReceiptLoader()
   };
+  let resetCachedLoaders = () => {
+    blockLoader.contents = makeBlockLoader();
+    transactionLoader.contents = makeTransactionLoader();
+    receiptLoader.contents = makeReceiptLoader();
+  };
   let getEventBlockOrThrow = makeThrowingGetEventBlock(blockNumber => LazyLoader.get(blockLoader.contents, blockNumber), lowercaseAddresses);
   let getEventTransactionOrThrow = makeThrowingGetEventTransaction(async transactionHash => {
     let json = await LazyLoader.get(transactionLoader.contents, transactionHash);
@@ -824,7 +877,7 @@ function make(param) {
   let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, partitionId, selection, param, retry, param$1) => {
     let startFetchingBatchTimeRef = Performance.now();
     let toBlock$1 = toBlock !== undefined ? Primitive_int.min(toBlock, knownHeight) : knownHeight;
-    let firstBlockParentPromise = fromBlock > 0 ? LazyLoader.get(blockLoader.contents, fromBlock - 1 | 0).then(json => parseBlockInfo(json)) : Promise.resolve(undefined);
+    let firstBlockPromise = fromBlock > 0 && fromBlock <= toBlock$1 ? LazyLoader.get(blockLoader.contents, fromBlock).then(json => parseBlockInfo(json)) : Promise.resolve(undefined);
     if (Utils.$$Array.isEmpty(selection.onEventRegistrations)) {
       throw {
         RE_EXN_ID: Source.GetItemsError,
@@ -887,8 +940,8 @@ function make(param) {
         let match;
         try {
           match = await Promise.all([
-            getEventBlockOrThrow(log, eventConfig.selectedBlockFields),
-            getEventTransactionOrThrow(log, eventConfig.selectedTransactionFields)
+            getEventBlockOrThrow(log, onEventRegistration.fieldSelection.blockFields),
+            getEventTransactionOrThrow(log, onEventRegistration.fieldSelection.transactionFields)
           ]);
         } catch (raw_exn) {
           let exn = Primitive_exceptions.internalToException(raw_exn);
@@ -942,67 +995,71 @@ function make(param) {
         };
       })();
     }));
-    let optFirstBlockParent = await firstBlockParentPromise;
+    let optFirstBlock = await firstBlockPromise;
     let totalTimeElapsed = Performance.secondsSince(startFetchingBatchTimeRef);
-    let blockHashes = [];
+    let observedBlocks = [];
     let pushBlockInfo = b => {
-      blockHashes.push({
+      observedBlocks.push({
+        blockNumber: b.number,
         blockHash: b.hash,
-        blockNumber: b.number
+        blockTimestamp: b.timestamp
       });
       if (b.number > 0) {
-        blockHashes.push({
-          blockHash: b.parentHash,
-          blockNumber: b.number - 1 | 0
+        observedBlocks.push({
+          blockNumber: b.number - 1 | 0,
+          blockHash: b.parentHash
         });
         return;
       }
     };
     pushBlockInfo(latestFetchedBlockInfo);
-    if (optFirstBlockParent !== undefined) {
-      pushBlockInfo(optFirstBlockParent);
+    if (optFirstBlock !== undefined) {
+      pushBlockInfo(optFirstBlock);
     }
     items.forEach(param => {
       let log = param.log;
-      blockHashes.push({
-        blockHash: log.blockHash,
-        blockNumber: log.blockNumber
+      observedBlocks.push({
+        blockNumber: log.blockNumber,
+        blockHash: log.blockHash
       });
     });
     return {
       knownHeight: knownHeight,
-      blockHashes: blockHashes,
       parsedQueueItems: parsedQueueItems,
       transactionStore: undefined,
-      blockStore: undefined,
+      blockStore: BlockStore.fromJs(observedBlocks, "evm", !lowercaseAddresses),
       fromBlockQueried: fromBlock,
       latestFetchedBlockNumber: latestFetchedBlockInfo.number,
-      latestFetchedBlockTimestamp: latestFetchedBlockInfo.timestamp,
       stats: {
         "total time elapsed (s)": totalTimeElapsed
       },
       requestStats: drainRequestStats()
     };
   };
-  let onReorg = param => {
-    blockLoader.contents = makeBlockLoader();
-    transactionLoader.contents = makeTransactionLoader();
-    receiptLoader.contents = makeReceiptLoader();
-  };
+  let onReorg = () => resetCachedLoaders();
   let getBlockHashes = (blockNumbers, _currentlyUnusedLogger) => Stdlib_Promise.$$catch(Promise.all(blockNumbers.map(blockNum => LazyLoader.get(blockLoader.contents, blockNum))).then(rawBlocks => {
-    let result = {
-      TAG: "Ok",
-      _0: rawBlocks.map(json => {
-        let b = parseBlockInfo(json);
-        return {
-          blockHash: b.hash,
-          blockNumber: b.number,
-          blockTimestamp: b.timestamp
-        };
-      })
-    };
+    let observedBlocks = [];
+    rawBlocks.forEach(json => {
+      let b = parseBlockInfo(json);
+      observedBlocks.push({
+        blockNumber: b.number,
+        blockHash: b.hash,
+        blockTimestamp: b.timestamp
+      });
+      if (b.number > 0) {
+        observedBlocks.push({
+          blockNumber: b.number - 1 | 0,
+          blockHash: b.parentHash
+        });
+        return;
+      }
+    });
+    let blockStore = BlockStore.fromJs(observedBlocks, "evm", !lowercaseAddresses);
     return {
-      result: result,
+      result: {
+        TAG: "Ok",
+        _0: blockStore
+      },
       requestStats: drainRequestStats()
     };
   }), exn => Promise.resolve({
@@ -1041,6 +1098,8 @@ function make(param) {
   };
 }
 
+let effectiveGasPriceKey = "effectiveGasPrice";
+
 export {
   TransactionDataNotFound,
   getKnownRawBlock,
@@ -1058,7 +1117,11 @@ export {
   makeFieldRegistry,
   fieldRegistryLowercase,
   fieldRegistryChecksum,
+  isRpcTransactionField,
   parseFieldsFromJson,
+  effectiveGasPriceKey,
+  effectiveGasPriceFallbackDef,
+  fillEffectiveGasPriceOrThrow,
   makeThrowingGetEventTransaction,
   make,
 }

package/src/sources/SimulateSource.res

@@ -3,6 +3,7 @@ let make = (
   ~endBlock: int,
   ~chainId: ChainId.t,
   ~addressStore: AddressStore.t,
+  ~ecosystem: Ecosystem.name=Evm,
 ): Source.t => {
   let reportedHeight = max(endBlock, 1)
 
@@ -13,7 +14,10 @@ let make = (
     poweredByHyperSync: false,
     pollingInterval: 0,
     getBlockHashes: (~blockNumbers as _, ~logger as _) => {
-      Promise.resolve({Source.result: Ok([]), requestStats: []})
+      Promise.resolve({
+        Source.result: Ok(BlockStore.fromJs([], ~ecosystem, ~shouldChecksum=false)),
+        requestStats: [],
+      })
     },
     getHeightOrThrow: () => {
       // Report at least height 1 so the engine doesn't treat 0 as "no blocks available"
@@ -101,14 +105,13 @@ let make = (
 
       Promise.resolve({
         Source.knownHeight: reportedHeight,
-        blockHashes: [],
         parsedQueueItems,
-        // Simulate keeps the transaction and block inline on the payload; no store pages.
+        // Simulate keeps the transaction and block inline on the payload; no
+        // transaction page and an empty block page (nothing to reorg-check).
         transactionStore: None,
-        blockStore: None,
+        blockStore: BlockStore.fromJs([], ~ecosystem, ~shouldChecksum=false),
         fromBlockQueried: fromBlock,
         latestFetchedBlockNumber: toBlockQueried,
-        latestFetchedBlockTimestamp: 0,
         stats: {
           totalTimeElapsed: 0.,
         },

package/src/sources/SimulateSource.res.mjs

@@ -1,8 +1,10 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as BlockStore from "./BlockStore.res.mjs";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 
-function make(items, endBlock, chainId, addressStore) {
+function make(items, endBlock, chainId, addressStore, ecosystemOpt) {
+  let ecosystem = ecosystemOpt !== undefined ? ecosystemOpt : "evm";
   let reportedHeight = Primitive_int.max(endBlock, 1);
   return {
     name: "SimulateSource",
@@ -13,7 +15,7 @@ function make(items, endBlock, chainId, addressStore) {
     getBlockHashes: (param, param$1) => Promise.resolve({
       result: {
         TAG: "Ok",
-        _0: []
+        _0: BlockStore.fromJs([], ecosystem, false)
       },
       requestStats: []
     }),
@@ -72,13 +74,11 @@ function make(items, endBlock, chainId, addressStore) {
       });
       return Promise.resolve({
         knownHeight: reportedHeight,
-        blockHashes: [],
         parsedQueueItems: parsedQueueItems,
         transactionStore: undefined,
-        blockStore: undefined,
+        blockStore: BlockStore.fromJs([], ecosystem, false),
         fromBlockQueried: fromBlock,
         latestFetchedBlockNumber: toBlockQueried,
-        latestFetchedBlockTimestamp: 0,
         stats: {
           "total time elapsed (s)": 0
         },
@@ -91,4 +91,4 @@ function make(items, endBlock, chainId, addressStore) {
 export {
   make,
 }
-/* No side effect */
+/* BlockStore Not a pure module */

package/src/sources/Source.res

@@ -10,30 +10,82 @@ type blockRangeFetchStats = {
 // A single backend request a source method actually made (cache/dedup hits
 // aren't requests), with the time it took. SourceManager aggregates these
 // per (source, method) into the envio_source_request_* metrics.
-type requestStat = {method: string, seconds: float}
+type requestStat = RequestStat.t = {method: string, seconds: float}
+
+// Native clients wrap a failure of a multi-request operation in a structured
+// payload, so the source can still return timings when SourceManager retries
+// it. `cause` carries the inner message as a plain error, ready for logging.
+type nativeRequestFailure = {
+  cause: exn,
+  message: option<string>,
+  requestStats: array<requestStat>,
+}
+
+// Prefix marking a napi error reason as a structured native-failure envelope.
+// Keep in sync with `request_stats.rs` `NATIVE_FAILURE_PREFIX`.
+let nativeFailurePrefix = "ENVIO_NATIVE_FAILURE:"
+
+// The envelope `request_stats.rs` writes after the prefix.
+type nativeFailurePayload = {message: string, requestStats: array<requestStat>}
+
+let nativeFailurePayloadSchema = S.schema(s => {
+  message: s.matches(S.string),
+  requestStats: s.matches(
+    S.array(
+      S.schema(s => {
+        method: s.matches(S.string),
+        seconds: s.matches(S.float),
+      }),
+    ),
+  ),
+})
+
+let unpackNativeRequestFailure = (exn: exn): nativeRequestFailure => {
+  let originalMessage = switch exn->JsExn.anyToExnInternal {
+  | JsExn(jsExn) => jsExn->JsExn.message
+  | _ => None
+  }
+  // Only a reason carrying our prefix is one of our envelopes; anything else
+  // keeps its original message and cause untouched.
+  let decoded = switch originalMessage {
+  | Some(message) if message->String.startsWith(nativeFailurePrefix) =>
+    try Some(
+      message
+      ->String.slice(~start=nativeFailurePrefix->String.length, ~end=message->String.length)
+      ->JSON.parseOrThrow
+      ->S.parseOrThrow(nativeFailurePayloadSchema),
+    ) catch {
+    | _ => None
+    }
+  | _ => None
+  }
+  switch decoded {
+  | Some({message, requestStats}) => {
+      cause: JsError.make(message)->(Utils.magic: JsError.t => exn),
+      message: Some(message),
+      requestStats,
+    }
+  | None => {cause: exn, message: originalMessage, requestStats: []}
+  }
+}
 
 /**
 Thes response returned from a block range fetch
 */
 type blockRangeFetchResponse = {
   knownHeight: int,
-  // Best-effort (blockNumber, blockHash) pairs observed while fetching this range.
-  // Used by reorg detection; gaps are OK, no extra requests are made to fill them.
-  // Duplicates with the same block number are allowed — registerReorgGuard treats
-  // a within-array hash mismatch on the same block number as a reorg.
-  blockHashes: array<ReorgDetection.blockData>,
   parsedQueueItems: array<Internal.item>,
   // Page of transactions for this response's items, keyed by (blockNumber,
   // transactionIndex); merged into the chain's store on apply. `None` for
   // sources that keep the transaction inline on the payload (RPC/Fuel/Simulate).
   transactionStore: option<TransactionStore.t>,
-  // Page of blocks for this response's items, keyed by block number; merged into
-  // the chain's store on apply. `None` for sources that keep the block fully
-  // inline on the payload (RPC/Fuel/Simulate).
-  blockStore: option<BlockStore.t>,
+  // Page of blocks observed while fetching this range, keyed by block number;
+  // merged into the chain's store on apply, where its hashes drive reorg
+  // detection. Sources that keep the block inline on the payload (RPC/Simulate)
+  // contribute hash-only rows built from the block hashes they saw.
+  blockStore: BlockStore.t,
   fromBlockQueried: int,
   latestFetchedBlockNumber: int,
-  latestFetchedBlockTimestamp: int,
   stats: blockRangeFetchStats,
   requestStats: array<requestStat>,
 }
@@ -41,16 +93,33 @@ type blockRangeFetchResponse = {
 type getHeightResponse = {height: int, requestStats: array<requestStat>}
 
 type getBlockHashesResponse = {
-  result: result<array<ReorgDetection.blockDataWithTimestamp>, exn>,
+  result: result<BlockStore.t, exn>,
   requestStats: array<requestStat>,
 }
 
+exception InconsistentResponse({
+  method: string,
+  blockNumber: option<int>,
+  storedHash: option<string>,
+  receivedHash: option<string>,
+  missingBlockNumbers: array<int>,
+})
+
+// The queried block hasn't reached the backend instance that served the
+// request. Load-balanced backends drift from each other around the head, so
+// this is expected there and resolves by retrying — SourceManager owns the
+// backoff and the decision to fail over, identically for every ecosystem.
+// Carries the timings of the requests the failed operation did make, so a
+// retried request still counts towards the source's metrics.
+exception SourceBehindHead({blockNumber: int, requestStats: array<requestStat>})
+
 type getItemsRetry =
   | WithSuggestedToBlock({toBlock: int})
   | WithBackoff({message: string, backoffMillis: int})
   | ImpossibleForTheQuery({message: string})
 
-exception RateLimited({resetMs: int})
+type rateLimited = {resetMs: int, requestStats: array<requestStat>}
+exception RateLimited(rateLimited)
 
 type getItemsError =
   | UnsupportedSelection({message: string})
@@ -92,7 +161,11 @@ type t = {
     ~logger: Pino.t,
   ) => promise<blockRangeFetchResponse>,
   createHeightSubscription?: (~onHeight: int => unit) => unit => unit,
-  // Invoked by SourceManager once a rollback target is known so the source can
-  // drop any state that may now point at an orphaned chain (e.g. RPC block cache).
-  onReorg?: (~rollbackTargetBlock: int) => unit,
+  // Invoked when a reorg or internally inconsistent response means local state
+  // may point at an orphaned chain (e.g. the RPC block cache): drop all of it.
+  // Deliberately takes no rollback target — the deepest reorged block isn't
+  // known until the depth search runs, and that search reads back through this
+  // very state, so pruning relative to a target would keep exactly the entries
+  // that make it answer wrong.
+  onReorg?: unit => unit,
 }

package/src/sources/Source.res.mjs

@@ -1,13 +1,62 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
+let nativeFailurePrefix = "ENVIO_NATIVE_FAILURE:";
+
+let nativeFailurePayloadSchema = S$RescriptSchema.schema(s => ({
+  message: s.m(S$RescriptSchema.string),
+  requestStats: s.m(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
+    method: s.m(S$RescriptSchema.string),
+    seconds: s.m(S$RescriptSchema.float)
+  }))))
+}));
+
+function unpackNativeRequestFailure(exn) {
+  let jsExn = Primitive_exceptions.internalToException(exn);
+  let originalMessage = jsExn.RE_EXN_ID === "JsExn" ? Stdlib_JsExn.message(jsExn._1) : undefined;
+  let decoded;
+  if (originalMessage !== undefined && originalMessage.startsWith(nativeFailurePrefix)) {
+    try {
+      decoded = S$RescriptSchema.parseOrThrow(JSON.parse(originalMessage.slice(nativeFailurePrefix.length, originalMessage.length)), nativeFailurePayloadSchema);
+    } catch (exn$1) {
+      decoded = undefined;
+    }
+  } else {
+    decoded = undefined;
+  }
+  if (decoded === undefined) {
+    return {
+      cause: exn,
+      message: originalMessage,
+      requestStats: []
+    };
+  }
+  let message = decoded.message;
+  return {
+    cause: new Error(message),
+    message: message,
+    requestStats: decoded.requestStats
+  };
+}
+
+let InconsistentResponse = /* @__PURE__ */Primitive_exceptions.create("Source.InconsistentResponse");
+
+let SourceBehindHead = /* @__PURE__ */Primitive_exceptions.create("Source.SourceBehindHead");
+
 let RateLimited = /* @__PURE__ */Primitive_exceptions.create("Source.RateLimited");
 
 let GetItemsError = /* @__PURE__ */Primitive_exceptions.create("Source.GetItemsError");
 
 export {
+  nativeFailurePrefix,
+  nativeFailurePayloadSchema,
+  unpackNativeRequestFailure,
+  InconsistentResponse,
+  SourceBehindHead,
   RateLimited,
   GetItemsError,
 }
-/* No side effect */
+/* nativeFailurePayloadSchema Not a pure module */