envio 3.6.1 → 3.7.0

package/src/sources/SourceManager.res.mjs

@@ -152,7 +152,7 @@ async function waitForRateLimitReset(sourceManager, resetMs, retry, logger) {
   let waitMs = Primitive_int.min(resetMs, 300000);
   let log = retry >= 2 ? Logging.childWarn : Logging.childTrace;
   log(logger, {
-    msg: `HyperSync source is rate-limited — not critical, the indexer will retry in ` + (waitMs / 1000 | 0).toString() + `s. For higher limits upgrade your plan at https://envio.dev/app/api-tokens.`,
+    msg: `HyperSync source is rate-limited - not critical, the indexer will retry in ` + (waitMs / 1000 | 0).toString() + `s. For higher limits upgrade your plan at https://envio.dev/app/api-tokens.`,
     retry: retry,
     waitMs: waitMs
   });
@@ -170,11 +170,40 @@ async function waitForRateLimitReset(sourceManager, resetMs, retry, logger) {
   sourceManager.activeRateLimitResetAtMs = undefined;
 }
 
-function onReorg(sourceManager, rollbackTargetBlock) {
+function validateResponseBlockStore(method, blockStore, requiredBlockNumbersOpt) {
+  let requiredBlockNumbers = requiredBlockNumbersOpt !== undefined ? requiredBlockNumbersOpt : [];
+  let match = blockStore.responseConflict();
+  if (match !== null) {
+    throw {
+      RE_EXN_ID: Source.InconsistentResponse,
+      method: method,
+      blockNumber: match.blockNumber,
+      storedHash: match.storedHash,
+      receivedHash: match.receivedHash,
+      missingBlockNumbers: [],
+      Error: new Error()
+    };
+  }
+  let missingBlockNumbers = blockStore.missingHashes(requiredBlockNumbers);
+  if (missingBlockNumbers.length === 0) {
+    return;
+  }
+  throw {
+    RE_EXN_ID: Source.InconsistentResponse,
+    method: method,
+    blockNumber: undefined,
+    storedHash: undefined,
+    receivedHash: undefined,
+    missingBlockNumbers: missingBlockNumbers,
+    Error: new Error()
+  };
+}
+
+function onReorg(sourceManager) {
   sourceManager.sourcesState.forEach(param => {
     let cb = param.source.onReorg;
     if (cb !== undefined) {
-      return cb(rollbackTargetBlock);
+      return cb();
     }
   });
 }
@@ -507,6 +536,71 @@ function getNextSource(sourceManager, isRealtime, excludedSources) {
   }
 }
 
+function retryBackoffMillis(retry) {
+  return Primitive_float.min(100 * 2 ** retry, 60000) | 0;
+}
+
+async function backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, minBackoffMillisOpt, excludedSources) {
+  let minBackoffMillis = minBackoffMillisOpt !== undefined ? minBackoffMillisOpt : 0;
+  let switchedToWorkingSource;
+  if (retry >= 2 && retry % 2 === 0) {
+    let now = Date.now();
+    sourceState.lastFailedAt = now;
+    let next = getNextSource(sourceManager, isRealtime, excludedSources);
+    if (next !== undefined) {
+      let failedAt = next.lastFailedAt;
+      switchedToWorkingSource = failedAt !== undefined ? now - failedAt >= sourceManager.recoveryTimeout : true;
+    } else {
+      switchedToWorkingSource = false;
+    }
+  } else {
+    switchedToWorkingSource = false;
+  }
+  if (switchedToWorkingSource) {
+    if (minBackoffMillis > 0) {
+      return await Utils.delay(minBackoffMillis);
+    } else {
+      return;
+    }
+  } else {
+    return await Utils.delay(Primitive_int.min(Primitive_int.max(backoffMillis, minBackoffMillis), 60000));
+  }
+}
+
+async function retryBehindHead(sourceManager, sourceState, retry, isRealtime, logger, blockNumber, method, err, excludedSources) {
+  let backoffMillis = retryBackoffMillis(retry);
+  let log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
+  log(logger, {
+    msg: `Block #` + blockNumber.toString() + ` is not available on the ` + sourceState.source.name + ` source yet. Instances of a load-balanced backend drift slightly around the head, so this is expected - indexing continues after an automatic retry.`,
+    method: method,
+    retry: retry,
+    backOffMilliseconds: backoffMillis,
+    err: Utils.prettifyExn(err)
+  });
+  return await backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, 50, excludedSources);
+}
+
+async function retryInconsistentResponse(sourceManager, sourceState, retry, isRealtime, logger, method, err, excludedSources) {
+  let backoffMillis = retryBackoffMillis(retry);
+  let msg = `Received a partial indicator of a possible reorg from the ` + sourceState.source.name + ` source while fetching ` + method + `. Retrying the request to better identify whether a reorg happened.`;
+  let match = retry >= 13 ? [
+      Logging.childError,
+      msg + " It has disagreed with itself on every attempt for several minutes now, so this chain has stopped making progress - the endpoint is likely serving blocks and logs from nodes on different chains."
+    ] : [
+      retry >= 2 ? Logging.childWarn : Logging.childTrace,
+      msg
+    ];
+  match[0](logger, {
+    msg: match[1],
+    method: method,
+    retry: retry,
+    backOffMilliseconds: backoffMillis,
+    err: Utils.prettifyExn(err)
+  });
+  Stdlib_Option.forEach(sourceState.source.onReorg, cb => cb());
+  return await backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, 50, excludedSources);
+}
+
 async function waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling) {
   let sourcesState = sourceManager.sourcesState;
   let logger = Logging.createChild({
@@ -606,31 +700,41 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
     try {
       let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addresses, knownHeight, query.partitionId, FetchState.narrowSelectionToRange(query.selection, toBlock), query.itemsTarget, retry, logger$2);
       recordRequestStats(sourceState, response.requestStats);
+      validateResponseBlockStore("getItems", response.blockStore, undefined);
       sourceState.lastFailedAt = undefined;
       if (response.knownHeight > sourceState.knownHeight) {
         sourceState.knownHeight = response.knownHeight;
       }
       responseRef = response;
-    } catch (raw_error) {
-      let error = Primitive_exceptions.internalToException(raw_error);
-      if (error.RE_EXN_ID === Source.RateLimited) {
-        await waitForRateLimitReset(sourceManager, error.resetMs, retry, logger$2);
+    } catch (raw_err) {
+      let err = Primitive_exceptions.internalToException(raw_err);
+      if (err.RE_EXN_ID === Source.RateLimited) {
+        let match = err._1;
+        recordRequestStats(sourceState, match.requestStats);
+        await waitForRateLimitReset(sourceManager, match.resetMs, retry, logger$2);
+        retryRef = retryRef + 1 | 0;
+      } else if (err.RE_EXN_ID === Source.SourceBehindHead) {
+        recordRequestStats(sourceState, err.requestStats);
+        await retryBehindHead(sourceManager, sourceState, retry, isRealtime, logger$2, err.blockNumber, "getItems", err, excludedSourcesRef);
+        retryRef = retryRef + 1 | 0;
+      } else if (err.RE_EXN_ID === Source.InconsistentResponse) {
+        await retryInconsistentResponse(sourceManager, sourceState, retry, isRealtime, logger$2, "getItems", err, excludedSourcesRef);
         retryRef = retryRef + 1 | 0;
-      } else if (error.RE_EXN_ID === Source.GetItemsError) {
-        let error$1 = error._1;
+      } else if (err.RE_EXN_ID === Source.GetItemsError) {
+        let error = err._1;
         let exit = 0;
-        switch (error$1.TAG) {
+        switch (error.TAG) {
           case "UnsupportedSelection" :
           case "FailedGettingFieldSelection" :
             exit = 1;
             break;
           case "FailedGettingItems" :
-            let match = error$1.retry;
-            let attemptedToBlock = error$1.attemptedToBlock;
-            let exn = error$1.exn;
-            switch (match.TAG) {
+            let match$1 = error.retry;
+            let attemptedToBlock = error.attemptedToBlock;
+            let exn = error.exn;
+            switch (match$1.TAG) {
               case "WithSuggestedToBlock" :
-                let toBlock$1 = match.toBlock;
+                let toBlock$1 = match$1.toBlock;
                 Logging.childTrace(logger$2, {
                   msg: "Failed getting data for the block range. Immediately retrying with the suggested block range from response.",
                   toBlock: attemptedToBlock,
@@ -640,33 +744,16 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
                 retryRef = 0;
                 break;
               case "WithBackoff" :
-                let backoffMillis = match.backoffMillis;
+                let backoffMillis = match$1.backoffMillis;
                 let log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
                 log(logger$2, {
-                  msg: match.message,
+                  msg: match$1.message,
                   toBlock: attemptedToBlock,
                   backOffMilliseconds: backoffMillis,
                   retry: retry,
                   err: Utils.prettifyExn(exn)
                 });
-                let shouldSwitch = retry !== 0 && retry !== 1 ? retry % 2 === 0 : false;
-                if (shouldSwitch) {
-                  let now = Date.now();
-                  sourceState.lastFailedAt = now;
-                  let nextSource = getNextSource(sourceManager, isRealtime, excludedSourcesRef);
-                  let hasWorkingAlternative;
-                  if (nextSource !== undefined) {
-                    let failedAt = nextSource.lastFailedAt;
-                    hasWorkingAlternative = failedAt !== undefined ? now - failedAt >= sourceManager.recoveryTimeout : true;
-                  } else {
-                    hasWorkingAlternative = false;
-                  }
-                  if (!hasWorkingAlternative) {
-                    await Utils.delay(Primitive_int.min(backoffMillis, 60000));
-                  }
-                } else {
-                  await Utils.delay(Primitive_int.min(backoffMillis, 60000));
-                }
+                await backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, undefined, excludedSourcesRef);
                 retryRef = retryRef + 1 | 0;
                 break;
               case "ImpossibleForTheQuery" :
@@ -681,7 +768,7 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
                 }
                 excludedSources.add(sourceState);
                 Logging.childWarn(logger$2, {
-                  msg: match.message + " - Attempting another source",
+                  msg: match$1.message + " - Attempting another source",
                   toBlock: attemptedToBlock,
                   err: Utils.prettifyExn(exn)
                 });
@@ -693,16 +780,16 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
         if (exit === 1) {
           let notAlreadyDisabled = disableSource(sourceManager, sourceState);
           if (notAlreadyDisabled) {
-            switch (error$1.TAG) {
+            switch (error.TAG) {
               case "UnsupportedSelection" :
-                Logging.childError(logger$2, error$1.message);
+                Logging.childError(logger$2, error.message);
                 break;
               case "FailedGettingFieldSelection" :
                 Logging.childError(logger$2, {
-                  msg: error$1.message,
-                  err: Utils.prettifyExn(error$1.exn),
-                  blockNumber: error$1.blockNumber,
-                  logIndex: error$1.logIndex
+                  msg: error.message,
+                  err: Utils.prettifyExn(error.exn),
+                  blockNumber: error.blockNumber,
+                  logIndex: error.logIndex
                 });
                 break;
               case "FailedGettingItems" :
@@ -712,7 +799,7 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
           retryRef = 0;
         }
       } else {
-        ErrorHandling.mkLogAndRaise(logger$2, "Failed to fetch block Range", error);
+        ErrorHandling.mkLogAndRaise(logger$2, "Failed to fetch block Range", err);
       }
     }
   };
@@ -747,30 +834,37 @@ async function getBlockHashes(sourceManager, blockNumbers, isRealtime) {
       recordRequestStats(sourceState, res.requestStats);
       let data = res.result;
       if (data.TAG === "Ok") {
+        let data$1 = data._0;
+        validateResponseBlockStore("getBlockHashes", data$1, blockNumbers);
         sourceState.lastFailedAt = undefined;
-        responseRef = data._0;
+        responseRef = Primitive_option.some(data$1);
       } else {
         throw data._0;
       }
-    } catch (raw_exn) {
-      let exn = Primitive_exceptions.internalToException(raw_exn);
-      if (exn.RE_EXN_ID === Source.RateLimited) {
-        await waitForRateLimitReset(sourceManager, exn.resetMs, retry, logger$1);
+    } catch (raw_err) {
+      let err = Primitive_exceptions.internalToException(raw_err);
+      if (err.RE_EXN_ID === Source.RateLimited) {
+        let match = err._1;
+        recordRequestStats(sourceState, match.requestStats);
+        await waitForRateLimitReset(sourceManager, match.resetMs, retry, logger$1);
+        retryRef = retryRef + 1 | 0;
+      } else if (err.RE_EXN_ID === Source.SourceBehindHead) {
+        recordRequestStats(sourceState, err.requestStats);
+        await retryBehindHead(sourceManager, sourceState, retry, isRealtime, logger$1, err.blockNumber, "getBlockHashes", err, undefined);
+        retryRef = retryRef + 1 | 0;
+      } else if (err.RE_EXN_ID === Source.InconsistentResponse) {
+        await retryInconsistentResponse(sourceManager, sourceState, retry, isRealtime, logger$1, "getBlockHashes", err, undefined);
         retryRef = retryRef + 1 | 0;
       } else {
-        let backoffMillis = retry !== 0 ? 1000 * retry | 0 : 500;
+        let backoffMillis = retryBackoffMillis(retry);
         let log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
         log(logger$1, {
           msg: "Failed to fetch block hashes. Retrying.",
           retry: retry,
           backOffMilliseconds: backoffMillis,
-          err: Utils.prettifyExn(exn)
+          err: Utils.prettifyExn(err)
         });
-        let shouldSwitch = retry !== 0 && retry !== 1 ? retry % 2 === 0 : false;
-        if (shouldSwitch) {
-          sourceState.lastFailedAt = Date.now();
-        }
-        await Utils.delay(Primitive_int.min(backoffMillis, 60000));
+        await backoffBeforeRetry(sourceManager, sourceState, retry, isRealtime, backoffMillis, 50, undefined);
         retryRef = retryRef + 1 | 0;
       }
     }

package/src/sources/SourceManager.resi

@@ -45,7 +45,7 @@ let queryingSeconds: t => float
 
 let inFlightCount: t => int
 
-let onReorg: (t, ~rollbackTargetBlock: int) => unit
+let onReorg: t => unit
 
 let dispatch: (
   t,
@@ -77,11 +77,7 @@ let makeGetHeightRetryInterval: (
   ~maxRetryInterval: int,
 ) => (~retry: int) => int
 
-let getBlockHashes: (
-  t,
-  ~blockNumbers: array<int>,
-  ~isRealtime: bool,
-) => promise<array<ReorgDetection.blockDataWithTimestamp>>
+let getBlockHashes: (t, ~blockNumbers: array<int>, ~isRealtime: bool) => promise<BlockStore.t>
 
 // Total time the indexer spent waiting on rate limits, including any
 // currently-in-progress window measured against Date.now() at call time.

package/src/sources/Svm.res

@@ -1,9 +1,3 @@
-let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
-    delete fields.hash
-    delete fields.height
-    delete fields.time
-  }`)
-
 // Ordered transaction field names, the field codes shared with the Rust store
 // (`SvmTxField`). Derived from the typed field list so the two can't drift;
 // `Internal.allSvmTransactionFields` is pinned to the Rust ordinal order by a test.
@@ -32,7 +26,6 @@ let make = (~logger: Pino.t): Ecosystem.t => {
   blockNumberName: "height",
   blockTimestampName: "time",
   blockHashName: "hash",
-  cleanUpRawEventFieldsInPlace,
   onBlockMethodName: "onSlot",
   // SVM filter shape: `{slot: {_gte?, _lte?, _every?}}`.
   // Inner range chunk parsed by `blockRangeSchema` in `Main.res`.

package/src/sources/Svm.res.mjs

@@ -12,12 +12,6 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as TransactionStore from "./TransactionStore.res.mjs";
 
-let cleanUpRawEventFieldsInPlace = (fields => {
-    delete fields.hash
-    delete fields.height
-    delete fields.time
-  });
-
 let eventTransactionFieldMask = TransactionStore.makeMaskFn(Internal.allSvmTransactionFields);
 
 let blockFields = [
@@ -37,7 +31,6 @@ function make(logger) {
     blockNumberName: "height",
     blockTimestampName: "time",
     blockHashName: "hash",
-    cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
     onBlockMethodName: "onSlot",
     onBlockFilterSchema: S$RescriptSchema.object(s => s.f("slot", S$RescriptSchema.option(S$RescriptSchema.unknown))),
     onEventBlockFilterSchema: S$RescriptSchema.object(param => {}),
@@ -99,7 +92,6 @@ function makeRPCSource(chainId, rpc, sourceForOpt) {
 let transactionFields = Internal.allSvmTransactionFields;
 
 export {
-  cleanUpRawEventFieldsInPlace,
   transactionFields,
   eventTransactionFieldMask,
   blockFields,

package/src/sources/SvmHyperSyncClient.res

@@ -4,7 +4,6 @@ type cfg = {
   /** Optional bearer token for the HyperSync server. */
   apiToken?: string,
   httpReqTimeoutMillis?: int,
-  maxNumRetries?: int,
   retryBaseMs?: int,
   retryCeilingMs?: int,
 }
@@ -67,10 +66,8 @@ module Registration = {
             },
           )
         ),
-        transactionFields: eventConfig.selectedTransactionFields->Utils.Set.toArray,
-        blockFields: eventConfig.selectedBlockFields
-        ->(Utils.magic: Utils.Set.t<Internal.svmBlockField> => Utils.Set.t<string>)
-        ->Utils.Set.toArray,
+        transactionFields: reg.fieldSelection.transactionFields->Utils.Set.toArray,
+        blockFields: reg.fieldSelection.blockFields->Utils.Set.toArray,
         accounts: eventConfig.accounts,
         argsJson: ?switch eventConfig.args {
         | JSON.Null => None
@@ -110,18 +107,20 @@ module QueryTypes = {
 
   type fieldSelection = {block?: array<blockField>, transaction?: array<transactionField>}
 
-  /** Filter for selecting instructions. All non-empty fields are AND-ed: an
-   instruction must match at least one value in every non-empty field.
+  /** Filter for selecting instruction calls. All non-empty fields are AND-ed:
+   an instruction must match at least one value in every non-empty field.
 
    Discriminator filters (d1..d8) take hex-encoded byte prefixes ("0x" optional).
    Account filters (a0..a9) take base58 pubkey strings. */
   type instructionSelection = {
-    programId?: array<string>,
+    executingAccount?: array<string>,
     d1?: array<string>,
     d2?: array<string>,
     d4?: array<string>,
     d8?: array<string>,
     isInner?: bool,
+    /** Success of the parent transaction; absent matches both. */
+    txSuccess?: bool,
   }
 
   // The `get` query surface, used only for block-data range queries; event
@@ -129,7 +128,7 @@ module QueryTypes = {
   type query = {
     fromSlot: int,
     toSlot?: int,
-    instructions?: array<instructionSelection>,
+    instructionCalls?: array<instructionSelection>,
     includeAllBlocks?: bool,
     fields?: fieldSelection,
     maxNumBlocks?: int,
@@ -155,24 +154,29 @@ module ResponseTypes = {
     extraAccounts: array<string>,
   }
 
-  type instruction = {
+  type instructionCall = {
     slot: int,
     transactionIndex: int,
     instructionAddress: array<int>,
-    programId: string,
-    accounts: array<string>,
+    /** The invoked program's account. */
+    executingAccount: string,
+    accountArguments: array<string>,
     data: string,
     d1?: string,
     d2?: string,
     d4?: string,
     d8?: string,
     isInner: bool,
-    isCommitted: bool,
+    /** Success of the parent transaction, not of this invocation. */
+    txSuccess: bool,
+    /** Per-invocation failure reason (e.g. "custom program error: 0x1"). */
+    error?: string,
+    computeUnitsConsumed?: bigint,
   }
 
   type queryResponseData = {
     blocks: array<block>,
-    instructions: array<instruction>,
+    instructionCalls: array<instructionCall>,
   }
 
   type queryResponse = {
@@ -241,6 +245,9 @@ type queryResponse = ResponseTypes.queryResponse
 
 type t = {
   getHeight: unit => promise<int>,
+  // Block-hash query construction, pagination, and cursor-backed skipped-slot
+  // coverage live in Rust.
+  getBlockHashes: (~blockNumbers: array<int>) => promise<(BlockStore.t, array<RequestStat.t>)>,
   // Block-data range queries only; the store pages it returns are empty.
   get: (~query: query) => promise<(queryResponse, TransactionStore.t, BlockStore.t)>,
   // Returns the routed items plus pages of raw transactions and blocks (kept
@@ -265,7 +272,6 @@ let make = (
   ~url,
   ~apiToken=?,
   ~httpReqTimeoutMillis=?,
-  ~maxNumRetries=?,
   ~retryBaseMs=?,
   ~retryCeilingMs=?,
   ~eventRegistrations=[],
@@ -277,7 +283,6 @@ let make = (
       url,
       ?apiToken,
       ?httpReqTimeoutMillis,
-      ?maxNumRetries,
       ?retryBaseMs,
       ?retryCeilingMs,
     },

package/src/sources/SvmHyperSyncClient.res.mjs

@@ -27,8 +27,8 @@ function fromOnEventRegistrations(onEventRegistrations) {
         position: filter.position,
         values: filter.values
       }))),
-      transactionFields: Array.from(eventConfig.selectedTransactionFields),
-      blockFields: Array.from(eventConfig.selectedBlockFields),
+      transactionFields: Array.from(reg.fieldSelection.transactionFields),
+      blockFields: Array.from(reg.fieldSelection.blockFields),
       accounts: eventConfig.accounts,
       argsJson: tmp,
       definedTypesJson: tmp$1
@@ -46,14 +46,13 @@ let ResponseTypes = {};
 
 let EventItems = {};
 
-function make(url, apiToken, httpReqTimeoutMillis, maxNumRetries, retryBaseMs, retryCeilingMs, eventRegistrationsOpt, addressStore) {
+function make(url, apiToken, httpReqTimeoutMillis, retryBaseMs, retryCeilingMs, eventRegistrationsOpt, addressStore) {
   let eventRegistrations = eventRegistrationsOpt !== undefined ? eventRegistrationsOpt : [];
   let envioVersion = Utils.EnvioPackage.value.version;
   return Core.getAddon().SvmHyperSyncClient.fromConfig({
     url: url,
     apiToken: apiToken,
     httpReqTimeoutMillis: httpReqTimeoutMillis,
-    maxNumRetries: maxNumRetries,
     retryBaseMs: retryBaseMs,
     retryCeilingMs: retryCeilingMs
   }, `hyperindex/` + envioVersion, eventRegistrations, addressStore);