@across-protocol/sdk 3.1.17 → 3.1.19

package/src/providers/retryProvider.ts

@@ -0,0 +1,262 @@
+import { ethers, logger } from "ethers";
+import { CachingMechanismInterface } from "../interfaces";
+import { delay, isDefined, isPromiseFulfilled, isPromiseRejected } from "../utils";
+import { getOriginFromURL } from "../utils/NetworkUtils";
+import { CacheProvider } from "./cachedProvider";
+import { compareRpcResults, createSendErrorWithMessage, formatProviderError } from "./utils";
+import { AugmentedLogger } from "@uma/logger";
+import { PROVIDER_CACHE_TTL } from "./constants";
+
+export class RetryProvider extends ethers.providers.StaticJsonRpcProvider {
+  readonly providers: ethers.providers.StaticJsonRpcProvider[];
+  constructor(
+    params: ConstructorParameters<typeof ethers.providers.StaticJsonRpcProvider>[],
+    chainId: number,
+    readonly nodeQuorumThreshold: number,
+    readonly retries: number,
+    readonly delay: number,
+    readonly maxConcurrency: number,
+    providerCacheNamespace: string,
+    pctRpcCallsLogged: number,
+    redisClient?: CachingMechanismInterface,
+    standardTtlBlockDistance?: number,
+    noTtlBlockDistance?: number,
+    providerCacheTtl = PROVIDER_CACHE_TTL,
+    logger?: AugmentedLogger
+  ) {
+    // Initialize the super just with the chainId, which stops it from trying to immediately send out a .send before
+    // this derived class is initialized.
+    super(undefined, chainId);
+    this.providers = params.map(
+      (inputs) =>
+        new CacheProvider(
+          providerCacheNamespace,
+          redisClient,
+          standardTtlBlockDistance,
+          noTtlBlockDistance,
+          providerCacheTtl,
+          maxConcurrency,
+          pctRpcCallsLogged,
+          logger,
+          ...inputs
+        )
+    );
+    if (this.nodeQuorumThreshold < 1 || !Number.isInteger(this.nodeQuorumThreshold)) {
+      throw new Error(
+        `nodeQuorum,Threshold cannot be < 1 and must be an integer. Currently set to ${this.nodeQuorumThreshold}`
+      );
+    }
+    if (this.retries < 0 || !Number.isInteger(this.retries)) {
+      throw new Error(`retries cannot be < 0 and must be an integer. Currently set to ${this.retries}`);
+    }
+    if (this.delay < 0) {
+      throw new Error(`delay cannot be < 0. Currently set to ${this.delay}`);
+    }
+    if (this.nodeQuorumThreshold > this.providers.length) {
+      throw new Error(
+        `nodeQuorumThreshold (${this.nodeQuorumThreshold}) must be <= the number of providers (${this.providers.length})`
+      );
+    }
+  }
+
+  override async send(method: string, params: Array<unknown>): Promise<unknown> {
+    const quorumThreshold = this._getQuorum(method, params);
+    const requiredProviders = this.providers.slice(0, quorumThreshold);
+    const fallbackProviders = this.providers.slice(quorumThreshold);
+    const errors: [ethers.providers.StaticJsonRpcProvider, string][] = [];
+
+    // This function is used to try to send with a provider and if it fails pop an element off the fallback list to try
+    // with that one. Once the fallback provider list is empty, the method throws. Because the fallback providers are
+    // removed, we ensure that no provider is used more than once because we care about quorum, making sure all
+    // considered responses come from unique providers.
+    const tryWithFallback = (
+      provider: ethers.providers.StaticJsonRpcProvider
+    ): Promise<[ethers.providers.StaticJsonRpcProvider, unknown]> => {
+      return this._trySend(provider, method, params)
+        .then((result): [ethers.providers.StaticJsonRpcProvider, unknown] => [provider, result])
+        .catch((err) => {
+          // Append the provider and error to the error array.
+          errors.push([provider, err?.stack || err?.toString()]);
+
+          // If there are no new fallback providers to use, terminate the recursion by throwing an error.
+          // Otherwise, we can try to call another provider.
+          if (fallbackProviders.length === 0) {
+            throw err;
+          }
+
+          // This line does two things:
+          // 1. Removes a fallback provider from the array so it cannot be used as a fallback for another required
+          // provider.
+          // 2. Recursively calls this method with that provider so it goes through the same try logic as the previous one.
+          return tryWithFallback(fallbackProviders.shift()!);
+        });
+    };
+
+    const results = await Promise.allSettled(requiredProviders.map(tryWithFallback));
+
+    if (!results.every(isPromiseFulfilled)) {
+      // Format the error so that it's very clear which providers failed and succeeded.
+      const errorTexts = errors.map(([provider, errorText]) => formatProviderError(provider, errorText));
+      const successfulProviderUrls = results.filter(isPromiseFulfilled).map((result) => result.value[0].connection.url);
+      throw createSendErrorWithMessage(
+        `Not enough providers succeeded. Errors:\n${errorTexts.join("\n")}\n` +
+          `Successful Providers:\n${successfulProviderUrls.join("\n")}`,
+        results.find(isPromiseRejected)?.reason
+      );
+    }
+
+    const values = results.map((result) => result.value);
+
+    // Start at element 1 and begin comparing.
+    // If _all_ values are equal, we have hit quorum, so return.
+    if (values.slice(1).every(([, output]) => compareRpcResults(method, values[0][1], output))) {
+      return values[0][1];
+    }
+
+    const throwQuorumError = () => {
+      const errorTexts = errors.map(([provider, errorText]) => formatProviderError(provider, errorText));
+      const successfulProviderUrls = values.map(([provider]) => provider.connection.url);
+      throw new Error(
+        "Not enough providers agreed to meet quorum.\n" +
+          "Providers that errored:\n" +
+          `${errorTexts.join("\n")}\n` +
+          "Providers that succeeded, but some failed to match:\n" +
+          successfulProviderUrls.join("\n")
+      );
+    };
+
+    // Exit early if there are no fallback providers left.
+    if (fallbackProviders.length === 0) {
+      throwQuorumError();
+    }
+
+    // Try each fallback provider in parallel.
+    const fallbackResults = await Promise.allSettled(
+      fallbackProviders.map((provider) =>
+        this._trySend(provider, method, params)
+          .then((result): [ethers.providers.StaticJsonRpcProvider, unknown] => [provider, result])
+          .catch((err) => {
+            errors.push([provider, err?.stack || err?.toString()]);
+            throw new Error("No fallbacks during quorum search");
+          })
+      )
+    );
+
+    // This filters only the fallbacks that succeeded.
+    const fallbackValues = fallbackResults.filter(isPromiseFulfilled).map((promise) => promise.value);
+
+    // Group the results by the count of that result.
+    const counts = [...values, ...fallbackValues].reduce(
+      (acc, curr) => {
+        const [, result] = curr;
+
+        // Find the first result that matches the return value.
+        const existingMatch = acc.find(([existingResult]) => compareRpcResults(method, existingResult, result));
+
+        // Increment the count if a match is found, else add a new element to the match array with a count of 1.
+        if (existingMatch) {
+          existingMatch[1]++;
+        } else {
+          acc.push([result, 1]);
+        }
+
+        // Return the same acc object because it was modified in place.
+        return acc;
+      },
+      [[undefined, 0]] as [unknown, number][] // Initialize with [undefined, 0] as the first element so something is always returned.
+    );
+
+    // Sort so the result with the highest count is first.
+    counts.sort(([, a], [, b]) => b - a);
+
+    // Extract the result by grabbing the first element.
+    const [quorumResult, count] = counts[0];
+
+    // If this count is less than we need for quorum, throw the quorum error.
+    if (count < quorumThreshold) {
+      throwQuorumError();
+    }
+
+    // If we've achieved quorum, then we should still log the providers that mismatched with the quorum result.
+    const mismatchedProviders = Object.fromEntries(
+      [...values, ...fallbackValues]
+        .filter(([, result]) => !compareRpcResults(method, result, quorumResult))
+        .map(([provider, result]) => [provider.connection.url, result])
+    );
+    const quorumProviders = [...values, ...fallbackValues]
+      .filter(([, result]) => compareRpcResults(method, result, quorumResult))
+      .map(([provider]) => provider.connection.url);
+    if (Object.keys(mismatchedProviders).length > 0 || errors.length > 0) {
+      logger.warn({
+        at: "ProviderUtils",
+        message: "Some providers mismatched with the quorum result or failed 🚸",
+        notificationPath: "across-warn",
+        method,
+        params,
+        quorumProviders,
+        mismatchedProviders,
+        erroringProviders: errors.map(([provider, errorText]) => formatProviderError(provider, errorText)),
+      });
+    }
+
+    return quorumResult;
+  }
+
+  _validateResponse(method: string, _: Array<unknown>, response: unknown): boolean {
+    // Basic validation logic to start.
+    // Note: eth_getTransactionReceipt is ignored here because null responses are expected in the case that ethers is
+    // polling for the transaction receipt and receiving null until it does.
+    return isDefined(response) || method === "eth_getTransactionReceipt";
+  }
+
+  async _sendAndValidate(
+    provider: ethers.providers.StaticJsonRpcProvider,
+    method: string,
+    params: Array<unknown>
+  ): Promise<unknown> {
+    const response = await provider.send(method, params);
+    if (!this._validateResponse(method, params, response)) {
+      // Not a warning to avoid spam since this could trigger a lot.
+      logger.debug({
+        at: "ProviderUtils",
+        message: "Provider returned invalid response",
+        provider: getOriginFromURL(provider.connection.url),
+        method,
+        params,
+        response,
+      });
+      throw new Error("Response failed validation");
+    }
+    return response;
+  }
+
+  _trySend(provider: ethers.providers.StaticJsonRpcProvider, method: string, params: Array<unknown>): Promise<unknown> {
+    let promise = this._sendAndValidate(provider, method, params);
+    for (let i = 0; i < this.retries; i++) {
+      promise = promise.catch(() => delay(this.delay).then(() => this._sendAndValidate(provider, method, params)));
+    }
+    return promise;
+  }
+
+  _getQuorum(method: string, params: Array<unknown>): number {
+    // Only use quorum if this is a historical query that doesn't depend on the current block number.
+
+    // All logs queries should use quorum.
+    if (method === "eth_getLogs") {
+      return this.nodeQuorumThreshold;
+    }
+
+    // getBlockByNumber should only use the quorum if it's not asking for the latest block.
+    if (method === "eth_getBlockByNumber" && params[0] !== "latest") {
+      return this.nodeQuorumThreshold;
+    }
+
+    // eth_call should only use quorum for queries at a specific past block.
+    if (method === "eth_call" && params[1] !== "latest") {
+      return this.nodeQuorumThreshold;
+    }
+
+    // All other calls should use quorum 1 to avoid errors due to sync differences.
+    return 1;
+  }
+}

package/src/providers/utils.ts

@@ -0,0 +1,119 @@
+// The async/queue library has a task-based interface for building a concurrent queue.
+
+import { providers } from "ethers";
+import { isDefined } from "../utils";
+import { isEqual } from "lodash";
+
+/**
+ * Deletes keys from an object and returns new copy of object without ignored keys
+ * @param ignoredKeys
+ * @param obj
+ * @returns Objects with ignored keys removed
+ */
+function deleteIgnoredKeys(ignoredKeys: string[], obj: Record<string, unknown>) {
+  if (!isDefined(obj)) {
+    return;
+  }
+  const newObj = { ...obj };
+  for (const key of ignoredKeys) {
+    delete newObj[key];
+  }
+  return newObj;
+}
+
+export function compareResultsAndFilterIgnoredKeys(
+  ignoredKeys: string[],
+  _objA: Record<string, unknown>,
+  _objB: Record<string, unknown>
+): boolean {
+  // Remove ignored keys from copied objects.
+  const filteredA = deleteIgnoredKeys(ignoredKeys, _objA);
+  const filteredB = deleteIgnoredKeys(ignoredKeys, _objB);
+
+  // Compare objects without the ignored keys.
+  return isEqual(filteredA, filteredB);
+}
+
+export function compareArrayResultsWithIgnoredKeys(ignoredKeys: string[], objA: unknown[], objB: unknown[]): boolean {
+  // Remove ignored keys from each element of copied arrays.
+  const filteredA = objA?.map((obj) => deleteIgnoredKeys(ignoredKeys, obj as Record<string, unknown>));
+  const filteredB = objB?.map((obj) => deleteIgnoredKeys(ignoredKeys, obj as Record<string, unknown>));
+
+  // Compare objects without the ignored keys.
+  return isDefined(filteredA) && isDefined(filteredB) && isEqual(filteredA, filteredB);
+}
+
+/**
+ * This is the type we pass to define a request "task".
+ */
+export interface RateLimitTask {
+  // These are the arguments to be passed to super.send().
+  sendArgs: [string, Array<unknown>];
+
+  // These are the promise callbacks that will cause the initial send call made by the user to either return a result
+  // or fail.
+  resolve: (result: unknown) => void;
+  reject: (err: unknown) => void;
+}
+
+/**
+ * A helper function to format an error message for a provider.
+ * @param provider The provider that failed.
+ * @param rawErrorText The raw error text.
+ * @returns The formatted error message.
+ */
+export function formatProviderError(provider: providers.StaticJsonRpcProvider, rawErrorText: string) {
+  return `Provider ${provider.connection.url} failed with error: ${rawErrorText}`;
+}
+
+export function createSendErrorWithMessage(message: string, sendError: Record<string, unknown>) {
+  const error = new Error(message);
+  return { ...sendError, ...error };
+}
+
+/**
+ * Compares two RPC results, filtering out fields that are known to differ between providers.
+ * Note: this function references `IGNORED_ERROR_CODES` which is a record of error codes that correspond to fields
+ *       that should be ignored when comparing RPC results.
+ * @param method The method that was called - conditionally filters out fields based on the method.
+ * @param rpcResultA The first RPC result.
+ * @param rpcResultB The second RPC result.
+ * @returns True if the results are equal, false otherwise.
+ */
+export function compareRpcResults(method: string, rpcResultA: unknown, rpcResultB: unknown): boolean {
+  if (method === "eth_getBlockByNumber") {
+    // We've seen RPC's disagree on the miner field, for example when Polygon nodes updated software that
+    // led alchemy and quicknode to disagree on the miner field's value.
+    return compareResultsAndFilterIgnoredKeys(
+      [
+        "miner", // polygon (sometimes)
+        "l1BatchNumber", // zkSync
+        "l1BatchTimestamp", // zkSync
+        "size", // Alchemy/Arbitrum (temporary)
+        "totalDifficulty", // Quicknode/Alchemy (sometimes)
+      ],
+      rpcResultA as Record<string, unknown>,
+      rpcResultB as Record<string, unknown>
+    );
+  } else if (method === "eth_getLogs") {
+    // We've seen some RPC's like QuickNode add in transactionLogIndex which isn't in the
+    // JSON RPC spec: https://ethereum.org/en/developers/docs/apis/json-rpc/#eth_getfilterchanges
+    // Additional reference: https://github.com/ethers-io/ethers.js/issues/1721
+    // 2023-08-31 Added blockHash because of upstream zkSync provider disagreements. Consider removing later.
+    // 2024-05-07 Added l1BatchNumber and logType due to Alchemy. Consider removing later.
+    // 2024-07-11 Added blockTimestamp after zkSync rolled out a new node release.
+    return compareArrayResultsWithIgnoredKeys(
+      ["blockTimestamp", "transactionLogIndex", "l1BatchNumber", "logType"],
+      rpcResultA as unknown[],
+      rpcResultB as unknown[]
+    );
+  } else {
+    return isEqual(rpcResultA, rpcResultB);
+  }
+}
+
+export enum CacheType {
+  NONE, // Do not cache
+  WITH_TTL, // Cache with TTL
+  NO_TTL, // Cache with infinite TTL
+}

package/src/relayFeeCalculator/chain-queries/factory.ts

@@ -1,9 +1,10 @@
+import assert from "assert";
 import { CHAIN_IDs, TOKEN_SYMBOLS_MAP } from "@across-protocol/constants";
 import { getDeployedAddress } from "@across-protocol/contracts";
 import { asL2Provider } from "@eth-optimism/sdk";
 import { providers } from "ethers";
 import { DEFAULT_SIMULATED_RELAYER_ADDRESS } from "../../constants";
-import { chainIsMatic, chainIsOPStack } from "../../utils";
+import { chainIsMatic, chainIsOPStack, isDefined } from "../../utils";
 import { QueryBase } from "./baseQuery";
 import { PolygonQueries } from "./polygon";
 import { DEFAULT_LOGGER, Logger } from "../relayFeeCalculator";
@@ -28,6 +29,8 @@ export class QueryBase__factory {
     gasMarkup = 0,
     coingeckoBaseCurrency = "eth"
   ): QueryBase {
+    assert(isDefined(spokePoolAddress));
+
     // Currently the only chain that has a custom query class is Polygon
     if (chainIsMatic(chainId)) {
       return new PolygonQueries(

package/src/relayFeeCalculator/chain-queries/polygon.ts

@@ -1,8 +1,10 @@
+import assert from "assert";
 import { getDeployedAddress } from "../../utils/DeploymentUtils";
 import { DEFAULT_LOGGER, Logger } from "../relayFeeCalculator";
 import { providers } from "ethers";
 import { CHAIN_IDs, DEFAULT_SIMULATED_RELAYER_ADDRESS, TOKEN_SYMBOLS_MAP } from "../../constants";
 import { Coingecko } from "../../coingecko/Coingecko";
+import { isDefined } from "../../utils";
 import { QueryBase } from "./baseQuery";
 
 export class PolygonQueries extends QueryBase {
@@ -15,6 +17,7 @@ export class PolygonQueries extends QueryBase {
     logger: Logger = DEFAULT_LOGGER,
     gasMarkup = 0
   ) {
+    assert(isDefined(spokePoolAddress));
     super(
       provider,
       symbolMapping,

package/src/utils/NetworkUtils.ts

@@ -115,3 +115,16 @@ export function chainIsCCTPEnabled(chainId: number): boolean {
 export function chainRequiresL1ToL2Finalization(chainId: number): boolean {
   return chainIsCCTPEnabled(chainId) || chainIsLinea(chainId);
 }
+
+/**
+ * Returns the origin of a URL.
+ * @param url A URL.
+ * @returns The origin of the URL, or "UNKNOWN" if the URL is invalid.
+ */
+export function getOriginFromURL(url: string): string {
+  try {
+    return new URL(url).origin;
+  } catch (e) {
+    return "UNKNOWN";
+  }
+}

package/src/utils/ObjectUtils.ts

@@ -2,6 +2,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 // Append value along the keyPath to object. For example assign(deposits, ['1337', '31337'], [{depositId:1}]) will create
 // deposits = {1337:{31337:[{depositId:1}]}}. Note that if the path into the object exists then this will append. This
+
 // function respects the destination type; if it is an object then deep merge and if an array effectively will push.
 export function assign(obj: any, keyPath: any[], value: any): void {
   const lastKeyIndex = keyPath.length - 1;