@paraswap/dex-lib 2.14.7 → 2.15.1

package/src/dex/hashflow/hashflow.ts

@@ -1,4 +1,3 @@
-import _ from 'lodash';
 import {
   Token,
   Address,
@@ -19,7 +18,7 @@ import {
   CACHE_PREFIX,
 } from '../../constants';
 import * as CALLDATA_GAS_COST from '../../calldata-gas-cost';
-import { getDexKeysWithNetwork, Utils } from '../../utils';
+import { getDexKeysWithNetwork } from '../../utils';
 import { IDex } from '../../dex/idex';
 import { IDexHelper } from '../../dex-helper/idex-helper';
 import {
@@ -32,12 +31,14 @@ import {
 import { SimpleExchange } from '../simple-exchange';
 import { Adapters, HashflowConfig } from './config';
 import { HashflowApi } from '@hashflow/taker-js';
+import { RateFetcher } from './rate-fetcher';
 import routerAbi from '../../abi/hashflow/HashflowRouter.abi.json';
 import BigNumber from 'bignumber.js';
 import { BN_0, BN_1, getBigNumberPow } from '../../bignumber-constants';
 import { Interface } from 'ethers/lib/utils';
 import { ChainId, ZERO_ADDRESS } from '@hashflow/sdk';
 import {
+  MarketMakersResponse,
   PriceLevelsResponse,
   RfqResponse,
 } from '@hashflow/taker-js/dist/types/rest';
@@ -45,22 +46,30 @@ import { assert } from 'ts-essentials';
 import {
   HASHFLOW_BLACKLIST_TTL_S,
   HASHFLOW_MM_RESTRICT_TTL_S,
-  PRICE_LEVELS_TTL_SECONDS,
+  HASHFLOW_API_CLIENT_NAME,
+  HASHFLOW_API_URL,
+  HASHFLOW_API_PRICES_POLLING_INTERVAL_MS,
+  HASHFLOW_API_MARKET_MAKERS_POLLING_INTERVAL_MS,
+  HASHFLOW_PRICES_CACHES_TTL_S,
+  HASHFLOW_MARKET_MAKERS_CACHES_TTL_S,
 } from './constants';
 import { BI_MAX_UINT256 } from '../../bigint-constants';
 
-const HASHFLOW_ASYNC_CALL_TIMEOUT = 150;
-
 export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
+  readonly isStatePollingDex = true;
   readonly hasConstantPriceLargeAmounts = false;
   readonly needWrapNative = false;
   readonly isFeeOnTransferSupported = false;
   private api: HashflowApi;
+  private rateFetcher: RateFetcher;
 
   private hashFlowAuthToken: string;
   private disabledMMs: Set<string>;
   private runtimeMMsRestrictHashMapKey: string;
 
+  private pricesCacheKey: string;
+  private marketMakersCacheKey: string;
+
   public static dexKeysWithNetwork: { key: string; networks: Network[] }[] =
     getDexKeysWithNetwork(HashflowConfig);
 
@@ -82,11 +91,65 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
       token !== undefined,
       'Hashflow auth token is not specified with env variable',
     );
+
     this.hashFlowAuthToken = token;
-    this.api = new HashflowApi('taker', 'paraswap', this.hashFlowAuthToken);
+    this.api = new HashflowApi(
+      'taker',
+      HASHFLOW_API_CLIENT_NAME,
+      this.hashFlowAuthToken,
+    );
+
+    this.pricesCacheKey = `${CACHE_PREFIX}_${this.dexHelper.config.data.network}_${this.dexKey}_prices`;
+    this.marketMakersCacheKey = `${CACHE_PREFIX}_${this.dexHelper.config.data.network}_${this.dexKey}_mms`;
+
     this.disabledMMs = new Set(dexHelper.config.data.hashFlowDisabledMMs);
     this.runtimeMMsRestrictHashMapKey =
       `${CACHE_PREFIX}_${this.dexKey}_${this.network}_restricted_mms`.toLowerCase();
+
+    this.rateFetcher = new RateFetcher(
+      this.dexHelper,
+      this.dexKey,
+      this.network,
+      this.logger,
+      {
+        rateConfig: {
+          pricesIntervalMs: HASHFLOW_API_PRICES_POLLING_INTERVAL_MS,
+          markerMakersIntervalMs:
+          HASHFLOW_API_MARKET_MAKERS_POLLING_INTERVAL_MS,
+          marketMakersReqParams: {
+            url: `${HASHFLOW_API_URL}/taker/v1/marketMakers`,
+            params: {
+              networkId: this.network,
+              source: HASHFLOW_API_CLIENT_NAME,
+            },
+            headers: { Authorization: this.hashFlowAuthToken },
+          },
+          pricesReqParams: {
+            url: `${HASHFLOW_API_URL}/taker/v2/price-levels`,
+            params: {
+              networkId: this.network,
+              source: HASHFLOW_API_CLIENT_NAME,
+              marketMakers: [],
+            },
+            headers: { Authorization: this.hashFlowAuthToken },
+          },
+          getCachedMarketMakers: this.getCachedMarketMakers.bind(this),
+          filterMarketMakers: this.getFilteredMarketMakers.bind(this),
+          pricesCacheKey: this.pricesCacheKey,
+          pricesCacheTTLSecs: HASHFLOW_PRICES_CACHES_TTL_S,
+          marketMakersCacheKey: this.marketMakersCacheKey,
+          marketMakersCacheTTLSecs: HASHFLOW_MARKET_MAKERS_CACHES_TTL_S,
+        },
+      },
+    );
+  }
+
+  async initializePricing(blockNumber: number): Promise<void> {
+    if (!this.dexHelper.config.isSlave) {
+      this.rateFetcher.start();
+    }
+
+    return;
   }
 
   getAdapters(side: SwapSide): { name: string; index: number }[] | null {
@@ -123,7 +186,7 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
       return [];
     }
 
-    const levels = await this.getLevelsWithCache();
+    const levels = (await this.getCachedLevels()) || {};
     const makers = Object.keys(levels);
 
     return makers
@@ -144,11 +207,10 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
       );
   }
 
-  private async getFilteredMarketMakers(chainId: ChainId): Promise<string[]> {
-    const [makers, cachedRestrictionUnparsed] = await Promise.all([
-      this.api.getMarketMakers(chainId),
-      this.dexHelper.cache.hgetAll(this.runtimeMMsRestrictHashMapKey),
-    ]);
+  private async getFilteredMarketMakers(makers: string[]): Promise<string[]> {
+    const cachedRestrictionUnparsed = await this.dexHelper.cache.hgetAll(
+      this.runtimeMMsRestrictHashMapKey,
+    );
 
     const runtimeRestrictedMMs = this.parseCacheRestrictionAndExpiryIfNeeded(
       cachedRestrictionUnparsed,
@@ -312,36 +374,30 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
     return undefined;
   }
 
-  async getLevelsWithCache(): Promise<PriceLevelsResponse['levels']> {
-    const cachedLevels = await this.dexHelper.cache.get(
-      this.dexKey,
-      this.network as ChainId,
-      `levels`,
+  async getCachedMarketMakers(): Promise<
+    MarketMakersResponse['marketMakers'] | null
+  > {
+    const cachedMarketMakers = await this.dexHelper.cache.rawget(
+      this.marketMakersCacheKey,
     );
-    if (cachedLevels) {
-      return JSON.parse(cachedLevels) as PriceLevelsResponse['levels'];
+
+    if (cachedMarketMakers) {
+      return JSON.parse(
+        cachedMarketMakers,
+      ) as MarketMakersResponse['marketMakers'];
     }
 
-    const makers = await Utils.timeoutPromise(
-      this.getFilteredMarketMakers(this.network as ChainId),
-      HASHFLOW_ASYNC_CALL_TIMEOUT,
-      'Hashflow: getFilteredMarketMakers timeout',
-    );
-    const levels = await Utils.timeoutPromise(
-      this.api.getPriceLevels(this.network as ChainId, makers),
-      HASHFLOW_ASYNC_CALL_TIMEOUT,
-      'Hashflow: getPriceLevels timeout',
-    );
+    return null;
+  }
 
-    await this.dexHelper.cache.setex(
-      this.dexKey,
-      this.network as ChainId,
-      `levels`,
-      PRICE_LEVELS_TTL_SECONDS,
-      JSON.stringify(levels),
-    );
+  async getCachedLevels(): Promise<PriceLevelsResponse['levels'] | null> {
+    const cachedLevels = await this.dexHelper.cache.rawget(this.pricesCacheKey);
 
-    return levels;
+    if (cachedLevels) {
+      return JSON.parse(cachedLevels) as PriceLevelsResponse['levels'];
+    }
+
+    return null;
   }
 
   // Hashflow protocol for native token expects 0x00000... instead of 0xeeeee...
@@ -381,7 +437,7 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
 
     const marketMakersToUse = pools.map(p => p.split(`${prefix}_`).pop());
 
-    const levelsMap = await this.getLevelsWithCache();
+    const levelsMap = (await this.getCachedLevels()) || {};
 
     Object.keys(levelsMap).forEach(mmKey => {
       if (!marketMakersToUse.includes(mmKey)) {
@@ -486,23 +542,19 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
 
     let rfq: RfqResponse;
     try {
-      rfq = await Utils.timeoutPromise(
-        this.api.requestQuote({
-          chainId,
-          baseToken: normalizedSrcToken.address,
-          quoteToken: normalizedDestToken.address,
-          ...(side === SwapSide.SELL
-            ? {
-                baseTokenAmount: optimalSwapExchange.srcAmount,
-              }
-            : { quoteTokenAmount: optimalSwapExchange.destAmount }),
-          wallet: this.augustusAddress.toLowerCase(),
-          effectiveTrader: options.txOrigin.toLowerCase(),
-          marketMakers: [mm],
-        }),
-        HASHFLOW_ASYNC_CALL_TIMEOUT,
-        'Hashflow: requestQuote timeout',
-      );
+      rfq = await this.api.requestQuote({
+        chainId,
+        baseToken: normalizedSrcToken.address,
+        quoteToken: normalizedDestToken.address,
+        ...(side === SwapSide.SELL
+          ? {
+              baseTokenAmount: optimalSwapExchange.srcAmount,
+            }
+          : { quoteTokenAmount: optimalSwapExchange.destAmount }),
+        wallet: this.augustusAddress.toLowerCase(),
+        effectiveTrader: options.txOrigin.toLowerCase(),
+        marketMakers: [mm],
+      });
 
       if (rfq.status !== 'success') {
         const message = `${this.dexKey}-${
@@ -816,16 +868,15 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
   ): Promise<PoolLiquidity[]> {
     const _tokenAddress = tokenAddress.toLowerCase();
 
-    const chainId = this.network as ChainId;
-
-    const makers = await this.getFilteredMarketMakers(chainId);
-    const pLevels = await this.api.getPriceLevels(chainId, makers);
+    const makers = (await this.getCachedMarketMakers()) || [];
+    const filteredMakers = await this.getFilteredMarketMakers(makers);
+    const pLevels = (await this.getCachedLevels()) || {};
 
     let baseToken: Token | undefined = undefined;
     // TODO: Improve efficiency of this part. Quite inefficient way to determine
     // Token address and decimals. But since it is not called frequently, not worth
     // optimizing now
-    for (const maker of makers) {
+    for (const maker of filteredMakers) {
       const baseTokenEntry = pLevels[maker]?.find(
         entry => entry.pair.baseToken.toLowerCase() === _tokenAddress,
       );
@@ -873,4 +924,10 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
       .sort((a, b) => b.liquidityUSD - a.liquidityUSD)
       .slice(0, limit);
   }
+
+  releaseResources(): void {
+    if (this.rateFetcher) {
+      this.rateFetcher.stop();
+    }
+  }
 }

package/src/dex/hashflow/rate-fetcher.ts

@@ -0,0 +1,120 @@
+import { IDexHelper } from '../../dex-helper';
+import { Fetcher, SkippingRequest } from '../../lib/fetcher/fetcher';
+import { validateAndCast } from '../../lib/validators';
+import { Logger } from '../../types';
+import {
+  HashflowRateFetcherConfig,
+  HashflowRatesResponse,
+  HashflowMarketMakersResponse,
+} from './types';
+import { pricesResponseValidator, marketMakersValidator } from './validators';
+import { Network } from '../../constants';
+
+export class RateFetcher {
+  private rateFetcher: Fetcher<HashflowRatesResponse>;
+  private pricesCacheKey: string;
+  private pricesCacheTTL: number;
+
+  private marketMakersFetcher: Fetcher<HashflowMarketMakersResponse>;
+  private marketMakersCacheKey: string;
+  private marketMakersCacheTTL: number;
+
+  constructor(
+    private dexHelper: IDexHelper,
+    private dexKey: string,
+    private network: Network,
+    private logger: Logger,
+    config: HashflowRateFetcherConfig,
+  ) {
+    this.pricesCacheKey = config.rateConfig.pricesCacheKey;
+    this.pricesCacheTTL = config.rateConfig.pricesCacheTTLSecs;
+    this.marketMakersCacheKey = config.rateConfig.marketMakersCacheKey;
+    this.marketMakersCacheTTL = config.rateConfig.marketMakersCacheTTLSecs;
+
+    this.marketMakersFetcher = new Fetcher<HashflowMarketMakersResponse>(
+      dexHelper.httpRequest,
+      {
+        info: {
+          requestOptions: config.rateConfig.marketMakersReqParams,
+          caster: (data: unknown) => {
+            return validateAndCast<HashflowMarketMakersResponse>(
+              data,
+              marketMakersValidator,
+            );
+          },
+        },
+        handler: this.handleMarketMakersResponse.bind(this),
+      },
+      config.rateConfig.markerMakersIntervalMs,
+      logger,
+    );
+
+    this.rateFetcher = new Fetcher<HashflowRatesResponse>(
+      dexHelper.httpRequest,
+      {
+        info: {
+          requestOptions: config.rateConfig.pricesReqParams,
+          requestFunc: async options => {
+            const { filterMarketMakers, getCachedMarketMakers } =
+              config.rateConfig;
+
+            const cachedMarketMakers = (await getCachedMarketMakers()) || [];
+            const filteredMarketMakers = await filterMarketMakers(
+              cachedMarketMakers,
+            );
+
+            if (filteredMarketMakers.length === 0) {
+              return new SkippingRequest(
+                `${dexKey}-${network}: got ${filteredMarketMakers.length} market makers.`,
+              );
+            }
+
+            const prices = await dexHelper.httpRequest.request({
+              ...options,
+              params: { ...options.params, marketMakers: filteredMarketMakers },
+            });
+
+            return prices;
+          },
+          caster: (data: unknown) => {
+            return validateAndCast<HashflowRatesResponse>(
+              data,
+              pricesResponseValidator,
+            );
+          },
+        },
+        handler: this.handleRatesResponse.bind(this),
+      },
+      config.rateConfig.pricesIntervalMs,
+      logger,
+    );
+  }
+
+  start() {
+    this.marketMakersFetcher.startPolling();
+    this.rateFetcher.startPolling();
+  }
+
+  stop() {
+    this.marketMakersFetcher.stopPolling();
+    this.rateFetcher.stopPolling();
+  }
+
+  private handleMarketMakersResponse(resp: HashflowMarketMakersResponse): void {
+    const { marketMakers } = resp;
+    this.dexHelper.cache.rawset(
+      this.marketMakersCacheKey,
+      JSON.stringify(marketMakers),
+      this.marketMakersCacheTTL,
+    );
+  }
+
+  private handleRatesResponse(resp: HashflowRatesResponse): void {
+    const { levels } = resp;
+    this.dexHelper.cache.rawset(
+      this.pricesCacheKey,
+      JSON.stringify(levels),
+      this.pricesCacheTTL,
+    );
+  }
+}

package/src/dex/hashflow/types.ts

@@ -1,4 +1,5 @@
 import { QuoteData } from '@hashflow/taker-js/dist/types/common';
+import { RequestHeaders } from '../../dex-helper';
 
 export type HashflowData = {
   mm: string;
@@ -24,3 +25,42 @@ export enum RFQType {
 }
 
 export class SlippageCheckError extends Error {}
+
+export type HashflowRatesLevel = {
+  pair: Record<string, string>;
+  levels: Array<Record<string, string>>;
+  includesFees: boolean;
+};
+
+export type HashflowMarketMakersResponse = {
+  marketMakers: string[];
+};
+
+export type HashflowRatesResponse = {
+  status: string;
+  networkId: string;
+  levels: Record<string, Array<HashflowRatesLevel>>;
+};
+
+export type HashflowRateFetcherConfig = {
+  rateConfig: {
+    marketMakersReqParams: {
+      url: string;
+      headers?: RequestHeaders;
+      params?: any;
+    };
+    pricesReqParams: {
+      url: string;
+      headers?: RequestHeaders;
+      params?: any;
+    };
+    pricesIntervalMs: number;
+    markerMakersIntervalMs: number;
+    getCachedMarketMakers: () => Promise<string[] | null>;
+    filterMarketMakers: (makers: string[]) => Promise<string[]>;
+    pricesCacheKey: string;
+    marketMakersCacheKey: string;
+    pricesCacheTTLSecs: number;
+    marketMakersCacheTTLSecs: number;
+  };
+};

package/src/dex/hashflow/validators.ts

@@ -0,0 +1,27 @@
+import joi from 'joi';
+
+export const levelValidator = joi.array().items(joi.object({
+  pair: joi.object({
+    baseTokenName: joi.string().min(1).required(),
+    quoteTokenName: joi.string().min(1).required(),
+    baseToken: joi.string().min(1).required(),
+    quoteToken: joi.string().min(1).required(),
+    baseTokenDecimals: joi.number().min(1).required(),
+    quoteTokenDecimals: joi.number().min(1).required(),
+  }),
+  levels: joi.array().items(joi.object({
+    level: joi.string().min(1).required(),
+    price: joi.string().min(1).required(),
+  })),
+  includesFees: joi.boolean().required(),
+}));
+
+export const pricesResponseValidator = joi.object({
+  status: joi.string().required(),
+  networkId: joi.number().required(),
+  levels: joi.object().pattern(joi.string().min(1), levelValidator),
+});
+
+export const marketMakersValidator = joi.object({
+  marketMakers: joi.array().items(joi.string().min(1).required()),
+});

package/src/lib/fetcher/fetcher.ts

@@ -1,12 +1,20 @@
 import { IRequestWrapper } from '../../dex-helper';
 import { Logger } from 'log4js';
 import { RequestConfig, Response } from '../../dex-helper/irequest-wrapper';
+import { isFunction } from 'lodash';
 
 const FETCH_TIMEOUT_MS = 10 * 1000;
 const FETCH_FAIL_MAX_ATTEMPT = 5;
 const FETCH_FAIL_RETRY_TIMEOUT_MS = 60 * 1000;
 
+export class SkippingRequest {
+  constructor(
+    public message = '',
+  ) {}
+};
+
 export type RequestInfo<T> = {
+  requestFunc?: (options: RequestConfig) => Promise<Response<T> | SkippingRequest>;
   requestOptions: RequestConfig;
   caster: (data: unknown) => T;
   authenticate?: (options: RequestConfig) => RequestConfig;
@@ -18,7 +26,7 @@ export type RequestInfoWithHandler<T> = {
   handler: (data: T) => void;
 };
 
-export default class Fetcher<T> {
+export class Fetcher<T> {
   private requests: RequestInfoWithHandler<T>[];
   public lastFetchSucceeded: boolean = false;
   private failedCount = 0;
@@ -52,19 +60,29 @@ export default class Fetcher<T> {
       return;
     }
 
-    const promises = this.requests.map<Promise<Error | Response<T>>>(
+    const promises = this.requests.map<Promise<Error | Response<T> | SkippingRequest>>(
       async (reqInfo: RequestInfoWithHandler<T>) => {
         const info = reqInfo.info;
         let options = info.requestOptions;
+
         if (info.authenticate) {
           info.authenticate(options);
         }
 
         try {
-          const result = await this.requestWrapper.request({
-            timeout: FETCH_TIMEOUT_MS,
-            ...options,
-          });
+          let result;
+          if (isFunction(info.requestFunc)) {
+            result = await info.requestFunc({
+              timeout: FETCH_TIMEOUT_MS,
+              ...options,
+            });
+          } else {
+            result = await this.requestWrapper.request({
+              timeout: FETCH_TIMEOUT_MS,
+              ...options,
+            });
+          }
+
           return result;
         } catch (e) {
           return e as Error;
@@ -76,17 +94,28 @@ export default class Fetcher<T> {
       .map((_, i) => i)
       .filter(i => results[i] instanceof Error);
 
-    results
-      .filter(result => !(result instanceof Error))
-      .map(result => {
-        debugger;
+    const skipped = results
+      .map((_, i) => i)
+      .filter(i => results[i] instanceof SkippingRequest);
+
+    results.map(result => {
+      if(!(result instanceof Error || result instanceof SkippingRequest)) {
         this.logger.info(
           'Results Data:',
           JSON.stringify((result as any).data)
             .replace(/(?:\r\n|\r|\n)/g, ' ')
             .substring(0, 1000),
         );
-      });
+      }
+    });
+
+    skipped.forEach(i => {
+      const skippedRes = results[i] as SkippingRequest;
+      this.logger.warn(
+        `skipped request ${this.requests[i].info.requestOptions.url} ${skippedRes.message}`,
+      );
+    });
+
     failures.forEach(i => {
       this.logger.warn(
         `failed polling ${this.requests[i].info.requestOptions.url} ${results[i]}`,
@@ -102,7 +131,7 @@ export default class Fetcher<T> {
 
     results
       .map((_, i) => i)
-      .filter(i => !(results[i] instanceof Error))
+      .filter(i => !(results[i] instanceof Error || results[i] instanceof SkippingRequest))
       .forEach(i => {
         const response = results[i] as Response<T>;
         const reqInfo = this.requests[i];

package/src/lib/validators.ts

@@ -0,0 +1,34 @@
+import joi from 'joi';
+
+export class ValidationError extends Error {
+  readonly rawMessage: string;
+
+  readonly key?: string;
+
+  constructor(message: string, key?: string) {
+    super(key !== undefined ? `'${key}': ${message}` : message);
+    this.rawMessage = message;
+    this.key = key;
+  }
+}
+
+export const validateAndCast = <T>(
+  value: unknown,
+  schema: joi.Schema,
+  name?: string,
+): T => {
+  const { error, value: parsed } = schema.validate(value);
+
+  if (error !== undefined) {
+    let message;
+    if (name) {
+      message = `"${name}" ${error.message.slice(
+        error.message.indexOf('must'),
+      )}`;
+    } else {
+      message = error.message;
+    }
+    throw new ValidationError(message);
+  }
+  return parsed as T;
+};