@paraswap/dex-lib 4.1.7-pub-sub.3 → 4.1.7-pub-sub.4

package/src/dex/dexalot/rate-fetcher.ts

@@ -2,9 +2,11 @@ import { Network } from '../../constants';
 import { IDexHelper } from '../../dex-helper';
 import { Fetcher } from '../../lib/fetcher/fetcher';
 import { validateAndCast } from '../../lib/validators';
-import { Logger, Token } from '../../types';
+import { Address, Logger, Token } from '../../types';
 import {
   DexalotBlacklistResponse,
+  TokenDataMap,
+  TokenAddrDataMap,
   DexalotPairsResponse,
   DexalotPricesResponse,
   DexalotRateFetcherConfig,
@@ -16,13 +18,21 @@ import {
   pairsResponseValidator,
   pricesResponseValidator,
 } from './validators';
+import {
+  DEXALOT_BLACKLIST_CACHES_TTL_S,
+  DEXALOT_RATE_LIMITED_TTL_S,
+  DEXALOT_RATELIMIT_CACHE_VALUE,
+  DEXALOT_RESTRICT_TTL_S,
+  DEXALOT_RESTRICTED_CACHE_KEY,
+} from './constants';
+import { ExpKeyValuePubSub, NonExpSetPubSub } from '../../lib/pub-sub';
 
 export class RateFetcher {
   private pairsFetcher: Fetcher<DexalotPairsResponse>;
   private pairsCacheKey: string;
-  private pairsCacheTTL: number;
 
   private rateFetcher: Fetcher<DexalotPricesResponse>;
+  private rateTokensPubSub: ExpKeyValuePubSub;
   private pricesCacheKey: string;
   private pricesCacheTTL: number;
 
@@ -31,9 +41,12 @@ export class RateFetcher {
   private tokensCacheTTL: number;
 
   private blacklistFetcher: Fetcher<DexalotBlacklistResponse>;
+  private blacklistPubSub: NonExpSetPubSub;
   private blacklistCacheKey: string;
   private blacklistCacheTTL: number;
 
+  private restrictedPoolPubSub: ExpKeyValuePubSub;
+
   constructor(
     private dexHelper: IDexHelper,
     private dexKey: string,
@@ -42,7 +55,6 @@ export class RateFetcher {
     config: DexalotRateFetcherConfig,
   ) {
     this.pairsCacheKey = config.rateConfig.pairsCacheKey;
-    this.pairsCacheTTL = config.rateConfig.pairsCacheTTLSecs;
     this.pricesCacheKey = config.rateConfig.pricesCacheKey;
     this.pricesCacheTTL = config.rateConfig.pricesCacheTTLSecs;
     this.tokensAddrCacheKey = config.rateConfig.tokensAddrCacheKey;
@@ -51,6 +63,12 @@ export class RateFetcher {
     this.blacklistCacheKey = config.rateConfig.blacklistCacheKey;
     this.blacklistCacheTTL = config.rateConfig.blacklistCacheTTLSecs;
 
+    this.rateTokensPubSub = new ExpKeyValuePubSub(
+      dexHelper,
+      dexKey,
+      'rateTokens',
+    );
+
     this.pairsFetcher = new Fetcher<DexalotPairsResponse>(
       dexHelper.httpRequest,
       {
@@ -87,6 +105,7 @@ export class RateFetcher {
       logger,
     );
 
+    this.blacklistPubSub = new NonExpSetPubSub(dexHelper, dexKey, 'blacklist');
     this.blacklistFetcher = new Fetcher<DexalotBlacklistResponse>(
       dexHelper.httpRequest,
       {
@@ -104,12 +123,30 @@ export class RateFetcher {
       config.rateConfig.blacklistIntervalMs,
       logger,
     );
+
+    this.restrictedPoolPubSub = new ExpKeyValuePubSub(
+      dexHelper,
+      dexKey,
+      'restricted-pool',
+      // using default value, we can lazy load non-restricted pools
+      // and restrict them by subscribing to this channel
+      'false',
+      DEXALOT_RESTRICT_TTL_S,
+    );
   }
 
-  start() {
-    this.pairsFetcher.startPolling();
-    this.rateFetcher.startPolling();
-    this.blacklistFetcher.startPolling();
+  async start() {
+    if (!this.dexHelper.config.isSlave) {
+      this.pairsFetcher.startPolling();
+      this.rateFetcher.startPolling();
+      this.blacklistFetcher.startPolling();
+    } else {
+      this.rateTokensPubSub.subscribe();
+      this.restrictedPoolPubSub.subscribe();
+
+      const initSet = await this.getAllBlacklisted();
+      this.blacklistPubSub.initializeAndSubscribe(initSet);
+    }
   }
 
   stop() {
@@ -145,7 +182,7 @@ export class RateFetcher {
       this.dexKey,
       this.network,
       this.pairsCacheKey,
-      this.pairsCacheTTL,
+      this.tokensCacheTTL,
       JSON.stringify(dexPairs),
     );
 
@@ -164,6 +201,15 @@ export class RateFetcher {
       this.tokensCacheTTL,
       JSON.stringify(tokenAddrMap),
     );
+
+    this.rateTokensPubSub.publish(
+      {
+        [this.pairsCacheKey]: dexPairs,
+        [this.tokensCacheKey]: tokenMap,
+        [this.tokensAddrCacheKey]: tokenAddrMap,
+      },
+      this.tokensCacheTTL,
+    );
   }
 
   private handleRatesResponse(resp: DexalotPricesResponse): void {
@@ -180,18 +226,182 @@ export class RateFetcher {
       this.pricesCacheTTL,
       JSON.stringify(dexPrices),
     );
+
+    this.rateTokensPubSub.publish(
+      { [this.pricesCacheKey]: dexPrices },
+      this.pricesCacheTTL,
+    );
   }
 
   private async handleBlacklistResponse(
     resp: DexalotBlacklistResponse,
   ): Promise<void> {
     const { blacklist } = resp;
+    const data = blacklist.map(item => item.toLowerCase());
     this.dexHelper.cache.setex(
       this.dexKey,
       this.network,
       this.blacklistCacheKey,
       this.blacklistCacheTTL,
-      JSON.stringify(blacklist.map(item => item.toLowerCase())),
+      JSON.stringify(data),
+    );
+
+    this.blacklistPubSub.publish(data);
+  }
+
+  async getCachedTokens(): Promise<TokenDataMap | null> {
+    const cachedTokens = await this.rateTokensPubSub.getAndCache(
+      this.tokensCacheKey,
+    );
+
+    if (cachedTokens) {
+      return cachedTokens as TokenDataMap;
+    }
+
+    return null;
+  }
+
+  async getCachedPairs(): Promise<PairDataMap | null> {
+    const cachedPairs = await this.rateTokensPubSub.getAndCache(
+      this.pairsCacheKey,
+    );
+
+    if (cachedPairs) {
+      return cachedPairs as PairDataMap;
+    }
+
+    return null;
+  }
+
+  async getCachedTokensAddr(): Promise<TokenAddrDataMap | null> {
+    const cachedTokensAddr = await this.rateTokensPubSub.getAndCache(
+      this.tokensAddrCacheKey,
+    );
+
+    if (cachedTokensAddr) {
+      return cachedTokensAddr as TokenAddrDataMap;
+    }
+
+    return null;
+  }
+
+  async setBlacklist(
+    txOrigin: Address,
+    ttl: number = DEXALOT_BLACKLIST_CACHES_TTL_S,
+  ): Promise<boolean> {
+    const blacklist = await this.getAllBlacklisted();
+
+    blacklist.push(txOrigin.toLowerCase());
+
+    this.dexHelper.cache.setex(
+      this.dexKey,
+      this.network,
+      this.blacklistCacheKey,
+      ttl,
+      JSON.stringify(blacklist),
+    );
+
+    this.blacklistPubSub.publish([txOrigin.toLowerCase()]);
+
+    return true;
+  }
+
+  async getAllBlacklisted(): Promise<string[]> {
+    const cachedBlacklist = await this.dexHelper.cache.get(
+      this.dexKey,
+      this.network,
+      this.blacklistCacheKey,
     );
+
+    if (cachedBlacklist) {
+      return JSON.parse(cachedBlacklist);
+    }
+
+    return [];
+  }
+
+  async isBlacklisted(txOrigin: Address): Promise<boolean> {
+    const blacklisted = await this.blacklistPubSub.has(txOrigin.toLowerCase());
+
+    return blacklisted;
+
+    /*
+     rate-limit check was only if blacklist data set was not available,
+     in the current implementation it should not be the case,
+     so skip next check
+    */
+    // To not show pricing for rate limited users
+    if (await this.isRateLimited(txOrigin)) {
+      return true;
+    }
+
+    return false;
+  }
+
+  async setRateLimited(txOrigin: Address, ttl = DEXALOT_RATE_LIMITED_TTL_S) {
+    await this.dexHelper.cache.setex(
+      this.dexKey,
+      this.network,
+      this.getRateLimitedKey(txOrigin),
+      ttl,
+      DEXALOT_RATELIMIT_CACHE_VALUE,
+    );
+    return true;
+  }
+
+  async isRateLimited(txOrigin: Address): Promise<boolean> {
+    const result = await this.dexHelper.cache.get(
+      this.dexKey,
+      this.network,
+      this.getRateLimitedKey(txOrigin),
+    );
+    return result === DEXALOT_RATELIMIT_CACHE_VALUE;
+  }
+
+  getRateLimitedKey(address: Address) {
+    return `rate_limited_${address}`.toLowerCase();
+  }
+
+  async getCachedPrices(): Promise<PriceDataMap | null> {
+    const cachedPrices = await this.rateTokensPubSub.getAndCache(
+      this.pricesCacheKey,
+    );
+
+    if (cachedPrices) {
+      return cachedPrices as PriceDataMap;
+    }
+
+    return null;
+  }
+
+  async restrictPool(
+    poolIdentifier: string,
+    ttl: number = DEXALOT_RESTRICT_TTL_S,
+  ): Promise<boolean> {
+    await this.dexHelper.cache.setex(
+      this.dexKey,
+      this.network,
+      this.getRestrictedPoolKey(poolIdentifier),
+      ttl,
+      'true',
+    );
+
+    await this.restrictedPoolPubSub.publish(
+      { [this.getRestrictedPoolKey(poolIdentifier)]: 'true' },
+      ttl,
+    );
+    return true;
+  }
+
+  async isRestrictedPool(poolIdentifier: string): Promise<boolean> {
+    const result = await this.restrictedPoolPubSub.getAndCache(
+      this.getRestrictedPoolKey(poolIdentifier),
+    );
+
+    return result === 'true';
+  }
+
+  getRestrictedPoolKey(poolIdentifier: string): string {
+    return `${DEXALOT_RESTRICTED_CACHE_KEY}-${poolIdentifier}`;
   }
 }

package/src/dex/dexalot/types.ts

@@ -117,7 +117,6 @@ export type DexalotRateFetcherConfig = {
     tokensCacheKey: string;
     blacklistCacheKey: string;
     blacklistCacheTTLSecs: number;
-    pairsCacheTTLSecs: number;
     pricesCacheTTLSecs: number;
     tokensCacheTTLSecs: number;
   };

package/src/dex/generic-rfq/rate-fetcher.ts

@@ -46,7 +46,6 @@ export const reversePrice = (price: PriceAndAmountBigNumber) =>
     price[1].times(price[0]),
   ] as PriceAndAmountBigNumber;
 
-const logPrefix = 'RateFetched[pub_sub]';
 export class RateFetcher {
   private tokensFetcher: Fetcher<TokensResponse>;
   private pairsFetcher: Fetcher<PairsResponse>;
@@ -168,7 +167,6 @@ export class RateFetcher {
   }
 
   async initialize() {
-    this.logger.info(`${logPrefix} Initializing rate fetcher`);
     const isContract = await isContractAddress(
       this.dexHelper.web3Provider,
       this.config.maker,
@@ -181,10 +179,8 @@ export class RateFetcher {
     }
 
     if (this.dexHelper.config.isSlave) {
-      this.logger.info(`${logPrefix} Subscribing to prices pubsub`);
       this.pricesPubSub.subscribe();
       if (this.blacklistPubSub) {
-        this.logger.info(`${logPrefix} Subscribing to blacklist pubsub`);
         const initSet = await this.dexHelper.cache.smembers(
           this.blackListCacheKey,
         );
@@ -194,7 +190,6 @@ export class RateFetcher {
   }
 
   start() {
-    this.logger.info(`${logPrefix} Starting rate fetcher`);
     this.tokensFetcher.startPolling();
     this.rateFetcher.startPolling();
     this.pairsFetcher.startPolling();
@@ -204,7 +199,6 @@ export class RateFetcher {
   }
 
   stop() {
-    this.logger.info(`${logPrefix} Stopping rate fetcher`);
     this.tokensFetcher.stopPolling();
     this.pairsFetcher.stopPolling();
     this.rateFetcher.stopPolling();
@@ -215,10 +209,6 @@ export class RateFetcher {
   }
 
   private handleTokensResponse(data: TokensResponse) {
-    this.logger.info(
-      `${logPrefix} Handling tokens response, tokens: `,
-      Object.keys(data.tokens).length,
-    );
     for (const tokenName of Object.keys(data.tokens)) {
       const token = data.tokens[tokenName];
       token.address = token.address.toLowerCase();
@@ -236,10 +226,6 @@ export class RateFetcher {
   }
 
   private handlePairsResponse(resp: PairsResponse) {
-    this.logger.info(
-      `${logPrefix} Handling pairs response, pairs: `,
-      Object.keys(resp.pairs).length,
-    );
     this.pairs = {};
 
     const pairs: PairMap = {};
@@ -251,19 +237,11 @@ export class RateFetcher {
   }
 
   private handleBlackListResponse(resp: BlackListResponse) {
-    this.logger.info(
-      `${logPrefix} Handling blacklist response, blacklist length: `,
-      resp.blacklist.length,
-    );
     for (const address of resp.blacklist) {
       this.dexHelper.cache.sadd(this.blackListCacheKey, address.toLowerCase());
     }
 
     if (this.blacklistPubSub) {
-      this.logger.info(
-        `${logPrefix} Publishing blacklist to pubsub, blacklist length: `,
-        resp.blacklist.length,
-      );
       this.blacklistPubSub.publish(resp.blacklist);
     }
   }
@@ -276,10 +254,6 @@ export class RateFetcher {
   }
 
   private handleRatesResponse(resp: RatesResponse) {
-    this.logger.info(
-      `${logPrefix} Handling rates response, prices length: `,
-      Object.keys(resp.prices).length,
-    );
     const pubSubData: Record<string, unknown> = {};
     const ttl = this.config.rateConfig.dataTTLS;
     const pairs = this.pairs;
@@ -395,10 +369,6 @@ export class RateFetcher {
 
   public async getAvailablePairs(): Promise<string[]> {
     const pairs = await this.pricesPubSub.getAndCache<string[]>(`pairs`);
-    this.logger.info(
-      `${logPrefix} Getting available pairs, pairs: `,
-      pairs?.length,
-    );
     if (!pairs) {
       return [];
     }

package/src/dex/hashflow/constants.ts

@@ -4,11 +4,11 @@ export const HASHFLOW_BLACKLIST_TTL_S = 60 * 60 * 24 * 7; // 7 days
 
 export const HASHFLOW_MM_RESTRICT_TTL_S = 60 * 60;
 
-export const HASHFLOW_PRICES_CACHES_TTL_S = 3;
+export const HASHFLOW_PRICES_CACHES_TTL_S = 5;
 
 export const HASHFLOW_MARKET_MAKERS_CACHES_TTL_S = 30;
 
-export const HASHFLOW_API_PRICES_POLLING_INTERVAL_MS = 1000;
+export const HASHFLOW_API_PRICES_POLLING_INTERVAL_MS = 2000;
 
 export const HASHFLOW_API_MARKET_MAKERS_POLLING_INTERVAL_MS = 28 * 1000; // 28 secs
 

package/src/dex/hashflow/hashflow.ts

@@ -1,10 +1,6 @@
 import { ChainId } from '@hashflow/sdk';
 import { Chain, ChainType, HashflowApi } from '@hashflow/taker-js';
-import {
-  MarketMakersResponse,
-  PriceLevelsResponse,
-  RfqResponse,
-} from '@hashflow/taker-js/dist/types/rest';
+import { RfqResponse } from '@hashflow/taker-js/dist/types/rest';
 import BigNumber from 'bignumber.js';
 import { Interface } from 'ethers/lib/utils';
 import { assert } from 'ts-essentials';
@@ -145,7 +141,6 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
             },
             headers: { Authorization: this.hashFlowAuthToken },
           },
-          getCachedMarketMakers: this.getCachedMarketMakers.bind(this),
           filterMarketMakers: this.getFilteredMarketMakers.bind(this),
           pricesCacheKey: this.pricesCacheKey,
           pricesCacheTTLSecs: HASHFLOW_PRICES_CACHES_TTL_S,
@@ -157,9 +152,7 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
   }
 
   async initializePricing(blockNumber: number): Promise<void> {
-    if (!this.dexHelper.config.isSlave) {
-      this.rateFetcher.start();
-    }
+    this.rateFetcher.start();
 
     return;
   }
@@ -198,7 +191,7 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
       return [];
     }
 
-    const levels = (await this.getCachedLevels()) || {};
+    const levels = (await this.rateFetcher.getCachedLevels()) || {};
     const makers = Object.keys(levels);
 
     return makers
@@ -381,32 +374,6 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
     return undefined;
   }
 
-  async getCachedMarketMakers(): Promise<
-    MarketMakersResponse['marketMakers'] | null
-  > {
-    const cachedMarketMakers = await this.dexHelper.cache.rawget(
-      this.marketMakersCacheKey,
-    );
-
-    if (cachedMarketMakers) {
-      return JSON.parse(
-        cachedMarketMakers,
-      ) as MarketMakersResponse['marketMakers'];
-    }
-
-    return null;
-  }
-
-  async getCachedLevels(): Promise<PriceLevelsResponse['levels'] | null> {
-    const cachedLevels = await this.dexHelper.cache.rawget(this.pricesCacheKey);
-
-    if (cachedLevels) {
-      return JSON.parse(cachedLevels) as PriceLevelsResponse['levels'];
-    }
-
-    return null;
-  }
-
   async getPricesVolume(
     srcToken: Token,
     destToken: Token,
@@ -434,7 +401,7 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
 
       const marketMakersToUse = pools.map(p => p.split(`${prefix}_`).pop());
 
-      const levelsMap = (await this.getCachedLevels()) || {};
+      const levelsMap = (await this.rateFetcher.getCachedLevels()) || {};
 
       Object.keys(levelsMap).forEach(mmKey => {
         if (!marketMakersToUse.includes(mmKey)) {
@@ -715,7 +682,7 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
         this.logger.warn(
           `${this.dexKey}-${this.network}: Encountered restricted user=${options.userAddress}. Adding to local blacklist cache`,
         );
-        await this.setBlacklist(options.userAddress);
+        await this.rateFetcher.setBlacklist(options.userAddress);
       } else {
         if (e instanceof TooStrictSlippageCheckError) {
           this.logger.warn(
@@ -911,33 +878,6 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
     };
   }
 
-  getBlackListKey(address: Address) {
-    return `blacklist_${address}`.toLowerCase();
-  }
-
-  async isBlacklisted(txOrigin: Address): Promise<boolean> {
-    const result = await this.dexHelper.cache.get(
-      this.dexKey,
-      this.network,
-      this.getBlackListKey(txOrigin),
-    );
-    return result === 'blacklisted';
-  }
-
-  async setBlacklist(
-    txOrigin: Address,
-    ttl: number = HASHFLOW_BLACKLIST_TTL_S,
-  ) {
-    await this.dexHelper.cache.setex(
-      this.dexKey,
-      this.network,
-      this.getBlackListKey(txOrigin),
-      ttl,
-      'blacklisted',
-    );
-    return true;
-  }
-
   async getSimpleParam(
     srcToken: string,
     destToken: string,
@@ -1058,9 +998,9 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
       .wrapETH(tokenAddress)
       .toLowerCase();
 
-    const makers = (await this.getCachedMarketMakers()) || [];
+    const makers = (await this.rateFetcher.getCachedMarketMakers()) || [];
     const filteredMakers = await this.getFilteredMarketMakers(makers);
-    const pLevels = (await this.getCachedLevels()) || {};
+    const pLevels = (await this.rateFetcher.getCachedLevels()) || {};
 
     let baseToken: Token | undefined = undefined;
     // TODO: Improve efficiency of this part. Quite inefficient way to determine
@@ -1120,4 +1060,8 @@ export class Hashflow extends SimpleExchange implements IDex<HashflowData> {
       this.rateFetcher.stop();
     }
   }
+
+  async isBlacklisted(txOrigin: Address): Promise<boolean> {
+    return this.rateFetcher.isBlacklisted(txOrigin);
+  }
 }