@paraswap/dex-lib 4.1.7-mwrappedm.4 → 4.1.7-pub-sub.2

package/src/dex/generic-rfq/generic-rfq.ts

@@ -447,12 +447,7 @@ export class GenericRFQ extends ParaSwapLimitOrders {
   }
 
   async setBlacklist(userAddress: string): Promise<boolean> {
-    await this.dexHelper.cache.hset(
-      this.rateFetcher.blackListCacheKey,
-      userAddress.toLowerCase(),
-      'true',
-    );
-    return true;
+    return this.rateFetcher.setBlacklist(userAddress);
   }
 
   releaseResources(): void {

package/src/dex/generic-rfq/rate-fetcher.ts

@@ -37,6 +37,7 @@ import {
   ERC1271Contract,
 } from '../../lib/erc1271-utils';
 import { isContractAddress } from '../../utils';
+import { ExpKeyValuePubSub, NonExpSetPubSub } from '../../lib/pub-sub';
 
 const GET_FIRM_RATE_TIMEOUT_MS = 2000;
 export const reversePrice = (price: PriceAndAmountBigNumber) =>
@@ -45,6 +46,7 @@ export const reversePrice = (price: PriceAndAmountBigNumber) =>
     price[1].times(price[0]),
   ] as PriceAndAmountBigNumber;
 
+const logPrefix = 'RateFetched[pub_sub]';
 export class RateFetcher {
   private tokensFetcher: Fetcher<TokensResponse>;
   private pairsFetcher: Fetcher<PairsResponse>;
@@ -55,6 +57,9 @@ export class RateFetcher {
   private addressToTokenMap: Record<string, TokenWithInfo> = {};
   private pairs: PairMap = {};
 
+  private pricesPubSub: ExpKeyValuePubSub;
+  private blacklistPubSub?: NonExpSetPubSub;
+
   private firmRateAuth?: (options: RequestConfig) => void;
 
   public blackListCacheKey: string;
@@ -123,6 +128,13 @@ export class RateFetcher {
       logger,
     );
 
+    this.pricesPubSub = new ExpKeyValuePubSub(
+      this.dexHelper,
+      this.dexKey,
+      'prices',
+    );
+
+    this.blackListCacheKey = `${this.dexHelper.config.data.network}_${this.dexKey}_blacklist`;
     if (config.blacklistConfig) {
       this.blackListFetcher = new Fetcher<BlackListResponse>(
         dexHelper.httpRequest,
@@ -142,15 +154,21 @@ export class RateFetcher {
         config.blacklistConfig.intervalMs,
         logger,
       );
+
+      this.blacklistPubSub = new NonExpSetPubSub(
+        this.dexHelper,
+        this.dexKey,
+        'blacklist',
+      );
     }
 
-    this.blackListCacheKey = `${this.dexHelper.config.data.network}_${this.dexKey}_blacklist`;
     if (this.config.firmRateConfig.secret) {
       this.firmRateAuth = this.authHttp(this.config.firmRateConfig.secret);
     }
   }
 
   async initialize() {
+    this.logger.info(`${logPrefix} Initializing rate fetcher`);
     const isContract = await isContractAddress(
       this.dexHelper.web3Provider,
       this.config.maker,
@@ -161,9 +179,22 @@ export class RateFetcher {
         this.config.maker,
       );
     }
+
+    if (this.dexHelper.config.isSlave) {
+      this.logger.info(`${logPrefix} Subscribing to prices pubsub`);
+      this.pricesPubSub.subscribe();
+      if (this.blacklistPubSub) {
+        this.logger.info(`${logPrefix} Subscribing to blacklist pubsub`);
+        const initSet = await this.dexHelper.cache.smembers(
+          this.blackListCacheKey,
+        );
+        this.blacklistPubSub.initializeAndSubscribe(initSet);
+      }
+    }
   }
 
   start() {
+    this.logger.info(`${logPrefix} Starting rate fetcher`);
     this.tokensFetcher.startPolling();
     this.rateFetcher.startPolling();
     this.pairsFetcher.startPolling();
@@ -173,6 +204,7 @@ export class RateFetcher {
   }
 
   stop() {
+    this.logger.info(`${logPrefix} Stopping rate fetcher`);
     this.tokensFetcher.stopPolling();
     this.pairsFetcher.stopPolling();
     this.rateFetcher.stopPolling();
@@ -183,6 +215,10 @@ export class RateFetcher {
   }
 
   private handleTokensResponse(data: TokensResponse) {
+    this.logger.info(
+      `${logPrefix} Handling tokens response, tokens: `,
+      Object.keys(data.tokens).length,
+    );
     for (const tokenName of Object.keys(data.tokens)) {
       const token = data.tokens[tokenName];
       token.address = token.address.toLowerCase();
@@ -200,6 +236,10 @@ export class RateFetcher {
   }
 
   private handlePairsResponse(resp: PairsResponse) {
+    this.logger.info(
+      `${logPrefix} Handling pairs response, pairs: `,
+      Object.keys(resp.pairs).length,
+    );
     this.pairs = {};
 
     const pairs: PairMap = {};
@@ -211,19 +251,37 @@ export class RateFetcher {
   }
 
   private handleBlackListResponse(resp: BlackListResponse) {
+    this.logger.info(
+      `${logPrefix} Handling blacklist response, blacklist length: `,
+      resp.blacklist.length,
+    );
     for (const address of resp.blacklist) {
       this.dexHelper.cache.sadd(this.blackListCacheKey, address.toLowerCase());
     }
+
+    if (this.blacklistPubSub) {
+      this.logger.info(
+        `${logPrefix} Publishing blacklist to pubsub, blacklist length: `,
+        resp.blacklist.length,
+      );
+      this.blacklistPubSub.publish(resp.blacklist);
+    }
   }
 
   public isBlackListed(userAddress: string) {
-    return this.dexHelper.cache.sismember(
-      this.blackListCacheKey,
-      userAddress.toLowerCase(),
-    );
+    if (this.blacklistPubSub) {
+      return this.blacklistPubSub.has(userAddress.toLowerCase());
+    }
+    return false;
   }
 
   private handleRatesResponse(resp: RatesResponse) {
+    this.logger.info(
+      `${logPrefix} Handling rates response, prices length: `,
+      Object.keys(resp.prices).length,
+    );
+    const pubSubData: Record<string, unknown> = {};
+    const ttl = this.config.rateConfig.dataTTLS;
     const pairs = this.pairs;
 
     if (isEmpty(pairs)) return;
@@ -252,37 +310,51 @@ export class RateFetcher {
       }
 
       if (prices.bids.length) {
+        const key = `${baseToken.address}_${quoteToken.address}_bids`;
+        const value = prices.bids;
+        pubSubData[key] = value;
+
         this.dexHelper.cache.setex(
           this.dexKey,
           this.dexHelper.config.data.network,
-          `${baseToken.address}_${quoteToken.address}_bids`,
-          this.config.rateConfig.dataTTLS,
-          JSON.stringify(prices.bids),
+          key,
+          ttl,
+          JSON.stringify(value),
         );
         currentPricePairs.add(`${baseToken.address}_${quoteToken.address}`);
       }
 
       if (prices.asks.length) {
+        const key = `${baseToken.address}_${quoteToken.address}_asks`;
+        const value = prices.asks;
+        pubSubData[key] = value;
+
         this.dexHelper.cache.setex(
           this.dexKey,
           this.dexHelper.config.data.network,
-          `${baseToken.address}_${quoteToken.address}_asks`,
-          this.config.rateConfig.dataTTLS,
-          JSON.stringify(prices.asks),
+          key,
+          ttl,
+          JSON.stringify(value),
         );
         currentPricePairs.add(`${quoteToken.address}_${baseToken.address}`);
       }
     });
 
     if (currentPricePairs.size > 0) {
+      const key = `pairs`;
+      const value = Array.from(currentPricePairs);
+      pubSubData[key] = value;
+
       this.dexHelper.cache.setex(
         this.dexKey,
         this.dexHelper.config.data.network,
-        `pairs`,
-        this.config.rateConfig.dataTTLS,
-        JSON.stringify(Array.from(currentPricePairs)),
+        key,
+        ttl,
+        JSON.stringify(value),
       );
     }
+
+    this.pricesPubSub.publish(pubSubData, ttl);
   }
 
   checkHealth(): boolean {
@@ -322,18 +394,16 @@ export class RateFetcher {
   }
 
   public async getAvailablePairs(): Promise<string[]> {
-    const pairs = await this.dexHelper.cache.getAndCacheLocally(
-      this.dexKey,
-      this.dexHelper.config.data.network,
-      `pairs`,
-      this.config.rateConfig.intervalMs / 1000,
+    const pairs = await this.pricesPubSub.getAndCache<string[]>(`pairs`);
+    this.logger.info(
+      `${logPrefix} Getting available pairs, pairs: `,
+      pairs?.length,
     );
-
     if (!pairs) {
       return [];
     }
 
-    return JSON.parse(pairs) as string[];
+    return pairs;
   }
 
   public async getOrderPrice(
@@ -343,53 +413,36 @@ export class RateFetcher {
   ): Promise<PriceAndAmountBigNumber[] | null> {
     let reversed = false;
 
-    let pricesAsString: string | null = null;
+    let prices: PriceAndAmount[] | null = null;
     if (side === SwapSide.SELL) {
-      pricesAsString = await this.dexHelper.cache.getAndCacheLocally(
-        this.dexKey,
-        this.dexHelper.config.data.network,
+      prices = await this.pricesPubSub.getAndCache(
         `${srcToken.address}_${destToken.address}_bids`,
-        this.config.rateConfig.intervalMs / 1000,
       );
 
-      if (!pricesAsString) {
-        pricesAsString = await this.dexHelper.cache.getAndCacheLocally(
-          this.dexKey,
-          this.dexHelper.config.data.network,
+      if (!prices) {
+        prices = await this.pricesPubSub.getAndCache(
           `${destToken.address}_${srcToken.address}_asks`,
-          this.config.rateConfig.intervalMs / 1000,
         );
         reversed = true;
       }
     } else {
-      pricesAsString = await this.dexHelper.cache.getAndCacheLocally(
-        this.dexKey,
-        this.dexHelper.config.data.network,
+      prices = await this.pricesPubSub.getAndCache(
         `${destToken.address}_${srcToken.address}_asks`,
-        this.config.rateConfig.intervalMs / 1000,
       );
 
-      if (!pricesAsString) {
-        pricesAsString = await this.dexHelper.cache.getAndCacheLocally(
-          this.dexKey,
-          this.dexHelper.config.data.network,
+      if (!prices) {
+        prices = await this.pricesPubSub.getAndCache(
           `${srcToken.address}_${destToken.address}_bids`,
-          this.config.rateConfig.intervalMs / 1000,
         );
         reversed = true;
       }
     }
 
-    if (!pricesAsString) {
-      return null;
-    }
-
-    const orderPricesAsString: PriceAndAmount[] = JSON.parse(pricesAsString);
-    if (!orderPricesAsString) {
+    if (!prices) {
       return null;
     }
 
-    let orderPrices = orderPricesAsString.map(price => [
+    let orderPrices = prices.map(price => [
       new BigNumber(price[0]),
       new BigNumber(price[1]),
     ]);
@@ -488,4 +541,16 @@ export class RateFetcher {
       throw e;
     }
   }
+
+  async setBlacklist(userAddress: string): Promise<boolean> {
+    await this.dexHelper.cache.hset(
+      this.blackListCacheKey,
+      userAddress.toLowerCase(),
+      'true',
+    );
+    if (this.blacklistPubSub) {
+      this.blacklistPubSub.publish([userAddress.toLowerCase()]);
+    }
+    return true;
+  }
 }

package/src/lib/pub-sub.ts

@@ -11,6 +11,108 @@ type KeyValuePubSubMsg = {
 
 type SetPubSubMsg = string[];
 
+export class ExpKeyValuePubSub {
+  channel: string;
+  network: Network;
+  localCache: NodeCache = new NodeCache();
+  logPrefix = 'ExpKeyValuePubSub';
+
+  logger: Logger;
+
+  constructor(
+    private dexHelper: IDexHelper,
+    private dexKey: string,
+    channel: string,
+    private defaultValue?: any,
+    private defaultTTL?: number,
+  ) {
+    this.network = this.dexHelper.config.data.network;
+    this.channel = `${this.network}_${this.dexKey}_${channel}`;
+
+    this.logger = this.dexHelper.getLogger(`ExpKeyValuePubSub_${this.channel}`);
+  }
+
+  subscribe() {
+    this.logger.info(`Subscribing to ${this.channel}`);
+
+    this.dexHelper.cache.subscribe(this.channel, (_, msg) => {
+      const decodedMsg = JSON.parse(msg) as KeyValuePubSubMsg;
+      this.logger.info(
+        `${this.logPrefix} Received msg: '${Object.keys(decodedMsg).length}'`,
+      );
+      this.handleSubscription(decodedMsg);
+    });
+  }
+
+  publish(data: Record<string, unknown>, ttl: number) {
+    const expiresAt = Math.round(Date.now() / 1000) + ttl;
+    this.logger.info(
+      `${this.logPrefix} Publishing msg: '${JSON.stringify({
+        expiresAt,
+        data: Object.keys(data).length,
+      })}'`,
+    );
+    this.dexHelper.cache.publish(
+      this.channel,
+      JSON.stringify({ expiresAt, data }),
+    );
+  }
+
+  handleSubscription(msg: KeyValuePubSubMsg) {
+    const { expiresAt, data } = msg;
+
+    const now = Math.round(Date.now() / 1000);
+    // calculating ttl as message might come with the delay
+    const ttl = expiresAt - now;
+
+    if (ttl > 0) {
+      const keys = Object.keys(data);
+      for (const key of keys) {
+        this.localCache.set(key, data[key], ttl);
+      }
+    } else {
+      this.logger.error('Message has expired', {
+        now,
+        expiresAt,
+        diffInSeconds: now - expiresAt,
+        keys: Object.keys(data),
+      });
+    }
+  }
+
+  async getAndCache<T>(key: string): Promise<T | null> {
+    const localValue = this.localCache.get<T>(key);
+
+    this.logger.info(
+      `${this.logPrefix} Getting and caching key: ${key}, localValue length: ${
+        localValue ? Object.keys(localValue).length : 0
+      }`,
+    );
+
+    if (localValue) {
+      return localValue;
+    }
+
+    const [value, ttl] = await Promise.all([
+      this.dexHelper.cache.get(this.dexKey, this.network, key),
+      this.dexHelper.cache.ttl(this.dexKey, this.network, key),
+    ]);
+
+    if (value && ttl > 0) {
+      const parsedValue = JSON.parse(value);
+      this.localCache.set(key, parsedValue, ttl);
+      return parsedValue;
+    }
+
+    if (this.defaultValue && this.defaultTTL && this.defaultTTL > 0) {
+      this.localCache.set(key, this.defaultValue, this.defaultTTL);
+      return this.defaultValue;
+    }
+
+    return null;
+  }
+}
+
 export class ExpStringPubSub {
   channel: string;
   network: Network;
@@ -178,6 +280,7 @@ export class NonExpSetPubSub {
   channel: string;
   network: Network;
   set = new Set<string>();
+  logPrefix = 'NonExpSetPubSub';
 
   logger: Logger;
 
@@ -193,7 +296,9 @@ export class NonExpSetPubSub {
   }
 
   async initializeAndSubscribe(initialSet: string[]) {
-    this.logger.info(`initializeAndSubscribe with ${initialSet}`);
+    this.logger.info(
+      `${this.logPrefix} initializeAndSubscribe with ${initialSet}`,
+    );
 
     for (const member of initialSet) {
       this.set.add(member);
@@ -207,19 +312,22 @@ export class NonExpSetPubSub {
 
     this.dexHelper.cache.subscribe(this.channel, (_, msg) => {
       const decodedMsg = JSON.parse(msg) as SetPubSubMsg;
+      this.logger.info(
+        `${this.logPrefix} Received subscription msg: '${decodedMsg.length}'`,
+      );
       this.handleSubscription(decodedMsg);
     });
   }
 
   publish(msg: SetPubSubMsg) {
     if (msg.length > 0) {
-      this.logger.info(`Publishing msg: '${msg}'`);
+      this.logger.info(`${this.logPrefix} Publishing msg: '${msg.length}'`);
       this.dexHelper.cache.publish(this.channel, JSON.stringify(msg));
     }
   }
 
   handleSubscription(set: SetPubSubMsg) {
-    this.logger.info(`Received subscription msg: '${set}'`);
+    this.logger.info(`${this.logPrefix} Received subscription msg: '${set}'`);
 
     for (const key of set) {
       this.set.add(key);

package/.idea/aws.xml

@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="accountSettings">
-    <option name="activeProfile" value="profile:default" />
-    <option name="activeRegion" value="us-east-1" />
-    <option name="recentlyUsedProfiles">
-      <list>
-        <option value="profile:default" />
-      </list>
-    </option>
-    <option name="recentlyUsedRegions">
-      <list>
-        <option value="us-east-1" />
-      </list>
-    </option>
-  </component>
-</project>

package/.idea/codeStyles/Project.xml

@@ -1,19 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <code_scheme name="Project" version="173">
-    <ScalaCodeStyleSettings>
-      <option name="MULTILINE_STRING_CLOSING_QUOTES_ON_NEW_LINE" value="true" />
-    </ScalaCodeStyleSettings>
-    <TypeScriptCodeStyleSettings version="0">
-      <option name="SPACE_WITHIN_ARRAY_INITIALIZER_BRACKETS" value="true" />
-      <option name="USE_DOUBLE_QUOTES" value="false" />
-      <option name="SPACES_WITHIN_IMPORTS" value="true" />
-    </TypeScriptCodeStyleSettings>
-    <codeStyleSettings language="TypeScript">
-      <indentOptions>
-        <option name="INDENT_SIZE" value="2" />
-        <option name="CONTINUATION_INDENT_SIZE" value="2" />
-        <option name="TAB_SIZE" value="2" />
-      </indentOptions>
-    </codeStyleSettings>
-  </code_scheme>
-</component>

package/.idea/codeStyles/codeStyleConfig.xml

@@ -1,5 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <state>
-    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
-  </state>
-</component>

package/.idea/misc.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectRootManager">
-    <output url="file://$PROJECT_DIR$/out" />
-  </component>
-</project>

package/.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/paraswap-dex-lib.iml" filepath="$PROJECT_DIR$/.idea/paraswap-dex-lib.iml" />
-    </modules>
-  </component>
-</project>

package/.idea/paraswap-dex-lib.iml

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="JAVA_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

package/.idea/prettier.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="PrettierConfiguration">
-    <option name="myRunOnSave" value="true" />
-    <option name="myRunOnReformat" value="true" />
-  </component>
-</project>

package/.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>