@paraswap/dex-lib 4.0.13 → 4.0.14

package/src/dex/curve-v1-factory/state-polling-pools/polling-manager.ts

@@ -1,9 +1,9 @@
 import { Logger } from 'log4js';
-import { Network } from '../../../constants';
 import { IDexHelper } from '../../../dex-helper';
 import { Utils, _require } from '../../../utils';
 import { PoolState } from '../types';
 import { MulticallReturnedTypes, PoolPollingBase } from './pool-polling-base';
+import { ExpHashPubSub } from '../../../lib/pub-sub';
 
 /*
  * Since we are updating all pools state at once, I need some generalized iterator without state,
@@ -11,7 +11,17 @@ import { MulticallReturnedTypes, PoolPollingBase } from './pool-polling-base';
  */
 
 export class StatePollingManager {
-  static async fetchAndSetStatesFromRPC(
+  private pubSub: ExpHashPubSub;
+
+  constructor(dexHelper: IDexHelper, cacheKey: string, ttl: number) {
+    this.pubSub = new ExpHashPubSub(dexHelper, cacheKey, ttl);
+  }
+
+  subscribe() {
+    this.pubSub.subscribe();
+  }
+
+  async fetchAndSetStatesFromRPC(
     dexHelper: IDexHelper,
     pools: PoolPollingBase[],
     blockNumber?: number,
@@ -94,7 +104,7 @@ export class StatePollingManager {
     return newStates;
   }
 
-  static async masterUpdatePoolsInBatch(
+  async masterUpdatePoolsInBatch(
     logger: Logger,
     dexHelper: IDexHelper,
     pools: PoolPollingBase[],
@@ -102,7 +112,7 @@ export class StatePollingManager {
   ) {
     const dexKey = pools.length > 0 ? pools[0].dexKey : 'CurveV1Factory';
     try {
-      const newStates = await StatePollingManager.fetchAndSetStatesFromRPC(
+      const newStates = await this.fetchAndSetStatesFromRPC(
         dexHelper,
         pools,
         blockNumber,
@@ -113,6 +123,17 @@ export class StatePollingManager {
         { poolLength: pools.length, newStatesLength: newStates.length },
         'newStates.length === pools.length',
       );
+
+      const dataToPublish: Record<string, PoolState | null> = newStates.reduce(
+        (acc, state, i) => {
+          acc[pools[i].poolIdentifier] = state;
+          return acc;
+        },
+        {} as Record<string, PoolState | null>,
+      );
+
+      this.pubSub.publish(dataToPublish);
+
       await Promise.all(
         pools.map(async (p, i) => {
           if (newStates[i] !== null) {
@@ -143,7 +164,7 @@ export class StatePollingManager {
     }
   }
 
-  static async slaveUpdatePoolsInBatch(
+  async slaveUpdatePoolsInBatch(
     logger: Logger,
     dexHelper: IDexHelper,
     pools: PoolPollingBase[],
@@ -155,15 +176,10 @@ export class StatePollingManager {
     await Promise.all(
       pools.map(async p => {
         try {
-          const unparsedStateFromCache = await dexHelper.cache.hget(
-            p.cacheStateKey,
+          const parsedState = await this.pubSub.getAndCache<PoolState>(
             p.poolIdentifier,
           );
-          if (unparsedStateFromCache !== null) {
-            const parsedState = Utils.Parse(
-              unparsedStateFromCache,
-            ) as PoolState;
-
+          if (parsedState !== null) {
             if (p.isStateUpToDate(parsedState)) {
               p.setState(parsedState);
               return;
@@ -197,7 +213,7 @@ export class StatePollingManager {
         }. Falling back to RPC`,
       );
 
-      await StatePollingManager.fetchAndSetStatesFromRPC(
+      await this.fetchAndSetStatesFromRPC(
         dexHelper,
         poolsForRPCUpdate,
         blockNumber,
@@ -217,7 +233,7 @@ export class StatePollingManager {
     }
   }
 
-  static async updatePoolsInBatch(
+  async updatePoolsInBatch(
     logger: Logger,
     dexHelper: IDexHelper,
     pools: PoolPollingBase[],
@@ -233,14 +249,9 @@ export class StatePollingManager {
     }
 
     if (dexHelper.config.isSlave) {
-      await StatePollingManager.slaveUpdatePoolsInBatch(
-        logger,
-        dexHelper,
-        pools,
-        blockNumber,
-      );
+      await this.slaveUpdatePoolsInBatch(logger, dexHelper, pools, blockNumber);
     } else {
-      await StatePollingManager.masterUpdatePoolsInBatch(
+      await this.masterUpdatePoolsInBatch(
         logger,
         dexHelper,
         pools,

package/src/dex/dexalot/constants.ts

@@ -1,6 +1,6 @@
 import BigNumber from 'bignumber.js';
 
-export const DEXALOT_PRICES_CACHES_TTL_S = 3;
+export const DEXALOT_PRICES_CACHES_TTL_S = 5;
 
 export const DEXALOT_PAIRS_CACHES_TTL_S = 21 * 60; // 21 mins
 
@@ -8,7 +8,7 @@ export const DEXALOT_TOKENS_CACHES_TTL_S = 21 * 60; // 21 mins
 
 export const DEXALOT_BLACKLIST_CACHES_TTL_S = 180 * 60; // 3 hours
 
-export const DEXALOT_API_PRICES_POLLING_INTERVAL_MS = 1000;
+export const DEXALOT_API_PRICES_POLLING_INTERVAL_MS = 2000;
 
 export const DEXALOT_API_PAIRS_POLLING_INTERVAL_MS = 1000 * 60 * 10; // 10 mins
 

package/src/dex/dexalot/dexalot.ts

@@ -239,10 +239,14 @@ export class Dexalot extends SimpleExchange implements IDex<DexalotData> {
   }
 
   async getCachedPairs(): Promise<PairDataMap | null> {
-    const cachedPairs = await this.dexHelper.cache.get(
+    const cachedPairs = await this.dexHelper.cache.getAndCacheLocally(
       this.dexKey,
       this.network,
       this.pairsCacheKey,
+      // as local cache just uses passed ttl (instead of getting actual ttl from cache)
+      // pass shorter interval to avoid getting stale data
+      // (same logic is used in other places)
+      DEXALOT_API_PAIRS_POLLING_INTERVAL_MS / 1000,
     );
 
     if (cachedPairs) {
@@ -253,10 +257,11 @@ export class Dexalot extends SimpleExchange implements IDex<DexalotData> {
   }
 
   async getCachedPrices(): Promise<PriceDataMap | null> {
-    const cachedPrices = await this.dexHelper.cache.get(
+    const cachedPrices = await this.dexHelper.cache.getAndCacheLocally(
       this.dexKey,
       this.network,
       this.pricesCacheKey,
+      DEXALOT_API_PRICES_POLLING_INTERVAL_MS / 1000,
     );
 
     if (cachedPrices) {
@@ -267,10 +272,11 @@ export class Dexalot extends SimpleExchange implements IDex<DexalotData> {
   }
 
   async getCachedTokensAddr(): Promise<TokenAddrDataMap | null> {
-    const cachedTokensAddr = await this.dexHelper.cache.get(
+    const cachedTokensAddr = await this.dexHelper.cache.getAndCacheLocally(
       this.dexKey,
       this.network,
       this.tokensAddrCacheKey,
+      DEXALOT_API_PAIRS_POLLING_INTERVAL_MS / 1000,
     );
 
     if (cachedTokensAddr) {
@@ -281,10 +287,11 @@ export class Dexalot extends SimpleExchange implements IDex<DexalotData> {
   }
 
   async getCachedTokens(): Promise<TokenDataMap | null> {
-    const cachedTokens = await this.dexHelper.cache.get(
+    const cachedTokens = await this.dexHelper.cache.getAndCacheLocally(
       this.dexKey,
       this.network,
       this.tokensCacheKey,
+      DEXALOT_API_PAIRS_POLLING_INTERVAL_MS / 1000,
     );
 
     if (cachedTokens) {

package/src/dex/dexalot/rate-fetcher.ts

@@ -1,21 +1,21 @@
+import { Network } from '../../constants';
 import { IDexHelper } from '../../dex-helper';
 import { Fetcher } from '../../lib/fetcher/fetcher';
 import { validateAndCast } from '../../lib/validators';
 import { Logger, Token } from '../../types';
 import {
-  DexalotRateFetcherConfig,
+  DexalotBlacklistResponse,
   DexalotPairsResponse,
-  PairDataMap,
   DexalotPricesResponse,
+  DexalotRateFetcherConfig,
+  PairDataMap,
   PriceDataMap,
-  DexalotBlacklistResponse,
 } from './types';
 import {
-  pricesResponseValidator,
-  pairsResponseValidator,
   blacklistResponseValidator,
+  pairsResponseValidator,
+  pricesResponseValidator,
 } from './validators';
-import { Network } from '../../constants';
 
 export class RateFetcher {
   private pairsFetcher: Fetcher<DexalotPairsResponse>;

package/src/dex/generic-rfq/rate-fetcher.ts

@@ -322,10 +322,11 @@ export class RateFetcher {
   }
 
   public async getAvailablePairs(): Promise<string[]> {
-    const pairs = await this.dexHelper.cache.get(
+    const pairs = await this.dexHelper.cache.getAndCacheLocally(
       this.dexKey,
       this.dexHelper.config.data.network,
       `pairs`,
+      this.config.rateConfig.dataTTLS,
     );
 
     if (!pairs) {
@@ -344,32 +345,36 @@ export class RateFetcher {
 
     let pricesAsString: string | null = null;
     if (side === SwapSide.SELL) {
-      pricesAsString = await this.dexHelper.cache.get(
+      pricesAsString = await this.dexHelper.cache.getAndCacheLocally(
         this.dexKey,
         this.dexHelper.config.data.network,
         `${srcToken.address}_${destToken.address}_bids`,
+        this.config.rateConfig.dataTTLS,
       );
 
       if (!pricesAsString) {
-        pricesAsString = await this.dexHelper.cache.get(
+        pricesAsString = await this.dexHelper.cache.getAndCacheLocally(
           this.dexKey,
           this.dexHelper.config.data.network,
           `${destToken.address}_${srcToken.address}_asks`,
+          this.config.rateConfig.dataTTLS,
         );
         reversed = true;
       }
     } else {
-      pricesAsString = await this.dexHelper.cache.get(
+      pricesAsString = await this.dexHelper.cache.getAndCacheLocally(
         this.dexKey,
         this.dexHelper.config.data.network,
         `${destToken.address}_${srcToken.address}_asks`,
+        this.config.rateConfig.dataTTLS,
       );
 
       if (!pricesAsString) {
-        pricesAsString = await this.dexHelper.cache.get(
+        pricesAsString = await this.dexHelper.cache.getAndCacheLocally(
           this.dexKey,
           this.dexHelper.config.data.network,
           `${srcToken.address}_${destToken.address}_bids`,
+          this.config.rateConfig.dataTTLS,
         );
         reversed = true;
       }

package/src/dex/swaap-v2/swaap-v2.ts

@@ -278,10 +278,11 @@ export class SwaapV2 extends SimpleExchange implements IDex<SwaapV2Data> {
   }
 
   async getCachedTokens(): Promise<TokensMap | null> {
-    const cachedTokens = await this.dexHelper.cache.get(
+    const cachedTokens = await this.dexHelper.cache.getAndCacheLocally(
       this.dexKey,
       this.network,
       SWAAP_TOKENS_CACHE_KEY,
+      SWAAP_RFQ_API_TOKENS_POLLING_INTERVAL_MS / 1000,
     );
 
     if (cachedTokens) {
@@ -292,10 +293,11 @@ export class SwaapV2 extends SimpleExchange implements IDex<SwaapV2Data> {
   }
 
   async getCachedLevels(): Promise<Record<string, SwaapV2PriceLevels> | null> {
-    const cachedLevels = await this.dexHelper.cache.get(
+    const cachedLevels = await this.dexHelper.cache.getAndCacheLocally(
       this.dexKey,
       this.network,
       SWAAP_PRICES_CACHE_KEY,
+      SWAAP_RFQ_API_PRICES_POLLING_INTERVAL_MS / 1000,
     );
 
     if (cachedLevels) {

package/src/dex-helper/dummy-dex-helper.ts

@@ -43,6 +43,23 @@ class DummyCache implements ICache {
     return null;
   }
 
+  async keys(
+    dexKey: string,
+    network: number,
+    cacheKey: string,
+  ): Promise<string[]> {
+    return [];
+  }
+
+  async ttl(
+    dexKey: string,
+    network: number,
+    cacheKey: string,
+  ): Promise<number> {
+    const key = `${network}_${dexKey}_${cacheKey}`.toLowerCase();
+    return this.storage[key] ? 1 : -1;
+  }
+
   async rawget(key: string): Promise<string | null> {
     return this.storage[key] ? this.storage[key] : null;
     return null;
@@ -139,6 +156,10 @@ class DummyCache implements ICache {
     return set.has(key);
   }
 
+  async smembers(setKey: string): Promise<string[]> {
+    return Array.from(this.setMap[setKey] ?? []);
+  }
+
   async hset(mapKey: string, key: string, value: string): Promise<void> {
     if (!this.hashStorage[mapKey]) this.hashStorage[mapKey] = {};
     this.hashStorage[mapKey][key] = value;

package/src/dex-helper/icache.ts

@@ -5,6 +5,10 @@ export interface ICache {
     cacheKey: string,
   ): Promise<string | null>;
 
+  ttl(dexKey: string, network: number, cacheKey: string): Promise<number>;
+
+  keys(dexKey: string, network: number, cacheKey: string): Promise<string[]>;
+
   rawget(key: string): Promise<string | null>;
 
   rawset(key: string, value: string, ttl: number): Promise<string | null>;
@@ -52,6 +56,8 @@ export interface ICache {
 
   sismember(setKey: string, key: string): Promise<boolean>;
 
+  smembers(setKey: string): Promise<string[]>;
+
   hset(mapKey: string, key: string, value: string): Promise<void>;
 
   hdel(mapKey: string, keys: string[]): Promise<number>;

package/src/lib/pub-sub.ts

@@ -0,0 +1,252 @@
+import NodeCache from 'node-cache';
+import { Network } from '../constants';
+import { IDexHelper } from '../dex-helper';
+import { Logger } from '../types';
+import { Utils } from '../utils';
+
+type KeyValuePubSubMsg = {
+  expiresAt: number;
+  data: Record<string, unknown>;
+};
+
+type SetPubSubMsg = string[];
+
+export class ExpStringPubSub {
+  channel: string;
+  network: Network;
+  localCache: NodeCache = new NodeCache();
+
+  logger: Logger;
+
+  constructor(
+    private dexHelper: IDexHelper,
+    private dexKey: string,
+    channel: string,
+    private defaultValue?: any,
+    private defaultTTL?: number,
+  ) {
+    this.network = this.dexHelper.config.data.network;
+    this.channel = `${this.network}_${this.dexKey}_${channel}`;
+
+    this.logger = this.dexHelper.getLogger(`ExpKeyValuePubSub_${this.channel}`);
+  }
+
+  subscribe() {
+    this.logger.info(`Subscribing to ${this.channel}`);
+
+    this.dexHelper.cache.subscribe(this.channel, (_, msg) => {
+      const before = Date.now();
+      const decodedMsg = JSON.parse(msg) as KeyValuePubSubMsg;
+      this.handleSubscription(decodedMsg);
+      const after = Date.now();
+      this.logger.info(`Decoding and handling took: ${after - before}ms`);
+    });
+  }
+
+  publish(data: Record<string, unknown>, ttl: number) {
+    const expiresAt = Math.round(Date.now() / 1000) + ttl;
+    this.logger.info(
+      `Publishing keys: '${Object.keys(data)}', expiresAt: '${expiresAt}'`,
+    );
+
+    this.dexHelper.cache.publish(
+      this.channel,
+      JSON.stringify({ expiresAt, data }),
+    );
+  }
+
+  handleSubscription(msg: KeyValuePubSubMsg) {
+    const { expiresAt, data } = msg;
+
+    const now = Math.round(Date.now() / 1000);
+    // calculating ttl as message might come with the delay
+    const ttl = expiresAt - now;
+
+    if (ttl > 0) {
+      const keys = Object.keys(data);
+      for (const key of keys) {
+        this.localCache.set(key, data[key], ttl);
+      }
+    } else {
+      this.logger.error('Message has expired', {
+        now,
+        expiresAt,
+        diffInSeconds: now - expiresAt,
+        keys: Object.keys(data),
+      });
+    }
+  }
+
+  async getAndCache<T>(key: string): Promise<T | null> {
+    const localValue = this.localCache.get<T>(key);
+
+    if (localValue) {
+      return localValue;
+    }
+
+    const [value, ttl] = await Promise.all([
+      this.dexHelper.cache.get(this.dexKey, this.network, key),
+      this.dexHelper.cache.ttl(this.dexKey, this.network, key),
+    ]);
+
+    if (value && ttl > 0) {
+      const parsedValue = JSON.parse(value);
+      this.localCache.set(key, parsedValue, ttl);
+      return parsedValue;
+    }
+
+    if (this.defaultValue && this.defaultTTL && this.defaultTTL > 0) {
+      this.localCache.set(key, this.defaultValue, this.defaultTTL);
+      return this.defaultValue;
+    }
+
+    return null;
+  }
+}
+
+export class ExpHashPubSub {
+  localCache: NodeCache = new NodeCache();
+
+  logger: Logger;
+
+  constructor(
+    private dexHelper: IDexHelper,
+    private hashKey: string,
+    // in seconds
+    private ttl: number,
+  ) {
+    this.hashKey = hashKey;
+    this.logger = this.dexHelper.getLogger(`ExpKeyValuePubSub_${this.hashKey}`);
+  }
+
+  subscribe() {
+    this.logger.info(`Subscribing`);
+
+    this.dexHelper.cache.subscribe(this.hashKey, (_, msg) => {
+      const before = Date.now();
+      const decodedMsg = Utils.Parse(msg) as KeyValuePubSubMsg;
+      this.handleSubscription(decodedMsg);
+      const after = Date.now();
+      this.logger.info(`Decoding and handling took: ${after - before}ms`);
+    });
+  }
+
+  publish(data: Record<string, unknown>) {
+    if (Object.keys(data).length > 0) {
+      const expiresAt = Math.round(Date.now() / 1000) + this.ttl;
+      this.logger.info(
+        `Publishing keys: '${Object.keys(data)}', expiresAt: '${expiresAt}'`,
+      );
+
+      this.dexHelper.cache.publish(
+        this.hashKey,
+        Utils.Serialize({ expiresAt, data }),
+      );
+    }
+  }
+
+  handleSubscription(msg: KeyValuePubSubMsg) {
+    const { expiresAt, data } = msg;
+    this.logger.info(
+      `Received subscription, keys: '${Object.keys(
+        data,
+      )}', expiresAt: '${expiresAt}'`,
+    );
+
+    const now = Math.round(Date.now() / 1000);
+    // calculating ttl as message might come with the delay
+    const ttl = expiresAt - now;
+
+    if (ttl > 0) {
+      const keys = Object.keys(data);
+      for (const key of keys) {
+        this.localCache.set(key, data[key], ttl);
+      }
+    } else {
+      this.logger.error('Message has expired', {
+        now,
+        expiresAt,
+        diffInSeconds: now - expiresAt,
+        keys: Object.keys(data),
+      });
+    }
+  }
+
+  async getAndCache<T>(key: string): Promise<T | null> {
+    const localValue = this.localCache.get<T>(key);
+
+    if (localValue !== undefined) {
+      this.logger.info(`Returning from local cache: '${key}'`);
+      return localValue;
+    }
+
+    this.logger.info(`Returning from external cache: '${key}'`);
+
+    const value = await this.dexHelper.cache.hget(this.hashKey, key);
+
+    if (value && this.ttl > 0) {
+      const parsedValue = Utils.Parse(value);
+      this.localCache.set(key, parsedValue, this.ttl);
+      return parsedValue;
+    }
+
+    return null;
+  }
+}
+
+export class NonExpSetPubSub {
+  channel: string;
+  network: Network;
+  set = new Set<string>();
+
+  logger: Logger;
+
+  constructor(
+    private dexHelper: IDexHelper,
+    private dexKey: string,
+    channel: string,
+  ) {
+    this.network = this.dexHelper.config.data.network;
+    this.channel = `${this.network}_${this.dexKey}_${channel}`;
+
+    this.logger = this.dexHelper.getLogger(`NonExpSetPubSub_${this.channel}`);
+  }
+
+  async initializeAndSubscribe(initialSet: string[]) {
+    this.logger.info(`initializeAndSubscribe with ${initialSet}`);
+
+    for (const member of initialSet) {
+      this.set.add(member);
+    }
+
+    this.subscribe();
+  }
+
+  subscribe() {
+    this.logger.info(`Subscribing`);
+
+    this.dexHelper.cache.subscribe(this.channel, (_, msg) => {
+      const decodedMsg = JSON.parse(msg) as SetPubSubMsg;
+      this.handleSubscription(decodedMsg);
+    });
+  }
+
+  publish(msg: SetPubSubMsg) {
+    if (msg.length > 0) {
+      this.logger.info(`Publishing msg: '${msg}'`);
+      this.dexHelper.cache.publish(this.channel, JSON.stringify(msg));
+    }
+  }
+
+  handleSubscription(set: SetPubSubMsg) {
+    this.logger.info(`Received subscription msg: '${set}'`);
+
+    for (const key of set) {
+      this.set.add(key);
+    }
+  }
+
+  async has(key: string) {
+    return this.set.has(key);
+  }
+}

package/.idea/aws.xml

@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="accountSettings">
-    <option name="activeProfile" value="profile:default" />
-    <option name="activeRegion" value="us-east-1" />
-    <option name="recentlyUsedProfiles">
-      <list>
-        <option value="profile:default" />
-      </list>
-    </option>
-    <option name="recentlyUsedRegions">
-      <list>
-        <option value="us-east-1" />
-      </list>
-    </option>
-  </component>
-</project>

package/.idea/codeStyles/Project.xml

@@ -1,19 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <code_scheme name="Project" version="173">
-    <ScalaCodeStyleSettings>
-      <option name="MULTILINE_STRING_CLOSING_QUOTES_ON_NEW_LINE" value="true" />
-    </ScalaCodeStyleSettings>
-    <TypeScriptCodeStyleSettings version="0">
-      <option name="SPACE_WITHIN_ARRAY_INITIALIZER_BRACKETS" value="true" />
-      <option name="USE_DOUBLE_QUOTES" value="false" />
-      <option name="SPACES_WITHIN_IMPORTS" value="true" />
-    </TypeScriptCodeStyleSettings>
-    <codeStyleSettings language="TypeScript">
-      <indentOptions>
-        <option name="INDENT_SIZE" value="2" />
-        <option name="CONTINUATION_INDENT_SIZE" value="2" />
-        <option name="TAB_SIZE" value="2" />
-      </indentOptions>
-    </codeStyleSettings>
-  </code_scheme>
-</component>

package/.idea/codeStyles/codeStyleConfig.xml

@@ -1,5 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <state>
-    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
-  </state>
-</component>

package/.idea/misc.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectRootManager">
-    <output url="file://$PROJECT_DIR$/out" />
-  </component>
-</project>

package/.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/paraswap-dex-lib.iml" filepath="$PROJECT_DIR$/.idea/paraswap-dex-lib.iml" />
-    </modules>
-  </component>
-</project>

package/.idea/paraswap-dex-lib.iml

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="JAVA_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>