@paraswap/dex-lib 2.16.0 → 2.16.1-generate-state-latest.0

package/src/dex/uniswap-v3/uniswap-v3.ts

@@ -113,7 +113,7 @@ export class UniswapV3
     // To receive revert reasons
     this.dexHelper.web3Provider.eth.handleRevert = false;
 
-    // Normalise once all config addresses and use across all scenarios
+    // Normalize once all config addresses and use across all scenarios
     this.config = this._toLowerForAllConfigAddresses();
 
     this.notExistingPoolSetKey =
@@ -133,7 +133,7 @@ export class UniswapV3
     return `${this.dexKey}_${tokenAddresses}_${fee}`;
   }
 
-  async initializePricing(blockNumber: number) {
+  async initializePricing(blockNumber: number | 'latest' = 'latest') {
     // This is only for testing, because cold pool fetching is goes out of
     // FETCH_POOL_INDENTIFIER_TIMEOUT range
     await Promise.all(
@@ -168,7 +168,7 @@ export class UniswapV3
     srcAddress: Address,
     destAddress: Address,
     fee: bigint,
-    blockNumber: number,
+    blockNumber: number | 'latest',
   ): Promise<UniswapV3EventPool | null> {
     let pool =
       this.eventPools[this.getPoolIdentifier(srcAddress, destAddress, fee)];
@@ -350,7 +350,8 @@ export class UniswapV3
       [],
     );
 
-    const balances = await getBalances(this.dexHelper.multiWrapper, requests);
+    const balances = (await getBalances(this.dexHelper.multiWrapper, requests))
+      .balances;
 
     pools = pools.filter((pool, index) => {
       const balance = balances[index].amounts[DEFAULT_ID_ERC20_AS_STRING];

package/src/dex/woo-fi-v2/woo-fi-v2.ts

@@ -106,7 +106,9 @@ export class WooFiV2 extends SimpleExchange implements IDex<WooFiV2Data> {
     return this._baseTokens;
   }
 
-  async initializePricing(blockNumber: number) {
+  async initializePricing(
+    blockNumber: number | 'latest' = 'latest',
+  ): Promise<void> {
     await this.initializeTokensAndAddresses();
     this._initializePollingPool();
   }

package/src/implementations/local-paraswap-sdk.ts

@@ -76,8 +76,7 @@ export class LocalParaswapSDK implements IParaSwapSDK {
   }
 
   async initializePricing() {
-    const blockNumber = await this.dexHelper.web3Provider.eth.getBlockNumber();
-    await this.pricingHelper.initialize(blockNumber, [this.dexKey]);
+    await this.pricingHelper.initialize('latest', [this.dexKey]);
   }
 
   async releaseResources() {

package/src/lib/generics-events-subscribers/erc20-event-subscriber.ts

@@ -2,7 +2,10 @@ import _ from 'lodash';
 import { LogDescription } from '@ethersproject/abi';
 import { DeepReadonly } from 'ts-essentials';
 import { IDexHelper } from '../../dex-helper';
-import { StatefulEventSubscriber } from '../../stateful-event-subscriber';
+import {
+  StatefulEventSubscriber,
+  StateWithBlock,
+} from '../../stateful-event-subscriber';
 import { Log, BlockHeader, Address } from '../../types';
 import { erc20Iface } from '../utils-interfaces';
 import {
@@ -108,10 +111,12 @@ export class ERC20EventSubscriber extends StatefulEventSubscriber<ERC20StateMap>
     blockHeader: Readonly<BlockHeader>,
   ): Promise<DeepReadonly<ERC20StateMap> | null> {
     const _state = _.cloneDeep(state);
-    let newState = await super.processBlockLogs(_state, logs, blockHeader);
+    const newState = await super.processBlockLogs(_state, logs, blockHeader);
     if (!newState) {
       this.logger.warn('received empty state generate new one');
-      newState = await this.generateState(blockHeader.number);
+      const newStateWithBn = await this.generateState('latest');
+      this.setState(newStateWithBn.state, newStateWithBn.blockNumber);
+      return newStateWithBn.state;
     }
 
     return newState;
@@ -153,7 +158,7 @@ export class ERC20EventSubscriber extends StatefulEventSubscriber<ERC20StateMap>
       blockNumber,
     );
 
-    return balances[0];
+    return balances.balances[0];
   }
 
   async subscribeToWalletBalanceChange(
@@ -185,7 +190,9 @@ export class ERC20EventSubscriber extends StatefulEventSubscriber<ERC20StateMap>
     this.setState(state, blockNumber);
   }
 
-  async generateState(blockNumber: number): Promise<Readonly<ERC20StateMap>> {
+  async generateState(
+    blockNumber: number | 'latest',
+  ): Promise<StateWithBlock<ERC20StateMap>> {
     const request = Array.from(this.walletAddresses).reduce((acc, wallet) => {
       acc.push({
         owner: wallet,
@@ -202,22 +209,26 @@ export class ERC20EventSubscriber extends StatefulEventSubscriber<ERC20StateMap>
       return acc;
     }, [] as BalanceRequest[]);
 
-    const balances = await getBalances(
+    const balancesAndBn = await getBalances(
       this.dexHelper.multiWrapper,
       request,
       blockNumber,
     );
-    return balances.reduce((acc, balance) => {
-      acc[balance.owner] = {
-        balance: balance.amounts[DEFAULT_ID_ERC20_AS_STRING],
-      };
-      this.dexHelper.cache.hset(
-        this.mapKey,
-        balance.owner,
-        balance.amounts[DEFAULT_ID_ERC20_AS_STRING].toString(),
-      );
-      return acc;
-    }, {} as ERC20StateMap);
+    return {
+      blockNumber: balancesAndBn.blockNumber,
+
+      state: balancesAndBn.balances.reduce((acc, balance) => {
+        acc[balance.owner] = {
+          balance: balance.amounts[DEFAULT_ID_ERC20_AS_STRING],
+        };
+        this.dexHelper.cache.hset(
+          this.mapKey,
+          balance.owner,
+          balance.amounts[DEFAULT_ID_ERC20_AS_STRING].toString(),
+        );
+        return acc;
+      }, {} as ERC20StateMap),
+    };
   }
 
   async getBalance(wallet: Address, blockNumber: number): Promise<bigint> {
@@ -249,7 +260,7 @@ export class ERC20EventSubscriber extends StatefulEventSubscriber<ERC20StateMap>
         );
         const balance = await this.getBalanceRPC(wallet, blockNumber);
 
-        // this is ok because we don't modify readonly data
+        // this is not ok because blockNumber might differ from subscriber blockNumber
         state[wallet] = {
           balance: balance.amounts[DEFAULT_ID_ERC20_AS_STRING],
         };

package/src/lib/multi-wrapper.ts

@@ -2,6 +2,7 @@ import { BytesLike } from 'ethers';
 import _ from 'lodash';
 import { Logger } from 'log4js';
 import { Contract } from 'web3-eth-contract';
+import { uint256DecodeToNumber } from './decoders';
 
 export type MultiResult<T> = {
   success: boolean;
@@ -54,10 +55,12 @@ export class MultiWrapper {
   async tryAggregate<T>(
     mandatory: boolean,
     calls: MultiCallParams<T>[],
-    blockNumber?: number | string,
+    blockNumber: number | 'latest' = 'latest',
     batchSize: number = this.defaultBatchSize,
     reportFails: boolean = true,
   ): Promise<MultiResult<T>[]> {
+    const _blockNumber = blockNumber === undefined ? 'latest' : blockNumber;
+
     const allCalls = new Array(Math.ceil(calls.length / batchSize));
     for (let i = 0; i < calls.length; i += batchSize) {
       const batch = calls.slice(i, i + batchSize);
@@ -68,7 +71,7 @@ export class MultiWrapper {
       allCalls.map(batch =>
         this.multi.methods
           .tryAggregate(mandatory, batch)
-          .call(undefined, blockNumber),
+          .call(undefined, _blockNumber),
       ),
     );
 
@@ -105,4 +108,40 @@ export class MultiWrapper {
 
     return results;
   }
+
+  // I removed here batching because it may give inconsistent blockNumber
+  // If we see that is a problem, we can add later
+  async blockTryAggregateWithoutBatching<T>(
+    mandatory: boolean,
+    calls: MultiCallParams<T>[],
+    requestedBlockNumber: number | 'latest' = 'latest',
+    reportFails: boolean = true,
+  ) {
+    const blockNumberCall: MultiCallParams<number> = {
+      target: this.multi.options.address,
+      callData: this.multi.methods.getBlockNumber().encodeABI() as string,
+      decodeFunction: uint256DecodeToNumber,
+    };
+
+    const callsWithBlockNumber: [
+      ...MultiCallParams<T>[],
+      MultiCallParams<number>,
+    ] = [...calls, blockNumberCall];
+
+    const results = await this.tryAggregate(
+      mandatory,
+      callsWithBlockNumber as MultiCallParams<T | number>[],
+      requestedBlockNumber,
+      callsWithBlockNumber.length,
+      reportFails,
+    );
+
+    // We guaranteed previously that at least one call is contained
+    const blockNumber = results.pop()!.returnData as number;
+
+    return {
+      blockNumber,
+      results: results as MultiResult<T>[],
+    };
+  }
 }

package/src/lib/tokens/balancer-fetcher.ts

@@ -135,11 +135,6 @@ export const getAllowanceCallParams = (
     }
     default:
       throw new Error(`missing case for assetType ${req.assetType}`);
-      return {
-        calls: [],
-        spenders: new Set(),
-        length: 0,
-      };
   }
 };
 
@@ -241,11 +236,16 @@ export const decodeBalanceAndAllowanceMultiResult = (
   return userBalance;
 };
 
+type getBalancesResult = {
+  blockNumber: number;
+  balances: UserBalance[];
+};
+
 export const getBalances = async (
   multiv2: MultiWrapper,
   reqs: BalanceRequest[],
   blockNumber: number | 'latest' = 'latest',
-): Promise<UserBalance[]> => {
+): Promise<getBalancesResult> => {
   const calls: MultiCallParams<MultiCallParamsType>[] = []; // TODO: compute the size on advance
 
   const decodingInfo: MultiCallResultDecodeInfo[] = [];
@@ -257,27 +257,31 @@ export const getBalances = async (
     params.length += balancesCalls.length;
     decodingInfo.push(params);
   }
-  const results = await multiv2.tryAggregate<MultiCallParamsType>(
-    false,
-    calls,
-    blockNumber,
-  );
+  const resultsAndBn =
+    await multiv2.blockTryAggregateWithoutBatching<MultiCallParamsType>(
+      false,
+      calls,
+      blockNumber,
+    );
 
   const chunks = [];
   let j = 0;
-  for (let i = 0; i < results.length; ) {
+  for (let i = 0; i < resultsAndBn.results.length; ) {
     const batchSize = decodingInfo[j].length;
-    const chunk = results.slice(i, i + batchSize);
+    const chunk = resultsAndBn.results.slice(i, i + batchSize);
     chunks.push(chunk);
     j++;
     i += batchSize;
   }
 
-  return chunks.map((chunk, index) =>
-    decodeBalanceAndAllowanceMultiResult(
-      reqs[index],
-      decodingInfo[index].spenders,
-      chunk,
+  return {
+    blockNumber: resultsAndBn.blockNumber,
+    balances: chunks.map((chunk, index) =>
+      decodeBalanceAndAllowanceMultiResult(
+        reqs[index],
+        decodingInfo[index].spenders,
+        chunk,
+      ),
     ),
-  );
+  };
 };

package/src/pricing-helper.ts

@@ -36,7 +36,10 @@ export class PricingHelper {
       );
   }
 
-  private async initializeDex(dexKey: string, blockNumber: number) {
+  private async initializeDex(
+    dexKey: string,
+    blockNumber: number | 'latest' = 'latest',
+  ) {
     try {
       const dexInstance = this.dexAdapterService.getDexByKey(dexKey);
 
@@ -80,7 +83,7 @@ export class PricingHelper {
     }
   }
 
-  public async initialize(blockNumber: number, dexKeys: string[]) {
+  public async initialize(blockNumber: number | 'latest', dexKeys: string[]) {
     return await Promise.all(
       dexKeys.map(key => this.initializeDex(key, blockNumber)),
     );

package/src/stateful-event-subscriber.ts

@@ -15,8 +15,13 @@ type StateCache<State> = {
   state: DeepReadonly<State>;
 };
 
+export type StateWithBlock<State> = {
+  state: DeepReadonly<State>;
+  blockNumber: number;
+};
+
 export type InitializeStateOptions<State> = {
-  state?: DeepReadonly<State>;
+  stateWithBn?: StateWithBlock<State>;
   initCallback?: (state: DeepReadonly<State>) => void;
 };
 
@@ -83,12 +88,13 @@ export abstract class StatefulEventSubscriber<State>
   //      will also publish a message to cache to tell one master version of dex-lib that this slave
   //      instance subscribed to a pool from dex this.parentName and name this.name.
   async initialize(
-    blockNumber: number,
+    blockNumber: number | 'latest',
     options?: InitializeStateOptions<State>,
   ) {
     let masterBn: undefined | number = undefined;
-    if (options && options.state) {
-      this.setState(options.state, blockNumber);
+    if (options && options.stateWithBn) {
+      this.setState(options.stateWithBn.state, options.stateWithBn.blockNumber);
+      masterBn = options.stateWithBn.blockNumber;
     } else {
       if (this.dexHelper.config.isSlave && this.masterPoolNeeded) {
         let stateAsString = await this.dexHelper.cache.hget(
@@ -104,37 +110,39 @@ export abstract class StatefulEventSubscriber<State>
             this.logger.warn(
               `${this.parentName}: ${this.name}: found null state in cache generate new one`,
             );
-            state.state = await this.generateState(blockNumber);
+            const stateAndBn = await this.generateState(blockNumber);
+
+            state.state = stateAndBn.state;
+            state.bn = stateAndBn.blockNumber;
+            masterBn = state.bn;
+            this.setState(state.state, state.bn);
           } else {
-            this.logger.info(
+            this.logger.trace(
               `${this.parentName}: ${this.name}: found state from cache`,
             );
-            blockNumber = state.bn;
-
             const _masterBn = await this.dexHelper.cache.rawget(
               this.dexHelper.config.masterBlockNumberCacheKey,
             );
             if (_masterBn) {
               masterBn = parseInt(_masterBn, 10);
-              this.logger.info(
-                `${this.dexHelper.config.data.network} found master blockNumber ${blockNumber}`,
+              this.logger.trace(
+                `${this.dexHelper.config.data.network} found master blockNumber ${masterBn}`,
               );
+              this.setState(state.state, state.bn);
             } else {
-              this.logger.error(
+              throw new Error(
                 `${this.dexHelper.config.data.network} did not found blockNumber in cache`,
               );
             }
           }
-          // set state and the according blockNumber. state.bn can be smaller, greater or equal
-          // to blockNumber
-          this.setState(state.state, blockNumber);
         } else {
           // if no state found in cache generate new state using rpc
           this.logger.info(
             `${this.parentName}: ${this.name}: did not found state on cache generating new one`,
           );
-          const state = await this.generateState(blockNumber);
-          this.setState(state, blockNumber);
+          const stateAndBn = await this.generateState(blockNumber);
+          masterBn = stateAndBn.blockNumber;
+          this.setState(stateAndBn.state, stateAndBn.blockNumber);
 
           // we should publish only if generateState succeeded
           this.dexHelper.cache.publish('new_pools', this.cacheName);
@@ -144,8 +152,9 @@ export abstract class StatefulEventSubscriber<State>
         this.logger.info(
           `${this.parentName}: ${this.name}: cache generating state`,
         );
-        const state = await this.generateState(blockNumber);
-        this.setState(state, blockNumber);
+        const stateAndBn = await this.generateState(blockNumber);
+        masterBn = stateAndBn.blockNumber;
+        this.setState(stateAndBn.state, stateAndBn.blockNumber);
       }
     }
 
@@ -160,7 +169,7 @@ export abstract class StatefulEventSubscriber<State>
     this.dexHelper.blockManager.subscribeToLogs(
       this,
       this.addressesSubscribed,
-      masterBn || blockNumber,
+      masterBn,
     );
     this.isInitialized = true;
   }
@@ -203,7 +212,7 @@ export abstract class StatefulEventSubscriber<State>
   //generate one from scratch.
   abstract generateState(
     blockNumber?: number | 'latest',
-  ): AsyncOrSync<DeepReadonly<State>>;
+  ): AsyncOrSync<StateWithBlock<State>>;
 
   restart(blockNumber: number): void {
     for (const _bn of Object.keys(this.stateHistory)) {
@@ -251,8 +260,8 @@ export abstract class StatefulEventSubscriber<State>
         ++indexBlockEnd;
       }
       if (!this.state) {
-        const freshState = await this.generateState(blockNumber);
-        this.setState(freshState, blockNumber);
+        const freshStateAndBn = await this.generateState('latest');
+        this.setState(freshStateAndBn.state, freshStateAndBn.blockNumber);
       }
       //Find the last state before the blockNumber of the logs
       let stateBeforeLog: DeepReadonly<State> | undefined;
@@ -285,18 +294,16 @@ export abstract class StatefulEventSubscriber<State>
         if (this.state !== null) {
           return true;
         }
-        const latestBlockNumber =
-          this.dexHelper.blockManager.getLatestBlockNumber();
         this.logger.warn(
-          `${network}: ${this.parentName}: ${this.name}: master generate (latest: ${latestBlockNumber}) new state because state is null`,
+          `${network}: ${this.parentName}: ${this.name}: master generate (latest) new state because state is null`,
         );
         try {
-          const state = await this.generateState(latestBlockNumber);
-          this.setState(state, latestBlockNumber);
+          const stateAndBn = await this.generateState('latest');
+          this.setState(stateAndBn.state, stateAndBn.blockNumber);
           return true;
         } catch (e) {
           this.logger.error(
-            `${network}: ${this.parentName} ${this.name}: (${latestBlockNumber}) failed fetch state:`,
+            `${network}: ${this.parentName} ${this.name}: (latest) failed fetch state:`,
             e,
           );
         }

package/src/utils.ts

@@ -319,7 +319,7 @@ export class Utils {
   ): Promise<T> {
     return Promise.race([
       promise,
-      new Promise<T>((resolve, reject) => {
+      new Promise<T>((_, reject) => {
         setTimeout(() => reject(message), timeout);
       }),
     ]);
@@ -353,8 +353,8 @@ export const isTruthy = <T>(x: T | undefined | null | '' | false | 0): x is T =>
   !!x;
 
 type MultiCallParams = {
-  target: any;
-  callData: any;
+  target: string;
+  callData: string;
 };
 
 type BlockAndTryAggregateResult = {
@@ -383,6 +383,5 @@ export const blockAndTryAggregate = async (
 
 export const isContractAddress = async (web3: Web3, addr: string) => {
   const contractCode = await web3.eth.getCode(addr);
-
   return contractCode !== '0x';
 };

package/tests/utils.test.ts

@@ -0,0 +1,44 @@
+import dotenv from 'dotenv';
+dotenv.config();
+
+import Web3 from 'web3';
+import { AbiItem } from 'web3-utils';
+import MultiAbi from '../src/abi/multi-v2.json';
+import { generateConfig } from '../src/config';
+import { Network } from '../src/constants';
+import { getBalanceERC20 } from '../src/lib/tokens/utils';
+import { blockAndTryAggregate } from '../src/utils';
+
+describe('MultiCall with blockNumber', () => {
+  const config = generateConfig(Network.MAINNET);
+
+  const web3 = new Web3(config.privateHttpProvider);
+  const multiContract = new web3.eth.Contract(
+    MultiAbi as AbiItem[],
+    config.multicallV2Address,
+  );
+
+  it('MultiCall with blockNumber', async () => {
+    const res = await blockAndTryAggregate(
+      true,
+      multiContract,
+      [
+        {
+          target: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',
+          callData: getBalanceERC20(
+            '0x05182e579fdfcf69e4390c3411d8fea1fb6467cf',
+          ),
+        },
+        {
+          target: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',
+          callData: getBalanceERC20(
+            '0x05182e579fdfcf69e4390c3411d8fea1fb6467cf',
+          ),
+        },
+      ],
+      'latest',
+    );
+
+    console.log(res);
+  });
+});