@ocap/resolver 1.19.1 → 1.19.3

package/lib/hooks.js

@@ -295,7 +295,14 @@ const onAccountMigrate = async (tx, resolver) => {
   }
 };
 
-const onCreateTx = (tx, ctx, resolver) => {
+const onCreateTx = async (tx, ctx, resolver) => {
+  // Always update distribution even if tx failed cause there might be had gas
+  try {
+    await resolver.tokenDistribution.updateByTx(tx, ctx);
+  } catch (e) {
+    resolver.logger.error('Failed to update token distribution', e);
+  }
+
   if (tx.code !== 'OK') {
     return;
   }

package/lib/index.js

@@ -1,3 +1,4 @@
+/* eslint-disable no-loop-func */
 /* eslint-disable no-await-in-loop */
 /* eslint-disable newline-per-chained-call */
 const get = require('lodash/get');
@@ -20,6 +21,7 @@ const { BN, toBN, fromTokenToUnit, toAddress } = require('@ocap/util');
 const { DEFAULT_TOKEN_DECIMAL } = require('@ocap/util/lib/constant');
 const { createExecutor } = require('@ocap/tx-protocols');
 const { decodeAnySafe } = require('@ocap/tx-protocols/lib/util');
+const { Joi } = require('@arcblock/validator');
 
 const {
   createIndexedAccount,
@@ -48,6 +50,7 @@ const hooks = require('./hooks');
 const tokenFlow = require('./token-flow');
 const { getInstance: getTokenCacheInstance } = require('./token-cache');
 const { MigrationChainManager } = require('./migration-chain');
+const { TokenDistributionManager } = require('./token-distribution');
 
 const noop = (x) => x;
 const CHAIN_ADDR = md5('OCAP_CHAIN_ADDR');
@@ -183,6 +186,7 @@ module.exports = class OCAPResolver {
 
     if (indexdb) {
       this.tokenCache = getTokenCacheInstance(indexdb.token);
+      this.tokenDistribution = new TokenDistributionManager(this);
     }
 
     if (this.tokenItx) {
@@ -842,6 +846,9 @@ module.exports = class OCAPResolver {
         }
       } catch (error) {
         console.error('create tx index failed', { account: x, error });
+      } finally {
+        // eslint-disable-next-line no-console
+        console.log(x);
       }
     });
 
@@ -1242,6 +1249,158 @@ module.exports = class OCAPResolver {
 
     return updatedTxs.filter(Boolean);
   }
+
+  async updateTokenDistribution({ tokenAddress = this.config.token.address, force = false } = {}) {
+    const paramsSchema = Joi.object({
+      tokenAddress: Joi.DID().prefix().role('ROLE_TOKEN').required(),
+      force: Joi.boolean().required(),
+    });
+    const validation = paramsSchema.validate({ tokenAddress, force });
+    if (validation.error) {
+      throw new CustomError('INVALID_PARAMS', validation.error.message);
+    }
+
+    const data = await this.tokenDistribution.updateByToken(tokenAddress, force);
+    return data;
+  }
+
+  async getTokenDistribution({ tokenAddress = this.config.token.address }) {
+    const data = await this.tokenDistribution.getDistribution(tokenAddress);
+    return data;
+  }
+
+  /**
+   * Fetch data in chunks to handle large datasets
+   * @param {Function} fn Function to fetch data
+   * @param {Object} param
+   * @param {number} [param.concurrency=3] Number of concurrent requests
+   * @param {number} [param.chunkSize=2000] Maximum number of items to return in each next() call
+   * @param {number} [param.pageSize=100] Number of items per page
+   * @param {Function} param.getTime Function to get timestamp from item
+   * @param {string} param.dataKey Key to access data in response
+   * @returns {Promise<Array>} Array of fetched items
+   */
+  listChunks(fn, { concurrency = 3, chunkSize = 2000, pageSize = 100, getTime, dataKey }) {
+    let totalPage;
+    let curPage;
+    let done = false;
+    let time;
+
+    const fetchFirstPage = async () => {
+      const { paging, page, [dataKey]: list } = await fn({ size: pageSize, cursor: '0' });
+      const total = paging?.total || page?.total;
+      curPage = 1;
+      totalPage = Math.ceil(total / pageSize);
+      return list;
+    };
+
+    const next = async () => {
+      if (done) {
+        return [];
+      }
+
+      let results = [];
+
+      // first page
+      if (!totalPage) {
+        const data = await fetchFirstPage();
+        // only 1 page
+        if (data.length < pageSize) {
+          done = true;
+          return data;
+        }
+
+        time = getTime(data[data.length - 1]);
+        results = results.concat(data);
+
+        // limit
+        if (data.length >= chunkSize) {
+          return results;
+        }
+      }
+
+      // next pages
+      for (; curPage < totalPage; curPage += concurrency) {
+        const batchResults = await Promise.all(
+          new Array(concurrency).fill(true).map(async (_, i) => {
+            const { [dataKey]: list } = await fn({
+              size: pageSize,
+              cursor: i * pageSize,
+              time,
+            });
+            return list;
+          })
+        );
+        const flatResults = batchResults.flat();
+
+        // finish
+        if (!flatResults.length) {
+          done = true;
+          return [];
+        }
+
+        results = results.concat(flatResults);
+        time = results.length ? getTime(results[results.length - 1]) : null;
+
+        // limit
+        if (results.length >= chunkSize) {
+          return results;
+        }
+      }
+
+      done = true;
+      return results;
+    };
+
+    return {
+      next,
+      done,
+    };
+  }
+
+  listTransactionsChunks(args = {}, { chunkSize = 2000, pageSize = 100 } = {}) {
+    return this.listChunks(
+      ({ size, time, cursor }) =>
+        this.listTransactions({
+          ...args,
+          paging: {
+            order: { field: 'time', type: 'asc' },
+            ...(args.paging || {}),
+            size,
+            cursor,
+          },
+          timeFilter: Object.assign(args.timeFilter || {}, time ? { startDateTime: time } : {}),
+        }),
+      {
+        dataKey: 'transactions',
+        getTime: (tx) => tx?.time,
+        chunkSize,
+        pageSize,
+      }
+    );
+  }
+
+  listStakeChunks(args = {}, { chunkSize = 2000, pageSize = 100 } = {}) {
+    return this.listChunks(
+      ({ size, time, cursor }) =>
+        this.listStakes({
+          ...args,
+          paging: {
+            order: { field: 'time', type: 'asc' },
+            ...(args.paging || {}),
+            size,
+            cursor,
+          },
+          timeFilter: Object.assign(args.timeFilter || {}, time ? { startDateTime: time } : {}),
+        }),
+      {
+        dataKey: 'stakes',
+        getTime: (state) => state?.time,
+        chunkSize,
+        pageSize,
+      }
+    );
+  }
 };
 
 module.exports.formatData = formatData;

package/lib/token-distribution.js

@@ -0,0 +1,334 @@
+/* eslint-disable no-loop-func */
+/* eslint-disable no-await-in-loop */
+/* eslint-disable consistent-return */
+const { BN, isSameDid, fromTokenToUnit } = require('@ocap/util');
+const { toTypeInfo } = require('@arcblock/did');
+const {
+  types: { RoleType },
+} = require('@ocap/mcrypto');
+const { createIndexedTokenDistribution } = require('@ocap/indexdb/lib/util');
+const { eachReceipts } = require('@ocap/state/lib/states/tx');
+const { isGasStakeAddress } = require('@ocap/tx-protocols/lib/util');
+
+const ZERO = new BN(0);
+
+class TokenDistributionManager {
+  constructor(resolver) {
+    this.resolver = resolver;
+    this.indexdb = resolver.indexdb;
+  }
+
+  formatDistribution(distribution) {
+    const { tokenAddress, account, gas, fee, slashedVault, stake, revokedStake, gasStake, other, txTime } =
+      distribution;
+    return {
+      tokenAddress,
+      account: new BN(account || 0),
+      gas: new BN(gas || 0),
+      fee: new BN(fee || 0),
+      slashedVault: new BN(slashedVault || 0),
+      stake: new BN(stake || 0),
+      revokedStake: new BN(revokedStake || 0),
+      gasStake: new BN(gasStake || 0),
+      other: new BN(other || 0),
+      txTime: txTime || new Date(0).toISOString(),
+    };
+  }
+
+  async getDistribution(tokenAddress) {
+    const data = await this.indexdb.tokenDistribution.get(tokenAddress);
+    return data && createIndexedTokenDistribution(data);
+  }
+
+  async saveDistribution(distribution) {
+    const data = createIndexedTokenDistribution(distribution);
+    const indexdbDistribution = await this.getDistribution(data.tokenAddress);
+
+    if (!indexdbDistribution) {
+      await this.indexdb.tokenDistribution.insert(data);
+    } else {
+      // ensure txTime is latest
+      const latestTime = Math.max(new Date(indexdbDistribution.txTime).getTime(), new Date(data.txTime).getTime());
+      data.txTime = new Date(latestTime).toISOString();
+      await this.indexdb.tokenDistribution.update(data.tokenAddress, data);
+    }
+
+    return data;
+  }
+
+  /**
+   * Calculate token distribution based on all transaction data.
+   * This method is usually used to update historical token distribution data
+   *
+   * @param {string} tokenAddress Token address
+   * @param {boolean} force If force is false, only calculate distributions for new transactions after txTime in indexdb. default is false
+   * @returns {Promise<Object>}
+   */
+  async updateByToken(tokenAddress, force) {
+    const { logger, config } = this.resolver;
+
+    const distribution = force
+      ? this.formatDistribution({ tokenAddress })
+      : this.formatDistribution((await this.getDistribution(tokenAddress)) || { tokenAddress });
+    const isDefaultToken = tokenAddress === config.token.address;
+
+    logger?.info(`Update distribution by token (${tokenAddress})`, {
+      distribution: createIndexedTokenDistribution(distribution),
+    });
+
+    if (force && isDefaultToken) {
+      this.handleModerator(distribution);
+    }
+
+    const { next } = await this.resolver.listTransactionsChunks({
+      timeFilter: { startDateTime: distribution.txTime },
+      // Default token is used for gas payment and may appear in any transaction, so we cannot filter by tokenFilter
+      tokenFilter: isDefaultToken ? {} : { tokenFilter: { tokens: [tokenAddress] } },
+    });
+    let nextData = await next();
+
+    // Process transactions in chunks and update indexdb
+    while (nextData.length) {
+      logger?.info('Updating token distribution in chunks', {
+        chunkSize: nextData.length,
+        startTime: nextData[0].time,
+        startHash: nextData[0].hash,
+        endTime: nextData[nextData.length - 1].time,
+      });
+
+      const handlePromises = nextData.map((tx) => this.handleTx(tx, distribution));
+      await Promise.all(handlePromises);
+
+      // update indexdb
+      await this.saveDistribution(distribution);
+      nextData = await next();
+    }
+
+    // We cannot distinguish between revokedStake and stake from tx receipts here,
+    // so we need to read all stake transactions and recalculate token distribution based on their revokeTokens and tokens
+    await this.splitStake(distribution, force);
+    await this.saveDistribution(distribution);
+
+    const result = createIndexedTokenDistribution(distribution);
+
+    logger.info(`Token distribution update completed (${tokenAddress})`, { distribution: result });
+
+    return result;
+  }
+
+  /**
+   * Split out revokedStake / gasStake from stake
+   *
+   * @param {Object} distribution
+   * @param {boolean} force If force is false, only process transactions after txTime in indexdb. Default is false
+   * @returns {Promise<Object>}
+   */
+  async splitStake(distribution, force) {
+    const { tokenAddress } = distribution;
+
+    const { next } = await this.resolver.listStakeChunks({
+      timeFilter: !force ? { startDateTime: distribution.txTime } : {},
+    });
+
+    let nextData = await next();
+
+    // Process transactions in chunks and update indexdb
+    while (nextData.length) {
+      nextData.forEach((stakeState) => {
+        const isGasStake = this.isGasStake(stakeState);
+        const token = stakeState.tokens.find((x) => x.address === tokenAddress);
+        const revokedToken = stakeState.revokedTokens.find((x) => x.address === tokenAddress);
+        const revokedBalance = new BN(revokedToken?.balance || 0);
+        const balance = new BN(token?.balance || 0);
+
+        // stake -->> revokedStake
+        distribution.revokedStake = distribution.revokedStake.add(revokedBalance);
+        distribution.stake = distribution.stake.sub(revokedBalance);
+
+        // stake -->> gasStake
+        if (isGasStake) {
+          distribution.gasStake = distribution.gasStake.add(balance);
+          distribution.stake = distribution.stake.sub(balance);
+        }
+      });
+
+      // continue
+      nextData = await next();
+    }
+
+    return distribution;
+  }
+
+  /**
+   * Update token distribution by a single transaction.
+   * This method is usually used when a tx is completed.
+   *
+   * @param {Object} tx The transaction object
+   * @param {Object} context The transaction context
+   * @returns {Promise<Object>} The updated token distributions
+   */
+  async updateByTx(tx, context) {
+    const formattedTx = await this.resolver.formatTx(tx);
+    const tokens = formattedTx.tokenSymbols.map(({ address }) => address);
+
+    // Parse all tokens in this transaction
+    const results = await Promise.all(
+      tokens.map(async (tokenAddress) => {
+        const distribution = this.formatDistribution((await this.getDistribution(tokenAddress)) || { tokenAddress });
+        await this.handleTx(tx, distribution, { context, isHandleStake: true });
+        const data = await this.saveDistribution(distribution);
+        return data;
+      })
+    );
+
+    return results;
+  }
+
+  /**
+   * Parse token distribution for a single transaction
+   * @param {Object} tx
+   * @param {String} distribution
+   * @param {Object} param
+   * @param {Object} param.context
+   * @param {Boolean} param.isHandleStake Whether to handle revoked / gas stake tokens
+   * @returns {Promise<Object>}  token distribution
+   */
+  async handleTx(tx, distribution, { context, isHandleStake = false } = {}) {
+    if (isHandleStake && !context) {
+      throw new Error('FORBIDDEN', 'context is missing, handle revoke stake is not supported without context');
+    }
+
+    const type = tx.tx?.itxJson?._type;
+    const receipts = tx.receipts || [];
+    const { tokenAddress } = distribution;
+
+    // Iterate through each transaction receipt and distribute tokens based on account type
+    const handlePromises = eachReceipts(receipts, async (address, change) => {
+      if (!isSameDid(change.target, tokenAddress)) return;
+      // In migrate tx, tokens are frozen for the old account and minted for the new account
+      // It behaves like minting new tokens in receipts
+      // So we should skip here
+      if (change.action === 'migrate') return;
+
+      const roleType = toTypeInfo(address).role;
+      const value = new BN(change.value);
+
+      // Stake
+      if (roleType === RoleType.ROLE_STAKE) {
+        if (isHandleStake) {
+          const stakeState = await this.getStakeState(address, context);
+          const isGasStake = this.isGasStake(stakeState);
+          // stake revokedTokens -->> account tokens
+          if (type === 'ClaimStakeTx') {
+            distribution.revokedStake = distribution.revokedStake.add(value);
+            return;
+          }
+          // stake tokens + stake revokedTokens  -->>  account tokens
+          if (type === 'SlashStakeTx') {
+            const beforeState = context.stateSnapshot[address];
+            const afterState = context.stakeState;
+            if (!beforeState || !afterState) {
+              throw new Error('INVALID_TOKEN_DISTRIBUTION', 'stake state is missing on slash stake tx');
+            }
+            const revokeTokenDiff = new BN(beforeState.revokedTokens[tokenAddress]).sub(
+              new BN(afterState.revokedTokens[tokenAddress])
+            );
+            const tokenDiff = new BN(beforeState.tokens[tokenAddress]).sub(new BN(afterState.tokens[tokenAddress]));
+
+            distribution.revokedStake = distribution.revokedStake.sub(revokeTokenDiff);
+            if (isGasStake) {
+              distribution.gasStake = distribution.gasStake.sub(tokenDiff);
+            } else {
+              distribution.stake = distribution.stake.sub(tokenDiff);
+            }
+            return;
+          }
+          // gasStake tokens -->> account tokens
+          if (isGasStake) {
+            distribution.gasStake = distribution.gasStake.add(value);
+            return;
+          }
+        }
+        distribution.stake = distribution.stake.add(value);
+      }
+      // Gas
+      else if (change.action === 'gas' && value.gt(ZERO)) {
+        distribution.gas = distribution.gas.add(value);
+      }
+      // Fee
+      else if (change.action === 'fee' && value.gt(ZERO)) {
+        distribution.fee = distribution.fee.add(value);
+      }
+      // SlashVault
+      else if (
+        type === 'SlashStakeTx' &&
+        value.gt(ZERO) &&
+        isSameDid(address, this.resolver.config.vaults.slashedStake)
+      ) {
+        distribution.slashedVault = distribution.slashedVault.add(value);
+      }
+      // Others: Account
+      else {
+        distribution.account = distribution.account.add(value);
+      }
+    });
+
+    await Promise.all(handlePromises);
+
+    if (isHandleStake && type === 'RevokeStakeTx') {
+      // RevokedStake has no receipts
+      // stake tokens -->> stake revokedTokens
+      const { address } = tx.tx.itxJson;
+      const stakeState = await this.getStakeState(address, context);
+      const isGasStake = this.isGasStake(stakeState);
+
+      tx.tx.itxJson.outputs.forEach((x) => {
+        x.tokens
+          .filter((change) => isSameDid(change.address, tokenAddress))
+          .forEach((change) => {
+            const value = new BN(change.value);
+            distribution.revokedStake = distribution.revokedStake.add(value);
+            if (isGasStake) {
+              distribution.gasStake = distribution.gasStake.sub(value);
+            } else {
+              distribution.stake = distribution.stake.sub(value);
+            }
+          });
+      });
+    }
+
+    distribution.txTime = tx.time;
+
+    return distribution;
+  }
+
+  handleModerator(distribution) {
+    const { config } = this.resolver;
+
+    if (distribution.tokenAddress !== config.token.address) {
+      return distribution;
+    }
+
+    const value = config.accounts
+      .filter((account) => account.balance > 0)
+      .map((account) => new BN(account.balance))
+      .reduce((cur, balance) => cur.add(balance), ZERO);
+
+    distribution.account = distribution.account.add(fromTokenToUnit(value.toString()));
+    return distribution;
+  }
+
+  async getStakeState(address, ctx) {
+    const stakeState =
+      ctx?.stateSnapshot?.[address] ||
+      (await this.resolver.runAsLambda((txn) => this.resolver.statedb.stake.get(address, { txn })));
+
+    return stakeState;
+  }
+
+  isGasStake(stakeState) {
+    return isGasStakeAddress(stakeState.sender, stakeState.address) && stakeState.message === 'stake-for-gas';
+  }
+}
+
+module.exports = { TokenDistributionManager };

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.19.1",
+  "version": "1.19.3",
   "description": "GraphQL resolver built upon ocap statedb and GQL layer",
   "main": "lib/index.js",
   "files": [
@@ -22,18 +22,18 @@
     "jest": "^29.7.0"
   },
   "dependencies": {
-    "@arcblock/did": "1.19.1",
-    "@arcblock/did-util": "1.19.1",
-    "@arcblock/validator": "1.19.1",
-    "@ocap/config": "1.19.1",
-    "@ocap/indexdb": "1.19.1",
-    "@ocap/mcrypto": "1.19.1",
-    "@ocap/message": "1.19.1",
-    "@ocap/state": "1.19.1",
-    "@ocap/tx-protocols": "1.19.1",
-    "@ocap/util": "1.19.1",
+    "@arcblock/did": "1.19.3",
+    "@arcblock/did-util": "1.19.3",
+    "@arcblock/validator": "1.19.3",
+    "@ocap/config": "1.19.3",
+    "@ocap/indexdb": "1.19.3",
+    "@ocap/mcrypto": "1.19.3",
+    "@ocap/message": "1.19.3",
+    "@ocap/state": "1.19.3",
+    "@ocap/tx-protocols": "1.19.3",
+    "@ocap/util": "1.19.3",
     "debug": "^4.3.6",
     "lodash": "^4.17.21"
   },
-  "gitHead": "21184488172c6c824ebd1714f728ff2aee4a3ac0"
+  "gitHead": "756076dad0df7468beecc95c8effd55f8c4c4f49"
 }