@ocap/resolver 1.19.4 → 1.19.6

package/lib/index.js

@@ -1046,11 +1046,11 @@ module.exports = class OCAPResolver {
     if (limit && paging.total > limit) {
       throw new CustomError('EXCEED_LIMIT', `Total exceeds limit ${limit}`);
     }
-    if (paging.total < pageSize) {
+    if (paging.total <= pageSize) {
       return firstPage;
     }
 
-    const totalPage = Math.floor(paging.total / pageSize);
+    const totalPage = Math.ceil(paging.total / pageSize) - 1;
     const cursors = new Array(totalPage).fill(true).map((_, i) => (i + 1) * pageSize);
     const step = process.env.RESOLVER_BATCH_CONCURRENCY || 3;
     let results = firstPage;
@@ -1282,23 +1282,15 @@ module.exports = class OCAPResolver {
    * @param {number} [param.concurrency=3] Number of concurrent requests
    * @param {number} [param.chunkSize=2000] Maximum number of items to return in each next() call
    * @param {number} [param.pageSize=100] Number of items per page
-   * @param {Function} param.getTime Function to get timestamp from item
+   * @param {string} param.timeKey Key to access time in response
    * @param {string} param.dataKey Key to access data in response
    * @returns {Promise<Array>} Array of fetched items
    */
-  listChunks(fn, { concurrency = 3, chunkSize = 2000, pageSize = 100, getTime, dataKey }) {
-    let totalPage;
-    let curPage;
+  listChunks(fn, { total, concurrency = 3, chunkSize = 2100, pageSize = 100, timeKey, dataKey }) {
     let done = false;
     let time;
-
-    const fetchFirstPage = async () => {
-      const { paging, page, [dataKey]: list } = await fn({ size: pageSize, cursor: '0' });
-      const total = paging?.total || page?.total;
-      curPage = 1;
-      totalPage = Math.ceil(total / pageSize);
-      return list;
-    };
+    const totalPage = Math.ceil(total / pageSize);
+    let curPage = 0;
 
     const next = async () => {
       if (done) {
@@ -1307,28 +1299,10 @@ module.exports = class OCAPResolver {
 
       let results = [];
 
-      // first page
-      if (!totalPage) {
-        const data = await fetchFirstPage();
-        // only 1 page
-        if (data.length < pageSize) {
-          done = true;
-          return data;
-        }
-
-        time = getTime(data[data.length - 1]);
-        results = results.concat(data);
-
-        // limit
-        if (data.length >= chunkSize) {
-          return results;
-        }
-      }
-
       // next pages
       for (; curPage < totalPage; curPage += concurrency) {
         const batchResults = await Promise.all(
-          new Array(concurrency).fill(true).map(async (_, i) => {
+          new Array(Math.min(concurrency, totalPage - curPage)).fill(true).map(async (_, i) => {
             const { [dataKey]: list } = await fn({
               size: pageSize,
               cursor: i * pageSize,
@@ -1342,11 +1316,11 @@ module.exports = class OCAPResolver {
         // finish
         if (!flatResults.length) {
           done = true;
-          return [];
+          return results;
         }
 
         results = results.concat(flatResults);
-        time = results.length ? getTime(results[results.length - 1]) : null;
+        time = results.length ? results[results.length - 1][timeKey] : null;
 
         // limit
         if (results.length >= chunkSize) {
@@ -1364,7 +1338,7 @@ module.exports = class OCAPResolver {
     };
   }
 
-  listTransactionsChunks(args = {}, { chunkSize = 2000, pageSize = 100 } = {}) {
+  async listTransactionsChunks(args = {}, { chunkSize = 2100, pageSize = 100 } = {}) {
     return this.listChunks(
       ({ size, time, cursor }) =>
         this.listTransactions({
@@ -1379,14 +1353,15 @@ module.exports = class OCAPResolver {
         }),
       {
         dataKey: 'transactions',
-        getTime: (tx) => tx?.time,
+        timeKey: 'time',
+        total: await this.indexdb.tx.count(),
         chunkSize,
         pageSize,
       }
     );
   }
 
-  listStakeChunks(args = {}, { chunkSize = 2000, pageSize = 100 } = {}) {
+  async listStakeChunks(args = {}, { chunkSize = 2100, pageSize = 100 } = {}) {
     return this.listChunks(
       ({ size, time, cursor }) =>
         this.listStakes({
@@ -1401,7 +1376,8 @@ module.exports = class OCAPResolver {
         }),
       {
         dataKey: 'stakes',
-        getTime: (state) => state?.time,
+        timeKey: 'renaissanceTime',
+        total: await this.indexdb.stake.count(),
         chunkSize,
         pageSize,
       }

package/lib/token-distribution.js

@@ -16,6 +16,7 @@ class TokenDistributionManager {
   constructor(resolver) {
     this.resolver = resolver;
     this.indexdb = resolver.indexdb;
+    this.isProcessing = false;
   }
 
   formatDistribution(distribution) {
@@ -40,7 +41,7 @@ class TokenDistributionManager {
     return data && createIndexedTokenDistribution(data);
   }
 
-  async saveDistribution(distribution) {
+  async saveDistribution(distribution, isEnsureLatest = true) {
     const data = createIndexedTokenDistribution(distribution);
     const indexdbDistribution = await this.getDistribution(data.tokenAddress);
 
@@ -48,8 +49,10 @@ class TokenDistributionManager {
       await this.indexdb.tokenDistribution.insert(data);
     } else {
       // ensure txTime is latest
-      const latestTime = Math.max(new Date(indexdbDistribution.txTime).getTime(), new Date(data.txTime).getTime());
-      data.txTime = new Date(latestTime).toISOString();
+      if (isEnsureLatest) {
+        const latestTime = Math.max(new Date(indexdbDistribution.txTime).getTime(), new Date(data.txTime).getTime());
+        data.txTime = new Date(latestTime).toISOString();
+      }
       await this.indexdb.tokenDistribution.update(data.tokenAddress, data);
     }
 
@@ -67,6 +70,11 @@ class TokenDistributionManager {
   async updateByToken(tokenAddress, force) {
     const { logger, config } = this.resolver;
 
+    if (this.isProcessing) {
+      logger?.logger('Token distribution is already in progress', { tokenAddress, force });
+      return;
+    }
+
     const distribution = force
       ? this.formatDistribution({ tokenAddress })
       : this.formatDistribution((await this.getDistribution(tokenAddress)) || { tokenAddress });
@@ -76,64 +84,77 @@ class TokenDistributionManager {
       distribution: createIndexedTokenDistribution(distribution),
     });
 
-    if (force && isDefaultToken) {
-      this.handleModerator(distribution);
-    }
+    this.isProcessing = true;
 
-    const { next } = await this.resolver.listTransactionsChunks({
-      timeFilter: { startDateTime: distribution.txTime },
-      // Default token is used for gas payment and may appear in any transaction, so we cannot filter by tokenFilter
-      tokenFilter: isDefaultToken ? {} : { tokenFilter: { tokens: [tokenAddress] } },
-    });
-    let nextData = await next();
+    try {
+      if (force && isDefaultToken) {
+        this.handleModerator(distribution);
+      }
 
-    // Process transactions in chunks and update indexdb
-    while (nextData.length) {
-      logger?.info('Updating token distribution in chunks', {
-        chunkSize: nextData.length,
-        startTime: nextData[0].time,
-        startHash: nextData[0].hash,
-        endTime: nextData[nextData.length - 1].time,
+      const { next } = await this.resolver.listTransactionsChunks({
+        timeFilter: { startDateTime: distribution.txTime },
+        // Default token is used for gas payment and may appear in any transaction, so we cannot filter by tokenFilter
+        tokenFilter: isDefaultToken ? {} : { tokenFilter: { tokens: [tokenAddress] } },
       });
+      let nextData = await next();
+
+      // Process transactions in chunks and update indexdb
+      while (nextData.length) {
+        logger?.info('Updating token distribution in chunks', {
+          chunkSize: nextData.length,
+          startTime: nextData[0].time,
+          startHash: nextData[0].hash,
+          endTime: nextData[nextData.length - 1].time,
+        });
+
+        const handlePromises = nextData.map((tx) => this.handleTx(tx, distribution));
+        await Promise.all(handlePromises);
+
+        // update indexdb
+        await this.saveDistribution(distribution, false);
+        nextData = await next();
+      }
 
-      const handlePromises = nextData.map((tx) => this.handleTx(tx, distribution));
-      await Promise.all(handlePromises);
-
-      // update indexdb
-      await this.saveDistribution(distribution);
-      nextData = await next();
-    }
-
-    // We cannot distinguish between revokedStake and stake from tx receipts here,
-    // so we need to read all stake transactions and recalculate token distribution based on their revokeTokens and tokens
-    await this.splitStake(distribution, force);
-    await this.saveDistribution(distribution);
-
-    const result = createIndexedTokenDistribution(distribution);
+      // We cannot distinguish between revokedStake and stake from tx receipts here,
+      // so we need to read all stake transactions and recalculate token distribution based on their revokeTokens and tokens
+      await this.splitStake(distribution);
+      await this.saveDistribution(distribution, false);
 
-    logger.info(`Token distribution update completed (${tokenAddress})`, { distribution: result });
+      const result = createIndexedTokenDistribution(distribution);
 
-    return result;
+      logger.info(`Token distribution update completed (${tokenAddress})`, { distribution: result });
+      return result;
+    } catch (e) {
+      logger?.error('Token distribution update failed', { error: e });
+      return null;
+    } finally {
+      this.isProcessing = false;
+    }
   }
 
   /**
    * Split out revokedStake / gasStake from stake
    *
    * @param {Object} distribution
-   * @param {boolean} force If force is false, only process transactions after txTime in indexdb. Default is false
    * @returns {Promise<Object>}
    */
-  async splitStake(distribution, force) {
+  async splitStake(distribution) {
+    const { logger } = this.resolver;
     const { tokenAddress } = distribution;
 
-    const { next } = await this.resolver.listStakeChunks({
-      timeFilter: !force ? { startDateTime: distribution.txTime } : {},
-    });
+    const { next } = await this.resolver.listStakeChunks();
 
     let nextData = await next();
 
     // Process transactions in chunks and update indexdb
     while (nextData.length) {
+      logger?.info('Updating stake distribution in chunks', {
+        chunkSize: nextData.length,
+        startTime: nextData[0].renaissanceTime,
+        startAddress: nextData[0].address,
+        endTime: nextData[nextData.length - 1].renaissanceTime,
+      });
+
       nextData.forEach((stakeState) => {
         const isGasStake = this.isGasStake(stakeState);
         const token = stakeState.tokens.find((x) => x.address === tokenAddress);
@@ -168,6 +189,13 @@ class TokenDistributionManager {
    * @returns {Promise<Object>} The updated token distributions
    */
   async updateByTx(tx, context) {
+    const { logger } = this.resolver;
+
+    if (this.isProcessing) {
+      logger?.logger('Token distribution is already in progress', { tx });
+      return;
+    }
+
     const formattedTx = await this.resolver.formatTx(tx);
     const tokens = formattedTx.tokenSymbols.map(({ address }) => address);
 

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.19.4",
+  "version": "1.19.6",
   "description": "GraphQL resolver built upon ocap statedb and GQL layer",
   "main": "lib/index.js",
   "files": [
@@ -22,18 +22,18 @@
     "jest": "^29.7.0"
   },
   "dependencies": {
-    "@arcblock/did": "1.19.4",
-    "@arcblock/did-util": "1.19.4",
-    "@arcblock/validator": "1.19.4",
-    "@ocap/config": "1.19.4",
-    "@ocap/indexdb": "1.19.4",
-    "@ocap/mcrypto": "1.19.4",
-    "@ocap/message": "1.19.4",
-    "@ocap/state": "1.19.4",
-    "@ocap/tx-protocols": "1.19.4",
-    "@ocap/util": "1.19.4",
+    "@arcblock/did": "1.19.6",
+    "@arcblock/did-util": "1.19.6",
+    "@arcblock/validator": "1.19.6",
+    "@ocap/config": "1.19.6",
+    "@ocap/indexdb": "1.19.6",
+    "@ocap/mcrypto": "1.19.6",
+    "@ocap/message": "1.19.6",
+    "@ocap/state": "1.19.6",
+    "@ocap/tx-protocols": "1.19.6",
+    "@ocap/util": "1.19.6",
     "debug": "^4.3.6",
     "lodash": "^4.17.21"
   },
-  "gitHead": "9b026c7ad59839ac1242a68b24e18b3a1534918d"
+  "gitHead": "a884f23e66c8b498f2bf8d1efbbb3b0d024f4a9d"
 }