@ocap/indexdb-elasticsearch 1.18.37 → 1.18.38

package/lib/db.js

@@ -2,7 +2,6 @@ const axios = require('axios');
 const { BaseIndexDB } = require('@ocap/indexdb');
 const { Client: ESClient } = require('@elastic/elasticsearch');
 const { formatPagination } = require('@ocap/indexdb/lib/util');
-const { parseDateTime } = require('@ocap/indexdb/lib/util');
 const { formatTxAfterRead } = require('@ocap/indexdb/lib/util');
 const { DEFAULT_TOKEN_DECIMAL } = require('@ocap/util/lib/constant');
 const { toChecksumAddress } = require('@arcblock/did/lib/type');
@@ -21,12 +20,13 @@ const RollupBlock = require('./table/rollup-block');
 const RollupValidator = require('./table/rollup-validator');
 
 const { name, version } = require('../package.json');
-const { formatQueryResult, getAuthHeaders } = require('./util');
+const { formatQueryResult, getAuthHeaders, getTimeCondition } = require('./util');
 
 // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-all-query.html
 const FALLBACK_QUERY = { match_all: { boost: 1.0 } };
 const REQUEST_TIMEOUT = 8000;
 const MAX_REQUEST_FACTORY_ADDRESS_SIZE = 100;
+const TRACK_TOTAL_HITS = +process.env.TRACK_TOTAL_HITS || 50000;
 
 class ESIndexDB extends BaseIndexDB {
   /**
@@ -114,9 +114,6 @@ class ESIndexDB extends BaseIndexDB {
     const { txs = [] } = txFilter;
     const { rollups = [] } = rollupFilter;
     const { stakes = [] } = stakeFilter;
-    let { startDateTime, endDateTime } = timeFilter;
-    startDateTime = parseDateTime(startDateTime);
-    endDateTime = parseDateTime(endDateTime);
 
     // OR is spelled should
     // AND is spelled must
@@ -185,12 +182,9 @@ class ESIndexDB extends BaseIndexDB {
       conditions.push({ terms: { stakes } });
     }
 
-    if (startDateTime && endDateTime) {
-      conditions.push({ range: { time: { gt: startDateTime, lte: endDateTime } } });
-    } else if (startDateTime) {
-      conditions.push({ range: { time: { gt: startDateTime } } });
-    } else if (endDateTime) {
-      conditions.push({ range: { time: { lte: endDateTime } } });
+    const condition = getTimeCondition(timeFilter, ['time'], 'time');
+    if (condition) {
+      conditions.push(condition);
     }
 
     if (validityFilter.validity && validityFilter.validity === 'VALID') {
@@ -206,6 +200,7 @@ class ESIndexDB extends BaseIndexDB {
       from: pagination.cursor,
       size: pagination.size,
       body: {
+        track_total_hits: TRACK_TOTAL_HITS,
         query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
         sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
         _source: { exclude: ['tx.itxJson.data', 'tx.itxJson.encoded_value'] }, // required to make api fast
@@ -224,24 +219,13 @@ class ESIndexDB extends BaseIndexDB {
     if (factoryAddress) {
       conditions.push({ term: { parent: factoryAddress } });
     }
-    if (conditions.length === 0) {
-      return { assets: [], account: null };
+    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime', 'consumedTime']);
+    if (condition) {
+      conditions.push(condition);
     }
 
-    // eslint-disable-next-line prefer-const
-    let { startDateTime, endDateTime, field = 'renaissanceTime' } = timeFilter;
-    if (['genesisTime', 'renaissanceTime', 'consumedTime'].includes(field) === false) {
-      throw new Error('invalid field specified in timeFilter');
-    }
-
-    startDateTime = parseDateTime(startDateTime);
-    endDateTime = parseDateTime(endDateTime);
-    if (startDateTime && endDateTime) {
-      conditions.push({ range: { [field]: { gt: startDateTime, lte: endDateTime } } });
-    } else if (startDateTime) {
-      conditions.push({ range: { [field]: { gt: startDateTime } } });
-    } else if (endDateTime) {
-      conditions.push({ range: { [field]: { lte: endDateTime } } });
+    if (conditions.length === 0) {
+      return { assets: [], account: null };
     }
 
     const pagination = formatPagination({ paging, defaultSortField: 'renaissanceTime' });
@@ -251,6 +235,7 @@ class ESIndexDB extends BaseIndexDB {
         from: pagination.cursor,
         size: pagination.size,
         body: {
+          track_total_hits: TRACK_TOTAL_HITS,
           query: { bool: { must: conditions } },
           sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
         },
@@ -262,8 +247,7 @@ class ESIndexDB extends BaseIndexDB {
     return { assets: items, account, paging: nextPaging };
   }
 
-  async listTopAccounts({ tokenAddress, paging } = {}) {
-    const filter = { term: { 'tokens.address': tokenAddress } };
+  async listTopAccounts({ tokenAddress, paging, timeFilter = {} } = {}) {
     const pagination = formatPagination({
       paging,
       defaultSortField: 'balance',
@@ -272,6 +256,10 @@ class ESIndexDB extends BaseIndexDB {
 
     let sort = null;
     if (pagination.order.field === 'balance') {
+      if (!tokenAddress) {
+        throw new Error('tokenAddress is required when order.field is balance');
+      }
+
       sort = [
         {
           'tokens.balance': {
@@ -279,7 +267,7 @@ class ESIndexDB extends BaseIndexDB {
             order: pagination.order.type,
             nested: {
               path: 'tokens',
-              filter,
+              filter: { term: { 'tokens.address': tokenAddress } },
             },
           },
         },
@@ -294,18 +282,32 @@ class ESIndexDB extends BaseIndexDB {
       ];
     }
 
+    const conditions = [];
+    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
+    if (condition) {
+      conditions.push(condition);
+    }
+    if (tokenAddress) {
+      conditions.push({
+        nested: {
+          path: 'tokens',
+          query: {
+            bool: {
+              must: [{ term: { 'tokens.address': tokenAddress } }],
+            },
+          },
+        },
+      });
+    }
+
     debug('listTopAccounts', { tokenAddress, paging, sort });
 
     const result = await this.account.search({
       from: pagination.cursor,
       size: pagination.size,
       body: {
-        query: {
-          nested: {
-            path: 'tokens',
-            query: { bool: { filter } },
-          },
-        },
+        track_total_hits: TRACK_TOTAL_HITS,
+        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
         sort,
       },
     });
@@ -324,9 +326,20 @@ class ESIndexDB extends BaseIndexDB {
     return { accounts: items.map((x) => Account.formatAfterRead(x)), paging: nextPaging };
   }
 
-  async listTokens({ issuerAddress, paging } = {}) {
+  async listTokens({ issuerAddress, paging, timeFilter = {} } = {}) {
     debug('listTokens', { issuerAddress, paging });
 
+    const conditions = [];
+
+    if (issuerAddress) {
+      conditions.push({ term: { issuer: issuerAddress } });
+    }
+
+    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
+    if (condition) {
+      conditions.push(condition);
+    }
+
     const pagination = formatPagination({
       paging,
       defaultSortField: 'renaissanceTime',
@@ -337,7 +350,8 @@ class ESIndexDB extends BaseIndexDB {
       from: pagination.cursor,
       size: pagination.size,
       body: {
-        query: issuerAddress ? { bool: { must: [{ term: { issuer: issuerAddress } }] } } : FALLBACK_QUERY,
+        track_total_hits: TRACK_TOTAL_HITS,
+        query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
         sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
       },
     });
@@ -346,7 +360,7 @@ class ESIndexDB extends BaseIndexDB {
     return { tokens: items.map((x) => Token.formatAfterRead(x)), paging: nextPaging };
   }
 
-  async listFactories({ ownerAddress, addressList, paging } = {}) {
+  async listFactories({ ownerAddress, addressList, paging, timeFilter = {} } = {}) {
     debug('listTokens', { ownerAddress, paging });
     const pagination = formatPagination({
       paging,
@@ -359,6 +373,11 @@ class ESIndexDB extends BaseIndexDB {
       conditions.push({ term: { owner: ownerAddress } });
     }
 
+    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
+    if (condition) {
+      conditions.push(condition);
+    }
+
     if (Array.isArray(addressList) && addressList.length > 0) {
       if (addressList.length > MAX_REQUEST_FACTORY_ADDRESS_SIZE) {
         throw new Error(`The length of 'addressList' cannot exceed the length of ${MAX_REQUEST_FACTORY_ADDRESS_SIZE}`);
@@ -375,6 +394,7 @@ class ESIndexDB extends BaseIndexDB {
       from: pagination.cursor,
       size: pagination.size,
       body: {
+        track_total_hits: TRACK_TOTAL_HITS,
         query: conditions.length > 0 ? { bool: { must: conditions } } : FALLBACK_QUERY,
         sort: [{ [sortFields[pagination.order.field] || pagination.order.field]: { order: pagination.order.type } }],
       },
@@ -393,10 +413,6 @@ class ESIndexDB extends BaseIndexDB {
 
     const { sender, receiver } = addressFilter;
     const { assets = [] } = assetFilter;
-    let { startDateTime, endDateTime, field } = timeFilter;
-    startDateTime = parseDateTime(startDateTime);
-    endDateTime = parseDateTime(endDateTime);
-    field = ['genesisTime', 'renaissanceTime'].includes(field) ? field : 'renaissanceTime';
 
     // OR is spelled should
     // AND is spelled must
@@ -422,12 +438,9 @@ class ESIndexDB extends BaseIndexDB {
       conditions.push({ terms: { assets } });
     }
 
-    if (startDateTime && endDateTime) {
-      conditions.push({ range: { [field]: { gt: startDateTime, lte: endDateTime } } });
-    } else if (startDateTime) {
-      conditions.push({ range: { [field]: { gt: startDateTime } } });
-    } else if (endDateTime) {
-      conditions.push({ range: { [field]: { lte: endDateTime } } });
+    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
+    if (condition) {
+      conditions.push(condition);
     }
 
     debug('listStakes query conditions', conditions);
@@ -436,6 +449,7 @@ class ESIndexDB extends BaseIndexDB {
       from: pagination.cursor,
       size: pagination.size,
       body: {
+        track_total_hits: TRACK_TOTAL_HITS,
         query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
         sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
         _source: { exclude: [] }, // to make api fast
@@ -446,7 +460,13 @@ class ESIndexDB extends BaseIndexDB {
     return { stakes: items.map((item) => Stake.formatAfterRead(item)), paging: nextPaging };
   }
 
-  async listRollups({ paging, tokenAddress = '', erc20TokenAddress = '', foreignTokenAddress = '' } = {}) {
+  async listRollups({
+    paging,
+    tokenAddress = '',
+    erc20TokenAddress = '',
+    foreignTokenAddress = '',
+    timeFilter = {},
+  } = {}) {
     const pagination = formatPagination({
       paging,
       defaultSortField: 'renaissanceTime',
@@ -457,6 +477,10 @@ class ESIndexDB extends BaseIndexDB {
     if (tokenAddress) {
       conditions.push({ term: { tokenAddress } });
     }
+    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
+    if (condition) {
+      conditions.push(condition);
+    }
     const foreignTokenAddr = foreignTokenAddress || erc20TokenAddress;
     if (foreignTokenAddr) {
       conditions.push({
@@ -480,6 +504,7 @@ class ESIndexDB extends BaseIndexDB {
       from: pagination.cursor,
       size: pagination.size,
       body: {
+        track_total_hits: TRACK_TOTAL_HITS,
         query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
         sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
       },
@@ -503,9 +528,6 @@ class ESIndexDB extends BaseIndexDB {
 
     const { txs = [] } = txFilter;
     const { validators = [] } = validatorFilter;
-    let { startDateTime, endDateTime, field = 'genesisTime' } = timeFilter; // eslint-disable-line
-    startDateTime = parseDateTime(startDateTime);
-    endDateTime = parseDateTime(endDateTime);
 
     const conditions = [];
 
@@ -538,12 +560,9 @@ class ESIndexDB extends BaseIndexDB {
       conditions.push({ terms: { validators } });
     }
 
-    if (startDateTime && endDateTime) {
-      conditions.push({ range: { [field]: { gt: startDateTime, lte: endDateTime } } });
-    } else if (startDateTime) {
-      conditions.push({ range: { [field]: { gt: startDateTime } } });
-    } else if (endDateTime) {
-      conditions.push({ range: { [field]: { lte: endDateTime } } });
+    const condition = getTimeCondition(timeFilter, ['genesisTime', 'renaissanceTime']);
+    if (condition) {
+      conditions.push(condition);
     }
 
     debug('listRollupBlocks query conditions', conditions);
@@ -552,6 +571,7 @@ class ESIndexDB extends BaseIndexDB {
       from: pagination.cursor,
       size: pagination.size,
       body: {
+        track_total_hits: TRACK_TOTAL_HITS,
         query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
         sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
       },
@@ -580,6 +600,7 @@ class ESIndexDB extends BaseIndexDB {
       from: pagination.cursor,
       size: pagination.size,
       body: {
+        track_total_hits: TRACK_TOTAL_HITS,
         query: conditions.length ? { bool: { must: conditions } } : FALLBACK_QUERY,
         sort: [{ [pagination.order.field]: { order: pagination.order.type } }],
       },

package/lib/util.js

@@ -1,4 +1,4 @@
-const { formatNextPagination } = require('@ocap/indexdb/lib/util');
+const { formatNextPagination, parseDateTime } = require('@ocap/indexdb/lib/util');
 
 const getTableName = (prefix, name) => (prefix ? [prefix, name].join('_') : name);
 
@@ -28,4 +28,27 @@ const getAuthHeaders = (auth) => {
   return headers;
 };
 
-module.exports = { getTableName, formatQueryResult, getAuthHeaders };
+const getTimeCondition = (
+  filter = {},
+  supportedFields = ['genesisTime', 'renaissanceTime'],
+  defaultField = 'renaissanceTime'
+) => {
+  let { startDateTime, endDateTime, field } = filter;
+  startDateTime = parseDateTime(startDateTime);
+  endDateTime = parseDateTime(endDateTime);
+  field = supportedFields.includes(field) ? field : defaultField;
+
+  if (startDateTime && endDateTime) {
+    return { range: { [field]: { gt: startDateTime, lte: endDateTime } } };
+  }
+  if (startDateTime) {
+    return { range: { [field]: { gt: startDateTime } } };
+  }
+  if (endDateTime) {
+    return { range: { [field]: { lte: endDateTime } } };
+  }
+
+  return null;
+};
+
+module.exports = { getTableName, formatQueryResult, getAuthHeaders, getTimeCondition };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ocap/indexdb-elasticsearch",
   "description": "OCAP indexdb adapter that uses elasticsearch as backend",
-  "version": "1.18.37",
+  "version": "1.18.38",
   "author": "wangshijun <shijun@arcblock.io> (https://www.arcblock.io)",
   "bugs": {
     "url": "https://github.com/ArcBlock/asset-chain/issues",
@@ -38,12 +38,12 @@
     "test": "jest --forceExit --detectOpenHandles",
     "coverage": "npm run test -- --coverage"
   },
-  "gitHead": "30215e61b8ff7abb8ccd44981cd5de30bb2b1355",
+  "gitHead": "0ea539a3a0e7a13e7a49daaa4d7e842dcee2192d",
   "dependencies": {
-    "@arcblock/did": "1.18.37",
+    "@arcblock/did": "1.18.38",
     "@elastic/elasticsearch": "7.13.0",
-    "@ocap/indexdb": "1.18.37",
-    "@ocap/util": "1.18.37",
+    "@ocap/indexdb": "1.18.38",
+    "@ocap/util": "1.18.38",
     "axios": "^0.25.0",
     "bn.js": "^5.2.1",
     "debug": "^4.3.4",