@cumulus/es-client 13.1.0 → 13.3.0

package/esDefaults.js

@@ -1,7 +1,10 @@
 const defaultESScrollSize = 1000;
 const defaultESScrollDuration = '6m';
 
+const defaultSortParams = [{ timestamp: 'desc' }, { _uid: 'asc' }];
+
 module.exports = {
   defaultESScrollSize,
   defaultESScrollDuration,
+  defaultSortParams,
 };

package/esSearchAfter.js

@@ -0,0 +1,57 @@
+const { Search } = require('./search');
+
+/**
+ * ES search_after wrapper class for Search.
+ *
+ * This class just allows a user to access the search_after api of elasticsearch for
+ * long lived requests that require multiple server requests.
+ *
+ * Use: Instantiate a class with the queryStringParameters, type, and index.
+ * and call the object.query() until no more results are returned.
+ */
+class ESSearchAfter extends Search {
+  /**
+   * Build search params for search-after API.
+   *
+   * @returns {Object} ES search params
+   */
+  _buildSearch() {
+    const params = super._buildSearch();
+    delete params.from;
+    delete params.to;
+
+    if (this.params.searchContext) {
+      // express seems to decode the URI params for us so we don't need to call decodeURIComponent
+      params.body.search_after = JSON.parse(this.params.searchContext);
+    }
+    return params;
+  }
+
+  /**
+   * Query ES search-after API.
+   *
+   * @returns {Promise<Object>} Object containing query meta and results
+   */
+  async query() {
+    if (!this.client) {
+      this.client = await super.constructor.es();
+    }
+    const searchParams = this._buildSearch();
+    const response = await this.client.search(searchParams);
+
+    const hits = response.body.hits.hits;
+    const meta = this._metaTemplate();
+
+    meta.count = response.body.hits.total;
+    meta.page = this.page;
+    if (hits.length > 0) {
+      meta.searchContext = encodeURIComponent(JSON.stringify(hits[hits.length - 1].sort));
+    }
+    return {
+      meta,
+      results: hits.map((s) => s._source),
+    };
+  }
+}
+
+module.exports = ESSearchAfter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/es-client",
-  "version": "13.1.0",
+  "version": "13.3.0",
   "description": "Utilities for working with Elasticsearch",
   "keywords": [
     "CUMULUS",
@@ -30,10 +30,10 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/common": "13.1.0",
-    "@cumulus/errors": "13.1.0",
-    "@cumulus/logger": "13.1.0",
-    "@cumulus/message": "13.1.0",
+    "@cumulus/common": "13.3.0",
+    "@cumulus/errors": "13.3.0",
+    "@cumulus/logger": "13.3.0",
+    "@cumulus/message": "13.3.0",
     "@elastic/elasticsearch": "^5.6.20",
     "aws-elasticsearch-connector": "8.2.0",
     "aws-sdk": "^2.585.0",
@@ -42,9 +42,9 @@
     "p-limit": "^1.2.0"
   },
   "devDependencies": {
-    "@cumulus/aws-client": "13.1.0",
-    "@cumulus/test-data": "13.1.0",
+    "@cumulus/aws-client": "13.3.0",
+    "@cumulus/test-data": "13.3.0",
     "p-each-series": "^2.1.0"
   },
-  "gitHead": "1966c485551680a65e665ecb30c05a8fadad0d08"
+  "gitHead": "eedae36804f1b1e753161a3e7eb105dacd40d889"
 }

package/queries.js

@@ -9,6 +9,7 @@
 
 const omit = require('lodash/omit');
 
+const { defaultSortParams } = require('./esDefaults');
 const { convertTextField } = require('./textFields');
 
 const regexes = {
@@ -57,7 +58,7 @@ const build = {
         return { [sortField]: { order: key.startsWith('-') ? 'desc' : 'asc' } };
       });
     } else {
-      sort = [{ timestamp: { order: 'desc' } }];
+      sort = defaultSortParams;
     }
 
     return sort;
@@ -226,6 +227,7 @@ module.exports = function query(params) {
       'prefix',
       'infix',
       'fields',
+      'searchContext',
     ]
   );
 

package/search.js

@@ -327,20 +327,22 @@ class BaseSearch {
       if (!this.client) {
         this.client = await this.constructor.es(null, this.metrics);
       }
-      const result = await this.client.search(searchParams);
-      const response = result.body.hits.hits.map((s) => s._source);
+      const response = await this.client.search(searchParams);
+      const hits = response.body.hits.hits;
 
       const meta = this._metaTemplate();
       meta.limit = this.size;
       meta.page = this.page;
-      meta.count = result.body.hits.total;
+      meta.count = response.body.hits.total;
+      if (hits.length > 0) {
+        meta.searchContext = encodeURIComponent(JSON.stringify(hits[hits.length - 1].sort));
+      }
 
       return {
         meta,
-        results: response,
+        results: hits.map((s) => s._source),
       };
     } catch (error) {
-      //log.error(e, logDetails);
       return error;
     }
   }