@cumulus/es-client 13.4.0 → 14.1.0

package/README.md

@@ -8,6 +8,13 @@ Utilities for working with Elasticsearch.
 npm install @cumulus/es-client
 ```
 
+## Notes
+
+### Sorting
+
+It is possible to sort the Elastic Search results by specifying `sort_by`,  `sort_key`, and `order` params.
+If not provided, the default sorting will be applied (`{ timestamp: { order: 'desc' } }` as of this writing).
+
 ## About Cumulus
 
 Cumulus is a cloud-based data ingest, archive, distribution and management prototype for NASA's

package/esDefaults.js

@@ -1,7 +1,7 @@
 const defaultESScrollSize = 1000;
 const defaultESScrollDuration = '6m';
 
-const defaultSortParams = [{ timestamp: 'desc' }, { _uid: 'asc' }];
+const defaultSortParams = [{ timestamp: { order: 'desc' } }];
 
 module.exports = {
   defaultESScrollSize,

package/indexer.js

@@ -1,3 +1,5 @@
+// @ts-check
+
 /* functions for transforming and indexing Cumulus Payloads
  * in Elasticsearch. These functions are specifically designed
  * to transform data for use in cumulus api
@@ -11,11 +13,13 @@
 'use strict';
 
 const cloneDeep = require('lodash/cloneDeep');
+const isEqual = require('lodash/isEqual');
 
 const Logger = require('@cumulus/logger');
 const { inTestMode } = require('@cumulus/common/test-utils');
-const { IndexExistsError } = require('@cumulus/errors');
+const { IndexExistsError, ValidationError } = require('@cumulus/errors');
 const { constructCollectionId } = require('@cumulus/message/Collections');
+const { removeNilProperties } = require('@cumulus/common/util');
 
 const { Search, defaultIndexAlias } = require('./search');
 const mappings = require('./config/mappings.json');
@@ -284,15 +288,27 @@ async function indexGranule(esClient, payload, index = defaultIndexAlias, type =
   );
 }
 
+const granuleInvalidNullFields = [
+  'granuleId',
+  'collectionId',
+  'status',
+  'updatedAt',
+  'execution',
+  'createdAt',
+];
+
 /**
  * Upserts a granule in Elasticsearch
  *
  * @param {Object} params
- * @param {Object} params.esClient - Elasticsearch Connection object
- * @param {Object} params.updates  - Updates to make
- * @param {string} params.index    - Elasticsearch index alias (default defined in search.js)
- * @param {string} params.type     - Elasticsearch type (default: granule)
- * @param {string} [params.refresh] - whether to refresh the index on update or not
+ * @param {Object} params.esClient          - Elasticsearch Connection object
+ * @param {Object} params.updates           - Updates to make
+ * @param {string | undefined} params.index - Elasticsearch index alias
+ *                                            (default defined in search.js)
+ * @param {string} [params.type]              - Elasticsearch type (default: granule)
+ * @param {string} [params.refresh]         - whether to refresh the index on update or not
+ * @param {boolean} writeConstraints      - boolean toggle restricting if conditionals should
+ *                                          be used to determine write eligibility
  * @returns {Promise} Elasticsearch response
  */
 async function upsertGranule({
@@ -301,7 +317,12 @@ async function upsertGranule({
   index = defaultIndexAlias,
   type = 'granule',
   refresh,
-}) {
+}, writeConstraints = true) {
+  Object.keys(updates).forEach((key) => {
+    if (updates[key] === null && granuleInvalidNullFields.includes(key)) {
+      throw new ValidationError(`Attempted DynamoDb write with invalid key ${key} set to null.  Please remove or change this field and retry`);
+    }
+  });
   // If the granule exists in 'deletedgranule', delete it first before inserting the granule
   // into ES.  Ignore 404 error, so the deletion still succeeds if the record doesn't exist.
   const delGranParams = {
@@ -313,7 +334,50 @@ async function upsertGranule({
   };
   await esClient.delete(delGranParams, { ignore: [404] });
 
-  const upsertDoc = updates;
+  // Remove nils in case there isn't a collision
+  const upsertDoc = removeNilProperties(updates);
+  let removeString = '';
+
+  // Set field removal for null values
+  Object.entries(updates).forEach(([fieldName, value]) => {
+    // File removal is a special case as null gets set to []
+    if (fieldName === 'files' && isEqual(value, [])) {
+      removeString += `ctx._source.remove('${fieldName}'); `;
+      delete upsertDoc.files; // Remove files in case this is not a scripted upsert
+    }
+    if (value === null) {
+      removeString += `ctx._source.remove('${fieldName}'); `;
+    }
+  });
+
+  let inlineDocWriteString = 'ctx._source.putAll(params.doc);';
+  if (removeString !== '') {
+    inlineDocWriteString += removeString;
+  }
+  let inline = inlineDocWriteString;
+
+  if (writeConstraints === true) {
+    // Because both API write and message write chains use the granule model to store records, in
+    // cases where createdAt does not exist on the granule, we assume overwrite protections are
+    // undesired behavior via business logic on the message write logic
+    inline = `
+    if ((ctx._source.createdAt === null || params.doc.createdAt >= ctx._source.createdAt)
+      && ((params.doc.status != 'running' && params.doc.status != 'queued') || ((params.doc.status == 'running' || params.doc.status == 'queued') && params.doc.execution != ctx._source.execution))) {
+      ${inlineDocWriteString}
+    } else {
+      ctx.op = 'none';
+    }
+    `;
+    if (!updates.createdAt) {
+      inline = `
+        if ((params.doc.status != 'running' && params.doc.status != 'queued') || ((params.doc.status == 'running' || params.doc.status == 'queued') && params.doc.execution != ctx._source.execution)) {
+          ${inlineDocWriteString}
+        } else {
+        ctx.op = 'none';
+      }
+      `;
+    }
+  }
 
   return await esClient.update({
     index,
@@ -323,14 +387,7 @@ async function upsertGranule({
     body: {
       script: {
         lang: 'painless',
-        inline: `
-          if ((ctx._source.createdAt === null || params.doc.createdAt >= ctx._source.createdAt)
-            && (params.doc.status != 'running' || (params.doc.status == 'running' && params.doc.execution != ctx._source.execution))) {
-            ctx._source.putAll(params.doc);
-          } else {
-            ctx.op = 'none';
-          }
-        `,
+        inline,
         params: {
           doc: upsertDoc,
         },
@@ -709,4 +766,5 @@ module.exports = {
   deleteGranule,
   deleteExecution,
   deleteReconciliationReport,
+  granuleInvalidNullFields,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/es-client",
-  "version": "13.4.0",
+  "version": "14.1.0",
   "description": "Utilities for working with Elasticsearch",
   "keywords": [
     "CUMULUS",
@@ -30,10 +30,10 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/common": "13.4.0",
-    "@cumulus/errors": "13.4.0",
-    "@cumulus/logger": "13.4.0",
-    "@cumulus/message": "13.4.0",
+    "@cumulus/common": "14.1.0",
+    "@cumulus/errors": "14.1.0",
+    "@cumulus/logger": "14.1.0",
+    "@cumulus/message": "14.1.0",
     "@elastic/elasticsearch": "^5.6.20",
     "aws-elasticsearch-connector": "8.2.0",
     "aws-sdk": "^2.585.0",
@@ -42,9 +42,9 @@
     "p-limit": "^1.2.0"
   },
   "devDependencies": {
-    "@cumulus/aws-client": "13.4.0",
-    "@cumulus/test-data": "13.4.0",
+    "@cumulus/aws-client": "14.1.0",
+    "@cumulus/test-data": "14.1.0",
     "p-each-series": "^2.1.0"
   },
-  "gitHead": "cd4377ccf97a1daba19775fb95f9f7fa0f2a29f3"
+  "gitHead": "d97b5b37913944c0f0ecf958f2a567ec3714816c"
 }