@cumulus/es-client 15.0.3 → 16.0.2-alpha.0

package/indexer.js

@@ -127,6 +127,14 @@ function updateAsyncOperation(esClient, id, updates, index = defaultIndexAlias,
   return updateExistingRecord(esClient, id, updates, index, type);
 }
 
+const executionInvalidNullFields = [
+  'arn',
+  'name',
+  'status',
+  'updatedAt',
+  'createdAt',
+];
+
 /**
  * Upsert an execution record in Elasticsearch
  *
@@ -136,6 +144,8 @@ function updateAsyncOperation(esClient, id, updates, index = defaultIndexAlias,
  * @param {string} params.index - Elasticsearch index alias (default defined in search.js)
  * @param {string} params.type - Elasticsearch type (default: execution)
  * @param {string} [params.refresh] - whether to refresh the index on update or not
+ * @param {boolean}   writeConstraints       - boolean toggle restricting if conditionals should
+ *                                          be used to determine write eligibility
  * @returns {Promise} elasticsearch update response
  */
 async function upsertExecution({
@@ -144,11 +154,42 @@ async function upsertExecution({
   index = defaultIndexAlias,
   type = 'execution',
   refresh,
-}) {
+}, writeConstraints = true) {
+  Object.keys(updates).forEach((key) => {
+    if (updates[key] === null && executionInvalidNullFields.includes(key)) {
+      throw new ValidationError(`Attempted Elasticsearch write with invalid key ${key} set to null.  Please remove or change this field and retry`);
+    }
+  });
+
   const upsertDoc = {
-    ...updates,
-    timestamp: Date.now(),
+    ...removeNilProperties(updates),
+    timestamp: updates.timestamp || Date.now(),
   };
+
+  let removeString = '';
+  // Set field removal for null values
+  Object.entries(updates).forEach(([fieldName, value]) => {
+    if (value === null) {
+      removeString += `ctx._source.remove('${fieldName}'); `;
+    }
+  });
+
+  let inlineDocWriteString = 'ctx._source.putAll(params.doc);';
+  if (removeString !== '') {
+    inlineDocWriteString += removeString;
+  }
+  let inline = inlineDocWriteString;
+  if (writeConstraints === true) {
+    inline = `
+          if (params.doc.status == "running") {
+            ctx._source.updatedAt = params.doc.updatedAt;
+            ctx._source.timestamp = params.doc.timestamp;
+            ctx._source.originalPayload = params.doc.originalPayload;
+          } else {
+            ${inlineDocWriteString}
+          }
+        `;
+  }
   return await esClient.update({
     index,
     type,
@@ -156,15 +197,7 @@ async function upsertExecution({
     body: {
       script: {
         lang: 'painless',
-        inline: `
-          if (params.doc.status == "running") {
-            ctx._source.updatedAt = params.doc.updatedAt;
-            ctx._source.timestamp = params.doc.timestamp;
-            ctx._source.originalPayload = params.doc.originalPayload;
-          } else {
-            ctx._source.putAll(params.doc)
-          }
-        `,
+        inline,
         params: {
           doc: upsertDoc,
         },
@@ -320,7 +353,7 @@ async function upsertGranule({
 }, writeConstraints = true) {
   Object.keys(updates).forEach((key) => {
     if (updates[key] === null && granuleInvalidNullFields.includes(key)) {
-      throw new ValidationError(`Attempted DynamoDb write with invalid key ${key} set to null.  Please remove or change this field and retry`);
+      throw new ValidationError(`Attempted Elasticsearch write with invalid key ${key} set to null.  Please remove or change this field and retry`);
     }
   });
   // If the granule exists in 'deletedgranule', delete it first before inserting the granule
@@ -766,5 +799,6 @@ module.exports = {
   deleteGranule,
   deleteExecution,
   deleteReconciliationReport,
+  executionInvalidNullFields,
   granuleInvalidNullFields,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/es-client",
-  "version": "15.0.3",
+  "version": "16.0.2-alpha.0",
   "description": "Utilities for working with Elasticsearch",
   "keywords": [
     "CUMULUS",
@@ -30,10 +30,10 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/common": "15.0.3",
-    "@cumulus/errors": "15.0.3",
-    "@cumulus/logger": "15.0.3",
-    "@cumulus/message": "15.0.3",
+    "@cumulus/common": "16.0.2-alpha.0",
+    "@cumulus/errors": "16.0.2-alpha.0",
+    "@cumulus/logger": "16.0.2-alpha.0",
+    "@cumulus/message": "16.0.2-alpha.0",
     "@elastic/elasticsearch": "^5.6.20",
     "aws-elasticsearch-connector": "8.2.0",
     "aws-sdk": "^2.585.0",
@@ -42,9 +42,9 @@
     "p-limit": "^1.2.0"
   },
   "devDependencies": {
-    "@cumulus/aws-client": "15.0.3",
-    "@cumulus/test-data": "15.0.3",
+    "@cumulus/aws-client": "16.0.2-alpha.0",
+    "@cumulus/test-data": "16.0.2-alpha.0",
     "p-each-series": "^2.1.0"
   },
-  "gitHead": "5fd5febe740ef96f365d1defc7b9f76ae64c9cc0"
+  "gitHead": "215833a8dbf46eff436099c9d056cf697df9903f"
 }

package/search.js

@@ -23,6 +23,8 @@ const logDetails = {
 };
 
 const defaultIndexAlias = 'cumulus-alias';
+const multipleRecordFoundString = 'More than one record was found!';
+const recordNotFoundString = 'Record not found';
 
 const getCredentials = () =>
   new Promise((resolve, reject) => aws.config.getCredentials((err) => {
@@ -135,12 +137,6 @@ class BaseSearch {
     this.frm = (page - 1) * this.size;
     this.page = Number.parseInt((params.skip) ? params.skip : page, 10);
     this.index = index || defaultIndexAlias;
-
-    if (this.type === process.env.CollectionsTable) {
-      this.hash = 'collectionName';
-    } else if (this.type === process.env.PdrsTable) {
-      this.hash = 'pdrName';
-    }
   }
 
   _buildSearch() {
@@ -234,10 +230,10 @@ class BaseSearch {
     }).then((response) => response.body);
 
     if (result.hits.total > 1) {
-      return { detail: 'More than one record was found!' };
+      return { detail: multipleRecordFoundString };
     }
     if (result.hits.total === 0) {
-      return { detail: 'Record not found' };
+      return { detail: recordNotFoundString };
     }
 
     const resp = result.hits.hits[0]._source;
@@ -247,76 +243,7 @@ class BaseSearch {
 
   async exists(id, parentId) {
     const response = await this.get(id, parentId);
-    return response.detail !== 'Record not found';
-  }
-
-  async granulesStats(key, value) {
-    const body = {
-      query: {
-        term: {
-          [`${key}.keyword`]: value,
-        },
-      },
-      aggs: {
-        statusCount: {
-          terms: {
-            field: 'status.keyword',
-          },
-        },
-        averageDuration: {
-          avg: {
-            field: 'duration',
-          },
-        },
-        granulesCount: {
-          value_count: {
-            field: 'granuleId.keyword',
-          },
-        },
-      },
-    };
-
-    const ag = await this.client.search({
-      index: this.index,
-      type: process.env.GranulesTable,
-      body: body,
-      size: 0,
-    });
-
-    const status = {
-      failed: 0,
-      ingesting: 0,
-      processing: 0,
-      archiving: 0,
-      cmr: 0,
-      completed: 0,
-    };
-
-    const item = ag.body.aggregations;
-
-    const newObj = {
-      averageDuration: item.averageDuration.value,
-      granules: item.granulesCount.value,
-      granulesStatus: { ...status },
-    };
-
-    item.statusCount.buckets.forEach((b) => {
-      newObj.granulesStatus[b.key] = b.doc_count;
-    });
-
-    if (newObj.granules > 0) {
-      newObj.progress = (
-        (
-          (newObj.granulesStatus.completed + newObj.granulesStatus.failed)
-          / newObj.granules
-        )
-        * 100
-      );
-    } else {
-      newObj.progress = 0;
-    }
-
-    return newObj;
+    return response.detail !== recordNotFoundString;
   }
 
   async query(searchParamsOverride) {
@@ -378,5 +305,7 @@ module.exports = {
   BaseSearch,
   Search,
   defaultIndexAlias,
+  multipleRecordFoundString,
+  recordNotFoundString,
   getLocalEsHost,
 };