@cumulus/es-client 10.1.2 → 11.0.0

package/indexer.js

@@ -1,5 +1,5 @@
 /* functions for transforming and indexing Cumulus Payloads
- * in ElasticSearch. These functions are specifically designed
+ * in Elasticsearch. These functions are specifically designed
  * to transform data for use in cumulus api
  *
  * The module accepts the following kinds of workflows (state machines):
@@ -46,9 +46,9 @@ async function createIndex(esClient, indexName) {
 }
 
 /**
- * Indexes a given record to the specified ElasticSearch index and type
+ * Indexes a given record to the specified Elasticsearch index and type
  *
- * @param  {Object} esClient - ElasticSearch Connection object
+ * @param  {Object} esClient - Elasticsearch Connection object
  * @param  {string} id       - the record id
  * @param  {Object} doc      - the record
  * @param  {string} index    - Elasticsearch index alias
@@ -84,11 +84,99 @@ async function genericRecordUpdate(esClient, id, doc, index, type, parent) {
   return indexResponse.body;
 }
 
+/**
+ * Updates a given record for the Elasticsearch index and type
+ *
+ * @param  {Object} esClient - Elasticsearch Connection object
+ * @param  {string} id       - the record id
+ * @param  {Object} doc      - the record
+ * @param  {string} index    - Elasticsearch index alias
+ * @param  {string} type     - Elasticsearch type
+ * @returns {Promise} Elasticsearch response
+ */
+async function updateExistingRecord(esClient, id, doc, index, type) {
+  return await esClient.update({
+    index,
+    type,
+    id,
+    body: {
+      doc: {
+        ...doc,
+        timestamp: Date.now(),
+      },
+    },
+    refresh: inTestMode(),
+  });
+}
+
+/**
+ * Updates an asyncOperation record in Elasticsearch
+ *
+ * @param  {Object} esClient - Elasticsearch Connection object
+ * @param  {Object} id - Record ID
+ * @param  {Object} updates - Document of updates to apply
+ * @param  {string} index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} type - Elasticsearch type (default: asyncOperation)
+ * @returns {Promise} elasticsearch update response
+ */
+function updateAsyncOperation(esClient, id, updates, index = defaultIndexAlias, type = 'asyncOperation') {
+  return updateExistingRecord(esClient, id, updates, index, type);
+}
+
+/**
+ * Upsert an execution record in Elasticsearch
+ *
+ * @param {Object} params
+ * @param {Object} params.esClient - Elasticsearch Connection object
+ * @param {Object} params.updates - Document of updates to apply
+ * @param {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param {string} params.type - Elasticsearch type (default: execution)
+ * @param {string} [params.refresh] - whether to refresh the index on update or not
+ * @returns {Promise} elasticsearch update response
+ */
+async function upsertExecution({
+  esClient,
+  updates,
+  index = defaultIndexAlias,
+  type = 'execution',
+  refresh,
+}) {
+  const upsertDoc = {
+    ...updates,
+    timestamp: Date.now(),
+  };
+  return await esClient.update({
+    index,
+    type,
+    id: upsertDoc.arn,
+    body: {
+      script: {
+        lang: 'painless',
+        inline: `
+          if (params.doc.status == "running") {
+            ctx._source.updatedAt = params.doc.updatedAt;
+            ctx._source.timestamp = params.doc.timestamp;
+            ctx._source.originalPayload = params.doc.originalPayload;
+          } else {
+            ctx._source.putAll(params.doc)
+          }
+        `,
+        params: {
+          doc: upsertDoc,
+        },
+      },
+      upsert: upsertDoc,
+    },
+    refresh: refresh !== undefined ? refresh : inTestMode(),
+    retry_on_conflict: 3,
+  });
+}
+
 /**
  * Indexes a step function message to Elastic Search. The message must
  * comply with the cumulus message protocol
  *
- * @param  {Object} esClient - ElasticSearch Connection object
+ * @param  {Object} esClient - Elasticsearch Connection object
  * @param  {Object} payload  - Cumulus Step Function message
  * @param  {string} index    - Elasticsearch index alias (default defined in search.js)
  * @param  {string} type     - Elasticsearch type (default: execution)
@@ -99,9 +187,9 @@ function indexExecution(esClient, payload, index = defaultIndexAlias, type = 'ex
 }
 
 /**
- * Indexes the asyncOperation type on ElasticSearch
+ * Indexes the asyncOperation type on Elasticsearch
  *
- * @param  {Object} esClient - ElasticSearch Connection object
+ * @param  {Object} esClient - Elasticsearch Connection object
  * @param  {Object} payload  - Cumulus Step Function message
  * @param  {string} index    - Elasticsearch index alias (default defined in search.js)
  * @param  {string} type     - Elasticsearch type (default: asyncOperation)
@@ -112,23 +200,23 @@ function indexAsyncOperation(esClient, payload, index = defaultIndexAlias, type
 }
 
 /**
- * Indexes the collection on ElasticSearch
+ * Indexes the collection on Elasticsearch
  *
- * @param  {Object} esClient - ElasticSearch Connection object
- * @param  {Object} meta     - the collection record
- * @param  {string} index    - Elasticsearch index alias (default defined in search.js)
- * @param  {string} type     - Elasticsearch type (default: collection)
+ * @param  {Object} esClient - Elasticsearch Connection object
+ * @param  {Object} collection - the collection record
+ * @param  {string} index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} type - Elasticsearch type (default: collection)
  * @returns {Promise} Elasticsearch response
  */
-function indexCollection(esClient, meta, index = defaultIndexAlias, type = 'collection') {
-  const collectionId = constructCollectionId(meta.name, meta.version);
-  return genericRecordUpdate(esClient, collectionId, meta, index, type);
+function indexCollection(esClient, collection, index = defaultIndexAlias, type = 'collection') {
+  const collectionId = constructCollectionId(collection.name, collection.version);
+  return genericRecordUpdate(esClient, collectionId, collection, index, type);
 }
 
 /**
- * Indexes the provider type on ElasticSearch
+ * Indexes the provider type on Elasticsearch
  *
- * @param  {Object} esClient - ElasticSearch Connection object
+ * @param  {Object} esClient - Elasticsearch Connection object
  * @param  {Object} payload  - the provider record
  * @param  {string} index    - Elasticsearch index alias (default defined in search.js)
  * @param  {string} type     - Elasticsearch type (default: provider)
@@ -139,9 +227,9 @@ function indexProvider(esClient, payload, index = defaultIndexAlias, type = 'pro
 }
 
 /**
- * Indexes the reconciliationReport type on ElasticSearch
+ * Indexes the reconciliationReport type on Elasticsearch
  *
- * @param  {Object} esClient - ElasticSearch Connection object
+ * @param  {Object} esClient - Elasticsearch Connection object
  * @param  {Object} payload  - the ReconciliationReport record
  * @param  {string} index    - Elasticsearch index alias (default defined in search.js)
  * @param  {string} type     - Elasticsearch type (default: reconciliationReport)
@@ -152,9 +240,9 @@ function indexReconciliationReport(esClient, payload, index = defaultIndexAlias,
 }
 
 /**
- * Indexes the rule type on ElasticSearch
+ * Indexes the rule type on Elasticsearch
  *
- * @param  {Object} esClient - ElasticSearch Connection object
+ * @param  {Object} esClient - Elasticsearch Connection object
  * @param  {Object} payload  - the Rule record
  * @param  {string} index    - Elasticsearch index alias (default defined in search.js)
  * @param  {string} type     - Elasticsearch type (default: rule)
@@ -166,9 +254,9 @@ function indexRule(esClient, payload, index = defaultIndexAlias, type = 'rule')
 }
 
 /**
- * Indexes the granule type on ElasticSearch
+ * Indexes the granule type on Elasticsearch
  *
- * @param  {Object} esClient - ElasticSearch Connection object
+ * @param  {Object} esClient - Elasticsearch Connection object
  * @param  {Object} payload  - Cumulus Step Function message
  * @param  {string} index    - Elasticsearch index alias (default defined in search.js)
  * @param  {string} type     - Elasticsearch type (default: granule)
@@ -197,9 +285,67 @@ async function indexGranule(esClient, payload, index = defaultIndexAlias, type =
 }
 
 /**
- * Indexes the pdr type on ElasticSearch
+ * Upserts a granule in Elasticsearch
+ *
+ * @param {Object} params
+ * @param {Object} params.esClient - Elasticsearch Connection object
+ * @param {Object} params.updates  - Updates to make
+ * @param {string} params.index    - Elasticsearch index alias (default defined in search.js)
+ * @param {string} params.type     - Elasticsearch type (default: granule)
+ * @param {string} [params.refresh] - whether to refresh the index on update or not
+ * @returns {Promise} Elasticsearch response
+ */
+async function upsertGranule({
+  esClient,
+  updates,
+  index = defaultIndexAlias,
+  type = 'granule',
+  refresh,
+}) {
+  // If the granule exists in 'deletedgranule', delete it first before inserting the granule
+  // into ES.  Ignore 404 error, so the deletion still succeeds if the record doesn't exist.
+  const delGranParams = {
+    index,
+    type: 'deletedgranule',
+    id: updates.granuleId,
+    parent: updates.collectionId,
+    refresh: inTestMode(),
+  };
+  await esClient.delete(delGranParams, { ignore: [404] });
+
+  const upsertDoc = updates;
+
+  return await esClient.update({
+    index,
+    type,
+    id: updates.granuleId,
+    parent: updates.collectionId,
+    body: {
+      script: {
+        lang: 'painless',
+        inline: `
+          if ((ctx._source.createdAt === null || params.doc.createdAt >= ctx._source.createdAt)
+            && (params.doc.status != 'running' || (params.doc.status == 'running' && params.doc.execution != ctx._source.execution))) {
+            ctx._source.putAll(params.doc);
+          } else {
+            ctx.op = 'none';
+          }
+        `,
+        params: {
+          doc: upsertDoc,
+        },
+      },
+      upsert: upsertDoc,
+    },
+    refresh: refresh !== undefined ? refresh : inTestMode(),
+    retry_on_conflict: 3,
+  });
+}
+
+/**
+ * Indexes the pdr type on Elasticsearch
  *
- * @param  {Object} esClient - ElasticSearch Connection object
+ * @param  {Object} esClient - Elasticsearch Connection object
  * @param  {Object} payload  - Cumulus Step Function message
  * @param  {string} index    - Elasticsearch index alias (default defined in search.js)
  * @param  {string} type     - Elasticsearch type (default: pdr)
@@ -216,10 +362,58 @@ async function indexPdr(esClient, payload, index = defaultIndexAlias, type = 'pd
 }
 
 /**
- * delete a record from ElasticSearch
+ * Upsert a PDR record in Elasticsearch
+ *
+ * @param {Object} params
+ * @param {Object} params.esClient - Elasticsearch Connection object
+ * @param {Object} params.updates - Document to upsert
+ * @param {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param {string} params.type - Elasticsearch type (default: execution)
+ * @param {string} [params.refresh] - whether to refresh the index on update or not
+ * @returns {Promise} elasticsearch update response
+ */
+async function upsertPdr({
+  esClient,
+  updates,
+  index = defaultIndexAlias,
+  type = 'pdr',
+  refresh,
+}) {
+  const upsertDoc = {
+    ...updates,
+    timestamp: Date.now(),
+  };
+  return await esClient.update({
+    index,
+    type,
+    id: upsertDoc.pdrName,
+    body: {
+      script: {
+        lang: 'painless',
+        inline: `
+          if ((ctx._source.createdAt === null || params.doc.createdAt >= ctx._source.createdAt)
+            && (params.doc.execution != ctx._source.execution || params.doc.progress > ctx._source.progress)) {
+            ctx._source.putAll(params.doc);
+          } else {
+            ctx.op = 'none';
+          }
+        `,
+        params: {
+          doc: upsertDoc,
+        },
+      },
+      upsert: upsertDoc,
+    },
+    refresh: refresh !== undefined ? refresh : inTestMode(),
+    retry_on_conflict: 3,
+  });
+}
+
+/**
+ * delete a record from Elasticsearch
  *
  * @param  {Object} params
- * @param  {Object} params.esClient - ElasticSearch Connection object
+ * @param  {Object} params.esClient - Elasticsearch Connection object
  * @param  {string} params.id       - id of the Elasticsearch record
  * @param  {string} params.type     - Elasticsearch type (default: execution)
  * @param  {strint} params.parent   - id of the parent (optional)
@@ -248,50 +442,271 @@ async function deleteRecord({
   if (ignore) options = { ignore };
 
   const actualEsClient = esClient || (await Search.es());
-
-  const getResponse = await actualEsClient.get(params, options);
   const deleteResponse = await actualEsClient.delete(params, options);
-
-  if (type === 'granule' && getResponse.body.found) {
-    const doc = getResponse.body._source;
-    doc.timestamp = Date.now();
-    doc.deletedAt = Date.now();
-
-    // When a 'granule' record is deleted, the record is added to 'deletedgranule' type
-    await genericRecordUpdate(
-      actualEsClient,
-      doc.granuleId,
-      doc,
-      index,
-      'deletedgranule',
-      parent
-    );
-  }
   return deleteResponse.body;
 }
 
 /**
- * Index a record to local Elasticsearch. Used when running API locally.
+ * Deletes the collection in Elasticsearch
+ *
+ * @param  {Object} params
+ * @param  {Object} params.esClient - Elasticsearch Connection object
+ * @param  {string} params.collectionId - the collection ID
+ * @param  {string[]} [params.ignore] - Array of response codes to ignore
+ * @param  {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} params.type - Elasticsearch type (default: collection)
+ * @returns {Promise} Elasticsearch response
+ */
+function deleteCollection({
+  esClient,
+  collectionId,
+  ignore,
+  index = defaultIndexAlias,
+  type = 'collection',
+}) {
+  return deleteRecord({
+    esClient,
+    id: collectionId,
+    index,
+    type,
+    ignore,
+  });
+}
+
+/**
+ * Deletes the provider in Elasticsearch
  *
- * @param {Object} record - Record object
- * @param {function} doIndex - Function to do indexing operation
- * @returns {Promise} - Promise of indexing operation
+ * @param  {Object} params
+ * @param  {Object} params.esClient - Elasticsearch Connection object
+ * @param  {string} params.id - the provider ID
+ * @param  {string[]} [params.ignore] - Array of response codes to ignore
+ * @param  {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} params.type - Elasticsearch type (default: provider)
+ * @returns {Promise} Elasticsearch response
  */
-async function addToLocalES(record, doIndex) {
-  const esClient = await Search.es(process.env.ES_HOST);
-  return doIndex(esClient, record, process.env.ES_INDEX);
+function deleteProvider({
+  esClient,
+  id,
+  ignore,
+  index = defaultIndexAlias,
+  type = 'provider',
+}) {
+  return deleteRecord({
+    esClient,
+    id,
+    index,
+    type,
+    ignore,
+  });
+}
+
+/**
+ * Deletes the rule in Elasticsearch
+ *
+ * @param  {Object} params
+ * @param  {Object} params.esClient - Elasticsearch Connection object
+ * @param  {string} params.name - the rule name
+ * @param  {string[]} [params.ignore] - Array of response codes to ignore
+ * @param  {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} params.type - Elasticsearch type (default: rule)
+ * @returns {Promise} Elasticsearch response
+ */
+function deleteRule({
+  esClient,
+  name,
+  ignore,
+  index = defaultIndexAlias,
+  type = 'rule',
+}) {
+  return deleteRecord({
+    esClient,
+    id: name,
+    index,
+    type,
+    ignore,
+  });
+}
+
+/**
+ * Deletes the PDR in Elasticsearch
+ *
+ * @param  {Object} params
+ * @param  {Object} params.esClient - Elasticsearch Connection object
+ * @param  {string} params.name - the PDR name
+ * @param  {string[]} [params.ignore] - Array of response codes to ignore
+ * @param  {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} params.type - Elasticsearch type (default: PDR)
+ * @returns {Promise} Elasticsearch response
+ */
+function deletePdr({
+  esClient,
+  name,
+  ignore,
+  index = defaultIndexAlias,
+  type = 'pdr',
+}) {
+  return deleteRecord({
+    esClient,
+    id: name,
+    index,
+    type,
+    ignore,
+  });
+}
+
+/**
+ * Deletes the execution in Elasticsearch
+ *
+ * @param  {Object} params
+ * @param  {Object} params.esClient - Elasticsearch Connection object
+ * @param  {string} params.arn - execution ARN
+ * @param  {string[]} [params.ignore] - Array of response codes to ignore
+ * @param  {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} params.type - Elasticsearch type (default: execution)
+ * @returns {Promise} Elasticsearch response
+ */
+function deleteExecution({
+  esClient,
+  arn,
+  ignore,
+  index = defaultIndexAlias,
+  type = 'execution',
+}) {
+  return deleteRecord({
+    esClient,
+    id: arn,
+    index,
+    type,
+    ignore,
+  });
+}
+
+/**
+ * Deletes the async operation in Elasticsearch
+ *
+ * @param {Object} params
+ * @param {Object} params.esClient - Elasticsearch Connection object
+ * @param {string} params.id - the async operation ID
+ * @param {string[]} [params.ignore] - Array of response codes to ignore
+ * @param {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param {string} params.type - Elasticsearch type (default: asyncOperation)
+ * @returns {Promise} Elasticsearch response
+*/
+function deleteAsyncOperation({
+  esClient,
+  id,
+  ignore,
+  index = defaultIndexAlias,
+  type = 'asyncOperation',
+}) {
+  return deleteRecord({
+    esClient,
+    id,
+    index,
+    type,
+    ignore,
+  });
+}
+
+/**
+ * Deletes the reconciliation report from Elasticsearch
+ *
+ * @param  {Object} params
+ * @param  {Object} params.esClient - Elasticsearch Connection object
+ * @param  {string} params.name - reconciliation report name
+ * @param  {string[]} [params.ignore] - Array of response codes to ignore
+ * @param  {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} params.type - Elasticsearch type (default: reconciliationReport)
+ * @returns {Promise} Elasticsearch response
+ */
+function deleteReconciliationReport({
+  esClient,
+  name,
+  ignore,
+  index = defaultIndexAlias,
+  type = 'reconciliationReport',
+}) {
+  return deleteRecord({
+    esClient,
+    id: name,
+    index,
+    type,
+    ignore,
+  });
+}
+
+/**
+ * Deletes the granule in Elasticsearch
+ *
+ * @param  {Object} params
+ * @param  {Object} params.esClient - Elasticsearch Connection object
+ * @param  {string} params.granuleId - the granule ID
+ * @param  {string} params.collectionId - the collection ID
+ * @param  {string[]} [params.ignore] - Array of response codes to ignore
+ * @param  {string} params.index - Elasticsearch index alias (default defined in search.js)
+ * @param  {string} params.type - Elasticsearch type (default: granule)
+ * @returns {Promise} Elasticsearch response
+ */
+async function deleteGranule({
+  esClient,
+  granuleId,
+  collectionId,
+  ignore,
+  index = defaultIndexAlias,
+  type = 'granule',
+}) {
+  const esGranulesClient = new Search(
+    {},
+    type,
+    index
+  );
+  const granuleEsRecord = await esGranulesClient.get(granuleId, collectionId);
+
+  // When a 'granule' record is deleted, the record is added to 'deletedgranule' type
+  const deletedGranuleDoc = granuleEsRecord;
+  delete deletedGranuleDoc._id;
+  deletedGranuleDoc.timestamp = Date.now();
+  deletedGranuleDoc.deletedAt = Date.now();
+  await genericRecordUpdate(
+    esClient,
+    granuleId,
+    deletedGranuleDoc,
+    index,
+    'deletedgranule',
+    collectionId
+  );
+
+  return await deleteRecord({
+    esClient,
+    id: granuleId,
+    parent: collectionId,
+    index,
+    type,
+    ignore,
+  });
 }
 
 module.exports = {
-  addToLocalES,
   createIndex,
   indexCollection,
   indexProvider,
   indexReconciliationReport,
   indexRule,
   indexGranule,
+  upsertGranule,
   indexPdr,
+  upsertPdr,
   indexExecution,
   indexAsyncOperation,
   deleteRecord,
+  deleteAsyncOperation,
+  updateAsyncOperation,
+  upsertExecution,
+  deleteCollection,
+  deleteProvider,
+  deleteRule,
+  deletePdr,
+  deleteGranule,
+  deleteExecution,
+  deleteReconciliationReport,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/es-client",
-  "version": "10.1.2",
+  "version": "11.0.0",
   "description": "Utilities for working with Elasticsearch",
   "keywords": [
     "CUMULUS",
@@ -30,10 +30,10 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/common": "10.1.2",
-    "@cumulus/errors": "10.1.2",
-    "@cumulus/logger": "10.1.2",
-    "@cumulus/message": "10.1.2",
+    "@cumulus/common": "11.0.0",
+    "@cumulus/errors": "11.0.0",
+    "@cumulus/logger": "11.0.0",
+    "@cumulus/message": "11.0.0",
     "@elastic/elasticsearch": "^5.6.20",
     "aws-elasticsearch-connector": "8.2.0",
     "aws-sdk": "^2.585.0",
@@ -42,9 +42,9 @@
     "p-limit": "^1.2.0"
   },
   "devDependencies": {
-    "@cumulus/aws-client": "10.1.2",
-    "@cumulus/test-data": "10.1.2",
+    "@cumulus/aws-client": "11.0.0",
+    "@cumulus/test-data": "11.0.0",
     "p-each-series": "^2.1.0"
   },
-  "gitHead": "3b59753fa85064549b70ad8ed4a4ee213c5af313"
+  "gitHead": "e922ad12fd94affa6cd60a97a184a465a756c50b"
 }

package/search.js

@@ -197,15 +197,30 @@ class BaseSearch {
     };
   }
 
-  async get(id) {
+  async get(id, parentId) {
     const body = {
       query: {
-        term: {
-          _id: id,
+        bool: {
+          must: [{
+            term: {
+              _id: id,
+            },
+          }],
         },
       },
     };
 
+    if (parentId) {
+      body.query.bool.must.push(
+        {
+          parent_id: {
+            id: parentId,
+            type: this.type,
+          },
+        }
+      );
+    }
+
     logDetails.granuleId = id;
 
     if (!this.client) {
@@ -215,7 +230,7 @@ class BaseSearch {
     const result = await this.client.search({
       index: this.index,
       type: this.type,
-      body: body,
+      body,
     }).then((response) => response.body);
 
     if (result.hits.total > 1) {
@@ -230,6 +245,11 @@ class BaseSearch {
     return resp;
   }
 
+  async exists(id, parentId) {
+    const response = await this.get(id, parentId);
+    return response.detail !== 'Record not found';
+  }
+
   async granulesStats(key, value) {
     const body = {
       query: {

package/testUtils.js

@@ -1,13 +1,14 @@
 const { randomString } = require('@cumulus/common/test-utils');
 
-const { bootstrapElasticSearch } = require('./bootstrap');
+const bootstrap = require('./bootstrap');
+
 const { Search } = require('./search');
 
 const createTestIndex = async () => {
   const esIndex = randomString();
   const esAlias = randomString();
   process.env.ES_INDEX = esIndex;
-  await bootstrapElasticSearch('fakehost', esIndex, esAlias);
+  await bootstrap.bootstrapElasticSearch('fakehost', esIndex, esAlias);
   const esClient = await Search.es('fakehost');
   return { esIndex, esClient };
 };