@cumulus/integration-tests 20.3.0 → 21.0.0-echo10

package/cmr.js

@@ -2,7 +2,7 @@
 
 const got = require('got');
 const pWaitFor = require('p-wait-for');
-const xml2js = require('xml2js');
+const js2xmlParser = require('js2xmlparser');
 const { s3 } = require('@cumulus/aws-client/services');
 const { buildS3Uri } = require('@cumulus/aws-client/S3');
 const { sleep } = require('@cumulus/common');
@@ -182,11 +182,13 @@ async function waitForConceptExistsOutcome(cmrLink, expectation) {
  * @param {Object} granule - granule object
  * @param {Object} collection - collection object
  * @param {Array<string>} additionalUrls - URLs to convert to online resources
+ * @param {boolean} matchFilesWithProducerGranuleId - When set to true, use the 'producerGranuleId'
  * @returns {Promise<Array<string>>} - Promise of the generated granule xml string
  * CMR xml files
  */
-function generateCmrXml(granule, collection, additionalUrls) {
-  const xmlObject = sampleEcho10Granule;
+function generateCmrXml(granule, collection, additionalUrls,
+  matchFilesWithProducerGranuleId = false) {
+  const xmlObject = structuredClone(sampleEcho10Granule);
   const oldGranuleId = xmlObject.Granule.GranuleUR;
   xmlObject.Granule.GranuleUR = granule.granuleId;
 
@@ -196,10 +198,17 @@ function generateCmrXml(granule, collection, additionalUrls) {
   };
 
   xmlObject.Granule.OnlineAccessURLs.forEach((url) => {
+    const replacementId = (matchFilesWithProducerGranuleId && granule.producerGranuleId)
+      ? granule.producerGranuleId : granule.granuleId;
     // eslint-disable-next-line no-param-reassign
-    url.OnlineAccessURL.URL = url.OnlineAccessURL.URL.replace(oldGranuleId, granule.granuleId);
+    url.OnlineAccessURL.URL = url.OnlineAccessURL.URL.replace(oldGranuleId, replacementId);
   });
 
+  xmlObject.Granule.DataGranule = new Map();
+  xmlObject.Granule.DataGranule.set('SizeMBDataGranule', '10');
+  xmlObject.Granule.DataGranule.set('DayNightFlag', 'BOTH');
+  xmlObject.Granule.DataGranule.set('ProductionDateTime', '2018-07-19T12:01:01Z');
+
   if (additionalUrls) {
     xmlObject.Granule.OnlineAccessURLs = additionalUrls.map((url) => ({
       OnlineAccessURL: {
@@ -208,8 +217,12 @@ function generateCmrXml(granule, collection, additionalUrls) {
       },
     }));
   }
+  const granuleMap = new Map();
+  Object.keys(xmlObject.Granule).forEach((key) => {
+    granuleMap.set(key, xmlObject.Granule[key]);
+  });
 
-  const xml = new xml2js.Builder().buildObject(xmlObject);
+  const xml = js2xmlParser.parse('Granule', granuleMap);
   return xml;
 }
 
@@ -220,21 +233,28 @@ function generateCmrXml(granule, collection, additionalUrls) {
  * @param {Object} collection - collection object
  * @param {string} bucket - bucket to save the xml file to
  * @param {Array<string>} additionalUrls - URLs to convert to online resources
+ * @param {string} stagingDir - staging directory
+ * @param {boolean} matchFilesWithProducerGranuleId - When set to true, use the 'producerGranuleId'
+ * instead default behavior of using 'granuleId' when generating filenames.
  * @returns {Promise<Array<string>>} - Promise of a list of granule files including the created
  * CMR xml files
  */
-async function generateAndStoreCmrXml(granule, collection, bucket, additionalUrls, stagingDir = 'file-staging') {
-  const xml = generateCmrXml(granule, collection, additionalUrls);
+async function generateAndStoreCmrXml(granule, collection, bucket, additionalUrls, stagingDir = 'file-staging',
+  matchFilesWithProducerGranuleId = false) {
+  const xml = generateCmrXml(granule, collection, additionalUrls, matchFilesWithProducerGranuleId);
   const granuleFiles = granule.files.map((f) => `s3://${f.bucket}/${f.key}`);
 
-  const fileKey = `${stagingDir}/${granule.granuleId}.cmr.xml`;
+  const id = (matchFilesWithProducerGranuleId && granule.producerGranuleId)
+    ? granule.producerGranuleId
+    : granule.granuleId;
+  const fileKey = `${stagingDir}/${granule.granuleId}/${id}.cmr.xml`;
 
   const params = {
     Bucket: bucket,
     Key: fileKey,
     Body: xml,
     ContentType: 'application/xml',
-    Tagging: `granuleId=${granule.granuleId}`,
+    Tagging: `granuleId=${id}`,
   };
 
   await s3().putObject(params);
@@ -268,6 +288,16 @@ function isUMMGMetadataFormat(cmrMetadataFormat) {
   return cmrMetadataFormat && cmrMetadataFormat.match(/umm_json_v/);
 }
 
+async function getCmrMetadataECHO10(cmrLink) {
+  const response = await got.get(cmrLink);
+
+  if (response.statusCode !== 200) {
+    console.log(`Error fetching CMR metadata, status code: ${response.statusCode}, response: ${JSON.stringify(response.body)}`);
+    return null;
+  }
+
+  return JSON.parse(response.body);
+}
 /**
  * Get the online resource links from the CMR objects for ECH010
  *
@@ -280,15 +310,17 @@ function isUMMGMetadataFormat(cmrMetadataFormat) {
     href: 'https://opendap.cr.usgs.gov/opendap/hyrax/MYD13Q1.006/contents.html' }
  */
 async function getOnlineResourcesECHO10(cmrLink) {
-  const response = await got.get(cmrLink);
+  const body = await getCmrMetadataECHO10(cmrLink);
+  return body.links;
+}
 
+async function getCmrMetadataUMMG(cmrLink) {
+  const response = await got.get(cmrLink);
   if (response.statusCode !== 200) {
+    console.log(`Error fetching CMR metadata, status code: ${response.statusCode}, response: ${JSON.stringify(response.body)}`);
     return null;
   }
-
-  const body = JSON.parse(response.body);
-
-  return body.links;
+  return JSON.parse(response.body);
 }
 
 /**
@@ -302,20 +334,36 @@ async function getOnlineResourcesECHO10(cmrLink) {
     Type: "GET DATA" }
  */
 async function getOnlineResourcesUMMG(cmrLink) {
-  const response = await got.get(cmrLink);
-
-  if (response.statusCode !== 200) {
-    return null;
-  }
-
-  const body = JSON.parse(response.body);
-
+  const body = await getCmrMetadataUMMG(cmrLink);
   const links = body.items.map((item) => item.umm.RelatedUrls);
 
   // Links is a list of a list, so flatten to be one list
   return [].concat(...links);
 }
 
+/**
+ * Fetches full granule object from CMR based on file type (ECHO10, UMM-G)
+ *
+ * @param {Object} granule
+ * @param {string} granule.cmrMetadataFormat - the cmr file type (e.g. echo10, umm-g)
+ * @param {Object} granule.cmrConceptId - the CMR granule concept ID
+ * @param {Object} granule.cmrLink - the metadata's granuleId
+ *
+ * @returns {Promise<Array<Object>>} - Promise returning array of links
+ */
+async function getCmrMetadata({ cmrMetadataFormat, cmrConceptId, cmrLink }) {
+  console.log('Running getCmrMetadata');
+  console.log(cmrLink);
+  console.log(`${getSearchUrl()}granules.umm_json?concept_id=${cmrConceptId}`);
+  if (cmrMetadataFormat === 'echo10') {
+    return await getCmrMetadataECHO10(cmrLink.replace(/(.echo10)$/, '.json'));
+  }
+  if (isUMMGMetadataFormat(cmrMetadataFormat)) {
+    return await getCmrMetadataUMMG(`${getSearchUrl()}granules.umm_json?concept_id=${cmrConceptId}`);
+  }
+  throw new Error(`Invalid cmrMetadataFormat passed to getOnlineResources: ${cmrMetadataFormat}`);
+}
+
 /**
  * Fetches online resources from CMR based on file type (ECHO10, UMM-G)
  *
@@ -347,6 +395,9 @@ async function getOnlineResources({ cmrMetadataFormat, cmrConceptId, cmrLink })
  * @param {string} bucket - bucket to save the xml file to
  * @param {Array<string>} additionalUrls - URLs to convert to related urls
  * @param {string} cmrMetadataFormat - CMR UMM-G version string <umm_json_v[x.y]>
+ * @param {string} stagingDir - staging directory
+ * @param {boolean} matchFilesWithProducerGranuleId - When set to true, use the 'producerGranuleId'
+ * instead default behavior of using 'granuleId' when generating filenames.
  * @returns {Promise<Array<string>>} - Promise of a list of granule files including the created
  * CMR files
  */
@@ -356,7 +407,8 @@ async function generateAndStoreCmrUmmJson(
   bucket,
   additionalUrls,
   cmrMetadataFormat,
-  stagingDir = 'file-staging'
+  stagingDir = 'file-staging',
+  matchFilesWithProducerGranuleId = false
 ) {
   const versionString = metadataFormatToVersion(cmrMetadataFormat);
   const jsonObject = sampleUmmGranule;
@@ -384,7 +436,10 @@ async function generateAndStoreCmrUmmJson(
     };
   }
 
-  const fileKey = `${stagingDir}/${granule.granuleId}.cmr.json`;
+  const fileNameBase = matchFilesWithProducerGranuleId
+    ? granule.producerGranuleId
+    : granule.granuleId;
+  const fileKey = `${stagingDir}/${granule.granuleId}/${fileNameBase}.cmr.json`;
 
   const params = {
     Bucket: bucket,
@@ -423,6 +478,7 @@ async function generateCmrFilesForGranules({
   cmrMetadataFormat,
   additionalUrls,
   stagingDir,
+  matchFilesWithProducerGranuleId = false,
 }) {
   let files;
 
@@ -437,7 +493,8 @@ async function generateCmrFilesForGranules({
           bucket,
           additionalUrls,
           cmrMetadataFormat,
-          stagingDir
+          stagingDir,
+          matchFilesWithProducerGranuleId
         ))
     );
   } else {
@@ -448,7 +505,8 @@ async function generateCmrFilesForGranules({
           collection,
           bucket,
           additionalUrls,
-          stagingDir
+          stagingDir,
+          matchFilesWithProducerGranuleId
         ))
     );
   }
@@ -459,6 +517,7 @@ async function generateCmrFilesForGranules({
 module.exports = {
   conceptExists,
   getOnlineResources,
+  getCmrMetadata,
   generateCmrFilesForGranules,
   generateCmrXml,
   waitForConceptExistsOutcome,