@cumulus/files-to-granules 20.3.0 → 21.0.0-echo10

package/dist/schemas/config.json

@@ -16,6 +16,9 @@
           "granuleId": {
             "type": "string"
           },
+          "producerGranuleId": {
+            "type": "string"
+          },
           "files": {
             "type": "array",
             "items": {
@@ -39,6 +42,15 @@
     "granuleIdExtraction": {
       "type": "string",
       "description": "The regex needed for extracting granuleId from filenames"
+    },
+    "matchFilesWithProducerGranuleId": {
+      "description": "When set to true/'true', use the 'producerGranuleId' instead default behavior of using 'granuleId' when mapping files to granules. Defaults to false",
+      "anyOf": [
+        { "type": "null"},
+        { "type": "boolean" },
+        { "type": "string", "enum": ["true", "false"] }
+      ],
+       "default": false
     }
   }
 }

package/index.js

@@ -8,6 +8,7 @@ const path = require('path');
 const { getObjectSize, parseS3Uri } = require('@cumulus/aws-client/S3');
 const { s3 } = require('@cumulus/aws-client/services');
 const cumulusMessageAdapter = require('@cumulus/cumulus-message-adapter-js');
+const { UnmetRequirementsError } = require('@cumulus/errors');
 
 const { getGranuleId } = require('./utils');
 
@@ -36,15 +37,28 @@ async function fileObjectFromS3URI(s3URI) {
  * Takes the files from input and granules and merges them into an object where
  * each file is associated with its granuleId.
  *
- * @param {Array<string>} inputFiles - list of s3 files to add to the inputgranules
- * @param {Array<Object>} inputGranules - an array of the granules
- * @param {string} regex - regex needed to extract granuleId from filenames
+ * @param {Object} params - params object
+ * @param {Array<string>} params.inputFiles - list of s3 files to add to the inputgranules
+ * @param {Array<Object>} params.inputGranules - an array of the granules
+ * @param {string} params.regex - regex needed to extract granuleId from filenames
+ * @param {boolean} params.matchFilesWithProducerGranuleId -
+ *  If true, match files to granules using producerGranuleId. Else, granuleId.
+ * @param {Object} params.testMocks - Mocks used for testing.
  * @returns {Object} inputGranules with updated file lists
  */
-async function mergeInputFilesWithInputGranules(inputFiles, inputGranules, regex) {
+async function mergeInputFilesWithInputGranules({
+  inputFiles,
+  inputGranules,
+  regex,
+  matchFilesWithProducerGranuleId,
+  testMocks,
+}) {
   // create hash list of the granules
   // and a list of files
-  const granulesHash = keyBy(inputGranules, 'granuleId');
+  const granulesHash = matchFilesWithProducerGranuleId ?
+    keyBy(inputGranules, 'producerGranuleId') :
+    keyBy(inputGranules, 'granuleId');
+
   const filesFromInputGranules = flatten(inputGranules.map((g) => g.files.map((f) => `s3://${f.bucket}/${f.key}`)));
 
   // add input files to corresponding granules
@@ -56,11 +70,20 @@ async function mergeInputFilesWithInputGranules(inputFiles, inputGranules, regex
   /* eslint-disable no-await-in-loop */
   for (let i = 0; i < filesToAdd.length; i += 1) {
     const f = filesToAdd[i];
-    const fileGranuleId = getGranuleId(f, regex);
+    const fileId = getGranuleId(f, regex);
     try {
-      granulesHash[fileGranuleId].files.push(await fileObjectFromS3URI(f));
+      granulesHash[fileId].files.push(
+        testMocks?.fileObjectFromS3URI ?
+          await testMocks.fileObjectFromS3URI(f) :
+          await fileObjectFromS3URI(f)
+      );
     } catch (error) {
-      throw new Error(`Failed adding ${f} to ${fileGranuleId}'s files: ${error.name} ${error.message}`);
+      if (!granulesHash[fileId]) {
+        throw new UnmetRequirementsError(
+          `fileId ${fileId} does not match an input granule. Check that 'matchFilesWithProducerGranuleId' is configured as expected.`
+        );
+      }
+      throw new Error(`Failed adding ${f} to ${fileId}'s files: ${error.name} ${error.message}`);
     }
   }
   /* eslint-enable no-await-in-loop */
@@ -79,17 +102,24 @@ async function mergeInputFilesWithInputGranules(inputFiles, inputGranules, regex
  *                                                    from filenames
  * @param {Array<Object>} event.config.inputGranules - an array of granules
  * @param {Array<string>} event.input - an array of s3 uris
+ * @param {Object} testMocks - Mocks used for testing.
  *
  * @returns {Object} Granules object
  */
-function filesToGranules(event) {
-  const granuleIdExtractionRegex = get(event.config, 'granuleIdExtraction', '(.*)');
+function filesToGranules(event, testMocks) {
+  const regex = get(event.config, 'granuleIdExtraction', '(.*)');
+  const matchFilesWithProducerGranuleIdConfigValue = get(event.config, 'matchFilesWithProducerGranuleId');
+  const matchFilesWithProducerGranuleId = [true, 'true'].includes(matchFilesWithProducerGranuleIdConfigValue);
   const inputGranules = event.config.inputGranules;
-  const inputFileList = event.input;
+  const inputFiles = event.input;
 
-  return mergeInputFilesWithInputGranules(
-    inputFileList, inputGranules, granuleIdExtractionRegex
-  );
+  return mergeInputFilesWithInputGranules({
+    inputFiles,
+    inputGranules,
+    regex,
+    matchFilesWithProducerGranuleId,
+    testMocks,
+  });
 }
 exports.filesToGranules = filesToGranules;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/files-to-granules",
-  "version": "20.3.0",
+  "version": "21.0.0-echo10",
   "description": "Converts array-of-files input into a granules object by extracting granuleId from filename",
   "main": "index.js",
   "directories": {
@@ -36,13 +36,14 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/aws-client": "20.3.0",
+    "@cumulus/aws-client": "21.0.0-echo10",
     "@cumulus/cumulus-message-adapter-js": "2.3.0",
+    "@cumulus/errors": "21.0.0-echo10",
     "lodash": "^4.17.21"
   },
   "devDependencies": {
-    "@cumulus/common": "20.3.0",
-    "@cumulus/schemas": "20.3.0"
+    "@cumulus/common": "21.0.0-echo10",
+    "@cumulus/schemas": "21.0.0-echo10"
   },
-  "gitHead": "0d3a61cd0185f62360b3e8ab6e59f69e93ac7410"
+  "gitHead": "b0560aa0ef1d3b7401eedc4a4ba90f57f16bcda3"
 }

package/schemas/config.json

@@ -16,6 +16,9 @@
           "granuleId": {
             "type": "string"
           },
+          "producerGranuleId": {
+            "type": "string"
+          },
           "files": {
             "type": "array",
             "items": {
@@ -39,6 +42,15 @@
     "granuleIdExtraction": {
       "type": "string",
       "description": "The regex needed for extracting granuleId from filenames"
+    },
+    "matchFilesWithProducerGranuleId": {
+      "description": "When set to true/'true', use the 'producerGranuleId' instead default behavior of using 'granuleId' when mapping files to granules. Defaults to false",
+      "anyOf": [
+        { "type": "null"},
+        { "type": "boolean" },
+        { "type": "string", "enum": ["true", "false"] }
+      ],
+       "default": false
     }
   }
 }