@cumulus/move-granules 9.8.0 → 10.0.1

package/dist/schemas/config.json

@@ -71,6 +71,10 @@
       "description": "Specifies how duplicate filenames should be handled. `error` will throw an error that, if not caught, will fail the task/workflow execution. `version` will add a suffix to the existing filename to avoid a clash.",
       "enum": ["replace", "version", "skip", "error"],
       "default": "error"
+    },
+    "s3MultipartChunksizeMb": {
+      "type": ["number", "null"],
+      "description": "S3 multipart upload chunk size in MB.  If none is specified, the default default_s3_multipart_chunksize_mb is used."
     }
   }
 }

package/dist/schemas/input.json

@@ -11,7 +11,10 @@
       "description": "Array of all granules",
       "items": {
         "type": "object",
-        "required": ["granuleId", "files"],
+        "required": [
+          "granuleId",
+          "files"
+        ],
         "properties": {
           "granuleId": {
             "type": "string"
@@ -19,19 +22,43 @@
           "files": {
             "type": "array",
             "items": {
+              "additionalProperties": false,
               "type": "object",
-              "required": ["name"],
+              "required": [
+                "bucket",
+                "key"
+              ],
               "properties": {
-                "name": {
+                "bucket": {
+                  "description": "Bucket where file is archived in S3",
                   "type": "string"
                 },
-                "filename": {
+                "checksum": {
+                  "description": "Checksum value for file",
                   "type": "string"
                 },
-                "type": {
+                "checksumType": {
+                  "description": "Type of checksum (e.g. md5, sha256, etc)",
                   "type": "string"
                 },
-                "bucket": {
+                "fileName": {
+                  "description": "Name of file (e.g. file.txt)",
+                  "type": "string"
+                },
+                "key": {
+                  "description": "S3 Key for archived file",
+                  "type": "string"
+                },
+                "size": {
+                  "description": "Size of file (in bytes)",
+                  "type": "number"
+                },
+                "source": {
+                  "description": "Source URI of the file from origin system (e.g. S3, FTP, HTTP)",
+                  "type": "string"
+                },
+                "type": {
+                  "description": "Type of file (e.g. data, metadata, browse)",
                   "type": "string"
                 }
               }
@@ -41,4 +68,4 @@
       }
     }
   }
-}
+}

package/dist/schemas/input.json.template

@@ -0,0 +1,24 @@
+{
+  "title": "MoveGranulesInput",
+  "description": "Describes the input expected by the move-granules task",
+  "type": "object",
+  "required": [
+    "granules"
+  ],
+  "properties": {
+    "granules": {
+      "type": "array",
+      "description": "Array of all granules",
+      "items": {
+        "type": "object",
+        "required": ["granuleId", "files"],
+        "properties": {
+          "granuleId": {
+            "type": "string"
+          },
+          "files": "{{files}}"
+        }
+      }
+    }
+  }
+}

package/dist/schemas/output.json

@@ -3,12 +3,40 @@
   "description": "Describes the output produced by the move-granules task",
   "type": "object",
   "properties": {
+    "granuleDuplicates": {
+      "type": "object",
+      "additionalProperties": {
+        "files": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": [
+              "bucket",
+              "key"
+            ],
+            "properties": {
+              "bucket": {
+                "description": "Bucket in-process file is being staged in in S3",
+                "type": "string"
+              },
+              "key": {
+                "description": "S3 Key for in-process file",
+                "type": "string"
+              }
+            }
+          }
+        }
+      }
+    },
     "granules": {
       "type": "array",
       "description": "Array of all granules",
       "items": {
         "type": "object",
-        "required": ["granuleId", "files"],
+        "required": [
+          "granuleId",
+          "files"
+        ],
         "properties": {
           "granuleId": {
             "type": "string"
@@ -16,33 +44,44 @@
           "files": {
             "type": "array",
             "items": {
+              "additionalProperties": false,
               "type": "object",
-              "required": ["name"],
+              "required": [
+                "bucket",
+                "key"
+              ],
               "properties": {
-                "name": {
+                "bucket": {
+                  "description": "Bucket where file is archived in S3",
                   "type": "string"
                 },
-                "filename": {
-                  "type": "string",
-                  "description": "The s3 bucket/path URI (e.g. s3://{bucket}{object})"
+                "checksum": {
+                  "description": "Checksum value for file",
+                  "type": "string"
                 },
-                "filepath": {
-                  "type": "string",
-                  "description": "The S3 object key"
+                "checksumType": {
+                  "description": "Type of checksum (e.g. md5, sha256, etc)",
+                  "type": "string"
                 },
-                "type": {
+                "fileName": {
+                  "description": "Name of file (e.g. file.txt)",
                   "type": "string"
                 },
-                "bucket": {
+                "key": {
+                  "description": "S3 Key for archived file",
                   "type": "string"
                 },
-                "etag": {
-                  "description": "Entity tag for the current version of the file",
+                "size": {
+                  "description": "Size of file (in bytes)",
+                  "type": "number"
+                },
+                "source": {
+                  "description": "Source URI of the file from origin system (e.g. S3, FTP, HTTP)",
                   "type": "string"
                 },
-                "duplicate_found": {
-                  "description": "whether a duplicate for this file was found",
-                  "type": "boolean"
+                "type": {
+                  "description": "Type of file (e.g. data, metadata, browse)",
+                  "type": "string"
                 }
               }
             }
@@ -51,4 +90,4 @@
       }
     }
   }
-}
+}

package/dist/schemas/output.json.template

@@ -0,0 +1,43 @@
+{
+  "title": "MoveGranulesOutput",
+  "description": "Describes the output produced by the move-granules task",
+  "type": "object",
+  "properties": {
+    "granuleDuplicates": {
+      "type": "object",
+      "additionalProperties": {
+        "files": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["bucket", "key"],
+            "properties": {
+              "bucket": {
+                "description": "Bucket in-process file is being staged in in S3",
+                "type": "string"
+              },
+              "key": {
+                "description": "S3 Key for in-process file",
+                "type": "string"
+              }
+            }
+          }
+        }
+      }
+    },
+    "granules": {
+      "type": "array",
+      "description": "Array of all granules",
+      "items": {
+        "type": "object",
+        "required": ["granuleId", "files"],
+        "properties": {
+          "granuleId": {
+            "type": "string"
+          },
+          "files": "{{files}}"
+        }
+      }
+    }
+  }
+}

package/index.js

@@ -7,13 +7,7 @@ const flatten = require('lodash/flatten');
 const keyBy = require('lodash/keyBy');
 const path = require('path');
 
-const {
-  buildS3Uri,
-  moveObject,
-  s3Join,
-  s3ObjectExists,
-  waitForObjectToExist,
-} = require('@cumulus/aws-client/S3');
+const S3 = require('@cumulus/aws-client/S3');
 
 const { InvalidArgument } = require('@cumulus/errors');
 
@@ -25,6 +19,7 @@ const {
 
 const {
   isCMRFile,
+  isISOFile,
   metadataObjectFromCMRFile,
   granulesToCmrFileObjects,
 } = require('@cumulus/cmrjs');
@@ -34,6 +29,25 @@ const BucketsConfig = require('@cumulus/common/BucketsConfig');
 const { urlPathTemplate } = require('@cumulus/ingest/url-path-template');
 const log = require('@cumulus/common/log');
 
+const MB = 1024 * 1024;
+
+function buildGranuleDuplicatesObject(movedGranulesByGranuleId) {
+  const duplicatesObject = {};
+  Object.keys(movedGranulesByGranuleId).forEach((k) => {
+    duplicatesObject[k] = {
+      files: movedGranulesByGranuleId[k].files.filter((file) => {
+        if (file.duplicate_found) {
+          // eslint-disable-next-line no-param-reassign
+          delete file.duplicate_found;
+          return true;
+        }
+        return false;
+      }),
+    };
+  });
+  return duplicatesObject;
+}
+
 /**
  * Validates the file matched only one collection.file and has a valid bucket
  * config.
@@ -73,7 +87,7 @@ function validateMatch(match, bucketsConfig, fileName, fileSpecs) {
  */
 async function updateGranuleMetadata(granulesObject, collection, cmrFiles, bucketsConfig) {
   const updatedGranules = {};
-  const cmrFileNames = cmrFiles.map((f) => path.basename(f.filename));
+  const cmrFileNames = cmrFiles.map((f) => path.basename(f.key));
   const fileSpecs = collection.files;
 
   await Promise.all(Object.keys(granulesObject).map(async (granuleId) => {
@@ -81,35 +95,33 @@ async function updateGranuleMetadata(granulesObject, collection, cmrFiles, bucke
     updatedGranules[granuleId] = { ...granulesObject[granuleId] };
 
     const cmrFile = cmrFiles.find((f) => f.granuleId === granuleId);
-    const cmrMetadata = cmrFile ? await metadataObjectFromCMRFile(cmrFile.filename) : {};
+    const cmrMetadata = cmrFile ? await metadataObjectFromCMRFile(`s3://${cmrFile.bucket}/${cmrFile.key}`) : {};
 
     granulesObject[granuleId].files.forEach((file) => {
       const cmrFileTypeObject = {};
-      if (cmrFileNames.includes(file.name) && !file.type) {
+      const fileName = path.basename(file.key);
+      if (cmrFileNames.includes(fileName) && !file.type) {
         cmrFileTypeObject.type = 'metadata';
       }
 
-      const match = fileSpecs.filter((cf) => unversionFilename(file.name).match(cf.regex));
-      validateMatch(match, bucketsConfig, file.name, fileSpecs);
+      const match = fileSpecs.filter((cf) => unversionFilename(fileName).match(cf.regex));
+      validateMatch(match, bucketsConfig, fileName, fileSpecs);
 
-      const URLPathTemplate = file.url_path || match[0].url_path || collection.url_path || '';
+      const URLPathTemplate = match[0].url_path || collection.url_path || '';
       const urlPath = urlPathTemplate(URLPathTemplate, {
         file,
         granule: granulesObject[granuleId],
         cmrMetadata,
       });
       const bucketName = bucketsConfig.nameByKey(match[0].bucket);
-      const filepath = s3Join(urlPath, file.name);
+      const updatedKey = S3.s3Join(urlPath, fileName);
 
       updatedFiles.push({
-        ...file, // keeps old info like "name" and "fileStagingDir"
+        ...file,
         ...cmrFileTypeObject, // Add type if the file is a CMR file
-        ...{
-          bucket: bucketName,
-          filepath,
-          filename: `s3://${s3Join(bucketName, filepath)}`,
-          url_path: URLPathTemplate,
-        },
+        bucket: bucketName,
+        sourceKey: file.key,
+        key: updatedKey,
       });
     });
     updatedGranules[granuleId].files = [...updatedFiles];
@@ -124,37 +136,35 @@ async function updateGranuleMetadata(granulesObject, collection, cmrFiles, bucke
  * @param {Object} file - granule file to be moved
  * @param {string} sourceBucket - source bucket location of files
  * @param {string} duplicateHandling - how to handle duplicate files
- * @param {BucketsConfig} bucketsConfig - BucketsConfig instance
  * @param {boolean} markDuplicates - Override to handle cmr metadata files that
  *                                   shouldn't be marked as duplicates
+ * @param {number} s3MultipartChunksizeMb - S3 multipart upload chunk size in MB
  * @returns {Array<Object>} returns the file moved and the renamed existing duplicates if any
  */
 async function moveFileRequest(
   file,
   sourceBucket,
   duplicateHandling,
-  bucketsConfig,
-  markDuplicates = true
+  markDuplicates = true,
+  s3MultipartChunksizeMb
 ) {
-  const fileStagingDir = file.fileStagingDir || 'file-staging';
   const source = {
     Bucket: sourceBucket,
-    Key: `${fileStagingDir}/${file.name}`,
+    Key: file.sourceKey,
   };
   const target = {
     Bucket: file.bucket,
-    Key: file.filepath,
+    Key: file.key,
   };
 
   // Due to S3's eventual consistency model, we need to make sure that the
   // source object is available in S3.
-  await waitForObjectToExist({ bucket: source.Bucket, key: source.Key });
-
+  await S3.waitForObjectToExist({ bucket: source.Bucket, key: source.Key });
   // the file moved to destination
   const fileMoved = { ...file };
-  delete fileMoved.fileStagingDir;
+  delete fileMoved.sourceKey;
 
-  const s3ObjAlreadyExists = await s3ObjectExists(target);
+  const s3ObjAlreadyExists = await S3.s3ObjectExists(target);
   log.debug(`file ${target.Key} exists in ${target.Bucket}: ${s3ObjAlreadyExists}`);
 
   let versionedFiles = [];
@@ -167,22 +177,21 @@ async function moveFileRequest(
       duplicateHandling,
     });
   } else {
-    await moveObject({
+    const chunkSize = s3MultipartChunksizeMb ? Number(s3MultipartChunksizeMb) * MB : undefined;
+    await S3.moveObject({
       sourceBucket: source.Bucket,
       sourceKey: source.Key,
       destinationBucket: target.Bucket,
       destinationKey: target.Key,
       copyTags: true,
+      chunkSize,
     });
   }
 
   const renamedFiles = versionedFiles.map((f) => ({
     bucket: f.Bucket,
-    name: path.basename(f.Key),
-    filename: buildS3Uri(f.Bucket, f.Key),
-    filepath: f.Key,
+    key: f.Key,
     size: f.size,
-    url_path: file.url_path,
   }));
 
   // return both file moved and renamed files
@@ -190,34 +199,32 @@ async function moveFileRequest(
 }
 
 /**
- * Move all files in a collection of granules from staging location fo final location,
+ * Move all files in a collection of granules from staging location to final location,
  * and update granule files to include renamed files if any.
  *
  * @param {Object} granulesObject - an object of the granules where the key is the granuleId
  * @param {string} sourceBucket - source bucket location of files
  * @param {string} duplicateHandling - how to handle duplicate files
- * @param {BucketsConfig} bucketsConfig - BucketsConfig instance
+ * @param {number} s3MultipartChunksizeMb - S3 multipart upload chunk size in MB
  * @returns {Object} the object with updated granules
  */
 async function moveFilesForAllGranules(
   granulesObject,
   sourceBucket,
   duplicateHandling,
-  bucketsConfig
+  s3MultipartChunksizeMb
 ) {
   const moveFileRequests = Object.keys(granulesObject).map(async (granuleKey) => {
     const granule = granulesObject[granuleKey];
     const filesToMove = granule.files.filter((file) => !isCMRFile(file));
     const cmrFiles = granule.files.filter((file) => isCMRFile(file));
     const filesMoved = await Promise.all(
-      filesToMove.map(
-        (file) => moveFileRequest(file, sourceBucket, duplicateHandling, bucketsConfig)
-      )
+      filesToMove.map((file) =>
+        moveFileRequest(file, sourceBucket, duplicateHandling, true, s3MultipartChunksizeMb))
     );
-    const markDuplicates = false;
     const cmrFilesMoved = await Promise.all(
       cmrFiles.map(
-        (file) => moveFileRequest(file, sourceBucket, 'replace', bucketsConfig, markDuplicates)
+        (file) => moveFileRequest(file, sourceBucket, 'replace', false)
       )
     );
     granule.files = flatten(filesMoved).concat(flatten(cmrFilesMoved));
@@ -250,11 +257,17 @@ async function moveGranules(event) {
   const bucketsConfig = new BucketsConfig(config.buckets);
 
   const moveStagedFiles = get(config, 'moveStagedFiles', true);
+  const s3MultipartChunksizeMb = config.s3MultipartChunksizeMb
+    ? config.s3MultipartChunksizeMb : process.env.default_s3_multipart_chunksize_mb;
 
   const duplicateHandling = duplicateHandlingType(event);
 
+  log.debug(`moveGranules config duplicateHandling: ${duplicateHandling}, moveStagedFiles: ${moveStagedFiles}, s3MultipartChunksizeMb: ${s3MultipartChunksizeMb}`);
+
   const granulesInput = event.input.granules;
-  const cmrFiles = granulesToCmrFileObjects(granulesInput);
+
+  const filterFunc = (fileobject) => isCMRFile(fileobject) || isISOFile(fileobject);
+  const cmrFiles = granulesToCmrFileObjects(granulesInput, filterFunc);
   const granulesByGranuleId = keyBy(granulesInput, 'granuleId');
 
   let movedGranulesByGranuleId;
@@ -269,13 +282,16 @@ async function moveGranules(event) {
 
     // Move files from staging location to final location
     movedGranulesByGranuleId = await moveFilesForAllGranules(
-      granulesToMove, config.bucket, duplicateHandling, bucketsConfig
+      granulesToMove, config.bucket, duplicateHandling, s3MultipartChunksizeMb
     );
   } else {
     movedGranulesByGranuleId = granulesByGranuleId;
   }
 
+  const granuleDuplicates = buildGranuleDuplicatesObject(movedGranulesByGranuleId);
+
   return {
+    granuleDuplicates,
     granules: Object.values(movedGranulesByGranuleId),
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/move-granules",
-  "version": "9.8.0",
+  "version": "10.0.1",
   "description": "Move granule files from staging to final location",
   "main": "index.js",
   "directories": {
@@ -18,7 +18,8 @@
     "node": ">=12.18.0"
   },
   "scripts": {
-    "build": "rm -rf dist && mkdir dist && cp -R schemas dist/ && ../../node_modules/.bin/webpack",
+    "generate-task-schemas": "npx generate-task-schemas . files",
+    "build": "rm -rf dist && mkdir dist && npm run generate-task-schemas && cp -R schemas dist/ && ../../node_modules/.bin/webpack",
     "package": "npm run build && (cd dist && node ../../../bin/zip.js lambda.zip index.js schemas)",
     "test": "../../node_modules/.bin/ava",
     "test:coverage": "../../node_modules/.bin/nyc npm test",
@@ -36,14 +37,17 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/aws-client": "9.8.0",
-    "@cumulus/cmrjs": "9.8.0",
-    "@cumulus/common": "9.8.0",
-    "@cumulus/cumulus-message-adapter-js": "2.0.0",
-    "@cumulus/distribution-utils": "9.8.0",
-    "@cumulus/errors": "9.8.0",
-    "@cumulus/ingest": "9.8.0",
+    "@cumulus/aws-client": "10.0.1",
+    "@cumulus/cmrjs": "10.0.1",
+    "@cumulus/common": "10.0.1",
+    "@cumulus/cumulus-message-adapter-js": "2.0.4",
+    "@cumulus/distribution-utils": "10.0.1",
+    "@cumulus/errors": "10.0.1",
+    "@cumulus/ingest": "10.0.1",
     "lodash": "^4.17.20"
   },
-  "gitHead": "913034ba6814e562b7f3d58bb39cf086255a6efd"
+  "devDependencies": {
+    "@cumulus/schemas": "10.0.1"
+  },
+  "gitHead": "49c3c88336838184f22f35fbce298c71cd269138"
 }

package/schemas/config.json

@@ -71,6 +71,10 @@
       "description": "Specifies how duplicate filenames should be handled. `error` will throw an error that, if not caught, will fail the task/workflow execution. `version` will add a suffix to the existing filename to avoid a clash.",
       "enum": ["replace", "version", "skip", "error"],
       "default": "error"
+    },
+    "s3MultipartChunksizeMb": {
+      "type": ["number", "null"],
+      "description": "S3 multipart upload chunk size in MB.  If none is specified, the default default_s3_multipart_chunksize_mb is used."
     }
   }
 }

package/schemas/input.json

@@ -11,7 +11,10 @@
       "description": "Array of all granules",
       "items": {
         "type": "object",
-        "required": ["granuleId", "files"],
+        "required": [
+          "granuleId",
+          "files"
+        ],
         "properties": {
           "granuleId": {
             "type": "string"
@@ -19,19 +22,43 @@
           "files": {
             "type": "array",
             "items": {
+              "additionalProperties": false,
               "type": "object",
-              "required": ["name"],
+              "required": [
+                "bucket",
+                "key"
+              ],
               "properties": {
-                "name": {
+                "bucket": {
+                  "description": "Bucket where file is archived in S3",
                   "type": "string"
                 },
-                "filename": {
+                "checksum": {
+                  "description": "Checksum value for file",
                   "type": "string"
                 },
-                "type": {
+                "checksumType": {
+                  "description": "Type of checksum (e.g. md5, sha256, etc)",
                   "type": "string"
                 },
-                "bucket": {
+                "fileName": {
+                  "description": "Name of file (e.g. file.txt)",
+                  "type": "string"
+                },
+                "key": {
+                  "description": "S3 Key for archived file",
+                  "type": "string"
+                },
+                "size": {
+                  "description": "Size of file (in bytes)",
+                  "type": "number"
+                },
+                "source": {
+                  "description": "Source URI of the file from origin system (e.g. S3, FTP, HTTP)",
+                  "type": "string"
+                },
+                "type": {
+                  "description": "Type of file (e.g. data, metadata, browse)",
                   "type": "string"
                 }
               }
@@ -41,4 +68,4 @@
       }
     }
   }
-}
+}

package/schemas/input.json.template

@@ -0,0 +1,24 @@
+{
+  "title": "MoveGranulesInput",
+  "description": "Describes the input expected by the move-granules task",
+  "type": "object",
+  "required": [
+    "granules"
+  ],
+  "properties": {
+    "granules": {
+      "type": "array",
+      "description": "Array of all granules",
+      "items": {
+        "type": "object",
+        "required": ["granuleId", "files"],
+        "properties": {
+          "granuleId": {
+            "type": "string"
+          },
+          "files": "{{files}}"
+        }
+      }
+    }
+  }
+}