@cumulus/aws-client 9.8.0 → 10.0.1

package/README.md

@@ -858,6 +858,7 @@ Copy an S3 object to another location in S3 using a multipart copy
 | [params.sourceObject] | <code>AWS.S3.HeadObjectOutput</code> |  | Output from https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property |
 | [params.ACL] | <code>string</code> |  | an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl) |
 | [params.copyTags] | <code>boolean</code> | <code>false</code> |  |
+| [params.chunkSize] | <code>number</code> |  | chunk size of the S3 multipart uploads |
 
 <a name="module_S3..moveObject"></a>
 
@@ -875,6 +876,7 @@ Move an S3 object to another location in S3
 | params.destinationKey | <code>string</code> |  |  |
 | [params.ACL] | <code>string</code> |  | an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl) |
 | [params.copyTags] | <code>boolean</code> | <code>false</code> |  |
+| [params.chunkSize] | <code>number</code> |  | chunk size of the S3 multipart uploads |
 
 <a name="module_SNS"></a>
 

package/S3.d.ts

@@ -282,7 +282,9 @@ export declare const deleteS3Files: (s3Objs: AWS.S3.DeleteObjectRequest[]) => Pr
 * @param {string} bucket - name of the bucket
 * @returns {Promise} the promised result of `S3.deleteBucket`
 **/
-export declare const recursivelyDeleteS3Bucket: (bucket: string) => Promise<void>;
+export declare const recursivelyDeleteS3Bucket: (bucket: string) => Promise<{
+    $response: import("aws-sdk").Response<{}, import("aws-sdk").AWSError>;
+}>;
 /**
 * Delete a list of buckets and all of their objects from S3
 *
@@ -412,6 +414,7 @@ export declare const createS3Buckets: (buckets: Array<string>) => Promise<any>;
  *   Output from https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
  * @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
  * @param {boolean} [params.copyTags=false]
+ * @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
  * @returns {Promise.<{ etag: string }>} object containing the ETag of the
  *    destination object
  */
@@ -423,7 +426,7 @@ export declare const multipartCopyObject: (params: {
     sourceObject?: AWS.S3.HeadObjectOutput;
     ACL?: AWS.S3.ObjectCannedACL;
     copyTags?: boolean;
-    copyMetadata?: boolean;
+    chunkSize?: number;
 }) => Promise<{
     etag: string;
 }>;
@@ -437,6 +440,7 @@ export declare const multipartCopyObject: (params: {
  * @param {string} params.destinationKey
  * @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
  * @param {boolean} [params.copyTags=false]
+ * @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
  * @returns {Promise<undefined>}
  */
 export declare const moveObject: (params: {
@@ -446,6 +450,7 @@ export declare const moveObject: (params: {
     destinationKey: string;
     ACL?: AWS.S3.ObjectCannedACL;
     copyTags?: boolean;
-}) => Promise<void>;
+    chunkSize?: number;
+}) => Promise<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/s3").DeleteObjectOutput, import("aws-sdk").AWSError>>;
 export {};
 //# sourceMappingURL=S3.d.ts.map

package/S3.js

@@ -166,7 +166,7 @@ exports.s3ObjectExists = s3ObjectExists;
  */
 const waitForObjectToExist = async (params) => {
     const { bucket, key, interval = 1000, timeout = 30 * 1000, } = params;
-    await (0, p_wait_for_1.default)(() => (0, exports.s3ObjectExists)({ Bucket: bucket, Key: key }), { interval, timeout });
+    return await (0, p_wait_for_1.default)(() => (0, exports.s3ObjectExists)({ Bucket: bucket, Key: key }), { interval, timeout });
 };
 exports.waitForObjectToExist = waitForObjectToExist;
 /**
@@ -320,9 +320,7 @@ exports.s3PutObjectTagging = (0, utils_1.improveStackTrace)((Bucket, Key, Taggin
  * @returns {Promise<AWS.S3.GetObjectOutput>} response from `AWS.S3.getObject()`
  *   as a Promise
  */
-const getObject = (
-// eslint-disable-next-line no-shadow
-s3Client, params) => s3Client.getObject(params).promise();
+const getObject = (s3Client, params) => s3Client.getObject(params).promise();
 exports.getObject = getObject;
 /**
  * Get an object from S3, waiting for it to exist and, if specified, have the
@@ -423,8 +421,7 @@ exports.getObjectReadStream = getObjectReadStream;
 **/
 const fileExists = async (bucket, key) => {
     try {
-        const r = await (0, services_1.s3)().headObject({ Key: key, Bucket: bucket }).promise();
-        return r;
+        return await (0, services_1.s3)().headObject({ Key: key, Bucket: bucket }).promise();
     }
     catch (error) {
         // if file is not return false
@@ -489,7 +486,7 @@ exports.recursivelyDeleteS3Bucket = (0, utils_1.improveStackTrace)(async (bucket
         };
     });
     await (0, exports.deleteS3Files)(s3Objects);
-    await (0, services_1.s3)().deleteBucket({ Bucket: bucket }).promise();
+    return await (0, services_1.s3)().deleteBucket({ Bucket: bucket }).promise();
 });
 /**
 * Delete a list of buckets and all of their objects from S3
@@ -743,12 +740,13 @@ const uploadPartCopy = async (params) => {
  *   Output from https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
  * @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
  * @param {boolean} [params.copyTags=false]
+ * @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
  * @returns {Promise.<{ etag: string }>} object containing the ETag of the
  *    destination object
  */
 const multipartCopyObject = async (params) => {
     var _a;
-    const { sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags = false, } = params;
+    const { sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags = false, chunkSize, } = params;
     const sourceObject = (_a = params.sourceObject) !== null && _a !== void 0 ? _a : await (0, exports.headObject)(sourceBucket, sourceKey);
     // Create a multi-part upload (copy) and get its UploadId
     const uploadId = await createMultipartUpload({
@@ -766,7 +764,7 @@ const multipartCopyObject = async (params) => {
         if (objectSize === undefined) {
             throw new Error(`Unable to determine size of s3://${sourceBucket}/${sourceKey}`);
         }
-        const chunks = S3MultipartUploads.createMultipartChunks(objectSize);
+        const chunks = S3MultipartUploads.createMultipartChunks(objectSize, chunkSize);
         // Submit all of the upload (copy) parts to S3
         const uploadPartCopyResponses = await Promise.all(chunks.map(({ start, end }, index) => uploadPartCopy({
             uploadId,
@@ -811,6 +809,7 @@ exports.multipartCopyObject = multipartCopyObject;
  * @param {string} params.destinationKey
  * @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
  * @param {boolean} [params.copyTags=false]
+ * @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
  * @returns {Promise<undefined>}
  */
 const moveObject = async (params) => {
@@ -821,8 +820,9 @@ const moveObject = async (params) => {
         destinationKey: params.destinationKey,
         ACL: params.ACL,
         copyTags: (0, isBoolean_1.default)(params.copyTags) ? params.copyTags : true,
+        chunkSize: params.chunkSize,
     });
-    await (0, exports.deleteS3Object)(params.sourceBucket, params.sourceKey);
+    return await (0, exports.deleteS3Object)(params.sourceBucket, params.sourceKey);
 };
 exports.moveObject = moveObject;
 //# sourceMappingURL=S3.js.map

package/StepFunctions.d.ts

@@ -65,7 +65,9 @@ export declare const getExecutionHistory: (params: import("aws-sdk/clients/stepf
     events: import("aws-sdk/clients/stepfunctions").HistoryEventList;
 }>;
 export declare const getExecutionStatus: (executionArn: string) => Promise<{
-    execution: import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/stepfunctions").DescribeExecutionOutput, import("aws-sdk").AWSError>;
+    execution: import("aws-sdk/clients/stepfunctions").DescribeExecutionOutput & {
+        $response: import("aws-sdk").Response<import("aws-sdk/clients/stepfunctions").DescribeExecutionOutput, import("aws-sdk").AWSError>;
+    };
     executionHistory: {
         events: import("aws-sdk/clients/stepfunctions").HistoryEventList;
     };

package/lib/S3MultipartUploads.d.ts

@@ -5,7 +5,24 @@ declare type Chunk = {
     start: number;
     end: number;
 };
-export declare const createMultipartChunks: (objectSize: number, maxChunkSize?: number) => Chunk[];
+/**
+ * Each part of a multi-part copy needs to specify a byte range to be copied.
+ * This byte range has a starting byte and an ending byte (inclusive) that makes
+ * up the part. The maximum allowed chunk size is 5368709120 bytes.
+ *
+ * This function takes a file size and an optional maxSize. It returns an array
+ * of objects, each containing a `start` and an `end` value. These will make up
+ * the ranges of the multi-part copy.
+ *
+ * From anecdotal testing, a chunk size of 250 MB seems to perform fairly well.
+ *
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+ *
+ * @param {number} objectSize - size of the object
+ * @param {number} chunkSize - chunk size of the S3 multipart uploads
+ * @returns {Promise<Array<Chunk>>} - array of chunks
+ */
+export declare const createMultipartChunks: (objectSize: number, chunkSize?: number) => Chunk[];
 export declare const createMultipartUpload: (params: AWS.S3.CreateMultipartUploadRequest) => Promise<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/s3").CreateMultipartUploadOutput, import("aws-sdk").AWSError>>;
 export declare const completeMultipartUpload: (params: AWS.S3.CompleteMultipartUploadRequest) => Promise<CompleteMultipartUploadOutput>;
 export declare const abortMultipartUpload: (params: AWS.S3.AbortMultipartUploadRequest) => Promise<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/s3").AbortMultipartUploadOutput, import("aws-sdk").AWSError>>;

package/lib/S3MultipartUploads.js

@@ -8,21 +8,27 @@ exports.uploadPartCopy = exports.abortMultipartUpload = exports.completeMultipar
 const range_1 = __importDefault(require("lodash/range"));
 const services_1 = require("../services");
 const MB = 1024 * 1024;
-// Each part of a multi-part copy needs to specify a byte range to be copied.
-// This byte range has a starting byte and an ending byte (inclusive) that makes
-// up the part. The maximum allowed chunk size is 5368709120 bytes.
-//
-// This function takes a file size and an optional maxSize. It returns an array
-// of objects, each containing a `start` and an `end` value. These will make up
-// the ranges of the multi-part copy.
-//
-// From anecdotal testing, a chunk size of 250 MB seems to perform fairly well.
-//
-// https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
-const createMultipartChunks = (objectSize, maxChunkSize = 250 * MB) => (0, range_1.default)(0, objectSize, maxChunkSize)
+/**
+ * Each part of a multi-part copy needs to specify a byte range to be copied.
+ * This byte range has a starting byte and an ending byte (inclusive) that makes
+ * up the part. The maximum allowed chunk size is 5368709120 bytes.
+ *
+ * This function takes a file size and an optional maxSize. It returns an array
+ * of objects, each containing a `start` and an `end` value. These will make up
+ * the ranges of the multi-part copy.
+ *
+ * From anecdotal testing, a chunk size of 250 MB seems to perform fairly well.
+ *
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+ *
+ * @param {number} objectSize - size of the object
+ * @param {number} chunkSize - chunk size of the S3 multipart uploads
+ * @returns {Promise<Array<Chunk>>} - array of chunks
+ */
+const createMultipartChunks = (objectSize, chunkSize = 250 * MB) => (0, range_1.default)(0, objectSize, chunkSize)
     .map((start) => ({
     start,
-    end: Math.min(start + maxChunkSize, objectSize) - 1,
+    end: Math.min(start + chunkSize, objectSize) - 1,
 }));
 exports.createMultipartChunks = createMultipartChunks;
 const createMultipartUpload = async (params) => await (0, services_1.s3)().createMultipartUpload(params).promise();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/aws-client",
-  "version": "9.8.0",
+  "version": "10.0.1",
   "description": "Utilities for working with AWS",
   "keywords": [
     "GIBS",
@@ -43,9 +43,9 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/checksum": "9.8.0",
-    "@cumulus/errors": "9.8.0",
-    "@cumulus/logger": "9.8.0",
+    "@cumulus/checksum": "10.0.1",
+    "@cumulus/errors": "10.0.1",
+    "@cumulus/logger": "10.0.1",
     "aws-sdk": "^2.814.0",
     "jsonpath-plus": "^1.1.0",
     "lodash": "~4.17.20",
@@ -54,5 +54,5 @@
     "p-wait-for": "^3.1.0",
     "pump": "^3.0.0"
   },
-  "gitHead": "913034ba6814e562b7f3d58bb39cf086255a6efd"
+  "gitHead": "49c3c88336838184f22f35fbce298c71cd269138"
 }