@cumulus/aws-client 11.1.4 → 11.1.6

package/README.md

@@ -855,7 +855,9 @@ Copy an S3 object to another location in S3 using a multipart copy
 
 **Kind**: inner method of [<code>S3</code>](#module_S3)  
 **Returns**: <code>Promise.&lt;{etag: string}&gt;</code> - object containing the ETag of the
-   destination object  
+   destination object
+
+note: this method may error if used with zero byte files. see CUMULUS-2557 and https://github.com/nasa/cumulus/pull/2117.  
 
 | Param | Type | Default | Description |
 | --- | --- | --- | --- |
@@ -1163,12 +1165,12 @@ Class to use when interacting with S3
 **Kind**: global class  
 
 * [S3ObjectStore](#S3ObjectStore)
-    * [.signGetObject(objectUrl, [options], [queryParams])](#S3ObjectStore+signGetObject) ⇒ <code>Promise.&lt;string&gt;</code>
-    * [.signHeadObject(objectUrl, [options], [queryParams])](#S3ObjectStore+signHeadObject) ⇒ <code>Promise.&lt;string&gt;</code>
+    * [.signGetObject(objectUrl, [options], [queryParams], presignOptions)](#S3ObjectStore+signGetObject) ⇒ <code>Promise.&lt;string&gt;</code>
+    * [.signHeadObject(objectUrl, [options], [queryParams], presignOptions)](#S3ObjectStore+signHeadObject) ⇒ <code>Promise.&lt;string&gt;</code>
 
 <a name="S3ObjectStore+signGetObject"></a>
 
-### s3ObjectStore.signGetObject(objectUrl, [options], [queryParams]) ⇒ <code>Promise.&lt;string&gt;</code>
+### s3ObjectStore.signGetObject(objectUrl, [options], [queryParams], presignOptions) ⇒ <code>Promise.&lt;string&gt;</code>
 Returns an HTTPS URL that can be used to perform a GET on the given object
 store URL
 
@@ -1184,10 +1186,11 @@ store URL
 | objectUrl | <code>string</code> | the URL of the object to sign |
 | [options] | <code>string</code> | options to pass to S3.getObject |
 | [queryParams] | <code>string</code> | a mapping of parameter key/values to put in the URL |
+| presignOptions | <code>RequestPresigningArguments</code> | presignOptions |
 
 <a name="S3ObjectStore+signHeadObject"></a>
 
-### s3ObjectStore.signHeadObject(objectUrl, [options], [queryParams]) ⇒ <code>Promise.&lt;string&gt;</code>
+### s3ObjectStore.signHeadObject(objectUrl, [options], [queryParams], presignOptions) ⇒ <code>Promise.&lt;string&gt;</code>
 Returns an HTTPS URL that can be used to perform a HEAD on the given object
 store URL
 
@@ -1203,6 +1206,7 @@ store URL
 | objectUrl | <code>string</code> | the URL of the object to sign |
 | [options] | <code>string</code> | options to pass to S3.getObject |
 | [queryParams] | <code>string</code> | a mapping of parameter key/values to put in the URL |
+| presignOptions | <code>RequestPresigningArguments</code> | presignOptions |
 
 
 ## About Cumulus

package/S3.d.ts

@@ -409,6 +409,8 @@ export declare const createS3Buckets: (buckets: Array<string>) => Promise<any>;
  * @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
  * @returns {Promise.<{ etag: string }>} object containing the ETag of the
  *    destination object
+ *
+ * note: this method may error if used with zero byte files. see CUMULUS-2557 and https://github.com/nasa/cumulus/pull/2117.
  */
 export declare const multipartCopyObject: (params: {
     sourceBucket: string;

package/S3.js

@@ -800,6 +800,8 @@ const uploadPartCopy = async (params) => {
  * @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
  * @returns {Promise.<{ etag: string }>} object containing the ETag of the
  *    destination object
+ *
+ * note: this method may error if used with zero byte files. see CUMULUS-2557 and https://github.com/nasa/cumulus/pull/2117.
  */
 const multipartCopyObject = async (params) => {
     var _a;
@@ -870,16 +872,38 @@ exports.multipartCopyObject = multipartCopyObject;
  * @returns {Promise<undefined>}
  */
 const moveObject = async (params) => {
-    await (0, exports.multipartCopyObject)({
-        sourceBucket: params.sourceBucket,
-        sourceKey: params.sourceKey,
-        destinationBucket: params.destinationBucket,
-        destinationKey: params.destinationKey,
-        ACL: params.ACL,
-        copyTags: (0, isBoolean_1.default)(params.copyTags) ? params.copyTags : true,
-        chunkSize: params.chunkSize,
-    });
-    return await (0, exports.deleteS3Object)(params.sourceBucket, params.sourceKey);
+    const { sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags, chunkSize, } = params;
+    const sourceObject = await (0, exports.headObject)(sourceBucket, sourceKey);
+    if (sourceObject.ContentLength === 0) {
+        // 0 byte files cannot be copied with multipart upload,
+        // so use a regular S3 PUT
+        const s3uri = (0, exports.buildS3Uri)(destinationBucket, destinationKey);
+        const { CopyObjectResult } = await (0, exports.s3CopyObject)({
+            CopySource: path_1.default.join(sourceBucket, sourceKey),
+            Bucket: destinationBucket,
+            Key: destinationKey,
+        });
+        // This error should never actually be reached in practice. It's a
+        // necessary workaround for bad typings in the AWS SDK.
+        // https://github.com/aws/aws-sdk-js/issues/1719
+        if (!CopyObjectResult || !CopyObjectResult.ETag) {
+            throw new Error(`ETag could not be determined for copy of ${(0, exports.buildS3Uri)(sourceBucket, sourceKey)} to ${s3uri}`);
+        }
+    }
+    else {
+        await (0, exports.multipartCopyObject)({
+            sourceBucket: sourceBucket,
+            sourceKey: sourceKey,
+            destinationBucket: destinationBucket,
+            destinationKey: destinationKey,
+            sourceObject: sourceObject,
+            ACL: ACL,
+            copyTags: (0, isBoolean_1.default)(copyTags) ? copyTags : true,
+            chunkSize: chunkSize,
+        });
+    }
+    const deleteS3ObjRes = await (0, exports.deleteS3Object)(sourceBucket, sourceKey);
+    return deleteS3ObjRes;
 };
 exports.moveObject = moveObject;
 //# sourceMappingURL=S3.js.map

package/S3ObjectStore.d.ts

@@ -1,6 +1,7 @@
 import { GetObjectCommand, HeadObjectCommand, GetObjectCommandInput, S3ClientConfig } from '@aws-sdk/client-s3';
+import { RequestPresigningArguments } from '@aws-sdk/types';
 declare type QueryParams = {
-    [key: string]: number | string;
+    [key: string]: string;
 };
 /**
  * Class to use when interacting with S3
@@ -13,7 +14,7 @@ declare class S3ObjectStore {
     constructor(config?: Partial<S3ClientConfig>);
     getQueryParams(): QueryParams;
     setQueryParams(queryParams: QueryParams): void;
-    getS3SignedUrlWithCustomQueryParams(command: GetObjectCommand | HeadObjectCommand): Promise<string>;
+    getS3SignedUrlWithCustomQueryParams(command: GetObjectCommand | HeadObjectCommand, presignOptions?: RequestPresigningArguments): Promise<string>;
     /**
      * Returns an HTTPS URL that can be used to perform a GET on the given object
      * store URL
@@ -21,10 +22,11 @@ declare class S3ObjectStore {
      * @param {string} objectUrl - the URL of the object to sign
      * @param {string} [options] - options to pass to S3.getObject
      * @param {string} [queryParams] - a mapping of parameter key/values to put in the URL
+     * @param {RequestPresigningArguments} presignOptions - presignOptions
      * @returns {Promise<string>} a signed URL
      * @throws TypeError - if the URL is not a recognized protocol or cannot be parsed
      */
-    signGetObject(objectUrl: string, options?: Partial<GetObjectCommandInput>, queryParams?: QueryParams): Promise<string>;
+    signGetObject(objectUrl: string, options?: Partial<GetObjectCommandInput>, queryParams?: QueryParams, presignOptions?: RequestPresigningArguments): Promise<string>;
     /**
      * Returns an HTTPS URL that can be used to perform a HEAD on the given object
      * store URL
@@ -32,12 +34,13 @@ declare class S3ObjectStore {
      * @param {string} objectUrl - the URL of the object to sign
      * @param {string} [options] - options to pass to S3.getObject
      * @param {string} [queryParams] - a mapping of parameter key/values to put in the URL
+     * @param {RequestPresigningArguments} presignOptions - presignOptions
      * @returns {Promise<string>} a signed URL
      * @throws TypeError - if the URL is not a recognized protocol or cannot be parsed
      */
     signHeadObject(objectUrl: string, options: {
         [key: string]: string;
-    } | undefined, queryParams: QueryParams): Promise<string>;
+    } | undefined, queryParams: QueryParams, presignOptions?: RequestPresigningArguments): Promise<string>;
 }
 export = S3ObjectStore;
 //# sourceMappingURL=S3ObjectStore.d.ts.map

package/S3ObjectStore.js

@@ -11,6 +11,7 @@ const S3_1 = require("./S3");
 const client_1 = __importDefault(require("./client"));
 // Code modified from https://github.com/nasa/harmony/blob/main/app/util/object-store.ts
 const log = new logger_1.default({ sender: '@cumulus/aws-client/S3ObjectStore' });
+const S3_LINK_EXPIRY_SECONDS_DEFAULT = 3600;
 /**
  * Class to use when interacting with S3
  *
@@ -29,7 +30,7 @@ class S3ObjectStore {
     setQueryParams(queryParams) {
         this.queryParams = queryParams;
     }
-    async getS3SignedUrlWithCustomQueryParams(command) {
+    async getS3SignedUrlWithCustomQueryParams(command, presignOptions = {}) {
         this.s3.middlewareStack.addRelativeTo((next) => (args) => {
             const { request } = args;
             request.query = {
@@ -42,7 +43,7 @@ class S3ObjectStore {
             relation: 'before',
             toMiddleware: 'presignInterceptMiddleware',
         });
-        const signedUrl = await (0, s3_request_presigner_1.getSignedUrl)(this.s3, command);
+        const signedUrl = await (0, s3_request_presigner_1.getSignedUrl)(this.s3, command, { expiresIn: S3_LINK_EXPIRY_SECONDS_DEFAULT, ...presignOptions });
         this.s3.middlewareStack.remove(this.middlewareName);
         return signedUrl;
     }
@@ -53,10 +54,11 @@ class S3ObjectStore {
      * @param {string} objectUrl - the URL of the object to sign
      * @param {string} [options] - options to pass to S3.getObject
      * @param {string} [queryParams] - a mapping of parameter key/values to put in the URL
+     * @param {RequestPresigningArguments} presignOptions - presignOptions
      * @returns {Promise<string>} a signed URL
      * @throws TypeError - if the URL is not a recognized protocol or cannot be parsed
      */
-    async signGetObject(objectUrl, options = {}, queryParams = {}) {
+    async signGetObject(objectUrl, options = {}, queryParams = {}, presignOptions = {}) {
         log.info(`Executing signGetObject with objectUrl: ${objectUrl}, options: ${JSON.stringify(options)}, queryParams: ${JSON.stringify(queryParams)}`);
         const url = new url_1.URL(objectUrl);
         if (url.protocol.toLowerCase() !== 's3:') {
@@ -66,7 +68,7 @@ class S3ObjectStore {
         await (0, S3_1.headObject)(Bucket, Key);
         const command = new client_s3_1.GetObjectCommand({ Bucket, Key, ...options });
         this.setQueryParams(queryParams);
-        const signedUrl = await this.getS3SignedUrlWithCustomQueryParams(command);
+        const signedUrl = await this.getS3SignedUrlWithCustomQueryParams(command, presignOptions);
         log.debug(`Signed GetObject request URL: ${signedUrl}`);
         return signedUrl;
     }
@@ -77,10 +79,11 @@ class S3ObjectStore {
      * @param {string} objectUrl - the URL of the object to sign
      * @param {string} [options] - options to pass to S3.getObject
      * @param {string} [queryParams] - a mapping of parameter key/values to put in the URL
+     * @param {RequestPresigningArguments} presignOptions - presignOptions
      * @returns {Promise<string>} a signed URL
      * @throws TypeError - if the URL is not a recognized protocol or cannot be parsed
      */
-    async signHeadObject(objectUrl, options = {}, queryParams) {
+    async signHeadObject(objectUrl, options = {}, queryParams, presignOptions = {}) {
         log.info(`Executing signHeadObject with objectUrl: ${objectUrl}, options: ${JSON.stringify(options)}, queryParams: ${JSON.stringify(queryParams)}`);
         const url = new url_1.URL(objectUrl);
         if (url.protocol.toLowerCase() !== 's3:') {
@@ -89,7 +92,7 @@ class S3ObjectStore {
         const { Bucket, Key } = (0, S3_1.parseS3Uri)(objectUrl);
         const command = new client_s3_1.HeadObjectCommand({ Bucket, Key, ...options });
         this.setQueryParams(queryParams);
-        const signedUrl = await this.getS3SignedUrlWithCustomQueryParams(command);
+        const signedUrl = await this.getS3SignedUrlWithCustomQueryParams(command, presignOptions);
         log.debug(`Signed HeadObject request URL: ${signedUrl}`);
         return signedUrl;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/aws-client",
-  "version": "11.1.4",
+  "version": "11.1.6",
   "description": "Utilities for working with AWS",
   "keywords": [
     "GIBS",
@@ -51,9 +51,9 @@
     "@aws-sdk/s3-request-presigner": "^3.53.0",
     "@aws-sdk/signature-v4-crt": "^3.53.0",
     "@aws-sdk/types": "^3.53.0",
-    "@cumulus/checksum": "11.1.4",
-    "@cumulus/errors": "11.1.4",
-    "@cumulus/logger": "11.1.4",
+    "@cumulus/checksum": "11.1.6",
+    "@cumulus/errors": "11.1.6",
+    "@cumulus/logger": "11.1.6",
     "aws-sdk": "^2.585.0",
     "jsonpath-plus": "^1.1.0",
     "lodash": "~4.17.21",
@@ -68,5 +68,5 @@
   "devDependencies": {
     "@types/uuid": "^8.0.0"
   },
-  "gitHead": "0c2cc0fa8bb66b2d668c95df1cc11a1a8c1d4332"
+  "gitHead": "cc17225ec1ded9cc2d2b1decc952da97ce0df072"
 }

package/test-utils.d.ts

@@ -1,3 +1,4 @@
+import { Readable } from 'stream';
 import { AWSClientTypes } from './types';
 export declare const inTestMode: () => boolean;
 declare const localStackPorts: {
@@ -75,5 +76,6 @@ export declare function getLocalstackAwsClientOptions<T extends AWSClientTypes>(
  * @private
  */
 export declare const throttleOnce: (fn: (...args: unknown[]) => unknown) => (...args: unknown[]) => unknown;
+export declare const streamToString: (stream: Readable) => Promise<unknown>;
 export {};
 //# sourceMappingURL=test-utils.d.ts.map

package/test-utils.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.throttleOnce = exports.getLocalstackAwsClientOptions = exports.localStackAwsClientOptions = exports.getLocalstackEndpoint = exports.inTestMode = void 0;
+exports.streamToString = exports.throttleOnce = exports.getLocalstackAwsClientOptions = exports.localStackAwsClientOptions = exports.getLocalstackEndpoint = exports.inTestMode = void 0;
 const errors_1 = require("@cumulus/errors");
 const utils_1 = require("./utils");
 const inTestMode = () => process.env.NODE_ENV === 'test';
@@ -130,4 +130,13 @@ const throttleOnce = (fn) => {
     };
 };
 exports.throttleOnce = throttleOnce;
+const streamToString = (stream) => {
+    let result = '';
+    // eslint-disable-next-line no-return-assign
+    stream.on('data', (chunk) => result += chunk.toString());
+    return new Promise((resolve) => {
+        stream.on('end', () => resolve(result));
+    });
+};
+exports.streamToString = streamToString;
 //# sourceMappingURL=test-utils.js.map