@cumulus/aws-client 11.1.0 → 11.1.1

package/S3ObjectStore.js

@@ -1,33 +1,11 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
-const querystring = __importStar(require("querystring"));
 const url_1 = require("url");
-const AWS = __importStar(require("aws-sdk"));
+const uuid_1 = require("uuid");
+const client_s3_1 = require("@aws-sdk/client-s3");
+const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
 const logger_1 = __importDefault(require("@cumulus/logger"));
 const S3_1 = require("./S3");
 const client_1 = __importDefault(require("./client"));
@@ -38,8 +16,35 @@ const log = new logger_1.default({ sender: '@cumulus/aws-client/S3ObjectStore' }
  *
  */
 class S3ObjectStore {
-    constructor() {
-        this.s3 = (0, client_1.default)(AWS.S3, '2006-03-01', { signatureVersion: 'v4' })();
+    constructor(config) {
+        this.s3 = (0, client_1.default)(client_s3_1.S3, '2006-03-01', {
+            signatureVersion: 'v4',
+        })(config);
+        this.queryParams = {};
+        this.middlewareName = `customQueryParams${(0, uuid_1.v4)()}`;
+    }
+    getQueryParams() {
+        return this.queryParams;
+    }
+    setQueryParams(queryParams) {
+        this.queryParams = queryParams;
+    }
+    async getS3SignedUrlWithCustomQueryParams(command) {
+        this.s3.middlewareStack.addRelativeTo((next) => (args) => {
+            const { request } = args;
+            request.query = {
+                ...this.getQueryParams(),
+                ...request.query,
+            };
+            return next(args);
+        }, {
+            name: this.middlewareName,
+            relation: 'before',
+            toMiddleware: 'presignInterceptMiddleware',
+        });
+        const signedUrl = await (0, s3_request_presigner_1.getSignedUrl)(this.s3, command);
+        this.s3.middlewareStack.remove(this.middlewareName);
+        return signedUrl;
     }
     /**
      * Returns an HTTPS URL that can be used to perform a GET on the given object
@@ -51,7 +56,7 @@ class S3ObjectStore {
      * @returns {Promise<string>} a signed URL
      * @throws TypeError - if the URL is not a recognized protocol or cannot be parsed
      */
-    async signGetObject(objectUrl, options = {}, queryParams) {
+    async signGetObject(objectUrl, options = {}, queryParams = {}) {
         log.info(`Executing signGetObject with objectUrl: ${objectUrl}, options: ${JSON.stringify(options)}, queryParams: ${JSON.stringify(queryParams)}`);
         const url = new url_1.URL(objectUrl);
         if (url.protocol.toLowerCase() !== 's3:') {
@@ -59,14 +64,11 @@ class S3ObjectStore {
         }
         const { Bucket, Key } = (0, S3_1.parseS3Uri)(objectUrl);
         await (0, S3_1.headObject)(Bucket, Key);
-        const req = this.s3.getObject({ Bucket, Key, ...options });
-        if (queryParams && req.on) {
-            (req.on('build', () => { req.httpRequest.path += `${options ? '&' : '?'}${querystring.stringify(queryParams)}`; }));
-        }
-        // TypeScript doesn't recognize that req has a presign method.  It does.
-        const result = await req.presign();
-        log.debug(`Signed GetObject request URL: ${result}`);
-        return result;
+        const command = new client_s3_1.GetObjectCommand({ Bucket, Key, ...options });
+        this.setQueryParams(queryParams);
+        const signedUrl = await this.getS3SignedUrlWithCustomQueryParams(command);
+        log.debug(`Signed GetObject request URL: ${signedUrl}`);
+        return signedUrl;
     }
     /**
      * Returns an HTTPS URL that can be used to perform a HEAD on the given object
@@ -85,14 +87,11 @@ class S3ObjectStore {
             throw new TypeError(`Invalid S3 URL: ${objectUrl}`);
         }
         const { Bucket, Key } = (0, S3_1.parseS3Uri)(objectUrl);
-        const req = this.s3.headObject({ Bucket, Key, ...options });
-        if (queryParams && req.on) {
-            (req.on('build', () => { req.httpRequest.path += `?${querystring.stringify(queryParams)}`; }));
-        }
-        // TypeScript doesn't recognize that req has a presign method.  It does.
-        const result = await req.presign();
-        log.debug(`Signed HeadObject request URL: ${result}`);
-        return result;
+        const command = new client_s3_1.HeadObjectCommand({ Bucket, Key, ...options });
+        this.setQueryParams(queryParams);
+        const signedUrl = await this.getS3SignedUrlWithCustomQueryParams(command);
+        log.debug(`Signed HeadObject request URL: ${signedUrl}`);
+        return signedUrl;
     }
 }
 module.exports = S3ObjectStore;

package/client.js

@@ -6,11 +6,7 @@ const aws_sdk_1 = __importDefault(require("aws-sdk"));
 const mem_1 = __importDefault(require("mem"));
 const test_utils_1 = require("./test-utils");
 const utils_1 = require("./utils");
-const noop = () => { }; // eslint-disable-line lodash/prefer-noop
 const getRegion = () => process.env.AWS_DEFAULT_REGION || process.env.AWS_REGION || 'us-east-1';
-// Workaround upload hangs. See: https://github.com/andrewrk/node-s3-client/issues/74
-// @ts-ignore - AWS.util is not part of the public API and may break
-aws_sdk_1.default.util.update(aws_sdk_1.default.S3.prototype, { addExpect100Continue: noop });
 aws_sdk_1.default.config.setPromisesDependency(Promise);
 const buildServiceClient = (Service, options) => {
     if ((0, test_utils_1.inTestMode)()) {

package/lib/S3MultipartUploads.d.ts

@@ -1,3 +1,4 @@
+import { AbortMultipartUploadRequest, CompleteMultipartUploadRequest, CreateMultipartUploadRequest, UploadPartCopyRequest } from '@aws-sdk/client-s3';
 export interface CompleteMultipartUploadOutput extends AWS.S3.CompleteMultipartUploadOutput {
     ETag: string;
 }
@@ -23,9 +24,9 @@ declare type Chunk = {
  * @returns {Promise<Array<Chunk>>} - array of chunks
  */
 export declare const createMultipartChunks: (objectSize: number, chunkSize?: number) => Chunk[];
-export declare const createMultipartUpload: (params: AWS.S3.CreateMultipartUploadRequest) => Promise<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/s3").CreateMultipartUploadOutput, import("aws-sdk").AWSError>>;
-export declare const completeMultipartUpload: (params: AWS.S3.CompleteMultipartUploadRequest) => Promise<CompleteMultipartUploadOutput>;
-export declare const abortMultipartUpload: (params: AWS.S3.AbortMultipartUploadRequest) => Promise<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/s3").AbortMultipartUploadOutput, import("aws-sdk").AWSError>>;
-export declare const uploadPartCopy: (params: AWS.S3.UploadPartCopyRequest) => Promise<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/s3").UploadPartCopyOutput, import("aws-sdk").AWSError>>;
+export declare const createMultipartUpload: (params: CreateMultipartUploadRequest) => Promise<import("@aws-sdk/client-s3").CreateMultipartUploadCommandOutput>;
+export declare const completeMultipartUpload: (params: CompleteMultipartUploadRequest) => Promise<CompleteMultipartUploadOutput>;
+export declare const abortMultipartUpload: (params: AbortMultipartUploadRequest) => Promise<import("@aws-sdk/client-s3").AbortMultipartUploadCommandOutput>;
+export declare const uploadPartCopy: (params: UploadPartCopyRequest) => Promise<import("@aws-sdk/client-s3").UploadPartCopyCommandOutput>;
 export {};
 //# sourceMappingURL=S3MultipartUploads.d.ts.map

package/lib/S3MultipartUploads.js

@@ -31,15 +31,15 @@ const createMultipartChunks = (objectSize, chunkSize = 250 * MB) => (0, range_1.
     end: Math.min(start + chunkSize, objectSize) - 1,
 }));
 exports.createMultipartChunks = createMultipartChunks;
-const createMultipartUpload = async (params) => await (0, services_1.s3)().createMultipartUpload(params).promise();
+const createMultipartUpload = async (params) => await (0, services_1.s3)().createMultipartUpload(params);
 exports.createMultipartUpload = createMultipartUpload;
 const completeMultipartUpload = async (params) => {
-    const result = await (0, services_1.s3)().completeMultipartUpload(params).promise();
+    const result = await (0, services_1.s3)().completeMultipartUpload(params);
     return result;
 };
 exports.completeMultipartUpload = completeMultipartUpload;
-const abortMultipartUpload = async (params) => await (0, services_1.s3)().abortMultipartUpload(params).promise();
+const abortMultipartUpload = async (params) => await (0, services_1.s3)().abortMultipartUpload(params);
 exports.abortMultipartUpload = abortMultipartUpload;
-const uploadPartCopy = async (params) => await (0, services_1.s3)().uploadPartCopy(params).promise();
+const uploadPartCopy = async (params) => await (0, services_1.s3)().uploadPartCopy(params);
 exports.uploadPartCopy = uploadPartCopy;
 //# sourceMappingURL=S3MultipartUploads.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/aws-client",
-  "version": "11.1.0",
+  "version": "11.1.1",
   "description": "Utilities for working with AWS",
   "keywords": [
     "GIBS",
@@ -45,11 +45,15 @@
   "dependencies": {
     "@aws-sdk/client-dynamodb": "^3.53.0",
     "@aws-sdk/client-dynamodb-streams": "^3.53.0",
+    "@aws-sdk/client-s3": "^3.53.0",
     "@aws-sdk/lib-dynamodb": "^3.53.0",
+    "@aws-sdk/lib-storage": "^3.53.0",
+    "@aws-sdk/s3-request-presigner": "^3.53.0",
+    "@aws-sdk/signature-v4-crt": "^3.53.0",
     "@aws-sdk/types": "^3.53.0",
-    "@cumulus/checksum": "11.1.0",
-    "@cumulus/errors": "11.1.0",
-    "@cumulus/logger": "11.1.0",
+    "@cumulus/checksum": "11.1.1",
+    "@cumulus/errors": "11.1.1",
+    "@cumulus/logger": "11.1.1",
     "aws-sdk": "^2.585.0",
     "jsonpath-plus": "^1.1.0",
     "lodash": "~4.17.21",
@@ -58,7 +62,11 @@
     "p-retry": "^4.2.0",
     "p-timeout": "^4.1.0",
     "p-wait-for": "^3.2.0",
-    "pump": "^3.0.0"
+    "pump": "^3.0.0",
+    "uuid": "^8.2.0"
   },
-  "gitHead": "a846e319e143684ebc408e35f8b51cc1a7195391"
+  "devDependencies": {
+    "@types/uuid": "^8.0.0"
+  },
+  "gitHead": "07fe682da23821434372759fb1948b081da429b5"
 }

package/services.d.ts

@@ -1,11 +1,12 @@
 import { DynamoDB } from '@aws-sdk/client-dynamodb';
 import { DynamoDBStreamsClient } from '@aws-sdk/client-dynamodb-streams';
+import { S3 } from '@aws-sdk/client-s3';
 import { DynamoDBDocument, TranslateConfig } from '@aws-sdk/lib-dynamodb';
 import * as AWS from 'aws-sdk';
 export declare const apigateway: (params?: object | undefined) => AWS.APIGateway;
 export declare const ecs: (params?: object | undefined) => AWS.ECS;
 export declare const ec2: (params?: object | undefined) => AWS.EC2;
-export declare const s3: (params?: object | undefined) => AWS.S3;
+export declare const s3: (params?: object | undefined) => S3;
 export declare const kinesis: (params?: object | undefined) => AWS.Kinesis;
 export declare const lambda: (params?: object | undefined) => AWS.Lambda;
 export declare const sqs: (params?: object | undefined) => AWS.SQS;

package/services.js

@@ -29,13 +29,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.sts = exports.es = exports.kms = exports.systemsManager = exports.secretsManager = exports.sns = exports.cf = exports.sfn = exports.dynamodbDocClient = exports.dynamodbstreams = exports.dynamodb = exports.cloudwatch = exports.cloudwatchlogs = exports.cloudwatchevents = exports.sqs = exports.lambda = exports.kinesis = exports.s3 = exports.ec2 = exports.ecs = exports.apigateway = void 0;
 const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
 const client_dynamodb_streams_1 = require("@aws-sdk/client-dynamodb-streams");
+const client_s3_1 = require("@aws-sdk/client-s3");
 const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
 const AWS = __importStar(require("aws-sdk"));
 const client_1 = __importDefault(require("./client"));
 exports.apigateway = (0, client_1.default)(AWS.APIGateway, '2015-07-09');
 exports.ecs = (0, client_1.default)(AWS.ECS, '2014-11-13');
 exports.ec2 = (0, client_1.default)(AWS.EC2, '2016-11-15');
-exports.s3 = (0, client_1.default)(AWS.S3, '2006-03-01');
+exports.s3 = (0, client_1.default)(client_s3_1.S3, '2006-03-01');
 exports.kinesis = (0, client_1.default)(AWS.Kinesis, '2013-12-02');
 exports.lambda = (0, client_1.default)(AWS.Lambda, '2015-03-31');
 exports.sqs = (0, client_1.default)(AWS.SQS, '2012-11-05');

package/test-utils.d.ts

@@ -20,7 +20,7 @@ declare const localStackPorts: {
     lambda: number;
     redshift: number;
     route53: number;
-    s3: number;
+    S3: number;
     secretsmanager: number;
     ses: number;
     sns: number;
@@ -48,7 +48,9 @@ export declare function getLocalstackEndpoint(identifier: keyof typeof localStac
  *
  * @private
  */
-export declare function localStackAwsClientOptions<T>(Service: new (params: object) => T, options?: object): {
+export declare function localStackAwsClientOptions<T>(Service: new (params: object) => T, options?: {
+    credentials?: object;
+}): {
     [key: string]: unknown;
 };
 /**
@@ -60,7 +62,9 @@ export declare function localStackAwsClientOptions<T>(Service: new (params: obje
  *
  * @private
  */
-export declare function getLocalstackAwsClientOptions<T extends AWSClientTypes>(Service: new (params: object) => T, options?: object): object;
+export declare function getLocalstackAwsClientOptions<T extends AWSClientTypes>(Service: new (params: object) => T, options?: {
+    credentials?: object;
+}): object;
 /**
  * Return a function that throws a ThrottlingException the first time it is called, then returns as
  * normal any other times.

package/test-utils.js

@@ -26,7 +26,7 @@ const localStackPorts = {
     lambda: 4566,
     redshift: 4566,
     route53: 4566,
-    s3: 4566,
+    S3: 4566,
     secretsmanager: 4566,
     ses: 4566,
     sns: 4566,
@@ -79,16 +79,17 @@ function localStackAwsClientOptions(Service, options = {}) {
     }
     const serviceIdentifier = (0, utils_1.getServiceIdentifer)(Service);
     const localStackOptions = {
+        region: 'us-east-1',
+        endpoint: getLocalstackEndpoint(serviceIdentifier),
         ...options,
         credentials: {
             accessKeyId: 'my-access-key-id',
             secretAccessKey: 'my-secret-access-key',
+            ...options.credentials,
         },
-        region: 'us-east-1',
-        endpoint: getLocalstackEndpoint(serviceIdentifier),
     };
     if (serviceIdentifier.toLowerCase() === 's3')
-        localStackOptions.s3ForcePathStyle = true;
+        localStackOptions.forcePathStyle = true;
     return localStackOptions;
 }
 exports.localStackAwsClientOptions = localStackAwsClientOptions;
@@ -101,7 +102,7 @@ exports.localStackAwsClientOptions = localStackAwsClientOptions;
  *
  * @private
  */
-function getLocalstackAwsClientOptions(Service, options) {
+function getLocalstackAwsClientOptions(Service, options = {}) {
     const serviceIdentifier = (0, utils_1.getServiceIdentifer)(Service);
     if (localstackSupportedService(serviceIdentifier)) {
         return localStackAwsClientOptions(Service, options);

package/types.d.ts

@@ -1,5 +1,6 @@
 import * as AWS from 'aws-sdk';
 import { DynamoDBStreamsClient } from '@aws-sdk/client-dynamodb-streams';
 import { DynamoDB, DynamoDBClient } from '@aws-sdk/client-dynamodb';
-export declare type AWSClientTypes = DynamoDB | DynamoDBClient | DynamoDBStreamsClient | AWS.Service | AWS.DynamoDB.DocumentClient;
+import { S3 } from '@aws-sdk/client-s3';
+export declare type AWSClientTypes = DynamoDB | DynamoDBClient | DynamoDBStreamsClient | S3 | AWS.Service | AWS.DynamoDB.DocumentClient;
 //# sourceMappingURL=types.d.ts.map