@cumulus/aws-client 9.2.3 → 9.4.1

package/README.md

@@ -328,6 +328,7 @@ Invoke a Lambda function
     * [.s3ObjectExists(params)](#module_S3.s3ObjectExists) ⇒ <code>Promise.&lt;boolean&gt;</code>
     * [.waitForObjectToExist(params)](#module_S3.waitForObjectToExist) ⇒ <code>Promise.&lt;undefined&gt;</code>
     * [.putFile(bucket, key, filename)](#module_S3.putFile) ⇒ <code>Promise</code>
+    * [.streamS3Upload(uploadStream, uploadParams)](#module_S3.streamS3Upload) ⇒ <code>Promise</code>
     * [.downloadS3File(s3Obj, filepath)](#module_S3.downloadS3File) ⇒ <code>Promise.&lt;string&gt;</code>
     * [.getObjectSize(params)](#module_S3.getObjectSize) ⇒ <code>Promise.&lt;(number\|undefined)&gt;</code>
     * [.getObject(s3, params)](#module_S3.getObject) ⇒ <code>Promise.&lt;AWS.S3.GetObjectOutput&gt;</code>
@@ -559,6 +560,22 @@ Upload a file to S3
 | key | <code>string</code> | the destination S3 key |
 | filename | <code>filename</code> | the local file to be uploaded |
 
+<a name="module_S3.streamS3Upload"></a>
+
+### S3.streamS3Upload(uploadStream, uploadParams) ⇒ <code>Promise</code>
+Upload data to S3 using a stream
+
+We are not using `s3.upload().promise()` due to errors observed in testing
+with uncaught exceptions. By creating our own promise, we can ensure any
+errors from the streams or upload cause this promise to reject.
+
+**Kind**: static method of [<code>S3</code>](#module_S3)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| uploadStream | <code>Readable</code> | Stream of data to upload |
+| uploadParams | <code>Object</code> | see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property) |
+
 <a name="module_S3.downloadS3File"></a>
 
 ### S3.downloadS3File(s3Obj, filepath) ⇒ <code>Promise.&lt;string&gt;</code>
@@ -881,7 +898,7 @@ errors, to allow more specific handling by the caller.
     * [.createQueue(QueueName)](#module_SQS.createQueue) ⇒ <code>Promise.&lt;string&gt;</code>
     * [.sendSQSMessage(queueUrl, message)](#module_SQS.sendSQSMessage) ⇒ <code>Promise</code>
     * [.receiveSQSMessages(queueUrl, options)](#module_SQS.receiveSQSMessages) ⇒ <code>Promise.&lt;Array&gt;</code>
-    * [.sqsQueueExists(queue)](#module_SQS.sqsQueueExists) ⇒ <code>Promise.&lt;boolean&gt;</code>
+    * [.sqsQueueExists(queueUrl)](#module_SQS.sqsQueueExists) ⇒ <code>Promise.&lt;boolean&gt;</code>
 
 <a name="module_SQS.deleteSQSMessage"></a>
 
@@ -940,16 +957,16 @@ can be set and the timeout is also adjustable.
 
 <a name="module_SQS.sqsQueueExists"></a>
 
-### SQS.sqsQueueExists(queue) ⇒ <code>Promise.&lt;boolean&gt;</code>
+### SQS.sqsQueueExists(queueUrl) ⇒ <code>Promise.&lt;boolean&gt;</code>
 Test if an SQS queue exists
 
 **Kind**: static method of [<code>SQS</code>](#module_SQS)  
-**Returns**: <code>Promise.&lt;boolean&gt;</code> - a Promise that will resolve to a boolean indicating
+**Returns**: <code>Promise.&lt;boolean&gt;</code> - - a Promise that will resolve to a boolean indicating
                               if the queue exists  
 
 | Param | Type | Description |
 | --- | --- | --- |
-| queue | <code>Object</code> | queue name or url |
+| queueUrl | <code>Object</code> | queue url |
 
 <a name="module_SecretsManager"></a>
 

package/S3.d.ts

@@ -126,6 +126,18 @@ export declare const s3CopyObject: (params: import("aws-sdk/clients/s3").CopyObj
  * @returns {Promise} see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
  */
 export declare const promiseS3Upload: (params: import("aws-sdk/clients/s3").PutObjectRequest) => Promise<import("aws-sdk/clients/s3").ManagedUpload.SendData>;
+/**
+ * Upload data to S3 using a stream
+ *
+ * We are not using `s3.upload().promise()` due to errors observed in testing
+ * with uncaught exceptions. By creating our own promise, we can ensure any
+ * errors from the streams or upload cause this promise to reject.
+ *
+ * @param {Readable} uploadStream - Stream of data to upload
+ * @param {Object} uploadParams - see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
+ * @returns {Promise}
+ */
+export declare const streamS3Upload: (uploadStream: Readable, uploadParams: AWS.S3.PutObjectRequest) => Promise<unknown>;
 /**
  * Downloads the given s3Obj to the given filename in a streaming manner
  *

package/S3.js

@@ -25,7 +25,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.moveObject = exports.multipartCopyObject = exports.createS3Buckets = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.deleteS3Buckets = exports.recursivelyDeleteS3Bucket = exports.deleteS3Files = exports.downloadS3Files = exports.fileExists = exports.getObjectReadStream = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
+exports.moveObject = exports.multipartCopyObject = exports.createS3Buckets = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.deleteS3Buckets = exports.recursivelyDeleteS3Bucket = exports.deleteS3Files = exports.downloadS3Files = exports.fileExists = exports.getObjectReadStream = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.streamS3Upload = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
 const fs_1 = __importDefault(require("fs"));
 const isBoolean_1 = __importDefault(require("lodash/isBoolean"));
 const path_1 = __importDefault(require("path"));
@@ -34,6 +34,7 @@ const p_retry_1 = __importDefault(require("p-retry"));
 const p_wait_for_1 = __importDefault(require("p-wait-for"));
 const pump_1 = __importDefault(require("pump"));
 const querystring_1 = __importDefault(require("querystring"));
+const stream_1 = require("stream");
 const util_1 = require("util");
 const checksum_1 = require("@cumulus/checksum");
 const errors_1 = require("@cumulus/errors");
@@ -204,6 +205,32 @@ exports.s3CopyObject = utils_1.improveStackTrace((params) => services_1.s3().cop
  * @returns {Promise} see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
  */
 exports.promiseS3Upload = utils_1.improveStackTrace((params) => services_1.s3().upload(params).promise());
+/**
+ * Upload data to S3 using a stream
+ *
+ * We are not using `s3.upload().promise()` due to errors observed in testing
+ * with uncaught exceptions. By creating our own promise, we can ensure any
+ * errors from the streams or upload cause this promise to reject.
+ *
+ * @param {Readable} uploadStream - Stream of data to upload
+ * @param {Object} uploadParams - see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
+ * @returns {Promise}
+ */
+exports.streamS3Upload = (uploadStream, uploadParams) => new Promise((resolve, reject) => {
+    const pass = new stream_1.PassThrough();
+    uploadStream.pipe(pass);
+    uploadStream.on('error', reject);
+    pass.on('error', reject);
+    return services_1.s3().upload({
+        ...uploadParams,
+        Body: pass,
+    }, (err, uploadResponse) => {
+        if (err) {
+            return reject(err);
+        }
+        return resolve(uploadResponse);
+    });
+});
 /**
  * Downloads the given s3Obj to the given filename in a streaming manner
  *

package/S3ObjectStore.js

@@ -24,9 +24,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 const querystring = __importStar(require("querystring"));
 const url_1 = require("url");
 const AWS = __importStar(require("aws-sdk"));
+const logger_1 = __importDefault(require("@cumulus/logger"));
 const S3_1 = require("./S3");
 const client_1 = __importDefault(require("./client"));
 // Code modified from https://github.com/nasa/harmony/blob/main/app/util/object-store.ts
+const log = new logger_1.default({ sender: '@cumulus/aws-client/S3ObjectStore' });
 /**
  * Class to use when interacting with S3
  *
@@ -46,6 +48,7 @@ class S3ObjectStore {
      * @throws TypeError - if the URL is not a recognized protocol or cannot be parsed
      */
     async signGetObject(objectUrl, options = {}, queryParams) {
+        log.info(`Executing signGetObject with objectUrl: ${objectUrl}, options: ${JSON.stringify(options)}, queryParams: ${JSON.stringify(queryParams)}`);
         const url = new url_1.URL(objectUrl);
         if (url.protocol.toLowerCase() !== 's3:') {
             throw new TypeError(`Invalid S3 URL: ${objectUrl}`);
@@ -58,6 +61,7 @@ class S3ObjectStore {
         }
         // TypeScript doesn't recognize that req has a presign method.  It does.
         const result = await req.presign();
+        log.debug(`Signed GetObject request URL: ${result}`);
         return result;
     }
     /**
@@ -71,6 +75,7 @@ class S3ObjectStore {
      * @throws TypeError - if the URL is not a recognized protocol or cannot be parsed
      */
     async signHeadObject(objectUrl, options = {}, queryParams) {
+        log.info(`Executing signHeadObject with objectUrl: ${objectUrl}, options: ${JSON.stringify(options)}, queryParams: ${JSON.stringify(queryParams)}`);
         const url = new url_1.URL(objectUrl);
         if (url.protocol.toLowerCase() !== 's3:') {
             throw new TypeError(`Invalid S3 URL: ${objectUrl}`);
@@ -82,6 +87,7 @@ class S3ObjectStore {
         }
         // TypeScript doesn't recognize that req has a presign method.  It does.
         const result = await req.presign();
+        log.debug(`Signed HeadObject request URL: ${result}`);
         return result;
     }
 }

package/SQS.d.ts

@@ -4,6 +4,7 @@
 export interface SQSMessage extends AWS.SQS.Message {
     ReceiptHandle: string;
 }
+export declare const getQueueNameFromUrl: (queueUrl: string) => string | undefined;
 export declare const getQueueUrl: (sourceArn: string, queueName: string) => string;
 export declare const getQueueUrlByName: (queueName: string) => Promise<string | undefined>;
 /**
@@ -62,10 +63,10 @@ export declare const deleteSQSMessage: (QueueUrl: string, ReceiptHandle: string)
 /**
  * Test if an SQS queue exists
  *
- * @param {Object} queue - queue name or url
- * @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
+ * @param {Object} queueUrl     - queue url
+ * @returns {Promise<boolean>}  - a Promise that will resolve to a boolean indicating
  *                               if the queue exists
  */
-export declare const sqsQueueExists: (queue: string) => Promise<boolean>;
+export declare const sqsQueueExists: (queueUrl: string) => Promise<boolean>;
 export {};
 //# sourceMappingURL=SQS.d.ts.map

package/SQS.js

@@ -25,7 +25,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sqsQueueExists = exports.deleteSQSMessage = exports.parseSQSMessageBody = exports.receiveSQSMessages = exports.sendSQSMessage = exports.getQueueAttributes = exports.deleteQueue = exports.createQueue = exports.getQueueUrlByName = exports.getQueueUrl = void 0;
+exports.sqsQueueExists = exports.deleteSQSMessage = exports.parseSQSMessageBody = exports.receiveSQSMessages = exports.sendSQSMessage = exports.getQueueAttributes = exports.deleteQueue = exports.createQueue = exports.getQueueUrlByName = exports.getQueueUrl = exports.getQueueNameFromUrl = void 0;
 const get_1 = __importDefault(require("lodash/get"));
 const isObject_1 = __importDefault(require("lodash/isObject"));
 const isString_1 = __importDefault(require("lodash/isString"));
@@ -34,6 +34,7 @@ const url = __importStar(require("url"));
 const services_1 = require("./services");
 const test_utils_1 = require("./test-utils");
 const utils_1 = require("./utils");
+exports.getQueueNameFromUrl = (queueUrl) => queueUrl.split('/').pop();
 exports.getQueueUrl = (sourceArn, queueName) => {
     const arnParts = sourceArn.split(':');
     return `https://sqs.${arnParts[3]}.amazonaws.com/${arnParts[4]}/${queueName}`;
@@ -148,14 +149,14 @@ exports.deleteSQSMessage = utils_1.improveStackTrace((QueueUrl, ReceiptHandle) =
 /**
  * Test if an SQS queue exists
  *
- * @param {Object} queue - queue name or url
- * @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
+ * @param {Object} queueUrl     - queue url
+ * @returns {Promise<boolean>}  - a Promise that will resolve to a boolean indicating
  *                               if the queue exists
  */
-exports.sqsQueueExists = async (queue) => {
-    const QueueName = queue.split('/').pop();
+exports.sqsQueueExists = async (queueUrl) => {
+    const QueueName = exports.getQueueNameFromUrl(queueUrl);
     if (!QueueName) {
-        throw new Error(`Unable to determine QueueName from ${queue}`);
+        throw new Error(`Unable to determine QueueName from ${queueUrl}`);
     }
     try {
         await services_1.sqs().getQueueUrl({ QueueName }).promise();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/aws-client",
-  "version": "9.2.3",
+  "version": "9.4.1",
   "description": "Utilities for working with AWS",
   "keywords": [
     "GIBS",
@@ -43,9 +43,9 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/checksum": "9.2.3",
-    "@cumulus/errors": "9.2.3",
-    "@cumulus/logger": "9.2.3",
+    "@cumulus/checksum": "9.4.1",
+    "@cumulus/errors": "9.4.1",
+    "@cumulus/logger": "9.4.1",
     "aws-sdk": "^2.814.0",
     "jsonpath-plus": "^1.1.0",
     "lodash": "~4.17.20",
@@ -54,5 +54,5 @@
     "p-wait-for": "^3.1.0",
     "pump": "^3.0.0"
   },
-  "gitHead": "91c94e8972edd3f35b6a1f7e22f81ab3f3f1c3ee"
+  "gitHead": "fce4415918c43a6a14d46ac7d9572f5a97a98a45"
 }