@cumulus/aws-client 21.0.0 → 21.1.0

package/README.md

@@ -83,6 +83,10 @@ for allowed params and return value.</p>
 <dt><a href="#decryptBase64String">decryptBase64String(ciphertext)</a> ⇒ <code>string</code></dt>
 <dd><p>Decrypt a KMS-encrypted string, Base 64 encoded</p>
 </dd>
+<dt><a href="#applyS3Jitter">applyS3Jitter(maxJitterMs, operation)</a> ⇒</dt>
+<dd><p>Introduces random jitter delay to stagger concurrent S3 operations.
+This helps prevent AWS S3 SlowDown errors when many operations occur simultaneously.</p>
+</dd>
 </dl>
 
 <a name="module_CloudFormation"></a>
@@ -1122,6 +1126,20 @@ Decrypt a KMS-encrypted string, Base 64 encoded
 | --- | --- | --- |
 | ciphertext | <code>string</code> | a KMS-encrypted value, Base 64 encoded |
 
+<a name="applyS3Jitter"></a>
+
+## applyS3Jitter(maxJitterMs, operation) ⇒
+Introduces random jitter delay to stagger concurrent S3 operations.
+This helps prevent AWS S3 SlowDown errors when many operations occur simultaneously.
+
+**Kind**: global function  
+**Returns**: A Promise that resolves after the random delay  
+
+| Param | Description |
+| --- | --- |
+| maxJitterMs | Maximum jitter time in milliseconds (0-59000).   If 0, no delay is applied. |
+| operation | Optional operation name for logging context |
+
 
 ## About Cumulus
 

package/S3.js

@@ -46,10 +46,13 @@ const checksum_1 = require("@cumulus/checksum");
 const errors_1 = require("@cumulus/errors");
 const logger_1 = __importDefault(require("@cumulus/logger"));
 const S3MultipartUploads = __importStar(require("./lib/S3MultipartUploads"));
+const s3_jitter_1 = require("./s3-jitter");
 const services_1 = require("./services");
 const test_utils_1 = require("./test-utils");
 const utils_1 = require("./utils");
 const log = new logger_1.default({ sender: 'aws-client/s3' });
+// S3 jitter configuration from environment variable
+const s3JitterMaxMs = Number(process.env.S3_JITTER_MAX_MS || 0);
 const buildDeprecationMessage = (name, version, alternative) => {
     let message = `${name} is deprecated after version ${version} and will be removed in a future release.`;
     if (alternative)
@@ -148,6 +151,7 @@ exports.deleteS3Objects = deleteS3Objects;
 * @returns  returns response from `S3.headObject` as a promise
 **/
 const headObject = (Bucket, Key, retryOptions = { retries: 0 }) => (0, p_retry_1.default)(async () => {
+    await (0, s3_jitter_1.applyS3Jitter)(s3JitterMaxMs, `headObject(${Bucket}/${Key})`);
     try {
         return await (0, services_1.s3)().headObject({ Bucket, Key });
     }
@@ -212,7 +216,10 @@ exports.waitForObjectToExist = waitForObjectToExist;
 *
 * @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
 **/
-const s3PutObject = (params) => (0, services_1.s3)().putObject(params);
+const s3PutObject = async (params) => {
+    await (0, s3_jitter_1.applyS3Jitter)(s3JitterMaxMs, `putObject(${params.Bucket}/${params.Key})`);
+    return (0, services_1.s3)().putObject(params);
+};
 exports.s3PutObject = s3PutObject;
 /**
  * Upload a file to S3
@@ -227,10 +234,13 @@ exports.putFile = putFile;
 /**
 * Copy an object from one location on S3 to another
 **/
-const s3CopyObject = (params) => (0, services_1.s3)().copyObject({
-    TaggingDirective: 'COPY',
-    ...params,
-});
+const s3CopyObject = async (params) => {
+    await (0, s3_jitter_1.applyS3Jitter)(s3JitterMaxMs, `copyObject(${params.Bucket}/${params.Key})`);
+    return (0, services_1.s3)().copyObject({
+        TaggingDirective: 'COPY',
+        ...params,
+    });
+};
 exports.s3CopyObject = s3CopyObject;
 /**
  * Upload data to S3
@@ -238,6 +248,7 @@ exports.s3CopyObject = s3CopyObject;
  * see https://github.com/aws/aws-sdk-js-v3/tree/main/lib/lib-storage
  */
 const promiseS3Upload = async (params) => {
+    await (0, s3_jitter_1.applyS3Jitter)(s3JitterMaxMs, `upload(${params.params?.Bucket}/${params.params?.Key})`);
     const parallelUploads = new lib_storage_1.Upload({
         ...params,
         client: (0, services_1.s3)(),
@@ -290,6 +301,7 @@ const downloadS3File = async (s3Obj, filepath) => {
     if (!s3Obj.Bucket || !s3Obj.Key) {
         throw new Error('Bucket and Key are required');
     }
+    await (0, s3_jitter_1.applyS3Jitter)(s3JitterMaxMs, `downloadS3File(${s3Obj.Bucket}/${s3Obj.Key})`);
     const fileWriteStream = fs_1.default.createWriteStream(filepath);
     const objectStream = await (0, exports.getObjectReadStream)({
         bucket: s3Obj.Bucket,
@@ -309,6 +321,7 @@ exports.downloadS3File = downloadS3File;
  */
 const getObjectSize = async (params) => {
     // eslint-disable-next-line no-shadow
+    await (0, s3_jitter_1.applyS3Jitter)(s3JitterMaxMs, `getObjectSize(${params.bucket}/${params.key})`);
     const { s3: s3Client, bucket, key } = params;
     const headObjectResponse = await s3Client.headObject({
         Bucket: bucket,
@@ -348,7 +361,10 @@ exports.s3PutObjectTagging = s3PutObjectTagging;
 /**
  * Gets an object from S3.
  */
-const getObject = (s3Client, params) => s3Client.getObject(params);
+const getObject = async (s3Client, params) => {
+    await (0, s3_jitter_1.applyS3Jitter)(s3JitterMaxMs, `getObject(${params.Bucket}/${params.Key})`);
+    return s3Client.getObject(params);
+};
 exports.getObject = getObject;
 /**
  * Get an object from S3, waiting for it to exist and, if specified, have the
@@ -754,6 +770,7 @@ const uploadPartCopy = async (params) => {
  */
 const multipartCopyObject = async (params) => {
     const { sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags = false, chunkSize, } = params;
+    await (0, s3_jitter_1.applyS3Jitter)(s3JitterMaxMs, `multipartCopyObject(${destinationBucket}/${destinationKey})`);
     const sourceObject = params.sourceObject ?? await (0, exports.headObject)(sourceBucket, sourceKey);
     // Create a multi-part upload (copy) and get its UploadId
     const uploadId = await createMultipartUpload({

package/SNS.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createSnsTopic = exports.publishSnsMessageWithRetry = void 0;
 const p_retry_1 = __importDefault(require("p-retry"));
 const logger_1 = __importDefault(require("@cumulus/logger"));
+const errors_1 = require("@cumulus/errors");
 const client_sns_1 = require("@aws-sdk/client-sns");
 const services_1 = require("./services");
 const log = new logger_1.default({ sender: 'aws-client/sns' });
@@ -33,7 +34,7 @@ const publishSnsMessageWithRetry = async (snsTopicArn, message, retryOptions = {
     await (0, services_1.sns)().send(new client_sns_1.PublishCommand(publishInput));
 }, {
     maxTimeout: 5000,
-    onFailedAttempt: (err) => log.debug(`publishSnsMessageWithRetry('${snsTopicArn}', '${JSON.stringify(message)}') failed with ${err.retriesLeft} retries left: ${JSON.stringify(err)}`),
+    onFailedAttempt: (err) => log.debug(`publishSnsMessageWithRetry('${snsTopicArn}', '${JSON.stringify(message)}') failed with ${err.retriesLeft} retries left: ${(0, errors_1.errorify)(err)}`),
     ...retryOptions,
 });
 exports.publishSnsMessageWithRetry = publishSnsMessageWithRetry;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/aws-client",
-  "version": "21.0.0",
+  "version": "21.1.0",
   "description": "Utilities for working with AWS",
   "keywords": [
     "GIBS",
@@ -69,10 +69,10 @@
     "@aws-sdk/s3-request-presigner": "^3.621.0",
     "@aws-sdk/signature-v4-crt": "^3.621.0",
     "@aws-sdk/types": "^3.609.0",
-    "@cumulus/checksum": "21.0.0",
-    "@cumulus/errors": "21.0.0",
-    "@cumulus/logger": "21.0.0",
-    "@cumulus/types": "21.0.0",
+    "@cumulus/checksum": "21.1.0",
+    "@cumulus/errors": "21.1.0",
+    "@cumulus/logger": "21.1.0",
+    "@cumulus/types": "21.1.0",
     "lodash": "~4.17.21",
     "mem": "^8.0.2",
     "p-map": "^1.2.0",
@@ -83,8 +83,8 @@
     "uuid": "^8.2.0"
   },
   "devDependencies": {
-    "@cumulus/test-data": "21.0.0",
+    "@cumulus/test-data": "21.1.0",
     "@types/uuid": "^8.0.0"
   },
-  "gitHead": "19bb3477969662a9e0b300f10f6df23b6c0654db"
+  "gitHead": "5d443a04647ed537903c85b48992d08ce3c3cd1d"
 }

package/s3-jitter.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Introduces random jitter delay to stagger concurrent S3 operations.
+ * This helps prevent AWS S3 SlowDown errors when many operations occur simultaneously.
+ *
+ * @param maxJitterMs - Maximum jitter time in milliseconds (0-59000).
+ *   If 0, no delay is applied.
+ * @param operation - Optional operation name for logging context
+ * @returns A Promise that resolves after the random delay
+ */
+export declare const applyS3Jitter: (maxJitterMs: number, operation?: string) => Promise<void>;
+//# sourceMappingURL=s3-jitter.d.ts.map

package/s3-jitter.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.applyS3Jitter = void 0;
+const logger_1 = __importDefault(require("@cumulus/logger"));
+const log = new logger_1.default({ sender: 's3-jitter' });
+/**
+ * Introduces random jitter delay to stagger concurrent S3 operations.
+ * This helps prevent AWS S3 SlowDown errors when many operations occur simultaneously.
+ *
+ * @param maxJitterMs - Maximum jitter time in milliseconds (0-59000).
+ *   If 0, no delay is applied.
+ * @param operation - Optional operation name for logging context
+ * @returns A Promise that resolves after the random delay
+ */
+const applyS3Jitter = async (maxJitterMs, operation) => {
+    if (maxJitterMs <= 0) {
+        return;
+    }
+    const jitterMs = Math.ceil(Math.random() * maxJitterMs);
+    const logContext = operation ? ` for ${operation}` : '';
+    log.info(`Applying S3 jitter: ${jitterMs}ms${logContext} (max: ${maxJitterMs}ms)`);
+    await new Promise((resolve) => setTimeout(resolve, jitterMs));
+    log.debug(`S3 jitter delay completed: ${jitterMs}ms${logContext}`);
+};
+exports.applyS3Jitter = applyS3Jitter;
+//# sourceMappingURL=s3-jitter.js.map