@cumulus/aws-client 9.7.1 → 9.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CloudFormation.js +10 -7
- package/CloudFormationGateway.js +2 -2
- package/CloudwatchEvents.js +8 -4
- package/DynamoDb.js +12 -11
- package/DynamoDbSearchQueue.d.ts +2 -2
- package/DynamoDbSearchQueue.js +1 -1
- package/KMS.js +8 -5
- package/Kinesis.js +3 -2
- package/Lambda.js +4 -3
- package/README.md +245 -237
- package/S3.d.ts +2 -2
- package/S3.js +111 -76
- package/S3ListObjectsV2Queue.js +1 -1
- package/S3ObjectStore.js +5 -5
- package/SNS.js +3 -2
- package/SQS.d.ts +6 -3
- package/SQS.js +31 -23
- package/SecretsManager.js +2 -1
- package/StepFunctions.d.ts +3 -1
- package/StepFunctions.js +15 -12
- package/client.js +2 -2
- package/index.js +2 -1
- package/lib/S3MultipartUploads.js +11 -6
- package/package.json +5 -5
- package/services.js +1 -1
- package/test-utils.js +4 -2
- package/utils.js +8 -5
package/S3.d.ts
CHANGED
|
@@ -193,13 +193,13 @@ export declare const s3PutObjectTagging: (Bucket: string, Key: string, Tagging:
|
|
|
193
193
|
* @example
|
|
194
194
|
* const obj = await getObject(s3(), { Bucket: 'b', Key: 'k' })
|
|
195
195
|
*
|
|
196
|
-
* @param {AWS.S3}
|
|
196
|
+
* @param {AWS.S3} s3Client - an `AWS.S3` instance
|
|
197
197
|
* @param {AWS.S3.GetObjectRequest} params - parameters object to pass through
|
|
198
198
|
* to `AWS.S3.getObject()`
|
|
199
199
|
* @returns {Promise<AWS.S3.GetObjectOutput>} response from `AWS.S3.getObject()`
|
|
200
200
|
* as a Promise
|
|
201
201
|
*/
|
|
202
|
-
export declare const getObject: (
|
|
202
|
+
export declare const getObject: (s3Client: {
|
|
203
203
|
getObject: GetObjectPromiseMethod;
|
|
204
204
|
}, params: AWS.S3.GetObjectRequest) => Promise<AWS.S3.GetObjectOutput>;
|
|
205
205
|
/**
|
package/S3.js
CHANGED
|
@@ -17,7 +17,7 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
|
|
|
17
17
|
var __importStar = (this && this.__importStar) || function (mod) {
|
|
18
18
|
if (mod && mod.__esModule) return mod;
|
|
19
19
|
var result = {};
|
|
20
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
20
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
21
21
|
__setModuleDefault(result, mod);
|
|
22
22
|
return result;
|
|
23
23
|
};
|
|
@@ -50,14 +50,14 @@ const buildDeprecationMessage = (name, version, alternative) => {
|
|
|
50
50
|
message += ` Use ${alternative} instead.`;
|
|
51
51
|
return log.buildMessage('warn', message);
|
|
52
52
|
};
|
|
53
|
-
const S3_RATE_LIMIT = test_utils_1.inTestMode() ? 1 : 20;
|
|
53
|
+
const S3_RATE_LIMIT = (0, test_utils_1.inTestMode)() ? 1 : 20;
|
|
54
54
|
/**
|
|
55
55
|
* Join strings into an S3 key without a leading slash
|
|
56
56
|
*
|
|
57
57
|
* @param {...string|Array<string>} args - the strings to join
|
|
58
58
|
* @returns {string} the full S3 key
|
|
59
59
|
*/
|
|
60
|
-
|
|
60
|
+
const s3Join = (...args) => {
|
|
61
61
|
let tokens;
|
|
62
62
|
if (typeof args[0] === 'string')
|
|
63
63
|
tokens = args;
|
|
@@ -75,13 +75,14 @@ exports.s3Join = (...args) => {
|
|
|
75
75
|
return `${key}/`;
|
|
76
76
|
return key;
|
|
77
77
|
};
|
|
78
|
+
exports.s3Join = s3Join;
|
|
78
79
|
/**
|
|
79
80
|
* parse an s3 uri to get the bucket and key
|
|
80
81
|
*
|
|
81
82
|
* @param {string} uri - must be a uri with the `s3://` protocol
|
|
82
83
|
* @returns {Object} Returns an object with `Bucket` and `Key` properties
|
|
83
84
|
**/
|
|
84
|
-
|
|
85
|
+
const parseS3Uri = (uri) => {
|
|
85
86
|
const match = uri.match('^s3://([^/]+)/(.*)$');
|
|
86
87
|
if (match === null) {
|
|
87
88
|
throw new TypeError(`Unable to parse S3 URI: ${uri}`);
|
|
@@ -91,6 +92,7 @@ exports.parseS3Uri = (uri) => {
|
|
|
91
92
|
Key: match[2],
|
|
92
93
|
};
|
|
93
94
|
};
|
|
95
|
+
exports.parseS3Uri = parseS3Uri;
|
|
94
96
|
/**
|
|
95
97
|
* Given a bucket and key, return an S3 URI
|
|
96
98
|
*
|
|
@@ -98,7 +100,8 @@ exports.parseS3Uri = (uri) => {
|
|
|
98
100
|
* @param {string} key - an S3 key
|
|
99
101
|
* @returns {string} an S3 URI
|
|
100
102
|
*/
|
|
101
|
-
|
|
103
|
+
const buildS3Uri = (bucket, key) => `s3://${bucket}/${key.replace(/^\/+/, '')}`;
|
|
104
|
+
exports.buildS3Uri = buildS3Uri;
|
|
102
105
|
/**
|
|
103
106
|
* Convert S3 TagSet Object to query string
|
|
104
107
|
* e.g. [{ Key: 'tag', Value: 'value }] to 'tag=value'
|
|
@@ -106,7 +109,8 @@ exports.buildS3Uri = (bucket, key) => `s3://${bucket}/${key.replace(/^\/+/, '')}
|
|
|
106
109
|
* @param {Array<Object>} tagset - S3 TagSet array
|
|
107
110
|
* @returns {string} tags query string
|
|
108
111
|
*/
|
|
109
|
-
|
|
112
|
+
const s3TagSetToQueryString = (tagset) => tagset.map(({ Key, Value }) => `${Key}=${Value}`).join('&');
|
|
113
|
+
exports.s3TagSetToQueryString = s3TagSetToQueryString;
|
|
110
114
|
/**
|
|
111
115
|
* Delete an object from S3
|
|
112
116
|
*
|
|
@@ -114,7 +118,7 @@ exports.s3TagSetToQueryString = (tagset) => tagset.map(({ Key, Value }) => `${Ke
|
|
|
114
118
|
* @param {string} key - key of the object to be deleted
|
|
115
119
|
* promise of the object being deleted
|
|
116
120
|
*/
|
|
117
|
-
exports.deleteS3Object = utils_1.improveStackTrace((bucket, key) => services_1.s3().deleteObject({ Bucket: bucket, Key: key }).promise());
|
|
121
|
+
exports.deleteS3Object = (0, utils_1.improveStackTrace)((bucket, key) => (0, services_1.s3)().deleteObject({ Bucket: bucket, Key: key }).promise());
|
|
118
122
|
/**
|
|
119
123
|
* Get an object header from S3
|
|
120
124
|
*
|
|
@@ -125,9 +129,9 @@ exports.deleteS3Object = utils_1.improveStackTrace((bucket, key) => services_1.s
|
|
|
125
129
|
* By default, retries will not be performed
|
|
126
130
|
* @returns {Promise} returns response from `S3.headObject` as a promise
|
|
127
131
|
**/
|
|
128
|
-
exports.headObject = utils_1.improveStackTrace((Bucket, Key, retryOptions = { retries: 0 }) => p_retry_1.default(async () => {
|
|
132
|
+
exports.headObject = (0, utils_1.improveStackTrace)((Bucket, Key, retryOptions = { retries: 0 }) => (0, p_retry_1.default)(async () => {
|
|
129
133
|
try {
|
|
130
|
-
return await services_1.s3().headObject({ Bucket, Key }).promise();
|
|
134
|
+
return await (0, services_1.s3)().headObject({ Bucket, Key }).promise();
|
|
131
135
|
}
|
|
132
136
|
catch (error) {
|
|
133
137
|
if (error.code === 'NotFound')
|
|
@@ -142,13 +146,14 @@ exports.headObject = utils_1.improveStackTrace((Bucket, Key, retryOptions = { re
|
|
|
142
146
|
* @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
|
|
143
147
|
* if the object exists
|
|
144
148
|
*/
|
|
145
|
-
|
|
149
|
+
const s3ObjectExists = (params) => (0, exports.headObject)(params.Bucket, params.Key)
|
|
146
150
|
.then(() => true)
|
|
147
151
|
.catch((error) => {
|
|
148
152
|
if (error.code === 'NotFound')
|
|
149
153
|
return false;
|
|
150
154
|
throw error;
|
|
151
155
|
});
|
|
156
|
+
exports.s3ObjectExists = s3ObjectExists;
|
|
152
157
|
/**
|
|
153
158
|
* Wait for an object to exist in S3
|
|
154
159
|
*
|
|
@@ -159,17 +164,18 @@ exports.s3ObjectExists = (params) => exports.headObject(params.Bucket, params.Ke
|
|
|
159
164
|
* @param {number} [params.timeout=30000] - timeout, in ms
|
|
160
165
|
* @returns {Promise<undefined>}
|
|
161
166
|
*/
|
|
162
|
-
|
|
167
|
+
const waitForObjectToExist = async (params) => {
|
|
163
168
|
const { bucket, key, interval = 1000, timeout = 30 * 1000, } = params;
|
|
164
|
-
await p_wait_for_1.default(() => exports.s3ObjectExists({ Bucket: bucket, Key: key }), { interval, timeout });
|
|
169
|
+
await (0, p_wait_for_1.default)(() => (0, exports.s3ObjectExists)({ Bucket: bucket, Key: key }), { interval, timeout });
|
|
165
170
|
};
|
|
171
|
+
exports.waitForObjectToExist = waitForObjectToExist;
|
|
166
172
|
/**
|
|
167
173
|
* Put an object on S3
|
|
168
174
|
*
|
|
169
175
|
* @param {Object} params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
|
|
170
176
|
* promise of the object being put
|
|
171
177
|
**/
|
|
172
|
-
exports.s3PutObject = utils_1.improveStackTrace((params) => services_1.s3().putObject({
|
|
178
|
+
exports.s3PutObject = (0, utils_1.improveStackTrace)((params) => (0, services_1.s3)().putObject({
|
|
173
179
|
ACL: 'private',
|
|
174
180
|
...params,
|
|
175
181
|
}).promise());
|
|
@@ -181,18 +187,19 @@ exports.s3PutObject = utils_1.improveStackTrace((params) => services_1.s3().putO
|
|
|
181
187
|
* @param {filename} filename - the local file to be uploaded
|
|
182
188
|
* @returns {Promise}
|
|
183
189
|
*/
|
|
184
|
-
|
|
190
|
+
const putFile = (bucket, key, filename) => (0, exports.s3PutObject)({
|
|
185
191
|
Bucket: bucket,
|
|
186
192
|
Key: key,
|
|
187
193
|
Body: fs_1.default.createReadStream(filename),
|
|
188
194
|
});
|
|
195
|
+
exports.putFile = putFile;
|
|
189
196
|
/**
|
|
190
197
|
* Copy an object from one location on S3 to another
|
|
191
198
|
*
|
|
192
199
|
* @param {Object} params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
|
|
193
200
|
* @returns {Promise} promise of the object being copied
|
|
194
201
|
**/
|
|
195
|
-
exports.s3CopyObject = utils_1.improveStackTrace((params) => services_1.s3().copyObject({
|
|
202
|
+
exports.s3CopyObject = (0, utils_1.improveStackTrace)((params) => (0, services_1.s3)().copyObject({
|
|
196
203
|
TaggingDirective: 'COPY',
|
|
197
204
|
...params,
|
|
198
205
|
}).promise());
|
|
@@ -204,7 +211,7 @@ exports.s3CopyObject = utils_1.improveStackTrace((params) => services_1.s3().cop
|
|
|
204
211
|
* @param {Object} params - see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
|
|
205
212
|
* @returns {Promise} see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
|
|
206
213
|
*/
|
|
207
|
-
exports.promiseS3Upload = utils_1.improveStackTrace((params) => services_1.s3().upload(params).promise());
|
|
214
|
+
exports.promiseS3Upload = (0, utils_1.improveStackTrace)((params) => (0, services_1.s3)().upload(params).promise());
|
|
208
215
|
/**
|
|
209
216
|
* Upload data to S3 using a stream
|
|
210
217
|
*
|
|
@@ -216,12 +223,12 @@ exports.promiseS3Upload = utils_1.improveStackTrace((params) => services_1.s3().
|
|
|
216
223
|
* @param {Object} uploadParams - see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
|
|
217
224
|
* @returns {Promise}
|
|
218
225
|
*/
|
|
219
|
-
|
|
226
|
+
const streamS3Upload = (uploadStream, uploadParams) => new Promise((resolve, reject) => {
|
|
220
227
|
const pass = new stream_1.PassThrough();
|
|
221
228
|
uploadStream.pipe(pass);
|
|
222
229
|
uploadStream.on('error', reject);
|
|
223
230
|
pass.on('error', reject);
|
|
224
|
-
return services_1.s3().upload({
|
|
231
|
+
return (0, services_1.s3)().upload({
|
|
225
232
|
...uploadParams,
|
|
226
233
|
Body: pass,
|
|
227
234
|
}, (err, uploadResponse) => {
|
|
@@ -231,6 +238,7 @@ exports.streamS3Upload = (uploadStream, uploadParams) => new Promise((resolve, r
|
|
|
231
238
|
return resolve(uploadResponse);
|
|
232
239
|
});
|
|
233
240
|
});
|
|
241
|
+
exports.streamS3Upload = streamS3Upload;
|
|
234
242
|
/**
|
|
235
243
|
* Downloads the given s3Obj to the given filename in a streaming manner
|
|
236
244
|
*
|
|
@@ -238,11 +246,11 @@ exports.streamS3Upload = (uploadStream, uploadParams) => new Promise((resolve, r
|
|
|
238
246
|
* @param {string} filepath - The filepath of the file that is downloaded
|
|
239
247
|
* @returns {Promise<string>} returns filename if successful
|
|
240
248
|
*/
|
|
241
|
-
|
|
249
|
+
const downloadS3File = (s3Obj, filepath) => {
|
|
242
250
|
const fileWriteStream = fs_1.default.createWriteStream(filepath);
|
|
243
251
|
return new Promise((resolve, reject) => {
|
|
244
|
-
const objectReadStream = services_1.s3().getObject(s3Obj).createReadStream();
|
|
245
|
-
pump_1.default(objectReadStream, fileWriteStream, (err) => {
|
|
252
|
+
const objectReadStream = (0, services_1.s3)().getObject(s3Obj).createReadStream();
|
|
253
|
+
(0, pump_1.default)(objectReadStream, fileWriteStream, (err) => {
|
|
246
254
|
if (err)
|
|
247
255
|
reject(err);
|
|
248
256
|
else
|
|
@@ -250,6 +258,7 @@ exports.downloadS3File = (s3Obj, filepath) => {
|
|
|
250
258
|
});
|
|
251
259
|
});
|
|
252
260
|
};
|
|
261
|
+
exports.downloadS3File = downloadS3File;
|
|
253
262
|
/**
|
|
254
263
|
* Get the size of an S3 object
|
|
255
264
|
*
|
|
@@ -259,15 +268,16 @@ exports.downloadS3File = (s3Obj, filepath) => {
|
|
|
259
268
|
* @param {AWS.S3} params.s3 - an S3 client instance
|
|
260
269
|
* @returns {Promise<number|undefined>} object size, in bytes
|
|
261
270
|
*/
|
|
262
|
-
|
|
271
|
+
const getObjectSize = async (params) => {
|
|
263
272
|
// eslint-disable-next-line no-shadow
|
|
264
|
-
const { s3, bucket, key } = params;
|
|
265
|
-
const headObjectResponse = await
|
|
273
|
+
const { s3: s3Client, bucket, key } = params;
|
|
274
|
+
const headObjectResponse = await s3Client.headObject({
|
|
266
275
|
Bucket: bucket,
|
|
267
276
|
Key: key,
|
|
268
277
|
}).promise();
|
|
269
278
|
return headObjectResponse.ContentLength;
|
|
270
279
|
};
|
|
280
|
+
exports.getObjectSize = getObjectSize;
|
|
271
281
|
/**
|
|
272
282
|
* Get object Tagging from S3
|
|
273
283
|
*
|
|
@@ -275,9 +285,9 @@ exports.getObjectSize = async (params) => {
|
|
|
275
285
|
* @param {string} key - key for object (filepath + filename)
|
|
276
286
|
* @returns {Promise<AWS.S3.GetObjectTaggingOutput>} the promised response from `S3.getObjectTagging`
|
|
277
287
|
**/
|
|
278
|
-
exports.s3GetObjectTagging = utils_1.improveStackTrace((bucket, key) => services_1.s3().getObjectTagging({ Bucket: bucket, Key: key }).promise());
|
|
288
|
+
exports.s3GetObjectTagging = (0, utils_1.improveStackTrace)((bucket, key) => (0, services_1.s3)().getObjectTagging({ Bucket: bucket, Key: key }).promise());
|
|
279
289
|
const getObjectTags = async (bucket, key) => {
|
|
280
|
-
const taggingResponse = await exports.s3GetObjectTagging(bucket, key);
|
|
290
|
+
const taggingResponse = await (0, exports.s3GetObjectTagging)(bucket, key);
|
|
281
291
|
return taggingResponse.TagSet.reduce((acc, { Key, Value }) => ({ ...acc, [Key]: Value }), {});
|
|
282
292
|
};
|
|
283
293
|
const getObjectTaggingString = async (bucket, key) => {
|
|
@@ -293,7 +303,7 @@ const getObjectTaggingString = async (bucket, key) => {
|
|
|
293
303
|
* @param {Object} Tagging - tagging object
|
|
294
304
|
* @returns {Promise} returns response from `S3.getObjectTagging` as a promise
|
|
295
305
|
**/
|
|
296
|
-
exports.s3PutObjectTagging = utils_1.improveStackTrace((Bucket, Key, Tagging) => services_1.s3().putObjectTagging({
|
|
306
|
+
exports.s3PutObjectTagging = (0, utils_1.improveStackTrace)((Bucket, Key, Tagging) => (0, services_1.s3)().putObjectTagging({
|
|
297
307
|
Bucket,
|
|
298
308
|
Key,
|
|
299
309
|
Tagging,
|
|
@@ -304,15 +314,16 @@ exports.s3PutObjectTagging = utils_1.improveStackTrace((Bucket, Key, Tagging) =>
|
|
|
304
314
|
* @example
|
|
305
315
|
* const obj = await getObject(s3(), { Bucket: 'b', Key: 'k' })
|
|
306
316
|
*
|
|
307
|
-
* @param {AWS.S3}
|
|
317
|
+
* @param {AWS.S3} s3Client - an `AWS.S3` instance
|
|
308
318
|
* @param {AWS.S3.GetObjectRequest} params - parameters object to pass through
|
|
309
319
|
* to `AWS.S3.getObject()`
|
|
310
320
|
* @returns {Promise<AWS.S3.GetObjectOutput>} response from `AWS.S3.getObject()`
|
|
311
321
|
* as a Promise
|
|
312
322
|
*/
|
|
313
|
-
|
|
323
|
+
const getObject = (
|
|
314
324
|
// eslint-disable-next-line no-shadow
|
|
315
|
-
|
|
325
|
+
s3Client, params) => s3Client.getObject(params).promise();
|
|
326
|
+
exports.getObject = getObject;
|
|
316
327
|
/**
|
|
317
328
|
* Get an object from S3, waiting for it to exist and, if specified, have the
|
|
318
329
|
* correct ETag.
|
|
@@ -322,9 +333,9 @@ s3, params) => s3.getObject(params).promise();
|
|
|
322
333
|
* @param {pRetry.Options} [retryOptions={}]
|
|
323
334
|
* @returns {Promise<AWS.S3.GetObjectOutput>}
|
|
324
335
|
*/
|
|
325
|
-
|
|
336
|
+
const waitForObject = (s3Client, params, retryOptions = {}) => (0, p_retry_1.default)(async () => {
|
|
326
337
|
try {
|
|
327
|
-
return await exports.getObject(s3Client, params);
|
|
338
|
+
return await (0, exports.getObject)(s3Client, params);
|
|
328
339
|
}
|
|
329
340
|
catch (error) {
|
|
330
341
|
// Retry if the object does not exist
|
|
@@ -337,6 +348,7 @@ exports.waitForObject = (s3Client, params, retryOptions = {}) => p_retry_1.defau
|
|
|
337
348
|
throw new p_retry_1.default.AbortError(error);
|
|
338
349
|
}
|
|
339
350
|
}, retryOptions);
|
|
351
|
+
exports.waitForObject = waitForObject;
|
|
340
352
|
/**
|
|
341
353
|
* Gets an object from S3.
|
|
342
354
|
*
|
|
@@ -349,7 +361,7 @@ exports.waitForObject = (s3Client, params, retryOptions = {}) => p_retry_1.defau
|
|
|
349
361
|
*
|
|
350
362
|
* @deprecated
|
|
351
363
|
*/
|
|
352
|
-
exports.getS3Object = util_1.deprecate(utils_1.improveStackTrace((Bucket, Key, retryOptions = { retries: 0 }) => exports.waitForObject(services_1.s3(), { Bucket, Key }, {
|
|
364
|
+
exports.getS3Object = (0, util_1.deprecate)((0, utils_1.improveStackTrace)((Bucket, Key, retryOptions = { retries: 0 }) => (0, exports.waitForObject)((0, services_1.s3)(), { Bucket, Key }, {
|
|
353
365
|
maxTimeout: 10000,
|
|
354
366
|
onFailedAttempt: (err) => log.debug(`getS3Object('${Bucket}', '${Key}') failed with ${err.retriesLeft} retries left: ${err.message}`),
|
|
355
367
|
...retryOptions,
|
|
@@ -361,29 +373,32 @@ exports.getS3Object = util_1.deprecate(utils_1.improveStackTrace((Bucket, Key, r
|
|
|
361
373
|
* @param {string} key - the S3 object's key
|
|
362
374
|
* @returns {Promise<string>} the contents of the S3 object
|
|
363
375
|
*/
|
|
364
|
-
|
|
376
|
+
const getTextObject = (bucket, key) => (0, exports.getS3Object)(bucket, key)
|
|
365
377
|
.then(({ Body }) => {
|
|
366
378
|
if (Body === undefined)
|
|
367
379
|
return undefined;
|
|
368
380
|
return Body.toString();
|
|
369
381
|
});
|
|
382
|
+
exports.getTextObject = getTextObject;
|
|
370
383
|
/**
|
|
371
384
|
* Fetch JSON stored in an S3 object
|
|
372
385
|
* @param {string} bucket - the S3 object's bucket
|
|
373
386
|
* @param {string} key - the S3 object's key
|
|
374
387
|
* @returns {Promise<*>} the contents of the S3 object, parsed as JSON
|
|
375
388
|
*/
|
|
376
|
-
|
|
389
|
+
const getJsonS3Object = (bucket, key) => (0, exports.getTextObject)(bucket, key)
|
|
377
390
|
.then((text) => {
|
|
378
391
|
if (text === undefined)
|
|
379
392
|
return undefined;
|
|
380
393
|
return JSON.parse(text);
|
|
381
394
|
});
|
|
382
|
-
exports.
|
|
395
|
+
exports.getJsonS3Object = getJsonS3Object;
|
|
396
|
+
const putJsonS3Object = (bucket, key, data) => (0, exports.s3PutObject)({
|
|
383
397
|
Bucket: bucket,
|
|
384
398
|
Key: key,
|
|
385
399
|
Body: JSON.stringify(data),
|
|
386
400
|
});
|
|
401
|
+
exports.putJsonS3Object = putJsonS3Object;
|
|
387
402
|
/**
|
|
388
403
|
* Get a readable stream for an S3 object
|
|
389
404
|
*
|
|
@@ -393,11 +408,12 @@ exports.putJsonS3Object = (bucket, key, data) => exports.s3PutObject({
|
|
|
393
408
|
* @param {string} params.key - the key of the requested object
|
|
394
409
|
* @returns {Readable}
|
|
395
410
|
*/
|
|
396
|
-
|
|
411
|
+
const getObjectReadStream = (params) => {
|
|
397
412
|
// eslint-disable-next-line no-shadow
|
|
398
|
-
const { s3, bucket, key } = params;
|
|
399
|
-
return
|
|
413
|
+
const { s3: s3Client, bucket, key } = params;
|
|
414
|
+
return s3Client.getObject({ Bucket: bucket, Key: key }).createReadStream();
|
|
400
415
|
};
|
|
416
|
+
exports.getObjectReadStream = getObjectReadStream;
|
|
401
417
|
/**
|
|
402
418
|
* Check if a file exists in an S3 object
|
|
403
419
|
*
|
|
@@ -405,9 +421,9 @@ exports.getObjectReadStream = (params) => {
|
|
|
405
421
|
* @param {string} key - key of the file in the S3 bucket
|
|
406
422
|
* @returns {Promise} returns the response from `S3.headObject` as a promise
|
|
407
423
|
**/
|
|
408
|
-
|
|
424
|
+
const fileExists = async (bucket, key) => {
|
|
409
425
|
try {
|
|
410
|
-
const r = await services_1.s3().headObject({ Key: key, Bucket: bucket }).promise();
|
|
426
|
+
const r = await (0, services_1.s3)().headObject({ Key: key, Bucket: bucket }).promise();
|
|
411
427
|
return r;
|
|
412
428
|
}
|
|
413
429
|
catch (error) {
|
|
@@ -418,7 +434,8 @@ exports.fileExists = async (bucket, key) => {
|
|
|
418
434
|
throw error;
|
|
419
435
|
}
|
|
420
436
|
};
|
|
421
|
-
exports.
|
|
437
|
+
exports.fileExists = fileExists;
|
|
438
|
+
const downloadS3Files = async (s3Objs, dir, s3opts = {}) => {
|
|
422
439
|
// Scrub s3Ojbs to avoid errors from the AWS SDK
|
|
423
440
|
const scrubbedS3Objs = s3Objs.map((s3Obj) => ({
|
|
424
441
|
Bucket: s3Obj.Bucket,
|
|
@@ -432,7 +449,7 @@ exports.downloadS3Files = async (s3Objs, dir, s3opts = {}) => {
|
|
|
432
449
|
const file = fs_1.default.createWriteStream(filename);
|
|
433
450
|
const opts = Object.assign(s3Obj, s3opts);
|
|
434
451
|
return new Promise((resolve, reject) => {
|
|
435
|
-
services_1.s3().getObject(opts)
|
|
452
|
+
(0, services_1.s3)().getObject(opts)
|
|
436
453
|
.createReadStream()
|
|
437
454
|
.pipe(file)
|
|
438
455
|
.on('finish', () => {
|
|
@@ -443,8 +460,9 @@ exports.downloadS3Files = async (s3Objs, dir, s3opts = {}) => {
|
|
|
443
460
|
.on('error', reject);
|
|
444
461
|
});
|
|
445
462
|
};
|
|
446
|
-
return await p_map_1.default(scrubbedS3Objs, promiseDownload, { concurrency: S3_RATE_LIMIT });
|
|
463
|
+
return await (0, p_map_1.default)(scrubbedS3Objs, promiseDownload, { concurrency: S3_RATE_LIMIT });
|
|
447
464
|
};
|
|
465
|
+
exports.downloadS3Files = downloadS3Files;
|
|
448
466
|
/**
|
|
449
467
|
* Delete files from S3
|
|
450
468
|
*
|
|
@@ -452,15 +470,16 @@ exports.downloadS3Files = async (s3Objs, dir, s3opts = {}) => {
|
|
|
452
470
|
* @returns {Promise} A promise that resolves to an Array of the data returned
|
|
453
471
|
* from the deletion operations
|
|
454
472
|
*/
|
|
455
|
-
|
|
473
|
+
const deleteS3Files = async (s3Objs) => await (0, p_map_1.default)(s3Objs, (s3Obj) => (0, services_1.s3)().deleteObject(s3Obj).promise(), { concurrency: S3_RATE_LIMIT });
|
|
474
|
+
exports.deleteS3Files = deleteS3Files;
|
|
456
475
|
/**
|
|
457
476
|
* Delete a bucket and all of its objects from S3
|
|
458
477
|
*
|
|
459
478
|
* @param {string} bucket - name of the bucket
|
|
460
479
|
* @returns {Promise} the promised result of `S3.deleteBucket`
|
|
461
480
|
**/
|
|
462
|
-
exports.recursivelyDeleteS3Bucket = utils_1.improveStackTrace(async (bucket) => {
|
|
463
|
-
const response = await services_1.s3().listObjects({ Bucket: bucket }).promise();
|
|
481
|
+
exports.recursivelyDeleteS3Bucket = (0, utils_1.improveStackTrace)(async (bucket) => {
|
|
482
|
+
const response = await (0, services_1.s3)().listObjects({ Bucket: bucket }).promise();
|
|
464
483
|
const s3Objects = (response.Contents || []).map((o) => {
|
|
465
484
|
if (!o.Key)
|
|
466
485
|
throw new Error(`Unable to determine S3 key of ${JSON.stringify(o)}`);
|
|
@@ -469,8 +488,8 @@ exports.recursivelyDeleteS3Bucket = utils_1.improveStackTrace(async (bucket) =>
|
|
|
469
488
|
Key: o.Key,
|
|
470
489
|
};
|
|
471
490
|
});
|
|
472
|
-
await exports.deleteS3Files(s3Objects);
|
|
473
|
-
await services_1.s3().deleteBucket({ Bucket: bucket }).promise();
|
|
491
|
+
await (0, exports.deleteS3Files)(s3Objects);
|
|
492
|
+
await (0, services_1.s3)().deleteBucket({ Bucket: bucket }).promise();
|
|
474
493
|
});
|
|
475
494
|
/**
|
|
476
495
|
* Delete a list of buckets and all of their objects from S3
|
|
@@ -478,8 +497,9 @@ exports.recursivelyDeleteS3Bucket = utils_1.improveStackTrace(async (bucket) =>
|
|
|
478
497
|
* @param {Array} buckets - list of bucket names
|
|
479
498
|
* @returns {Promise} the promised result of `S3.deleteBucket`
|
|
480
499
|
**/
|
|
481
|
-
|
|
482
|
-
exports.
|
|
500
|
+
const deleteS3Buckets = async (buckets) => await Promise.all(buckets.map(exports.recursivelyDeleteS3Bucket));
|
|
501
|
+
exports.deleteS3Buckets = deleteS3Buckets;
|
|
502
|
+
const uploadS3Files = async (files, defaultBucket, keyPath, s3opts = {}) => {
|
|
483
503
|
let i = 0;
|
|
484
504
|
const n = files.length;
|
|
485
505
|
if (n > 1) {
|
|
@@ -493,7 +513,7 @@ exports.uploadS3Files = async (files, defaultBucket, keyPath, s3opts = {}) => {
|
|
|
493
513
|
bucket = defaultBucket;
|
|
494
514
|
filename = file;
|
|
495
515
|
if (typeof keyPath === 'string') {
|
|
496
|
-
key = exports.s3Join(keyPath, path_1.default.basename(file));
|
|
516
|
+
key = (0, exports.s3Join)(keyPath, path_1.default.basename(file));
|
|
497
517
|
}
|
|
498
518
|
else {
|
|
499
519
|
key = keyPath(file);
|
|
@@ -504,7 +524,7 @@ exports.uploadS3Files = async (files, defaultBucket, keyPath, s3opts = {}) => {
|
|
|
504
524
|
filename = file.filename;
|
|
505
525
|
key = file.key;
|
|
506
526
|
}
|
|
507
|
-
await exports.promiseS3Upload({
|
|
527
|
+
await (0, exports.promiseS3Upload)({
|
|
508
528
|
Bucket: bucket,
|
|
509
529
|
Key: key,
|
|
510
530
|
Body: fs_1.default.createReadStream(filename),
|
|
@@ -514,8 +534,9 @@ exports.uploadS3Files = async (files, defaultBucket, keyPath, s3opts = {}) => {
|
|
|
514
534
|
log.info(`Progress: [${i} of ${n}] ${filename} -> s3://${bucket}/${key}`);
|
|
515
535
|
return { key, bucket };
|
|
516
536
|
};
|
|
517
|
-
return await p_map_1.default(files, promiseUpload, { concurrency: S3_RATE_LIMIT });
|
|
537
|
+
return await (0, p_map_1.default)(files, promiseUpload, { concurrency: S3_RATE_LIMIT });
|
|
518
538
|
};
|
|
539
|
+
exports.uploadS3Files = uploadS3Files;
|
|
519
540
|
/**
|
|
520
541
|
* Upload the file associated with the given stream to an S3 bucket
|
|
521
542
|
*
|
|
@@ -525,12 +546,13 @@ exports.uploadS3Files = async (files, defaultBucket, keyPath, s3opts = {}) => {
|
|
|
525
546
|
* @param {Object} s3opts - Options to pass to the AWS sdk call (defaults to `{}`)
|
|
526
547
|
* @returns {Promise} A promise
|
|
527
548
|
*/
|
|
528
|
-
|
|
549
|
+
const uploadS3FileStream = (fileStream, bucket, key, s3opts = {}) => (0, exports.promiseS3Upload)({
|
|
529
550
|
Bucket: bucket,
|
|
530
551
|
Key: key,
|
|
531
552
|
Body: fileStream,
|
|
532
553
|
...s3opts,
|
|
533
554
|
});
|
|
555
|
+
exports.uploadS3FileStream = uploadS3FileStream;
|
|
534
556
|
/**
|
|
535
557
|
* List the objects in an S3 bucket
|
|
536
558
|
*
|
|
@@ -543,14 +565,14 @@ exports.uploadS3FileStream = (fileStream, bucket, key, s3opts = {}) => exports.p
|
|
|
543
565
|
* object is represented as a JS object with the following attributes: `Key`,
|
|
544
566
|
* `ETag`, `LastModified`, `Owner`, `Size`, `StorageClass`.
|
|
545
567
|
*/
|
|
546
|
-
|
|
568
|
+
const listS3Objects = async (bucket, prefix, skipFolders = true) => {
|
|
547
569
|
log.info(`Listing objects in s3://${bucket}`);
|
|
548
570
|
const params = {
|
|
549
571
|
Bucket: bucket,
|
|
550
572
|
};
|
|
551
573
|
if (prefix)
|
|
552
574
|
params.Prefix = prefix;
|
|
553
|
-
const data = await services_1.s3().listObjects(params).promise();
|
|
575
|
+
const data = await (0, services_1.s3)().listObjects(params).promise();
|
|
554
576
|
let contents = data.Contents || [];
|
|
555
577
|
if (skipFolders) {
|
|
556
578
|
// Filter out any references to folders
|
|
@@ -558,6 +580,7 @@ exports.listS3Objects = async (bucket, prefix, skipFolders = true) => {
|
|
|
558
580
|
}
|
|
559
581
|
return contents;
|
|
560
582
|
};
|
|
583
|
+
exports.listS3Objects = listS3Objects;
|
|
561
584
|
/**
|
|
562
585
|
* Fetch complete list of S3 objects
|
|
563
586
|
*
|
|
@@ -574,14 +597,14 @@ exports.listS3Objects = async (bucket, prefix, skipFolders = true) => {
|
|
|
574
597
|
*
|
|
575
598
|
* @static
|
|
576
599
|
*/
|
|
577
|
-
|
|
600
|
+
const listS3ObjectsV2 = async (params) => {
|
|
578
601
|
// Fetch the first list of objects from S3
|
|
579
|
-
let listObjectsResponse = (await services_1.s3().listObjectsV2(params).promise());
|
|
602
|
+
let listObjectsResponse = (await (0, services_1.s3)().listObjectsV2(params).promise());
|
|
580
603
|
let discoveredObjects = listObjectsResponse.Contents;
|
|
581
604
|
// Keep listing more objects from S3 until we have all of them
|
|
582
605
|
while (listObjectsResponse.IsTruncated) {
|
|
583
606
|
// eslint-disable-next-line no-await-in-loop
|
|
584
|
-
listObjectsResponse = (await services_1.s3().listObjectsV2(
|
|
607
|
+
listObjectsResponse = (await (0, services_1.s3)().listObjectsV2(
|
|
585
608
|
// Update the params with a Continuation Token
|
|
586
609
|
{
|
|
587
610
|
...params,
|
|
@@ -591,6 +614,7 @@ exports.listS3ObjectsV2 = async (params) => {
|
|
|
591
614
|
}
|
|
592
615
|
return discoveredObjects;
|
|
593
616
|
};
|
|
617
|
+
exports.listS3ObjectsV2 = listS3ObjectsV2;
|
|
594
618
|
/**
|
|
595
619
|
* Calculate the cryptographic hash of an S3 object
|
|
596
620
|
*
|
|
@@ -601,12 +625,17 @@ exports.listS3ObjectsV2 = async (params) => {
|
|
|
601
625
|
* @param {string} params.bucket
|
|
602
626
|
* @param {string} params.key
|
|
603
627
|
*/
|
|
604
|
-
|
|
628
|
+
const calculateObjectHash = async (params) => {
|
|
605
629
|
// eslint-disable-next-line no-shadow
|
|
606
|
-
const { algorithm, bucket, key, s3 } = params;
|
|
607
|
-
const stream = exports.getObjectReadStream({
|
|
608
|
-
|
|
630
|
+
const { algorithm, bucket, key, s3: s3Client } = params;
|
|
631
|
+
const stream = (0, exports.getObjectReadStream)({
|
|
632
|
+
s3: s3Client,
|
|
633
|
+
bucket,
|
|
634
|
+
key,
|
|
635
|
+
});
|
|
636
|
+
return await (0, checksum_1.generateChecksumFromStream)(algorithm, stream);
|
|
609
637
|
};
|
|
638
|
+
exports.calculateObjectHash = calculateObjectHash;
|
|
610
639
|
/**
|
|
611
640
|
* Validate S3 object checksum against expected sum
|
|
612
641
|
*
|
|
@@ -620,15 +649,16 @@ exports.calculateObjectHash = async (params) => {
|
|
|
620
649
|
* @throws {InvalidChecksum} - Throws error if validation fails
|
|
621
650
|
* @returns {Promise<boolean>} returns true for success
|
|
622
651
|
*/
|
|
623
|
-
|
|
652
|
+
const validateS3ObjectChecksum = async (params) => {
|
|
624
653
|
const { algorithm, bucket, key, expectedSum, options } = params;
|
|
625
|
-
const fileStream = exports.getObjectReadStream({ s3: services_1.s3(), bucket, key });
|
|
626
|
-
if (await checksum_1.validateChecksumFromStream(algorithm, fileStream, expectedSum, options)) {
|
|
654
|
+
const fileStream = (0, exports.getObjectReadStream)({ s3: (0, services_1.s3)(), bucket, key });
|
|
655
|
+
if (await (0, checksum_1.validateChecksumFromStream)(algorithm, fileStream, expectedSum, options)) {
|
|
627
656
|
return true;
|
|
628
657
|
}
|
|
629
658
|
const msg = `Invalid checksum for S3 object s3://${bucket}/${key} with type ${algorithm} and expected sum ${expectedSum}`;
|
|
630
659
|
throw new errors_1.InvalidChecksum(msg);
|
|
631
660
|
};
|
|
661
|
+
exports.validateS3ObjectChecksum = validateS3ObjectChecksum;
|
|
632
662
|
/**
|
|
633
663
|
* Extract the S3 bucket and key from the URL path parameters
|
|
634
664
|
*
|
|
@@ -636,7 +666,7 @@ exports.validateS3ObjectChecksum = async (params) => {
|
|
|
636
666
|
* bucket/key in the form of
|
|
637
667
|
* @returns {Array<string>} `[Bucket, Key]`
|
|
638
668
|
*/
|
|
639
|
-
|
|
669
|
+
const getFileBucketAndKey = (pathParams) => {
|
|
640
670
|
const [Bucket, ...fields] = pathParams.split('/');
|
|
641
671
|
const Key = fields.join('/');
|
|
642
672
|
if (Bucket.length === 0 || Key.length === 0) {
|
|
@@ -644,20 +674,23 @@ exports.getFileBucketAndKey = (pathParams) => {
|
|
|
644
674
|
}
|
|
645
675
|
return [Bucket, Key];
|
|
646
676
|
};
|
|
677
|
+
exports.getFileBucketAndKey = getFileBucketAndKey;
|
|
647
678
|
/**
|
|
648
679
|
* Create an S3 bucket
|
|
649
680
|
*
|
|
650
681
|
* @param {string} Bucket - the name of the S3 bucket to create
|
|
651
682
|
* @returns {Promise}
|
|
652
683
|
*/
|
|
653
|
-
|
|
684
|
+
const createBucket = (Bucket) => (0, services_1.s3)().createBucket({ Bucket }).promise();
|
|
685
|
+
exports.createBucket = createBucket;
|
|
654
686
|
/**
|
|
655
687
|
* Create multiple S3 buckets
|
|
656
688
|
*
|
|
657
689
|
* @param {Array<string>} buckets - the names of the S3 buckets to create
|
|
658
690
|
* @returns {Promise}
|
|
659
691
|
*/
|
|
660
|
-
|
|
692
|
+
const createS3Buckets = async (buckets) => await Promise.all(buckets.map(exports.createBucket));
|
|
693
|
+
exports.createS3Buckets = createS3Buckets;
|
|
661
694
|
const createMultipartUpload = async (params) => {
|
|
662
695
|
const uploadParams = {
|
|
663
696
|
Bucket: params.destinationBucket,
|
|
@@ -713,10 +746,10 @@ const uploadPartCopy = async (params) => {
|
|
|
713
746
|
* @returns {Promise.<{ etag: string }>} object containing the ETag of the
|
|
714
747
|
* destination object
|
|
715
748
|
*/
|
|
716
|
-
|
|
749
|
+
const multipartCopyObject = async (params) => {
|
|
717
750
|
var _a;
|
|
718
751
|
const { sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags = false, } = params;
|
|
719
|
-
const sourceObject = (_a = params.sourceObject) !== null && _a !== void 0 ? _a : await exports.headObject(sourceBucket, sourceKey);
|
|
752
|
+
const sourceObject = (_a = params.sourceObject) !== null && _a !== void 0 ? _a : await (0, exports.headObject)(sourceBucket, sourceKey);
|
|
720
753
|
// Create a multi-part upload (copy) and get its UploadId
|
|
721
754
|
const uploadId = await createMultipartUpload({
|
|
722
755
|
sourceBucket,
|
|
@@ -767,6 +800,7 @@ exports.multipartCopyObject = async (params) => {
|
|
|
767
800
|
throw error;
|
|
768
801
|
}
|
|
769
802
|
};
|
|
803
|
+
exports.multipartCopyObject = multipartCopyObject;
|
|
770
804
|
/**
|
|
771
805
|
* Move an S3 object to another location in S3
|
|
772
806
|
*
|
|
@@ -779,15 +813,16 @@ exports.multipartCopyObject = async (params) => {
|
|
|
779
813
|
* @param {boolean} [params.copyTags=false]
|
|
780
814
|
* @returns {Promise<undefined>}
|
|
781
815
|
*/
|
|
782
|
-
|
|
783
|
-
await exports.multipartCopyObject({
|
|
816
|
+
const moveObject = async (params) => {
|
|
817
|
+
await (0, exports.multipartCopyObject)({
|
|
784
818
|
sourceBucket: params.sourceBucket,
|
|
785
819
|
sourceKey: params.sourceKey,
|
|
786
820
|
destinationBucket: params.destinationBucket,
|
|
787
821
|
destinationKey: params.destinationKey,
|
|
788
822
|
ACL: params.ACL,
|
|
789
|
-
copyTags: isBoolean_1.default(params.copyTags) ? params.copyTags : true,
|
|
823
|
+
copyTags: (0, isBoolean_1.default)(params.copyTags) ? params.copyTags : true,
|
|
790
824
|
});
|
|
791
|
-
await exports.deleteS3Object(params.sourceBucket, params.sourceKey);
|
|
825
|
+
await (0, exports.deleteS3Object)(params.sourceBucket, params.sourceKey);
|
|
792
826
|
};
|
|
827
|
+
exports.moveObject = moveObject;
|
|
793
828
|
//# sourceMappingURL=S3.js.map
|