@cumulus/aws-client 20.1.0 → 20.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -181
- package/S3.d.ts +34 -107
- package/S3.js +26 -104
- package/package.json +8 -7
package/README.md
CHANGED
|
@@ -287,7 +287,7 @@ Invoke a Lambda function
|
|
|
287
287
|
|
|
288
288
|
* [S3](#module_S3)
|
|
289
289
|
* _static_
|
|
290
|
-
* ~~[.getS3Object](#module_S3.getS3Object)
|
|
290
|
+
* ~~[.getS3Object](#module_S3.getS3Object)~~
|
|
291
291
|
* [.recursivelyDeleteS3Bucket](#module_S3.recursivelyDeleteS3Bucket) ⇒
|
|
292
292
|
* [.listS3ObjectsV2(params)](#module_S3.listS3ObjectsV2) ⇒ <code>Promise.<Array></code>
|
|
293
293
|
* [.listS3ObjectsV2Batch(params)](#module_S3.listS3ObjectsV2Batch)
|
|
@@ -299,24 +299,24 @@ Invoke a Lambda function
|
|
|
299
299
|
* [~deleteS3Object(bucket, key)](#module_S3..deleteS3Object) ⇒ <code>Promise</code>
|
|
300
300
|
* [~headObject(Bucket, Key, retryOptions)](#module_S3..headObject) ⇒
|
|
301
301
|
* [~s3ObjectExists(params)](#module_S3..s3ObjectExists) ⇒ <code>Promise.<boolean></code>
|
|
302
|
-
* [~waitForObjectToExist(
|
|
303
|
-
* [~s3PutObject(params)](#module_S3..s3PutObject)
|
|
304
|
-
* [~putFile(
|
|
305
|
-
* [~s3CopyObject(
|
|
306
|
-
* [~promiseS3Upload(
|
|
307
|
-
* [~streamS3Upload(
|
|
308
|
-
* [~getObjectReadStream(
|
|
309
|
-
* [~downloadS3File(
|
|
310
|
-
* [~getObjectSize(
|
|
311
|
-
* [~s3GetObjectTagging(
|
|
312
|
-
* [~s3PutObjectTagging(
|
|
313
|
-
* [~getObject(
|
|
314
|
-
* [~waitForObject(
|
|
315
|
-
* [~getObjectStreamContents(
|
|
316
|
-
* [~getTextObject(
|
|
317
|
-
* [~getJsonS3Object(
|
|
318
|
-
* [~fileExists(
|
|
319
|
-
* [~deleteS3Files(s3Objs)](#module_S3..deleteS3Files)
|
|
302
|
+
* [~waitForObjectToExist()](#module_S3..waitForObjectToExist)
|
|
303
|
+
* [~s3PutObject(params)](#module_S3..s3PutObject)
|
|
304
|
+
* [~putFile()](#module_S3..putFile)
|
|
305
|
+
* [~s3CopyObject()](#module_S3..s3CopyObject)
|
|
306
|
+
* [~promiseS3Upload()](#module_S3..promiseS3Upload)
|
|
307
|
+
* [~streamS3Upload()](#module_S3..streamS3Upload)
|
|
308
|
+
* [~getObjectReadStream()](#module_S3..getObjectReadStream)
|
|
309
|
+
* [~downloadS3File()](#module_S3..downloadS3File)
|
|
310
|
+
* [~getObjectSize()](#module_S3..getObjectSize)
|
|
311
|
+
* [~s3GetObjectTagging()](#module_S3..s3GetObjectTagging)
|
|
312
|
+
* [~s3PutObjectTagging()](#module_S3..s3PutObjectTagging)
|
|
313
|
+
* [~getObject()](#module_S3..getObject)
|
|
314
|
+
* [~waitForObject()](#module_S3..waitForObject)
|
|
315
|
+
* [~getObjectStreamContents()](#module_S3..getObjectStreamContents)
|
|
316
|
+
* [~getTextObject()](#module_S3..getTextObject)
|
|
317
|
+
* [~getJsonS3Object()](#module_S3..getJsonS3Object)
|
|
318
|
+
* [~fileExists()](#module_S3..fileExists)
|
|
319
|
+
* [~deleteS3Files(s3Objs)](#module_S3..deleteS3Files)
|
|
320
320
|
* [~uploadS3FileStream(fileStream, bucket, key, s3opts)](#module_S3..uploadS3FileStream) ⇒ <code>Promise</code>
|
|
321
321
|
* [~listS3Objects()](#module_S3..listS3Objects)
|
|
322
322
|
* [~deleteS3Buckets(buckets)](#module_S3..deleteS3Buckets) ⇒ <code>Promise</code>
|
|
@@ -326,24 +326,17 @@ Invoke a Lambda function
|
|
|
326
326
|
* [~createBucket(Bucket)](#module_S3..createBucket) ⇒ <code>Promise</code>
|
|
327
327
|
* [~createS3Buckets(buckets)](#module_S3..createS3Buckets) ⇒ <code>Promise</code>
|
|
328
328
|
* [~multipartCopyObject(params)](#module_S3..multipartCopyObject) ⇒ <code>Promise.<{etag: string}></code>
|
|
329
|
-
* [~
|
|
329
|
+
* [~copyObject()](#module_S3..copyObject)
|
|
330
|
+
* [~moveObject()](#module_S3..moveObject) ⇒ <code>Promise.<undefined></code>
|
|
330
331
|
|
|
331
332
|
<a name="module_S3.getS3Object"></a>
|
|
332
333
|
|
|
333
|
-
### ~~S3.getS3Object
|
|
334
|
+
### ~~S3.getS3Object~~
|
|
334
335
|
***Deprecated***
|
|
335
336
|
|
|
336
337
|
Gets an object from S3.
|
|
337
338
|
|
|
338
339
|
**Kind**: static property of [<code>S3</code>](#module_S3)
|
|
339
|
-
**Returns**: <code>Promise</code> - returns response from `S3.getObject` as a promise
|
|
340
|
-
|
|
341
|
-
| Param | Type | Description |
|
|
342
|
-
| --- | --- | --- |
|
|
343
|
-
| Bucket | <code>string</code> | name of bucket |
|
|
344
|
-
| Key | <code>string</code> | key for object (filepath + filename) |
|
|
345
|
-
| retryOptions | <code>Object</code> | options to control retry behavior when an object does not exist. See https://github.com/tim-kos/node-retry#retryoperationoptions By default, retries will not be performed |
|
|
346
|
-
|
|
347
340
|
<a name="module_S3.recursivelyDeleteS3Bucket"></a>
|
|
348
341
|
|
|
349
342
|
### S3.recursivelyDeleteS3Bucket ⇒
|
|
@@ -478,253 +471,139 @@ Test if an object exists in S3
|
|
|
478
471
|
|
|
479
472
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
480
473
|
**Returns**: <code>Promise.<boolean></code> - a Promise that will resolve to a boolean indicating
|
|
481
|
-
|
|
474
|
+
if the object exists
|
|
482
475
|
|
|
483
|
-
| Param |
|
|
484
|
-
| --- | --- |
|
|
485
|
-
| params |
|
|
476
|
+
| Param | Description |
|
|
477
|
+
| --- | --- |
|
|
478
|
+
| params | same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property |
|
|
486
479
|
|
|
487
480
|
<a name="module_S3..waitForObjectToExist"></a>
|
|
488
481
|
|
|
489
|
-
### S3~waitForObjectToExist(
|
|
482
|
+
### S3~waitForObjectToExist()
|
|
490
483
|
Wait for an object to exist in S3
|
|
491
484
|
|
|
492
485
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
493
486
|
|
|
494
|
-
| Param |
|
|
495
|
-
| --- | --- | --- |
|
|
496
|
-
| params | <code>
|
|
497
|
-
| params.
|
|
498
|
-
| params.key | <code>string</code> | | |
|
|
499
|
-
| [params.interval] | <code>number</code> | <code>1000</code> | interval before retries, in ms |
|
|
500
|
-
| [params.timeout] | <code>number</code> | <code>30000</code> | timeout, in ms |
|
|
487
|
+
| Param | Default | Description |
|
|
488
|
+
| --- | --- | --- |
|
|
489
|
+
| params.interval | <code>1000</code> | interval before retries, in ms |
|
|
490
|
+
| params.timeout | <code>30000</code> | timeout, in ms |
|
|
501
491
|
|
|
502
492
|
<a name="module_S3..s3PutObject"></a>
|
|
503
493
|
|
|
504
|
-
### S3~s3PutObject(params)
|
|
494
|
+
### S3~s3PutObject(params)
|
|
505
495
|
Put an object on S3
|
|
506
496
|
|
|
507
497
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
508
498
|
|
|
509
|
-
| Param |
|
|
510
|
-
| --- | --- |
|
|
511
|
-
| params |
|
|
499
|
+
| Param | Description |
|
|
500
|
+
| --- | --- |
|
|
501
|
+
| params | same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property |
|
|
512
502
|
|
|
513
503
|
<a name="module_S3..putFile"></a>
|
|
514
504
|
|
|
515
|
-
### S3~putFile(
|
|
505
|
+
### S3~putFile()
|
|
516
506
|
Upload a file to S3
|
|
517
507
|
|
|
518
508
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
519
|
-
|
|
520
|
-
| Param | Type | Description |
|
|
521
|
-
| --- | --- | --- |
|
|
522
|
-
| bucket | <code>string</code> | the destination S3 bucket |
|
|
523
|
-
| key | <code>string</code> | the destination S3 key |
|
|
524
|
-
| filename | <code>filename</code> | the local file to be uploaded |
|
|
525
|
-
|
|
526
509
|
<a name="module_S3..s3CopyObject"></a>
|
|
527
510
|
|
|
528
|
-
### S3~s3CopyObject(
|
|
511
|
+
### S3~s3CopyObject()
|
|
529
512
|
Copy an object from one location on S3 to another
|
|
530
513
|
|
|
531
514
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
532
|
-
**Returns**: <code>Promise</code> - promise of the object being copied
|
|
533
|
-
|
|
534
|
-
| Param | Type | Description |
|
|
535
|
-
| --- | --- | --- |
|
|
536
|
-
| params | <code>Object</code> | same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property |
|
|
537
|
-
|
|
538
515
|
<a name="module_S3..promiseS3Upload"></a>
|
|
539
516
|
|
|
540
|
-
### S3~promiseS3Upload(
|
|
517
|
+
### S3~promiseS3Upload()
|
|
541
518
|
Upload data to S3
|
|
542
519
|
|
|
543
520
|
see https://github.com/aws/aws-sdk-js-v3/tree/main/lib/lib-storage
|
|
544
521
|
|
|
545
522
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
546
|
-
|
|
547
|
-
| Param | Type |
|
|
548
|
-
| --- | --- |
|
|
549
|
-
| params | <code>UploadOptions</code> |
|
|
550
|
-
|
|
551
523
|
<a name="module_S3..streamS3Upload"></a>
|
|
552
524
|
|
|
553
|
-
### S3~streamS3Upload(
|
|
525
|
+
### S3~streamS3Upload()
|
|
554
526
|
Upload data to S3 using a stream
|
|
555
527
|
|
|
556
528
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
557
|
-
|
|
558
|
-
| Param | Type | Description |
|
|
559
|
-
| --- | --- | --- |
|
|
560
|
-
| uploadStream | <code>Readable</code> | Stream of data to upload |
|
|
561
|
-
| uploadParams | <code>Object</code> | |
|
|
562
|
-
|
|
563
529
|
<a name="module_S3..getObjectReadStream"></a>
|
|
564
530
|
|
|
565
|
-
### S3~getObjectReadStream(
|
|
531
|
+
### S3~getObjectReadStream()
|
|
566
532
|
Get a readable stream for an S3 object
|
|
567
533
|
|
|
568
534
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
569
|
-
|
|
570
|
-
| Param | Type | Description |
|
|
571
|
-
| --- | --- | --- |
|
|
572
|
-
| params | <code>Object</code> | |
|
|
573
|
-
| params.s3 | <code>S3</code> | an S3 instance |
|
|
574
|
-
| params.bucket | <code>string</code> | the bucket of the requested object |
|
|
575
|
-
| params.key | <code>string</code> | the key of the requested object |
|
|
576
|
-
|
|
577
535
|
<a name="module_S3..downloadS3File"></a>
|
|
578
536
|
|
|
579
|
-
### S3~downloadS3File(
|
|
537
|
+
### S3~downloadS3File()
|
|
580
538
|
Downloads the given s3Obj to the given filename in a streaming manner
|
|
581
539
|
|
|
582
540
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
583
|
-
**Returns**: <code>Promise.<string></code> - returns filename if successful
|
|
584
|
-
|
|
585
|
-
| Param | Type | Description |
|
|
586
|
-
| --- | --- | --- |
|
|
587
|
-
| s3Obj | <code>Object</code> | The parameters to send to S3 getObject call |
|
|
588
|
-
| filepath | <code>string</code> | The filepath of the file that is downloaded |
|
|
589
|
-
|
|
590
541
|
<a name="module_S3..getObjectSize"></a>
|
|
591
542
|
|
|
592
|
-
### S3~getObjectSize(
|
|
543
|
+
### S3~getObjectSize()
|
|
593
544
|
Get the size of an S3 object
|
|
594
545
|
|
|
595
546
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
596
|
-
**Returns**: <code>Promise.<(number\|undefined)></code> - object size, in bytes
|
|
597
|
-
|
|
598
|
-
| Param | Type | Description |
|
|
599
|
-
| --- | --- | --- |
|
|
600
|
-
| params | <code>Object</code> | |
|
|
601
|
-
| params.bucket | <code>string</code> | |
|
|
602
|
-
| params.key | <code>string</code> | |
|
|
603
|
-
| params.s3 | <code>S3</code> | an S3 client instance |
|
|
604
|
-
|
|
605
547
|
<a name="module_S3..s3GetObjectTagging"></a>
|
|
606
548
|
|
|
607
|
-
### S3~s3GetObjectTagging(
|
|
549
|
+
### S3~s3GetObjectTagging()
|
|
608
550
|
Get object Tagging from S3
|
|
609
551
|
|
|
610
552
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
611
|
-
**Returns**: <code>Promise.<GetObjectTaggingOutput></code> - the promised response from `S3.getObjectTagging`
|
|
612
|
-
|
|
613
|
-
| Param | Type | Description |
|
|
614
|
-
| --- | --- | --- |
|
|
615
|
-
| bucket | <code>string</code> | name of bucket |
|
|
616
|
-
| key | <code>string</code> | key for object (filepath + filename) |
|
|
617
|
-
|
|
618
553
|
<a name="module_S3..s3PutObjectTagging"></a>
|
|
619
554
|
|
|
620
|
-
### S3~s3PutObjectTagging(
|
|
555
|
+
### S3~s3PutObjectTagging()
|
|
621
556
|
Puts object Tagging in S3
|
|
622
557
|
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObjectTagging-property
|
|
623
558
|
|
|
624
559
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
625
|
-
**Returns**: <code>Promise</code> - returns response from `S3.getObjectTagging` as a promise
|
|
626
|
-
|
|
627
|
-
| Param | Type | Description |
|
|
628
|
-
| --- | --- | --- |
|
|
629
|
-
| Bucket | <code>string</code> | name of bucket |
|
|
630
|
-
| Key | <code>string</code> | key for object (filepath + filename) |
|
|
631
|
-
| ObjectTagging | <code>Object</code> | tagging object |
|
|
632
|
-
|
|
633
560
|
<a name="module_S3..getObject"></a>
|
|
634
561
|
|
|
635
|
-
### S3~getObject(
|
|
562
|
+
### S3~getObject()
|
|
636
563
|
Gets an object from S3.
|
|
637
564
|
|
|
638
565
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
639
|
-
**Returns**: <code>Promise.<GetObjectOutput></code> - response from `S3.getObject()`
|
|
640
|
-
as a Promise
|
|
641
|
-
|
|
642
|
-
| Param | Type | Description |
|
|
643
|
-
| --- | --- | --- |
|
|
644
|
-
| s3Client | <code>S3</code> | an `S3` instance |
|
|
645
|
-
| params | <code>GetObjectCommandInput</code> | parameters object to pass through to `S3.getObject()` |
|
|
646
|
-
|
|
647
|
-
**Example**
|
|
648
|
-
```js
|
|
649
|
-
const obj = await getObject(s3(), { Bucket: 'b', Key: 'k' })
|
|
650
|
-
```
|
|
651
566
|
<a name="module_S3..waitForObject"></a>
|
|
652
567
|
|
|
653
|
-
### S3~waitForObject(
|
|
568
|
+
### S3~waitForObject()
|
|
654
569
|
Get an object from S3, waiting for it to exist and, if specified, have the
|
|
655
570
|
correct ETag.
|
|
656
571
|
|
|
657
572
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
658
|
-
|
|
659
|
-
| Param | Type | Default |
|
|
660
|
-
| --- | --- | --- |
|
|
661
|
-
| s3Client | <code>S3</code> | |
|
|
662
|
-
| params | <code>GetObjectCommandInput</code> | |
|
|
663
|
-
| [retryOptions] | <code>pRetry.Options</code> | <code>{}</code> |
|
|
664
|
-
|
|
665
573
|
<a name="module_S3..getObjectStreamContents"></a>
|
|
666
574
|
|
|
667
|
-
### S3~getObjectStreamContents(
|
|
575
|
+
### S3~getObjectStreamContents()
|
|
668
576
|
Transform streaming response from S3 object to text content
|
|
669
577
|
|
|
670
578
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
671
|
-
**Returns**: <code>Promise.<string></code> - the contents of the S3 object
|
|
672
|
-
|
|
673
|
-
| Param | Type | Description |
|
|
674
|
-
| --- | --- | --- |
|
|
675
|
-
| objectReadStream | <code>Readable</code> | Readable stream of S3 object |
|
|
676
|
-
|
|
677
579
|
<a name="module_S3..getTextObject"></a>
|
|
678
580
|
|
|
679
|
-
### S3~getTextObject(
|
|
581
|
+
### S3~getTextObject()
|
|
680
582
|
Fetch the contents of an S3 object
|
|
681
583
|
|
|
682
584
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
683
|
-
**Returns**: <code>Promise.<string></code> - the contents of the S3 object
|
|
684
|
-
|
|
685
|
-
| Param | Type | Description |
|
|
686
|
-
| --- | --- | --- |
|
|
687
|
-
| bucket | <code>string</code> | the S3 object's bucket |
|
|
688
|
-
| key | <code>string</code> | the S3 object's key |
|
|
689
|
-
|
|
690
585
|
<a name="module_S3..getJsonS3Object"></a>
|
|
691
586
|
|
|
692
|
-
### S3~getJsonS3Object(
|
|
587
|
+
### S3~getJsonS3Object()
|
|
693
588
|
Fetch JSON stored in an S3 object
|
|
694
589
|
|
|
695
590
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
696
|
-
**Returns**: <code>Promise.<\*></code> - the contents of the S3 object, parsed as JSON
|
|
697
|
-
|
|
698
|
-
| Param | Type | Description |
|
|
699
|
-
| --- | --- | --- |
|
|
700
|
-
| bucket | <code>string</code> | the S3 object's bucket |
|
|
701
|
-
| key | <code>string</code> | the S3 object's key |
|
|
702
|
-
|
|
703
591
|
<a name="module_S3..fileExists"></a>
|
|
704
592
|
|
|
705
|
-
### S3~fileExists(
|
|
593
|
+
### S3~fileExists()
|
|
706
594
|
Check if a file exists in an S3 object
|
|
707
595
|
|
|
708
596
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
709
|
-
**Returns**: <code>Promise</code> - returns the response from `S3.headObject` as a promise
|
|
710
|
-
|
|
711
|
-
| Param | Type | Description |
|
|
712
|
-
| --- | --- | --- |
|
|
713
|
-
| bucket | <code>string</code> | name of the S3 bucket |
|
|
714
|
-
| key | <code>string</code> | key of the file in the S3 bucket |
|
|
715
|
-
|
|
716
597
|
<a name="module_S3..deleteS3Files"></a>
|
|
717
598
|
|
|
718
|
-
### S3~deleteS3Files(s3Objs)
|
|
599
|
+
### S3~deleteS3Files(s3Objs)
|
|
719
600
|
Delete files from S3
|
|
720
601
|
|
|
721
602
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
722
|
-
**Returns**: <code>Promise</code> - A promise that resolves to an Array of the data returned
|
|
723
|
-
from the deletion operations
|
|
724
603
|
|
|
725
|
-
| Param |
|
|
726
|
-
| --- | --- |
|
|
727
|
-
| s3Objs |
|
|
604
|
+
| Param | Description |
|
|
605
|
+
| --- | --- |
|
|
606
|
+
| s3Objs | An array of objects containing keys 'Bucket' and 'Key' |
|
|
728
607
|
|
|
729
608
|
<a name="module_S3..uploadS3FileStream"></a>
|
|
730
609
|
|
|
@@ -852,20 +731,28 @@ note: this method may error if used with zero byte files. see CUMULUS-2557 and h
|
|
|
852
731
|
| [params.copyTags] | <code>boolean</code> | <code>false</code> | |
|
|
853
732
|
| [params.chunkSize] | <code>number</code> | | chunk size of the S3 multipart uploads |
|
|
854
733
|
|
|
734
|
+
<a name="module_S3..copyObject"></a>
|
|
735
|
+
|
|
736
|
+
### S3~copyObject()
|
|
737
|
+
Copy an S3 object to another location in S3
|
|
738
|
+
|
|
739
|
+
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
740
|
+
|
|
741
|
+
| Param | Default | Description |
|
|
742
|
+
| --- | --- | --- |
|
|
743
|
+
| params.ACL | | an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl) |
|
|
744
|
+
| params.copyTags | <code>false</code> | |
|
|
745
|
+
| params.chunkSize | | chunk size of the S3 multipart uploads |
|
|
746
|
+
|
|
855
747
|
<a name="module_S3..moveObject"></a>
|
|
856
748
|
|
|
857
|
-
### S3~moveObject(
|
|
749
|
+
### S3~moveObject() ⇒ <code>Promise.<undefined></code>
|
|
858
750
|
Move an S3 object to another location in S3
|
|
859
751
|
|
|
860
752
|
**Kind**: inner method of [<code>S3</code>](#module_S3)
|
|
861
753
|
|
|
862
754
|
| Param | Type | Default | Description |
|
|
863
755
|
| --- | --- | --- | --- |
|
|
864
|
-
| params | <code>Object</code> | | |
|
|
865
|
-
| params.sourceBucket | <code>string</code> | | |
|
|
866
|
-
| params.sourceKey | <code>string</code> | | |
|
|
867
|
-
| params.destinationBucket | <code>string</code> | | |
|
|
868
|
-
| params.destinationKey | <code>string</code> | | |
|
|
869
756
|
| [params.ACL] | <code>string</code> | | an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl) |
|
|
870
757
|
| [params.copyTags] | <code>boolean</code> | <code>false</code> | |
|
|
871
758
|
| [params.chunkSize] | <code>number</code> | | chunk size of the S3 multipart uploads |
|
package/S3.d.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
/// <reference types="node" />
|
|
6
6
|
import pRetry from 'p-retry';
|
|
7
7
|
import { Readable, TransformOptions } from 'stream';
|
|
8
|
-
import { CopyObjectCommandInput, DeleteObjectRequest, DeleteBucketCommandOutput, GetObjectCommandInput, GetObjectOutput, HeadObjectOutput, ListObjectsV2Request, ObjectCannedACL, PutObjectCommandInput, PutObjectRequest, S3, Tagging, ListObjectsCommandOutput } from '@aws-sdk/client-s3';
|
|
8
|
+
import { CopyObjectCommandInput, DeleteObjectRequest, DeleteBucketCommandOutput, GetObjectCommandInput, GetObjectOutput, HeadObjectOutput, ListObjectsV2Request, ObjectCannedACL, PutObjectCommandInput, PutObjectRequest, S3, Tagging, ListObjectsCommandOutput, DeleteObjectCommandOutput, PutObjectCommandOutput, CopyObjectCommandOutput, CompleteMultipartUploadCommandOutput, GetObjectTaggingCommandOutput } from '@aws-sdk/client-s3';
|
|
9
9
|
import { Options as UploadOptions } from '@aws-sdk/lib-storage';
|
|
10
10
|
import { s3 } from './services';
|
|
11
11
|
export declare type GetObjectMethod = (params: GetObjectCommandInput) => Promise<GetObjectOutput>;
|
|
@@ -49,7 +49,7 @@ export declare const s3TagSetToQueryString: (tagset: Tagging['TagSet']) => strin
|
|
|
49
49
|
* @param {string} key - key of the object to be deleted
|
|
50
50
|
* @returns {Promise} promise of the object being deleted
|
|
51
51
|
*/
|
|
52
|
-
export declare const deleteS3Object: (bucket: string, key: string) => Promise<
|
|
52
|
+
export declare const deleteS3Object: (bucket: string, key: string) => Promise<DeleteObjectCommandOutput>;
|
|
53
53
|
export declare const deleteS3Objects: (params: {
|
|
54
54
|
client: S3;
|
|
55
55
|
bucket: string;
|
|
@@ -69,9 +69,9 @@ export declare const headObject: (Bucket: string, Key: string, retryOptions?: pR
|
|
|
69
69
|
/**
|
|
70
70
|
* Test if an object exists in S3
|
|
71
71
|
*
|
|
72
|
-
* @param
|
|
72
|
+
* @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
|
|
73
73
|
* @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
|
|
74
|
-
*
|
|
74
|
+
* if the object exists
|
|
75
75
|
*/
|
|
76
76
|
export declare const s3ObjectExists: (params: {
|
|
77
77
|
Bucket: string;
|
|
@@ -79,13 +79,8 @@ export declare const s3ObjectExists: (params: {
|
|
|
79
79
|
}) => Promise<boolean>;
|
|
80
80
|
/**
|
|
81
81
|
* Wait for an object to exist in S3
|
|
82
|
-
*
|
|
83
|
-
* @param
|
|
84
|
-
* @param {string} params.bucket
|
|
85
|
-
* @param {string} params.key
|
|
86
|
-
* @param {number} [params.interval=1000] - interval before retries, in ms
|
|
87
|
-
* @param {number} [params.timeout=30000] - timeout, in ms
|
|
88
|
-
* @returns {Promise<undefined>}
|
|
82
|
+
* @param params.interval=1000 - interval before retries, in ms
|
|
83
|
+
* @param params.timeout=30000 - timeout, in ms
|
|
89
84
|
*/
|
|
90
85
|
export declare const waitForObjectToExist: (params: {
|
|
91
86
|
bucket: string;
|
|
@@ -96,34 +91,22 @@ export declare const waitForObjectToExist: (params: {
|
|
|
96
91
|
/**
|
|
97
92
|
* Put an object on S3
|
|
98
93
|
*
|
|
99
|
-
* @param
|
|
100
|
-
* promise of the object being put
|
|
101
|
-
* @returns {Promise}
|
|
94
|
+
* @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
|
|
102
95
|
**/
|
|
103
|
-
export declare const s3PutObject: (params: PutObjectCommandInput) => Promise<
|
|
96
|
+
export declare const s3PutObject: (params: PutObjectCommandInput) => Promise<PutObjectCommandOutput>;
|
|
104
97
|
/**
|
|
105
98
|
* Upload a file to S3
|
|
106
99
|
*
|
|
107
|
-
* @param {string} bucket - the destination S3 bucket
|
|
108
|
-
* @param {string} key - the destination S3 key
|
|
109
|
-
* @param {filename} filename - the local file to be uploaded
|
|
110
|
-
* @returns {Promise}
|
|
111
100
|
*/
|
|
112
|
-
export declare const putFile: (bucket: string, key: string, filename: string) => Promise<
|
|
101
|
+
export declare const putFile: (bucket: string, key: string, filename: string) => Promise<PutObjectCommandOutput>;
|
|
113
102
|
/**
|
|
114
103
|
* Copy an object from one location on S3 to another
|
|
115
|
-
*
|
|
116
|
-
* @param {Object} params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
|
|
117
|
-
* @returns {Promise} promise of the object being copied
|
|
118
104
|
**/
|
|
119
|
-
export declare const s3CopyObject: (params: CopyObjectCommandInput) => Promise<
|
|
105
|
+
export declare const s3CopyObject: (params: CopyObjectCommandInput) => Promise<CopyObjectCommandOutput>;
|
|
120
106
|
/**
|
|
121
107
|
* Upload data to S3
|
|
122
108
|
*
|
|
123
109
|
* see https://github.com/aws/aws-sdk-js-v3/tree/main/lib/lib-storage
|
|
124
|
-
*
|
|
125
|
-
* @param {UploadOptions} params
|
|
126
|
-
* @returns {Promise}
|
|
127
110
|
*/
|
|
128
111
|
export declare const promiseS3Upload: (params: Omit<UploadOptions, 'client'>) => Promise<{
|
|
129
112
|
[key: string]: any;
|
|
@@ -131,20 +114,10 @@ export declare const promiseS3Upload: (params: Omit<UploadOptions, 'client'>) =>
|
|
|
131
114
|
}>;
|
|
132
115
|
/**
|
|
133
116
|
* Upload data to S3 using a stream
|
|
134
|
-
*
|
|
135
|
-
* @param {Readable} uploadStream - Stream of data to upload
|
|
136
|
-
* @param {Object} uploadParams
|
|
137
|
-
* @returns {Promise}
|
|
138
117
|
*/
|
|
139
|
-
export declare const streamS3Upload: (uploadStream: Readable, uploadParams: UploadOptions) => Promise<
|
|
118
|
+
export declare const streamS3Upload: (uploadStream: Readable, uploadParams: UploadOptions) => Promise<CompleteMultipartUploadCommandOutput>;
|
|
140
119
|
/**
|
|
141
120
|
* Get a readable stream for an S3 object
|
|
142
|
-
*
|
|
143
|
-
* @param {Object} params
|
|
144
|
-
* @param {S3} params.s3 - an S3 instance
|
|
145
|
-
* @param {string} params.bucket - the bucket of the requested object
|
|
146
|
-
* @param {string} params.key - the key of the requested object
|
|
147
|
-
* @returns {Promise<Readable>}
|
|
148
121
|
*/
|
|
149
122
|
export declare const getObjectReadStream: (params: {
|
|
150
123
|
s3: {
|
|
@@ -155,20 +128,10 @@ export declare const getObjectReadStream: (params: {
|
|
|
155
128
|
}) => Promise<Readable>;
|
|
156
129
|
/**
|
|
157
130
|
* Downloads the given s3Obj to the given filename in a streaming manner
|
|
158
|
-
*
|
|
159
|
-
* @param {Object} s3Obj - The parameters to send to S3 getObject call
|
|
160
|
-
* @param {string} filepath - The filepath of the file that is downloaded
|
|
161
|
-
* @returns {Promise<string>} returns filename if successful
|
|
162
131
|
*/
|
|
163
132
|
export declare const downloadS3File: (s3Obj: GetObjectCommandInput, filepath: string) => Promise<string>;
|
|
164
133
|
/**
|
|
165
134
|
* Get the size of an S3 object
|
|
166
|
-
*
|
|
167
|
-
* @param {Object} params
|
|
168
|
-
* @param {string} params.bucket
|
|
169
|
-
* @param {string} params.key
|
|
170
|
-
* @param {S3} params.s3 - an S3 client instance
|
|
171
|
-
* @returns {Promise<number|undefined>} object size, in bytes
|
|
172
135
|
*/
|
|
173
136
|
export declare const getObjectSize: (params: {
|
|
174
137
|
s3: S3;
|
|
@@ -177,98 +140,51 @@ export declare const getObjectSize: (params: {
|
|
|
177
140
|
}) => Promise<number | undefined>;
|
|
178
141
|
/**
|
|
179
142
|
* Get object Tagging from S3
|
|
180
|
-
*
|
|
181
|
-
* @param {string} bucket - name of bucket
|
|
182
|
-
* @param {string} key - key for object (filepath + filename)
|
|
183
|
-
* @returns {Promise<GetObjectTaggingOutput>} the promised response from `S3.getObjectTagging`
|
|
184
143
|
**/
|
|
185
|
-
export declare const s3GetObjectTagging: (bucket: string, key: string) => Promise<
|
|
144
|
+
export declare const s3GetObjectTagging: (bucket: string, key: string) => Promise<GetObjectTaggingCommandOutput>;
|
|
186
145
|
/**
|
|
187
146
|
* Puts object Tagging in S3
|
|
188
147
|
* https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObjectTagging-property
|
|
189
|
-
*
|
|
190
|
-
* @param {string} Bucket - name of bucket
|
|
191
|
-
* @param {string} Key - key for object (filepath + filename)
|
|
192
|
-
* @param {Object} ObjectTagging - tagging object
|
|
193
|
-
* @returns {Promise} returns response from `S3.getObjectTagging` as a promise
|
|
194
148
|
**/
|
|
195
|
-
export declare const s3PutObjectTagging: (Bucket: string, Key: string, ObjectTagging: Tagging) => Promise<
|
|
149
|
+
export declare const s3PutObjectTagging: (Bucket: string, Key: string, ObjectTagging: Tagging) => Promise<PutObjectCommandOutput>;
|
|
196
150
|
/**
|
|
197
151
|
* Gets an object from S3.
|
|
198
|
-
*
|
|
199
|
-
* @example
|
|
200
|
-
* const obj = await getObject(s3(), { Bucket: 'b', Key: 'k' })
|
|
201
|
-
*
|
|
202
|
-
* @param {S3} s3Client - an `S3` instance
|
|
203
|
-
* @param {GetObjectCommandInput} params - parameters object to pass through
|
|
204
|
-
* to `S3.getObject()`
|
|
205
|
-
* @returns {Promise<GetObjectOutput>} response from `S3.getObject()`
|
|
206
|
-
* as a Promise
|
|
207
152
|
*/
|
|
208
153
|
export declare const getObject: (s3Client: S3, params: GetObjectCommandInput) => Promise<GetObjectOutput>;
|
|
209
154
|
/**
|
|
210
155
|
* Get an object from S3, waiting for it to exist and, if specified, have the
|
|
211
156
|
* correct ETag.
|
|
212
|
-
*
|
|
213
|
-
* @param {S3} s3Client
|
|
214
|
-
* @param {GetObjectCommandInput} params
|
|
215
|
-
* @param {pRetry.Options} [retryOptions={}]
|
|
216
|
-
* @returns {Promise<GetObjectOutput>}
|
|
217
157
|
*/
|
|
218
158
|
export declare const waitForObject: (s3Client: S3, params: GetObjectCommandInput, retryOptions?: pRetry.Options) => Promise<GetObjectOutput>;
|
|
219
159
|
/**
|
|
220
160
|
* Gets an object from S3.
|
|
221
|
-
*
|
|
222
|
-
* @param {string} Bucket - name of bucket
|
|
223
|
-
* @param {string} Key - key for object (filepath + filename)
|
|
224
|
-
* @param {Object} retryOptions - options to control retry behavior when an
|
|
225
|
-
* object does not exist. See https://github.com/tim-kos/node-retry#retryoperationoptions
|
|
226
|
-
* By default, retries will not be performed
|
|
227
|
-
* @returns {Promise} returns response from `S3.getObject` as a promise
|
|
228
|
-
*
|
|
229
161
|
* @deprecated
|
|
230
162
|
*/
|
|
231
163
|
export declare const getS3Object: (Bucket: string, Key: string, retryOptions?: pRetry.Options) => Promise<GetObjectOutput>;
|
|
232
164
|
export declare const getObjectStreamBuffers: (objectReadStream: Readable) => Promise<Buffer[]>;
|
|
233
165
|
/**
|
|
234
166
|
* Transform streaming response from S3 object to text content
|
|
235
|
-
*
|
|
236
|
-
* @param {Readable} objectReadStream - Readable stream of S3 object
|
|
237
|
-
* @returns {Promise<string>} the contents of the S3 object
|
|
238
167
|
*/
|
|
239
168
|
export declare const getObjectStreamContents: (objectReadStream: Readable) => Promise<string>;
|
|
240
169
|
/**
|
|
241
170
|
* Fetch the contents of an S3 object
|
|
242
|
-
*
|
|
243
|
-
* @param {string} bucket - the S3 object's bucket
|
|
244
|
-
* @param {string} key - the S3 object's key
|
|
245
|
-
* @returns {Promise<string>} the contents of the S3 object
|
|
246
171
|
*/
|
|
247
172
|
export declare const getTextObject: (bucket: string, key: string) => Promise<string>;
|
|
248
173
|
/**
|
|
249
174
|
* Fetch JSON stored in an S3 object
|
|
250
|
-
* @param {string} bucket - the S3 object's bucket
|
|
251
|
-
* @param {string} key - the S3 object's key
|
|
252
|
-
* @returns {Promise<*>} the contents of the S3 object, parsed as JSON
|
|
253
175
|
*/
|
|
254
176
|
export declare const getJsonS3Object: (bucket: string, key: string) => Promise<any>;
|
|
255
|
-
export declare const putJsonS3Object: (bucket: string, key: string, data: any) => Promise<
|
|
177
|
+
export declare const putJsonS3Object: (bucket: string, key: string, data: any) => Promise<PutObjectCommandOutput>;
|
|
256
178
|
/**
|
|
257
179
|
* Check if a file exists in an S3 object
|
|
258
|
-
*
|
|
259
|
-
* @param {string} bucket - name of the S3 bucket
|
|
260
|
-
* @param {string} key - key of the file in the S3 bucket
|
|
261
|
-
* @returns {Promise} returns the response from `S3.headObject` as a promise
|
|
262
180
|
**/
|
|
263
|
-
export declare const fileExists: (bucket: string, key: string) => Promise<
|
|
181
|
+
export declare const fileExists: (bucket: string, key: string) => Promise<HeadObjectOutput | false>;
|
|
264
182
|
/**
|
|
265
183
|
* Delete files from S3
|
|
266
184
|
*
|
|
267
|
-
* @param
|
|
268
|
-
* @returns {Promise} A promise that resolves to an Array of the data returned
|
|
269
|
-
* from the deletion operations
|
|
185
|
+
* @param s3Objs - An array of objects containing keys 'Bucket' and 'Key'
|
|
270
186
|
*/
|
|
271
|
-
export declare const deleteS3Files: (s3Objs: DeleteObjectRequest[]) => Promise<
|
|
187
|
+
export declare const deleteS3Files: (s3Objs: DeleteObjectRequest[]) => Promise<DeleteObjectCommandOutput[]>;
|
|
272
188
|
declare type FileInfo = {
|
|
273
189
|
filename: string;
|
|
274
190
|
key: string;
|
|
@@ -434,14 +350,25 @@ export declare const multipartCopyObject: (params: {
|
|
|
434
350
|
}) => Promise<{
|
|
435
351
|
etag: string;
|
|
436
352
|
}>;
|
|
353
|
+
/**
|
|
354
|
+
* Copy an S3 object to another location in S3
|
|
355
|
+
*
|
|
356
|
+
* @param params.ACL - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
|
|
357
|
+
* @param params.copyTags=false
|
|
358
|
+
* @param params.chunkSize - chunk size of the S3 multipart uploads
|
|
359
|
+
*/
|
|
360
|
+
export declare const copyObject: ({ sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags, chunkSize, }: {
|
|
361
|
+
sourceBucket: string;
|
|
362
|
+
sourceKey: string;
|
|
363
|
+
destinationBucket: string;
|
|
364
|
+
destinationKey: string;
|
|
365
|
+
ACL?: string | undefined;
|
|
366
|
+
copyTags?: boolean | undefined;
|
|
367
|
+
chunkSize?: number | undefined;
|
|
368
|
+
}) => Promise<void>;
|
|
437
369
|
/**
|
|
438
370
|
* Move an S3 object to another location in S3
|
|
439
371
|
*
|
|
440
|
-
* @param {Object} params
|
|
441
|
-
* @param {string} params.sourceBucket
|
|
442
|
-
* @param {string} params.sourceKey
|
|
443
|
-
* @param {string} params.destinationBucket
|
|
444
|
-
* @param {string} params.destinationKey
|
|
445
372
|
* @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
|
|
446
373
|
* @param {boolean} [params.copyTags=false]
|
|
447
374
|
* @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
|
|
@@ -455,6 +382,6 @@ export declare const moveObject: (params: {
|
|
|
455
382
|
ACL?: string;
|
|
456
383
|
copyTags?: boolean;
|
|
457
384
|
chunkSize?: number;
|
|
458
|
-
}) => Promise<
|
|
385
|
+
}) => Promise<DeleteObjectCommandOutput>;
|
|
459
386
|
export {};
|
|
460
387
|
//# sourceMappingURL=S3.d.ts.map
|
package/S3.js
CHANGED
|
@@ -29,7 +29,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
29
29
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
30
30
|
};
|
|
31
31
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
32
|
-
exports.moveObject = exports.multipartCopyObject = exports.createS3Buckets = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.deleteS3Buckets = exports.recursivelyDeleteS3Bucket = exports.listS3ObjectsV2Batch = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.deleteS3Files = exports.fileExists = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getObjectStreamContents = exports.getObjectStreamBuffers = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.getObjectReadStream = exports.streamS3Upload = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Objects = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
|
|
32
|
+
exports.moveObject = exports.copyObject = exports.multipartCopyObject = exports.createS3Buckets = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.deleteS3Buckets = exports.recursivelyDeleteS3Bucket = exports.listS3ObjectsV2Batch = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.deleteS3Files = exports.fileExists = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getObjectStreamContents = exports.getObjectStreamBuffers = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.getObjectReadStream = exports.streamS3Upload = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Objects = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
|
|
33
33
|
const fs_1 = __importDefault(require("fs"));
|
|
34
34
|
const isBoolean_1 = __importDefault(require("lodash/isBoolean"));
|
|
35
35
|
const path_1 = __importDefault(require("path"));
|
|
@@ -161,9 +161,9 @@ exports.headObject = headObject;
|
|
|
161
161
|
/**
|
|
162
162
|
* Test if an object exists in S3
|
|
163
163
|
*
|
|
164
|
-
* @param
|
|
164
|
+
* @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property
|
|
165
165
|
* @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
|
|
166
|
-
*
|
|
166
|
+
* if the object exists
|
|
167
167
|
*/
|
|
168
168
|
const s3ObjectExists = (params) => (0, exports.headObject)(params.Bucket, params.Key)
|
|
169
169
|
.then(() => true)
|
|
@@ -175,13 +175,8 @@ const s3ObjectExists = (params) => (0, exports.headObject)(params.Bucket, params
|
|
|
175
175
|
exports.s3ObjectExists = s3ObjectExists;
|
|
176
176
|
/**
|
|
177
177
|
* Wait for an object to exist in S3
|
|
178
|
-
*
|
|
179
|
-
* @param
|
|
180
|
-
* @param {string} params.bucket
|
|
181
|
-
* @param {string} params.key
|
|
182
|
-
* @param {number} [params.interval=1000] - interval before retries, in ms
|
|
183
|
-
* @param {number} [params.timeout=30000] - timeout, in ms
|
|
184
|
-
* @returns {Promise<undefined>}
|
|
178
|
+
* @param params.interval=1000 - interval before retries, in ms
|
|
179
|
+
* @param params.timeout=30000 - timeout, in ms
|
|
185
180
|
*/
|
|
186
181
|
const waitForObjectToExist = async (params) => {
|
|
187
182
|
const { bucket, key, interval = 1000, timeout = 30 * 1000, } = params;
|
|
@@ -202,19 +197,13 @@ exports.waitForObjectToExist = waitForObjectToExist;
|
|
|
202
197
|
/**
|
|
203
198
|
* Put an object on S3
|
|
204
199
|
*
|
|
205
|
-
* @param
|
|
206
|
-
* promise of the object being put
|
|
207
|
-
* @returns {Promise}
|
|
200
|
+
* @param params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
|
|
208
201
|
**/
|
|
209
202
|
const s3PutObject = (params) => (0, services_1.s3)().putObject(params);
|
|
210
203
|
exports.s3PutObject = s3PutObject;
|
|
211
204
|
/**
|
|
212
205
|
* Upload a file to S3
|
|
213
206
|
*
|
|
214
|
-
* @param {string} bucket - the destination S3 bucket
|
|
215
|
-
* @param {string} key - the destination S3 key
|
|
216
|
-
* @param {filename} filename - the local file to be uploaded
|
|
217
|
-
* @returns {Promise}
|
|
218
207
|
*/
|
|
219
208
|
const putFile = (bucket, key, filename) => (0, exports.s3PutObject)({
|
|
220
209
|
Bucket: bucket,
|
|
@@ -224,9 +213,6 @@ const putFile = (bucket, key, filename) => (0, exports.s3PutObject)({
|
|
|
224
213
|
exports.putFile = putFile;
|
|
225
214
|
/**
|
|
226
215
|
* Copy an object from one location on S3 to another
|
|
227
|
-
*
|
|
228
|
-
* @param {Object} params - same params as https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
|
|
229
|
-
* @returns {Promise} promise of the object being copied
|
|
230
216
|
**/
|
|
231
217
|
const s3CopyObject = (params) => (0, services_1.s3)().copyObject({
|
|
232
218
|
TaggingDirective: 'COPY',
|
|
@@ -237,9 +223,6 @@ exports.s3CopyObject = s3CopyObject;
|
|
|
237
223
|
* Upload data to S3
|
|
238
224
|
*
|
|
239
225
|
* see https://github.com/aws/aws-sdk-js-v3/tree/main/lib/lib-storage
|
|
240
|
-
*
|
|
241
|
-
* @param {UploadOptions} params
|
|
242
|
-
* @returns {Promise}
|
|
243
226
|
*/
|
|
244
227
|
const promiseS3Upload = async (params) => {
|
|
245
228
|
const parallelUploads = new lib_storage_1.Upload({
|
|
@@ -255,10 +238,6 @@ const promiseS3Upload = async (params) => {
|
|
|
255
238
|
exports.promiseS3Upload = promiseS3Upload;
|
|
256
239
|
/**
|
|
257
240
|
* Upload data to S3 using a stream
|
|
258
|
-
*
|
|
259
|
-
* @param {Readable} uploadStream - Stream of data to upload
|
|
260
|
-
* @param {Object} uploadParams
|
|
261
|
-
* @returns {Promise}
|
|
262
241
|
*/
|
|
263
242
|
const streamS3Upload = (uploadStream, uploadParams) => {
|
|
264
243
|
const parallelUploads3 = new lib_storage_1.Upload({
|
|
@@ -277,12 +256,6 @@ const streamS3Upload = (uploadStream, uploadParams) => {
|
|
|
277
256
|
exports.streamS3Upload = streamS3Upload;
|
|
278
257
|
/**
|
|
279
258
|
* Get a readable stream for an S3 object
|
|
280
|
-
*
|
|
281
|
-
* @param {Object} params
|
|
282
|
-
* @param {S3} params.s3 - an S3 instance
|
|
283
|
-
* @param {string} params.bucket - the bucket of the requested object
|
|
284
|
-
* @param {string} params.key - the key of the requested object
|
|
285
|
-
* @returns {Promise<Readable>}
|
|
286
259
|
*/
|
|
287
260
|
const getObjectReadStream = async (params) => {
|
|
288
261
|
// eslint-disable-next-line no-shadow
|
|
@@ -299,10 +272,6 @@ const getObjectReadStream = async (params) => {
|
|
|
299
272
|
exports.getObjectReadStream = getObjectReadStream;
|
|
300
273
|
/**
|
|
301
274
|
* Downloads the given s3Obj to the given filename in a streaming manner
|
|
302
|
-
*
|
|
303
|
-
* @param {Object} s3Obj - The parameters to send to S3 getObject call
|
|
304
|
-
* @param {string} filepath - The filepath of the file that is downloaded
|
|
305
|
-
* @returns {Promise<string>} returns filename if successful
|
|
306
275
|
*/
|
|
307
276
|
const downloadS3File = async (s3Obj, filepath) => {
|
|
308
277
|
if (!s3Obj.Bucket || !s3Obj.Key) {
|
|
@@ -324,12 +293,6 @@ const downloadS3File = async (s3Obj, filepath) => {
|
|
|
324
293
|
exports.downloadS3File = downloadS3File;
|
|
325
294
|
/**
|
|
326
295
|
* Get the size of an S3 object
|
|
327
|
-
*
|
|
328
|
-
* @param {Object} params
|
|
329
|
-
* @param {string} params.bucket
|
|
330
|
-
* @param {string} params.key
|
|
331
|
-
* @param {S3} params.s3 - an S3 client instance
|
|
332
|
-
* @returns {Promise<number|undefined>} object size, in bytes
|
|
333
296
|
*/
|
|
334
297
|
const getObjectSize = async (params) => {
|
|
335
298
|
// eslint-disable-next-line no-shadow
|
|
@@ -343,10 +306,6 @@ const getObjectSize = async (params) => {
|
|
|
343
306
|
exports.getObjectSize = getObjectSize;
|
|
344
307
|
/**
|
|
345
308
|
* Get object Tagging from S3
|
|
346
|
-
*
|
|
347
|
-
* @param {string} bucket - name of bucket
|
|
348
|
-
* @param {string} key - key for object (filepath + filename)
|
|
349
|
-
* @returns {Promise<GetObjectTaggingOutput>} the promised response from `S3.getObjectTagging`
|
|
350
309
|
**/
|
|
351
310
|
const s3GetObjectTagging = (bucket, key) => (0, services_1.s3)().getObjectTagging({ Bucket: bucket, Key: key });
|
|
352
311
|
exports.s3GetObjectTagging = s3GetObjectTagging;
|
|
@@ -366,11 +325,6 @@ const getObjectTaggingString = async (bucket, key) => {
|
|
|
366
325
|
/**
|
|
367
326
|
* Puts object Tagging in S3
|
|
368
327
|
* https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObjectTagging-property
|
|
369
|
-
*
|
|
370
|
-
* @param {string} Bucket - name of bucket
|
|
371
|
-
* @param {string} Key - key for object (filepath + filename)
|
|
372
|
-
* @param {Object} ObjectTagging - tagging object
|
|
373
|
-
* @returns {Promise} returns response from `S3.getObjectTagging` as a promise
|
|
374
328
|
**/
|
|
375
329
|
const s3PutObjectTagging = (Bucket, Key, ObjectTagging) => (0, services_1.s3)().putObjectTagging({
|
|
376
330
|
Bucket,
|
|
@@ -380,26 +334,12 @@ const s3PutObjectTagging = (Bucket, Key, ObjectTagging) => (0, services_1.s3)().
|
|
|
380
334
|
exports.s3PutObjectTagging = s3PutObjectTagging;
|
|
381
335
|
/**
|
|
382
336
|
* Gets an object from S3.
|
|
383
|
-
*
|
|
384
|
-
* @example
|
|
385
|
-
* const obj = await getObject(s3(), { Bucket: 'b', Key: 'k' })
|
|
386
|
-
*
|
|
387
|
-
* @param {S3} s3Client - an `S3` instance
|
|
388
|
-
* @param {GetObjectCommandInput} params - parameters object to pass through
|
|
389
|
-
* to `S3.getObject()`
|
|
390
|
-
* @returns {Promise<GetObjectOutput>} response from `S3.getObject()`
|
|
391
|
-
* as a Promise
|
|
392
337
|
*/
|
|
393
338
|
const getObject = (s3Client, params) => s3Client.getObject(params);
|
|
394
339
|
exports.getObject = getObject;
|
|
395
340
|
/**
|
|
396
341
|
* Get an object from S3, waiting for it to exist and, if specified, have the
|
|
397
342
|
* correct ETag.
|
|
398
|
-
*
|
|
399
|
-
* @param {S3} s3Client
|
|
400
|
-
* @param {GetObjectCommandInput} params
|
|
401
|
-
* @param {pRetry.Options} [retryOptions={}]
|
|
402
|
-
* @returns {Promise<GetObjectOutput>}
|
|
403
343
|
*/
|
|
404
344
|
const waitForObject = (s3Client, params, retryOptions = {}) => (0, p_retry_1.default)(async () => {
|
|
405
345
|
try {
|
|
@@ -419,14 +359,6 @@ const waitForObject = (s3Client, params, retryOptions = {}) => (0, p_retry_1.def
|
|
|
419
359
|
exports.waitForObject = waitForObject;
|
|
420
360
|
/**
|
|
421
361
|
* Gets an object from S3.
|
|
422
|
-
*
|
|
423
|
-
* @param {string} Bucket - name of bucket
|
|
424
|
-
* @param {string} Key - key for object (filepath + filename)
|
|
425
|
-
* @param {Object} retryOptions - options to control retry behavior when an
|
|
426
|
-
* object does not exist. See https://github.com/tim-kos/node-retry#retryoperationoptions
|
|
427
|
-
* By default, retries will not be performed
|
|
428
|
-
* @returns {Promise} returns response from `S3.getObject` as a promise
|
|
429
|
-
*
|
|
430
362
|
* @deprecated
|
|
431
363
|
*/
|
|
432
364
|
exports.getS3Object = (0, util_1.deprecate)((Bucket, Key, retryOptions = { retries: 0 }) => (0, exports.waitForObject)((0, services_1.s3)(), { Bucket, Key }, {
|
|
@@ -450,9 +382,6 @@ const getObjectStreamBuffers = (objectReadStream) => new Promise((resolve, rejec
|
|
|
450
382
|
exports.getObjectStreamBuffers = getObjectStreamBuffers;
|
|
451
383
|
/**
|
|
452
384
|
* Transform streaming response from S3 object to text content
|
|
453
|
-
*
|
|
454
|
-
* @param {Readable} objectReadStream - Readable stream of S3 object
|
|
455
|
-
* @returns {Promise<string>} the contents of the S3 object
|
|
456
385
|
*/
|
|
457
386
|
const getObjectStreamContents = async (objectReadStream) => {
|
|
458
387
|
const buffers = await (0, exports.getObjectStreamBuffers)(objectReadStream);
|
|
@@ -461,19 +390,12 @@ const getObjectStreamContents = async (objectReadStream) => {
|
|
|
461
390
|
exports.getObjectStreamContents = getObjectStreamContents;
|
|
462
391
|
/**
|
|
463
392
|
* Fetch the contents of an S3 object
|
|
464
|
-
*
|
|
465
|
-
* @param {string} bucket - the S3 object's bucket
|
|
466
|
-
* @param {string} key - the S3 object's key
|
|
467
|
-
* @returns {Promise<string>} the contents of the S3 object
|
|
468
393
|
*/
|
|
469
394
|
const getTextObject = (bucket, key) => (0, exports.getObjectReadStream)({ s3: (0, services_1.s3)(), bucket, key })
|
|
470
395
|
.then((objectReadStream) => (0, exports.getObjectStreamContents)(objectReadStream));
|
|
471
396
|
exports.getTextObject = getTextObject;
|
|
472
397
|
/**
|
|
473
398
|
* Fetch JSON stored in an S3 object
|
|
474
|
-
* @param {string} bucket - the S3 object's bucket
|
|
475
|
-
* @param {string} key - the S3 object's key
|
|
476
|
-
* @returns {Promise<*>} the contents of the S3 object, parsed as JSON
|
|
477
399
|
*/
|
|
478
400
|
const getJsonS3Object = (bucket, key) => (0, exports.getTextObject)(bucket, key)
|
|
479
401
|
.then((text) => {
|
|
@@ -490,10 +412,6 @@ const putJsonS3Object = (bucket, key, data) => (0, exports.s3PutObject)({
|
|
|
490
412
|
exports.putJsonS3Object = putJsonS3Object;
|
|
491
413
|
/**
|
|
492
414
|
* Check if a file exists in an S3 object
|
|
493
|
-
*
|
|
494
|
-
* @param {string} bucket - name of the S3 bucket
|
|
495
|
-
* @param {string} key - key of the file in the S3 bucket
|
|
496
|
-
* @returns {Promise} returns the response from `S3.headObject` as a promise
|
|
497
415
|
**/
|
|
498
416
|
const fileExists = async (bucket, key) => {
|
|
499
417
|
try {
|
|
@@ -511,9 +429,7 @@ exports.fileExists = fileExists;
|
|
|
511
429
|
/**
|
|
512
430
|
* Delete files from S3
|
|
513
431
|
*
|
|
514
|
-
* @param
|
|
515
|
-
* @returns {Promise} A promise that resolves to an Array of the data returned
|
|
516
|
-
* from the deletion operations
|
|
432
|
+
* @param s3Objs - An array of objects containing keys 'Bucket' and 'Key'
|
|
517
433
|
*/
|
|
518
434
|
const deleteS3Files = async (s3Objs) => await (0, p_map_1.default)(s3Objs, (s3Obj) => (0, services_1.s3)().deleteObject(s3Obj), { concurrency: S3_RATE_LIMIT });
|
|
519
435
|
exports.deleteS3Files = deleteS3Files;
|
|
@@ -878,20 +794,13 @@ const multipartCopyObject = async (params) => {
|
|
|
878
794
|
};
|
|
879
795
|
exports.multipartCopyObject = multipartCopyObject;
|
|
880
796
|
/**
|
|
881
|
-
*
|
|
797
|
+
* Copy an S3 object to another location in S3
|
|
882
798
|
*
|
|
883
|
-
* @param
|
|
884
|
-
* @param
|
|
885
|
-
* @param
|
|
886
|
-
* @param {string} params.destinationBucket
|
|
887
|
-
* @param {string} params.destinationKey
|
|
888
|
-
* @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
|
|
889
|
-
* @param {boolean} [params.copyTags=false]
|
|
890
|
-
* @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
|
|
891
|
-
* @returns {Promise<undefined>}
|
|
799
|
+
* @param params.ACL - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
|
|
800
|
+
* @param params.copyTags=false
|
|
801
|
+
* @param params.chunkSize - chunk size of the S3 multipart uploads
|
|
892
802
|
*/
|
|
893
|
-
const
|
|
894
|
-
const { sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags, chunkSize, } = params;
|
|
803
|
+
const copyObject = async ({ sourceBucket, sourceKey, destinationBucket, destinationKey, ACL, copyTags, chunkSize, }) => {
|
|
895
804
|
const sourceObject = await (0, exports.headObject)(sourceBucket, sourceKey);
|
|
896
805
|
if (sourceObject.ContentLength === 0) {
|
|
897
806
|
// 0 byte files cannot be copied with multipart upload,
|
|
@@ -905,7 +814,7 @@ const moveObject = async (params) => {
|
|
|
905
814
|
// This error should never actually be reached in practice. It's a
|
|
906
815
|
// necessary workaround for bad typings in the AWS SDK.
|
|
907
816
|
// https://github.com/aws/aws-sdk-js/issues/1719
|
|
908
|
-
if (!CopyObjectResult
|
|
817
|
+
if (!CopyObjectResult?.ETag) {
|
|
909
818
|
throw new Error(`ETag could not be determined for copy of ${(0, exports.buildS3Uri)(sourceBucket, sourceKey)} to ${s3uri}`);
|
|
910
819
|
}
|
|
911
820
|
}
|
|
@@ -921,6 +830,19 @@ const moveObject = async (params) => {
|
|
|
921
830
|
chunkSize: chunkSize,
|
|
922
831
|
});
|
|
923
832
|
}
|
|
833
|
+
};
|
|
834
|
+
exports.copyObject = copyObject;
|
|
835
|
+
/**
|
|
836
|
+
* Move an S3 object to another location in S3
|
|
837
|
+
*
|
|
838
|
+
* @param {string} [params.ACL] - an [S3 Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl)
|
|
839
|
+
* @param {boolean} [params.copyTags=false]
|
|
840
|
+
* @param {number} [params.chunkSize] - chunk size of the S3 multipart uploads
|
|
841
|
+
* @returns {Promise<undefined>}
|
|
842
|
+
*/
|
|
843
|
+
const moveObject = async (params) => {
|
|
844
|
+
const { sourceBucket, sourceKey, } = params;
|
|
845
|
+
await (0, exports.copyObject)(params);
|
|
924
846
|
const deleteS3ObjRes = await (0, exports.deleteS3Object)(sourceBucket, sourceKey);
|
|
925
847
|
return deleteS3ObjRes;
|
|
926
848
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cumulus/aws-client",
|
|
3
|
-
"version": "20.1.
|
|
3
|
+
"version": "20.1.2",
|
|
4
4
|
"description": "Utilities for working with AWS",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"GIBS",
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
"tests/**/*.js"
|
|
42
42
|
],
|
|
43
43
|
"verbose": true,
|
|
44
|
+
"timeout": "80s",
|
|
44
45
|
"failFast": true
|
|
45
46
|
},
|
|
46
47
|
"author": "Cumulus Authors",
|
|
@@ -68,10 +69,10 @@
|
|
|
68
69
|
"@aws-sdk/s3-request-presigner": "^3.621.0",
|
|
69
70
|
"@aws-sdk/signature-v4-crt": "^3.621.0",
|
|
70
71
|
"@aws-sdk/types": "^3.609.0",
|
|
71
|
-
"@cumulus/checksum": "20.1.
|
|
72
|
-
"@cumulus/errors": "20.1.
|
|
73
|
-
"@cumulus/logger": "20.1.
|
|
74
|
-
"@cumulus/types": "20.1.
|
|
72
|
+
"@cumulus/checksum": "20.1.2",
|
|
73
|
+
"@cumulus/errors": "20.1.2",
|
|
74
|
+
"@cumulus/logger": "20.1.2",
|
|
75
|
+
"@cumulus/types": "20.1.2",
|
|
75
76
|
"lodash": "~4.17.21",
|
|
76
77
|
"mem": "^8.0.2",
|
|
77
78
|
"p-map": "^1.2.0",
|
|
@@ -82,8 +83,8 @@
|
|
|
82
83
|
"uuid": "^8.2.0"
|
|
83
84
|
},
|
|
84
85
|
"devDependencies": {
|
|
85
|
-
"@cumulus/test-data": "20.1.
|
|
86
|
+
"@cumulus/test-data": "20.1.2",
|
|
86
87
|
"@types/uuid": "^8.0.0"
|
|
87
88
|
},
|
|
88
|
-
"gitHead": "
|
|
89
|
+
"gitHead": "051ef1428ac21105fd55370d15e26009f4b49f57"
|
|
89
90
|
}
|