@aws-sdk/lib-transfer-manager 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
const { isReadableStream, sdkStreamMixin } = require("@smithy/core/serde");
|
|
2
2
|
const { Readable } = require("node:stream");
|
|
3
3
|
const { S3Client, GetObjectCommand, ListObjectsV2Command, HeadObjectCommand, PutObjectCommand, CreateMultipartUploadCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, UploadPartCommand } = require("@aws-sdk/client-s3");
|
|
4
|
+
const { setFeature } = require("@aws-sdk/core/client");
|
|
4
5
|
const { isThrottlingError, isTransientError } = require("@smithy/core/retry");
|
|
5
6
|
const { ReadStream, lstatSync, existsSync, unlinkSync, createReadStream } = require("node:fs");
|
|
6
7
|
const { stat, mkdir, realpath, open, rename, unlink, opendir } = require("node:fs/promises");
|
|
@@ -488,6 +489,7 @@ class OrderedPartQueue {
|
|
|
488
489
|
}
|
|
489
490
|
}
|
|
490
491
|
|
|
492
|
+
const TRANSFER_MANAGER_DIRECTORY_FEATURE = Symbol("transferManagerDirectoryFeature");
|
|
491
493
|
class S3TransferManagerBase {
|
|
492
494
|
static MIN_PART_SIZE = 5 * 1024 * 1024;
|
|
493
495
|
s3;
|
|
@@ -541,6 +543,21 @@ class S3TransferManagerBase {
|
|
|
541
543
|
};
|
|
542
544
|
this.validateConfig();
|
|
543
545
|
}
|
|
546
|
+
send(command, options) {
|
|
547
|
+
const directoryFeature = options?.[TRANSFER_MANAGER_DIRECTORY_FEATURE];
|
|
548
|
+
command.middlewareStack.add((next, context) => async (args) => {
|
|
549
|
+
setFeature(context, "S3_TRANSFER", "G");
|
|
550
|
+
if (directoryFeature) {
|
|
551
|
+
setFeature(context, directoryFeature, directoryFeature === "S3_TRANSFER_UPLOAD_DIRECTORY" ? "9" : "+");
|
|
552
|
+
}
|
|
553
|
+
return next(args);
|
|
554
|
+
}, {
|
|
555
|
+
name: "s3TransferManagerFeature",
|
|
556
|
+
step: "initialize",
|
|
557
|
+
override: true,
|
|
558
|
+
});
|
|
559
|
+
return this.s3.send(command, (options ?? {}));
|
|
560
|
+
}
|
|
544
561
|
addEventListener(type, callback, options) {
|
|
545
562
|
const eventType = type;
|
|
546
563
|
const listeners = this.eventListeners[eventType];
|
|
@@ -807,7 +824,7 @@ class S3TransferManagerBase {
|
|
|
807
824
|
};
|
|
808
825
|
let initialResponse;
|
|
809
826
|
try {
|
|
810
|
-
initialResponse = await this.
|
|
827
|
+
initialResponse = await this.send(new GetObjectCommand(initialPartRequest), transferOptions);
|
|
811
828
|
}
|
|
812
829
|
catch (error) {
|
|
813
830
|
removeLocalEventListeners();
|
|
@@ -903,7 +920,7 @@ class S3TransferManagerBase {
|
|
|
903
920
|
expectedLength,
|
|
904
921
|
resultToken,
|
|
905
922
|
};
|
|
906
|
-
await this.
|
|
923
|
+
await this.send(new GetObjectCommand(partGetRequest), {
|
|
907
924
|
...transferOptions,
|
|
908
925
|
downloadDataToFile: downloadOptions,
|
|
909
926
|
});
|
|
@@ -917,7 +934,7 @@ class S3TransferManagerBase {
|
|
|
917
934
|
});
|
|
918
935
|
}
|
|
919
936
|
else {
|
|
920
|
-
const partResponse = await this.
|
|
937
|
+
const partResponse = await this.send(new GetObjectCommand(partGetRequest), transferOptions);
|
|
921
938
|
if (partResponse.Body && partResponse.Body instanceof Readable) {
|
|
922
939
|
partResponse.Body.on("error", () => { });
|
|
923
940
|
}
|
|
@@ -1006,7 +1023,7 @@ class S3TransferManagerBase {
|
|
|
1006
1023
|
Range: `bytes=0-${rangeSize - 1}`,
|
|
1007
1024
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1008
1025
|
};
|
|
1009
|
-
const initialResponse = await this.
|
|
1026
|
+
const initialResponse = await this.send(new GetObjectCommand(initialRangeRequest), transferOptions);
|
|
1010
1027
|
if (initialResponse.Body && initialResponse.Body instanceof Readable) {
|
|
1011
1028
|
initialResponse.Body.on("error", () => { });
|
|
1012
1029
|
}
|
|
@@ -1094,7 +1111,7 @@ class S3TransferManagerBase {
|
|
|
1094
1111
|
expectedLength,
|
|
1095
1112
|
resultToken,
|
|
1096
1113
|
};
|
|
1097
|
-
const rangeResponse = await this.
|
|
1114
|
+
const rangeResponse = await this.send(new GetObjectCommand(rangeGetRequest), {
|
|
1098
1115
|
...transferOptions,
|
|
1099
1116
|
downloadDataToFile: downloadOptions,
|
|
1100
1117
|
});
|
|
@@ -1109,7 +1126,7 @@ class S3TransferManagerBase {
|
|
|
1109
1126
|
});
|
|
1110
1127
|
}
|
|
1111
1128
|
else {
|
|
1112
|
-
const rangeResponse = await this.
|
|
1129
|
+
const rangeResponse = await this.send(new GetObjectCommand(rangeGetRequest), transferOptions);
|
|
1113
1130
|
this.validateRangeDownload(`bytes=${start}-${end}`, rangeResponse.ContentRange);
|
|
1114
1131
|
if (rangeResponse.Body && rangeResponse.Body instanceof Readable) {
|
|
1115
1132
|
rangeResponse.Body.on("error", () => { });
|
|
@@ -1284,6 +1301,7 @@ class S3TransferManagerBase {
|
|
|
1284
1301
|
}
|
|
1285
1302
|
await this.uploadInternal(putRequest, {
|
|
1286
1303
|
abortSignal: abortController.signal,
|
|
1304
|
+
[TRANSFER_MANAGER_DIRECTORY_FEATURE]: "S3_TRANSFER_UPLOAD_DIRECTORY",
|
|
1287
1305
|
}, false);
|
|
1288
1306
|
objectsUploaded++;
|
|
1289
1307
|
transferredFiles++;
|
|
@@ -1418,8 +1436,9 @@ class S3TransferManagerBase {
|
|
|
1418
1436
|
Delimiter: undefined,
|
|
1419
1437
|
ContinuationToken: continuationToken,
|
|
1420
1438
|
};
|
|
1421
|
-
const listResponse = await this.
|
|
1439
|
+
const listResponse = await this.send(new ListObjectsV2Command(listRequest), {
|
|
1422
1440
|
abortSignal: abortController.signal,
|
|
1441
|
+
[TRANSFER_MANAGER_DIRECTORY_FEATURE]: "S3_TRANSFER_DOWNLOAD_DIRECTORY",
|
|
1423
1442
|
});
|
|
1424
1443
|
continuationToken = listResponse.IsTruncated ? listResponse.NextContinuationToken : undefined;
|
|
1425
1444
|
for (const object of listResponse.Contents ?? []) {
|
|
@@ -1463,6 +1482,7 @@ class S3TransferManagerBase {
|
|
|
1463
1482
|
destination: localPath,
|
|
1464
1483
|
}, {
|
|
1465
1484
|
abortSignal: abortController.signal,
|
|
1485
|
+
[TRANSFER_MANAGER_DIRECTORY_FEATURE]: "S3_TRANSFER_DOWNLOAD_DIRECTORY",
|
|
1466
1486
|
}, false);
|
|
1467
1487
|
objectsDownloaded++;
|
|
1468
1488
|
transferredFiles++;
|
|
@@ -1555,7 +1575,7 @@ class S3TransferManagerBase {
|
|
|
1555
1575
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1556
1576
|
};
|
|
1557
1577
|
try {
|
|
1558
|
-
const initialPart = await this.
|
|
1578
|
+
const initialPart = await this.send(new GetObjectCommand(initialPartRequest), transferOptions);
|
|
1559
1579
|
await internalEventHandler.afterInitialGetObject();
|
|
1560
1580
|
const partSize = initialPart.ContentLength;
|
|
1561
1581
|
const initialETag = initialPart.ETag ?? undefined;
|
|
@@ -1588,8 +1608,7 @@ class S3TransferManagerBase {
|
|
|
1588
1608
|
partRequests.push(getObjectRequest);
|
|
1589
1609
|
partTasks.push(() => {
|
|
1590
1610
|
this.checkAborted(transferOptions);
|
|
1591
|
-
return this.
|
|
1592
|
-
.send(new GetObjectCommand(getObjectRequest), transferOptions)
|
|
1611
|
+
return this.send(new GetObjectCommand(getObjectRequest), transferOptions)
|
|
1593
1612
|
.then((response) => {
|
|
1594
1613
|
this.validatePartDownload(response.ContentRange, part, partSize ?? 0);
|
|
1595
1614
|
if (response.Body && typeof response.Body.getReader === "function") {
|
|
@@ -1630,7 +1649,7 @@ class S3TransferManagerBase {
|
|
|
1630
1649
|
ChecksumMode: "ENABLED",
|
|
1631
1650
|
}),
|
|
1632
1651
|
};
|
|
1633
|
-
const getObject = await this.
|
|
1652
|
+
const getObject = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1634
1653
|
totalSize = getObject.ContentRange ? Number.parseInt(getObject.ContentRange.split("/")[1]) : 0;
|
|
1635
1654
|
this.dispatchTransferInitiatedEvent(request, totalSize);
|
|
1636
1655
|
if (getObject.Body) {
|
|
@@ -1652,13 +1671,13 @@ class S3TransferManagerBase {
|
|
|
1652
1671
|
async downloadByRange(request, transferOptions, streams, requests, metadata, checksumValidationEnabled) {
|
|
1653
1672
|
let streamConsumedCallback;
|
|
1654
1673
|
this.checkAborted(transferOptions);
|
|
1655
|
-
const headResponse = await this.
|
|
1674
|
+
const headResponse = await this.send(new HeadObjectCommand({ Bucket: request.Bucket, Key: request.Key }), transferOptions);
|
|
1656
1675
|
if (headResponse.ContentLength === 0) {
|
|
1657
1676
|
const getObjectRequest = {
|
|
1658
1677
|
...request,
|
|
1659
1678
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1660
1679
|
};
|
|
1661
|
-
const response = await this.
|
|
1680
|
+
const response = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1662
1681
|
this.dispatchTransferInitiatedEvent(request, 0);
|
|
1663
1682
|
if (response.Body)
|
|
1664
1683
|
streams.push(Promise.resolve(response.Body));
|
|
@@ -1685,7 +1704,7 @@ class S3TransferManagerBase {
|
|
|
1685
1704
|
Range: `bytes=${left}-${right}`,
|
|
1686
1705
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1687
1706
|
};
|
|
1688
|
-
const initialRangeGet = await this.
|
|
1707
|
+
const initialRangeGet = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1689
1708
|
await internalEventHandler.afterInitialGetObject();
|
|
1690
1709
|
this.validateRangeDownload(`bytes=${left}-${right}`, initialRangeGet.ContentRange);
|
|
1691
1710
|
initialETag = initialRangeGet.ETag ?? undefined;
|
|
@@ -1732,8 +1751,7 @@ class S3TransferManagerBase {
|
|
|
1732
1751
|
rangeRequests.push(getObjectRequest);
|
|
1733
1752
|
rangeTasks.push(() => {
|
|
1734
1753
|
this.checkAborted(transferOptions);
|
|
1735
|
-
return this.
|
|
1736
|
-
.send(new GetObjectCommand(getObjectRequest), transferOptions)
|
|
1754
|
+
return this.send(new GetObjectCommand(getObjectRequest), transferOptions)
|
|
1737
1755
|
.then((response) => {
|
|
1738
1756
|
this.validateRangeDownload(range, response.ContentRange);
|
|
1739
1757
|
if (response.Body && typeof response.Body.getReader === "function") {
|
|
@@ -1776,7 +1794,7 @@ class S3TransferManagerBase {
|
|
|
1776
1794
|
PartNumber: 1,
|
|
1777
1795
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1778
1796
|
};
|
|
1779
|
-
const initialPart = await this.
|
|
1797
|
+
const initialPart = await this.send(new GetObjectCommand(initialPartRequest), transferOptions);
|
|
1780
1798
|
await internalEventHandler.afterInitialGetObject();
|
|
1781
1799
|
const partSize = initialPart.ContentLength;
|
|
1782
1800
|
const initialETag = initialPart.ETag ?? undefined;
|
|
@@ -1856,8 +1874,7 @@ class S3TransferManagerBase {
|
|
|
1856
1874
|
checksumAlgorithm: checksumValidationEnabled ? "CRC32" : undefined,
|
|
1857
1875
|
resultToken,
|
|
1858
1876
|
};
|
|
1859
|
-
this.
|
|
1860
|
-
.send(new GetObjectCommand(partGetRequest), {
|
|
1877
|
+
this.send(new GetObjectCommand(partGetRequest), {
|
|
1861
1878
|
...transferOptions,
|
|
1862
1879
|
downloadStream: transferOptions_,
|
|
1863
1880
|
})
|
|
@@ -1923,14 +1940,14 @@ class S3TransferManagerBase {
|
|
|
1923
1940
|
}
|
|
1924
1941
|
async downloadByRangeWithWorkers(request, transferOptions, streams, requests, metadata, checksumValidationEnabled) {
|
|
1925
1942
|
this.checkAborted(transferOptions);
|
|
1926
|
-
const headResponse = await this.
|
|
1943
|
+
const headResponse = await this.send(new HeadObjectCommand({ Bucket: request.Bucket, Key: request.Key }), transferOptions);
|
|
1927
1944
|
await internalEventHandler.afterInitialGetObject();
|
|
1928
1945
|
if (headResponse.ContentLength === 0) {
|
|
1929
1946
|
const getObjectRequest = {
|
|
1930
1947
|
...request,
|
|
1931
1948
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1932
1949
|
};
|
|
1933
|
-
const response = await this.
|
|
1950
|
+
const response = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1934
1951
|
this.dispatchTransferInitiatedEvent(request, 0);
|
|
1935
1952
|
if (response.Body)
|
|
1936
1953
|
streams.push(Promise.resolve(response.Body));
|
|
@@ -2008,8 +2025,7 @@ class S3TransferManagerBase {
|
|
|
2008
2025
|
checksumAlgorithm: checksumValidationEnabled ? "CRC32" : undefined,
|
|
2009
2026
|
resultToken,
|
|
2010
2027
|
};
|
|
2011
|
-
this.
|
|
2012
|
-
.send(new GetObjectCommand(rangeGetRequest), {
|
|
2028
|
+
this.send(new GetObjectCommand(rangeGetRequest), {
|
|
2013
2029
|
...transferOptions,
|
|
2014
2030
|
downloadStream: transferOpts,
|
|
2015
2031
|
})
|
|
@@ -2280,7 +2296,7 @@ class S3TransferManagerBase {
|
|
|
2280
2296
|
async uploadSinglePart(request, contentLength, transferOptions, emitEvents) {
|
|
2281
2297
|
const putObjectRequest = { ...request };
|
|
2282
2298
|
this.checkAborted(transferOptions);
|
|
2283
|
-
const response = await this.
|
|
2299
|
+
const response = await this.send(new PutObjectCommand(putObjectRequest), transferOptions);
|
|
2284
2300
|
if (emitEvents) {
|
|
2285
2301
|
this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
|
|
2286
2302
|
request,
|
|
@@ -2370,7 +2386,7 @@ class S3TransferManagerBase {
|
|
|
2370
2386
|
if (!createMpuRequest.ChecksumAlgorithm && this.requestChecksumCalculation === "WHEN_SUPPORTED") {
|
|
2371
2387
|
createMpuRequest.ChecksumAlgorithm = request.ChecksumAlgorithm ?? "CRC32";
|
|
2372
2388
|
}
|
|
2373
|
-
const createMpuResponse = await this.
|
|
2389
|
+
const createMpuResponse = await this.send(new CreateMultipartUploadCommand(createMpuRequest), transferOptions);
|
|
2374
2390
|
const uploadId = createMpuResponse.UploadId;
|
|
2375
2391
|
const checksumAlgorithm = createMpuRequest.ChecksumAlgorithm;
|
|
2376
2392
|
const checksumHeader = checksumAlgorithm ? `x-amz-checksum-${checksumAlgorithm.toLowerCase()}` : undefined;
|
|
@@ -2395,7 +2411,7 @@ class S3TransferManagerBase {
|
|
|
2395
2411
|
}),
|
|
2396
2412
|
};
|
|
2397
2413
|
try {
|
|
2398
|
-
return await this.
|
|
2414
|
+
return await this.send(new UploadPartCommand(partRequest), {
|
|
2399
2415
|
...transferOptions,
|
|
2400
2416
|
dataSource,
|
|
2401
2417
|
});
|
|
@@ -2484,7 +2500,7 @@ class S3TransferManagerBase {
|
|
|
2484
2500
|
completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
|
|
2485
2501
|
}
|
|
2486
2502
|
try {
|
|
2487
|
-
return await this.
|
|
2503
|
+
return await this.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
|
|
2488
2504
|
}
|
|
2489
2505
|
catch (completeMpuError) {
|
|
2490
2506
|
this.logger.warn(`CompleteMultipartUpload failed for upload ID ${uploadId}: ${completeMpuError.message}`);
|
|
@@ -2498,7 +2514,7 @@ class S3TransferManagerBase {
|
|
|
2498
2514
|
UploadId: uploadId,
|
|
2499
2515
|
};
|
|
2500
2516
|
try {
|
|
2501
|
-
await this.
|
|
2517
|
+
await this.send(new AbortMultipartUploadCommand(abortRequest), transferOptions);
|
|
2502
2518
|
}
|
|
2503
2519
|
catch (abortError) {
|
|
2504
2520
|
this.logger.warn(`Failed to abort multipart upload ${uploadId}:`, abortError);
|
|
@@ -2522,7 +2538,7 @@ class S3TransferManagerBase {
|
|
|
2522
2538
|
if (!createMpuRequest.ChecksumAlgorithm && this.requestChecksumCalculation === "WHEN_SUPPORTED") {
|
|
2523
2539
|
createMpuRequest.ChecksumAlgorithm = request.ChecksumAlgorithm ?? "CRC32";
|
|
2524
2540
|
}
|
|
2525
|
-
const createMpuResponse = await this.
|
|
2541
|
+
const createMpuResponse = await this.send(new CreateMultipartUploadCommand(createMpuRequest), transferOptions);
|
|
2526
2542
|
const uploadId = createMpuResponse.UploadId;
|
|
2527
2543
|
const abortController = new AbortController();
|
|
2528
2544
|
const uploadAbortSignal = abortController.signal;
|
|
@@ -2547,7 +2563,7 @@ class S3TransferManagerBase {
|
|
|
2547
2563
|
ChecksumAlgorithm: request.ChecksumAlgorithm,
|
|
2548
2564
|
}),
|
|
2549
2565
|
};
|
|
2550
|
-
const partResponse = await this.
|
|
2566
|
+
const partResponse = await this.send(new UploadPartCommand(partRequest), transferOptions);
|
|
2551
2567
|
const checksumAlgorithm = partRequest.ChecksumAlgorithm;
|
|
2552
2568
|
if (checksumAlgorithm &&
|
|
2553
2569
|
!partResponse.ChecksumCRC32 &&
|
|
@@ -2616,7 +2632,7 @@ class S3TransferManagerBase {
|
|
|
2616
2632
|
completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
|
|
2617
2633
|
}
|
|
2618
2634
|
try {
|
|
2619
|
-
return await this.
|
|
2635
|
+
return await this.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
|
|
2620
2636
|
}
|
|
2621
2637
|
catch (completeMpuError) {
|
|
2622
2638
|
this.logger.warn(`CompleteMultipartUpload failed for upload ID ${uploadId}: ${completeMpuError.message}`);
|
|
@@ -2630,7 +2646,7 @@ class S3TransferManagerBase {
|
|
|
2630
2646
|
UploadId: uploadId,
|
|
2631
2647
|
};
|
|
2632
2648
|
try {
|
|
2633
|
-
await this.
|
|
2649
|
+
await this.send(new AbortMultipartUploadCommand(abortRequest), transferOptions);
|
|
2634
2650
|
}
|
|
2635
2651
|
catch (abortError) {
|
|
2636
2652
|
this.logger.warn(`Failed to abort multipart upload ${uploadId}:`, abortError);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client, UploadPartCommand, } from "@aws-sdk/client-s3";
|
|
2
|
+
import { setFeature } from "@aws-sdk/core/client";
|
|
2
3
|
import { isThrottlingError, isTransientError } from "@smithy/core/retry";
|
|
3
4
|
import { createReadStream, existsSync } from "node:fs";
|
|
4
5
|
import { open, opendir, realpath, stat } from "node:fs/promises";
|
|
@@ -9,6 +10,7 @@ import { createDestinationDirectory, createLocalParentDirectories, deriveLocalPa
|
|
|
9
10
|
import { FileManager } from "./file-manager";
|
|
10
11
|
import { LogLevel } from "./log-level";
|
|
11
12
|
import { OrderedPartQueue } from "./ordered-part-queue";
|
|
13
|
+
const TRANSFER_MANAGER_DIRECTORY_FEATURE = Symbol("transferManagerDirectoryFeature");
|
|
12
14
|
class S3TransferManagerBase {
|
|
13
15
|
static MIN_PART_SIZE = 5 * 1024 * 1024;
|
|
14
16
|
s3;
|
|
@@ -62,6 +64,21 @@ class S3TransferManagerBase {
|
|
|
62
64
|
};
|
|
63
65
|
this.validateConfig();
|
|
64
66
|
}
|
|
67
|
+
send(command, options) {
|
|
68
|
+
const directoryFeature = options?.[TRANSFER_MANAGER_DIRECTORY_FEATURE];
|
|
69
|
+
command.middlewareStack.add((next, context) => async (args) => {
|
|
70
|
+
setFeature(context, "S3_TRANSFER", "G");
|
|
71
|
+
if (directoryFeature) {
|
|
72
|
+
setFeature(context, directoryFeature, directoryFeature === "S3_TRANSFER_UPLOAD_DIRECTORY" ? "9" : "+");
|
|
73
|
+
}
|
|
74
|
+
return next(args);
|
|
75
|
+
}, {
|
|
76
|
+
name: "s3TransferManagerFeature",
|
|
77
|
+
step: "initialize",
|
|
78
|
+
override: true,
|
|
79
|
+
});
|
|
80
|
+
return this.s3.send(command, (options ?? {}));
|
|
81
|
+
}
|
|
65
82
|
addEventListener(type, callback, options) {
|
|
66
83
|
const eventType = type;
|
|
67
84
|
const listeners = this.eventListeners[eventType];
|
|
@@ -328,7 +345,7 @@ class S3TransferManagerBase {
|
|
|
328
345
|
};
|
|
329
346
|
let initialResponse;
|
|
330
347
|
try {
|
|
331
|
-
initialResponse = await this.
|
|
348
|
+
initialResponse = await this.send(new GetObjectCommand(initialPartRequest), transferOptions);
|
|
332
349
|
}
|
|
333
350
|
catch (error) {
|
|
334
351
|
removeLocalEventListeners();
|
|
@@ -424,7 +441,7 @@ class S3TransferManagerBase {
|
|
|
424
441
|
expectedLength,
|
|
425
442
|
resultToken,
|
|
426
443
|
};
|
|
427
|
-
await this.
|
|
444
|
+
await this.send(new GetObjectCommand(partGetRequest), {
|
|
428
445
|
...transferOptions,
|
|
429
446
|
downloadDataToFile: downloadOptions,
|
|
430
447
|
});
|
|
@@ -438,7 +455,7 @@ class S3TransferManagerBase {
|
|
|
438
455
|
});
|
|
439
456
|
}
|
|
440
457
|
else {
|
|
441
|
-
const partResponse = await this.
|
|
458
|
+
const partResponse = await this.send(new GetObjectCommand(partGetRequest), transferOptions);
|
|
442
459
|
if (partResponse.Body && partResponse.Body instanceof Readable) {
|
|
443
460
|
partResponse.Body.on("error", () => { });
|
|
444
461
|
}
|
|
@@ -527,7 +544,7 @@ class S3TransferManagerBase {
|
|
|
527
544
|
Range: `bytes=0-${rangeSize - 1}`,
|
|
528
545
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
529
546
|
};
|
|
530
|
-
const initialResponse = await this.
|
|
547
|
+
const initialResponse = await this.send(new GetObjectCommand(initialRangeRequest), transferOptions);
|
|
531
548
|
if (initialResponse.Body && initialResponse.Body instanceof Readable) {
|
|
532
549
|
initialResponse.Body.on("error", () => { });
|
|
533
550
|
}
|
|
@@ -615,7 +632,7 @@ class S3TransferManagerBase {
|
|
|
615
632
|
expectedLength,
|
|
616
633
|
resultToken,
|
|
617
634
|
};
|
|
618
|
-
const rangeResponse = await this.
|
|
635
|
+
const rangeResponse = await this.send(new GetObjectCommand(rangeGetRequest), {
|
|
619
636
|
...transferOptions,
|
|
620
637
|
downloadDataToFile: downloadOptions,
|
|
621
638
|
});
|
|
@@ -630,7 +647,7 @@ class S3TransferManagerBase {
|
|
|
630
647
|
});
|
|
631
648
|
}
|
|
632
649
|
else {
|
|
633
|
-
const rangeResponse = await this.
|
|
650
|
+
const rangeResponse = await this.send(new GetObjectCommand(rangeGetRequest), transferOptions);
|
|
634
651
|
this.validateRangeDownload(`bytes=${start}-${end}`, rangeResponse.ContentRange);
|
|
635
652
|
if (rangeResponse.Body && rangeResponse.Body instanceof Readable) {
|
|
636
653
|
rangeResponse.Body.on("error", () => { });
|
|
@@ -805,6 +822,7 @@ class S3TransferManagerBase {
|
|
|
805
822
|
}
|
|
806
823
|
await this.uploadInternal(putRequest, {
|
|
807
824
|
abortSignal: abortController.signal,
|
|
825
|
+
[TRANSFER_MANAGER_DIRECTORY_FEATURE]: "S3_TRANSFER_UPLOAD_DIRECTORY",
|
|
808
826
|
}, false);
|
|
809
827
|
objectsUploaded++;
|
|
810
828
|
transferredFiles++;
|
|
@@ -939,8 +957,9 @@ class S3TransferManagerBase {
|
|
|
939
957
|
Delimiter: undefined,
|
|
940
958
|
ContinuationToken: continuationToken,
|
|
941
959
|
};
|
|
942
|
-
const listResponse = await this.
|
|
960
|
+
const listResponse = await this.send(new ListObjectsV2Command(listRequest), {
|
|
943
961
|
abortSignal: abortController.signal,
|
|
962
|
+
[TRANSFER_MANAGER_DIRECTORY_FEATURE]: "S3_TRANSFER_DOWNLOAD_DIRECTORY",
|
|
944
963
|
});
|
|
945
964
|
continuationToken = listResponse.IsTruncated ? listResponse.NextContinuationToken : undefined;
|
|
946
965
|
for (const object of listResponse.Contents ?? []) {
|
|
@@ -984,6 +1003,7 @@ class S3TransferManagerBase {
|
|
|
984
1003
|
destination: localPath,
|
|
985
1004
|
}, {
|
|
986
1005
|
abortSignal: abortController.signal,
|
|
1006
|
+
[TRANSFER_MANAGER_DIRECTORY_FEATURE]: "S3_TRANSFER_DOWNLOAD_DIRECTORY",
|
|
987
1007
|
}, false);
|
|
988
1008
|
objectsDownloaded++;
|
|
989
1009
|
transferredFiles++;
|
|
@@ -1076,7 +1096,7 @@ class S3TransferManagerBase {
|
|
|
1076
1096
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1077
1097
|
};
|
|
1078
1098
|
try {
|
|
1079
|
-
const initialPart = await this.
|
|
1099
|
+
const initialPart = await this.send(new GetObjectCommand(initialPartRequest), transferOptions);
|
|
1080
1100
|
await internalEventHandler.afterInitialGetObject();
|
|
1081
1101
|
const partSize = initialPart.ContentLength;
|
|
1082
1102
|
const initialETag = initialPart.ETag ?? undefined;
|
|
@@ -1109,8 +1129,7 @@ class S3TransferManagerBase {
|
|
|
1109
1129
|
partRequests.push(getObjectRequest);
|
|
1110
1130
|
partTasks.push(() => {
|
|
1111
1131
|
this.checkAborted(transferOptions);
|
|
1112
|
-
return this.
|
|
1113
|
-
.send(new GetObjectCommand(getObjectRequest), transferOptions)
|
|
1132
|
+
return this.send(new GetObjectCommand(getObjectRequest), transferOptions)
|
|
1114
1133
|
.then((response) => {
|
|
1115
1134
|
this.validatePartDownload(response.ContentRange, part, partSize ?? 0);
|
|
1116
1135
|
if (response.Body && typeof response.Body.getReader === "function") {
|
|
@@ -1151,7 +1170,7 @@ class S3TransferManagerBase {
|
|
|
1151
1170
|
ChecksumMode: "ENABLED",
|
|
1152
1171
|
}),
|
|
1153
1172
|
};
|
|
1154
|
-
const getObject = await this.
|
|
1173
|
+
const getObject = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1155
1174
|
totalSize = getObject.ContentRange ? Number.parseInt(getObject.ContentRange.split("/")[1]) : 0;
|
|
1156
1175
|
this.dispatchTransferInitiatedEvent(request, totalSize);
|
|
1157
1176
|
if (getObject.Body) {
|
|
@@ -1173,13 +1192,13 @@ class S3TransferManagerBase {
|
|
|
1173
1192
|
async downloadByRange(request, transferOptions, streams, requests, metadata, checksumValidationEnabled) {
|
|
1174
1193
|
let streamConsumedCallback;
|
|
1175
1194
|
this.checkAborted(transferOptions);
|
|
1176
|
-
const headResponse = await this.
|
|
1195
|
+
const headResponse = await this.send(new HeadObjectCommand({ Bucket: request.Bucket, Key: request.Key }), transferOptions);
|
|
1177
1196
|
if (headResponse.ContentLength === 0) {
|
|
1178
1197
|
const getObjectRequest = {
|
|
1179
1198
|
...request,
|
|
1180
1199
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1181
1200
|
};
|
|
1182
|
-
const response = await this.
|
|
1201
|
+
const response = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1183
1202
|
this.dispatchTransferInitiatedEvent(request, 0);
|
|
1184
1203
|
if (response.Body)
|
|
1185
1204
|
streams.push(Promise.resolve(response.Body));
|
|
@@ -1206,7 +1225,7 @@ class S3TransferManagerBase {
|
|
|
1206
1225
|
Range: `bytes=${left}-${right}`,
|
|
1207
1226
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1208
1227
|
};
|
|
1209
|
-
const initialRangeGet = await this.
|
|
1228
|
+
const initialRangeGet = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1210
1229
|
await internalEventHandler.afterInitialGetObject();
|
|
1211
1230
|
this.validateRangeDownload(`bytes=${left}-${right}`, initialRangeGet.ContentRange);
|
|
1212
1231
|
initialETag = initialRangeGet.ETag ?? undefined;
|
|
@@ -1253,8 +1272,7 @@ class S3TransferManagerBase {
|
|
|
1253
1272
|
rangeRequests.push(getObjectRequest);
|
|
1254
1273
|
rangeTasks.push(() => {
|
|
1255
1274
|
this.checkAborted(transferOptions);
|
|
1256
|
-
return this.
|
|
1257
|
-
.send(new GetObjectCommand(getObjectRequest), transferOptions)
|
|
1275
|
+
return this.send(new GetObjectCommand(getObjectRequest), transferOptions)
|
|
1258
1276
|
.then((response) => {
|
|
1259
1277
|
this.validateRangeDownload(range, response.ContentRange);
|
|
1260
1278
|
if (response.Body && typeof response.Body.getReader === "function") {
|
|
@@ -1297,7 +1315,7 @@ class S3TransferManagerBase {
|
|
|
1297
1315
|
PartNumber: 1,
|
|
1298
1316
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1299
1317
|
};
|
|
1300
|
-
const initialPart = await this.
|
|
1318
|
+
const initialPart = await this.send(new GetObjectCommand(initialPartRequest), transferOptions);
|
|
1301
1319
|
await internalEventHandler.afterInitialGetObject();
|
|
1302
1320
|
const partSize = initialPart.ContentLength;
|
|
1303
1321
|
const initialETag = initialPart.ETag ?? undefined;
|
|
@@ -1377,8 +1395,7 @@ class S3TransferManagerBase {
|
|
|
1377
1395
|
checksumAlgorithm: checksumValidationEnabled ? "CRC32" : undefined,
|
|
1378
1396
|
resultToken,
|
|
1379
1397
|
};
|
|
1380
|
-
this.
|
|
1381
|
-
.send(new GetObjectCommand(partGetRequest), {
|
|
1398
|
+
this.send(new GetObjectCommand(partGetRequest), {
|
|
1382
1399
|
...transferOptions,
|
|
1383
1400
|
downloadStream: transferOptions_,
|
|
1384
1401
|
})
|
|
@@ -1444,14 +1461,14 @@ class S3TransferManagerBase {
|
|
|
1444
1461
|
}
|
|
1445
1462
|
async downloadByRangeWithWorkers(request, transferOptions, streams, requests, metadata, checksumValidationEnabled) {
|
|
1446
1463
|
this.checkAborted(transferOptions);
|
|
1447
|
-
const headResponse = await this.
|
|
1464
|
+
const headResponse = await this.send(new HeadObjectCommand({ Bucket: request.Bucket, Key: request.Key }), transferOptions);
|
|
1448
1465
|
await internalEventHandler.afterInitialGetObject();
|
|
1449
1466
|
if (headResponse.ContentLength === 0) {
|
|
1450
1467
|
const getObjectRequest = {
|
|
1451
1468
|
...request,
|
|
1452
1469
|
...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
|
|
1453
1470
|
};
|
|
1454
|
-
const response = await this.
|
|
1471
|
+
const response = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1455
1472
|
this.dispatchTransferInitiatedEvent(request, 0);
|
|
1456
1473
|
if (response.Body)
|
|
1457
1474
|
streams.push(Promise.resolve(response.Body));
|
|
@@ -1529,8 +1546,7 @@ class S3TransferManagerBase {
|
|
|
1529
1546
|
checksumAlgorithm: checksumValidationEnabled ? "CRC32" : undefined,
|
|
1530
1547
|
resultToken,
|
|
1531
1548
|
};
|
|
1532
|
-
this.
|
|
1533
|
-
.send(new GetObjectCommand(rangeGetRequest), {
|
|
1549
|
+
this.send(new GetObjectCommand(rangeGetRequest), {
|
|
1534
1550
|
...transferOptions,
|
|
1535
1551
|
downloadStream: transferOpts,
|
|
1536
1552
|
})
|
|
@@ -1801,7 +1817,7 @@ class S3TransferManagerBase {
|
|
|
1801
1817
|
async uploadSinglePart(request, contentLength, transferOptions, emitEvents) {
|
|
1802
1818
|
const putObjectRequest = { ...request };
|
|
1803
1819
|
this.checkAborted(transferOptions);
|
|
1804
|
-
const response = await this.
|
|
1820
|
+
const response = await this.send(new PutObjectCommand(putObjectRequest), transferOptions);
|
|
1805
1821
|
if (emitEvents) {
|
|
1806
1822
|
this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
|
|
1807
1823
|
request,
|
|
@@ -1891,7 +1907,7 @@ class S3TransferManagerBase {
|
|
|
1891
1907
|
if (!createMpuRequest.ChecksumAlgorithm && this.requestChecksumCalculation === "WHEN_SUPPORTED") {
|
|
1892
1908
|
createMpuRequest.ChecksumAlgorithm = request.ChecksumAlgorithm ?? "CRC32";
|
|
1893
1909
|
}
|
|
1894
|
-
const createMpuResponse = await this.
|
|
1910
|
+
const createMpuResponse = await this.send(new CreateMultipartUploadCommand(createMpuRequest), transferOptions);
|
|
1895
1911
|
const uploadId = createMpuResponse.UploadId;
|
|
1896
1912
|
const checksumAlgorithm = createMpuRequest.ChecksumAlgorithm;
|
|
1897
1913
|
const checksumHeader = checksumAlgorithm ? `x-amz-checksum-${checksumAlgorithm.toLowerCase()}` : undefined;
|
|
@@ -1916,7 +1932,7 @@ class S3TransferManagerBase {
|
|
|
1916
1932
|
}),
|
|
1917
1933
|
};
|
|
1918
1934
|
try {
|
|
1919
|
-
return await this.
|
|
1935
|
+
return await this.send(new UploadPartCommand(partRequest), {
|
|
1920
1936
|
...transferOptions,
|
|
1921
1937
|
dataSource,
|
|
1922
1938
|
});
|
|
@@ -2005,7 +2021,7 @@ class S3TransferManagerBase {
|
|
|
2005
2021
|
completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
|
|
2006
2022
|
}
|
|
2007
2023
|
try {
|
|
2008
|
-
return await this.
|
|
2024
|
+
return await this.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
|
|
2009
2025
|
}
|
|
2010
2026
|
catch (completeMpuError) {
|
|
2011
2027
|
this.logger.warn(`CompleteMultipartUpload failed for upload ID ${uploadId}: ${completeMpuError.message}`);
|
|
@@ -2019,7 +2035,7 @@ class S3TransferManagerBase {
|
|
|
2019
2035
|
UploadId: uploadId,
|
|
2020
2036
|
};
|
|
2021
2037
|
try {
|
|
2022
|
-
await this.
|
|
2038
|
+
await this.send(new AbortMultipartUploadCommand(abortRequest), transferOptions);
|
|
2023
2039
|
}
|
|
2024
2040
|
catch (abortError) {
|
|
2025
2041
|
this.logger.warn(`Failed to abort multipart upload ${uploadId}:`, abortError);
|
|
@@ -2043,7 +2059,7 @@ class S3TransferManagerBase {
|
|
|
2043
2059
|
if (!createMpuRequest.ChecksumAlgorithm && this.requestChecksumCalculation === "WHEN_SUPPORTED") {
|
|
2044
2060
|
createMpuRequest.ChecksumAlgorithm = request.ChecksumAlgorithm ?? "CRC32";
|
|
2045
2061
|
}
|
|
2046
|
-
const createMpuResponse = await this.
|
|
2062
|
+
const createMpuResponse = await this.send(new CreateMultipartUploadCommand(createMpuRequest), transferOptions);
|
|
2047
2063
|
const uploadId = createMpuResponse.UploadId;
|
|
2048
2064
|
const abortController = new AbortController();
|
|
2049
2065
|
const uploadAbortSignal = abortController.signal;
|
|
@@ -2068,7 +2084,7 @@ class S3TransferManagerBase {
|
|
|
2068
2084
|
ChecksumAlgorithm: request.ChecksumAlgorithm,
|
|
2069
2085
|
}),
|
|
2070
2086
|
};
|
|
2071
|
-
const partResponse = await this.
|
|
2087
|
+
const partResponse = await this.send(new UploadPartCommand(partRequest), transferOptions);
|
|
2072
2088
|
const checksumAlgorithm = partRequest.ChecksumAlgorithm;
|
|
2073
2089
|
if (checksumAlgorithm &&
|
|
2074
2090
|
!partResponse.ChecksumCRC32 &&
|
|
@@ -2137,7 +2153,7 @@ class S3TransferManagerBase {
|
|
|
2137
2153
|
completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
|
|
2138
2154
|
}
|
|
2139
2155
|
try {
|
|
2140
|
-
return await this.
|
|
2156
|
+
return await this.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
|
|
2141
2157
|
}
|
|
2142
2158
|
catch (completeMpuError) {
|
|
2143
2159
|
this.logger.warn(`CompleteMultipartUpload failed for upload ID ${uploadId}: ${completeMpuError.message}`);
|
|
@@ -2151,7 +2167,7 @@ class S3TransferManagerBase {
|
|
|
2151
2167
|
UploadId: uploadId,
|
|
2152
2168
|
};
|
|
2153
2169
|
try {
|
|
2154
|
-
await this.
|
|
2170
|
+
await this.send(new AbortMultipartUploadCommand(abortRequest), transferOptions);
|
|
2155
2171
|
}
|
|
2156
2172
|
catch (abortError) {
|
|
2157
2173
|
this.logger.warn(`Failed to abort multipart upload ${uploadId}:`, abortError);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aws-sdk/lib-transfer-manager",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "S3 Transfer Manager for AWS SDK JS v3",
|
|
5
5
|
"homepage": "https://github.com/aws/aws-sdk-js-v3/tree/main/preview/lib-transfer-manager",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -85,15 +85,16 @@
|
|
|
85
85
|
"test:browser:watch": "yarn g:vitest watch -c vitest.config.browser.mts"
|
|
86
86
|
},
|
|
87
87
|
"dependencies": {
|
|
88
|
-
"@aws-sdk/checksums": "^3.1001.
|
|
89
|
-
"@
|
|
88
|
+
"@aws-sdk/checksums": "^3.1001.5",
|
|
89
|
+
"@aws-sdk/core": "^3.978.5",
|
|
90
|
+
"@smithy/core": "^3.36.1",
|
|
90
91
|
"@smithy/node-http-handler": "^4.12.1",
|
|
91
92
|
"@smithy/types": "^4.19.0",
|
|
92
93
|
"tslib": "^2.6.2"
|
|
93
94
|
},
|
|
94
95
|
"devDependencies": {
|
|
95
|
-
"@aws-sdk/client-s3": "3.
|
|
96
|
-
"@aws-sdk/xhr-http-handler": "3.
|
|
96
|
+
"@aws-sdk/client-s3": "3.1155.0",
|
|
97
|
+
"@aws-sdk/xhr-http-handler": "3.1155.0",
|
|
97
98
|
"@tsconfig/recommended": "1.0.1",
|
|
98
99
|
"@types/node": "^20.14.8",
|
|
99
100
|
"concurrently": "7.0.0",
|
|
@@ -102,7 +103,7 @@
|
|
|
102
103
|
"typescript": "~7.0.2"
|
|
103
104
|
},
|
|
104
105
|
"peerDependencies": {
|
|
105
|
-
"@aws-sdk/client-s3": "^3.
|
|
106
|
+
"@aws-sdk/client-s3": "^3.1155.0"
|
|
106
107
|
},
|
|
107
108
|
"engines": {
|
|
108
109
|
"node": ">=20.0.0"
|