@aws-sdk/lib-transfer-manager 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-cjs/submodules/transfer-manager/index.js +94 -47
- package/dist-cjs/submodules/worker/index.js +12 -6
- package/dist-es/submodules/transfer-manager/S3TransferManager.js +80 -40
- package/dist-es/submodules/transfer-manager/chunker.js +2 -1
- package/dist-es/submodules/transfer-manager/join-streams.browser.js +2 -1
- package/dist-es/submodules/transfer-manager/join-streams.js +2 -1
- package/dist-es/submodules/transfer-manager/ordered-part-queue.js +2 -1
- package/dist-es/submodules/transfer-manager/worker-http-handler.js +8 -4
- package/dist-es/submodules/worker/http-request-worker.js +12 -6
- package/package.json +8 -8
|
@@ -101,8 +101,9 @@ async function* iterateStreams(promises, eventListeners) {
|
|
|
101
101
|
}
|
|
102
102
|
async function destroyStreams(promises) {
|
|
103
103
|
await Promise.all(promises.map(async (streamPromise) => {
|
|
104
|
-
if (!streamPromise)
|
|
104
|
+
if (!streamPromise) {
|
|
105
105
|
return;
|
|
106
|
+
}
|
|
106
107
|
return streamPromise
|
|
107
108
|
.then((stream) => {
|
|
108
109
|
if (stream instanceof Readable) {
|
|
@@ -199,8 +200,9 @@ async function* getDataReadableStream(data) {
|
|
|
199
200
|
try {
|
|
200
201
|
while (true) {
|
|
201
202
|
const { done, value } = await reader.read();
|
|
202
|
-
if (done)
|
|
203
|
+
if (done) {
|
|
203
204
|
return;
|
|
205
|
+
}
|
|
204
206
|
if (Buffer$1.isBuffer(value) || value instanceof Uint8Array) {
|
|
205
207
|
yield value;
|
|
206
208
|
}
|
|
@@ -441,8 +443,9 @@ class OrderedPartQueue {
|
|
|
441
443
|
this.totalParts = totalParts;
|
|
442
444
|
}
|
|
443
445
|
enqueue(rangeIndex, buffer, byteLength) {
|
|
444
|
-
if (this.error)
|
|
446
|
+
if (this.error) {
|
|
445
447
|
return;
|
|
448
|
+
}
|
|
446
449
|
if (rangeIndex === this.nextExpected && this.waitingConsumer) {
|
|
447
450
|
const consumer = this.waitingConsumer;
|
|
448
451
|
this.waitingConsumer = null;
|
|
@@ -891,8 +894,9 @@ class S3TransferManagerBase {
|
|
|
891
894
|
}
|
|
892
895
|
const remainingPartTasks = [];
|
|
893
896
|
for (let part = 2; part <= partsCount; part++) {
|
|
894
|
-
if (failed || abortController.signal.aborted)
|
|
897
|
+
if (failed || abortController.signal.aborted) {
|
|
895
898
|
break;
|
|
899
|
+
}
|
|
896
900
|
this.checkAborted(transferOptions);
|
|
897
901
|
await semaphore.acquire();
|
|
898
902
|
if (failed || abortController.signal.aborted) {
|
|
@@ -1081,8 +1085,9 @@ class S3TransferManagerBase {
|
|
|
1081
1085
|
const rangeTasks = [];
|
|
1082
1086
|
let rangeIndex = 1;
|
|
1083
1087
|
for (let offset = rangeSize; offset < totalContentLength; offset += rangeSize) {
|
|
1084
|
-
if (failed || abortController.signal.aborted)
|
|
1088
|
+
if (failed || abortController.signal.aborted) {
|
|
1085
1089
|
break;
|
|
1090
|
+
}
|
|
1086
1091
|
this.checkAborted(transferOptions);
|
|
1087
1092
|
await semaphore.acquire();
|
|
1088
1093
|
if (failed || abortController.signal.aborted) {
|
|
@@ -1268,13 +1273,15 @@ class S3TransferManagerBase {
|
|
|
1268
1273
|
recursive: request.recursive ?? false,
|
|
1269
1274
|
followSymbolicLinks: request.followSymbolicLinks ?? false,
|
|
1270
1275
|
})) {
|
|
1271
|
-
if (terminated || abortController.signal.aborted)
|
|
1276
|
+
if (terminated || abortController.signal.aborted) {
|
|
1272
1277
|
break;
|
|
1278
|
+
}
|
|
1273
1279
|
this.checkAborted(transferOptions);
|
|
1274
1280
|
if (request.filter) {
|
|
1275
1281
|
const include = request.filter instanceof RegExp ? matchesFilterRegExp(request.filter, filePath) : request.filter(filePath);
|
|
1276
|
-
if (!include)
|
|
1282
|
+
if (!include) {
|
|
1277
1283
|
continue;
|
|
1284
|
+
}
|
|
1278
1285
|
}
|
|
1279
1286
|
const fileStat = await stat(filePath);
|
|
1280
1287
|
discoveredFiles++;
|
|
@@ -1427,8 +1434,9 @@ class S3TransferManagerBase {
|
|
|
1427
1434
|
try {
|
|
1428
1435
|
let continuationToken;
|
|
1429
1436
|
do {
|
|
1430
|
-
if (terminated || abortController.signal.aborted)
|
|
1437
|
+
if (terminated || abortController.signal.aborted) {
|
|
1431
1438
|
break;
|
|
1439
|
+
}
|
|
1432
1440
|
this.checkAborted(transferOptions);
|
|
1433
1441
|
const listRequest = {
|
|
1434
1442
|
Bucket: request.bucket,
|
|
@@ -1442,8 +1450,9 @@ class S3TransferManagerBase {
|
|
|
1442
1450
|
});
|
|
1443
1451
|
continuationToken = listResponse.IsTruncated ? listResponse.NextContinuationToken : undefined;
|
|
1444
1452
|
for (const object of listResponse.Contents ?? []) {
|
|
1445
|
-
if (terminated || abortController.signal.aborted)
|
|
1453
|
+
if (terminated || abortController.signal.aborted) {
|
|
1446
1454
|
break;
|
|
1455
|
+
}
|
|
1447
1456
|
this.checkAborted(transferOptions);
|
|
1448
1457
|
if (isFolderObject(object)) {
|
|
1449
1458
|
continue;
|
|
@@ -1679,8 +1688,9 @@ class S3TransferManagerBase {
|
|
|
1679
1688
|
};
|
|
1680
1689
|
const response = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1681
1690
|
this.dispatchTransferInitiatedEvent(request, 0);
|
|
1682
|
-
if (response.Body)
|
|
1691
|
+
if (response.Body) {
|
|
1683
1692
|
streams.push(Promise.resolve(response.Body));
|
|
1693
|
+
}
|
|
1684
1694
|
requests.push(getObjectRequest);
|
|
1685
1695
|
this.processResponseMetadata(response, metadata, 0);
|
|
1686
1696
|
return { totalSize: 0 };
|
|
@@ -1850,12 +1860,14 @@ class S3TransferManagerBase {
|
|
|
1850
1860
|
};
|
|
1851
1861
|
(async () => {
|
|
1852
1862
|
for (let partNumber = 2; partNumber <= partsCount; partNumber++) {
|
|
1853
|
-
if (abortController.signal.aborted || queue.hasError())
|
|
1863
|
+
if (abortController.signal.aborted || queue.hasError()) {
|
|
1854
1864
|
break;
|
|
1865
|
+
}
|
|
1855
1866
|
this.checkAborted(transferOptions);
|
|
1856
1867
|
await waitForSlot();
|
|
1857
|
-
if (abortController.signal.aborted || queue.hasError())
|
|
1868
|
+
if (abortController.signal.aborted || queue.hasError()) {
|
|
1858
1869
|
break;
|
|
1870
|
+
}
|
|
1859
1871
|
inFlight++;
|
|
1860
1872
|
const rangeIndex = partNumber - 2;
|
|
1861
1873
|
const partGetRequest = {
|
|
@@ -1904,8 +1916,9 @@ class S3TransferManagerBase {
|
|
|
1904
1916
|
let reading = false;
|
|
1905
1917
|
const transferStream = new Readable({
|
|
1906
1918
|
read() {
|
|
1907
|
-
if (reading)
|
|
1919
|
+
if (reading) {
|
|
1908
1920
|
return;
|
|
1921
|
+
}
|
|
1909
1922
|
reading = true;
|
|
1910
1923
|
queue
|
|
1911
1924
|
.dequeue()
|
|
@@ -1949,8 +1962,9 @@ class S3TransferManagerBase {
|
|
|
1949
1962
|
};
|
|
1950
1963
|
const response = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1951
1964
|
this.dispatchTransferInitiatedEvent(request, 0);
|
|
1952
|
-
if (response.Body)
|
|
1965
|
+
if (response.Body) {
|
|
1953
1966
|
streams.push(Promise.resolve(response.Body));
|
|
1967
|
+
}
|
|
1954
1968
|
requests.push(getObjectRequest);
|
|
1955
1969
|
this.processResponseMetadata(response, metadata, 0);
|
|
1956
1970
|
return { totalSize: 0 };
|
|
@@ -1999,12 +2013,14 @@ class S3TransferManagerBase {
|
|
|
1999
2013
|
};
|
|
2000
2014
|
(async () => {
|
|
2001
2015
|
for (let rangeIndex = 0; rangeIndex < totalParts; rangeIndex++) {
|
|
2002
|
-
if (abortController.signal.aborted || queue.hasError())
|
|
2016
|
+
if (abortController.signal.aborted || queue.hasError()) {
|
|
2003
2017
|
break;
|
|
2018
|
+
}
|
|
2004
2019
|
this.checkAborted(transferOptions);
|
|
2005
2020
|
await waitForSlot();
|
|
2006
|
-
if (abortController.signal.aborted || queue.hasError())
|
|
2021
|
+
if (abortController.signal.aborted || queue.hasError()) {
|
|
2007
2022
|
break;
|
|
2023
|
+
}
|
|
2008
2024
|
inFlight++;
|
|
2009
2025
|
const start = rangeIndex * partSize;
|
|
2010
2026
|
const end = Math.min(start + partSize - 1, totalSize - 1);
|
|
@@ -2055,8 +2071,9 @@ class S3TransferManagerBase {
|
|
|
2055
2071
|
let reading = false;
|
|
2056
2072
|
const transferStream = new Readable({
|
|
2057
2073
|
read() {
|
|
2058
|
-
if (reading)
|
|
2074
|
+
if (reading) {
|
|
2059
2075
|
return;
|
|
2076
|
+
}
|
|
2060
2077
|
reading = true;
|
|
2061
2078
|
queue
|
|
2062
2079
|
.dequeue()
|
|
@@ -2214,8 +2231,9 @@ class S3TransferManagerBase {
|
|
|
2214
2231
|
}
|
|
2215
2232
|
}
|
|
2216
2233
|
createConcurrentTaskPool(tasks, maxConcurrent) {
|
|
2217
|
-
if (tasks.length === 0)
|
|
2234
|
+
if (tasks.length === 0) {
|
|
2218
2235
|
return { promises: [], onStreamConsumed: () => { } };
|
|
2236
|
+
}
|
|
2219
2237
|
const resolvers = [];
|
|
2220
2238
|
const promises = tasks.map(() => {
|
|
2221
2239
|
let resolve;
|
|
@@ -2242,8 +2260,9 @@ class S3TransferManagerBase {
|
|
|
2242
2260
|
launchTask(nextToLaunch++);
|
|
2243
2261
|
}
|
|
2244
2262
|
const onStreamConsumed = (_index) => {
|
|
2245
|
-
if (failed || nextToLaunch >= tasks.length)
|
|
2263
|
+
if (failed || nextToLaunch >= tasks.length) {
|
|
2246
2264
|
return;
|
|
2265
|
+
}
|
|
2247
2266
|
launchTask(nextToLaunch++);
|
|
2248
2267
|
};
|
|
2249
2268
|
return { promises, onStreamConsumed };
|
|
@@ -2253,8 +2272,9 @@ class S3TransferManagerBase {
|
|
|
2253
2272
|
throw new Error(`Missing ContentRange for part ${partNumber}.`);
|
|
2254
2273
|
}
|
|
2255
2274
|
const match = contentRange.match(/bytes (\d+)-(\d+)\/(\d+)/);
|
|
2256
|
-
if (!match)
|
|
2275
|
+
if (!match) {
|
|
2257
2276
|
throw new Error(`Invalid ContentRange format: ${contentRange}`);
|
|
2277
|
+
}
|
|
2258
2278
|
const start = Number.parseInt(match[1]);
|
|
2259
2279
|
const end = Number.parseInt(match[2]);
|
|
2260
2280
|
const total = Number.parseInt(match[3]) - 1;
|
|
@@ -2272,14 +2292,16 @@ class S3TransferManagerBase {
|
|
|
2272
2292
|
throw new Error(`Missing ContentRange for range ${requestRange}.`);
|
|
2273
2293
|
}
|
|
2274
2294
|
const match = responseRange.match(/bytes (\d+)-(\d+)\/(\d+)/);
|
|
2275
|
-
if (!match)
|
|
2295
|
+
if (!match) {
|
|
2276
2296
|
throw new Error(`Invalid ContentRange format: ${responseRange}`);
|
|
2297
|
+
}
|
|
2277
2298
|
const start = Number.parseInt(match[1]);
|
|
2278
2299
|
const end = Number.parseInt(match[2]);
|
|
2279
2300
|
const total = Number.parseInt(match[3]) - 1;
|
|
2280
2301
|
const rangeMatch = requestRange.match(/bytes=(\d+)-(\d+)/);
|
|
2281
|
-
if (!rangeMatch)
|
|
2302
|
+
if (!rangeMatch) {
|
|
2282
2303
|
throw new Error(`Invalid Range format: ${requestRange}`);
|
|
2304
|
+
}
|
|
2283
2305
|
const expectedStart = Number.parseInt(rangeMatch[1]);
|
|
2284
2306
|
const expectedEnd = Number.parseInt(rangeMatch[2]);
|
|
2285
2307
|
if (start !== expectedStart) {
|
|
@@ -2434,8 +2456,9 @@ class S3TransferManagerBase {
|
|
|
2434
2456
|
const uploadPart = async () => {
|
|
2435
2457
|
while (true) {
|
|
2436
2458
|
const partNumber = nextPartNumber++;
|
|
2437
|
-
if (partNumber > expectedPartsCount)
|
|
2459
|
+
if (partNumber > expectedPartsCount) {
|
|
2438
2460
|
return;
|
|
2461
|
+
}
|
|
2439
2462
|
if (uploadAbortSignal.aborted) {
|
|
2440
2463
|
throw Object.assign(new Error("Upload aborted due to part failure."), { name: "AbortError" });
|
|
2441
2464
|
}
|
|
@@ -2447,16 +2470,21 @@ class S3TransferManagerBase {
|
|
|
2447
2470
|
PartNumber: partNumber,
|
|
2448
2471
|
ETag: partResponse.ETag,
|
|
2449
2472
|
};
|
|
2450
|
-
if (partResponse.ChecksumCRC32)
|
|
2473
|
+
if (partResponse.ChecksumCRC32) {
|
|
2451
2474
|
completedPart.ChecksumCRC32 = partResponse.ChecksumCRC32;
|
|
2452
|
-
|
|
2475
|
+
}
|
|
2476
|
+
if (partResponse.ChecksumCRC32C) {
|
|
2453
2477
|
completedPart.ChecksumCRC32C = partResponse.ChecksumCRC32C;
|
|
2454
|
-
|
|
2478
|
+
}
|
|
2479
|
+
if (partResponse.ChecksumCRC64NVME) {
|
|
2455
2480
|
completedPart.ChecksumCRC64NVME = partResponse.ChecksumCRC64NVME;
|
|
2456
|
-
|
|
2481
|
+
}
|
|
2482
|
+
if (partResponse.ChecksumSHA1) {
|
|
2457
2483
|
completedPart.ChecksumSHA1 = partResponse.ChecksumSHA1;
|
|
2458
|
-
|
|
2484
|
+
}
|
|
2485
|
+
if (partResponse.ChecksumSHA256) {
|
|
2459
2486
|
completedPart.ChecksumSHA256 = partResponse.ChecksumSHA256;
|
|
2487
|
+
}
|
|
2460
2488
|
completedParts.push(completedPart);
|
|
2461
2489
|
bytesUploaded += length;
|
|
2462
2490
|
if (emitEvents) {
|
|
@@ -2492,12 +2520,15 @@ class S3TransferManagerBase {
|
|
|
2492
2520
|
};
|
|
2493
2521
|
if (hasFullObjectChecksum) {
|
|
2494
2522
|
completeRequest.ChecksumType = "FULL_OBJECT";
|
|
2495
|
-
if (request.ChecksumCRC32)
|
|
2523
|
+
if (request.ChecksumCRC32) {
|
|
2496
2524
|
completeRequest.ChecksumCRC32 = request.ChecksumCRC32;
|
|
2497
|
-
|
|
2525
|
+
}
|
|
2526
|
+
if (request.ChecksumCRC32C) {
|
|
2498
2527
|
completeRequest.ChecksumCRC32C = request.ChecksumCRC32C;
|
|
2499
|
-
|
|
2528
|
+
}
|
|
2529
|
+
if (request.ChecksumCRC64NVME) {
|
|
2500
2530
|
completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
|
|
2531
|
+
}
|
|
2501
2532
|
}
|
|
2502
2533
|
try {
|
|
2503
2534
|
return await this.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
|
|
@@ -2578,16 +2609,21 @@ class S3TransferManagerBase {
|
|
|
2578
2609
|
PartNumber: dataPart.partNumber,
|
|
2579
2610
|
ETag: partResponse.ETag,
|
|
2580
2611
|
};
|
|
2581
|
-
if (partResponse.ChecksumCRC32)
|
|
2612
|
+
if (partResponse.ChecksumCRC32) {
|
|
2582
2613
|
completedPart.ChecksumCRC32 = partResponse.ChecksumCRC32;
|
|
2583
|
-
|
|
2614
|
+
}
|
|
2615
|
+
if (partResponse.ChecksumCRC32C) {
|
|
2584
2616
|
completedPart.ChecksumCRC32C = partResponse.ChecksumCRC32C;
|
|
2585
|
-
|
|
2617
|
+
}
|
|
2618
|
+
if (partResponse.ChecksumCRC64NVME) {
|
|
2586
2619
|
completedPart.ChecksumCRC64NVME = partResponse.ChecksumCRC64NVME;
|
|
2587
|
-
|
|
2620
|
+
}
|
|
2621
|
+
if (partResponse.ChecksumSHA1) {
|
|
2588
2622
|
completedPart.ChecksumSHA1 = partResponse.ChecksumSHA1;
|
|
2589
|
-
|
|
2623
|
+
}
|
|
2624
|
+
if (partResponse.ChecksumSHA256) {
|
|
2590
2625
|
completedPart.ChecksumSHA256 = partResponse.ChecksumSHA256;
|
|
2626
|
+
}
|
|
2591
2627
|
completedParts.push(completedPart);
|
|
2592
2628
|
const partLength = byteLength(dataPart.data);
|
|
2593
2629
|
bytesUploaded += partLength;
|
|
@@ -2624,12 +2660,15 @@ class S3TransferManagerBase {
|
|
|
2624
2660
|
};
|
|
2625
2661
|
if (hasFullObjectChecksum) {
|
|
2626
2662
|
completeRequest.ChecksumType = "FULL_OBJECT";
|
|
2627
|
-
if (request.ChecksumCRC32)
|
|
2663
|
+
if (request.ChecksumCRC32) {
|
|
2628
2664
|
completeRequest.ChecksumCRC32 = request.ChecksumCRC32;
|
|
2629
|
-
|
|
2665
|
+
}
|
|
2666
|
+
if (request.ChecksumCRC32C) {
|
|
2630
2667
|
completeRequest.ChecksumCRC32C = request.ChecksumCRC32C;
|
|
2631
|
-
|
|
2668
|
+
}
|
|
2669
|
+
if (request.ChecksumCRC64NVME) {
|
|
2632
2670
|
completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
|
|
2671
|
+
}
|
|
2633
2672
|
}
|
|
2634
2673
|
try {
|
|
2635
2674
|
return await this.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
|
|
@@ -2660,8 +2699,9 @@ class S3TransferManagerBase {
|
|
|
2660
2699
|
for await (const entry of dir) {
|
|
2661
2700
|
const fullPath = join(entry.parentPath ?? entry.path ?? source, entry.name);
|
|
2662
2701
|
if (entry.isSymbolicLink()) {
|
|
2663
|
-
if (!options.followSymbolicLinks)
|
|
2702
|
+
if (!options.followSymbolicLinks) {
|
|
2664
2703
|
continue;
|
|
2704
|
+
}
|
|
2665
2705
|
const realPath = await realpath(fullPath);
|
|
2666
2706
|
const linkStat = await stat(realPath);
|
|
2667
2707
|
if (linkStat.isDirectory()) {
|
|
@@ -2672,8 +2712,9 @@ class S3TransferManagerBase {
|
|
|
2672
2712
|
yield* this.traverseDirectory(realPath, options);
|
|
2673
2713
|
continue;
|
|
2674
2714
|
}
|
|
2675
|
-
if (!linkStat.isFile())
|
|
2715
|
+
if (!linkStat.isFile()) {
|
|
2676
2716
|
continue;
|
|
2717
|
+
}
|
|
2677
2718
|
}
|
|
2678
2719
|
else if (!entry.isFile()) {
|
|
2679
2720
|
continue;
|
|
@@ -2686,8 +2727,9 @@ class S3TransferManagerBase {
|
|
|
2686
2727
|
if (sep !== "/") {
|
|
2687
2728
|
relativePath = relativePath.split(sep).join("/");
|
|
2688
2729
|
}
|
|
2689
|
-
if (!s3Prefix)
|
|
2730
|
+
if (!s3Prefix) {
|
|
2690
2731
|
return relativePath;
|
|
2732
|
+
}
|
|
2691
2733
|
const normalizedPrefix = s3Prefix.endsWith("/") ? s3Prefix : s3Prefix + "/";
|
|
2692
2734
|
return normalizedPrefix + relativePath;
|
|
2693
2735
|
}
|
|
@@ -2714,8 +2756,9 @@ function matchesFilterRegExp(filter, value) {
|
|
|
2714
2756
|
return filter.test(value);
|
|
2715
2757
|
}
|
|
2716
2758
|
function parseContentRangeStart(contentRange) {
|
|
2717
|
-
if (!contentRange)
|
|
2759
|
+
if (!contentRange) {
|
|
2718
2760
|
return undefined;
|
|
2761
|
+
}
|
|
2719
2762
|
const match = contentRange.match(/^bytes\s+(\d+)-/);
|
|
2720
2763
|
return match ? Number(match[1]) : undefined;
|
|
2721
2764
|
}
|
|
@@ -2725,10 +2768,12 @@ const internalEventHandler = {
|
|
|
2725
2768
|
|
|
2726
2769
|
function buildDownloadError(msg) {
|
|
2727
2770
|
const error = new Error(msg.error);
|
|
2728
|
-
if (msg.name)
|
|
2771
|
+
if (msg.name) {
|
|
2729
2772
|
error.name = msg.name;
|
|
2730
|
-
|
|
2773
|
+
}
|
|
2774
|
+
if (msg.code) {
|
|
2731
2775
|
error.code = msg.code;
|
|
2776
|
+
}
|
|
2732
2777
|
return error;
|
|
2733
2778
|
}
|
|
2734
2779
|
function createEmptyReadable() {
|
|
@@ -2793,10 +2838,12 @@ class WorkerHttpHandler {
|
|
|
2793
2838
|
return { workerPath: submodulePath };
|
|
2794
2839
|
}
|
|
2795
2840
|
ensureInitialized() {
|
|
2796
|
-
if (this.initialized)
|
|
2841
|
+
if (this.initialized) {
|
|
2797
2842
|
return Promise.resolve();
|
|
2798
|
-
|
|
2843
|
+
}
|
|
2844
|
+
if (this.initPromise) {
|
|
2799
2845
|
return this.initPromise;
|
|
2846
|
+
}
|
|
2800
2847
|
this.initPromise = new Promise((resolve, reject) => {
|
|
2801
2848
|
const { workerPath, workerOptions } = this.resolveWorkerConfig();
|
|
2802
2849
|
let readyCount = 0;
|
|
@@ -15,8 +15,9 @@ const DNS_TTL_MS = 1000;
|
|
|
15
15
|
const dnsCache = new Map();
|
|
16
16
|
const MAX_ERROR_BODY_BYTES = 16 * 1024;
|
|
17
17
|
const readErrorBodyBytes = async (body) => {
|
|
18
|
-
if (!body)
|
|
18
|
+
if (!body) {
|
|
19
19
|
return undefined;
|
|
20
|
+
}
|
|
20
21
|
try {
|
|
21
22
|
const chunks = [];
|
|
22
23
|
let total = 0;
|
|
@@ -24,8 +25,9 @@ const readErrorBodyBytes = async (body) => {
|
|
|
24
25
|
const buf = typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk);
|
|
25
26
|
chunks.push(buf);
|
|
26
27
|
total += buf.length;
|
|
27
|
-
if (total >= MAX_ERROR_BODY_BYTES)
|
|
28
|
+
if (total >= MAX_ERROR_BODY_BYTES) {
|
|
28
29
|
break;
|
|
30
|
+
}
|
|
29
31
|
}
|
|
30
32
|
return concatBytes(chunks);
|
|
31
33
|
}
|
|
@@ -148,15 +150,18 @@ async function finalizeChecksumToBase64(checksum) {
|
|
|
148
150
|
return Buffer.from(bytes).toString("base64");
|
|
149
151
|
}
|
|
150
152
|
function parseContentRange(contentRange) {
|
|
151
|
-
if (!contentRange)
|
|
153
|
+
if (!contentRange) {
|
|
152
154
|
return undefined;
|
|
155
|
+
}
|
|
153
156
|
const match = contentRange.match(/^bytes\s+(\d+)-(\d+)\//);
|
|
154
|
-
if (!match)
|
|
157
|
+
if (!match) {
|
|
155
158
|
return undefined;
|
|
159
|
+
}
|
|
156
160
|
const start = Number(match[1]);
|
|
157
161
|
const end = Number(match[2]);
|
|
158
|
-
if (end < start)
|
|
162
|
+
if (end < start) {
|
|
159
163
|
return undefined;
|
|
164
|
+
}
|
|
160
165
|
return { start, end };
|
|
161
166
|
}
|
|
162
167
|
function getChecksumHeaderValue(headers, algorithm) {
|
|
@@ -197,8 +202,9 @@ if (parentPort) {
|
|
|
197
202
|
let read = 0;
|
|
198
203
|
while (read < length) {
|
|
199
204
|
const n = readSync(fd, buffer, read, length - read, offset + read);
|
|
200
|
-
if (n === 0)
|
|
205
|
+
if (n === 0) {
|
|
201
206
|
break;
|
|
207
|
+
}
|
|
202
208
|
read += n;
|
|
203
209
|
}
|
|
204
210
|
return buffer;
|
|
@@ -412,8 +412,9 @@ class S3TransferManagerBase {
|
|
|
412
412
|
}
|
|
413
413
|
const remainingPartTasks = [];
|
|
414
414
|
for (let part = 2; part <= partsCount; part++) {
|
|
415
|
-
if (failed || abortController.signal.aborted)
|
|
415
|
+
if (failed || abortController.signal.aborted) {
|
|
416
416
|
break;
|
|
417
|
+
}
|
|
417
418
|
this.checkAborted(transferOptions);
|
|
418
419
|
await semaphore.acquire();
|
|
419
420
|
if (failed || abortController.signal.aborted) {
|
|
@@ -602,8 +603,9 @@ class S3TransferManagerBase {
|
|
|
602
603
|
const rangeTasks = [];
|
|
603
604
|
let rangeIndex = 1;
|
|
604
605
|
for (let offset = rangeSize; offset < totalContentLength; offset += rangeSize) {
|
|
605
|
-
if (failed || abortController.signal.aborted)
|
|
606
|
+
if (failed || abortController.signal.aborted) {
|
|
606
607
|
break;
|
|
608
|
+
}
|
|
607
609
|
this.checkAborted(transferOptions);
|
|
608
610
|
await semaphore.acquire();
|
|
609
611
|
if (failed || abortController.signal.aborted) {
|
|
@@ -789,13 +791,15 @@ class S3TransferManagerBase {
|
|
|
789
791
|
recursive: request.recursive ?? false,
|
|
790
792
|
followSymbolicLinks: request.followSymbolicLinks ?? false,
|
|
791
793
|
})) {
|
|
792
|
-
if (terminated || abortController.signal.aborted)
|
|
794
|
+
if (terminated || abortController.signal.aborted) {
|
|
793
795
|
break;
|
|
796
|
+
}
|
|
794
797
|
this.checkAborted(transferOptions);
|
|
795
798
|
if (request.filter) {
|
|
796
799
|
const include = request.filter instanceof RegExp ? matchesFilterRegExp(request.filter, filePath) : request.filter(filePath);
|
|
797
|
-
if (!include)
|
|
800
|
+
if (!include) {
|
|
798
801
|
continue;
|
|
802
|
+
}
|
|
799
803
|
}
|
|
800
804
|
const fileStat = await stat(filePath);
|
|
801
805
|
discoveredFiles++;
|
|
@@ -948,8 +952,9 @@ class S3TransferManagerBase {
|
|
|
948
952
|
try {
|
|
949
953
|
let continuationToken;
|
|
950
954
|
do {
|
|
951
|
-
if (terminated || abortController.signal.aborted)
|
|
955
|
+
if (terminated || abortController.signal.aborted) {
|
|
952
956
|
break;
|
|
957
|
+
}
|
|
953
958
|
this.checkAborted(transferOptions);
|
|
954
959
|
const listRequest = {
|
|
955
960
|
Bucket: request.bucket,
|
|
@@ -963,8 +968,9 @@ class S3TransferManagerBase {
|
|
|
963
968
|
});
|
|
964
969
|
continuationToken = listResponse.IsTruncated ? listResponse.NextContinuationToken : undefined;
|
|
965
970
|
for (const object of listResponse.Contents ?? []) {
|
|
966
|
-
if (terminated || abortController.signal.aborted)
|
|
971
|
+
if (terminated || abortController.signal.aborted) {
|
|
967
972
|
break;
|
|
973
|
+
}
|
|
968
974
|
this.checkAborted(transferOptions);
|
|
969
975
|
if (isFolderObject(object)) {
|
|
970
976
|
continue;
|
|
@@ -1200,8 +1206,9 @@ class S3TransferManagerBase {
|
|
|
1200
1206
|
};
|
|
1201
1207
|
const response = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1202
1208
|
this.dispatchTransferInitiatedEvent(request, 0);
|
|
1203
|
-
if (response.Body)
|
|
1209
|
+
if (response.Body) {
|
|
1204
1210
|
streams.push(Promise.resolve(response.Body));
|
|
1211
|
+
}
|
|
1205
1212
|
requests.push(getObjectRequest);
|
|
1206
1213
|
this.processResponseMetadata(response, metadata, 0);
|
|
1207
1214
|
return { totalSize: 0 };
|
|
@@ -1371,12 +1378,14 @@ class S3TransferManagerBase {
|
|
|
1371
1378
|
};
|
|
1372
1379
|
const dispatchPromise = (async () => {
|
|
1373
1380
|
for (let partNumber = 2; partNumber <= partsCount; partNumber++) {
|
|
1374
|
-
if (abortController.signal.aborted || queue.hasError())
|
|
1381
|
+
if (abortController.signal.aborted || queue.hasError()) {
|
|
1375
1382
|
break;
|
|
1383
|
+
}
|
|
1376
1384
|
this.checkAborted(transferOptions);
|
|
1377
1385
|
await waitForSlot();
|
|
1378
|
-
if (abortController.signal.aborted || queue.hasError())
|
|
1386
|
+
if (abortController.signal.aborted || queue.hasError()) {
|
|
1379
1387
|
break;
|
|
1388
|
+
}
|
|
1380
1389
|
inFlight++;
|
|
1381
1390
|
const rangeIndex = partNumber - 2;
|
|
1382
1391
|
const partGetRequest = {
|
|
@@ -1425,8 +1434,9 @@ class S3TransferManagerBase {
|
|
|
1425
1434
|
let reading = false;
|
|
1426
1435
|
const transferStream = new Readable({
|
|
1427
1436
|
read() {
|
|
1428
|
-
if (reading)
|
|
1437
|
+
if (reading) {
|
|
1429
1438
|
return;
|
|
1439
|
+
}
|
|
1430
1440
|
reading = true;
|
|
1431
1441
|
queue
|
|
1432
1442
|
.dequeue()
|
|
@@ -1470,8 +1480,9 @@ class S3TransferManagerBase {
|
|
|
1470
1480
|
};
|
|
1471
1481
|
const response = await this.send(new GetObjectCommand(getObjectRequest), transferOptions);
|
|
1472
1482
|
this.dispatchTransferInitiatedEvent(request, 0);
|
|
1473
|
-
if (response.Body)
|
|
1483
|
+
if (response.Body) {
|
|
1474
1484
|
streams.push(Promise.resolve(response.Body));
|
|
1485
|
+
}
|
|
1475
1486
|
requests.push(getObjectRequest);
|
|
1476
1487
|
this.processResponseMetadata(response, metadata, 0);
|
|
1477
1488
|
return { totalSize: 0 };
|
|
@@ -1520,12 +1531,14 @@ class S3TransferManagerBase {
|
|
|
1520
1531
|
};
|
|
1521
1532
|
const dispatchPromise = (async () => {
|
|
1522
1533
|
for (let rangeIndex = 0; rangeIndex < totalParts; rangeIndex++) {
|
|
1523
|
-
if (abortController.signal.aborted || queue.hasError())
|
|
1534
|
+
if (abortController.signal.aborted || queue.hasError()) {
|
|
1524
1535
|
break;
|
|
1536
|
+
}
|
|
1525
1537
|
this.checkAborted(transferOptions);
|
|
1526
1538
|
await waitForSlot();
|
|
1527
|
-
if (abortController.signal.aborted || queue.hasError())
|
|
1539
|
+
if (abortController.signal.aborted || queue.hasError()) {
|
|
1528
1540
|
break;
|
|
1541
|
+
}
|
|
1529
1542
|
inFlight++;
|
|
1530
1543
|
const start = rangeIndex * partSize;
|
|
1531
1544
|
const end = Math.min(start + partSize - 1, totalSize - 1);
|
|
@@ -1576,8 +1589,9 @@ class S3TransferManagerBase {
|
|
|
1576
1589
|
let reading = false;
|
|
1577
1590
|
const transferStream = new Readable({
|
|
1578
1591
|
read() {
|
|
1579
|
-
if (reading)
|
|
1592
|
+
if (reading) {
|
|
1580
1593
|
return;
|
|
1594
|
+
}
|
|
1581
1595
|
reading = true;
|
|
1582
1596
|
queue
|
|
1583
1597
|
.dequeue()
|
|
@@ -1735,8 +1749,9 @@ class S3TransferManagerBase {
|
|
|
1735
1749
|
}
|
|
1736
1750
|
}
|
|
1737
1751
|
createConcurrentTaskPool(tasks, maxConcurrent) {
|
|
1738
|
-
if (tasks.length === 0)
|
|
1752
|
+
if (tasks.length === 0) {
|
|
1739
1753
|
return { promises: [], onStreamConsumed: () => { } };
|
|
1754
|
+
}
|
|
1740
1755
|
const resolvers = [];
|
|
1741
1756
|
const promises = tasks.map(() => {
|
|
1742
1757
|
let resolve;
|
|
@@ -1763,8 +1778,9 @@ class S3TransferManagerBase {
|
|
|
1763
1778
|
launchTask(nextToLaunch++);
|
|
1764
1779
|
}
|
|
1765
1780
|
const onStreamConsumed = (_index) => {
|
|
1766
|
-
if (failed || nextToLaunch >= tasks.length)
|
|
1781
|
+
if (failed || nextToLaunch >= tasks.length) {
|
|
1767
1782
|
return;
|
|
1783
|
+
}
|
|
1768
1784
|
launchTask(nextToLaunch++);
|
|
1769
1785
|
};
|
|
1770
1786
|
return { promises, onStreamConsumed };
|
|
@@ -1774,8 +1790,9 @@ class S3TransferManagerBase {
|
|
|
1774
1790
|
throw new Error(`Missing ContentRange for part ${partNumber}.`);
|
|
1775
1791
|
}
|
|
1776
1792
|
const match = contentRange.match(/bytes (\d+)-(\d+)\/(\d+)/);
|
|
1777
|
-
if (!match)
|
|
1793
|
+
if (!match) {
|
|
1778
1794
|
throw new Error(`Invalid ContentRange format: ${contentRange}`);
|
|
1795
|
+
}
|
|
1779
1796
|
const start = Number.parseInt(match[1]);
|
|
1780
1797
|
const end = Number.parseInt(match[2]);
|
|
1781
1798
|
const total = Number.parseInt(match[3]) - 1;
|
|
@@ -1793,14 +1810,16 @@ class S3TransferManagerBase {
|
|
|
1793
1810
|
throw new Error(`Missing ContentRange for range ${requestRange}.`);
|
|
1794
1811
|
}
|
|
1795
1812
|
const match = responseRange.match(/bytes (\d+)-(\d+)\/(\d+)/);
|
|
1796
|
-
if (!match)
|
|
1813
|
+
if (!match) {
|
|
1797
1814
|
throw new Error(`Invalid ContentRange format: ${responseRange}`);
|
|
1815
|
+
}
|
|
1798
1816
|
const start = Number.parseInt(match[1]);
|
|
1799
1817
|
const end = Number.parseInt(match[2]);
|
|
1800
1818
|
const total = Number.parseInt(match[3]) - 1;
|
|
1801
1819
|
const rangeMatch = requestRange.match(/bytes=(\d+)-(\d+)/);
|
|
1802
|
-
if (!rangeMatch)
|
|
1820
|
+
if (!rangeMatch) {
|
|
1803
1821
|
throw new Error(`Invalid Range format: ${requestRange}`);
|
|
1822
|
+
}
|
|
1804
1823
|
const expectedStart = Number.parseInt(rangeMatch[1]);
|
|
1805
1824
|
const expectedEnd = Number.parseInt(rangeMatch[2]);
|
|
1806
1825
|
if (start !== expectedStart) {
|
|
@@ -1955,8 +1974,9 @@ class S3TransferManagerBase {
|
|
|
1955
1974
|
const uploadPart = async () => {
|
|
1956
1975
|
while (true) {
|
|
1957
1976
|
const partNumber = nextPartNumber++;
|
|
1958
|
-
if (partNumber > expectedPartsCount)
|
|
1977
|
+
if (partNumber > expectedPartsCount) {
|
|
1959
1978
|
return;
|
|
1979
|
+
}
|
|
1960
1980
|
if (uploadAbortSignal.aborted) {
|
|
1961
1981
|
throw Object.assign(new Error("Upload aborted due to part failure."), { name: "AbortError" });
|
|
1962
1982
|
}
|
|
@@ -1968,16 +1988,21 @@ class S3TransferManagerBase {
|
|
|
1968
1988
|
PartNumber: partNumber,
|
|
1969
1989
|
ETag: partResponse.ETag,
|
|
1970
1990
|
};
|
|
1971
|
-
if (partResponse.ChecksumCRC32)
|
|
1991
|
+
if (partResponse.ChecksumCRC32) {
|
|
1972
1992
|
completedPart.ChecksumCRC32 = partResponse.ChecksumCRC32;
|
|
1973
|
-
|
|
1993
|
+
}
|
|
1994
|
+
if (partResponse.ChecksumCRC32C) {
|
|
1974
1995
|
completedPart.ChecksumCRC32C = partResponse.ChecksumCRC32C;
|
|
1975
|
-
|
|
1996
|
+
}
|
|
1997
|
+
if (partResponse.ChecksumCRC64NVME) {
|
|
1976
1998
|
completedPart.ChecksumCRC64NVME = partResponse.ChecksumCRC64NVME;
|
|
1977
|
-
|
|
1999
|
+
}
|
|
2000
|
+
if (partResponse.ChecksumSHA1) {
|
|
1978
2001
|
completedPart.ChecksumSHA1 = partResponse.ChecksumSHA1;
|
|
1979
|
-
|
|
2002
|
+
}
|
|
2003
|
+
if (partResponse.ChecksumSHA256) {
|
|
1980
2004
|
completedPart.ChecksumSHA256 = partResponse.ChecksumSHA256;
|
|
2005
|
+
}
|
|
1981
2006
|
completedParts.push(completedPart);
|
|
1982
2007
|
bytesUploaded += length;
|
|
1983
2008
|
if (emitEvents) {
|
|
@@ -2013,12 +2038,15 @@ class S3TransferManagerBase {
|
|
|
2013
2038
|
};
|
|
2014
2039
|
if (hasFullObjectChecksum) {
|
|
2015
2040
|
completeRequest.ChecksumType = "FULL_OBJECT";
|
|
2016
|
-
if (request.ChecksumCRC32)
|
|
2041
|
+
if (request.ChecksumCRC32) {
|
|
2017
2042
|
completeRequest.ChecksumCRC32 = request.ChecksumCRC32;
|
|
2018
|
-
|
|
2043
|
+
}
|
|
2044
|
+
if (request.ChecksumCRC32C) {
|
|
2019
2045
|
completeRequest.ChecksumCRC32C = request.ChecksumCRC32C;
|
|
2020
|
-
|
|
2046
|
+
}
|
|
2047
|
+
if (request.ChecksumCRC64NVME) {
|
|
2021
2048
|
completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
|
|
2049
|
+
}
|
|
2022
2050
|
}
|
|
2023
2051
|
try {
|
|
2024
2052
|
return await this.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
|
|
@@ -2099,16 +2127,21 @@ class S3TransferManagerBase {
|
|
|
2099
2127
|
PartNumber: dataPart.partNumber,
|
|
2100
2128
|
ETag: partResponse.ETag,
|
|
2101
2129
|
};
|
|
2102
|
-
if (partResponse.ChecksumCRC32)
|
|
2130
|
+
if (partResponse.ChecksumCRC32) {
|
|
2103
2131
|
completedPart.ChecksumCRC32 = partResponse.ChecksumCRC32;
|
|
2104
|
-
|
|
2132
|
+
}
|
|
2133
|
+
if (partResponse.ChecksumCRC32C) {
|
|
2105
2134
|
completedPart.ChecksumCRC32C = partResponse.ChecksumCRC32C;
|
|
2106
|
-
|
|
2135
|
+
}
|
|
2136
|
+
if (partResponse.ChecksumCRC64NVME) {
|
|
2107
2137
|
completedPart.ChecksumCRC64NVME = partResponse.ChecksumCRC64NVME;
|
|
2108
|
-
|
|
2138
|
+
}
|
|
2139
|
+
if (partResponse.ChecksumSHA1) {
|
|
2109
2140
|
completedPart.ChecksumSHA1 = partResponse.ChecksumSHA1;
|
|
2110
|
-
|
|
2141
|
+
}
|
|
2142
|
+
if (partResponse.ChecksumSHA256) {
|
|
2111
2143
|
completedPart.ChecksumSHA256 = partResponse.ChecksumSHA256;
|
|
2144
|
+
}
|
|
2112
2145
|
completedParts.push(completedPart);
|
|
2113
2146
|
const partLength = byteLength(dataPart.data);
|
|
2114
2147
|
bytesUploaded += partLength;
|
|
@@ -2145,12 +2178,15 @@ class S3TransferManagerBase {
|
|
|
2145
2178
|
};
|
|
2146
2179
|
if (hasFullObjectChecksum) {
|
|
2147
2180
|
completeRequest.ChecksumType = "FULL_OBJECT";
|
|
2148
|
-
if (request.ChecksumCRC32)
|
|
2181
|
+
if (request.ChecksumCRC32) {
|
|
2149
2182
|
completeRequest.ChecksumCRC32 = request.ChecksumCRC32;
|
|
2150
|
-
|
|
2183
|
+
}
|
|
2184
|
+
if (request.ChecksumCRC32C) {
|
|
2151
2185
|
completeRequest.ChecksumCRC32C = request.ChecksumCRC32C;
|
|
2152
|
-
|
|
2186
|
+
}
|
|
2187
|
+
if (request.ChecksumCRC64NVME) {
|
|
2153
2188
|
completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
|
|
2189
|
+
}
|
|
2154
2190
|
}
|
|
2155
2191
|
try {
|
|
2156
2192
|
return await this.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
|
|
@@ -2181,8 +2217,9 @@ class S3TransferManagerBase {
|
|
|
2181
2217
|
for await (const entry of dir) {
|
|
2182
2218
|
const fullPath = join(entry.parentPath ?? entry.path ?? source, entry.name);
|
|
2183
2219
|
if (entry.isSymbolicLink()) {
|
|
2184
|
-
if (!options.followSymbolicLinks)
|
|
2220
|
+
if (!options.followSymbolicLinks) {
|
|
2185
2221
|
continue;
|
|
2222
|
+
}
|
|
2186
2223
|
const realPath = await realpath(fullPath);
|
|
2187
2224
|
const linkStat = await stat(realPath);
|
|
2188
2225
|
if (linkStat.isDirectory()) {
|
|
@@ -2193,8 +2230,9 @@ class S3TransferManagerBase {
|
|
|
2193
2230
|
yield* this.traverseDirectory(realPath, options);
|
|
2194
2231
|
continue;
|
|
2195
2232
|
}
|
|
2196
|
-
if (!linkStat.isFile())
|
|
2233
|
+
if (!linkStat.isFile()) {
|
|
2197
2234
|
continue;
|
|
2235
|
+
}
|
|
2198
2236
|
}
|
|
2199
2237
|
else if (!entry.isFile()) {
|
|
2200
2238
|
continue;
|
|
@@ -2207,8 +2245,9 @@ class S3TransferManagerBase {
|
|
|
2207
2245
|
if (sep !== "/") {
|
|
2208
2246
|
relativePath = relativePath.split(sep).join("/");
|
|
2209
2247
|
}
|
|
2210
|
-
if (!s3Prefix)
|
|
2248
|
+
if (!s3Prefix) {
|
|
2211
2249
|
return relativePath;
|
|
2250
|
+
}
|
|
2212
2251
|
const normalizedPrefix = s3Prefix.endsWith("/") ? s3Prefix : s3Prefix + "/";
|
|
2213
2252
|
return normalizedPrefix + relativePath;
|
|
2214
2253
|
}
|
|
@@ -2235,8 +2274,9 @@ function matchesFilterRegExp(filter, value) {
|
|
|
2235
2274
|
return filter.test(value);
|
|
2236
2275
|
}
|
|
2237
2276
|
function parseContentRangeStart(contentRange) {
|
|
2238
|
-
if (!contentRange)
|
|
2277
|
+
if (!contentRange) {
|
|
2239
2278
|
return undefined;
|
|
2279
|
+
}
|
|
2240
2280
|
const match = contentRange.match(/^bytes\s+(\d+)-/);
|
|
2241
2281
|
return match ? Number(match[1]) : undefined;
|
|
2242
2282
|
}
|
|
@@ -88,8 +88,9 @@ export async function* iterateStreams(promises, eventListeners) {
|
|
|
88
88
|
}
|
|
89
89
|
export async function destroyStreams(promises) {
|
|
90
90
|
await Promise.all(promises.map(async (streamPromise) => {
|
|
91
|
-
if (!streamPromise)
|
|
91
|
+
if (!streamPromise) {
|
|
92
92
|
return;
|
|
93
|
+
}
|
|
93
94
|
return streamPromise
|
|
94
95
|
.then((stream) => {
|
|
95
96
|
if (isReadableStream(stream)) {
|
|
@@ -87,8 +87,9 @@ export async function* iterateStreams(promises, eventListeners) {
|
|
|
87
87
|
}
|
|
88
88
|
export async function destroyStreams(promises) {
|
|
89
89
|
await Promise.all(promises.map(async (streamPromise) => {
|
|
90
|
-
if (!streamPromise)
|
|
90
|
+
if (!streamPromise) {
|
|
91
91
|
return;
|
|
92
|
+
}
|
|
92
93
|
return streamPromise
|
|
93
94
|
.then((stream) => {
|
|
94
95
|
if (stream instanceof Readable) {
|
|
@@ -8,8 +8,9 @@ export class OrderedPartQueue {
|
|
|
8
8
|
this.totalParts = totalParts;
|
|
9
9
|
}
|
|
10
10
|
enqueue(rangeIndex, buffer, byteLength) {
|
|
11
|
-
if (this.error)
|
|
11
|
+
if (this.error) {
|
|
12
12
|
return;
|
|
13
|
+
}
|
|
13
14
|
if (rangeIndex === this.nextExpected && this.waitingConsumer) {
|
|
14
15
|
const consumer = this.waitingConsumer;
|
|
15
16
|
this.waitingConsumer = null;
|
|
@@ -7,10 +7,12 @@ import { Readable } from "node:stream";
|
|
|
7
7
|
import { Worker } from "node:worker_threads";
|
|
8
8
|
function buildDownloadError(msg) {
|
|
9
9
|
const error = new Error(msg.error);
|
|
10
|
-
if (msg.name)
|
|
10
|
+
if (msg.name) {
|
|
11
11
|
error.name = msg.name;
|
|
12
|
-
|
|
12
|
+
}
|
|
13
|
+
if (msg.code) {
|
|
13
14
|
error.code = msg.code;
|
|
15
|
+
}
|
|
14
16
|
return error;
|
|
15
17
|
}
|
|
16
18
|
export function createEmptyReadable() {
|
|
@@ -75,10 +77,12 @@ export class WorkerHttpHandler {
|
|
|
75
77
|
return { workerPath: submodulePath };
|
|
76
78
|
}
|
|
77
79
|
ensureInitialized() {
|
|
78
|
-
if (this.initialized)
|
|
80
|
+
if (this.initialized) {
|
|
79
81
|
return Promise.resolve();
|
|
80
|
-
|
|
82
|
+
}
|
|
83
|
+
if (this.initPromise) {
|
|
81
84
|
return this.initPromise;
|
|
85
|
+
}
|
|
82
86
|
this.initPromise = new Promise((resolve, reject) => {
|
|
83
87
|
const { workerPath, workerOptions } = this.resolveWorkerConfig();
|
|
84
88
|
let readyCount = 0;
|
|
@@ -14,8 +14,9 @@ const DNS_TTL_MS = 1000;
|
|
|
14
14
|
const dnsCache = new Map();
|
|
15
15
|
const MAX_ERROR_BODY_BYTES = 16 * 1024;
|
|
16
16
|
const readErrorBodyBytes = async (body) => {
|
|
17
|
-
if (!body)
|
|
17
|
+
if (!body) {
|
|
18
18
|
return undefined;
|
|
19
|
+
}
|
|
19
20
|
try {
|
|
20
21
|
const chunks = [];
|
|
21
22
|
let total = 0;
|
|
@@ -23,8 +24,9 @@ const readErrorBodyBytes = async (body) => {
|
|
|
23
24
|
const buf = typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk);
|
|
24
25
|
chunks.push(buf);
|
|
25
26
|
total += buf.length;
|
|
26
|
-
if (total >= MAX_ERROR_BODY_BYTES)
|
|
27
|
+
if (total >= MAX_ERROR_BODY_BYTES) {
|
|
27
28
|
break;
|
|
29
|
+
}
|
|
28
30
|
}
|
|
29
31
|
return concatBytes(chunks);
|
|
30
32
|
}
|
|
@@ -147,15 +149,18 @@ async function finalizeChecksumToBase64(checksum) {
|
|
|
147
149
|
return Buffer.from(bytes).toString("base64");
|
|
148
150
|
}
|
|
149
151
|
function parseContentRange(contentRange) {
|
|
150
|
-
if (!contentRange)
|
|
152
|
+
if (!contentRange) {
|
|
151
153
|
return undefined;
|
|
154
|
+
}
|
|
152
155
|
const match = contentRange.match(/^bytes\s+(\d+)-(\d+)\//);
|
|
153
|
-
if (!match)
|
|
156
|
+
if (!match) {
|
|
154
157
|
return undefined;
|
|
158
|
+
}
|
|
155
159
|
const start = Number(match[1]);
|
|
156
160
|
const end = Number(match[2]);
|
|
157
|
-
if (end < start)
|
|
161
|
+
if (end < start) {
|
|
158
162
|
return undefined;
|
|
163
|
+
}
|
|
159
164
|
return { start, end };
|
|
160
165
|
}
|
|
161
166
|
function getChecksumHeaderValue(headers, algorithm) {
|
|
@@ -196,8 +201,9 @@ if (parentPort) {
|
|
|
196
201
|
let read = 0;
|
|
197
202
|
while (read < length) {
|
|
198
203
|
const n = readSync(fd, buffer, read, length - read, offset + read);
|
|
199
|
-
if (n === 0)
|
|
204
|
+
if (n === 0) {
|
|
200
205
|
break;
|
|
206
|
+
}
|
|
201
207
|
read += n;
|
|
202
208
|
}
|
|
203
209
|
return buffer;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aws-sdk/lib-transfer-manager",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "S3 Transfer Manager for AWS SDK JS v3",
|
|
5
5
|
"homepage": "https://github.com/aws/aws-sdk-js-v3/tree/main/preview/lib-transfer-manager",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -85,16 +85,16 @@
|
|
|
85
85
|
"test:browser:watch": "yarn g:vitest watch -c vitest.config.browser.mts"
|
|
86
86
|
},
|
|
87
87
|
"dependencies": {
|
|
88
|
-
"@aws-sdk/checksums": "^3.1001.
|
|
89
|
-
"@aws-sdk/core": "^3.978.
|
|
90
|
-
"@smithy/core": "^3.
|
|
91
|
-
"@smithy/node-http-handler": "^4.
|
|
88
|
+
"@aws-sdk/checksums": "^3.1001.7",
|
|
89
|
+
"@aws-sdk/core": "^3.978.7",
|
|
90
|
+
"@smithy/core": "^3.37.0",
|
|
91
|
+
"@smithy/node-http-handler": "^4.13.0",
|
|
92
92
|
"@smithy/types": "^4.19.0",
|
|
93
93
|
"tslib": "^2.6.2"
|
|
94
94
|
},
|
|
95
95
|
"devDependencies": {
|
|
96
|
-
"@aws-sdk/client-s3": "3.
|
|
97
|
-
"@aws-sdk/xhr-http-handler": "3.
|
|
96
|
+
"@aws-sdk/client-s3": "3.1159.0",
|
|
97
|
+
"@aws-sdk/xhr-http-handler": "3.1159.0",
|
|
98
98
|
"@tsconfig/recommended": "1.0.1",
|
|
99
99
|
"@types/node": "^20.14.8",
|
|
100
100
|
"concurrently": "7.0.0",
|
|
@@ -103,7 +103,7 @@
|
|
|
103
103
|
"typescript": "~7.0.2"
|
|
104
104
|
},
|
|
105
105
|
"peerDependencies": {
|
|
106
|
-
"@aws-sdk/client-s3": "^3.
|
|
106
|
+
"@aws-sdk/client-s3": "^3.1159.0"
|
|
107
107
|
},
|
|
108
108
|
"engines": {
|
|
109
109
|
"node": ">=20.0.0"
|