@aws-sdk/lib-transfer-manager 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +303 -2
  2. package/dist-cjs/index.js +2 -0
  3. package/dist-cjs/submodules/transfer-manager/index.js +3166 -0
  4. package/dist-cjs/submodules/worker/index.js +579 -0
  5. package/dist-es/index.js +1 -0
  6. package/dist-es/submodules/transfer-manager/S3TransferManager.js +2229 -0
  7. package/dist-es/submodules/transfer-manager/chunker.js +115 -0
  8. package/dist-es/submodules/transfer-manager/directory-transfer-utils.js +102 -0
  9. package/dist-es/submodules/transfer-manager/event-listener-types.js +1 -0
  10. package/dist-es/submodules/transfer-manager/file-manager.js +70 -0
  11. package/dist-es/submodules/transfer-manager/index.browser.js +11 -0
  12. package/dist-es/submodules/transfer-manager/index.js +11 -0
  13. package/dist-es/submodules/transfer-manager/join-streams.browser.js +101 -0
  14. package/dist-es/submodules/transfer-manager/join-streams.js +105 -0
  15. package/dist-es/submodules/transfer-manager/log-level.js +30 -0
  16. package/dist-es/submodules/transfer-manager/ordered-part-queue.js +57 -0
  17. package/dist-es/submodules/transfer-manager/types.js +1 -0
  18. package/dist-es/submodules/transfer-manager/worker-http-handler.browser.js +25 -0
  19. package/dist-es/submodules/transfer-manager/worker-http-handler.js +453 -0
  20. package/dist-es/submodules/worker/http-request-worker.js +578 -0
  21. package/dist-es/submodules/worker/index.js +1 -0
  22. package/dist-types/index.d.ts +2 -0
  23. package/dist-types/submodules/transfer-manager/S3TransferManager.d.ts +60 -0
  24. package/dist-types/submodules/transfer-manager/chunker.d.ts +7 -0
  25. package/dist-types/submodules/transfer-manager/directory-transfer-utils.d.ts +90 -0
  26. package/dist-types/submodules/transfer-manager/event-listener-types.d.ts +54 -0
  27. package/dist-types/submodules/transfer-manager/file-manager.d.ts +51 -0
  28. package/dist-types/submodules/transfer-manager/index.browser.d.ts +10 -0
  29. package/dist-types/submodules/transfer-manager/index.d.ts +10 -0
  30. package/dist-types/submodules/transfer-manager/join-streams.browser.d.ts +21 -0
  31. package/dist-types/submodules/transfer-manager/join-streams.d.ts +26 -0
  32. package/dist-types/submodules/transfer-manager/log-level.d.ts +13 -0
  33. package/dist-types/submodules/transfer-manager/ordered-part-queue.d.ts +73 -0
  34. package/dist-types/submodules/transfer-manager/types.d.ts +473 -0
  35. package/dist-types/submodules/transfer-manager/worker-http-handler.browser.d.ts +20 -0
  36. package/dist-types/submodules/transfer-manager/worker-http-handler.d.ts +410 -0
  37. package/dist-types/submodules/worker/http-request-worker.d.ts +1 -0
  38. package/dist-types/submodules/worker/index.d.ts +1 -0
  39. package/dist-types/ts3.4/index.d.ts +2 -0
  40. package/dist-types/ts3.4/submodules/transfer-manager/S3TransferManager.d.ts +38 -0
  41. package/dist-types/ts3.4/submodules/transfer-manager/chunker.d.ts +10 -0
  42. package/dist-types/ts3.4/submodules/transfer-manager/directory-transfer-utils.d.ts +23 -0
  43. package/dist-types/ts3.4/submodules/transfer-manager/event-listener-types.d.ts +12 -0
  44. package/dist-types/ts3.4/submodules/transfer-manager/file-manager.d.ts +9 -0
  45. package/dist-types/ts3.4/submodules/transfer-manager/index.browser.d.ts +3 -0
  46. package/dist-types/ts3.4/submodules/transfer-manager/index.d.ts +3 -0
  47. package/dist-types/ts3.4/submodules/transfer-manager/join-streams.browser.d.ts +13 -0
  48. package/dist-types/ts3.4/submodules/transfer-manager/join-streams.d.ts +13 -0
  49. package/dist-types/ts3.4/submodules/transfer-manager/log-level.d.ts +13 -0
  50. package/dist-types/ts3.4/submodules/transfer-manager/ordered-part-queue.d.ts +18 -0
  51. package/dist-types/ts3.4/submodules/transfer-manager/types.d.ts +184 -0
  52. package/dist-types/ts3.4/submodules/transfer-manager/worker-http-handler.browser.d.ts +17 -0
  53. package/dist-types/ts3.4/submodules/transfer-manager/worker-http-handler.d.ts +241 -0
  54. package/dist-types/ts3.4/submodules/worker/http-request-worker.d.ts +1 -0
  55. package/dist-types/ts3.4/submodules/worker/index.d.ts +1 -0
  56. package/package.json +108 -4
  57. package/transfer-manager.d.ts +5 -0
  58. package/transfer-manager.js +5 -0
  59. package/worker.d.ts +5 -0
  60. package/worker.js +5 -0
@@ -0,0 +1,2229 @@
1
+ import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client, UploadPartCommand, } from "@aws-sdk/client-s3";
2
+ import { isThrottlingError, isTransientError } from "@smithy/core/retry";
3
+ import { createReadStream, existsSync } from "node:fs";
4
+ import { open, opendir, realpath, stat } from "node:fs/promises";
5
+ import { join, relative, resolve, sep } from "node:path";
6
+ import { Readable } from "node:stream";
7
+ import { byteLength, getChunk } from "./chunker";
8
+ import { createDestinationDirectory, createLocalParentDirectories, deriveLocalPath, handleFailure, isFolderObject, Semaphore, validateDirectory, } from "./directory-transfer-utils";
9
+ import { FileManager } from "./file-manager";
10
+ import { LogLevel } from "./log-level";
11
+ import { OrderedPartQueue } from "./ordered-part-queue";
12
+ class S3TransferManagerBase {
13
+ static MIN_PART_SIZE = 5 * 1024 * 1024;
14
+ s3;
15
+ targetPartSizeBytes;
16
+ multipartUploadThresholdBytes;
17
+ requestChecksumCalculation;
18
+ responseChecksumValidation;
19
+ multipartDownloadType;
20
+ eventListeners;
21
+ abortCleanupFunctions = new WeakMap();
22
+ maxConcurrentDownloads;
23
+ maxConcurrentUploads;
24
+ workerThreadCount;
25
+ workerHttpHandler;
26
+ logger;
27
+ deps;
28
+ constructor(config, deps) {
29
+ this.deps = deps;
30
+ this.requestChecksumCalculation = config.requestChecksumCalculation ?? "WHEN_SUPPORTED";
31
+ this.responseChecksumValidation = config.responseChecksumValidation ?? "WHEN_SUPPORTED";
32
+ this.maxConcurrentUploads = config.maxConcurrentUploads ?? 32;
33
+ this.maxConcurrentDownloads = config.maxConcurrentDownloads ?? 32;
34
+ this.workerThreadCount = config.workerThreadCount ?? this.deps.defaultWorkerCount();
35
+ this.logger = config.logger ?? new LogLevel("warn");
36
+ this.s3 =
37
+ config.s3 ??
38
+ new S3Client({
39
+ requestChecksumCalculation: this.requestChecksumCalculation,
40
+ responseChecksumValidation: this.responseChecksumValidation,
41
+ });
42
+ if (this.workerThreadCount > 1) {
43
+ this.workerHttpHandler = new this.deps.WorkerHttpHandler({
44
+ workerThreadCount: this.workerThreadCount,
45
+ maxConcurrentUploads: this.maxConcurrentUploads,
46
+ maxConcurrentDownloads: this.maxConcurrentDownloads,
47
+ useODirect: config.useODirect,
48
+ logger: this.logger,
49
+ });
50
+ if (this.s3.config) {
51
+ this.s3.config.requestHandler = this.workerHttpHandler;
52
+ }
53
+ }
54
+ this.targetPartSizeBytes = config.targetPartSizeBytes ?? 8 * 1024 * 1024;
55
+ this.multipartUploadThresholdBytes = config.multipartUploadThresholdBytes ?? 16 * 1024 * 1024;
56
+ this.multipartDownloadType = config.multipartDownloadType ?? "PART";
57
+ this.eventListeners = {
58
+ transferInitiated: config.eventListeners?.transferInitiated ?? [],
59
+ bytesTransferred: config.eventListeners?.bytesTransferred ?? [],
60
+ transferComplete: config.eventListeners?.transferComplete ?? [],
61
+ transferFailed: config.eventListeners?.transferFailed ?? [],
62
+ };
63
+ this.validateConfig();
64
+ }
65
+ addEventListener(type, callback, options) {
66
+ const eventType = type;
67
+ const listeners = this.eventListeners[eventType];
68
+ if (!listeners) {
69
+ throw new Error(`Unknown event type: ${eventType}`);
70
+ }
71
+ const once = typeof options !== "boolean" && options?.once;
72
+ const signal = typeof options !== "boolean" ? options?.signal : undefined;
73
+ let updatedCallback = callback;
74
+ if (signal?.aborted) {
75
+ return;
76
+ }
77
+ if (signal) {
78
+ const removeListenerAfterAbort = () => {
79
+ this.removeEventListener(eventType, updatedCallback);
80
+ this.abortCleanupFunctions.delete(signal);
81
+ };
82
+ this.abortCleanupFunctions.set(signal, () => signal.removeEventListener("abort", removeListenerAfterAbort));
83
+ signal.addEventListener("abort", removeListenerAfterAbort, {
84
+ once: true,
85
+ });
86
+ }
87
+ if (once) {
88
+ updatedCallback = (event) => {
89
+ if (typeof callback === "function") {
90
+ callback(event);
91
+ }
92
+ else {
93
+ callback.handleEvent(event);
94
+ }
95
+ this.removeEventListener(eventType, updatedCallback);
96
+ };
97
+ }
98
+ listeners.push(updatedCallback);
99
+ }
100
+ dispatchEvent(event) {
101
+ const eventType = event.type;
102
+ const listeners = this.eventListeners[eventType];
103
+ if (listeners) {
104
+ for (const listener of listeners) {
105
+ if (typeof listener === "function") {
106
+ listener(event);
107
+ }
108
+ else {
109
+ listener.handleEvent(event);
110
+ }
111
+ }
112
+ }
113
+ return true;
114
+ }
115
+ removeEventListener(type, callback, options) {
116
+ const eventType = type;
117
+ const listeners = this.eventListeners[eventType];
118
+ if (listeners) {
119
+ if (eventType === "transferInitiated" ||
120
+ eventType === "bytesTransferred" ||
121
+ eventType === "transferFailed" ||
122
+ eventType === "transferComplete") {
123
+ const eventListener = callback;
124
+ let index = listeners.indexOf(eventListener);
125
+ while (index !== -1) {
126
+ listeners.splice(index, 1);
127
+ index = listeners.indexOf(eventListener);
128
+ }
129
+ }
130
+ else {
131
+ throw new Error(`Unknown event type: ${type}`);
132
+ }
133
+ }
134
+ }
135
+ async upload(request, transferOptions) {
136
+ return this.uploadInternal(request, transferOptions, true);
137
+ }
138
+ async uploadInternal(request, transferOptions, emitEvents = true) {
139
+ const totalContentLength = request.ContentLength ?? byteLength(request.Body);
140
+ if (totalContentLength === undefined) {
141
+ throw new Error("Unable to determine content length for upload");
142
+ }
143
+ this.checkAborted(transferOptions);
144
+ if (emitEvents) {
145
+ this.addEventListeners(transferOptions?.eventListeners);
146
+ this.dispatchTransferInitiatedEvent(request, totalContentLength);
147
+ }
148
+ const removeLocalEventListeners = () => {
149
+ this.removeEventListeners(transferOptions?.eventListeners);
150
+ if (transferOptions?.abortSignal) {
151
+ this.abortCleanupFunctions.get(transferOptions.abortSignal)?.();
152
+ this.abortCleanupFunctions.delete(transferOptions.abortSignal);
153
+ }
154
+ };
155
+ try {
156
+ let response;
157
+ if (totalContentLength < this.multipartUploadThresholdBytes) {
158
+ response = await this.uploadSinglePart(request, totalContentLength, transferOptions, emitEvents);
159
+ }
160
+ else if (this.workerThreadCount > 1 && this.isFileBody(request.Body)) {
161
+ response = await this.threadedUploadInPartsFromFile(request, totalContentLength, transferOptions, emitEvents);
162
+ }
163
+ else if (this.workerThreadCount > 1 && this.isInMemoryBody(request.Body)) {
164
+ if (typeof request.Body === "string") {
165
+ request = {
166
+ ...request,
167
+ Body: new TextEncoder().encode(request.Body),
168
+ };
169
+ }
170
+ response = await this.threadedUploadInParts(request, totalContentLength, transferOptions, emitEvents);
171
+ }
172
+ else {
173
+ response = await this.uploadInParts(request, totalContentLength, transferOptions, emitEvents);
174
+ }
175
+ if (emitEvents) {
176
+ this.dispatchEvent(Object.assign(new Event("transferComplete"), {
177
+ request,
178
+ response,
179
+ snapshot: {
180
+ transferredBytes: totalContentLength,
181
+ totalBytes: totalContentLength,
182
+ },
183
+ }));
184
+ }
185
+ removeLocalEventListeners();
186
+ return response;
187
+ }
188
+ catch (error) {
189
+ if (emitEvents) {
190
+ this.dispatchTransferFailedEvent(request, totalContentLength, error);
191
+ }
192
+ removeLocalEventListeners();
193
+ throw error;
194
+ }
195
+ }
196
+ async download(request, transferOptions) {
197
+ const partNumber = request.PartNumber;
198
+ if (typeof partNumber === "number") {
199
+ throw new Error("partNumber included: S3 Transfer Manager does not support downloads for specific parts. Use GetObjectCommand instead");
200
+ }
201
+ const metadata = {};
202
+ const streams = [];
203
+ const requests = [];
204
+ let totalSize;
205
+ this.checkAborted(transferOptions);
206
+ this.addEventListeners(transferOptions?.eventListeners);
207
+ const checksumValidationEnabled = request.ChecksumMode === "ENABLED" || this.responseChecksumValidation === "WHEN_SUPPORTED";
208
+ let onStreamConsumed;
209
+ const removeLocalEventListeners = () => {
210
+ this.removeEventListeners(transferOptions?.eventListeners);
211
+ if (transferOptions?.abortSignal) {
212
+ this.abortCleanupFunctions.get(transferOptions.abortSignal)?.();
213
+ this.abortCleanupFunctions.delete(transferOptions.abortSignal);
214
+ }
215
+ };
216
+ const resolvedTransferOptions = transferOptions ?? {};
217
+ const downloadArgs = [
218
+ request,
219
+ resolvedTransferOptions,
220
+ streams,
221
+ requests,
222
+ metadata,
223
+ checksumValidationEnabled,
224
+ ];
225
+ try {
226
+ let responseMetadata;
227
+ if (this.multipartDownloadType === "PART") {
228
+ if (this.workerHttpHandler) {
229
+ responseMetadata = await this.downloadByPartWithWorkers(...downloadArgs);
230
+ }
231
+ else {
232
+ responseMetadata = await this.downloadByPart(...downloadArgs);
233
+ }
234
+ }
235
+ else if (this.multipartDownloadType === "RANGE") {
236
+ if (this.workerHttpHandler && !request.Range) {
237
+ responseMetadata = await this.downloadByRangeWithWorkers(...downloadArgs);
238
+ }
239
+ else {
240
+ responseMetadata = await this.downloadByRange(...downloadArgs);
241
+ }
242
+ }
243
+ totalSize = responseMetadata.totalSize;
244
+ onStreamConsumed = responseMetadata.onStreamConsumed;
245
+ }
246
+ catch (error) {
247
+ await this.deps.destroyStreams(streams);
248
+ removeLocalEventListeners();
249
+ throw error;
250
+ }
251
+ const response = {
252
+ ...metadata,
253
+ Body: await this.deps.joinStreams(streams, {
254
+ onBytes: (byteLength, index) => {
255
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
256
+ request: requests[index],
257
+ snapshot: {
258
+ transferredBytes: byteLength,
259
+ totalBytes: totalSize,
260
+ },
261
+ }));
262
+ },
263
+ onCompletion: (byteLength, index) => {
264
+ this.dispatchEvent(Object.assign(new Event("transferComplete"), {
265
+ request: requests[index],
266
+ response,
267
+ snapshot: {
268
+ transferredBytes: byteLength,
269
+ totalBytes: totalSize,
270
+ },
271
+ }));
272
+ removeLocalEventListeners();
273
+ },
274
+ onFailure: (error, index) => {
275
+ this.dispatchEvent(Object.assign(new Event("transferFailed"), {
276
+ request: requests[index],
277
+ snapshot: {
278
+ transferredBytes: error,
279
+ totalBytes: totalSize,
280
+ },
281
+ }));
282
+ removeLocalEventListeners();
283
+ },
284
+ onStreamConsumed,
285
+ }),
286
+ };
287
+ return response;
288
+ }
289
+ async downloadToFile(request, transferOptions) {
290
+ return this.downloadToFileInternal(request, transferOptions, true);
291
+ }
292
+ async downloadToFileInternal(request, transferOptions, emitEvents = true) {
293
+ if (!request.destination || request.destination === "") {
294
+ throw new Error("destination must be a non-empty string");
295
+ }
296
+ const resolvedDestination = resolve(request.destination);
297
+ if (request.failIfExists && existsSync(resolvedDestination)) {
298
+ throw new Error(`File already exists at destination: ${resolvedDestination}`);
299
+ }
300
+ this.checkAborted(transferOptions);
301
+ if (emitEvents) {
302
+ this.addEventListeners(transferOptions?.eventListeners);
303
+ }
304
+ const removeLocalEventListeners = () => {
305
+ this.removeEventListeners(transferOptions?.eventListeners);
306
+ if (transferOptions?.abortSignal) {
307
+ this.abortCleanupFunctions.get(transferOptions.abortSignal)?.();
308
+ this.abortCleanupFunctions.delete(transferOptions.abortSignal);
309
+ }
310
+ };
311
+ const checksumValidationEnabled = request.ChecksumMode === "ENABLED" || this.responseChecksumValidation === "WHEN_SUPPORTED";
312
+ const { destination, failIfExists, ...getObjectFields } = request;
313
+ if (this.multipartDownloadType === "RANGE") {
314
+ try {
315
+ const result = await this.downloadToFileByRange(request, resolvedDestination, transferOptions, emitEvents);
316
+ removeLocalEventListeners();
317
+ return result;
318
+ }
319
+ catch (error) {
320
+ removeLocalEventListeners();
321
+ throw error;
322
+ }
323
+ }
324
+ const initialPartRequest = {
325
+ ...getObjectFields,
326
+ PartNumber: 1,
327
+ ...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
328
+ };
329
+ let initialResponse;
330
+ try {
331
+ initialResponse = await this.s3.send(new GetObjectCommand(initialPartRequest), transferOptions);
332
+ }
333
+ catch (error) {
334
+ removeLocalEventListeners();
335
+ throw error;
336
+ }
337
+ if (initialResponse.Body && initialResponse.Body instanceof Readable) {
338
+ initialResponse.Body.on("error", () => { });
339
+ }
340
+ const totalSize = initialResponse.ContentRange
341
+ ? Number.parseInt(initialResponse.ContentRange.split("/")[1])
342
+ : (initialResponse.ContentLength ?? 0);
343
+ const partsCount = initialResponse.PartsCount ?? 1;
344
+ const eTag = initialResponse.ETag ?? undefined;
345
+ if (emitEvents) {
346
+ this.dispatchTransferInitiatedEvent(request, totalSize);
347
+ }
348
+ const fileManager = new FileManager();
349
+ let tempFilePath;
350
+ try {
351
+ if (partsCount === 1) {
352
+ tempFilePath = await fileManager.createTempFile(resolvedDestination, totalSize);
353
+ fileManager.registerCleanupHandler(tempFilePath);
354
+ await this.writeResponseBodyToFile(initialResponse.Body, tempFilePath, initialResponse.ContentRange, 0);
355
+ if (emitEvents) {
356
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
357
+ request,
358
+ snapshot: {
359
+ transferredBytes: totalSize,
360
+ totalBytes: totalSize,
361
+ },
362
+ }));
363
+ }
364
+ await fileManager.atomicRename(tempFilePath, resolvedDestination);
365
+ fileManager.unregisterCleanupHandler(tempFilePath);
366
+ const response = this.buildDownloadToFileResponse(initialResponse, totalSize, request, emitEvents);
367
+ removeLocalEventListeners();
368
+ return response;
369
+ }
370
+ tempFilePath = await fileManager.createTempFile(resolvedDestination, totalSize);
371
+ fileManager.registerCleanupHandler(tempFilePath);
372
+ let transferredBytes = 0;
373
+ const partSize = initialResponse.ContentLength ?? 0;
374
+ await this.writeResponseBodyToFile(initialResponse.Body, tempFilePath, initialResponse.ContentRange, 0);
375
+ transferredBytes += partSize;
376
+ if (emitEvents) {
377
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
378
+ request,
379
+ snapshot: {
380
+ transferredBytes,
381
+ totalBytes: totalSize,
382
+ },
383
+ }));
384
+ }
385
+ const partResults = [];
386
+ partResults.push({ partNumber: 1, bytesWritten: partSize });
387
+ const semaphore = new Semaphore(this.maxConcurrentDownloads);
388
+ const abortController = new AbortController();
389
+ let failed = false;
390
+ let failureError;
391
+ const userSignal = transferOptions?.abortSignal;
392
+ if (userSignal) {
393
+ const onUserAbort = () => abortController.abort();
394
+ userSignal.addEventListener("abort", onUserAbort, { once: true });
395
+ }
396
+ const remainingPartTasks = [];
397
+ for (let part = 2; part <= partsCount; part++) {
398
+ if (failed || abortController.signal.aborted)
399
+ break;
400
+ this.checkAborted(transferOptions);
401
+ await semaphore.acquire();
402
+ if (failed || abortController.signal.aborted) {
403
+ semaphore.release();
404
+ break;
405
+ }
406
+ const partNumber = part;
407
+ const partOffset = (partNumber - 1) * partSize;
408
+ const expectedLength = Math.min(partSize, totalSize - partOffset);
409
+ const partGetRequest = {
410
+ ...getObjectFields,
411
+ PartNumber: partNumber,
412
+ IfMatch: eTag,
413
+ ...(checksumValidationEnabled && {
414
+ ChecksumMode: "ENABLED",
415
+ }),
416
+ };
417
+ const task = (async () => {
418
+ try {
419
+ if (this.workerHttpHandler) {
420
+ const resultToken = `download-${partNumber}-${Date.now()}`;
421
+ const downloadOptions = {
422
+ filePath: tempFilePath,
423
+ offset: partOffset,
424
+ expectedLength,
425
+ resultToken,
426
+ };
427
+ await this.s3.send(new GetObjectCommand(partGetRequest), {
428
+ ...transferOptions,
429
+ downloadDataToFile: downloadOptions,
430
+ });
431
+ const downloadResult = this.s3.config.requestHandler.getDownloadResult(resultToken);
432
+ if (!downloadResult) {
433
+ throw new Error(`Missing download result for part ${partNumber}`);
434
+ }
435
+ partResults.push({
436
+ partNumber,
437
+ bytesWritten: downloadResult.bytesWritten,
438
+ });
439
+ }
440
+ else {
441
+ const partResponse = await this.s3.send(new GetObjectCommand(partGetRequest), transferOptions);
442
+ if (partResponse.Body && partResponse.Body instanceof Readable) {
443
+ partResponse.Body.on("error", () => { });
444
+ }
445
+ this.validatePartDownload(partResponse.ContentRange, partNumber, partSize);
446
+ await this.writeResponseBodyToFile(partResponse.Body, tempFilePath, partResponse.ContentRange, partOffset);
447
+ partResults.push({
448
+ partNumber,
449
+ bytesWritten: partResponse.ContentLength ?? expectedLength,
450
+ });
451
+ }
452
+ transferredBytes += expectedLength;
453
+ if (emitEvents) {
454
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
455
+ request: partGetRequest,
456
+ snapshot: {
457
+ transferredBytes,
458
+ totalBytes: totalSize,
459
+ },
460
+ }));
461
+ }
462
+ }
463
+ catch (error) {
464
+ if (!failed) {
465
+ failed = true;
466
+ failureError = error;
467
+ abortController.abort();
468
+ }
469
+ }
470
+ finally {
471
+ semaphore.release();
472
+ }
473
+ })();
474
+ remainingPartTasks.push(task);
475
+ }
476
+ await Promise.allSettled(remainingPartTasks);
477
+ if (abortController.signal.aborted && !failed) {
478
+ const abortError = Object.assign(new Error("Transfer aborted."), {
479
+ name: "AbortError",
480
+ });
481
+ await this.handleDownloadCleanup(fileManager, tempFilePath, request, totalSize, abortError, emitEvents);
482
+ removeLocalEventListeners();
483
+ throw abortError;
484
+ }
485
+ if (failed) {
486
+ await this.handleDownloadCleanup(fileManager, tempFilePath, request, totalSize, failureError, emitEvents);
487
+ removeLocalEventListeners();
488
+ throw failureError;
489
+ }
490
+ if (partResults.length !== partsCount) {
491
+ const error = new Error(`The number of parts downloaded (${partResults.length}) does not match the expected number (${partsCount})`);
492
+ await this.handleDownloadCleanup(fileManager, tempFilePath, request, totalSize, error, emitEvents);
493
+ removeLocalEventListeners();
494
+ throw error;
495
+ }
496
+ const totalBytesWritten = partResults.reduce((sum, p) => sum + p.bytesWritten, 0);
497
+ if (totalBytesWritten !== totalSize) {
498
+ const error = new Error(`Total bytes written (${totalBytesWritten}) does not match expected object size (${totalSize})`);
499
+ await this.handleDownloadCleanup(fileManager, tempFilePath, request, totalSize, error, emitEvents);
500
+ removeLocalEventListeners();
501
+ throw error;
502
+ }
503
+ await fileManager.atomicRename(tempFilePath, resolvedDestination);
504
+ fileManager.unregisterCleanupHandler(tempFilePath);
505
+ const response = this.buildDownloadToFileResponse(initialResponse, totalSize, request, emitEvents);
506
+ removeLocalEventListeners();
507
+ return response;
508
+ }
509
+ catch (error) {
510
+ if (tempFilePath) {
511
+ await fileManager.deleteTempFile(tempFilePath).catch(() => { });
512
+ fileManager.unregisterCleanupHandler(tempFilePath);
513
+ }
514
+ removeLocalEventListeners();
515
+ throw error;
516
+ }
517
+ }
518
+ async downloadToFileByRange(request, resolvedDestination, transferOptions, emitEvents) {
519
+ const { destination, failIfExists, ...getObjectFields } = request;
520
+ const checksumValidationEnabled = request.ChecksumMode === "ENABLED" || this.responseChecksumValidation === "WHEN_SUPPORTED";
521
+ const fileManager = new FileManager();
522
+ let tempFilePath;
523
+ try {
524
+ const rangeSize = this.targetPartSizeBytes;
525
+ const initialRangeRequest = {
526
+ ...getObjectFields,
527
+ Range: `bytes=0-${rangeSize - 1}`,
528
+ ...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
529
+ };
530
+ const initialResponse = await this.s3.send(new GetObjectCommand(initialRangeRequest), transferOptions);
531
+ if (initialResponse.Body && initialResponse.Body instanceof Readable) {
532
+ initialResponse.Body.on("error", () => { });
533
+ }
534
+ this.validateRangeDownload(`bytes=0-${rangeSize - 1}`, initialResponse.ContentRange);
535
+ const totalContentLength = Number.parseInt(initialResponse.ContentRange.split("/")[1]);
536
+ const responseContentLength = initialResponse.ContentLength ?? 0;
537
+ if (responseContentLength === totalContentLength) {
538
+ tempFilePath = await fileManager.createTempFile(resolvedDestination, totalContentLength);
539
+ fileManager.registerCleanupHandler(tempFilePath);
540
+ await this.writeResponseBodyToFile(initialResponse.Body, tempFilePath, initialResponse.ContentRange, 0);
541
+ await fileManager.atomicRename(tempFilePath, resolvedDestination);
542
+ fileManager.unregisterCleanupHandler(tempFilePath);
543
+ if (emitEvents) {
544
+ this.dispatchTransferInitiatedEvent(request, totalContentLength);
545
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
546
+ request,
547
+ snapshot: {
548
+ transferredBytes: totalContentLength,
549
+ totalBytes: totalContentLength,
550
+ },
551
+ }));
552
+ }
553
+ return this.buildDownloadToFileResponse(initialResponse, totalContentLength, request, emitEvents);
554
+ }
555
+ const eTag = initialResponse.ETag ?? undefined;
556
+ const expectedRequestCount = Math.ceil(totalContentLength / rangeSize);
557
+ if (emitEvents) {
558
+ this.dispatchTransferInitiatedEvent(request, totalContentLength);
559
+ }
560
+ tempFilePath = await fileManager.createTempFile(resolvedDestination, totalContentLength);
561
+ fileManager.registerCleanupHandler(tempFilePath);
562
+ await this.writeResponseBodyToFile(initialResponse.Body, tempFilePath, initialResponse.ContentRange, 0);
563
+ const rangeResults = [];
564
+ rangeResults.push({ index: 0, bytesWritten: responseContentLength });
565
+ let transferredBytes = responseContentLength;
566
+ if (emitEvents) {
567
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
568
+ request: initialRangeRequest,
569
+ snapshot: {
570
+ transferredBytes,
571
+ totalBytes: totalContentLength,
572
+ },
573
+ }));
574
+ }
575
+ const semaphore = new Semaphore(this.maxConcurrentDownloads);
576
+ const abortController = new AbortController();
577
+ let failed = false;
578
+ let failureError;
579
+ const userSignal = transferOptions?.abortSignal;
580
+ if (userSignal) {
581
+ userSignal.addEventListener("abort", () => abortController.abort(), {
582
+ once: true,
583
+ });
584
+ }
585
+ const rangeTasks = [];
586
+ let rangeIndex = 1;
587
+ for (let offset = rangeSize; offset < totalContentLength; offset += rangeSize) {
588
+ if (failed || abortController.signal.aborted)
589
+ break;
590
+ this.checkAborted(transferOptions);
591
+ await semaphore.acquire();
592
+ if (failed || abortController.signal.aborted) {
593
+ semaphore.release();
594
+ break;
595
+ }
596
+ const start = offset;
597
+ const end = Math.min(offset + rangeSize - 1, totalContentLength - 1);
598
+ const currentIndex = rangeIndex++;
599
+ const rangeGetRequest = {
600
+ ...getObjectFields,
601
+ Range: `bytes=${start}-${end}`,
602
+ IfMatch: eTag,
603
+ ...(checksumValidationEnabled && {
604
+ ChecksumMode: "ENABLED",
605
+ }),
606
+ };
607
+ const task = (async () => {
608
+ try {
609
+ const expectedLength = end - start + 1;
610
+ if (this.workerHttpHandler) {
611
+ const resultToken = `download-range-${currentIndex}-${Date.now()}`;
612
+ const downloadOptions = {
613
+ filePath: tempFilePath,
614
+ offset: start,
615
+ expectedLength,
616
+ resultToken,
617
+ };
618
+ const rangeResponse = await this.s3.send(new GetObjectCommand(rangeGetRequest), {
619
+ ...transferOptions,
620
+ downloadDataToFile: downloadOptions,
621
+ });
622
+ this.validateRangeDownload(`bytes=${start}-${end}`, rangeResponse.ContentRange);
623
+ const downloadResult = this.s3.config.requestHandler.getDownloadResult(resultToken);
624
+ if (!downloadResult) {
625
+ throw new Error(`Missing download result for range ${start}-${end}`);
626
+ }
627
+ rangeResults.push({
628
+ index: currentIndex,
629
+ bytesWritten: downloadResult.bytesWritten,
630
+ });
631
+ }
632
+ else {
633
+ const rangeResponse = await this.s3.send(new GetObjectCommand(rangeGetRequest), transferOptions);
634
+ this.validateRangeDownload(`bytes=${start}-${end}`, rangeResponse.ContentRange);
635
+ if (rangeResponse.Body && rangeResponse.Body instanceof Readable) {
636
+ rangeResponse.Body.on("error", () => { });
637
+ }
638
+ await this.writeResponseBodyToFile(rangeResponse.Body, tempFilePath, rangeResponse.ContentRange, start);
639
+ rangeResults.push({
640
+ index: currentIndex,
641
+ bytesWritten: rangeResponse.ContentLength ?? expectedLength,
642
+ });
643
+ }
644
+ transferredBytes += end - start + 1;
645
+ if (emitEvents) {
646
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
647
+ request: rangeGetRequest,
648
+ snapshot: {
649
+ transferredBytes,
650
+ totalBytes: totalContentLength,
651
+ },
652
+ }));
653
+ }
654
+ }
655
+ catch (error) {
656
+ if (!failed) {
657
+ failed = true;
658
+ failureError = error;
659
+ abortController.abort();
660
+ }
661
+ }
662
+ finally {
663
+ semaphore.release();
664
+ }
665
+ })();
666
+ rangeTasks.push(task);
667
+ }
668
+ await Promise.allSettled(rangeTasks);
669
+ if (abortController.signal.aborted && !failed) {
670
+ const abortError = Object.assign(new Error("Transfer aborted."), {
671
+ name: "AbortError",
672
+ });
673
+ await this.handleDownloadCleanup(fileManager, tempFilePath, request, totalContentLength, abortError, emitEvents);
674
+ throw abortError;
675
+ }
676
+ if (failed) {
677
+ await this.handleDownloadCleanup(fileManager, tempFilePath, request, totalContentLength, failureError, emitEvents);
678
+ throw failureError;
679
+ }
680
+ const actualRequestCount = rangeResults.length;
681
+ if (actualRequestCount !== expectedRequestCount) {
682
+ const error = new Error(`Expected ${expectedRequestCount} ranged GET requests but sent ${actualRequestCount}`);
683
+ await this.handleDownloadCleanup(fileManager, tempFilePath, request, totalContentLength, error, emitEvents);
684
+ throw error;
685
+ }
686
+ const totalBytesWritten = rangeResults.reduce((sum, r) => sum + r.bytesWritten, 0);
687
+ if (totalBytesWritten !== totalContentLength) {
688
+ const error = new Error(`Total bytes written (${totalBytesWritten}) does not match expected object size (${totalContentLength})`);
689
+ await this.handleDownloadCleanup(fileManager, tempFilePath, request, totalContentLength, error, emitEvents);
690
+ throw error;
691
+ }
692
+ await fileManager.atomicRename(tempFilePath, resolvedDestination);
693
+ fileManager.unregisterCleanupHandler(tempFilePath);
694
+ return this.buildDownloadToFileResponse(initialResponse, totalContentLength, request, emitEvents);
695
+ }
696
+ catch (error) {
697
+ if (tempFilePath) {
698
+ await fileManager.deleteTempFile(tempFilePath).catch(() => { });
699
+ fileManager.unregisterCleanupHandler(tempFilePath);
700
+ }
701
+ throw error;
702
+ }
703
+ }
704
+ async writeResponseBodyToFile(body, filePath, contentRange, fallbackOffset) {
705
+ const start = parseContentRangeStart(contentRange) ?? fallbackOffset;
706
+ const fd = await open(filePath, "r+");
707
+ try {
708
+ let offset = start;
709
+ if (typeof body[Symbol.asyncIterator] === "function") {
710
+ for await (const chunk of body) {
711
+ await fd.write(chunk, 0, chunk.length, offset);
712
+ offset += chunk.length;
713
+ }
714
+ }
715
+ else if (typeof body.transformToByteArray === "function") {
716
+ const bytes = await body.transformToByteArray();
717
+ await fd.write(bytes, 0, bytes.length, offset);
718
+ offset += bytes.length;
719
+ }
720
+ return offset - start;
721
+ }
722
+ finally {
723
+ await fd.close();
724
+ }
725
+ }
726
+ async uploadDirectory(request, transferOptions) {
727
+ const maxConcurrency = request.maxConcurrency ?? 100;
728
+ if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
729
+ throw new Error("maxConcurrency must be a positive integer");
730
+ }
731
+ const absoluteSource = await validateDirectory(request.source);
732
+ const failurePolicy = request.failurePolicy ?? "terminate";
733
+ const semaphore = new Semaphore(maxConcurrency);
734
+ const abortController = new AbortController();
735
+ const inFlight = new Set();
736
+ let objectsUploaded = 0;
737
+ let objectsFailed = 0;
738
+ let terminated = false;
739
+ let terminationError;
740
+ this.checkAborted(transferOptions);
741
+ this.addEventListeners(transferOptions?.eventListeners);
742
+ let transferredBytes = 0;
743
+ let transferredFiles = 0;
744
+ let totalFiles = undefined;
745
+ let totalBytes = undefined;
746
+ let traversalComplete = false;
747
+ const userSignal = transferOptions?.abortSignal;
748
+ const onUserAbort = () => {
749
+ if (!terminated) {
750
+ terminated = true;
751
+ terminationError = Object.assign(new Error("Transfer aborted."), { name: "AbortError" });
752
+ }
753
+ abortController.abort();
754
+ };
755
+ const removeLocalEventListeners = () => {
756
+ this.removeEventListeners(transferOptions?.eventListeners);
757
+ userSignal?.removeEventListener("abort", onUserAbort);
758
+ };
759
+ this.dispatchEvent(Object.assign(new Event("transferInitiated"), {
760
+ request,
761
+ snapshot: {
762
+ transferredBytes: 0,
763
+ totalBytes: undefined,
764
+ transferredFiles: 0,
765
+ totalFiles: undefined,
766
+ },
767
+ }));
768
+ userSignal?.addEventListener("abort", onUserAbort, { once: true });
769
+ let discoveredFiles = 0;
770
+ let discoveredBytes = 0;
771
+ for await (const filePath of this.traverseDirectory(absoluteSource, {
772
+ recursive: request.recursive ?? false,
773
+ followSymbolicLinks: request.followSymbolicLinks ?? false,
774
+ })) {
775
+ if (terminated || abortController.signal.aborted)
776
+ break;
777
+ this.checkAborted(transferOptions);
778
+ if (request.filter) {
779
+ const include = request.filter instanceof RegExp ? matchesFilterRegExp(request.filter, filePath) : request.filter(filePath);
780
+ if (!include)
781
+ continue;
782
+ }
783
+ const fileStat = await stat(filePath);
784
+ discoveredFiles++;
785
+ discoveredBytes += fileStat.size;
786
+ const count = fileStat.size >= this.multipartUploadThresholdBytes
787
+ ? Math.min(Math.ceil(fileStat.size / this.targetPartSizeBytes), this.maxConcurrentUploads)
788
+ : 1;
789
+ await semaphore.acquire(count);
790
+ if (terminated || abortController.signal.aborted) {
791
+ semaphore.release(count);
792
+ break;
793
+ }
794
+ const task = (async () => {
795
+ try {
796
+ const key = this.deriveS3Key(absoluteSource, filePath, request.s3Prefix);
797
+ let putRequest = {
798
+ Bucket: request.bucket,
799
+ Key: key,
800
+ Body: createReadStream(filePath),
801
+ ContentLength: fileStat.size,
802
+ };
803
+ if (request.uploadObjectRequestModifier) {
804
+ putRequest = request.uploadObjectRequestModifier(putRequest);
805
+ }
806
+ await this.uploadInternal(putRequest, {
807
+ abortSignal: abortController.signal,
808
+ }, false);
809
+ objectsUploaded++;
810
+ transferredFiles++;
811
+ transferredBytes += fileStat.size;
812
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
813
+ request,
814
+ snapshot: {
815
+ transferredBytes,
816
+ totalBytes: traversalComplete ? totalBytes : undefined,
817
+ transferredFiles,
818
+ totalFiles: traversalComplete ? totalFiles : undefined,
819
+ },
820
+ }));
821
+ }
822
+ catch (error) {
823
+ if (terminated || abortController.signal.aborted) {
824
+ objectsFailed++;
825
+ return;
826
+ }
827
+ const context = {
828
+ request,
829
+ objectRequest: {
830
+ Bucket: request.bucket,
831
+ Key: this.deriveS3Key(absoluteSource, filePath, request.s3Prefix),
832
+ },
833
+ error,
834
+ };
835
+ const result = await handleFailure(failurePolicy, context, abortController);
836
+ if (result === "terminate") {
837
+ terminated = true;
838
+ if (!terminationError) {
839
+ terminationError = error;
840
+ }
841
+ }
842
+ objectsFailed++;
843
+ }
844
+ finally {
845
+ semaphore.release(count);
846
+ }
847
+ })();
848
+ inFlight.add(task);
849
+ task.finally(() => inFlight.delete(task));
850
+ }
851
+ traversalComplete = true;
852
+ totalFiles = discoveredFiles;
853
+ totalBytes = discoveredBytes;
854
+ await Promise.allSettled([...inFlight]);
855
+ if (terminated && terminationError) {
856
+ this.dispatchEvent(Object.assign(new Event("transferFailed"), {
857
+ request,
858
+ snapshot: {
859
+ transferredBytes,
860
+ totalBytes,
861
+ transferredFiles,
862
+ totalFiles,
863
+ },
864
+ }));
865
+ removeLocalEventListeners();
866
+ throw terminationError;
867
+ }
868
+ this.dispatchEvent(Object.assign(new Event("transferComplete"), {
869
+ request,
870
+ response: { objectsUploaded, objectsFailed },
871
+ snapshot: {
872
+ transferredBytes,
873
+ totalBytes,
874
+ transferredFiles,
875
+ totalFiles,
876
+ },
877
+ }));
878
+ removeLocalEventListeners();
879
+ return { objectsUploaded, objectsFailed };
880
+ }
881
+ async downloadDirectory(request, transferOptions) {
882
+ const maxConcurrency = request.maxConcurrency ?? 100;
883
+ if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
884
+ throw new Error("maxConcurrency must be a positive integer");
885
+ }
886
+ const destination = await createDestinationDirectory(request.destination);
887
+ const failurePolicy = request.failurePolicy ?? "terminate";
888
+ const semaphore = new Semaphore(maxConcurrency);
889
+ const abortController = new AbortController();
890
+ const inFlight = new Set();
891
+ let objectsDownloaded = 0;
892
+ let objectsFailed = 0;
893
+ let terminated = false;
894
+ let terminationError;
895
+ this.checkAborted(transferOptions);
896
+ this.addEventListeners(transferOptions?.eventListeners);
897
+ let transferredBytes = 0;
898
+ let transferredFiles = 0;
899
+ let discoveredFiles = 0;
900
+ let discoveredBytes = 0;
901
+ let totalFiles = undefined;
902
+ let totalBytes = undefined;
903
+ let traversalComplete = false;
904
+ const userSignal = transferOptions?.abortSignal;
905
+ const onUserAbort = () => {
906
+ if (!terminated) {
907
+ terminated = true;
908
+ terminationError = Object.assign(new Error("Transfer aborted."), { name: "AbortError" });
909
+ }
910
+ abortController.abort();
911
+ };
912
+ const removeLocalEventListeners = () => {
913
+ this.removeEventListeners(transferOptions?.eventListeners);
914
+ userSignal?.removeEventListener("abort", onUserAbort);
915
+ if (transferOptions?.abortSignal) {
916
+ this.abortCleanupFunctions.get(transferOptions.abortSignal)?.();
917
+ this.abortCleanupFunctions.delete(transferOptions.abortSignal);
918
+ }
919
+ };
920
+ this.dispatchEvent(Object.assign(new Event("transferInitiated"), {
921
+ request,
922
+ snapshot: {
923
+ transferredBytes: 0,
924
+ totalBytes: undefined,
925
+ transferredFiles: 0,
926
+ totalFiles: undefined,
927
+ },
928
+ }));
929
+ userSignal?.addEventListener("abort", onUserAbort, { once: true });
930
+ try {
931
+ let continuationToken;
932
+ do {
933
+ if (terminated || abortController.signal.aborted)
934
+ break;
935
+ this.checkAborted(transferOptions);
936
+ const listRequest = {
937
+ Bucket: request.bucket,
938
+ Prefix: request.s3Prefix,
939
+ Delimiter: undefined,
940
+ ContinuationToken: continuationToken,
941
+ };
942
+ const listResponse = await this.s3.send(new ListObjectsV2Command(listRequest), {
943
+ abortSignal: abortController.signal,
944
+ });
945
+ continuationToken = listResponse.IsTruncated ? listResponse.NextContinuationToken : undefined;
946
+ for (const object of listResponse.Contents ?? []) {
947
+ if (terminated || abortController.signal.aborted)
948
+ break;
949
+ this.checkAborted(transferOptions);
950
+ if (isFolderObject(object)) {
951
+ continue;
952
+ }
953
+ if (!object.Key) {
954
+ continue;
955
+ }
956
+ if (request.filter) {
957
+ const include = request.filter instanceof RegExp
958
+ ? matchesFilterRegExp(request.filter, object.Key)
959
+ : request.filter(object);
960
+ if (!include) {
961
+ continue;
962
+ }
963
+ }
964
+ const currentObject = object;
965
+ const key = currentObject.Key;
966
+ await semaphore.acquire();
967
+ if (terminated || abortController.signal.aborted) {
968
+ semaphore.release();
969
+ break;
970
+ }
971
+ const task = (async () => {
972
+ let objectRequest = {
973
+ Bucket: request.bucket,
974
+ Key: key,
975
+ };
976
+ try {
977
+ const localPath = deriveLocalPath(destination, key);
978
+ await createLocalParentDirectories(destination, localPath);
979
+ if (request.downloadObjectRequestModifier) {
980
+ objectRequest = request.downloadObjectRequestModifier(objectRequest);
981
+ }
982
+ await this.downloadToFileInternal({
983
+ ...objectRequest,
984
+ destination: localPath,
985
+ }, {
986
+ abortSignal: abortController.signal,
987
+ }, false);
988
+ objectsDownloaded++;
989
+ transferredFiles++;
990
+ transferredBytes += currentObject.Size ?? 0;
991
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
992
+ request,
993
+ snapshot: {
994
+ transferredBytes,
995
+ totalBytes: traversalComplete ? totalBytes : undefined,
996
+ transferredFiles,
997
+ totalFiles: traversalComplete ? totalFiles : undefined,
998
+ },
999
+ }));
1000
+ }
1001
+ catch (error) {
1002
+ if (terminated || abortController.signal.aborted) {
1003
+ objectsFailed++;
1004
+ return;
1005
+ }
1006
+ const context = {
1007
+ request,
1008
+ objectRequest,
1009
+ error,
1010
+ };
1011
+ const result = await handleFailure(failurePolicy, context, abortController);
1012
+ if (result === "terminate") {
1013
+ terminated = true;
1014
+ if (!terminationError) {
1015
+ terminationError = error;
1016
+ }
1017
+ }
1018
+ objectsFailed++;
1019
+ }
1020
+ finally {
1021
+ semaphore.release();
1022
+ }
1023
+ })();
1024
+ inFlight.add(task);
1025
+ task.finally(() => inFlight.delete(task));
1026
+ discoveredFiles++;
1027
+ discoveredBytes += currentObject.Size ?? 0;
1028
+ }
1029
+ } while (continuationToken);
1030
+ }
1031
+ catch (error) {
1032
+ if (!terminated) {
1033
+ terminated = true;
1034
+ terminationError = error;
1035
+ }
1036
+ abortController.abort();
1037
+ }
1038
+ traversalComplete = true;
1039
+ totalFiles = discoveredFiles;
1040
+ totalBytes = discoveredBytes;
1041
+ await Promise.allSettled([...inFlight]);
1042
+ if (terminated && terminationError) {
1043
+ this.dispatchEvent(Object.assign(new Event("transferFailed"), {
1044
+ request,
1045
+ snapshot: {
1046
+ transferredBytes,
1047
+ totalBytes,
1048
+ transferredFiles,
1049
+ totalFiles,
1050
+ },
1051
+ }));
1052
+ removeLocalEventListeners();
1053
+ throw terminationError;
1054
+ }
1055
+ this.dispatchEvent(Object.assign(new Event("transferComplete"), {
1056
+ request,
1057
+ response: { objectsDownloaded, objectsFailed },
1058
+ snapshot: {
1059
+ transferredBytes,
1060
+ totalBytes,
1061
+ transferredFiles,
1062
+ totalFiles,
1063
+ },
1064
+ }));
1065
+ removeLocalEventListeners();
1066
+ return { objectsDownloaded, objectsFailed };
1067
+ }
1068
+ async downloadByPart(request, transferOptions, streams, requests, metadata, checksumValidationEnabled) {
1069
+ let totalSize;
1070
+ let streamConsumedCallback;
1071
+ this.checkAborted(transferOptions);
1072
+ if (request.Range == null) {
1073
+ const initialPartRequest = {
1074
+ ...request,
1075
+ PartNumber: 1,
1076
+ ...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
1077
+ };
1078
+ try {
1079
+ const initialPart = await this.s3.send(new GetObjectCommand(initialPartRequest), transferOptions);
1080
+ await internalEventHandler.afterInitialGetObject();
1081
+ const partSize = initialPart.ContentLength;
1082
+ const initialETag = initialPart.ETag ?? undefined;
1083
+ totalSize = initialPart.ContentRange ? Number.parseInt(initialPart.ContentRange.split("/")[1]) : 0;
1084
+ this.dispatchTransferInitiatedEvent(request, totalSize);
1085
+ if (initialPart.Body) {
1086
+ if (initialPart.Body && typeof initialPart.Body.getReader === "function") {
1087
+ const reader = initialPart.Body.getReader();
1088
+ initialPart.Body.getReader = function () {
1089
+ return reader;
1090
+ };
1091
+ }
1092
+ streams.push(Promise.resolve(initialPart.Body));
1093
+ requests.push(initialPartRequest);
1094
+ }
1095
+ this.processResponseMetadata(initialPart, metadata, totalSize);
1096
+ let partCount = 1;
1097
+ if (initialPart.PartsCount > 1) {
1098
+ const partTasks = [];
1099
+ const partRequests = [];
1100
+ for (let part = 2; part <= initialPart.PartsCount; part++) {
1101
+ const getObjectRequest = {
1102
+ ...request,
1103
+ PartNumber: part,
1104
+ IfMatch: initialETag,
1105
+ ...(checksumValidationEnabled && {
1106
+ ChecksumMode: "ENABLED",
1107
+ }),
1108
+ };
1109
+ partRequests.push(getObjectRequest);
1110
+ partTasks.push(() => {
1111
+ this.checkAborted(transferOptions);
1112
+ return this.s3
1113
+ .send(new GetObjectCommand(getObjectRequest), transferOptions)
1114
+ .then((response) => {
1115
+ this.validatePartDownload(response.ContentRange, part, partSize ?? 0);
1116
+ if (response.Body && typeof response.Body.getReader === "function") {
1117
+ const reader = response.Body.getReader();
1118
+ response.Body.getReader = function () {
1119
+ return reader;
1120
+ };
1121
+ }
1122
+ return response.Body;
1123
+ })
1124
+ .catch((error) => {
1125
+ this.dispatchTransferFailedEvent(getObjectRequest, totalSize, error);
1126
+ throw error;
1127
+ });
1128
+ });
1129
+ }
1130
+ const { promises: pendingStreams, onStreamConsumed } = this.createConcurrentTaskPool(partTasks, this.maxConcurrentDownloads);
1131
+ streams.push(...pendingStreams);
1132
+ requests.push(...partRequests);
1133
+ streamConsumedCallback = onStreamConsumed;
1134
+ partCount += partTasks.length;
1135
+ if (partCount !== initialPart.PartsCount) {
1136
+ throw new Error(`The number of parts downloaded (${partCount}) does not match the expected number (${initialPart.PartsCount})`);
1137
+ }
1138
+ }
1139
+ }
1140
+ catch (error) {
1141
+ this.dispatchTransferFailedEvent(request, totalSize, error);
1142
+ throw error;
1143
+ }
1144
+ }
1145
+ else {
1146
+ this.checkAborted(transferOptions);
1147
+ try {
1148
+ const getObjectRequest = {
1149
+ ...request,
1150
+ ...(checksumValidationEnabled && {
1151
+ ChecksumMode: "ENABLED",
1152
+ }),
1153
+ };
1154
+ const getObject = await this.s3.send(new GetObjectCommand(getObjectRequest), transferOptions);
1155
+ totalSize = getObject.ContentRange ? Number.parseInt(getObject.ContentRange.split("/")[1]) : 0;
1156
+ this.dispatchTransferInitiatedEvent(request, totalSize);
1157
+ if (getObject.Body) {
1158
+ streams.push(Promise.resolve(getObject.Body));
1159
+ requests.push(getObjectRequest);
1160
+ }
1161
+ this.processResponseMetadata(getObject, metadata, totalSize);
1162
+ }
1163
+ catch (error) {
1164
+ this.dispatchTransferFailedEvent(request, undefined, error);
1165
+ throw error;
1166
+ }
1167
+ }
1168
+ return {
1169
+ totalSize,
1170
+ onStreamConsumed: streamConsumedCallback,
1171
+ };
1172
+ }
1173
+ async downloadByRange(request, transferOptions, streams, requests, metadata, checksumValidationEnabled) {
1174
+ let streamConsumedCallback;
1175
+ this.checkAborted(transferOptions);
1176
+ const headResponse = await this.s3.send(new HeadObjectCommand({ Bucket: request.Bucket, Key: request.Key }), transferOptions);
1177
+ if (headResponse.ContentLength === 0) {
1178
+ const getObjectRequest = {
1179
+ ...request,
1180
+ ...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
1181
+ };
1182
+ const response = await this.s3.send(new GetObjectCommand(getObjectRequest), transferOptions);
1183
+ this.dispatchTransferInitiatedEvent(request, 0);
1184
+ if (response.Body)
1185
+ streams.push(Promise.resolve(response.Body));
1186
+ requests.push(getObjectRequest);
1187
+ this.processResponseMetadata(response, metadata, 0);
1188
+ return { totalSize: 0 };
1189
+ }
1190
+ let left = 0;
1191
+ let right = this.targetPartSizeBytes - 1;
1192
+ let maxRange = Number.POSITIVE_INFINITY;
1193
+ let remainingLength = 1;
1194
+ let totalSize;
1195
+ let initialETag;
1196
+ if (request.Range != null) {
1197
+ const [userRangeLeft, userRangeRight] = request.Range.replace("bytes=", "").split("-").map(Number);
1198
+ maxRange = userRangeRight;
1199
+ left = userRangeLeft;
1200
+ right = Math.min(userRangeRight, left + S3TransferManagerBase.MIN_PART_SIZE - 1);
1201
+ totalSize = userRangeRight + 1;
1202
+ }
1203
+ try {
1204
+ const getObjectRequest = {
1205
+ ...request,
1206
+ Range: `bytes=${left}-${right}`,
1207
+ ...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
1208
+ };
1209
+ const initialRangeGet = await this.s3.send(new GetObjectCommand(getObjectRequest), transferOptions);
1210
+ await internalEventHandler.afterInitialGetObject();
1211
+ this.validateRangeDownload(`bytes=${left}-${right}`, initialRangeGet.ContentRange);
1212
+ initialETag = initialRangeGet.ETag ?? undefined;
1213
+ if (!totalSize) {
1214
+ totalSize = initialRangeGet.ContentRange
1215
+ ? Number.parseInt(initialRangeGet.ContentRange.split("/")[1])
1216
+ : undefined;
1217
+ }
1218
+ if (initialRangeGet.Body && typeof initialRangeGet.Body.getReader === "function") {
1219
+ const reader = initialRangeGet.Body.getReader();
1220
+ initialRangeGet.Body.getReader = function () {
1221
+ return reader;
1222
+ };
1223
+ }
1224
+ this.dispatchTransferInitiatedEvent(request, totalSize);
1225
+ streams.push(Promise.resolve(initialRangeGet.Body));
1226
+ requests.push(getObjectRequest);
1227
+ this.processResponseMetadata(initialRangeGet, metadata, totalSize);
1228
+ }
1229
+ catch (error) {
1230
+ this.dispatchTransferFailedEvent(request, totalSize, error);
1231
+ throw error;
1232
+ }
1233
+ let expectedRequestCount = 1;
1234
+ if (totalSize) {
1235
+ const contentLength = totalSize;
1236
+ const remainingBytes = Math.max(0, contentLength - (right - left + 1));
1237
+ const additionalRequests = Math.ceil(remainingBytes / S3TransferManagerBase.MIN_PART_SIZE);
1238
+ expectedRequestCount += additionalRequests;
1239
+ }
1240
+ left = right + 1;
1241
+ right = Math.min(left + S3TransferManagerBase.MIN_PART_SIZE - 1, maxRange);
1242
+ remainingLength = totalSize ? Math.min(right - left + 1, Math.max(0, totalSize - left)) : 0;
1243
+ const rangeTasks = [];
1244
+ const rangeRequests = [];
1245
+ while (remainingLength > 0) {
1246
+ const range = `bytes=${left}-${right}`;
1247
+ const getObjectRequest = {
1248
+ ...request,
1249
+ Range: range,
1250
+ IfMatch: initialETag,
1251
+ ...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
1252
+ };
1253
+ rangeRequests.push(getObjectRequest);
1254
+ rangeTasks.push(() => {
1255
+ this.checkAborted(transferOptions);
1256
+ return this.s3
1257
+ .send(new GetObjectCommand(getObjectRequest), transferOptions)
1258
+ .then((response) => {
1259
+ this.validateRangeDownload(range, response.ContentRange);
1260
+ if (response.Body && typeof response.Body.getReader === "function") {
1261
+ const reader = response.Body.getReader();
1262
+ response.Body.getReader = function () {
1263
+ return reader;
1264
+ };
1265
+ }
1266
+ return response.Body;
1267
+ })
1268
+ .catch((error) => {
1269
+ this.dispatchTransferFailedEvent(getObjectRequest, totalSize, error);
1270
+ throw error;
1271
+ });
1272
+ });
1273
+ left = right + 1;
1274
+ right = Math.min(left + S3TransferManagerBase.MIN_PART_SIZE - 1, maxRange);
1275
+ remainingLength = totalSize ? Math.min(right - left + 1, Math.max(0, totalSize - left)) : 0;
1276
+ }
1277
+ if (rangeTasks.length > 0) {
1278
+ const { promises: pendingStreams, onStreamConsumed } = this.createConcurrentTaskPool(rangeTasks, this.maxConcurrentDownloads);
1279
+ streams.push(...pendingStreams);
1280
+ requests.push(...rangeRequests);
1281
+ streamConsumedCallback = onStreamConsumed;
1282
+ }
1283
+ const actualRequestCount = 1 + rangeTasks.length;
1284
+ if (expectedRequestCount !== actualRequestCount) {
1285
+ throw new Error(`The number of ranged GET requests sent (${actualRequestCount}) does not match the expected number (${expectedRequestCount})`);
1286
+ }
1287
+ return {
1288
+ totalSize,
1289
+ onStreamConsumed: streamConsumedCallback,
1290
+ };
1291
+ }
1292
+ async downloadByPartWithWorkers(request, transferOptions, streams, requests, metadata, checksumValidationEnabled) {
1293
+ this.checkAborted(transferOptions);
1294
+ const { Range: _range, ...requestWithoutRange } = request;
1295
+ const initialPartRequest = {
1296
+ ...requestWithoutRange,
1297
+ PartNumber: 1,
1298
+ ...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
1299
+ };
1300
+ const initialPart = await this.s3.send(new GetObjectCommand(initialPartRequest), transferOptions);
1301
+ await internalEventHandler.afterInitialGetObject();
1302
+ const partSize = initialPart.ContentLength;
1303
+ const initialETag = initialPart.ETag ?? undefined;
1304
+ const totalSize = initialPart.ContentRange
1305
+ ? Number.parseInt(initialPart.ContentRange.split("/")[1])
1306
+ : (initialPart.ContentLength ?? 0);
1307
+ const partsCount = initialPart.PartsCount ?? 1;
1308
+ this.dispatchTransferInitiatedEvent(request, totalSize);
1309
+ if (initialPart.Body) {
1310
+ if (typeof initialPart.Body.getReader === "function") {
1311
+ const reader = initialPart.Body.getReader();
1312
+ initialPart.Body.getReader = function () {
1313
+ return reader;
1314
+ };
1315
+ }
1316
+ streams.push(Promise.resolve(initialPart.Body));
1317
+ requests.push(initialPartRequest);
1318
+ }
1319
+ this.processResponseMetadata(initialPart, metadata, totalSize);
1320
+ if (partsCount <= 1) {
1321
+ return { totalSize };
1322
+ }
1323
+ this.checkAborted(transferOptions);
1324
+ const remainingParts = partsCount - 1;
1325
+ const queue = new OrderedPartQueue(remainingParts);
1326
+ const abortController = new AbortController();
1327
+ const userSignal = transferOptions?.abortSignal;
1328
+ if (userSignal) {
1329
+ userSignal.addEventListener("abort", () => abortController.abort(), {
1330
+ once: true,
1331
+ });
1332
+ }
1333
+ let inFlight = 0;
1334
+ let nextDispatchResolve = null;
1335
+ const waitForSlot = () => {
1336
+ if (inFlight < this.maxConcurrentDownloads) {
1337
+ return Promise.resolve();
1338
+ }
1339
+ return new Promise((resolve) => {
1340
+ nextDispatchResolve = resolve;
1341
+ });
1342
+ };
1343
+ const releaseSlot = () => {
1344
+ if (inFlight <= 0) {
1345
+ throw new Error("releaseSlot called with no in-flight requests");
1346
+ }
1347
+ inFlight--;
1348
+ if (nextDispatchResolve && inFlight < this.maxConcurrentDownloads) {
1349
+ const resolve = nextDispatchResolve;
1350
+ nextDispatchResolve = null;
1351
+ resolve();
1352
+ }
1353
+ };
1354
+ const dispatchPromise = (async () => {
1355
+ for (let partNumber = 2; partNumber <= partsCount; partNumber++) {
1356
+ if (abortController.signal.aborted || queue.hasError())
1357
+ break;
1358
+ this.checkAborted(transferOptions);
1359
+ await waitForSlot();
1360
+ if (abortController.signal.aborted || queue.hasError())
1361
+ break;
1362
+ inFlight++;
1363
+ const rangeIndex = partNumber - 2;
1364
+ const partGetRequest = {
1365
+ ...requestWithoutRange,
1366
+ PartNumber: partNumber,
1367
+ IfMatch: initialETag,
1368
+ ...(checksumValidationEnabled && {
1369
+ ChecksumMode: "ENABLED",
1370
+ }),
1371
+ };
1372
+ requests.push(partGetRequest);
1373
+ const resultToken = `download-transfer-part-${rangeIndex}-${Date.now()}`;
1374
+ const transferOptions_ = {
1375
+ expectedSize: partSize,
1376
+ rangeIndex,
1377
+ checksumAlgorithm: checksumValidationEnabled ? "CRC32" : undefined,
1378
+ resultToken,
1379
+ };
1380
+ this.s3
1381
+ .send(new GetObjectCommand(partGetRequest), {
1382
+ ...transferOptions,
1383
+ downloadStream: transferOptions_,
1384
+ })
1385
+ .then(() => {
1386
+ const transferResult = this.s3.config.requestHandler.getStreamDownloadResult(resultToken);
1387
+ if (!transferResult) {
1388
+ queue.setError(new Error(`Missing download result for part ${partNumber}`));
1389
+ abortController.abort();
1390
+ return;
1391
+ }
1392
+ this.validatePartDownload(transferResult.headers["content-range"], partNumber, partSize);
1393
+ queue.enqueue(rangeIndex, transferResult.buffer, transferResult.byteLength);
1394
+ })
1395
+ .catch((error) => {
1396
+ queue.setError(error);
1397
+ abortController.abort();
1398
+ })
1399
+ .finally(() => {
1400
+ releaseSlot();
1401
+ });
1402
+ }
1403
+ })().catch((err) => {
1404
+ if (!queue.hasError()) {
1405
+ queue.setError(err);
1406
+ }
1407
+ });
1408
+ let reading = false;
1409
+ const transferStream = new Readable({
1410
+ read() {
1411
+ if (reading)
1412
+ return;
1413
+ reading = true;
1414
+ queue
1415
+ .dequeue()
1416
+ .then((item) => {
1417
+ reading = false;
1418
+ if (item === null) {
1419
+ if (queue.hasError()) {
1420
+ this.destroy(queue.getError());
1421
+ return;
1422
+ }
1423
+ this.push(null);
1424
+ return;
1425
+ }
1426
+ const buf = Buffer.from(item.buffer, 0, item.byteLength);
1427
+ this.push(buf);
1428
+ })
1429
+ .catch((err) => {
1430
+ reading = false;
1431
+ this.destroy(err);
1432
+ });
1433
+ },
1434
+ destroy(err, callback) {
1435
+ if (err) {
1436
+ queue.setError(err);
1437
+ abortController.abort();
1438
+ }
1439
+ callback(err);
1440
+ },
1441
+ });
1442
+ streams.push(Promise.resolve(transferStream));
1443
+ return { totalSize };
1444
+ }
1445
+ async downloadByRangeWithWorkers(request, transferOptions, streams, requests, metadata, checksumValidationEnabled) {
1446
+ this.checkAborted(transferOptions);
1447
+ const headResponse = await this.s3.send(new HeadObjectCommand({ Bucket: request.Bucket, Key: request.Key }), transferOptions);
1448
+ await internalEventHandler.afterInitialGetObject();
1449
+ if (headResponse.ContentLength === 0) {
1450
+ const getObjectRequest = {
1451
+ ...request,
1452
+ ...(checksumValidationEnabled && { ChecksumMode: "ENABLED" }),
1453
+ };
1454
+ const response = await this.s3.send(new GetObjectCommand(getObjectRequest), transferOptions);
1455
+ this.dispatchTransferInitiatedEvent(request, 0);
1456
+ if (response.Body)
1457
+ streams.push(Promise.resolve(response.Body));
1458
+ requests.push(getObjectRequest);
1459
+ this.processResponseMetadata(response, metadata, 0);
1460
+ return { totalSize: 0 };
1461
+ }
1462
+ const totalSize = headResponse.ContentLength;
1463
+ const partSize = this.targetPartSizeBytes;
1464
+ const totalParts = Math.ceil(totalSize / partSize);
1465
+ const eTag = headResponse.ETag ?? undefined;
1466
+ this.dispatchTransferInitiatedEvent(request, totalSize);
1467
+ Object.assign(metadata, {
1468
+ ContentLength: totalSize,
1469
+ ContentRange: `bytes 0-${totalSize - 1}/${totalSize}`,
1470
+ ContentType: headResponse.ContentType,
1471
+ ETag: headResponse.ETag,
1472
+ LastModified: headResponse.LastModified,
1473
+ $metadata: headResponse.$metadata,
1474
+ });
1475
+ const queue = new OrderedPartQueue(totalParts);
1476
+ const abortController = new AbortController();
1477
+ const userSignal = transferOptions?.abortSignal;
1478
+ if (userSignal) {
1479
+ userSignal.addEventListener("abort", () => abortController.abort(), {
1480
+ once: true,
1481
+ });
1482
+ }
1483
+ let inFlight = 0;
1484
+ let nextDispatchResolve = null;
1485
+ const waitForSlot = () => {
1486
+ if (inFlight < this.maxConcurrentDownloads) {
1487
+ return Promise.resolve();
1488
+ }
1489
+ return new Promise((resolve) => {
1490
+ nextDispatchResolve = resolve;
1491
+ });
1492
+ };
1493
+ const releaseSlot = () => {
1494
+ if (inFlight <= 0) {
1495
+ throw new Error("releaseSlot called with no in-flight requests");
1496
+ }
1497
+ inFlight--;
1498
+ if (nextDispatchResolve && inFlight < this.maxConcurrentDownloads) {
1499
+ const resolve = nextDispatchResolve;
1500
+ nextDispatchResolve = null;
1501
+ resolve();
1502
+ }
1503
+ };
1504
+ const dispatchPromise = (async () => {
1505
+ for (let rangeIndex = 0; rangeIndex < totalParts; rangeIndex++) {
1506
+ if (abortController.signal.aborted || queue.hasError())
1507
+ break;
1508
+ this.checkAborted(transferOptions);
1509
+ await waitForSlot();
1510
+ if (abortController.signal.aborted || queue.hasError())
1511
+ break;
1512
+ inFlight++;
1513
+ const start = rangeIndex * partSize;
1514
+ const end = Math.min(start + partSize - 1, totalSize - 1);
1515
+ const expectedLength = end - start + 1;
1516
+ const rangeGetRequest = {
1517
+ ...request,
1518
+ Range: `bytes=${start}-${end}`,
1519
+ IfMatch: eTag,
1520
+ ...(checksumValidationEnabled && {
1521
+ ChecksumMode: "ENABLED",
1522
+ }),
1523
+ };
1524
+ requests.push(rangeGetRequest);
1525
+ const resultToken = `download-transfer-range-${rangeIndex}-${Date.now()}`;
1526
+ const transferOpts = {
1527
+ expectedSize: expectedLength,
1528
+ rangeIndex,
1529
+ checksumAlgorithm: checksumValidationEnabled ? "CRC32" : undefined,
1530
+ resultToken,
1531
+ };
1532
+ this.s3
1533
+ .send(new GetObjectCommand(rangeGetRequest), {
1534
+ ...transferOptions,
1535
+ downloadStream: transferOpts,
1536
+ })
1537
+ .then(() => {
1538
+ const transferResult = this.s3.config.requestHandler.getStreamDownloadResult(resultToken);
1539
+ if (!transferResult) {
1540
+ queue.setError(new Error(`Missing download result for range ${start}-${end}`));
1541
+ abortController.abort();
1542
+ return;
1543
+ }
1544
+ this.validateRangeDownload(`bytes=${start}-${end}`, transferResult.headers["content-range"]);
1545
+ queue.enqueue(rangeIndex, transferResult.buffer, transferResult.byteLength);
1546
+ })
1547
+ .catch((error) => {
1548
+ queue.setError(error);
1549
+ abortController.abort();
1550
+ })
1551
+ .finally(() => {
1552
+ releaseSlot();
1553
+ });
1554
+ }
1555
+ })().catch((err) => {
1556
+ if (!queue.hasError()) {
1557
+ queue.setError(err);
1558
+ }
1559
+ });
1560
+ let reading = false;
1561
+ const transferStream = new Readable({
1562
+ read() {
1563
+ if (reading)
1564
+ return;
1565
+ reading = true;
1566
+ queue
1567
+ .dequeue()
1568
+ .then((item) => {
1569
+ reading = false;
1570
+ if (item === null) {
1571
+ if (queue.hasError()) {
1572
+ this.destroy(queue.getError());
1573
+ return;
1574
+ }
1575
+ this.push(null);
1576
+ return;
1577
+ }
1578
+ const buf = Buffer.from(item.buffer, 0, item.byteLength);
1579
+ this.push(buf);
1580
+ })
1581
+ .catch((err) => {
1582
+ reading = false;
1583
+ this.destroy(err);
1584
+ });
1585
+ },
1586
+ destroy(err, callback) {
1587
+ if (err) {
1588
+ queue.setError(err);
1589
+ abortController.abort();
1590
+ }
1591
+ callback(err);
1592
+ },
1593
+ });
1594
+ streams.push(Promise.resolve(transferStream));
1595
+ return { totalSize };
1596
+ }
1597
+ addEventListeners(eventListeners) {
1598
+ for (const listeners of this.iterateListeners(eventListeners)) {
1599
+ for (const listener of listeners) {
1600
+ this.addEventListener(listener.eventType, listener.callback);
1601
+ }
1602
+ }
1603
+ }
1604
+ removeEventListeners(eventListeners) {
1605
+ for (const listeners of this.iterateListeners(eventListeners)) {
1606
+ for (const listener of listeners) {
1607
+ this.removeEventListener(listener.eventType, listener.callback);
1608
+ }
1609
+ }
1610
+ }
1611
+ assignMetadata(container, response) {
1612
+ for (const key in response) {
1613
+ if (key === "Body") {
1614
+ continue;
1615
+ }
1616
+ container[key] = response[key];
1617
+ }
1618
+ }
1619
+ buildDownloadToFileResponse(initialResponse, totalSize, request, emitEvents) {
1620
+ const metadata = {};
1621
+ this.assignMetadata(metadata, initialResponse);
1622
+ metadata.ContentLength = totalSize;
1623
+ metadata.ContentRange = `bytes 0-${totalSize - 1}/${totalSize}`;
1624
+ if (initialResponse.ChecksumType === "COMPOSITE") {
1625
+ metadata.ChecksumCRC32 = undefined;
1626
+ metadata.ChecksumCRC32C = undefined;
1627
+ metadata.ChecksumSHA1 = undefined;
1628
+ metadata.ChecksumSHA256 = undefined;
1629
+ }
1630
+ const response = {
1631
+ ...metadata,
1632
+ bytesWritten: totalSize,
1633
+ };
1634
+ if (emitEvents) {
1635
+ this.dispatchEvent(Object.assign(new Event("transferComplete"), {
1636
+ request,
1637
+ response,
1638
+ snapshot: {
1639
+ transferredBytes: totalSize,
1640
+ totalBytes: totalSize,
1641
+ },
1642
+ }));
1643
+ }
1644
+ return response;
1645
+ }
1646
+ async handleDownloadCleanup(fileManager, tempFilePath, request, totalSize, error, emitEvents) {
1647
+ await fileManager.deleteTempFile(tempFilePath);
1648
+ fileManager.unregisterCleanupHandler(tempFilePath);
1649
+ if (emitEvents) {
1650
+ this.dispatchTransferFailedEvent(request, totalSize, error);
1651
+ }
1652
+ }
1653
+ updateResponseLengthAndRange(response, totalSize) {
1654
+ if (totalSize !== undefined) {
1655
+ response.ContentLength = totalSize;
1656
+ response.ContentRange = `bytes 0-${totalSize - 1}/${totalSize}`;
1657
+ }
1658
+ }
1659
+ updateChecksumValues(initialPart, metadata) {
1660
+ if (initialPart.ChecksumType === "COMPOSITE") {
1661
+ metadata.ChecksumCRC32 = undefined;
1662
+ metadata.ChecksumCRC32C = undefined;
1663
+ metadata.ChecksumSHA1 = undefined;
1664
+ metadata.ChecksumSHA256 = undefined;
1665
+ }
1666
+ }
1667
+ processResponseMetadata(response, metadata, totalSize) {
1668
+ this.updateResponseLengthAndRange(response, totalSize);
1669
+ this.assignMetadata(metadata, response);
1670
+ this.updateChecksumValues(response, metadata);
1671
+ }
1672
+ checkAborted(transferOptions) {
1673
+ if (transferOptions?.abortSignal?.aborted) {
1674
+ throw Object.assign(new Error("Transfer aborted."), {
1675
+ name: "AbortError",
1676
+ });
1677
+ }
1678
+ }
1679
+ validateConfig() {
1680
+ if (this.targetPartSizeBytes < S3TransferManagerBase.MIN_PART_SIZE) {
1681
+ throw new Error(`targetPartSizeBytes must be at least ${S3TransferManagerBase.MIN_PART_SIZE} bytes`);
1682
+ }
1683
+ }
1684
+ dispatchTransferInitiatedEvent(request, totalSize) {
1685
+ this.dispatchEvent(Object.assign(new Event("transferInitiated"), {
1686
+ request,
1687
+ snapshot: {
1688
+ transferredBytes: 0,
1689
+ totalBytes: totalSize,
1690
+ },
1691
+ }));
1692
+ return true;
1693
+ }
1694
+ dispatchTransferFailedEvent(request, totalSize, error) {
1695
+ this.dispatchEvent(Object.assign(new Event("transferFailed"), {
1696
+ request,
1697
+ error,
1698
+ snapshot: {
1699
+ transferredBytes: 0,
1700
+ totalBytes: totalSize,
1701
+ },
1702
+ }));
1703
+ return true;
1704
+ }
1705
+ *iterateListeners(eventListeners = {}) {
1706
+ for (const key in eventListeners) {
1707
+ const eventType = key;
1708
+ const listeners = eventListeners[eventType];
1709
+ if (Array.isArray(listeners)) {
1710
+ for (const callback of listeners) {
1711
+ yield [
1712
+ {
1713
+ eventType: eventType,
1714
+ callback: callback,
1715
+ },
1716
+ ];
1717
+ }
1718
+ }
1719
+ }
1720
+ }
1721
+ createConcurrentTaskPool(tasks, maxConcurrent) {
1722
+ if (tasks.length === 0)
1723
+ return { promises: [], onStreamConsumed: () => { } };
1724
+ const resolvers = [];
1725
+ const promises = tasks.map(() => {
1726
+ let resolve;
1727
+ let reject;
1728
+ const promise = new Promise((res, rej) => {
1729
+ resolve = res;
1730
+ reject = rej;
1731
+ });
1732
+ resolvers.push({ resolve, reject });
1733
+ return promise;
1734
+ });
1735
+ let nextToLaunch = 0;
1736
+ let failed = false;
1737
+ const launchTask = (index) => {
1738
+ tasks[index]().then((result) => {
1739
+ resolvers[index].resolve(result);
1740
+ }, (error) => {
1741
+ failed = true;
1742
+ resolvers[index].reject(error);
1743
+ });
1744
+ };
1745
+ const initialBatch = Math.min(maxConcurrent, tasks.length);
1746
+ for (let i = 0; i < initialBatch; i++) {
1747
+ launchTask(nextToLaunch++);
1748
+ }
1749
+ const onStreamConsumed = (_index) => {
1750
+ if (failed || nextToLaunch >= tasks.length)
1751
+ return;
1752
+ launchTask(nextToLaunch++);
1753
+ };
1754
+ return { promises, onStreamConsumed };
1755
+ }
1756
+ validatePartDownload(contentRange, partNumber, partSize) {
1757
+ if (!contentRange) {
1758
+ throw new Error(`Missing ContentRange for part ${partNumber}.`);
1759
+ }
1760
+ const match = contentRange.match(/bytes (\d+)-(\d+)\/(\d+)/);
1761
+ if (!match)
1762
+ throw new Error(`Invalid ContentRange format: ${contentRange}`);
1763
+ const start = Number.parseInt(match[1]);
1764
+ const end = Number.parseInt(match[2]);
1765
+ const total = Number.parseInt(match[3]) - 1;
1766
+ const expectedStart = (partNumber - 1) * partSize;
1767
+ const expectedEnd = Math.min(expectedStart + partSize - 1, total);
1768
+ if (start !== expectedStart) {
1769
+ throw new Error(`Expected part ${partNumber} to start at ${expectedStart} but got ${start}`);
1770
+ }
1771
+ if (end !== expectedEnd) {
1772
+ throw new Error(`Expected part ${partNumber} to end at ${expectedEnd} but got ${end}`);
1773
+ }
1774
+ }
1775
+ validateRangeDownload(requestRange, responseRange) {
1776
+ if (!responseRange) {
1777
+ throw new Error(`Missing ContentRange for range ${requestRange}.`);
1778
+ }
1779
+ const match = responseRange.match(/bytes (\d+)-(\d+)\/(\d+)/);
1780
+ if (!match)
1781
+ throw new Error(`Invalid ContentRange format: ${responseRange}`);
1782
+ const start = Number.parseInt(match[1]);
1783
+ const end = Number.parseInt(match[2]);
1784
+ const total = Number.parseInt(match[3]) - 1;
1785
+ const rangeMatch = requestRange.match(/bytes=(\d+)-(\d+)/);
1786
+ if (!rangeMatch)
1787
+ throw new Error(`Invalid Range format: ${requestRange}`);
1788
+ const expectedStart = Number.parseInt(rangeMatch[1]);
1789
+ const expectedEnd = Number.parseInt(rangeMatch[2]);
1790
+ if (start !== expectedStart) {
1791
+ throw new Error(`Expected range to start at ${expectedStart} but got ${start}`);
1792
+ }
1793
+ if (end === expectedEnd) {
1794
+ return;
1795
+ }
1796
+ if (end === total) {
1797
+ return;
1798
+ }
1799
+ throw new Error(`Expected range to end at ${expectedEnd} but got ${end}`);
1800
+ }
1801
+ async uploadSinglePart(request, contentLength, transferOptions, emitEvents) {
1802
+ const putObjectRequest = { ...request };
1803
+ this.checkAborted(transferOptions);
1804
+ const response = await this.s3.send(new PutObjectCommand(putObjectRequest), transferOptions);
1805
+ if (emitEvents) {
1806
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
1807
+ request,
1808
+ snapshot: {
1809
+ transferredBytes: contentLength,
1810
+ totalBytes: contentLength,
1811
+ },
1812
+ }));
1813
+ }
1814
+ return response;
1815
+ }
1816
+ calculatePartSize(contentLength) {
1817
+ const partSize = Math.max(this.targetPartSizeBytes, Math.ceil(contentLength / 10_000));
1818
+ const expectedPartsCount = Math.ceil(contentLength / partSize);
1819
+ return { partSize, expectedPartsCount };
1820
+ }
1821
+ isFileBody(body) {
1822
+ return typeof body?.pipe === "function" && typeof body.path === "string";
1823
+ }
1824
+ isInMemoryBody(body) {
1825
+ return body instanceof Uint8Array || typeof body === "string";
1826
+ }
1827
+ async threadedUploadInPartsFromFile(request, contentLength, transferOptions, emitEvents) {
1828
+ const { partSize } = this.calculatePartSize(contentLength);
1829
+ const body = request.Body;
1830
+ const filePath = body.path;
1831
+ const startOffset = typeof body.start === "number" ? body.start : 0;
1832
+ const buildDataSource = (checksumAlgorithm, checksumHeader) => ({
1833
+ type: "file",
1834
+ filePath,
1835
+ partSize,
1836
+ totalFileSize: contentLength,
1837
+ startOffset,
1838
+ checksumAlgorithm,
1839
+ checksumHeader,
1840
+ });
1841
+ return this.threadedMultipartUpload(request, contentLength, buildDataSource, transferOptions, emitEvents);
1842
+ }
1843
+ async threadedUploadInParts(request, contentLength, transferOptions, emitEvents) {
1844
+ const { partSize } = this.calculatePartSize(contentLength);
1845
+ const body = request.Body;
1846
+ const isSharedBufferBody = body.buffer instanceof SharedArrayBuffer;
1847
+ const sharedBuffer = isSharedBufferBody ? body.buffer : new SharedArrayBuffer(body.byteLength);
1848
+ if (!isSharedBufferBody) {
1849
+ new Uint8Array(sharedBuffer).set(body);
1850
+ }
1851
+ const startOffset = isSharedBufferBody ? body.byteOffset : 0;
1852
+ const buildDataSource = (checksumAlgorithm, checksumHeader) => ({
1853
+ type: "sharedBuffer",
1854
+ sharedBuffer,
1855
+ startOffset,
1856
+ partSize,
1857
+ totalSize: contentLength,
1858
+ checksumAlgorithm,
1859
+ checksumHeader,
1860
+ });
1861
+ return this.threadedMultipartUpload(request, contentLength, buildDataSource, transferOptions, emitEvents);
1862
+ }
1863
+ async getWorkerUploadMaxAttempts() {
1864
+ const maxAttemptsProvider = this.s3.config.maxAttempts;
1865
+ return maxAttemptsProvider();
1866
+ }
1867
+ isRetryableWorkerUploadError(error) {
1868
+ const sdkError = error;
1869
+ return isThrottlingError(sdkError) || isTransientError(sdkError);
1870
+ }
1871
+ async waitForWorkerUploadRetry(attempt) {
1872
+ const maxDelayMs = 1_000;
1873
+ const baseDelayMs = 100;
1874
+ const exponentialDelayMs = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1));
1875
+ const jitteredDelayMs = Math.round(exponentialDelayMs * (0.5 + Math.random()));
1876
+ await new Promise((resolve) => setTimeout(resolve, jitteredDelayMs));
1877
+ }
1878
+ async threadedMultipartUpload(request, contentLength, buildDataSource, transferOptions, emitEvents) {
1879
+ const { partSize, expectedPartsCount } = this.calculatePartSize(contentLength);
1880
+ this.checkAborted(transferOptions);
1881
+ const createMpuRequest = { ...request };
1882
+ const hasFullObjectChecksum = request.ContentMD5 ||
1883
+ request.ChecksumCRC32 ||
1884
+ request.ChecksumCRC32C ||
1885
+ request.ChecksumCRC64NVME ||
1886
+ request.ChecksumSHA1 ||
1887
+ request.ChecksumSHA256;
1888
+ if (hasFullObjectChecksum) {
1889
+ createMpuRequest.ChecksumType = "FULL_OBJECT";
1890
+ }
1891
+ if (!createMpuRequest.ChecksumAlgorithm && this.requestChecksumCalculation === "WHEN_SUPPORTED") {
1892
+ createMpuRequest.ChecksumAlgorithm = request.ChecksumAlgorithm ?? "CRC32";
1893
+ }
1894
+ const createMpuResponse = await this.s3.send(new CreateMultipartUploadCommand(createMpuRequest), transferOptions);
1895
+ const uploadId = createMpuResponse.UploadId;
1896
+ const checksumAlgorithm = createMpuRequest.ChecksumAlgorithm;
1897
+ const checksumHeader = checksumAlgorithm ? `x-amz-checksum-${checksumAlgorithm.toLowerCase()}` : undefined;
1898
+ const dataSource = buildDataSource(checksumAlgorithm, checksumHeader);
1899
+ const abortController = new AbortController();
1900
+ const uploadAbortSignal = abortController.signal;
1901
+ const maxWorkerUploadAttempts = await this.getWorkerUploadMaxAttempts();
1902
+ const sendUploadPart = async (partNumber, length) => {
1903
+ for (let attempt = 1;; attempt++) {
1904
+ if (uploadAbortSignal.aborted) {
1905
+ throw Object.assign(new Error("Upload aborted due to part failure."), { name: "AbortError" });
1906
+ }
1907
+ this.checkAborted(transferOptions);
1908
+ const partRequest = {
1909
+ ...request,
1910
+ Body: this.deps.createEmptyReadable(),
1911
+ UploadId: uploadId,
1912
+ PartNumber: partNumber,
1913
+ ContentLength: length,
1914
+ ...(checksumAlgorithm && {
1915
+ ChecksumAlgorithm: checksumAlgorithm,
1916
+ }),
1917
+ };
1918
+ try {
1919
+ return await this.s3.send(new UploadPartCommand(partRequest), {
1920
+ ...transferOptions,
1921
+ dataSource,
1922
+ });
1923
+ }
1924
+ catch (error) {
1925
+ if (attempt >= maxWorkerUploadAttempts || !this.isRetryableWorkerUploadError(error)) {
1926
+ throw error;
1927
+ }
1928
+ const errorName = error.name ?? "unknown error";
1929
+ this.logger.warn(`Retrying UploadPart ${partNumber} for ${request.Key} after ${errorName} ` +
1930
+ `(attempt ${attempt + 1}/${maxWorkerUploadAttempts}).`);
1931
+ await this.waitForWorkerUploadRetry(attempt);
1932
+ }
1933
+ }
1934
+ };
1935
+ try {
1936
+ const completedParts = [];
1937
+ let bytesUploaded = 0;
1938
+ let nextPartNumber = 1;
1939
+ const uploadPart = async () => {
1940
+ while (true) {
1941
+ const partNumber = nextPartNumber++;
1942
+ if (partNumber > expectedPartsCount)
1943
+ return;
1944
+ if (uploadAbortSignal.aborted) {
1945
+ throw Object.assign(new Error("Upload aborted due to part failure."), { name: "AbortError" });
1946
+ }
1947
+ this.checkAborted(transferOptions);
1948
+ const offset = (partNumber - 1) * partSize;
1949
+ const length = Math.min(partSize, contentLength - offset);
1950
+ const partResponse = await sendUploadPart(partNumber, length);
1951
+ const completedPart = {
1952
+ PartNumber: partNumber,
1953
+ ETag: partResponse.ETag,
1954
+ };
1955
+ if (partResponse.ChecksumCRC32)
1956
+ completedPart.ChecksumCRC32 = partResponse.ChecksumCRC32;
1957
+ if (partResponse.ChecksumCRC32C)
1958
+ completedPart.ChecksumCRC32C = partResponse.ChecksumCRC32C;
1959
+ if (partResponse.ChecksumCRC64NVME)
1960
+ completedPart.ChecksumCRC64NVME = partResponse.ChecksumCRC64NVME;
1961
+ if (partResponse.ChecksumSHA1)
1962
+ completedPart.ChecksumSHA1 = partResponse.ChecksumSHA1;
1963
+ if (partResponse.ChecksumSHA256)
1964
+ completedPart.ChecksumSHA256 = partResponse.ChecksumSHA256;
1965
+ completedParts.push(completedPart);
1966
+ bytesUploaded += length;
1967
+ if (emitEvents) {
1968
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
1969
+ request,
1970
+ snapshot: {
1971
+ transferredBytes: bytesUploaded,
1972
+ totalBytes: contentLength,
1973
+ },
1974
+ }));
1975
+ }
1976
+ }
1977
+ };
1978
+ const partUploads = [];
1979
+ for (let i = 0; i < this.maxConcurrentUploads; i++) {
1980
+ partUploads.push(uploadPart().catch((error) => {
1981
+ abortController.abort();
1982
+ throw error;
1983
+ }));
1984
+ }
1985
+ await Promise.all(partUploads);
1986
+ if (completedParts.length !== expectedPartsCount) {
1987
+ throw new Error(`Expected ${expectedPartsCount} parts but uploaded ${completedParts.length} parts`);
1988
+ }
1989
+ completedParts.sort((a, b) => a.PartNumber - b.PartNumber);
1990
+ this.checkAborted(transferOptions);
1991
+ const { Body: _body, ...completeRequestFields } = request;
1992
+ const completeRequest = {
1993
+ ...completeRequestFields,
1994
+ UploadId: uploadId,
1995
+ MultipartUpload: { Parts: completedParts },
1996
+ MpuObjectSize: contentLength,
1997
+ };
1998
+ if (hasFullObjectChecksum) {
1999
+ completeRequest.ChecksumType = "FULL_OBJECT";
2000
+ if (request.ChecksumCRC32)
2001
+ completeRequest.ChecksumCRC32 = request.ChecksumCRC32;
2002
+ if (request.ChecksumCRC32C)
2003
+ completeRequest.ChecksumCRC32C = request.ChecksumCRC32C;
2004
+ if (request.ChecksumCRC64NVME)
2005
+ completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
2006
+ }
2007
+ try {
2008
+ return await this.s3.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
2009
+ }
2010
+ catch (completeMpuError) {
2011
+ this.logger.warn(`CompleteMultipartUpload failed for upload ID ${uploadId}: ${completeMpuError.message}`);
2012
+ throw completeMpuError;
2013
+ }
2014
+ }
2015
+ catch (error) {
2016
+ const { Body: _abortBody, ...abortRequestFields } = request;
2017
+ const abortRequest = {
2018
+ ...abortRequestFields,
2019
+ UploadId: uploadId,
2020
+ };
2021
+ try {
2022
+ await this.s3.send(new AbortMultipartUploadCommand(abortRequest), transferOptions);
2023
+ }
2024
+ catch (abortError) {
2025
+ this.logger.warn(`Failed to abort multipart upload ${uploadId}:`, abortError);
2026
+ }
2027
+ throw error;
2028
+ }
2029
+ }
2030
+ async uploadInParts(request, contentLength, transferOptions, emitEvents) {
2031
+ const { partSize, expectedPartsCount } = this.calculatePartSize(contentLength);
2032
+ this.checkAborted(transferOptions);
2033
+ const createMpuRequest = { ...request };
2034
+ const hasFullObjectChecksum = request.ContentMD5 ||
2035
+ request.ChecksumCRC32 ||
2036
+ request.ChecksumCRC32C ||
2037
+ request.ChecksumCRC64NVME ||
2038
+ request.ChecksumSHA1 ||
2039
+ request.ChecksumSHA256;
2040
+ if (hasFullObjectChecksum) {
2041
+ createMpuRequest.ChecksumType = "FULL_OBJECT";
2042
+ }
2043
+ if (!createMpuRequest.ChecksumAlgorithm && this.requestChecksumCalculation === "WHEN_SUPPORTED") {
2044
+ createMpuRequest.ChecksumAlgorithm = request.ChecksumAlgorithm ?? "CRC32";
2045
+ }
2046
+ const createMpuResponse = await this.s3.send(new CreateMultipartUploadCommand(createMpuRequest), transferOptions);
2047
+ const uploadId = createMpuResponse.UploadId;
2048
+ const abortController = new AbortController();
2049
+ const uploadAbortSignal = abortController.signal;
2050
+ try {
2051
+ const completedParts = [];
2052
+ const dataFeeder = getChunk(request.Body, partSize);
2053
+ let bytesUploaded = 0;
2054
+ const uploadPart = async (dataFeeder) => {
2055
+ for await (const dataPart of dataFeeder) {
2056
+ if (uploadAbortSignal.aborted) {
2057
+ throw Object.assign(new Error("Upload aborted due to part failure."), { name: "AbortError" });
2058
+ }
2059
+ this.checkAborted(transferOptions);
2060
+ this.validateUploadPart(dataPart, partSize);
2061
+ const partRequest = {
2062
+ ...request,
2063
+ Body: dataPart.data,
2064
+ UploadId: uploadId,
2065
+ PartNumber: dataPart.partNumber,
2066
+ ContentLength: byteLength(dataPart.data),
2067
+ ...(this.requestChecksumCalculation === "WHEN_SUPPORTED" && {
2068
+ ChecksumAlgorithm: request.ChecksumAlgorithm,
2069
+ }),
2070
+ };
2071
+ const partResponse = await this.s3.send(new UploadPartCommand(partRequest), transferOptions);
2072
+ const checksumAlgorithm = partRequest.ChecksumAlgorithm;
2073
+ if (checksumAlgorithm &&
2074
+ !partResponse.ChecksumCRC32 &&
2075
+ !partResponse.ChecksumCRC32C &&
2076
+ !partResponse.ChecksumCRC64NVME &&
2077
+ !partResponse.ChecksumSHA1 &&
2078
+ !partResponse.ChecksumSHA256) {
2079
+ throw new Error(`Checksum was requested for part ${dataPart.partNumber} using ${checksumAlgorithm}, but no checksum was returned in the response. ` +
2080
+ `If running in a browser, ensure your S3 bucket CORS configuration is enabled. `);
2081
+ }
2082
+ const completedPart = {
2083
+ PartNumber: dataPart.partNumber,
2084
+ ETag: partResponse.ETag,
2085
+ };
2086
+ if (partResponse.ChecksumCRC32)
2087
+ completedPart.ChecksumCRC32 = partResponse.ChecksumCRC32;
2088
+ if (partResponse.ChecksumCRC32C)
2089
+ completedPart.ChecksumCRC32C = partResponse.ChecksumCRC32C;
2090
+ if (partResponse.ChecksumCRC64NVME)
2091
+ completedPart.ChecksumCRC64NVME = partResponse.ChecksumCRC64NVME;
2092
+ if (partResponse.ChecksumSHA1)
2093
+ completedPart.ChecksumSHA1 = partResponse.ChecksumSHA1;
2094
+ if (partResponse.ChecksumSHA256)
2095
+ completedPart.ChecksumSHA256 = partResponse.ChecksumSHA256;
2096
+ completedParts.push(completedPart);
2097
+ const partLength = byteLength(dataPart.data);
2098
+ bytesUploaded += partLength;
2099
+ if (emitEvents) {
2100
+ this.dispatchEvent(Object.assign(new Event("bytesTransferred"), {
2101
+ request,
2102
+ snapshot: {
2103
+ transferredBytes: bytesUploaded,
2104
+ totalBytes: contentLength,
2105
+ },
2106
+ }));
2107
+ }
2108
+ }
2109
+ };
2110
+ const partUploads = [];
2111
+ for (let i = 0; i < this.maxConcurrentUploads; i++) {
2112
+ partUploads.push(uploadPart(dataFeeder).catch((error) => {
2113
+ abortController.abort();
2114
+ throw error;
2115
+ }));
2116
+ }
2117
+ await Promise.all(partUploads);
2118
+ if (completedParts.length !== expectedPartsCount) {
2119
+ throw new Error(`Expected ${expectedPartsCount} parts but uploaded ${completedParts.length} parts`);
2120
+ }
2121
+ completedParts.sort((a, b) => a.PartNumber - b.PartNumber);
2122
+ this.checkAborted(transferOptions);
2123
+ const { Body: _body, ...completeRequestFields } = request;
2124
+ const completeRequest = {
2125
+ ...completeRequestFields,
2126
+ UploadId: uploadId,
2127
+ MultipartUpload: { Parts: completedParts },
2128
+ MpuObjectSize: contentLength,
2129
+ };
2130
+ if (hasFullObjectChecksum) {
2131
+ completeRequest.ChecksumType = "FULL_OBJECT";
2132
+ if (request.ChecksumCRC32)
2133
+ completeRequest.ChecksumCRC32 = request.ChecksumCRC32;
2134
+ if (request.ChecksumCRC32C)
2135
+ completeRequest.ChecksumCRC32C = request.ChecksumCRC32C;
2136
+ if (request.ChecksumCRC64NVME)
2137
+ completeRequest.ChecksumCRC64NVME = request.ChecksumCRC64NVME;
2138
+ }
2139
+ try {
2140
+ return await this.s3.send(new CompleteMultipartUploadCommand(completeRequest), transferOptions);
2141
+ }
2142
+ catch (completeMpuError) {
2143
+ this.logger.warn(`CompleteMultipartUpload failed for upload ID ${uploadId}: ${completeMpuError.message}`);
2144
+ throw completeMpuError;
2145
+ }
2146
+ }
2147
+ catch (error) {
2148
+ const { Body: _abortBody, ...abortRequestFields } = request;
2149
+ const abortRequest = {
2150
+ ...abortRequestFields,
2151
+ UploadId: uploadId,
2152
+ };
2153
+ try {
2154
+ await this.s3.send(new AbortMultipartUploadCommand(abortRequest), transferOptions);
2155
+ }
2156
+ catch (abortError) {
2157
+ this.logger.warn(`Failed to abort multipart upload ${uploadId}:`, abortError);
2158
+ }
2159
+ throw error;
2160
+ }
2161
+ }
2162
+ async *traverseDirectory(source, options) {
2163
+ const visited = new Set();
2164
+ const dir = await opendir(source, { recursive: options.recursive });
2165
+ for await (const entry of dir) {
2166
+ const fullPath = join(entry.parentPath ?? entry.path ?? source, entry.name);
2167
+ if (entry.isSymbolicLink()) {
2168
+ if (!options.followSymbolicLinks)
2169
+ continue;
2170
+ const realPath = await realpath(fullPath);
2171
+ const linkStat = await stat(realPath);
2172
+ if (linkStat.isDirectory()) {
2173
+ if (visited.has(realPath)) {
2174
+ throw new Error(`Circular symbolic link detected: ${fullPath} -> ${realPath}`);
2175
+ }
2176
+ visited.add(realPath);
2177
+ yield* this.traverseDirectory(realPath, options);
2178
+ continue;
2179
+ }
2180
+ if (!linkStat.isFile())
2181
+ continue;
2182
+ }
2183
+ else if (!entry.isFile()) {
2184
+ continue;
2185
+ }
2186
+ yield fullPath;
2187
+ }
2188
+ }
2189
+ deriveS3Key(source, filePath, s3Prefix) {
2190
+ let relativePath = relative(source, filePath);
2191
+ if (sep !== "/") {
2192
+ relativePath = relativePath.split(sep).join("/");
2193
+ }
2194
+ if (!s3Prefix)
2195
+ return relativePath;
2196
+ const normalizedPrefix = s3Prefix.endsWith("/") ? s3Prefix : s3Prefix + "/";
2197
+ return normalizedPrefix + relativePath;
2198
+ }
2199
+ validateUploadPart(dataPart, partSize) {
2200
+ const actualPartSize = byteLength(dataPart.data);
2201
+ if (actualPartSize === undefined) {
2202
+ throw new Error(`A dataPart was generated without a measurable data chunk size for part number ${dataPart.partNumber}`);
2203
+ }
2204
+ if (dataPart.partNumber === 1 && dataPart.lastPart) {
2205
+ return;
2206
+ }
2207
+ if (!dataPart.lastPart && actualPartSize !== partSize) {
2208
+ throw new Error(`The byte size for part number ${dataPart.partNumber}, size ${actualPartSize} does not match expected size ${partSize}`);
2209
+ }
2210
+ }
2211
+ }
2212
+ export const bindS3TransferManager = (deps) => class S3TransferManager extends S3TransferManagerBase {
2213
+ constructor(config = {}) {
2214
+ super(config, deps);
2215
+ }
2216
+ };
2217
+ function matchesFilterRegExp(filter, value) {
2218
+ filter.lastIndex = 0;
2219
+ return filter.test(value);
2220
+ }
2221
+ function parseContentRangeStart(contentRange) {
2222
+ if (!contentRange)
2223
+ return undefined;
2224
+ const match = contentRange.match(/^bytes\s+(\d+)-/);
2225
+ return match ? Number(match[1]) : undefined;
2226
+ }
2227
+ export const internalEventHandler = {
2228
+ async afterInitialGetObject() { },
2229
+ };