@aws-sdk/lib-storage 3.578.0 → 3.583.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -4
- package/dist-cjs/index.js +102 -74
- package/dist-es/Upload.js +92 -70
- package/dist-types/Upload.d.ts +20 -8
- package/dist-types/ts3.4/Upload.d.ts +11 -7
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -16,12 +16,23 @@ try {
|
|
|
16
16
|
client: new S3({}) || new S3Client({}),
|
|
17
17
|
params: { Bucket, Key, Body },
|
|
18
18
|
|
|
19
|
+
// optional tags
|
|
19
20
|
tags: [
|
|
20
21
|
/*...*/
|
|
21
|
-
],
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
],
|
|
23
|
+
|
|
24
|
+
// additional optional fields show default values below:
|
|
25
|
+
|
|
26
|
+
// (optional) concurrency configuration
|
|
27
|
+
queueSize: 4,
|
|
28
|
+
|
|
29
|
+
// (optional) size of each part, in bytes, at least 5MB
|
|
30
|
+
partSize: 1024 * 1024 * 5,
|
|
31
|
+
|
|
32
|
+
// (optional) when true, do not automatically call AbortMultipartUpload when
|
|
33
|
+
// a multipart upload fails to complete. You should then manually handle
|
|
34
|
+
// the leftover parts.
|
|
35
|
+
leavePartsOnError: false,
|
|
25
36
|
});
|
|
26
37
|
|
|
27
38
|
parallelUploads3.on("httpUploadProgress", (progress) => {
|
package/dist-cjs/index.js
CHANGED
|
@@ -175,7 +175,7 @@ var _Upload = class _Upload extends import_events.EventEmitter {
|
|
|
175
175
|
constructor(options) {
|
|
176
176
|
super();
|
|
177
177
|
/**
|
|
178
|
-
* S3 multipart upload does not allow more than
|
|
178
|
+
* S3 multipart upload does not allow more than 10,000 parts.
|
|
179
179
|
*/
|
|
180
180
|
this.MAX_PARTS = 1e4;
|
|
181
181
|
// Defaults.
|
|
@@ -184,8 +184,11 @@ var _Upload = class _Upload extends import_events.EventEmitter {
|
|
|
184
184
|
this.leavePartsOnError = false;
|
|
185
185
|
this.tags = [];
|
|
186
186
|
this.concurrentUploaders = [];
|
|
187
|
+
this.abortMultipartUploadCommand = null;
|
|
187
188
|
this.uploadedParts = [];
|
|
189
|
+
this.uploadEnqueuedPartsCount = 0;
|
|
188
190
|
this.isMultiPart = true;
|
|
191
|
+
this.sent = false;
|
|
189
192
|
this.queueSize = options.queueSize || this.queueSize;
|
|
190
193
|
this.partSize = options.partSize || this.partSize;
|
|
191
194
|
this.leavePartsOnError = options.leavePartsOnError || this.leavePartsOnError;
|
|
@@ -201,6 +204,12 @@ var _Upload = class _Upload extends import_events.EventEmitter {
|
|
|
201
204
|
this.abortController.abort();
|
|
202
205
|
}
|
|
203
206
|
async done() {
|
|
207
|
+
if (this.sent) {
|
|
208
|
+
throw new Error(
|
|
209
|
+
"@aws-sdk/lib-storage: this instance of Upload has already executed .done(). Create a new instance."
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
this.sent = true;
|
|
204
213
|
return await Promise.race([this.__doMultipartUpload(), this.__abortTimeout(this.abortController.signal)]);
|
|
205
214
|
}
|
|
206
215
|
on(event, listener) {
|
|
@@ -275,84 +284,49 @@ var _Upload = class _Upload extends import_events.EventEmitter {
|
|
|
275
284
|
async __createMultipartUpload() {
|
|
276
285
|
if (!this.createMultiPartPromise) {
|
|
277
286
|
const createCommandParams = { ...this.params, Body: void 0 };
|
|
278
|
-
this.createMultiPartPromise = this.client.send(new import_client_s3.CreateMultipartUploadCommand(createCommandParams))
|
|
287
|
+
this.createMultiPartPromise = this.client.send(new import_client_s3.CreateMultipartUploadCommand(createCommandParams)).then((createMpuResponse) => {
|
|
288
|
+
this.abortMultipartUploadCommand = new import_client_s3.AbortMultipartUploadCommand({
|
|
289
|
+
Bucket: this.params.Bucket,
|
|
290
|
+
Key: this.params.Key,
|
|
291
|
+
UploadId: createMpuResponse.UploadId
|
|
292
|
+
});
|
|
293
|
+
return createMpuResponse;
|
|
294
|
+
});
|
|
279
295
|
}
|
|
280
296
|
return this.createMultiPartPromise;
|
|
281
297
|
}
|
|
282
298
|
async __doConcurrentUpload(dataFeeder) {
|
|
283
299
|
for await (const dataPart of dataFeeder) {
|
|
284
|
-
if (this.
|
|
300
|
+
if (this.uploadEnqueuedPartsCount > this.MAX_PARTS) {
|
|
285
301
|
throw new Error(
|
|
286
|
-
`Exceeded ${this.MAX_PARTS}
|
|
302
|
+
`Exceeded ${this.MAX_PARTS} parts in multipart upload to Bucket: ${this.params.Bucket} Key: ${this.params.Key}.`
|
|
287
303
|
);
|
|
288
304
|
}
|
|
289
|
-
|
|
305
|
+
if (this.abortController.signal.aborted) {
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
if (dataPart.partNumber === 1 && dataPart.lastPart) {
|
|
309
|
+
return await this.__uploadUsingPut(dataPart);
|
|
310
|
+
}
|
|
311
|
+
if (!this.uploadId) {
|
|
312
|
+
const { UploadId } = await this.__createMultipartUpload();
|
|
313
|
+
this.uploadId = UploadId;
|
|
290
314
|
if (this.abortController.signal.aborted) {
|
|
291
315
|
return;
|
|
292
316
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
const partSize = byteLength(dataPart.data) || 0;
|
|
304
|
-
const requestHandler = this.client.config.requestHandler;
|
|
305
|
-
const eventEmitter = requestHandler instanceof import_events.EventEmitter ? requestHandler : null;
|
|
306
|
-
let lastSeenBytes = 0;
|
|
307
|
-
const uploadEventListener = /* @__PURE__ */ __name((event, request) => {
|
|
308
|
-
const requestPartSize = Number(request.query["partNumber"]) || -1;
|
|
309
|
-
if (requestPartSize !== dataPart.partNumber) {
|
|
310
|
-
return;
|
|
311
|
-
}
|
|
312
|
-
if (event.total && partSize) {
|
|
313
|
-
this.bytesUploadedSoFar += event.loaded - lastSeenBytes;
|
|
314
|
-
lastSeenBytes = event.loaded;
|
|
315
|
-
}
|
|
316
|
-
this.__notifyProgress({
|
|
317
|
-
loaded: this.bytesUploadedSoFar,
|
|
318
|
-
total: this.totalBytes,
|
|
319
|
-
part: dataPart.partNumber,
|
|
320
|
-
Key: this.params.Key,
|
|
321
|
-
Bucket: this.params.Bucket
|
|
322
|
-
});
|
|
323
|
-
}, "uploadEventListener");
|
|
324
|
-
if (eventEmitter !== null) {
|
|
325
|
-
eventEmitter.on("xhr.upload.progress", uploadEventListener);
|
|
326
|
-
}
|
|
327
|
-
const partResult = await this.client.send(
|
|
328
|
-
new import_client_s3.UploadPartCommand({
|
|
329
|
-
...this.params,
|
|
330
|
-
UploadId: this.uploadId,
|
|
331
|
-
Body: dataPart.data,
|
|
332
|
-
PartNumber: dataPart.partNumber
|
|
333
|
-
})
|
|
334
|
-
);
|
|
335
|
-
if (eventEmitter !== null) {
|
|
336
|
-
eventEmitter.off("xhr.upload.progress", uploadEventListener);
|
|
337
|
-
}
|
|
338
|
-
if (this.abortController.signal.aborted) {
|
|
317
|
+
}
|
|
318
|
+
const partSize = byteLength(dataPart.data) || 0;
|
|
319
|
+
const requestHandler = this.client.config.requestHandler;
|
|
320
|
+
const eventEmitter = requestHandler instanceof import_events.EventEmitter ? requestHandler : null;
|
|
321
|
+
let lastSeenBytes = 0;
|
|
322
|
+
const uploadEventListener = /* @__PURE__ */ __name((event, request) => {
|
|
323
|
+
const requestPartSize = Number(request.query["partNumber"]) || -1;
|
|
324
|
+
if (requestPartSize !== dataPart.partNumber) {
|
|
339
325
|
return;
|
|
340
326
|
}
|
|
341
|
-
if (
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
);
|
|
345
|
-
}
|
|
346
|
-
this.uploadedParts.push({
|
|
347
|
-
PartNumber: dataPart.partNumber,
|
|
348
|
-
ETag: partResult.ETag,
|
|
349
|
-
...partResult.ChecksumCRC32 && { ChecksumCRC32: partResult.ChecksumCRC32 },
|
|
350
|
-
...partResult.ChecksumCRC32C && { ChecksumCRC32C: partResult.ChecksumCRC32C },
|
|
351
|
-
...partResult.ChecksumSHA1 && { ChecksumSHA1: partResult.ChecksumSHA1 },
|
|
352
|
-
...partResult.ChecksumSHA256 && { ChecksumSHA256: partResult.ChecksumSHA256 }
|
|
353
|
-
});
|
|
354
|
-
if (eventEmitter === null) {
|
|
355
|
-
this.bytesUploadedSoFar += partSize;
|
|
327
|
+
if (event.total && partSize) {
|
|
328
|
+
this.bytesUploadedSoFar += event.loaded - lastSeenBytes;
|
|
329
|
+
lastSeenBytes = event.loaded;
|
|
356
330
|
}
|
|
357
331
|
this.__notifyProgress({
|
|
358
332
|
loaded: this.bytesUploadedSoFar,
|
|
@@ -361,24 +335,66 @@ var _Upload = class _Upload extends import_events.EventEmitter {
|
|
|
361
335
|
Key: this.params.Key,
|
|
362
336
|
Bucket: this.params.Bucket
|
|
363
337
|
});
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
}
|
|
368
|
-
if (this.leavePartsOnError) {
|
|
369
|
-
throw e;
|
|
370
|
-
}
|
|
338
|
+
}, "uploadEventListener");
|
|
339
|
+
if (eventEmitter !== null) {
|
|
340
|
+
eventEmitter.on("xhr.upload.progress", uploadEventListener);
|
|
371
341
|
}
|
|
342
|
+
this.uploadEnqueuedPartsCount += 1;
|
|
343
|
+
const partResult = await this.client.send(
|
|
344
|
+
new import_client_s3.UploadPartCommand({
|
|
345
|
+
...this.params,
|
|
346
|
+
UploadId: this.uploadId,
|
|
347
|
+
Body: dataPart.data,
|
|
348
|
+
PartNumber: dataPart.partNumber
|
|
349
|
+
})
|
|
350
|
+
);
|
|
351
|
+
if (eventEmitter !== null) {
|
|
352
|
+
eventEmitter.off("xhr.upload.progress", uploadEventListener);
|
|
353
|
+
}
|
|
354
|
+
if (this.abortController.signal.aborted) {
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
if (!partResult.ETag) {
|
|
358
|
+
throw new Error(
|
|
359
|
+
`Part ${dataPart.partNumber} is missing ETag in UploadPart response. Missing Bucket CORS configuration for ETag header?`
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
this.uploadedParts.push({
|
|
363
|
+
PartNumber: dataPart.partNumber,
|
|
364
|
+
ETag: partResult.ETag,
|
|
365
|
+
...partResult.ChecksumCRC32 && { ChecksumCRC32: partResult.ChecksumCRC32 },
|
|
366
|
+
...partResult.ChecksumCRC32C && { ChecksumCRC32C: partResult.ChecksumCRC32C },
|
|
367
|
+
...partResult.ChecksumSHA1 && { ChecksumSHA1: partResult.ChecksumSHA1 },
|
|
368
|
+
...partResult.ChecksumSHA256 && { ChecksumSHA256: partResult.ChecksumSHA256 }
|
|
369
|
+
});
|
|
370
|
+
if (eventEmitter === null) {
|
|
371
|
+
this.bytesUploadedSoFar += partSize;
|
|
372
|
+
}
|
|
373
|
+
this.__notifyProgress({
|
|
374
|
+
loaded: this.bytesUploadedSoFar,
|
|
375
|
+
total: this.totalBytes,
|
|
376
|
+
part: dataPart.partNumber,
|
|
377
|
+
Key: this.params.Key,
|
|
378
|
+
Bucket: this.params.Bucket
|
|
379
|
+
});
|
|
372
380
|
}
|
|
373
381
|
}
|
|
374
382
|
async __doMultipartUpload() {
|
|
375
383
|
const dataFeeder = getChunk(this.params.Body, this.partSize);
|
|
384
|
+
const concurrentUploaderFailures = [];
|
|
376
385
|
for (let index = 0; index < this.queueSize; index++) {
|
|
377
|
-
const currentUpload = this.__doConcurrentUpload(dataFeeder)
|
|
386
|
+
const currentUpload = this.__doConcurrentUpload(dataFeeder).catch((err) => {
|
|
387
|
+
concurrentUploaderFailures.push(err);
|
|
388
|
+
});
|
|
378
389
|
this.concurrentUploaders.push(currentUpload);
|
|
379
390
|
}
|
|
380
391
|
await Promise.all(this.concurrentUploaders);
|
|
392
|
+
if (concurrentUploaderFailures.length >= 1) {
|
|
393
|
+
await this.markUploadAsAborted();
|
|
394
|
+
throw concurrentUploaderFailures[0];
|
|
395
|
+
}
|
|
381
396
|
if (this.abortController.signal.aborted) {
|
|
397
|
+
await this.markUploadAsAborted();
|
|
382
398
|
throw Object.assign(new Error("Upload aborted."), { name: "AbortError" });
|
|
383
399
|
}
|
|
384
400
|
let result;
|
|
@@ -399,6 +415,7 @@ var _Upload = class _Upload extends import_events.EventEmitter {
|
|
|
399
415
|
} else {
|
|
400
416
|
result = this.singleUploadResult;
|
|
401
417
|
}
|
|
418
|
+
this.abortMultipartUploadCommand = null;
|
|
402
419
|
if (this.tags.length) {
|
|
403
420
|
await this.client.send(
|
|
404
421
|
new import_client_s3.PutObjectTaggingCommand({
|
|
@@ -411,6 +428,17 @@ var _Upload = class _Upload extends import_events.EventEmitter {
|
|
|
411
428
|
}
|
|
412
429
|
return result;
|
|
413
430
|
}
|
|
431
|
+
/**
|
|
432
|
+
* Abort the last multipart upload in progress
|
|
433
|
+
* if we know the upload id, the user did not specify to leave the parts, and
|
|
434
|
+
* we have a prepared AbortMultipartUpload command.
|
|
435
|
+
*/
|
|
436
|
+
async markUploadAsAborted() {
|
|
437
|
+
if (this.uploadId && !this.leavePartsOnError && null !== this.abortMultipartUploadCommand) {
|
|
438
|
+
await this.client.send(this.abortMultipartUploadCommand);
|
|
439
|
+
this.abortMultipartUploadCommand = null;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
414
442
|
__notifyProgress(progress) {
|
|
415
443
|
if (this.uploadEvent) {
|
|
416
444
|
this.emit(this.uploadEvent, progress);
|
package/dist-es/Upload.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CompleteMultipartUploadCommand, CreateMultipartUploadCommand, PutObjectCommand, PutObjectTaggingCommand, UploadPartCommand, } from "@aws-sdk/client-s3";
|
|
1
|
+
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, PutObjectCommand, PutObjectTaggingCommand, UploadPartCommand, } from "@aws-sdk/client-s3";
|
|
2
2
|
import { AbortController } from "@smithy/abort-controller";
|
|
3
3
|
import { getEndpointFromInstructions, toEndpointV1, } from "@smithy/middleware-endpoint";
|
|
4
4
|
import { extendedEncodeURIComponent } from "@smithy/smithy-client";
|
|
@@ -15,8 +15,11 @@ export class Upload extends EventEmitter {
|
|
|
15
15
|
this.leavePartsOnError = false;
|
|
16
16
|
this.tags = [];
|
|
17
17
|
this.concurrentUploaders = [];
|
|
18
|
+
this.abortMultipartUploadCommand = null;
|
|
18
19
|
this.uploadedParts = [];
|
|
20
|
+
this.uploadEnqueuedPartsCount = 0;
|
|
19
21
|
this.isMultiPart = true;
|
|
22
|
+
this.sent = false;
|
|
20
23
|
this.queueSize = options.queueSize || this.queueSize;
|
|
21
24
|
this.partSize = options.partSize || this.partSize;
|
|
22
25
|
this.leavePartsOnError = options.leavePartsOnError || this.leavePartsOnError;
|
|
@@ -32,6 +35,10 @@ export class Upload extends EventEmitter {
|
|
|
32
35
|
this.abortController.abort();
|
|
33
36
|
}
|
|
34
37
|
async done() {
|
|
38
|
+
if (this.sent) {
|
|
39
|
+
throw new Error("@aws-sdk/lib-storage: this instance of Upload has already executed .done(). Create a new instance.");
|
|
40
|
+
}
|
|
41
|
+
this.sent = true;
|
|
35
42
|
return await Promise.race([this.__doMultipartUpload(), this.__abortTimeout(this.abortController.signal)]);
|
|
36
43
|
}
|
|
37
44
|
on(event, listener) {
|
|
@@ -106,78 +113,49 @@ export class Upload extends EventEmitter {
|
|
|
106
113
|
async __createMultipartUpload() {
|
|
107
114
|
if (!this.createMultiPartPromise) {
|
|
108
115
|
const createCommandParams = { ...this.params, Body: undefined };
|
|
109
|
-
this.createMultiPartPromise = this.client
|
|
116
|
+
this.createMultiPartPromise = this.client
|
|
117
|
+
.send(new CreateMultipartUploadCommand(createCommandParams))
|
|
118
|
+
.then((createMpuResponse) => {
|
|
119
|
+
this.abortMultipartUploadCommand = new AbortMultipartUploadCommand({
|
|
120
|
+
Bucket: this.params.Bucket,
|
|
121
|
+
Key: this.params.Key,
|
|
122
|
+
UploadId: createMpuResponse.UploadId,
|
|
123
|
+
});
|
|
124
|
+
return createMpuResponse;
|
|
125
|
+
});
|
|
110
126
|
}
|
|
111
127
|
return this.createMultiPartPromise;
|
|
112
128
|
}
|
|
113
129
|
async __doConcurrentUpload(dataFeeder) {
|
|
114
130
|
for await (const dataPart of dataFeeder) {
|
|
115
|
-
if (this.
|
|
116
|
-
throw new Error(`Exceeded ${this.MAX_PARTS}
|
|
131
|
+
if (this.uploadEnqueuedPartsCount > this.MAX_PARTS) {
|
|
132
|
+
throw new Error(`Exceeded ${this.MAX_PARTS} parts in multipart upload to Bucket: ${this.params.Bucket} Key: ${this.params.Key}.`);
|
|
117
133
|
}
|
|
118
|
-
|
|
134
|
+
if (this.abortController.signal.aborted) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
if (dataPart.partNumber === 1 && dataPart.lastPart) {
|
|
138
|
+
return await this.__uploadUsingPut(dataPart);
|
|
139
|
+
}
|
|
140
|
+
if (!this.uploadId) {
|
|
141
|
+
const { UploadId } = await this.__createMultipartUpload();
|
|
142
|
+
this.uploadId = UploadId;
|
|
119
143
|
if (this.abortController.signal.aborted) {
|
|
120
144
|
return;
|
|
121
145
|
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
const partSize = byteLength(dataPart.data) || 0;
|
|
133
|
-
const requestHandler = this.client.config.requestHandler;
|
|
134
|
-
const eventEmitter = requestHandler instanceof EventEmitter ? requestHandler : null;
|
|
135
|
-
let lastSeenBytes = 0;
|
|
136
|
-
const uploadEventListener = (event, request) => {
|
|
137
|
-
const requestPartSize = Number(request.query["partNumber"]) || -1;
|
|
138
|
-
if (requestPartSize !== dataPart.partNumber) {
|
|
139
|
-
return;
|
|
140
|
-
}
|
|
141
|
-
if (event.total && partSize) {
|
|
142
|
-
this.bytesUploadedSoFar += event.loaded - lastSeenBytes;
|
|
143
|
-
lastSeenBytes = event.loaded;
|
|
144
|
-
}
|
|
145
|
-
this.__notifyProgress({
|
|
146
|
-
loaded: this.bytesUploadedSoFar,
|
|
147
|
-
total: this.totalBytes,
|
|
148
|
-
part: dataPart.partNumber,
|
|
149
|
-
Key: this.params.Key,
|
|
150
|
-
Bucket: this.params.Bucket,
|
|
151
|
-
});
|
|
152
|
-
};
|
|
153
|
-
if (eventEmitter !== null) {
|
|
154
|
-
eventEmitter.on("xhr.upload.progress", uploadEventListener);
|
|
155
|
-
}
|
|
156
|
-
const partResult = await this.client.send(new UploadPartCommand({
|
|
157
|
-
...this.params,
|
|
158
|
-
UploadId: this.uploadId,
|
|
159
|
-
Body: dataPart.data,
|
|
160
|
-
PartNumber: dataPart.partNumber,
|
|
161
|
-
}));
|
|
162
|
-
if (eventEmitter !== null) {
|
|
163
|
-
eventEmitter.off("xhr.upload.progress", uploadEventListener);
|
|
164
|
-
}
|
|
165
|
-
if (this.abortController.signal.aborted) {
|
|
146
|
+
}
|
|
147
|
+
const partSize = byteLength(dataPart.data) || 0;
|
|
148
|
+
const requestHandler = this.client.config.requestHandler;
|
|
149
|
+
const eventEmitter = requestHandler instanceof EventEmitter ? requestHandler : null;
|
|
150
|
+
let lastSeenBytes = 0;
|
|
151
|
+
const uploadEventListener = (event, request) => {
|
|
152
|
+
const requestPartSize = Number(request.query["partNumber"]) || -1;
|
|
153
|
+
if (requestPartSize !== dataPart.partNumber) {
|
|
166
154
|
return;
|
|
167
155
|
}
|
|
168
|
-
if (
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
this.uploadedParts.push({
|
|
172
|
-
PartNumber: dataPart.partNumber,
|
|
173
|
-
ETag: partResult.ETag,
|
|
174
|
-
...(partResult.ChecksumCRC32 && { ChecksumCRC32: partResult.ChecksumCRC32 }),
|
|
175
|
-
...(partResult.ChecksumCRC32C && { ChecksumCRC32C: partResult.ChecksumCRC32C }),
|
|
176
|
-
...(partResult.ChecksumSHA1 && { ChecksumSHA1: partResult.ChecksumSHA1 }),
|
|
177
|
-
...(partResult.ChecksumSHA256 && { ChecksumSHA256: partResult.ChecksumSHA256 }),
|
|
178
|
-
});
|
|
179
|
-
if (eventEmitter === null) {
|
|
180
|
-
this.bytesUploadedSoFar += partSize;
|
|
156
|
+
if (event.total && partSize) {
|
|
157
|
+
this.bytesUploadedSoFar += event.loaded - lastSeenBytes;
|
|
158
|
+
lastSeenBytes = event.loaded;
|
|
181
159
|
}
|
|
182
160
|
this.__notifyProgress({
|
|
183
161
|
loaded: this.bytesUploadedSoFar,
|
|
@@ -186,25 +164,62 @@ export class Upload extends EventEmitter {
|
|
|
186
164
|
Key: this.params.Key,
|
|
187
165
|
Bucket: this.params.Bucket,
|
|
188
166
|
});
|
|
167
|
+
};
|
|
168
|
+
if (eventEmitter !== null) {
|
|
169
|
+
eventEmitter.on("xhr.upload.progress", uploadEventListener);
|
|
189
170
|
}
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
171
|
+
this.uploadEnqueuedPartsCount += 1;
|
|
172
|
+
const partResult = await this.client.send(new UploadPartCommand({
|
|
173
|
+
...this.params,
|
|
174
|
+
UploadId: this.uploadId,
|
|
175
|
+
Body: dataPart.data,
|
|
176
|
+
PartNumber: dataPart.partNumber,
|
|
177
|
+
}));
|
|
178
|
+
if (eventEmitter !== null) {
|
|
179
|
+
eventEmitter.off("xhr.upload.progress", uploadEventListener);
|
|
180
|
+
}
|
|
181
|
+
if (this.abortController.signal.aborted) {
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
if (!partResult.ETag) {
|
|
185
|
+
throw new Error(`Part ${dataPart.partNumber} is missing ETag in UploadPart response. Missing Bucket CORS configuration for ETag header?`);
|
|
197
186
|
}
|
|
187
|
+
this.uploadedParts.push({
|
|
188
|
+
PartNumber: dataPart.partNumber,
|
|
189
|
+
ETag: partResult.ETag,
|
|
190
|
+
...(partResult.ChecksumCRC32 && { ChecksumCRC32: partResult.ChecksumCRC32 }),
|
|
191
|
+
...(partResult.ChecksumCRC32C && { ChecksumCRC32C: partResult.ChecksumCRC32C }),
|
|
192
|
+
...(partResult.ChecksumSHA1 && { ChecksumSHA1: partResult.ChecksumSHA1 }),
|
|
193
|
+
...(partResult.ChecksumSHA256 && { ChecksumSHA256: partResult.ChecksumSHA256 }),
|
|
194
|
+
});
|
|
195
|
+
if (eventEmitter === null) {
|
|
196
|
+
this.bytesUploadedSoFar += partSize;
|
|
197
|
+
}
|
|
198
|
+
this.__notifyProgress({
|
|
199
|
+
loaded: this.bytesUploadedSoFar,
|
|
200
|
+
total: this.totalBytes,
|
|
201
|
+
part: dataPart.partNumber,
|
|
202
|
+
Key: this.params.Key,
|
|
203
|
+
Bucket: this.params.Bucket,
|
|
204
|
+
});
|
|
198
205
|
}
|
|
199
206
|
}
|
|
200
207
|
async __doMultipartUpload() {
|
|
201
208
|
const dataFeeder = getChunk(this.params.Body, this.partSize);
|
|
209
|
+
const concurrentUploaderFailures = [];
|
|
202
210
|
for (let index = 0; index < this.queueSize; index++) {
|
|
203
|
-
const currentUpload = this.__doConcurrentUpload(dataFeeder)
|
|
211
|
+
const currentUpload = this.__doConcurrentUpload(dataFeeder).catch((err) => {
|
|
212
|
+
concurrentUploaderFailures.push(err);
|
|
213
|
+
});
|
|
204
214
|
this.concurrentUploaders.push(currentUpload);
|
|
205
215
|
}
|
|
206
216
|
await Promise.all(this.concurrentUploaders);
|
|
217
|
+
if (concurrentUploaderFailures.length >= 1) {
|
|
218
|
+
await this.markUploadAsAborted();
|
|
219
|
+
throw concurrentUploaderFailures[0];
|
|
220
|
+
}
|
|
207
221
|
if (this.abortController.signal.aborted) {
|
|
222
|
+
await this.markUploadAsAborted();
|
|
208
223
|
throw Object.assign(new Error("Upload aborted."), { name: "AbortError" });
|
|
209
224
|
}
|
|
210
225
|
let result;
|
|
@@ -226,6 +241,7 @@ export class Upload extends EventEmitter {
|
|
|
226
241
|
else {
|
|
227
242
|
result = this.singleUploadResult;
|
|
228
243
|
}
|
|
244
|
+
this.abortMultipartUploadCommand = null;
|
|
229
245
|
if (this.tags.length) {
|
|
230
246
|
await this.client.send(new PutObjectTaggingCommand({
|
|
231
247
|
...this.params,
|
|
@@ -236,6 +252,12 @@ export class Upload extends EventEmitter {
|
|
|
236
252
|
}
|
|
237
253
|
return result;
|
|
238
254
|
}
|
|
255
|
+
async markUploadAsAborted() {
|
|
256
|
+
if (this.uploadId && !this.leavePartsOnError && null !== this.abortMultipartUploadCommand) {
|
|
257
|
+
await this.client.send(this.abortMultipartUploadCommand);
|
|
258
|
+
this.abortMultipartUploadCommand = null;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
239
261
|
__notifyProgress(progress) {
|
|
240
262
|
if (this.uploadEvent) {
|
|
241
263
|
this.emit(this.uploadEvent, progress);
|
package/dist-types/Upload.d.ts
CHANGED
|
@@ -9,25 +9,31 @@ export interface RawDataPart {
|
|
|
9
9
|
}
|
|
10
10
|
export declare class Upload extends EventEmitter {
|
|
11
11
|
/**
|
|
12
|
-
* S3 multipart upload does not allow more than
|
|
12
|
+
* S3 multipart upload does not allow more than 10,000 parts.
|
|
13
13
|
*/
|
|
14
14
|
private MAX_PARTS;
|
|
15
|
-
private queueSize;
|
|
16
|
-
private partSize;
|
|
17
|
-
private leavePartsOnError;
|
|
18
|
-
private tags;
|
|
19
|
-
private client;
|
|
20
|
-
private params;
|
|
15
|
+
private readonly queueSize;
|
|
16
|
+
private readonly partSize;
|
|
17
|
+
private readonly leavePartsOnError;
|
|
18
|
+
private readonly tags;
|
|
19
|
+
private readonly client;
|
|
20
|
+
private readonly params;
|
|
21
21
|
private totalBytes?;
|
|
22
22
|
private bytesUploadedSoFar;
|
|
23
23
|
private abortController;
|
|
24
24
|
private concurrentUploaders;
|
|
25
25
|
private createMultiPartPromise?;
|
|
26
|
+
private abortMultipartUploadCommand;
|
|
26
27
|
private uploadedParts;
|
|
27
|
-
private
|
|
28
|
+
private uploadEnqueuedPartsCount;
|
|
29
|
+
/**
|
|
30
|
+
* Last UploadId if the upload was done with MultipartUpload and not PutObject.
|
|
31
|
+
*/
|
|
32
|
+
uploadId?: string;
|
|
28
33
|
uploadEvent?: string;
|
|
29
34
|
private isMultiPart;
|
|
30
35
|
private singleUploadResult?;
|
|
36
|
+
private sent;
|
|
31
37
|
constructor(options: Options);
|
|
32
38
|
abort(): Promise<void>;
|
|
33
39
|
done(): Promise<CompleteMultipartUploadCommandOutput>;
|
|
@@ -36,6 +42,12 @@ export declare class Upload extends EventEmitter {
|
|
|
36
42
|
private __createMultipartUpload;
|
|
37
43
|
private __doConcurrentUpload;
|
|
38
44
|
private __doMultipartUpload;
|
|
45
|
+
/**
|
|
46
|
+
* Abort the last multipart upload in progress
|
|
47
|
+
* if we know the upload id, the user did not specify to leave the parts, and
|
|
48
|
+
* we have a prepared AbortMultipartUpload command.
|
|
49
|
+
*/
|
|
50
|
+
private markUploadAsAborted;
|
|
39
51
|
private __notifyProgress;
|
|
40
52
|
private __abortTimeout;
|
|
41
53
|
private __validateInput;
|
|
@@ -8,22 +8,25 @@ export interface RawDataPart {
|
|
|
8
8
|
}
|
|
9
9
|
export declare class Upload extends EventEmitter {
|
|
10
10
|
private MAX_PARTS;
|
|
11
|
-
private queueSize;
|
|
12
|
-
private partSize;
|
|
13
|
-
private leavePartsOnError;
|
|
14
|
-
private tags;
|
|
15
|
-
private client;
|
|
16
|
-
private params;
|
|
11
|
+
private readonly queueSize;
|
|
12
|
+
private readonly partSize;
|
|
13
|
+
private readonly leavePartsOnError;
|
|
14
|
+
private readonly tags;
|
|
15
|
+
private readonly client;
|
|
16
|
+
private readonly params;
|
|
17
17
|
private totalBytes?;
|
|
18
18
|
private bytesUploadedSoFar;
|
|
19
19
|
private abortController;
|
|
20
20
|
private concurrentUploaders;
|
|
21
21
|
private createMultiPartPromise?;
|
|
22
|
+
private abortMultipartUploadCommand;
|
|
22
23
|
private uploadedParts;
|
|
23
|
-
private
|
|
24
|
+
private uploadEnqueuedPartsCount;
|
|
25
|
+
uploadId?: string;
|
|
24
26
|
uploadEvent?: string;
|
|
25
27
|
private isMultiPart;
|
|
26
28
|
private singleUploadResult?;
|
|
29
|
+
private sent;
|
|
27
30
|
constructor(options: Options);
|
|
28
31
|
abort(): Promise<void>;
|
|
29
32
|
done(): Promise<CompleteMultipartUploadCommandOutput>;
|
|
@@ -32,6 +35,7 @@ export declare class Upload extends EventEmitter {
|
|
|
32
35
|
private __createMultipartUpload;
|
|
33
36
|
private __doConcurrentUpload;
|
|
34
37
|
private __doMultipartUpload;
|
|
38
|
+
private markUploadAsAborted;
|
|
35
39
|
private __notifyProgress;
|
|
36
40
|
private __abortTimeout;
|
|
37
41
|
private __validateInput;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aws-sdk/lib-storage",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.583.0",
|
|
4
4
|
"description": "Storage higher order operation",
|
|
5
5
|
"main": "./dist-cjs/index.js",
|
|
6
6
|
"module": "./dist-es/index.js",
|
|
@@ -28,17 +28,17 @@
|
|
|
28
28
|
"dependencies": {
|
|
29
29
|
"@smithy/abort-controller": "^3.0.0",
|
|
30
30
|
"@smithy/middleware-endpoint": "^3.0.0",
|
|
31
|
-
"@smithy/smithy-client": "^3.0.
|
|
31
|
+
"@smithy/smithy-client": "^3.0.1",
|
|
32
32
|
"buffer": "5.6.0",
|
|
33
33
|
"events": "3.3.0",
|
|
34
34
|
"stream-browserify": "3.0.0",
|
|
35
35
|
"tslib": "^2.6.2"
|
|
36
36
|
},
|
|
37
37
|
"peerDependencies": {
|
|
38
|
-
"@aws-sdk/client-s3": "^3.
|
|
38
|
+
"@aws-sdk/client-s3": "^3.583.0"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
|
-
"@aws-sdk/client-s3": "3.
|
|
41
|
+
"@aws-sdk/client-s3": "3.583.0",
|
|
42
42
|
"@smithy/types": "^3.0.0",
|
|
43
43
|
"@tsconfig/recommended": "1.0.1",
|
|
44
44
|
"@types/node": "^16.18.96",
|