@aws-sdk/lib-storage 3.32.0 → 3.36.0

package/dist-es/Upload.js

@@ -0,0 +1,261 @@
+import { __assign, __asyncValues, __awaiter, __extends, __generator } from "tslib";
+import { CompleteMultipartUploadCommand, CreateMultipartUploadCommand, PutObjectCommand, PutObjectTaggingCommand, UploadPartCommand, } from "@aws-sdk/client-s3";
+import { EventEmitter } from "events";
+import { getChunk } from "./chunker";
+import { byteLength } from "./bytelength";
+import { AbortController } from "@aws-sdk/abort-controller";
+var MIN_PART_SIZE = 1024 * 1024 * 5;
+var Upload = (function (_super) {
+    __extends(Upload, _super);
+    function Upload(options) {
+        var _this = _super.call(this) || this;
+        _this.MAX_PARTS = 10000;
+        _this.queueSize = 4;
+        _this.partSize = MIN_PART_SIZE;
+        _this.leavePartsOnError = false;
+        _this.tags = [];
+        _this.concurrentUploaders = [];
+        _this.uploadedParts = [];
+        _this.isMultiPart = true;
+        _this.queueSize = options.queueSize || _this.queueSize;
+        _this.partSize = options.partSize || _this.partSize;
+        _this.leavePartsOnError = options.leavePartsOnError || _this.leavePartsOnError;
+        _this.tags = options.tags || _this.tags;
+        _this.client = options.client;
+        _this.params = options.params;
+        _this.__validateInput();
+        _this.totalBytes = byteLength(_this.params.Body);
+        _this.bytesUploadedSoFar = 0;
+        _this.abortController = new AbortController();
+        return _this;
+    }
+    Upload.prototype.abort = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                this.abortController.abort();
+                return [2];
+            });
+        });
+    };
+    Upload.prototype.done = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4, Promise.race([this.__doMultipartUpload(), this.__abortTimeout(this.abortController.signal)])];
+                    case 1: return [2, _a.sent()];
+                }
+            });
+        });
+    };
+    Upload.prototype.on = function (event, listener) {
+        this.uploadEvent = event;
+        _super.prototype.on.call(this, event, listener);
+    };
+    Upload.prototype.__uploadUsingPut = function (dataPart) {
+        return __awaiter(this, void 0, void 0, function () {
+            var params, putResult, totalSize;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        this.isMultiPart = false;
+                        params = __assign(__assign({}, this.params), { Body: dataPart.data });
+                        return [4, this.client.send(new PutObjectCommand(params))];
+                    case 1:
+                        putResult = _a.sent();
+                        this.putResponse = putResult;
+                        totalSize = byteLength(dataPart.data);
+                        this.__notifyProgress({
+                            loaded: totalSize,
+                            total: totalSize,
+                            part: 1,
+                            Key: this.params.Key,
+                            Bucket: this.params.Bucket,
+                        });
+                        return [2];
+                }
+            });
+        });
+    };
+    Upload.prototype.__createMultipartUpload = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var createCommandParams, createMultipartUploadResult;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.createMultiPartPromise) {
+                            createCommandParams = __assign(__assign({}, this.params), { Body: undefined });
+                            this.createMultiPartPromise = this.client.send(new CreateMultipartUploadCommand(createCommandParams));
+                        }
+                        return [4, this.createMultiPartPromise];
+                    case 1:
+                        createMultipartUploadResult = _a.sent();
+                        this.uploadId = createMultipartUploadResult.UploadId;
+                        return [2];
+                }
+            });
+        });
+    };
+    Upload.prototype.__doConcurrentUpload = function (dataFeeder) {
+        var dataFeeder_1, dataFeeder_1_1;
+        var e_1, _a;
+        return __awaiter(this, void 0, void 0, function () {
+            var dataPart, partResult, e_2, e_1_1;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _b.trys.push([0, 12, 13, 18]);
+                        dataFeeder_1 = __asyncValues(dataFeeder);
+                        _b.label = 1;
+                    case 1: return [4, dataFeeder_1.next()];
+                    case 2:
+                        if (!(dataFeeder_1_1 = _b.sent(), !dataFeeder_1_1.done)) return [3, 11];
+                        dataPart = dataFeeder_1_1.value;
+                        if (this.uploadedParts.length > this.MAX_PARTS) {
+                            throw new Error("Exceeded " + this.MAX_PARTS + " as part of the upload to " + this.params.Key + " and " + this.params.Bucket + ".");
+                        }
+                        _b.label = 3;
+                    case 3:
+                        _b.trys.push([3, 9, , 10]);
+                        if (this.abortController.signal.aborted) {
+                            return [2];
+                        }
+                        if (!(dataPart.partNumber === 1 && dataPart.lastPart)) return [3, 5];
+                        return [4, this.__uploadUsingPut(dataPart)];
+                    case 4: return [2, _b.sent()];
+                    case 5:
+                        if (!!this.uploadId) return [3, 7];
+                        return [4, this.__createMultipartUpload()];
+                    case 6:
+                        _b.sent();
+                        if (this.abortController.signal.aborted) {
+                            return [2];
+                        }
+                        _b.label = 7;
+                    case 7: return [4, this.client.send(new UploadPartCommand(__assign(__assign({}, this.params), { UploadId: this.uploadId, Body: dataPart.data, PartNumber: dataPart.partNumber })))];
+                    case 8:
+                        partResult = _b.sent();
+                        if (this.abortController.signal.aborted) {
+                            return [2];
+                        }
+                        this.uploadedParts.push({
+                            PartNumber: dataPart.partNumber,
+                            ETag: partResult.ETag,
+                        });
+                        this.bytesUploadedSoFar += byteLength(dataPart.data);
+                        this.__notifyProgress({
+                            loaded: this.bytesUploadedSoFar,
+                            total: this.totalBytes,
+                            part: dataPart.partNumber,
+                            Key: this.params.Key,
+                            Bucket: this.params.Bucket,
+                        });
+                        return [3, 10];
+                    case 9:
+                        e_2 = _b.sent();
+                        if (!this.uploadId) {
+                            throw e_2;
+                        }
+                        if (this.leavePartsOnError) {
+                            throw e_2;
+                        }
+                        return [3, 10];
+                    case 10: return [3, 1];
+                    case 11: return [3, 18];
+                    case 12:
+                        e_1_1 = _b.sent();
+                        e_1 = { error: e_1_1 };
+                        return [3, 18];
+                    case 13:
+                        _b.trys.push([13, , 16, 17]);
+                        if (!(dataFeeder_1_1 && !dataFeeder_1_1.done && (_a = dataFeeder_1.return))) return [3, 15];
+                        return [4, _a.call(dataFeeder_1)];
+                    case 14:
+                        _b.sent();
+                        _b.label = 15;
+                    case 15: return [3, 17];
+                    case 16:
+                        if (e_1) throw e_1.error;
+                        return [7];
+                    case 17: return [7];
+                    case 18: return [2];
+                }
+            });
+        });
+    };
+    Upload.prototype.__doMultipartUpload = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var dataFeeder, index, currentUpload, result, uploadCompleteParams;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        dataFeeder = getChunk(this.params.Body, this.partSize);
+                        for (index = 0; index < this.queueSize; index++) {
+                            currentUpload = this.__doConcurrentUpload(dataFeeder);
+                            this.concurrentUploaders.push(currentUpload);
+                        }
+                        return [4, Promise.all(this.concurrentUploaders)];
+                    case 1:
+                        _a.sent();
+                        if (this.abortController.signal.aborted) {
+                            throw Object.assign(new Error("Upload aborted."), { name: "AbortError" });
+                        }
+                        if (!this.isMultiPart) return [3, 3];
+                        this.uploadedParts.sort(function (a, b) { return a.PartNumber - b.PartNumber; });
+                        uploadCompleteParams = __assign(__assign({}, this.params), { Body: undefined, UploadId: this.uploadId, MultipartUpload: {
+                                Parts: this.uploadedParts,
+                            } });
+                        return [4, this.client.send(new CompleteMultipartUploadCommand(uploadCompleteParams))];
+                    case 2:
+                        result = _a.sent();
+                        return [3, 4];
+                    case 3:
+                        result = this.putResponse;
+                        _a.label = 4;
+                    case 4:
+                        if (!this.tags.length) return [3, 6];
+                        return [4, this.client.send(new PutObjectTaggingCommand(__assign(__assign({}, this.params), { Tagging: {
+                                    TagSet: this.tags,
+                                } })))];
+                    case 5:
+                        _a.sent();
+                        _a.label = 6;
+                    case 6: return [2, result];
+                }
+            });
+        });
+    };
+    Upload.prototype.__notifyProgress = function (progress) {
+        if (this.uploadEvent) {
+            this.emit(this.uploadEvent, progress);
+        }
+    };
+    Upload.prototype.__abortTimeout = function (abortSignal) {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                return [2, new Promise(function (resolve, reject) {
+                        abortSignal.onabort = function () {
+                            var abortError = new Error("Upload aborted.");
+                            abortError.name = "AbortError";
+                            reject(abortError);
+                        };
+                    })];
+            });
+        });
+    };
+    Upload.prototype.__validateInput = function () {
+        if (!this.params) {
+            throw new Error("InputError: Upload requires params to be passed to upload.");
+        }
+        if (!this.client) {
+            throw new Error("InputError: Upload requires a AWS client to do uploads with.");
+        }
+        if (this.partSize < MIN_PART_SIZE) {
+            throw new Error("EntityTooSmall: Your proposed upload partsize [" + this.partSize + "] is smaller than the minimum allowed size [" + MIN_PART_SIZE + "] (5MB)");
+        }
+        if (this.queueSize < 1) {
+            throw new Error("Queue size: Must have at least one uploading queue.");
+        }
+    };
+    return Upload;
+}(EventEmitter));
+export { Upload };

package/dist-es/bytelength.js

@@ -0,0 +1,25 @@
+import { ClientDefaultValues } from "./runtimeConfig";
+export var byteLength = function (input) {
+    if (input === null || input === undefined)
+        return 0;
+    if (typeof input === "string")
+        input = Buffer.from(input);
+    if (typeof input.byteLength === "number") {
+        return input.byteLength;
+    }
+    else if (typeof input.length === "number") {
+        return input.length;
+    }
+    else if (typeof input.size === "number") {
+        return input.size;
+    }
+    else if (typeof input.path === "string") {
+        try {
+            return ClientDefaultValues.lstatSync(input.path).size;
+        }
+        catch (error) {
+            return undefined;
+        }
+    }
+    return undefined;
+};

package/dist-es/chunker.js

@@ -0,0 +1,26 @@
+import { Readable } from "stream";
+import { Buffer } from "buffer";
+import { getChunkBuffer } from "./chunks/getChunkBuffer";
+import { getChunkStream } from "./chunks/getChunkStream";
+import { getDataReadableStream } from "./chunks/getDataReadableStream";
+import { getDataReadable } from "./chunks/getDataReadable";
+export var getChunk = function (data, partSize) {
+    if (data instanceof Buffer) {
+        return getChunkBuffer(data, partSize);
+    }
+    else if (data instanceof Readable) {
+        return getChunkStream(data, partSize, getDataReadable);
+    }
+    else if (data instanceof String || typeof data === "string" || data instanceof Uint8Array) {
+        return getChunkBuffer(Buffer.from(data), partSize);
+    }
+    if (typeof data.stream === "function") {
+        return getChunkStream(data.stream(), partSize, getDataReadableStream);
+    }
+    else if (data instanceof ReadableStream) {
+        return getChunkStream(data, partSize, getDataReadableStream);
+    }
+    else {
+        throw new Error("Body Data is unsupported format, expected data to be one of: string | Uint8Array | Buffer | Readable | ReadableStream | Blob;.");
+    }
+};

package/dist-es/chunks/getChunkBuffer.js

@@ -0,0 +1,37 @@
+import { __asyncGenerator, __await, __generator } from "tslib";
+export function getChunkBuffer(data, partSize) {
+    return __asyncGenerator(this, arguments, function getChunkBuffer_1() {
+        var partNumber, startByte, endByte;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    partNumber = 1;
+                    startByte = 0;
+                    endByte = partSize;
+                    _a.label = 1;
+                case 1:
+                    if (!(endByte < data.byteLength)) return [3, 4];
+                    return [4, __await({
+                            partNumber: partNumber,
+                            data: data.slice(startByte, endByte),
+                        })];
+                case 2: return [4, _a.sent()];
+                case 3:
+                    _a.sent();
+                    partNumber += 1;
+                    startByte = endByte;
+                    endByte = startByte + partSize;
+                    return [3, 1];
+                case 4: return [4, __await({
+                        partNumber: partNumber,
+                        data: data.slice(startByte),
+                        lastPart: true,
+                    })];
+                case 5: return [4, _a.sent()];
+                case 6:
+                    _a.sent();
+                    return [2];
+            }
+        });
+    });
+}

package/dist-es/chunks/getChunkStream.js

@@ -0,0 +1,68 @@
+import { __asyncGenerator, __asyncValues, __await, __generator } from "tslib";
+import { Buffer } from "buffer";
+export function getChunkStream(data, partSize, getNextData) {
+    return __asyncGenerator(this, arguments, function getChunkStream_1() {
+        var partNumber, currentBuffer, _a, _b, datum, dataChunk, e_1_1;
+        var e_1, _c;
+        return __generator(this, function (_d) {
+            switch (_d.label) {
+                case 0:
+                    partNumber = 1;
+                    currentBuffer = { chunks: [], length: 0 };
+                    _d.label = 1;
+                case 1:
+                    _d.trys.push([1, 9, 10, 15]);
+                    _a = __asyncValues(getNextData(data));
+                    _d.label = 2;
+                case 2: return [4, __await(_a.next())];
+                case 3:
+                    if (!(_b = _d.sent(), !_b.done)) return [3, 8];
+                    datum = _b.value;
+                    currentBuffer.chunks.push(datum);
+                    currentBuffer.length += datum.length;
+                    _d.label = 4;
+                case 4:
+                    if (!(currentBuffer.length >= partSize)) return [3, 7];
+                    dataChunk = currentBuffer.chunks.length > 1 ? Buffer.concat(currentBuffer.chunks) : currentBuffer.chunks[0];
+                    return [4, __await({
+                            partNumber: partNumber,
+                            data: dataChunk.slice(0, partSize),
+                        })];
+                case 5: return [4, _d.sent()];
+                case 6:
+                    _d.sent();
+                    currentBuffer.chunks = [dataChunk.slice(partSize)];
+                    currentBuffer.length = currentBuffer.chunks[0].length;
+                    partNumber += 1;
+                    return [3, 4];
+                case 7: return [3, 2];
+                case 8: return [3, 15];
+                case 9:
+                    e_1_1 = _d.sent();
+                    e_1 = { error: e_1_1 };
+                    return [3, 15];
+                case 10:
+                    _d.trys.push([10, , 13, 14]);
+                    if (!(_b && !_b.done && (_c = _a.return))) return [3, 12];
+                    return [4, __await(_c.call(_a))];
+                case 11:
+                    _d.sent();
+                    _d.label = 12;
+                case 12: return [3, 14];
+                case 13:
+                    if (e_1) throw e_1.error;
+                    return [7];
+                case 14: return [7];
+                case 15: return [4, __await({
+                        partNumber: partNumber,
+                        data: Buffer.concat(currentBuffer.chunks),
+                        lastPart: true,
+                    })];
+                case 16: return [4, _d.sent()];
+                case 17:
+                    _d.sent();
+                    return [2];
+            }
+        });
+    });
+}

package/dist-es/chunks/getDataReadable.js

@@ -0,0 +1,44 @@
+import { __asyncGenerator, __asyncValues, __await, __generator } from "tslib";
+import { Buffer } from "buffer";
+export function getDataReadable(data) {
+    return __asyncGenerator(this, arguments, function getDataReadable_1() {
+        var data_1, data_1_1, chunk, e_1_1;
+        var e_1, _a;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    _b.trys.push([0, 7, 8, 13]);
+                    data_1 = __asyncValues(data);
+                    _b.label = 1;
+                case 1: return [4, __await(data_1.next())];
+                case 2:
+                    if (!(data_1_1 = _b.sent(), !data_1_1.done)) return [3, 6];
+                    chunk = data_1_1.value;
+                    return [4, __await(Buffer.from(chunk))];
+                case 3: return [4, _b.sent()];
+                case 4:
+                    _b.sent();
+                    _b.label = 5;
+                case 5: return [3, 1];
+                case 6: return [3, 13];
+                case 7:
+                    e_1_1 = _b.sent();
+                    e_1 = { error: e_1_1 };
+                    return [3, 13];
+                case 8:
+                    _b.trys.push([8, , 11, 12]);
+                    if (!(data_1_1 && !data_1_1.done && (_a = data_1.return))) return [3, 10];
+                    return [4, __await(_a.call(data_1))];
+                case 9:
+                    _b.sent();
+                    _b.label = 10;
+                case 10: return [3, 12];
+                case 11:
+                    if (e_1) throw e_1.error;
+                    return [7];
+                case 12: return [7];
+                case 13: return [2];
+            }
+        });
+    });
+}

package/dist-es/chunks/getDataReadableStream.js

@@ -0,0 +1,38 @@
+import { __asyncGenerator, __await, __generator } from "tslib";
+import { Buffer } from "buffer";
+export function getDataReadableStream(data) {
+    return __asyncGenerator(this, arguments, function getDataReadableStream_1() {
+        var reader, _a, done, value, e_1;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    reader = data.getReader();
+                    _b.label = 1;
+                case 1:
+                    _b.trys.push([1, 9, 10, 11]);
+                    _b.label = 2;
+                case 2:
+                    if (!true) return [3, 8];
+                    return [4, __await(reader.read())];
+                case 3:
+                    _a = _b.sent(), done = _a.done, value = _a.value;
+                    if (!done) return [3, 5];
+                    return [4, __await(void 0)];
+                case 4: return [2, _b.sent()];
+                case 5: return [4, __await(Buffer.from(value))];
+                case 6: return [4, _b.sent()];
+                case 7:
+                    _b.sent();
+                    return [3, 2];
+                case 8: return [3, 11];
+                case 9:
+                    e_1 = _b.sent();
+                    throw e_1;
+                case 10:
+                    reader.releaseLock();
+                    return [7];
+                case 11: return [2];
+            }
+        });
+    });
+}

package/dist-es/runtimeConfig.browser.js

@@ -0,0 +1,3 @@
+import { __assign } from "tslib";
+import { ClientSharedValues } from "./runtimeConfig.shared";
+export var ClientDefaultValues = __assign(__assign({}, ClientSharedValues), { runtime: "browser" });

package/dist-es/runtimeConfig.js

@@ -0,0 +1,4 @@
+import { __assign } from "tslib";
+import { ClientSharedValues } from "./runtimeConfig.shared";
+import { lstatSync } from "fs";
+export var ClientDefaultValues = __assign(__assign({}, ClientSharedValues), { runtime: "node", lstatSync: lstatSync });

package/dist-es/runtimeConfig.native.js

@@ -0,0 +1,3 @@
+import { __assign } from "tslib";
+import { ClientDefaultValues as BrowserDefaults } from "./runtimeConfig.browser";
+export var ClientDefaultValues = __assign(__assign({}, BrowserDefaults), { runtime: "react-native" });

package/dist-es/runtimeConfig.shared.js

@@ -0,0 +1,3 @@
+export var ClientSharedValues = {
+    lstatSync: function () { },
+};

package/dist-types/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./Upload";
+export * from "./types";

package/dist-types/runtimeConfig.browser.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @internal
+ */
+export declare const ClientDefaultValues: {
+    runtime: string;
+    lstatSync: () => void;
+};

package/dist-types/runtimeConfig.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import { lstatSync } from "fs";
+/**
+ * @internal
+ */
+export declare const ClientDefaultValues: {
+    runtime: string;
+    lstatSync: typeof lstatSync;
+};

package/dist-types/runtimeConfig.native.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @internal
+ */
+export declare const ClientDefaultValues: {
+    runtime: string;
+    lstatSync: () => void;
+};

package/dist-types/runtimeConfig.shared.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @internal
+ */
+export declare const ClientSharedValues: {
+    lstatSync: () => void;
+};

package/dist-types/ts3.4/runtimeConfig.browser.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @internal
+ */
+export declare const ClientDefaultValues: {
+    runtime: string;
+    lstatSync: () => void;
+};

package/dist-types/ts3.4/runtimeConfig.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import { lstatSync } from "fs";
+/**
+ * @internal
+ */
+export declare const ClientDefaultValues: {
+    runtime: string;
+    lstatSync: typeof lstatSync;
+};

package/dist-types/ts3.4/runtimeConfig.native.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @internal
+ */
+export declare const ClientDefaultValues: {
+    runtime: string;
+    lstatSync: () => void;
+};

package/dist-types/ts3.4/runtimeConfig.shared.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @internal
+ */
+export declare const ClientSharedValues: {
+    lstatSync: () => void;
+};