@azure/storage-file-share 12.27.0 → 12.28.0-beta.1

package/dist/commonjs/utils/BufferScheduler.js

@@ -0,0 +1,268 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BufferScheduler = void 0;
+const events_1 = require("events");
+/**
+ * This class accepts a Node.js Readable stream as input, and keeps reading data
+ * from the stream into the internal buffer structure, until it reaches maxBuffers.
+ * Every available buffer will try to trigger outgoingHandler.
+ *
+ * The internal buffer structure includes an incoming buffer array, and a outgoing
+ * buffer array. The incoming buffer array includes the "empty" buffers can be filled
+ * with new incoming data. The outgoing array includes the filled buffers to be
+ * handled by outgoingHandler. Every above buffer size is defined by parameter bufferSize.
+ *
+ * NUM_OF_ALL_BUFFERS = BUFFERS_IN_INCOMING + BUFFERS_IN_OUTGOING + BUFFERS_UNDER_HANDLING
+ *
+ * NUM_OF_ALL_BUFFERS lesser than or equal to maxBuffers
+ *
+ * PERFORMANCE IMPROVEMENT TIPS:
+ * 1. Input stream highWaterMark is better to set a same value with bufferSize
+ *    parameter, which will avoid Buffer.concat() operations.
+ * 2. Concurrency should set a smaller value than maxBuffers, which is helpful to
+ *    reduce the possibility when a outgoing handler waits for the stream data.
+ *    in this situation, outgoing handlers are blocked.
+ *    Outgoing queue shouldn't be empty.
+ */
+class BufferScheduler {
+    /**
+     * Creates an instance of BufferScheduler.
+     *
+     * @param readable - A Node.js Readable stream
+     * @param bufferSize - Buffer size of every maintained buffer
+     * @param maxBuffers - How many buffers can be allocated
+     * @param outgoingHandler - An async function scheduled to be
+     *                                          triggered when a buffer fully filled
+     *                                          with stream data
+     * @param concurrency - Concurrency of executing outgoingHandlers (greater than 0)
+     * @param encoding - [Optional] Encoding of Readable stream when it's a string stream
+     */
+    constructor(readable, bufferSize, maxBuffers, outgoingHandler, concurrency, encoding) {
+        /**
+         * An internal event emitter.
+         */
+        this.emitter = new events_1.EventEmitter();
+        /**
+         * An internal offset marker to track data offset in bytes of next outgoingHandler.
+         */
+        this.offset = 0;
+        /**
+         * An internal marker to track whether stream is end.
+         */
+        this.isStreamEnd = false;
+        /**
+         * An internal marker to track whether stream or outgoingHandler returns error.
+         */
+        this.isError = false;
+        /**
+         * How many handlers are executing.
+         */
+        this.executingOutgoingHandlers = 0;
+        /**
+         * How many buffers have been allocated.
+         */
+        this.numBuffers = 0;
+        /**
+         * Because this class doesn't know how much data every time stream pops, which
+         * is defined by highWaterMarker of the stream. So BufferScheduler will cache
+         * data received from the stream, when data in unresolvedDataArray exceeds the
+         * blockSize defined, it will try to concat a blockSize of buffer, fill into available
+         * buffers from incoming and push to outgoing array.
+         */
+        this.unresolvedDataArray = [];
+        /**
+         * How much data consisted in unresolvedDataArray.
+         */
+        this.unresolvedLength = 0;
+        /**
+         * The array includes all the available buffers can be used to fill data from stream.
+         */
+        this.incoming = [];
+        /**
+         * The array (queue) includes all the buffers filled from stream data.
+         */
+        this.outgoing = [];
+        if (bufferSize <= 0) {
+            throw new RangeError(`bufferSize must be larger than 0, current is ${bufferSize}`);
+        }
+        if (maxBuffers <= 0) {
+            throw new RangeError(`maxBuffers must be larger than 0, current is ${maxBuffers}`);
+        }
+        if (concurrency <= 0) {
+            throw new RangeError(`concurrency must be larger than 0, current is ${concurrency}`);
+        }
+        this.bufferSize = bufferSize;
+        this.maxBuffers = maxBuffers;
+        this.readable = readable;
+        this.outgoingHandler = outgoingHandler;
+        this.concurrency = concurrency;
+        this.encoding = encoding;
+    }
+    /**
+     * Start the scheduler, will return error when stream of any of the outgoingHandlers
+     * returns error.
+     *
+     */
+    async do() {
+        return new Promise((resolve, reject) => {
+            this.readable.on("data", (data) => {
+                data = typeof data === "string" ? Buffer.from(data, this.encoding) : data;
+                this.appendUnresolvedData(data);
+                if (!this.resolveData()) {
+                    this.readable.pause();
+                }
+            });
+            this.readable.on("error", (err) => {
+                this.emitter.emit("error", err);
+            });
+            this.readable.on("end", () => {
+                this.isStreamEnd = true;
+                this.emitter.emit("checkEnd");
+            });
+            this.emitter.on("error", (err) => {
+                this.isError = true;
+                this.readable.pause();
+                reject(err);
+            });
+            this.emitter.on("checkEnd", () => {
+                if (this.outgoing.length > 0) {
+                    this.triggerOutgoingHandlers();
+                    return;
+                }
+                if (this.isStreamEnd && this.executingOutgoingHandlers === 0) {
+                    if (this.unresolvedLength > 0 && this.unresolvedLength < this.bufferSize) {
+                        this.outgoingHandler(this.shiftBufferFromUnresolvedDataArray(), this.offset)
+                            .then(resolve)
+                            .catch(reject);
+                    }
+                    else if (this.unresolvedLength >= this.bufferSize) {
+                        return;
+                    }
+                    else {
+                        resolve();
+                    }
+                }
+            });
+        });
+    }
+    /**
+     * Insert a new data into unresolved array.
+     *
+     * @param data -
+     */
+    appendUnresolvedData(data) {
+        this.unresolvedDataArray.push(data);
+        this.unresolvedLength += data.length;
+    }
+    /**
+     * Try to shift a buffer with size in blockSize. The buffer returned may be less
+     * than blockSize when data in unresolvedDataArray is less than bufferSize.
+     *
+     */
+    shiftBufferFromUnresolvedDataArray() {
+        if (this.unresolvedLength >= this.bufferSize) {
+            if (this.bufferSize === this.unresolvedDataArray[0].length) {
+                this.unresolvedLength -= this.bufferSize;
+                return this.unresolvedDataArray.shift();
+            }
+            // Lazy concat because Buffer.concat highly drops performance
+            let merged = Buffer.concat(this.unresolvedDataArray, this.unresolvedLength);
+            const buffer = merged.slice(0, this.bufferSize);
+            merged = merged.slice(this.bufferSize);
+            this.unresolvedDataArray = [merged];
+            this.unresolvedLength -= buffer.length;
+            return buffer;
+        }
+        else if (this.unresolvedLength > 0) {
+            const merged = Buffer.concat(this.unresolvedDataArray, this.unresolvedLength);
+            this.unresolvedDataArray = [];
+            this.unresolvedLength = 0;
+            return merged;
+        }
+        else {
+            return Buffer.allocUnsafe(0);
+        }
+    }
+    /**
+     * Resolve data in unresolvedDataArray. For every buffer with size in blockSize
+     * shifted, it will try to get (or allocate a buffer) from incoming, and fill it,
+     * then push it into outgoing to be handled by outgoing handler.
+     *
+     * Return false when available buffers in incoming are not enough, else true.
+     *
+     * @returns Return false when buffers in incoming are not enough, else true.
+     */
+    resolveData() {
+        while (this.unresolvedLength >= this.bufferSize) {
+            let buffer;
+            if (this.incoming.length > 0) {
+                buffer = this.incoming.shift();
+            }
+            else {
+                if (this.numBuffers < this.maxBuffers) {
+                    buffer = Buffer.allocUnsafe(this.bufferSize);
+                    this.numBuffers++;
+                }
+                else {
+                    // No available buffer, wait for buffer returned
+                    return false;
+                }
+            }
+            buffer.fill(this.shiftBufferFromUnresolvedDataArray());
+            this.outgoing.push(buffer);
+            this.triggerOutgoingHandlers();
+        }
+        return true;
+    }
+    /**
+     * Try to trigger a outgoing handler for every buffer in outgoing. Stop when
+     * concurrency reaches.
+     */
+    async triggerOutgoingHandlers() {
+        let buffer;
+        do {
+            if (this.executingOutgoingHandlers >= this.concurrency) {
+                return;
+            }
+            buffer = this.outgoing.shift();
+            if (buffer) {
+                this.triggerOutgoingHandler(buffer);
+            }
+        } while (buffer);
+    }
+    /**
+     * Trigger a outgoing handler for a buffer shifted from outgoing.
+     *
+     * @param buffer -
+     */
+    async triggerOutgoingHandler(buffer) {
+        const bufferLength = buffer.length;
+        this.executingOutgoingHandlers++;
+        this.offset += bufferLength;
+        try {
+            await this.outgoingHandler(buffer, this.offset - bufferLength);
+        }
+        catch (err) {
+            this.emitter.emit("error", err);
+            return;
+        }
+        this.executingOutgoingHandlers--;
+        this.reuseBuffer(buffer);
+        this.emitter.emit("checkEnd");
+    }
+    /**
+     * Return buffer used by outgoing handler into incoming.
+     *
+     * @param buffer -
+     */
+    reuseBuffer(buffer) {
+        this.incoming.push(buffer);
+        if (!this.isError && this.resolveData() && !this.isStreamEnd) {
+            this.readable.resume();
+        }
+    }
+}
+exports.BufferScheduler = BufferScheduler;
+//# sourceMappingURL=BufferScheduler.js.map

package/dist/commonjs/utils/BufferScheduler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BufferScheduler.js","sourceRoot":"","sources":["../../../src/utils/BufferScheduler.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AAElC,mCAAsC;AAOtC;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAa,eAAe;IAuF1B;;;;;;;;;;;OAWG;IACH,YACE,QAA+B,EAC/B,UAAkB,EAClB,UAAkB,EAClB,eAAgC,EAChC,WAAmB,EACnB,QAAyB;QAlF3B;;WAEG;QACc,YAAO,GAAiB,IAAI,qBAAY,EAAE,CAAC;QAO5D;;WAEG;QACK,WAAM,GAAW,CAAC,CAAC;QAE3B;;WAEG;QACK,gBAAW,GAAY,KAAK,CAAC;QAErC;;WAEG;QACK,YAAO,GAAY,KAAK,CAAC;QAEjC;;WAEG;QACK,8BAAyB,GAAW,CAAC,CAAC;QAO9C;;WAEG;QACK,eAAU,GAAW,CAAC,CAAC;QAE/B;;;;;;WAMG;QACK,wBAAmB,GAAa,EAAE,CAAC;QAE3C;;WAEG;QACK,qBAAgB,GAAW,CAAC,CAAC;QAErC;;WAEG;QACK,aAAQ,GAAa,EAAE,CAAC;QAEhC;;WAEG;QACK,aAAQ,GAAa,EAAE,CAAC;QAsB9B,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,IAAI,UAAU,CAAC,gDAAgD,UAAU,EAAE,CAAC,CAAC;QACrF,CAAC;QAED,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,IAAI,UAAU,CAAC,gDAAgD,UAAU,EAAE,CAAC,CAAC;QACrF,CAAC;QAED,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,UAAU,CAAC,iDAAiD,WAAW,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,EAAE;QACb,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;gBAChC,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC1E,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;gBAEhC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACxB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;gBAC/B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7B,IAAI,CAAC,uBAAuB,EAAE,CAAC;oBAC/B,OAAO;gBACT,CAAC;gBAED,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,yBAAyB,KAAK,CAAC,EAAE,CAAC;oBAC7D,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,IAAI,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;wBACzE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC;6BACzE,IAAI,CAAC,OAAO,CAAC;6BACb,KAAK,CAAC,MAAM,CAAC,CAAC;oBACnB,CAAC;yBAAM,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpD,OAAO;oBACT,CAAC;yBAAM,CAAC;wBACN,OAAO,EAAE,CAAC;oBACZ,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,oBAAoB,CAAC,IAAY;QACvC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACK,kCAAkC;QACxC,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC7C,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC3D,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,UAAU,CAAC;gBACzC,OAAO,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAG,CAAC;YAC3C,CAAC;YAED,6DAA6D;YAC7D,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC5E,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAChD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACvC,IAAI,CAAC,mBAAmB,GAAG,CAAC,MAAM,CAAC,CAAC;YACpC,IAAI,CAAC,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC;YACvC,OAAO,MAAM,CAAC;QAChB,CAAC;aAAM,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC9E,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC;YAC9B,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;YAC1B,OAAO,MAAM,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,WAAW;QACjB,OAAO,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAChD,IAAI,MAAc,CAAC;YAEnB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAG,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;oBACtC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oBAC7C,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,CAAC;qBAAM,CAAC;oBACN,gDAAgD;oBAChD,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,EAAE,CAAC,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,uBAAuB;QACnC,IAAI,MAA0B,CAAC;QAC/B,GAAG,CAAC;YACF,IAAI,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACvD,OAAO;YACT,CAAC;YAED,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;YACtC,CAAC;QACH,CAAC,QAAQ,MAAM,EAAE;IACnB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,sBAAsB,CAAC,MAAc;QACjD,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;QAEnC,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC;QAE5B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QAED,IAAI,CAAC,yBAAyB,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,MAAc;QAChC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC7D,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;CACF;AA9SD,0CA8SC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { EventEmitter } from \"events\";\n\n/**\n * OutgoingHandler is an async function triggered by BufferScheduler.\n */\nexport declare type OutgoingHandler = (buffer: Buffer, offset?: number) => Promise<any>;\n\n/**\n * This class accepts a Node.js Readable stream as input, and keeps reading data\n * from the stream into the internal buffer structure, until it reaches maxBuffers.\n * Every available buffer will try to trigger outgoingHandler.\n *\n * The internal buffer structure includes an incoming buffer array, and a outgoing\n * buffer array. The incoming buffer array includes the \"empty\" buffers can be filled\n * with new incoming data. The outgoing array includes the filled buffers to be\n * handled by outgoingHandler. Every above buffer size is defined by parameter bufferSize.\n *\n * NUM_OF_ALL_BUFFERS = BUFFERS_IN_INCOMING + BUFFERS_IN_OUTGOING + BUFFERS_UNDER_HANDLING\n *\n * NUM_OF_ALL_BUFFERS lesser than or equal to maxBuffers\n *\n * PERFORMANCE IMPROVEMENT TIPS:\n * 1. Input stream highWaterMark is better to set a same value with bufferSize\n *    parameter, which will avoid Buffer.concat() operations.\n * 2. Concurrency should set a smaller value than maxBuffers, which is helpful to\n *    reduce the possibility when a outgoing handler waits for the stream data.\n *    in this situation, outgoing handlers are blocked.\n *    Outgoing queue shouldn't be empty.\n */\nexport class BufferScheduler {\n  /**\n   * Size of buffers in incoming and outgoing queues. This class will try to align\n   * data read from Readable stream into buffer chunks with bufferSize defined.\n   */\n  private readonly bufferSize: number;\n\n  /**\n   * How many buffers can be created or maintained.\n   */\n  private readonly maxBuffers: number;\n\n  /**\n   * A Node.js Readable stream.\n   */\n  private readonly readable: NodeJS.ReadableStream;\n\n  /**\n   * OutgoingHandler is an async function triggered by BufferScheduler when there\n   * are available buffers in outgoing array.\n   */\n  private readonly outgoingHandler: OutgoingHandler;\n\n  /**\n   * An internal event emitter.\n   */\n  private readonly emitter: EventEmitter = new EventEmitter();\n\n  /**\n   * Concurrency of executing outgoingHandlers. (0 lesser than concurrency lesser than or equal to maxBuffers)\n   */\n  private readonly concurrency: number;\n\n  /**\n   * An internal offset marker to track data offset in bytes of next outgoingHandler.\n   */\n  private offset: number = 0;\n\n  /**\n   * An internal marker to track whether stream is end.\n   */\n  private isStreamEnd: boolean = false;\n\n  /**\n   * An internal marker to track whether stream or outgoingHandler returns error.\n   */\n  private isError: boolean = false;\n\n  /**\n   * How many handlers are executing.\n   */\n  private executingOutgoingHandlers: number = 0;\n\n  /**\n   * Encoding of the input Readable stream which has string data type instead of Buffer.\n   */\n  private encoding?: BufferEncoding;\n\n  /**\n   * How many buffers have been allocated.\n   */\n  private numBuffers: number = 0;\n\n  /**\n   * Because this class doesn't know how much data every time stream pops, which\n   * is defined by highWaterMarker of the stream. So BufferScheduler will cache\n   * data received from the stream, when data in unresolvedDataArray exceeds the\n   * blockSize defined, it will try to concat a blockSize of buffer, fill into available\n   * buffers from incoming and push to outgoing array.\n   */\n  private unresolvedDataArray: Buffer[] = [];\n\n  /**\n   * How much data consisted in unresolvedDataArray.\n   */\n  private unresolvedLength: number = 0;\n\n  /**\n   * The array includes all the available buffers can be used to fill data from stream.\n   */\n  private incoming: Buffer[] = [];\n\n  /**\n   * The array (queue) includes all the buffers filled from stream data.\n   */\n  private outgoing: Buffer[] = [];\n\n  /**\n   * Creates an instance of BufferScheduler.\n   *\n   * @param readable - A Node.js Readable stream\n   * @param bufferSize - Buffer size of every maintained buffer\n   * @param maxBuffers - How many buffers can be allocated\n   * @param outgoingHandler - An async function scheduled to be\n   *                                          triggered when a buffer fully filled\n   *                                          with stream data\n   * @param concurrency - Concurrency of executing outgoingHandlers (greater than 0)\n   * @param encoding - [Optional] Encoding of Readable stream when it's a string stream\n   */\n  constructor(\n    readable: NodeJS.ReadableStream,\n    bufferSize: number,\n    maxBuffers: number,\n    outgoingHandler: OutgoingHandler,\n    concurrency: number,\n    encoding?: BufferEncoding,\n  ) {\n    if (bufferSize <= 0) {\n      throw new RangeError(`bufferSize must be larger than 0, current is ${bufferSize}`);\n    }\n\n    if (maxBuffers <= 0) {\n      throw new RangeError(`maxBuffers must be larger than 0, current is ${maxBuffers}`);\n    }\n\n    if (concurrency <= 0) {\n      throw new RangeError(`concurrency must be larger than 0, current is ${concurrency}`);\n    }\n\n    this.bufferSize = bufferSize;\n    this.maxBuffers = maxBuffers;\n    this.readable = readable;\n    this.outgoingHandler = outgoingHandler;\n    this.concurrency = concurrency;\n    this.encoding = encoding;\n  }\n\n  /**\n   * Start the scheduler, will return error when stream of any of the outgoingHandlers\n   * returns error.\n   *\n   */\n  public async do(): Promise<void> {\n    return new Promise<void>((resolve, reject) => {\n      this.readable.on(\"data\", (data) => {\n        data = typeof data === \"string\" ? Buffer.from(data, this.encoding) : data;\n        this.appendUnresolvedData(data);\n\n        if (!this.resolveData()) {\n          this.readable.pause();\n        }\n      });\n\n      this.readable.on(\"error\", (err) => {\n        this.emitter.emit(\"error\", err);\n      });\n\n      this.readable.on(\"end\", () => {\n        this.isStreamEnd = true;\n        this.emitter.emit(\"checkEnd\");\n      });\n\n      this.emitter.on(\"error\", (err) => {\n        this.isError = true;\n        this.readable.pause();\n        reject(err);\n      });\n\n      this.emitter.on(\"checkEnd\", () => {\n        if (this.outgoing.length > 0) {\n          this.triggerOutgoingHandlers();\n          return;\n        }\n\n        if (this.isStreamEnd && this.executingOutgoingHandlers === 0) {\n          if (this.unresolvedLength > 0 && this.unresolvedLength < this.bufferSize) {\n            this.outgoingHandler(this.shiftBufferFromUnresolvedDataArray(), this.offset)\n              .then(resolve)\n              .catch(reject);\n          } else if (this.unresolvedLength >= this.bufferSize) {\n            return;\n          } else {\n            resolve();\n          }\n        }\n      });\n    });\n  }\n\n  /**\n   * Insert a new data into unresolved array.\n   *\n   * @param data -\n   */\n  private appendUnresolvedData(data: Buffer): void {\n    this.unresolvedDataArray.push(data);\n    this.unresolvedLength += data.length;\n  }\n\n  /**\n   * Try to shift a buffer with size in blockSize. The buffer returned may be less\n   * than blockSize when data in unresolvedDataArray is less than bufferSize.\n   *\n   */\n  private shiftBufferFromUnresolvedDataArray(): Buffer {\n    if (this.unresolvedLength >= this.bufferSize) {\n      if (this.bufferSize === this.unresolvedDataArray[0].length) {\n        this.unresolvedLength -= this.bufferSize;\n        return this.unresolvedDataArray.shift()!;\n      }\n\n      // Lazy concat because Buffer.concat highly drops performance\n      let merged = Buffer.concat(this.unresolvedDataArray, this.unresolvedLength);\n      const buffer = merged.slice(0, this.bufferSize);\n      merged = merged.slice(this.bufferSize);\n      this.unresolvedDataArray = [merged];\n      this.unresolvedLength -= buffer.length;\n      return buffer;\n    } else if (this.unresolvedLength > 0) {\n      const merged = Buffer.concat(this.unresolvedDataArray, this.unresolvedLength);\n      this.unresolvedDataArray = [];\n      this.unresolvedLength = 0;\n      return merged;\n    } else {\n      return Buffer.allocUnsafe(0);\n    }\n  }\n\n  /**\n   * Resolve data in unresolvedDataArray. For every buffer with size in blockSize\n   * shifted, it will try to get (or allocate a buffer) from incoming, and fill it,\n   * then push it into outgoing to be handled by outgoing handler.\n   *\n   * Return false when available buffers in incoming are not enough, else true.\n   *\n   * @returns Return false when buffers in incoming are not enough, else true.\n   */\n  private resolveData(): boolean {\n    while (this.unresolvedLength >= this.bufferSize) {\n      let buffer: Buffer;\n\n      if (this.incoming.length > 0) {\n        buffer = this.incoming.shift()!;\n      } else {\n        if (this.numBuffers < this.maxBuffers) {\n          buffer = Buffer.allocUnsafe(this.bufferSize);\n          this.numBuffers++;\n        } else {\n          // No available buffer, wait for buffer returned\n          return false;\n        }\n      }\n\n      buffer.fill(this.shiftBufferFromUnresolvedDataArray());\n      this.outgoing.push(buffer);\n      this.triggerOutgoingHandlers();\n    }\n    return true;\n  }\n\n  /**\n   * Try to trigger a outgoing handler for every buffer in outgoing. Stop when\n   * concurrency reaches.\n   */\n  private async triggerOutgoingHandlers(): Promise<void> {\n    let buffer: Buffer | undefined;\n    do {\n      if (this.executingOutgoingHandlers >= this.concurrency) {\n        return;\n      }\n\n      buffer = this.outgoing.shift();\n      if (buffer) {\n        this.triggerOutgoingHandler(buffer);\n      }\n    } while (buffer);\n  }\n\n  /**\n   * Trigger a outgoing handler for a buffer shifted from outgoing.\n   *\n   * @param buffer -\n   */\n  private async triggerOutgoingHandler(buffer: Buffer): Promise<any> {\n    const bufferLength = buffer.length;\n\n    this.executingOutgoingHandlers++;\n    this.offset += bufferLength;\n\n    try {\n      await this.outgoingHandler(buffer, this.offset - bufferLength);\n    } catch (err: any) {\n      this.emitter.emit(\"error\", err);\n      return;\n    }\n\n    this.executingOutgoingHandlers--;\n    this.reuseBuffer(buffer);\n    this.emitter.emit(\"checkEnd\");\n  }\n\n  /**\n   * Return buffer used by outgoing handler into incoming.\n   *\n   * @param buffer -\n   */\n  private reuseBuffer(buffer: Buffer): void {\n    this.incoming.push(buffer);\n    if (!this.isError && this.resolveData() && !this.isStreamEnd) {\n      this.readable.resume();\n    }\n  }\n}\n"]}

package/dist/commonjs/utils/RetriableReadableStream.d.ts

@@ -0,0 +1,58 @@
+import type { TransferProgressEvent } from "@azure/core-rest-pipeline";
+import { Readable } from "node:stream";
+export type ReadableStreamGetter = (offset: number) => Promise<NodeJS.ReadableStream>;
+export interface RetriableReadableStreamOptions {
+    /**
+     * Max retry count (greater than or equal to 0), undefined or invalid value means no retry
+     */
+    maxRetryRequests?: number;
+    /**
+     * Read progress event handler
+     */
+    onProgress?: (progress: TransferProgressEvent) => void;
+    /**
+     * Debug purpose only. Used to inject an unexpected end to existing internal stream,
+     * to test stream retry works well or not.
+     *
+     * When assign it to true, for next incoming "data" event of internal stream,
+     * RetriableReadableStream will try to emit an "end" event to existing internal
+     * stream to force it end and start retry from the breaking point.
+     * The value will then update to "undefined", once the injection works.
+     */
+    doInjectErrorOnce?: boolean;
+}
+/**
+ * ONLY AVAILABLE IN NODE.JS RUNTIME.
+ *
+ * A Node.js ReadableStream will internally retry when internal ReadableStream unexpected ends.
+ */
+export declare class RetriableReadableStream extends Readable {
+    private start;
+    private offset;
+    private end;
+    private getter;
+    private source;
+    private retries;
+    private maxRetryRequests;
+    private onProgress?;
+    private options;
+    /**
+     * Creates an instance of RetriableReadableStream.
+     *
+     * @param source - The current ReadableStream returned from getter
+     * @param getter - A method calling downloading request returning
+     *                                      a new ReadableStream from specified offset
+     * @param offset - Offset position in original data source to read
+     * @param count - How much data in original data source to read
+     * @param options -
+     */
+    constructor(source: NodeJS.ReadableStream, getter: ReadableStreamGetter, offset: number, count: number, options?: RetriableReadableStreamOptions);
+    _read(): void;
+    private setSourceEventHandlers;
+    private removeSourceEventHandlers;
+    private sourceDataHandler;
+    private sourceAbortedHandler;
+    private sourceErrorOrEndHandler;
+    _destroy(error: Error | null, callback: (error?: Error) => void): void;
+}
+//# sourceMappingURL=RetriableReadableStream.d.ts.map

package/dist/commonjs/utils/RetriableReadableStream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RetriableReadableStream.d.ts","sourceRoot":"","sources":["../../../src/utils/RetriableReadableStream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,MAAM,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAEtF,MAAM,WAAW,8BAA8B;IAC7C;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAEvD;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;;;GAIG;AACH,qBAAa,uBAAwB,SAAQ,QAAQ;IACnD,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAwB;IACtC,OAAO,CAAC,OAAO,CAAa;IAC5B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,UAAU,CAAC,CAA4C;IAC/D,OAAO,CAAC,OAAO,CAAiC;IAEhD;;;;;;;;;OASG;gBAED,MAAM,EAAE,MAAM,CAAC,cAAc,EAC7B,MAAM,EAAE,oBAAoB,EAC5B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,8BAAmC;IAgBvC,KAAK,IAAI,IAAI;IAIpB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,yBAAyB;IAOjC,OAAO,CAAC,iBAAiB,CAmBvB;IAEF,OAAO,CAAC,oBAAoB,CAG1B;IAEF,OAAO,CAAC,uBAAuB,CAiD7B;IAEF,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;CAOvE"}

package/dist/commonjs/utils/RetriableReadableStream.js

@@ -0,0 +1,123 @@
+"use strict";
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RetriableReadableStream = void 0;
+const abort_controller_1 = require("@azure/abort-controller");
+const node_stream_1 = require("node:stream");
+/**
+ * ONLY AVAILABLE IN NODE.JS RUNTIME.
+ *
+ * A Node.js ReadableStream will internally retry when internal ReadableStream unexpected ends.
+ */
+class RetriableReadableStream extends node_stream_1.Readable {
+    /**
+     * Creates an instance of RetriableReadableStream.
+     *
+     * @param source - The current ReadableStream returned from getter
+     * @param getter - A method calling downloading request returning
+     *                                      a new ReadableStream from specified offset
+     * @param offset - Offset position in original data source to read
+     * @param count - How much data in original data source to read
+     * @param options -
+     */
+    constructor(source, getter, offset, count, options = {}) {
+        super();
+        this.retries = 0;
+        this.sourceDataHandler = (data) => {
+            if (this.options.doInjectErrorOnce) {
+                this.options.doInjectErrorOnce = undefined;
+                this.source.pause();
+                this.sourceErrorOrEndHandler();
+                this.source.destroy();
+                return;
+            }
+            // console.log(
+            //   `Offset: ${this.offset}, Received ${data.length} from internal stream`
+            // );
+            this.offset += data.length;
+            if (this.onProgress) {
+                this.onProgress({ loadedBytes: this.offset - this.start });
+            }
+            if (!this.push(data)) {
+                this.source.pause();
+            }
+        };
+        this.sourceAbortedHandler = () => {
+            const abortError = new abort_controller_1.AbortError("The operation was aborted.");
+            this.destroy(abortError);
+        };
+        this.sourceErrorOrEndHandler = (err) => {
+            if (err && err.name === "AbortError") {
+                this.destroy(err);
+                return;
+            }
+            // console.log(
+            //   `Source stream emits end or error, offset: ${
+            //     this.offset
+            //   }, dest end : ${this.end}`
+            // );
+            this.removeSourceEventHandlers();
+            if (this.offset - 1 === this.end) {
+                this.push(null);
+            }
+            else if (this.offset <= this.end) {
+                // console.log(
+                //   `retries: ${this.retries}, max retries: ${this.maxRetries}`
+                // );
+                if (this.retries < this.maxRetryRequests) {
+                    this.retries += 1;
+                    this.getter(this.offset)
+                        .then((newSource) => {
+                        this.source = newSource;
+                        this.setSourceEventHandlers();
+                        return;
+                    })
+                        .catch((error) => {
+                        this.destroy(error);
+                    });
+                }
+                else {
+                    this.destroy(new Error(`Data corruption failure: received less data than required and reached maxRetires limitation. Received data offset: ${this.offset - 1}, data needed offset: ${this.end}, retries: ${this.retries}, max retries: ${this.maxRetryRequests}`));
+                }
+            }
+            else {
+                this.destroy(new Error(`Data corruption failure: Received more data than original request, data needed offset is ${this.end}, received offset: ${this.offset - 1}`));
+            }
+        };
+        this.getter = getter;
+        this.source = source;
+        this.start = offset;
+        this.offset = offset;
+        this.end = offset + count - 1;
+        this.maxRetryRequests =
+            options.maxRetryRequests && options.maxRetryRequests >= 0 ? options.maxRetryRequests : 0;
+        this.onProgress = options.onProgress;
+        this.options = options;
+        this.setSourceEventHandlers();
+    }
+    _read() {
+        this.source.resume();
+    }
+    setSourceEventHandlers() {
+        this.source.on("data", this.sourceDataHandler);
+        this.source.on("end", this.sourceErrorOrEndHandler);
+        this.source.on("error", this.sourceErrorOrEndHandler);
+        // needed for Node14
+        this.source.on("aborted", this.sourceAbortedHandler);
+    }
+    removeSourceEventHandlers() {
+        this.source.removeListener("data", this.sourceDataHandler);
+        this.source.removeListener("end", this.sourceErrorOrEndHandler);
+        this.source.removeListener("error", this.sourceErrorOrEndHandler);
+        this.source.removeListener("aborted", this.sourceAbortedHandler);
+    }
+    _destroy(error, callback) {
+        // remove listener from source and release source
+        this.removeSourceEventHandlers();
+        this.source.destroy();
+        callback(error === null ? undefined : error);
+    }
+}
+exports.RetriableReadableStream = RetriableReadableStream;
+//# sourceMappingURL=RetriableReadableStream.js.map

package/dist/commonjs/utils/RetriableReadableStream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RetriableReadableStream.js","sourceRoot":"","sources":["../../../src/utils/RetriableReadableStream.ts"],"names":[],"mappings":";AAAA,uCAAuC;AACvC,kCAAkC;;;AAElC,8DAAqD;AAErD,6CAAuC;AA2BvC;;;;GAIG;AACH,MAAa,uBAAwB,SAAQ,sBAAQ;IAWnD;;;;;;;;;OASG;IACH,YACE,MAA6B,EAC7B,MAA4B,EAC5B,MAAc,EACd,KAAa,EACb,UAA0C,EAAE;QAE5C,KAAK,EAAE,CAAC;QAtBF,YAAO,GAAW,CAAC,CAAC;QAuDpB,sBAAiB,GAAG,CAAC,IAAY,EAAE,EAAE;YAC3C,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBACnC,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACpB,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAmB,CAAC,OAAO,EAAE,CAAC;gBACpC,OAAO;YACT,CAAC;YAED,eAAe;YACf,2EAA2E;YAC3E,KAAK;YACL,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;YAC3B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAC7D,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;QACH,CAAC,CAAC;QAEM,yBAAoB,GAAG,GAAG,EAAE;YAClC,MAAM,UAAU,GAAG,IAAI,6BAAU,CAAC,4BAA4B,CAAC,CAAC;YAChE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC3B,CAAC,CAAC;QAEM,4BAAuB,GAAG,CAAC,GAAW,EAAE,EAAE;YAChD,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAClB,OAAO;YACT,CAAC;YAED,eAAe;YACf,kDAAkD;YAClD,kBAAkB;YAClB,+BAA+B;YAC/B,KAAK;YACL,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;gBACjC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACnC,eAAe;gBACf,gEAAgE;gBAChE,KAAK;gBACL,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACzC,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;yBACrB,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE;wBAClB,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;wBACxB,IAAI,CAAC,sBAAsB,EAAE,CAAC;wBAC9B,OAAO;oBACT,CAAC,CAAC;yBACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;wBACf,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACtB,CAAC,CAAC,CAAC;gBACP,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,OAAO,CACV,IAAI,KAAK,CACP,sHACE,IAAI,CAAC,MAAM,GAAG,CAChB,yBAAyB,IAAI,CAAC,GAAG,cAAc,IAAI,CAAC,OAAO,kBACzD,IAAI,CAAC,gBACP,EAAE,CACH,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,OAAO,CACV,IAAI,KAAK,CACP,4FACE,IAAI,CAAC,GACP,sBAAsB,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CACxC,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC;QA3GA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,GAAG,GAAG,MAAM,GAAG,KAAK,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,gBAAgB;YACnB,OAAO,CAAC,gBAAgB,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3F,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QAEvB,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAChC,CAAC;IAEM,KAAK;QACV,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC;IAEO,sBAAsB;QAC5B,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACtD,oBAAoB;QACpB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACvD,CAAC;IAEO,yBAAyB;QAC/B,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC3D,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAChE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACnE,CAAC;IA+ED,QAAQ,CAAC,KAAmB,EAAE,QAAiC;QAC7D,iDAAiD;QACjD,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAChC,IAAI,CAAC,MAAmB,CAAC,OAAO,EAAE,CAAC;QAEpC,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC;CACF;AAjJD,0DAiJC","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\n\nimport { AbortError } from \"@azure/abort-controller\";\nimport type { TransferProgressEvent } from \"@azure/core-rest-pipeline\";\nimport { Readable } from \"node:stream\";\n\nexport type ReadableStreamGetter = (offset: number) => Promise<NodeJS.ReadableStream>;\n\nexport interface RetriableReadableStreamOptions {\n  /**\n   * Max retry count (greater than or equal to 0), undefined or invalid value means no retry\n   */\n  maxRetryRequests?: number;\n\n  /**\n   * Read progress event handler\n   */\n  onProgress?: (progress: TransferProgressEvent) => void;\n\n  /**\n   * Debug purpose only. Used to inject an unexpected end to existing internal stream,\n   * to test stream retry works well or not.\n   *\n   * When assign it to true, for next incoming \"data\" event of internal stream,\n   * RetriableReadableStream will try to emit an \"end\" event to existing internal\n   * stream to force it end and start retry from the breaking point.\n   * The value will then update to \"undefined\", once the injection works.\n   */\n  doInjectErrorOnce?: boolean;\n}\n\n/**\n * ONLY AVAILABLE IN NODE.JS RUNTIME.\n *\n * A Node.js ReadableStream will internally retry when internal ReadableStream unexpected ends.\n */\nexport class RetriableReadableStream extends Readable {\n  private start: number;\n  private offset: number;\n  private end: number;\n  private getter: ReadableStreamGetter;\n  private source: NodeJS.ReadableStream;\n  private retries: number = 0;\n  private maxRetryRequests: number;\n  private onProgress?: (progress: TransferProgressEvent) => void;\n  private options: RetriableReadableStreamOptions;\n\n  /**\n   * Creates an instance of RetriableReadableStream.\n   *\n   * @param source - The current ReadableStream returned from getter\n   * @param getter - A method calling downloading request returning\n   *                                      a new ReadableStream from specified offset\n   * @param offset - Offset position in original data source to read\n   * @param count - How much data in original data source to read\n   * @param options -\n   */\n  public constructor(\n    source: NodeJS.ReadableStream,\n    getter: ReadableStreamGetter,\n    offset: number,\n    count: number,\n    options: RetriableReadableStreamOptions = {},\n  ) {\n    super();\n    this.getter = getter;\n    this.source = source;\n    this.start = offset;\n    this.offset = offset;\n    this.end = offset + count - 1;\n    this.maxRetryRequests =\n      options.maxRetryRequests && options.maxRetryRequests >= 0 ? options.maxRetryRequests : 0;\n    this.onProgress = options.onProgress;\n    this.options = options;\n\n    this.setSourceEventHandlers();\n  }\n\n  public _read(): void {\n    this.source.resume();\n  }\n\n  private setSourceEventHandlers() {\n    this.source.on(\"data\", this.sourceDataHandler);\n    this.source.on(\"end\", this.sourceErrorOrEndHandler);\n    this.source.on(\"error\", this.sourceErrorOrEndHandler);\n    // needed for Node14\n    this.source.on(\"aborted\", this.sourceAbortedHandler);\n  }\n\n  private removeSourceEventHandlers() {\n    this.source.removeListener(\"data\", this.sourceDataHandler);\n    this.source.removeListener(\"end\", this.sourceErrorOrEndHandler);\n    this.source.removeListener(\"error\", this.sourceErrorOrEndHandler);\n    this.source.removeListener(\"aborted\", this.sourceAbortedHandler);\n  }\n\n  private sourceDataHandler = (data: Buffer) => {\n    if (this.options.doInjectErrorOnce) {\n      this.options.doInjectErrorOnce = undefined;\n      this.source.pause();\n      this.sourceErrorOrEndHandler();\n      (this.source as Readable).destroy();\n      return;\n    }\n\n    // console.log(\n    //   `Offset: ${this.offset}, Received ${data.length} from internal stream`\n    // );\n    this.offset += data.length;\n    if (this.onProgress) {\n      this.onProgress({ loadedBytes: this.offset - this.start });\n    }\n    if (!this.push(data)) {\n      this.source.pause();\n    }\n  };\n\n  private sourceAbortedHandler = () => {\n    const abortError = new AbortError(\"The operation was aborted.\");\n    this.destroy(abortError);\n  };\n\n  private sourceErrorOrEndHandler = (err?: Error) => {\n    if (err && err.name === \"AbortError\") {\n      this.destroy(err);\n      return;\n    }\n\n    // console.log(\n    //   `Source stream emits end or error, offset: ${\n    //     this.offset\n    //   }, dest end : ${this.end}`\n    // );\n    this.removeSourceEventHandlers();\n    if (this.offset - 1 === this.end) {\n      this.push(null);\n    } else if (this.offset <= this.end) {\n      // console.log(\n      //   `retries: ${this.retries}, max retries: ${this.maxRetries}`\n      // );\n      if (this.retries < this.maxRetryRequests) {\n        this.retries += 1;\n        this.getter(this.offset)\n          .then((newSource) => {\n            this.source = newSource;\n            this.setSourceEventHandlers();\n            return;\n          })\n          .catch((error) => {\n            this.destroy(error);\n          });\n      } else {\n        this.destroy(\n          new Error(\n            `Data corruption failure: received less data than required and reached maxRetires limitation. Received data offset: ${\n              this.offset - 1\n            }, data needed offset: ${this.end}, retries: ${this.retries}, max retries: ${\n              this.maxRetryRequests\n            }`,\n          ),\n        );\n      }\n    } else {\n      this.destroy(\n        new Error(\n          `Data corruption failure: Received more data than original request, data needed offset is ${\n            this.end\n          }, received offset: ${this.offset - 1}`,\n        ),\n      );\n    }\n  };\n\n  _destroy(error: Error | null, callback: (error?: Error) => void): void {\n    // remove listener from source and release source\n    this.removeSourceEventHandlers();\n    (this.source as Readable).destroy();\n\n    callback(error === null ? undefined : error);\n  }\n}\n"]}

package/dist/commonjs/utils/constants.d.ts

@@ -0,0 +1,55 @@
+export declare const SDK_VERSION: string;
+export declare const SERVICE_VERSION: string;
+export declare const FILE_MAX_SIZE_BYTES: number;
+export declare const FILE_RANGE_MAX_SIZE_BYTES: number;
+export declare const DEFAULT_MAX_DOWNLOAD_RETRY_REQUESTS: number;
+export declare const DEFAULT_HIGH_LEVEL_CONCURRENCY: number;
+export declare const REQUEST_TIMEOUT: number;
+/**
+ * The OAuth scope to use with Azure Storage.
+ */
+export declare const StorageOAuthScopes: string | string[];
+export declare const URLConstants: {
+    Parameters: {
+        FORCE_BROWSER_NO_CACHE: string;
+        SHARE_SNAPSHOT: string;
+        SIGNATURE: string;
+        TIMEOUT: string;
+    };
+};
+export declare const HttpUrlConnection: {
+    HTTP_CONFLICT: number;
+    HTTP_NOT_FOUND: number;
+    HTTP_PRECON_FAILED: number;
+    HTTP_RANGE_NOT_SATISFIABLE: number;
+};
+export declare const HeaderConstants: {
+    AUTHORIZATION: string;
+    AUTHORIZATION_SCHEME: string;
+    CONTENT_ENCODING: string;
+    CONTENT_LANGUAGE: string;
+    CONTENT_LENGTH: string;
+    CONTENT_MD5: string;
+    CONTENT_TYPE: string;
+    COOKIE: string;
+    DATE: string;
+    IF_MATCH: string;
+    IF_MODIFIED_SINCE: string;
+    IF_NONE_MATCH: string;
+    IF_UNMODIFIED_SINCE: string;
+    PREFIX_FOR_STORAGE: string;
+    RANGE: string;
+    USER_AGENT: string;
+    X_MS_CLIENT_REQUEST_ID: string;
+    X_MS_COPY_SOURCE: string;
+    X_MS_CopySourceErrorCode: string;
+    X_MS_DATE: string;
+};
+export declare const ETagNone = "";
+export declare const ETagAny = "*";
+export declare const FileAttributesPreserve = "Preserve";
+export declare const FileAttributesNone = "None";
+export declare const StorageFileLoggingAllowedHeaderNames: string[];
+export declare const StorageFileLoggingAllowedQueryParameters: string[];
+export declare const PathStylePorts: string[];
+//# sourceMappingURL=constants.d.ts.map

package/dist/commonjs/utils/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../src/utils/constants.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,WAAW,EAAE,MAAyB,CAAC;AACpD,eAAO,MAAM,eAAe,EAAE,MAAqB,CAAC;AAEpD,eAAO,MAAM,mBAAmB,EAAE,MAAsC,CAAC;AACzE,eAAO,MAAM,yBAAyB,EAAE,MAAwB,CAAC;AACjE,eAAO,MAAM,mCAAmC,EAAE,MAAU,CAAC;AAC7D,eAAO,MAAM,8BAA8B,EAAE,MAAU,CAAC;AACxD,eAAO,MAAM,eAAe,EAAE,MAAmB,CAAC;AAClD;;GAEG;AACH,eAAO,MAAM,kBAAkB,EAAE,MAAM,GAAG,MAAM,EAAyC,CAAC;AAE1F,eAAO,MAAM,YAAY;;;;;;;CAOxB,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;CAK7B,CAAC;AAEF,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;CAqB3B,CAAC;AAEF,eAAO,MAAM,QAAQ,KAAK,CAAC;AAC3B,eAAO,MAAM,OAAO,MAAM,CAAC;AAC3B,eAAO,MAAM,sBAAsB,aAAa,CAAC;AACjD,eAAO,MAAM,kBAAkB,SAAS,CAAC;AAEzC,eAAO,MAAM,oCAAoC,UA8DhD,CAAC;AAEF,eAAO,MAAM,wCAAwC,UAoBpD,CAAC;AAIF,eAAO,MAAM,cAAc,UAqB1B,CAAC"}