@aws-sdk/lib-transfer-manager 0.0.0-stage → 0.1.1

package/dist-es/submodules/transfer-manager/chunker.js

@@ -0,0 +1,115 @@
+import { Buffer } from "buffer";
+import { lstatSync, ReadStream } from "node:fs";
+export const byteLength = (input) => {
+    if (input == null) {
+        return 0;
+    }
+    if (typeof input === "string") {
+        return Buffer.byteLength(input);
+    }
+    if (typeof input.byteLength === "number") {
+        return input.byteLength;
+    }
+    else if (typeof input.length === "number") {
+        return input.length;
+    }
+    else if (typeof input.size === "number") {
+        return input.size;
+    }
+    else if (input instanceof ReadStream) {
+        const { start: rangeStart, end: rangeEnd } = input;
+        let size;
+        try {
+            size = lstatSync(input.path).size;
+        }
+        catch {
+            return undefined;
+        }
+        const start = typeof rangeStart === "number" ? rangeStart : 0;
+        const endExclusive = Number.isFinite(rangeEnd) ? Math.min(rangeEnd + 1, size) : size;
+        return Math.max(0, endExclusive - start);
+    }
+    else if (typeof input.start === "number" && typeof input.end === "number") {
+        return input.end + 1 - input.start;
+    }
+    return undefined;
+};
+async function* getChunkStream(data, partSize, getNextData) {
+    let partNumber = 1;
+    const currentBuffer = { chunks: [], length: 0 };
+    for await (const datum of getNextData(data)) {
+        currentBuffer.chunks.push(datum);
+        currentBuffer.length += datum.byteLength;
+        while (currentBuffer.length > partSize) {
+            const dataChunk = currentBuffer.chunks.length > 1 ? Buffer.concat(currentBuffer.chunks) : currentBuffer.chunks[0];
+            yield { partNumber, data: dataChunk.subarray(0, partSize) };
+            currentBuffer.chunks = [dataChunk.subarray(partSize)];
+            currentBuffer.length = currentBuffer.chunks[0].byteLength;
+            partNumber += 1;
+        }
+    }
+    yield {
+        partNumber,
+        data: currentBuffer.chunks.length !== 1 ? Buffer.concat(currentBuffer.chunks) : currentBuffer.chunks[0],
+        lastPart: true,
+    };
+}
+async function* getChunkUint8Array(data, partSize) {
+    let partNumber = 1;
+    let startByte = 0;
+    let endByte = partSize;
+    while (endByte < data.byteLength) {
+        yield { partNumber, data: data.subarray(startByte, endByte) };
+        partNumber += 1;
+        startByte = endByte;
+        endByte = startByte + partSize;
+    }
+    yield { partNumber, data: data.subarray(startByte), lastPart: true };
+}
+async function* getDataReadable(data) {
+    for await (const chunk of data) {
+        if (Buffer.isBuffer(chunk) || chunk instanceof Uint8Array) {
+            yield chunk;
+        }
+        else {
+            yield Buffer.from(chunk);
+        }
+    }
+}
+async function* getDataReadableStream(data) {
+    const reader = data.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                return;
+            if (Buffer.isBuffer(value) || value instanceof Uint8Array) {
+                yield value;
+            }
+            else {
+                yield Buffer.from(value);
+            }
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+export const getChunk = (data, partSize) => {
+    if (data instanceof Uint8Array) {
+        return getChunkUint8Array(data, partSize);
+    }
+    if (data instanceof String || typeof data === "string") {
+        return getChunkUint8Array(Buffer.from(data), partSize);
+    }
+    if (data instanceof ReadableStream) {
+        return getChunkStream(data, partSize, getDataReadableStream);
+    }
+    if (typeof data.stream === "function") {
+        return getChunkStream(data.stream(), partSize, getDataReadableStream);
+    }
+    if (Symbol.asyncIterator in data || typeof data[Symbol.asyncIterator] === "function") {
+        return getChunkStream(data, partSize, getDataReadable);
+    }
+    throw new Error("Body Data is unsupported format, expected data to be one of: string | Uint8Array | Buffer | Readable | ReadableStream | Blob;.");
+};

package/dist-es/submodules/transfer-manager/directory-transfer-utils.js

@@ -0,0 +1,102 @@
+import { existsSync } from "node:fs";
+import { mkdir, stat, realpath } from "node:fs/promises";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+export class Semaphore {
+    limit;
+    inFlightRequests = 0;
+    waiters = [];
+    constructor(limit) {
+        this.limit = limit;
+    }
+    async acquire(count = 1) {
+        if (this.inFlightRequests < this.limit) {
+            this.inFlightRequests += count;
+            return;
+        }
+        await new Promise((resolve) => this.waiters.push(resolve));
+        this.inFlightRequests += count;
+    }
+    release(count = 1) {
+        this.inFlightRequests -= count;
+        while (this.waiters.length > 0 && this.inFlightRequests < this.limit) {
+            const next = this.waiters.shift();
+            if (next) {
+                next();
+            }
+        }
+    }
+}
+export async function validateDirectory(dirPath) {
+    const absolutePath = resolve(dirPath);
+    let stats;
+    try {
+        stats = await stat(absolutePath);
+    }
+    catch (error) {
+        throw new Error(`Cannot access directory at: ${absolutePath}, ${error.message}`);
+    }
+    if (!stats.isDirectory()) {
+        throw new Error(`Path is not a directory: ${absolutePath}`);
+    }
+    return absolutePath;
+}
+export async function handleFailure(policy, context, abortController) {
+    if (policy === "terminate") {
+        abortController.abort();
+        return "terminate";
+    }
+    if (policy === "continue") {
+        return "continue";
+    }
+    try {
+        const result = await policy(context);
+        if (result === "terminate") {
+            abortController.abort();
+        }
+        return result;
+    }
+    catch {
+        abortController.abort();
+        return "terminate";
+    }
+}
+export async function createDestinationDirectory(dirPath) {
+    const absolutePath = resolve(dirPath);
+    if (existsSync(absolutePath)) {
+        return validateDirectory(absolutePath);
+    }
+    await mkdir(absolutePath, { recursive: true });
+    return absolutePath;
+}
+export function isFolderObject(object) {
+    return (object.Key?.endsWith("/") ?? false) && (object.Size ?? 0) === 0;
+}
+export function deriveLocalPath(destination, key) {
+    const normalizedKey = sep === "/" ? key : key.split("/").join(sep);
+    if (isAbsolute(normalizedKey)) {
+        throw new Error(`Object key resolves to an absolute path and would escape the destination directory: ${key}`);
+    }
+    const fullPath = resolve(destination, normalizedKey);
+    const destinationWithSep = destination.endsWith(sep) ? destination : destination + sep;
+    if (fullPath !== destination && !fullPath.startsWith(destinationWithSep)) {
+        throw new Error(`Object key "${key}" resolves to a path outside the destination directory. Path traversal is not allowed.`);
+    }
+    return fullPath;
+}
+export async function createLocalParentDirectories(destination, localPath) {
+    const canonicalDestination = await realpath(destination);
+    const relativeParent = relative(destination, dirname(localPath));
+    let currentDirectory = destination;
+    for (const segment of relativeParent.split(sep)) {
+        if (!segment || segment === ".") {
+            continue;
+        }
+        currentDirectory = join(currentDirectory, segment);
+        await mkdir(currentDirectory, { recursive: true });
+        const canonicalDirectory = await realpath(currentDirectory);
+        const canonicalRelative = relative(canonicalDestination, canonicalDirectory);
+        if (canonicalRelative === ".." || canonicalRelative.startsWith(`..${sep}`) || isAbsolute(canonicalRelative)) {
+            throw new Error("Symbolic links outside the destination are not allowed.");
+        }
+    }
+}

package/dist-es/submodules/transfer-manager/event-listener-types.js

@@ -0,0 +1 @@
+export {};

package/dist-es/submodules/transfer-manager/file-manager.js

@@ -0,0 +1,70 @@
+import { randomBytes } from "node:crypto";
+import { unlinkSync } from "node:fs";
+import { open, rename, unlink } from "node:fs/promises";
+export class FileManager {
+    cleanupHandlers = new Map();
+    async createTempFile(destination, totalSize) {
+        const tempFilePath = `${destination}.s3tmp.${this.generateUniqueId()}`;
+        const fd = await open(tempFilePath, "wx");
+        try {
+            await fd.truncate(totalSize);
+        }
+        finally {
+            await fd.close();
+        }
+        return tempFilePath;
+    }
+    async atomicRename(tempFilePath, destination) {
+        try {
+            await rename(tempFilePath, destination);
+        }
+        catch (renameError) {
+            if (renameError.code === "EEXIST" || renameError.code === "EPERM") {
+                try {
+                    await unlink(destination);
+                    await rename(tempFilePath, destination);
+                }
+                catch (fallbackError) {
+                    await this.deleteTempFile(tempFilePath);
+                    throw new Error(`Failed to rename temp file to destination: ${fallbackError.message || fallbackError.code}`);
+                }
+            }
+            else {
+                await this.deleteTempFile(tempFilePath);
+                throw new Error(`Failed to rename temp file to destination: ${renameError.message || renameError.code}`);
+            }
+        }
+    }
+    async deleteTempFile(tempFilePath) {
+        this.unregisterCleanupHandler(tempFilePath);
+        try {
+            await unlink(tempFilePath);
+        }
+        catch (error) {
+            if (error.code !== "ENOENT") {
+                throw error;
+            }
+        }
+    }
+    registerCleanupHandler(tempFilePath) {
+        const exitHandler = () => {
+            try {
+                unlinkSync(tempFilePath);
+            }
+            catch {
+            }
+        };
+        process.on("exit", exitHandler);
+        this.cleanupHandlers.set(tempFilePath, { exitHandler });
+    }
+    unregisterCleanupHandler(tempFilePath) {
+        const handlers = this.cleanupHandlers.get(tempFilePath);
+        if (handlers) {
+            process.removeListener("exit", handlers.exitHandler);
+            this.cleanupHandlers.delete(tempFilePath);
+        }
+    }
+    generateUniqueId() {
+        return randomBytes(4).toString("hex");
+    }
+}

package/dist-es/submodules/transfer-manager/index.browser.js

@@ -0,0 +1,11 @@
+import { destroyStreams, joinStreams } from "./join-streams.browser";
+import { bindS3TransferManager } from "./S3TransferManager";
+import { createEmptyReadable, defaultWorkerCount, WorkerHttpHandler } from "./worker-http-handler.browser";
+export class S3TransferManager extends bindS3TransferManager({
+    joinStreams,
+    destroyStreams,
+    WorkerHttpHandler,
+    createEmptyReadable,
+    defaultWorkerCount,
+}) {
+}

package/dist-es/submodules/transfer-manager/index.js

@@ -0,0 +1,11 @@
+import { destroyStreams, joinStreams } from "./join-streams";
+import { bindS3TransferManager } from "./S3TransferManager";
+import { createEmptyReadable, defaultWorkerCount, WorkerHttpHandler } from "./worker-http-handler";
+export class S3TransferManager extends bindS3TransferManager({
+    joinStreams,
+    destroyStreams,
+    WorkerHttpHandler,
+    createEmptyReadable,
+    defaultWorkerCount,
+}) {
+}

package/dist-es/submodules/transfer-manager/join-streams.browser.js

@@ -0,0 +1,101 @@
+import { isReadableStream, sdkStreamMixin } from "@smithy/core/serde";
+export async function joinStreams(streams, eventListeners) {
+    const firstStream = await streams[0];
+    if (isReadableStream(firstStream) || firstStream instanceof Blob) {
+        const newReadableStream = new ReadableStream({
+            async start(controller) {
+                for await (const chunk of iterateStreams(streams, eventListeners)) {
+                    controller.enqueue(chunk);
+                }
+                controller.close();
+            },
+        });
+        return sdkStreamMixin(newReadableStream);
+    }
+    else {
+        throw new Error("Unsupported Stream Type");
+    }
+}
+export async function* iterateStreams(promises, eventListeners) {
+    let bytesTransferred = 0;
+    let index = 0;
+    for (const streamPromise of promises) {
+        let stream;
+        try {
+            stream = await streamPromise;
+        }
+        catch (e) {
+            await destroyStreams(promises);
+            eventListeners?.onFailure?.(e, index);
+            throw e;
+        }
+        if (isReadableStream(stream)) {
+            const reader = stream.getReader();
+            try {
+                while (true) {
+                    const { done, value } = await reader.read();
+                    if (done) {
+                        break;
+                    }
+                    yield value;
+                    bytesTransferred += value.byteLength;
+                    eventListeners?.onBytes?.(bytesTransferred, index);
+                }
+            }
+            catch (e) {
+                reader.releaseLock();
+                await destroyStreams(promises);
+                eventListeners?.onFailure?.(e, index);
+                throw e;
+            }
+            finally {
+                reader.releaseLock();
+            }
+        }
+        else if (stream instanceof Blob) {
+            const blobStream = stream.stream();
+            const reader = blobStream.getReader();
+            try {
+                while (true) {
+                    const { done, value } = await reader.read();
+                    if (done) {
+                        break;
+                    }
+                    yield value;
+                    bytesTransferred += value.byteLength;
+                    eventListeners?.onBytes?.(bytesTransferred, index);
+                }
+            }
+            catch (e) {
+                await destroyStreams(promises);
+                eventListeners?.onFailure?.(e, index);
+                throw e;
+            }
+            finally {
+                reader.releaseLock();
+            }
+        }
+        else {
+            await destroyStreams(promises);
+            const failure = new Error(`unhandled stream type ${stream?.constructor?.name}`);
+            eventListeners?.onFailure?.(failure, index);
+            throw failure;
+        }
+        eventListeners?.onStreamConsumed?.(index);
+        index++;
+    }
+    eventListeners?.onCompletion?.(bytesTransferred, index - 1);
+}
+export async function destroyStreams(promises) {
+    await Promise.all(promises.map(async (streamPromise) => {
+        if (!streamPromise)
+            return;
+        return streamPromise
+            .then((stream) => {
+            if (isReadableStream(stream)) {
+                return stream.cancel();
+            }
+        })
+            .catch((e) => { });
+    }));
+}

package/dist-es/submodules/transfer-manager/join-streams.js

@@ -0,0 +1,105 @@
+import { isReadableStream, sdkStreamMixin } from "@smithy/core/serde";
+import { Readable } from "node:stream";
+export async function joinStreams(streams, eventListeners) {
+    const firstStream = await streams[0];
+    if (isReadableStream(firstStream)) {
+        const newReadableStream = new ReadableStream({
+            async start(controller) {
+                for await (const chunk of iterateStreams(streams, eventListeners)) {
+                    controller.enqueue(chunk);
+                }
+                controller.close();
+            },
+        });
+        return sdkStreamMixin(newReadableStream);
+    }
+    else {
+        streams.forEach((stream) => stream?.catch(() => { }));
+        return sdkStreamMixin(Readable.from(iterateStreams(streams, eventListeners)));
+    }
+}
+export async function* iterateStreams(promises, eventListeners) {
+    for (const p of promises) {
+        p?.catch(() => { });
+    }
+    let bytesTransferred = 0;
+    let index = 0;
+    for (const streamPromise of promises) {
+        if (!streamPromise) {
+            throw new Error("Stream promise is undefined - lazy prefetch may not have completed");
+        }
+        let stream;
+        try {
+            stream = await streamPromise;
+        }
+        catch (e) {
+            await destroyStreams(promises);
+            eventListeners?.onFailure?.(e, index);
+            throw e;
+        }
+        if (isReadableStream(stream)) {
+            const reader = stream.getReader();
+            try {
+                while (true) {
+                    const { done, value } = await reader.read();
+                    if (done) {
+                        break;
+                    }
+                    yield value;
+                    bytesTransferred += value.byteLength;
+                    eventListeners?.onBytes?.(bytesTransferred, index);
+                }
+            }
+            catch (e) {
+                await destroyStreams(promises);
+                eventListeners?.onFailure?.(e, index);
+                throw e;
+            }
+            finally {
+                reader.releaseLock();
+            }
+        }
+        else if (stream instanceof Readable) {
+            try {
+                for await (const chunk of stream) {
+                    yield chunk;
+                    const chunkSize = Buffer.isBuffer(chunk) ? chunk.length : Buffer.byteLength(chunk);
+                    bytesTransferred += chunkSize;
+                    eventListeners?.onBytes?.(bytesTransferred, index);
+                }
+            }
+            catch (e) {
+                await destroyStreams(promises);
+                eventListeners?.onFailure?.(e, index);
+                throw e;
+            }
+        }
+        else {
+            await destroyStreams(promises);
+            const failure = new Error(`unhandled stream type ${stream?.constructor?.name}`);
+            eventListeners?.onFailure?.(failure, index);
+            throw failure;
+        }
+        eventListeners?.onStreamConsumed?.(index);
+        index++;
+    }
+    eventListeners?.onCompletion?.(bytesTransferred, index - 1);
+}
+export async function destroyStreams(promises) {
+    await Promise.all(promises.map(async (streamPromise) => {
+        if (!streamPromise)
+            return;
+        return streamPromise
+            .then((stream) => {
+            if (stream instanceof Readable) {
+                stream.on("error", () => { });
+                stream.destroy();
+                return;
+            }
+            else if (isReadableStream(stream)) {
+                return stream.cancel();
+            }
+        })
+            .catch((e) => { });
+    }));
+}

package/dist-es/submodules/transfer-manager/log-level.js

@@ -0,0 +1,30 @@
+export class LogLevel {
+    minimum;
+    logger;
+    static LEVELS = ["trace", "debug", "info", "warn", "error"];
+    level = 5;
+    constructor(minimum, logger = console) {
+        this.minimum = minimum;
+        this.logger = logger;
+        const index = LogLevel.LEVELS.indexOf(minimum);
+        if (index === -1) {
+            throw new Error(`Log level must be one of ${LogLevel.LEVELS.join(", ")}`);
+        }
+        this.level = index;
+    }
+    trace(...args) {
+        this.level <= 0 && this.logger.trace?.(...args);
+    }
+    debug(...args) {
+        this.level <= 1 && this.logger.debug?.(...args);
+    }
+    info(...args) {
+        this.level <= 2 && this.logger.info?.(...args);
+    }
+    warn(...args) {
+        this.level <= 3 && this.logger.warn?.(...args);
+    }
+    error(...args) {
+        this.level <= 4 && this.logger.error?.(...args);
+    }
+}

package/dist-es/submodules/transfer-manager/ordered-part-queue.js

@@ -0,0 +1,57 @@
+export class OrderedPartQueue {
+    pending = new Map();
+    nextExpected = 0;
+    totalParts;
+    waitingConsumer = null;
+    error = undefined;
+    constructor(totalParts) {
+        this.totalParts = totalParts;
+    }
+    enqueue(rangeIndex, buffer, byteLength) {
+        if (this.error)
+            return;
+        if (rangeIndex === this.nextExpected && this.waitingConsumer) {
+            const consumer = this.waitingConsumer;
+            this.waitingConsumer = null;
+            this.nextExpected++;
+            consumer({ buffer, byteLength });
+        }
+        else {
+            this.pending.set(rangeIndex, { buffer, byteLength });
+        }
+    }
+    async dequeue() {
+        if (this.error) {
+            throw this.error;
+        }
+        if (this.nextExpected >= this.totalParts) {
+            return null;
+        }
+        const ready = this.pending.get(this.nextExpected);
+        if (ready) {
+            this.pending.delete(this.nextExpected);
+            this.nextExpected++;
+            return ready;
+        }
+        return new Promise((resolve) => {
+            this.waitingConsumer = resolve;
+        });
+    }
+    setError(error) {
+        this.error = error;
+        if (this.waitingConsumer) {
+            const consumer = this.waitingConsumer;
+            this.waitingConsumer = null;
+            consumer(null);
+        }
+    }
+    hasError() {
+        return this.error !== undefined;
+    }
+    getError() {
+        return this.error;
+    }
+    get pendingCount() {
+        return this.pending.size;
+    }
+}

package/dist-es/submodules/transfer-manager/types.js

@@ -0,0 +1 @@
+export {};

package/dist-es/submodules/transfer-manager/worker-http-handler.browser.js

@@ -0,0 +1,25 @@
+export function defaultWorkerCount() {
+    return 1;
+}
+export function createEmptyReadable() {
+    throw new Error("WorkerHttpHandler is not supported in browser environments.");
+}
+export class WorkerHttpHandler {
+    constructor() {
+        throw new Error("WorkerHttpHandler is not supported in browser environments.");
+    }
+    async handle() {
+        throw new Error("WorkerHttpHandler is not supported in browser environments.");
+    }
+    updateHttpClientConfig() { }
+    httpHandlerConfigs() {
+        return {};
+    }
+    getDownloadResult() {
+        throw new Error("WorkerHttpHandler is not supported in browser environments.");
+    }
+    getStreamDownloadResult() {
+        throw new Error("WorkerHttpHandler is not supported in browser environments.");
+    }
+    destroy() { }
+}