@aws-sdk/lib-transfer-manager 0.0.0-stage → 0.1.0

package/dist-es/submodules/worker/http-request-worker.js

@@ -0,0 +1,578 @@
+import { HttpRequest } from "@smithy/core/protocols";
+import { concatBytes } from "@smithy/core/serde";
+import { NodeHttpHandler } from "@smithy/node-http-handler";
+import dns from "node:dns";
+import { openSync, readSync, closeSync, writeSync, unlinkSync, constants as fsConstants } from "node:fs";
+import { Crc32, Crc32c, Crc64Nvme } from "@aws-sdk/checksums/crc";
+import { Sha1, Sha256 } from "@aws-sdk/checksums/sha";
+import { Md5 } from "@aws-sdk/checksums/md5";
+import { open } from "node:fs/promises";
+import { dirname, join as pathJoin } from "node:path";
+import { Agent as httpsAgent } from "node:https";
+import { parentPort } from "node:worker_threads";
+const DNS_TTL_MS = 1000;
+const dnsCache = new Map();
+const MAX_ERROR_BODY_BYTES = 16 * 1024;
+const readErrorBodyBytes = async (body) => {
+    if (!body)
+        return undefined;
+    try {
+        const chunks = [];
+        let total = 0;
+        for await (const chunk of body) {
+            const buf = typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk);
+            chunks.push(buf);
+            total += buf.length;
+            if (total >= MAX_ERROR_BODY_BYTES)
+                break;
+        }
+        return concatBytes(chunks);
+    }
+    catch {
+        return undefined;
+    }
+};
+const buildHttpStatusError = async (id, response) => {
+    return {
+        type: "httpDownloadHttpError",
+        id,
+        statusCode: response.statusCode,
+        headers: response.headers,
+        body: await readErrorBodyBytes(response.body),
+    };
+};
+const O_DIRECT_ALIGN = 4096;
+const O_DIRECT_FLAG = fsConstants.O_DIRECT ?? 0;
+const O_DIRECT_SUPPORTED = process.platform === "linux" && O_DIRECT_FLAG > 0;
+let oDirectEnabled = false;
+let oDirectRequested = false;
+let oDirectProbed = false;
+let oDirectUsable = false;
+function postLog(level, message) {
+    parentPort?.postMessage({ type: "workerLog", level, message });
+}
+function probeAlignedPad(dir, buf) {
+    const probePath = pathJoin(dir, `.tm-odirect-probe-${process.pid}-${Math.random().toString(36).slice(2)}`);
+    let found = -1;
+    for (let pad = 0; pad < O_DIRECT_ALIGN; pad += 8) {
+        let fd;
+        try {
+            fd = openSync(probePath, fsConstants.O_RDWR | fsConstants.O_CREAT | fsConstants.O_TRUNC | O_DIRECT_FLAG, 0o600);
+            const n = writeSync(fd, buf, pad, O_DIRECT_ALIGN, 0);
+            closeSync(fd);
+            fd = undefined;
+            if (n === O_DIRECT_ALIGN) {
+                found = pad;
+                break;
+            }
+        }
+        catch {
+            if (fd !== undefined) {
+                try {
+                    closeSync(fd);
+                }
+                catch {
+                }
+            }
+        }
+    }
+    try {
+        unlinkSync(probePath);
+    }
+    catch {
+    }
+    return found;
+}
+function ensureODirectProbed(dir) {
+    if (oDirectProbed) {
+        return;
+    }
+    oDirectProbed = true;
+    const testBuf = Buffer.allocUnsafeSlow(O_DIRECT_ALIGN * 2);
+    oDirectUsable = probeAlignedPad(dir, testBuf) >= 0;
+    if (!oDirectUsable) {
+        postLog("warn", `useODirect was requested but O_DIRECT is not usable on the filesystem at "${dir}"; ` +
+            `falling back to buffered writes.`);
+    }
+    else {
+        postLog("debug", `O_DIRECT probe succeeded for filesystem at "${dir}"; using O_DIRECT writes.`);
+    }
+}
+function spreadLookup(hostname, options, callback) {
+    if (options && options.family === 6) {
+        return dns.lookup(hostname, options, callback);
+    }
+    const deliver = (ips) => {
+        const ip = ips[Math.floor(Math.random() * ips.length)];
+        if (options && options.all) {
+            callback(null, [{ address: ip, family: 4 }]);
+        }
+        else {
+            callback(null, ip, 4);
+        }
+    };
+    const cached = dnsCache.get(hostname);
+    if (cached && cached.ips.length && Date.now() - cached.ts < DNS_TTL_MS) {
+        return deliver(cached.ips);
+    }
+    dns.resolve4(hostname, (err, ips) => {
+        if (err || !ips?.length) {
+            return dns.lookup(hostname, options, callback);
+        }
+        dnsCache.set(hostname, { ips, ts: Date.now() });
+        deliver(ips);
+    });
+}
+const UNVERIFIABLE_ALGORITHMS = ["SHA512", "XXHASH3", "XXHASH64", "XXHASH128"];
+function createChecksum(algorithm) {
+    switch (algorithm) {
+        case "CRC64NVME":
+            return new Crc64Nvme();
+        case "CRC32C":
+            return new Crc32c();
+        case "CRC32":
+            return new Crc32();
+        case "SHA1":
+            return new Sha1();
+        case "SHA256":
+            return new Sha256();
+        case "MD5":
+            return new Md5();
+        default:
+            return undefined;
+    }
+}
+async function finalizeChecksumToBase64(checksum) {
+    const bytes = await checksum.digest();
+    return Buffer.from(bytes).toString("base64");
+}
+function parseContentRange(contentRange) {
+    if (!contentRange)
+        return undefined;
+    const match = contentRange.match(/^bytes\s+(\d+)-(\d+)\//);
+    if (!match)
+        return undefined;
+    const start = Number(match[1]);
+    const end = Number(match[2]);
+    if (end < start)
+        return undefined;
+    return { start, end };
+}
+function getChecksumHeaderValue(headers, algorithm) {
+    const headerName = `x-amz-checksum-${algorithm.toLowerCase()}`;
+    return headers[headerName];
+}
+function resolveDownloadChecksumAlgorithm(headers) {
+    const candidates = ["CRC64NVME", "CRC32C", "CRC32", "SHA256", "SHA1"];
+    for (const algorithm of candidates) {
+        if (getChecksumHeaderValue(headers, algorithm)) {
+            return algorithm;
+        }
+    }
+    for (const algorithm of UNVERIFIABLE_ALGORITHMS) {
+        if (getChecksumHeaderValue(headers, algorithm)) {
+            throw Object.assign(new Error(`S3 returned a ${algorithm} checksum, which worker threads cannot compute. ` +
+                `Set workerThreadCount: 1 to validate this object, or set ` +
+                `responseChecksumValidation: "WHEN_REQUIRED" to download it without validation.`), { name: "ChecksumValidationError", code: "CHECKSUM_ALGORITHM_UNSUPPORTED" });
+        }
+    }
+    return undefined;
+}
+if (parentPort) {
+    let handler;
+    const port = parentPort;
+    const fdCache = new Map();
+    function getFd(filePath) {
+        let fd = fdCache.get(filePath);
+        if (fd === undefined) {
+            fd = openSync(filePath, "r");
+            fdCache.set(filePath, fd);
+        }
+        return fd;
+    }
+    const readFileSlice = (filePath, offset, length) => {
+        const fd = getFd(filePath);
+        const buffer = Buffer.allocUnsafe(length);
+        let read = 0;
+        while (read < length) {
+            const n = readSync(fd, buffer, read, length - read, offset + read);
+            if (n === 0)
+                break;
+            read += n;
+        }
+        return buffer;
+    };
+    const computeDeclaredChecksum = async (data, algorithm) => {
+        const checksum = createChecksum(algorithm);
+        if (!checksum) {
+            throw new Error(`Unsupported checksum algorithm for threaded upload: ${algorithm}`);
+        }
+        checksum.update(data);
+        return finalizeChecksumToBase64(checksum);
+    };
+    const buildAwsChunkedBody = (data, checksumHeader, checksumValue) => {
+        const hexLen = data.byteLength.toString(16);
+        const parts = [Buffer.from(`${hexLen}\r\n`), data, Buffer.from("\r\n0\r\n")];
+        if (checksumHeader && checksumValue) {
+            parts.push(Buffer.from(`${checksumHeader}:${checksumValue}\r\n`));
+        }
+        parts.push(Buffer.from("\r\n"));
+        return Buffer.concat(parts);
+    };
+    const processRequest = async (msg) => {
+        const { id, request: serialized } = msg;
+        try {
+            let body;
+            if (msg.type === "httpRequestFromRAM") {
+                const fileData = Buffer.from(msg.sharedBuffer, msg.offset, msg.length);
+                if (msg.checksumAlgorithm && msg.checksumHeader) {
+                    const checksumValue = await computeDeclaredChecksum(fileData, msg.checksumAlgorithm);
+                    body = buildAwsChunkedBody(fileData, msg.checksumHeader, checksumValue);
+                }
+                else {
+                    body = fileData;
+                }
+            }
+            else if (msg.type === "httpRequestFromFile") {
+                const fileData = readFileSlice(msg.filePath, msg.offset, msg.length);
+                if (msg.checksumAlgorithm && msg.checksumHeader) {
+                    const checksumValue = await computeDeclaredChecksum(fileData, msg.checksumAlgorithm);
+                    body = buildAwsChunkedBody(fileData, msg.checksumHeader, checksumValue);
+                }
+                else {
+                    body = fileData;
+                }
+            }
+            const request = new HttpRequest({
+                method: serialized.method,
+                protocol: serialized.protocol,
+                hostname: serialized.hostname,
+                port: serialized.port,
+                path: serialized.path,
+                query: serialized.query,
+                headers: serialized.headers,
+                body,
+            });
+            const { response } = await handler.handle(request);
+            const chunks = [];
+            if (response.body) {
+                for await (const chunk of response.body) {
+                    chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
+                }
+            }
+            let responseBody;
+            const transferList = [];
+            if (chunks.length > 0) {
+                const concatenated = Buffer.concat(chunks);
+                const ab = new ArrayBuffer(concatenated.byteLength);
+                new Uint8Array(ab).set(concatenated);
+                responseBody = new Uint8Array(ab);
+                transferList.push(ab);
+            }
+            port.postMessage({
+                type: "httpResponse",
+                id,
+                response: {
+                    statusCode: response.statusCode,
+                    headers: response.headers,
+                    body: responseBody,
+                },
+            }, transferList);
+        }
+        catch (err) {
+            port.postMessage({
+                type: "httpError",
+                id,
+                error: err.message ?? String(err),
+                code: err.code,
+                name: err.name,
+            });
+        }
+    };
+    const processDownloadToFile = async (msg) => {
+        const { id, request: serialized, filePath, offset, expectedLength } = msg;
+        try {
+            const request = new HttpRequest({
+                method: serialized.method,
+                protocol: serialized.protocol,
+                hostname: serialized.hostname,
+                port: serialized.port,
+                path: serialized.path,
+                query: serialized.query,
+                headers: serialized.headers,
+            });
+            const { response } = await handler.handle(request);
+            if (response.statusCode < 200 || response.statusCode >= 300) {
+                port.postMessage(await buildHttpStatusError(id, response));
+                return;
+            }
+            const contentRange = response.headers["content-range"];
+            const range = parseContentRange(contentRange);
+            const writeOffset = range ? range.start : offset;
+            const targetLength = range ? range.end - range.start + 1 : expectedLength;
+            const fh = await open(filePath, "r+");
+            try {
+                const verifiableDownloadChecksumAlgorithm = resolveDownloadChecksumAlgorithm(response.headers);
+                let checksum;
+                if (verifiableDownloadChecksumAlgorithm) {
+                    checksum = createChecksum(verifiableDownloadChecksumAlgorithm);
+                }
+                let bytesWritten = 0;
+                let alignedBuf;
+                let pad = -1;
+                if (oDirectEnabled && writeOffset % O_DIRECT_ALIGN === 0 && targetLength >= O_DIRECT_ALIGN) {
+                    ensureODirectProbed(dirname(filePath));
+                    if (oDirectUsable) {
+                        alignedBuf = Buffer.allocUnsafeSlow(targetLength + O_DIRECT_ALIGN);
+                        pad = probeAlignedPad(dirname(filePath), alignedBuf);
+                    }
+                }
+                else if (oDirectEnabled) {
+                    postLog("trace", `O_DIRECT skipped: part not 4096-block-aligned (offset=${writeOffset}, length=${targetLength}, block=${O_DIRECT_ALIGN}), using buffered writes.`);
+                }
+                if (alignedBuf && pad >= 0) {
+                    if (response.body) {
+                        for await (const chunk of response.body) {
+                            const buf = typeof chunk === "string" ? Buffer.from(chunk) : Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                            if (bytesWritten + buf.length > targetLength) {
+                                await fh.close();
+                                port.postMessage({
+                                    type: "httpDownloadError",
+                                    id,
+                                    error: `Bytes written (${bytesWritten + buf.length}) exceeds expected length (${targetLength})`,
+                                    code: "BYTES_EXCEEDED",
+                                    name: "DownloadValidationError",
+                                });
+                                return;
+                            }
+                            buf.copy(alignedBuf, pad + bytesWritten, 0, buf.length);
+                            if (checksum) {
+                                checksum.update(buf);
+                            }
+                            bytesWritten += buf.length;
+                        }
+                    }
+                    const alignedLength = Math.floor(bytesWritten / O_DIRECT_ALIGN) * O_DIRECT_ALIGN;
+                    if (alignedLength > 0) {
+                        const directFh = await open(filePath, fsConstants.O_RDWR | O_DIRECT_FLAG);
+                        try {
+                            await directFh.write(alignedBuf, pad, alignedLength, writeOffset);
+                        }
+                        finally {
+                            await directFh.close().catch(() => { });
+                        }
+                    }
+                    const tailLength = bytesWritten - alignedLength;
+                    if (tailLength > 0) {
+                        await fh.write(alignedBuf, pad + alignedLength, tailLength, writeOffset + alignedLength);
+                    }
+                }
+                else {
+                    if (response.body) {
+                        for await (const chunk of response.body) {
+                            const buf = typeof chunk === "string" ? Buffer.from(chunk) : Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                            if (bytesWritten + buf.length > targetLength) {
+                                await fh.close();
+                                port.postMessage({
+                                    type: "httpDownloadError",
+                                    id,
+                                    error: `Bytes written (${bytesWritten + buf.length}) exceeds expected length (${targetLength})`,
+                                    code: "BYTES_EXCEEDED",
+                                    name: "DownloadValidationError",
+                                });
+                                return;
+                            }
+                            await fh.write(buf, 0, buf.length, writeOffset + bytesWritten);
+                            if (checksum) {
+                                checksum.update(buf);
+                            }
+                            bytesWritten += buf.length;
+                        }
+                    }
+                }
+                await fh.close();
+                if (bytesWritten !== targetLength) {
+                    port.postMessage({
+                        type: "httpDownloadError",
+                        id,
+                        error: `Bytes written (${bytesWritten}) does not match expected length (${targetLength})`,
+                        code: "LENGTH_MISMATCH",
+                        name: "DownloadValidationError",
+                    });
+                    return;
+                }
+                let checksumBase64;
+                if (checksum && verifiableDownloadChecksumAlgorithm) {
+                    checksumBase64 = await finalizeChecksumToBase64(checksum);
+                    const s3ChecksumValue = getChecksumHeaderValue(response.headers, verifiableDownloadChecksumAlgorithm);
+                    if (s3ChecksumValue && s3ChecksumValue !== checksumBase64) {
+                        port.postMessage({
+                            type: "httpDownloadError",
+                            id,
+                            error: `Checksum mismatch: computed ${checksumBase64}, S3 returned ${s3ChecksumValue}`,
+                            code: "CHECKSUM_MISMATCH",
+                            name: "ChecksumValidationError",
+                        });
+                        return;
+                    }
+                }
+                port.postMessage({
+                    type: "httpDownloadResult",
+                    id,
+                    statusCode: response.statusCode,
+                    headers: response.headers,
+                    bytesWritten,
+                    checksum: checksumBase64,
+                });
+            }
+            catch (err) {
+                await fh.close().catch(() => { });
+                throw err;
+            }
+        }
+        catch (err) {
+            port.postMessage({
+                type: "httpDownloadError",
+                id,
+                error: err.message ?? String(err),
+                code: err.code,
+                name: err.name,
+            });
+        }
+    };
+    const reusableBufferPool = [];
+    function acquireTransferBuffer(size) {
+        for (let i = 0; i < reusableBufferPool.length; i++) {
+            if (reusableBufferPool[i].byteLength >= size) {
+                return reusableBufferPool.splice(i, 1)[0];
+            }
+        }
+        return Buffer.allocUnsafeSlow(size).buffer;
+    }
+    const processDownloadToTransfer = async (msg) => {
+        const { id, request: serialized, expectedSize, rangeIndex } = msg;
+        try {
+            const request = new HttpRequest({
+                method: serialized.method,
+                protocol: serialized.protocol,
+                hostname: serialized.hostname,
+                port: serialized.port,
+                path: serialized.path,
+                query: serialized.query,
+                headers: serialized.headers,
+            });
+            const { response } = await handler.handle(request);
+            if (response.statusCode < 200 || response.statusCode >= 300) {
+                port.postMessage(await buildHttpStatusError(id, response));
+                return;
+            }
+            const ab = acquireTransferBuffer(expectedSize);
+            const view = new Uint8Array(ab, 0, expectedSize);
+            const verifiableDownloadChecksumAlgorithm = resolveDownloadChecksumAlgorithm(response.headers);
+            let checksum;
+            if (verifiableDownloadChecksumAlgorithm) {
+                checksum = createChecksum(verifiableDownloadChecksumAlgorithm);
+            }
+            let bytesWritten = 0;
+            if (response.body) {
+                for await (const chunk of response.body) {
+                    const buf = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
+                    if (bytesWritten + buf.length > expectedSize) {
+                        port.postMessage({
+                            type: "httpDownloadError",
+                            id,
+                            error: `Bytes received (${bytesWritten + buf.length}) exceeds expected size (${expectedSize}) for range index ${rangeIndex}`,
+                            code: "BYTES_EXCEEDED",
+                            name: "DownloadValidationError",
+                        });
+                        return;
+                    }
+                    view.set(buf, bytesWritten);
+                    if (checksum) {
+                        checksum.update(buf);
+                    }
+                    bytesWritten += buf.length;
+                }
+            }
+            let checksumBase64;
+            if (checksum && verifiableDownloadChecksumAlgorithm) {
+                checksumBase64 = await finalizeChecksumToBase64(checksum);
+                const s3ChecksumValue = getChecksumHeaderValue(response.headers, verifiableDownloadChecksumAlgorithm);
+                if (s3ChecksumValue && s3ChecksumValue !== checksumBase64) {
+                    port.postMessage({
+                        type: "httpDownloadError",
+                        id,
+                        error: `Checksum mismatch: computed ${checksumBase64}, S3 returned ${s3ChecksumValue}`,
+                        code: "CHECKSUM_MISMATCH",
+                        name: "ChecksumValidationError",
+                    });
+                    return;
+                }
+            }
+            port.postMessage({
+                type: "httpDownloadStreamResult",
+                id,
+                rangeIndex,
+                buffer: ab,
+                byteLength: bytesWritten,
+                statusCode: response.statusCode,
+                headers: response.headers,
+                checksum: checksumBase64,
+            }, [ab]);
+        }
+        catch (err) {
+            port.postMessage({
+                type: "httpDownloadError",
+                id,
+                error: err.message ?? String(err),
+                code: err.code,
+                name: err.name,
+            });
+        }
+    };
+    port.on("message", (msg) => {
+        if (msg.type === "done") {
+            for (const fd of fdCache.values()) {
+                try {
+                    closeSync(fd);
+                }
+                catch {
+                }
+            }
+            handler?.destroy();
+            process.exit(0);
+            return;
+        }
+        if (msg.type === "config") {
+            const { maxSockets, useODirect } = msg;
+            oDirectRequested = useODirect === true;
+            oDirectEnabled = oDirectRequested && O_DIRECT_SUPPORTED;
+            if (oDirectRequested && !O_DIRECT_SUPPORTED) {
+                postLog("warn", `useODirect was requested but O_DIRECT is not available on this platform ` +
+                    `(platform=${process.platform}, O_DIRECT flag ${O_DIRECT_FLAG > 0 ? "present" : "absent"}); ` +
+                    `falling back to buffered writes.`);
+            }
+            handler = new NodeHttpHandler({
+                httpsAgent: new httpsAgent({
+                    maxSockets,
+                    keepAlive: true,
+                    lookup: spreadLookup,
+                }),
+            });
+            return;
+        }
+        if (msg.type === "httpRequestFromFile" || msg.type === "httpRequestFromRAM") {
+            processRequest(msg);
+        }
+        if (msg.type === "httpDownloadToFile") {
+            processDownloadToFile(msg);
+        }
+        if (msg.type === "httpDownloadStream") {
+            processDownloadToTransfer(msg);
+        }
+        if (msg.type === "returnBuffer") {
+            reusableBufferPool.push(msg.buffer);
+        }
+    });
+    port.postMessage({ type: "ready" });
+}

package/dist-es/submodules/worker/index.js

@@ -0,0 +1 @@
+import "./http-request-worker";

package/dist-types/index.d.ts

@@ -0,0 +1,2 @@
+export { S3TransferManager } from "@aws-sdk/lib-transfer-manager/transfer-manager";
+export type { IS3TransferManager } from "@aws-sdk/lib-transfer-manager/transfer-manager";

package/dist-types/submodules/transfer-manager/S3TransferManager.d.ts

@@ -0,0 +1,60 @@
+import type { Logger } from "@smithy/types";
+import type { IS3TransferManager, S3TransferManagerConfig } from "./types";
+import type { DownloadStreamResult } from "./worker-http-handler";
+import type { destroyStreams, joinStreams } from "./join-streams";
+/**
+ * Structural contract for the worker HTTP handler. Both the Node.js
+ * `WorkerHttpHandler` and the browser stub satisfy this shape, keeping the
+ * shared implementation environment-agnostic.
+ *
+ * @internal
+ */
+export interface IWorkerHttpHandler {
+    getDownloadResult(token: string): {
+        bytesWritten: number;
+        checksum?: string;
+    } | undefined;
+    getStreamDownloadResult(token: string): DownloadStreamResult | undefined;
+}
+/**
+ * @internal
+ */
+export interface WorkerHttpHandlerConstructor {
+    new (options?: {
+        workerThreadCount?: number;
+        maxConcurrentUploads?: number;
+        maxConcurrentDownloads?: number;
+        useODirect?: boolean;
+        logger?: Logger;
+    }): IWorkerHttpHandler;
+}
+/**
+ * @internal
+ */
+export interface S3TransferManagerRuntimeDependencies {
+    joinStreams: typeof joinStreams;
+    destroyStreams: typeof destroyStreams;
+    WorkerHttpHandler: WorkerHttpHandlerConstructor;
+    createEmptyReadable: () => any;
+    defaultWorkerCount: () => number;
+}
+/**
+ * @internal
+ */
+export interface S3TransferManagerConstructor {
+    new (config?: S3TransferManagerConfig): IS3TransferManager;
+}
+/**
+ * Binds environment runtime dependencies.
+ *
+ * @internal
+ */
+export declare const bindS3TransferManager: (deps: S3TransferManagerRuntimeDependencies) => S3TransferManagerConstructor;
+/**
+ * Internal event handler for download lifecycle hooks.
+ *
+ * @internal
+ */
+export declare const internalEventHandler: {
+    afterInitialGetObject(): Promise<void>;
+};

package/dist-types/submodules/transfer-manager/chunker.d.ts

@@ -0,0 +1,7 @@
+export interface RawDataPart {
+    partNumber: number;
+    data: Uint8Array;
+    lastPart?: boolean;
+}
+export declare const byteLength: (input: any) => number | undefined;
+export declare const getChunk: (data: any, partSize: number) => AsyncGenerator<RawDataPart, void, undefined>;