@aws-sdk/lib-transfer-manager 0.0.0-stage → 0.1.1

package/dist-types/submodules/transfer-manager/worker-http-handler.d.ts

@@ -0,0 +1,410 @@
+/**
+ * HTTP handler that delegates signed HTTP requests to a pool of worker threads.
+ *
+ * For UploadPart requests with a file source, the handler forwards the signed
+ * headers (including aws-chunked encoding headers) and file offset info to a
+ * worker. The worker reads the file slice, computes CRC32, and sends the data
+ * in aws-chunked format with a trailing checksum — fulfilling the contract
+ * that the checksum middleware set up on the main thread.
+ *
+ * @internal
+ */
+import { HttpResponse } from "@smithy/core/protocols";
+import type { HttpHandlerOptions, HttpRequest, HttpResponse as HttpResponseShape, Logger } from "@smithy/types";
+import type { ChecksumAlgorithm } from "@aws-sdk/client-s3";
+import { Readable } from "node:stream";
+/**
+ * Creates an empty Readable stream that immediately ends.
+ * Used as a placeholder body for UploadPart in threaded uploads.
+ * @internal
+ */
+export declare function createEmptyReadable(): Readable;
+/**
+ * Serializable subset of HttpRequest.
+ * @internal
+ */
+export interface WorkerHttpRequest extends Omit<HttpRequest, "body"> {
+    body?: Uint8Array;
+}
+/**
+ * Serializable subset of HttpResponse.
+ * @internal
+ */
+export interface WorkerHttpResponse extends Omit<HttpResponseShape, "body"> {
+    body?: Uint8Array;
+}
+interface BaseHttpWorkerRequestMessage {
+    id: number;
+    request: WorkerHttpRequest;
+}
+export interface HttpWorkerFileRequestMessage extends BaseHttpWorkerRequestMessage {
+    type: "httpRequestFromFile";
+    filePath: string;
+    offset: number;
+    length: number;
+    checksumAlgorithm?: ChecksumAlgorithm;
+    checksumHeader?: string;
+}
+export interface HttpWorkerRAMRequestMessage extends BaseHttpWorkerRequestMessage {
+    type: "httpRequestFromRAM";
+    sharedBuffer: SharedArrayBuffer;
+    offset: number;
+    length: number;
+    checksumAlgorithm?: ChecksumAlgorithm;
+    checksumHeader?: string;
+}
+export type HttpWorkerRequestMessage = HttpWorkerFileRequestMessage | HttpWorkerRAMRequestMessage;
+/**
+ * Main thread → Worker: download a part and write directly to a file offset.
+ * @internal
+ */
+export interface HttpWorkerDownloadToFileMessage extends BaseHttpWorkerRequestMessage {
+    type: "httpDownloadToFile";
+    filePath: string;
+    offset: number;
+    expectedLength: number;
+    checksumAlgorithm?: ChecksumAlgorithm;
+}
+/**
+ * Main thread → Worker: download a part and stream the response back.
+ * @internal
+ */
+export interface HttpWorkerDownloadStreamMessage extends BaseHttpWorkerRequestMessage {
+    type: "httpDownloadStream";
+    /**
+     * Expected byte length of the part (used for buffer pre-allocation).
+     */
+    expectedSize: number;
+    /**
+     * Part/range index for in-order delivery on the main thread.
+     */
+    rangeIndex: number;
+    /**
+     * Optional CRC algorithm for inline checksum validation against S3 response headers.
+     */
+    checksumAlgorithm?: ChecksumAlgorithm;
+}
+/**
+ * Main thread → Worker: return a consumed ArrayBuffer for reuse.
+ * Sent after the main-thread consumer has finished reading a transferred buffer.
+ * @internal
+ */
+export interface HttpWorkerReturnBufferMessage {
+    type: "returnBuffer";
+    /**
+     * The ArrayBuffer being returned (transferred back to the worker).
+     */
+    buffer: ArrayBuffer;
+}
+export interface HttpWorkerConfigMessage {
+    type: "config";
+    maxSockets: number;
+    useODirect?: boolean;
+}
+export interface HttpWorkerDoneMessage {
+    type: "done";
+}
+export type HttpWorkerInboundMessage = HttpWorkerRequestMessage | HttpWorkerDownloadToFileMessage | HttpWorkerDownloadStreamMessage | HttpWorkerReturnBufferMessage | HttpWorkerConfigMessage | HttpWorkerDoneMessage;
+export interface HttpWorkerResponseMessage {
+    type: "httpResponse";
+    id: number;
+    response: WorkerHttpResponse;
+}
+export interface HttpWorkerErrorMessage {
+    type: "httpError";
+    id: number;
+    error: string;
+    code?: string;
+    name?: string;
+}
+export interface HttpWorkerReadyMessage {
+    type: "ready";
+}
+/**
+ * Worker → Main thread: a diagnostic log line relayed to the TM logger.
+ * The worker thread has no access to the client logger, so O_DIRECT fallback
+ * diagnostics are forwarded here and logged on the main thread.
+ * @internal
+ */
+export interface HttpWorkerLogMessage {
+    type: "workerLog";
+    level: "warn" | "debug" | "trace";
+    message: string;
+}
+/**
+ * Worker → Main thread: download part completed successfully.
+ * @internal
+ */
+export interface HttpWorkerDownloadResultMessage {
+    type: "httpDownloadResult";
+    id: number;
+    statusCode: number;
+    headers: Record<string, string>;
+    bytesWritten: number;
+    checksum?: string;
+}
+/**
+ * Worker → Main thread: download failed with no HTTP response
+ * (checksum/length mismatch, socket error).
+ * @internal
+ */
+export interface HttpWorkerDownloadErrorMessage {
+    type: "httpDownloadError";
+    id: number;
+    error: string;
+    code?: string;
+    name?: string;
+}
+/**
+ * Worker → Main thread: download GET returned a non-2xx response. Forwards the
+ * raw status/headers/body so handle() can return it as an HttpResponse and let
+ * the SDK deserialize the typed error and retry it.
+ * @internal
+ */
+export interface HttpWorkerDownloadHttpErrorMessage {
+    type: "httpDownloadHttpError";
+    id: number;
+    statusCode: number;
+    headers: Record<string, string>;
+    body?: Uint8Array;
+}
+/**
+ * Worker → Main thread: stream-mode download completed.
+ * The ArrayBuffer is transferred (zero-copy) via the postMessage transfer list.
+ * @internal
+ */
+export interface HttpWorkerDownloadStreamResultMessage {
+    type: "httpDownloadStreamResult";
+    id: number;
+    rangeIndex: number;
+    /**
+     * The ArrayBuffer containing the downloaded part data (transferred, not copied).
+     */
+    buffer: ArrayBuffer;
+    byteLength: number;
+    statusCode: number;
+    headers: Record<string, string>;
+    checksum?: string;
+}
+export type HttpWorkerOutboundMessage = HttpWorkerResponseMessage | HttpWorkerErrorMessage | HttpWorkerReadyMessage | HttpWorkerLogMessage | HttpWorkerDownloadResultMessage | HttpWorkerDownloadErrorMessage | HttpWorkerDownloadHttpErrorMessage | HttpWorkerDownloadStreamResultMessage;
+/** @internal */
+export declare function defaultWorkerCount(): number;
+/** @internal */
+export interface FileSource {
+    type: "file";
+    filePath: string;
+    partSize: number;
+    totalFileSize: number;
+    startOffset?: number;
+    checksumAlgorithm?: ChecksumAlgorithm;
+    checksumHeader?: string;
+}
+/** @internal */
+export interface SharedBufferSource {
+    type: "sharedBuffer";
+    sharedBuffer: SharedArrayBuffer;
+    /** Byte offset of the caller's Uint8Array view into sharedBuffer. */
+    startOffset?: number;
+    partSize: number;
+    totalSize: number;
+    checksumAlgorithm?: ChecksumAlgorithm;
+    checksumHeader?: string;
+}
+/**
+ * Data source passed per-request to handle().
+ * Tells the worker where to read the body from.
+ * @internal
+ */
+export type DataSource = FileSource | SharedBufferSource;
+/**
+ * Options for WorkerHttpHandler.handle() that extend standard HttpHandlerOptions
+ * with an optional data source for UploadPart requests.
+ * @internal
+ */
+export interface WorkerHttpHandlerOptions extends HttpHandlerOptions {
+    dataSource?: DataSource;
+}
+/**
+ * Metadata passed to the worker instructing it to write the HTTP response body
+ * to a specific file offset rather than returning it over the message channel.
+ * @internal
+ */
+export interface DownloadDataToFile {
+    /**
+     * Absolute path to the temp file where the part body should be written.
+     */
+    filePath: string;
+    /**
+     * Absolute byte offset in the file where writing should begin. Used only as a
+     * fallback: the worker prefers the start byte from the response's ContentRange
+     * header, since part sizes are not guaranteed to be uniform.
+     */
+    offset: number;
+    /**
+     * Expected byte length of the part body. Used only as a fallback, for the same
+     * reason as {@link DownloadDataToFile.offset}.
+     */
+    expectedLength: number;
+    /**
+     * Optional algorithm for inline checksum validation against S3 response headers.
+     */
+    checksumAlgorithm?: ChecksumAlgorithm;
+    /**
+     * Unique token for correlating the download result with the caller.
+     */
+    resultToken?: string;
+}
+/**
+ * Metadata passed to the worker instructing it to download a part into a
+ * worker-owned ArrayBuffer and transfer ownership to main (zero-copy).
+ * @internal
+ */
+export interface DownloadStreamOptions {
+    /**
+     * Expected byte length of the part (used for buffer pre-allocation).
+     */
+    expectedSize: number;
+    /**
+     * Part/range index for in-order delivery on the main thread.
+     */
+    rangeIndex: number;
+    /**
+     * Optional algorithm for inline checksum validation against S3 response headers.
+     */
+    checksumAlgorithm?: ChecksumAlgorithm;
+    /**
+     * Unique token for correlating the download result with the caller.
+     */
+    resultToken?: string;
+}
+/**
+ * Options for WorkerHttpHandler.handle() that extend standard HttpHandlerOptions
+ * with download-to-file metadata for Part GET requests.
+ * @internal
+ */
+export interface WorkerHttpHandlerDownloadOptions extends HttpHandlerOptions {
+    downloadDataToFile?: DownloadDataToFile;
+    downloadStream?: DownloadStreamOptions;
+}
+/**
+ * Result returned when a download-to-file request completes successfully.
+ * @internal
+ */
+export interface DownloadToFileResult {
+    statusCode: number;
+    headers: Record<string, string>;
+    bytesWritten: number;
+    checksum?: string;
+    /**
+     * Set only on non-2xx responses: the raw error body, returned as an
+     * HttpResponse for the SDK to deserialize.
+     */
+    errorBody?: Uint8Array;
+}
+/**
+ * Result returned when a transfer-mode download completes successfully.
+ * Contains the transferred ArrayBuffer (zero-copy from worker).
+ * @internal
+ */
+export interface DownloadStreamResult {
+    rangeIndex: number;
+    /**
+     * The ArrayBuffer transferred from the worker (zero-copy). Empty on non-2xx
+     * responses; see {@link DownloadStreamResult.errorBody}.
+     */
+    buffer: ArrayBuffer;
+    byteLength: number;
+    statusCode: number;
+    headers: Record<string, string>;
+    checksum?: string;
+    /**
+     * Set only on non-2xx responses: the raw error body, returned as an
+     * HttpResponse for the SDK to deserialize.
+     */
+    errorBody?: Uint8Array;
+}
+export declare class WorkerHttpHandler {
+    private workers;
+    private inflightRequests;
+    private inflightDownloads;
+    private inflightStreamDownloads;
+    private nextRequestId;
+    private workerInflightCounts;
+    private workerThreadCount;
+    private maxConcurrentUploads;
+    private maxConcurrentDownloads;
+    private useODirect;
+    private logger;
+    private initialized;
+    private initPromise;
+    private fallbackHandler;
+    /**
+     * Stores completed download results keyed by a caller-provided token.
+     * The TM passes a unique token in the downloadDataToFile options and
+     * retrieves the result after this.s3.send() completes.
+     * @internal
+     */
+    private completedDownloads;
+    /**
+     * Stores completed stream-mode download results keyed by a caller-provided token.
+     * Unlike completedDownloads, this includes the ArrayBuffer with the response body.
+     * @internal
+     */
+    private completedTransfers;
+    readonly metadata: {
+        handlerProtocol: string;
+    };
+    constructor(options?: {
+        workerThreadCount?: number;
+        maxConcurrentUploads?: number;
+        maxConcurrentDownloads?: number;
+        useODirect?: boolean;
+        logger?: Logger;
+    });
+    /**
+     * Retrieves and removes the completed download result for a given token.
+     * Returns undefined if no result is stored for that token.
+     * @internal
+     */
+    getDownloadResult(token: string): {
+        bytesWritten: number;
+        checksum?: string;
+    } | undefined;
+    /**
+     * Retrieves and removes the completed stream-mode download result for a given token.
+     * Includes the ArrayBuffer with the response body (zero-copy from worker).
+     * @internal
+     */
+    getStreamDownloadResult(token: string): DownloadStreamResult | undefined;
+    /**
+     * Returns the worker script file path.
+     */
+    private resolveWorkerConfig;
+    private ensureInitialized;
+    /**
+     * Returns the index of the worker with the fewest in-flight requests.
+     */
+    private pickWorkerIndex;
+    /**
+     * Dispatches a message to the least-busy worker and tracks the in-flight count.
+     */
+    private dispatchToWorker;
+    /**
+     * Dispatches a download-to-file message to the least-busy worker and tracks the in-flight count.
+     */
+    private dispatchDownloadToWorker;
+    private extractPartNumber;
+    /**
+     * Returns a consumed ArrayBuffer to the worker that originally sent it,
+     * so it can be reused for future downloads.
+     * @internal
+     */
+    returnBuffer(buffer: ArrayBuffer, workerIndex?: number): void;
+    handle(request: HttpRequest, options?: WorkerHttpHandlerOptions | WorkerHttpHandlerDownloadOptions): Promise<{
+        response: HttpResponse;
+    }>;
+    updateHttpClientConfig(_key: string, _value: unknown): void;
+    httpHandlerConfigs(): Record<string, unknown>;
+    destroy(): void;
+}
+export {};

package/dist-types/submodules/worker/http-request-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist-types/submodules/worker/index.d.ts

@@ -0,0 +1 @@
+import "./http-request-worker";

package/dist-types/ts3.4/index.d.ts

@@ -0,0 +1,2 @@
+export { S3TransferManager } from "@aws-sdk/lib-transfer-manager/transfer-manager";
+export { IS3TransferManager } from "@aws-sdk/lib-transfer-manager/transfer-manager";

package/dist-types/ts3.4/submodules/transfer-manager/S3TransferManager.d.ts

@@ -0,0 +1,38 @@
+import { Logger } from "@smithy/types";
+import { IS3TransferManager, S3TransferManagerConfig } from "./types";
+import { DownloadStreamResult } from "./worker-http-handler";
+import { destroyStreams, joinStreams } from "./join-streams";
+export interface IWorkerHttpHandler {
+  getDownloadResult(token: string):
+    | {
+        bytesWritten: number;
+        checksum?: string;
+      }
+    | undefined;
+  getStreamDownloadResult(token: string): DownloadStreamResult | undefined;
+}
+export interface WorkerHttpHandlerConstructor {
+  new (options?: {
+    workerThreadCount?: number;
+    maxConcurrentUploads?: number;
+    maxConcurrentDownloads?: number;
+    useODirect?: boolean;
+    logger?: Logger;
+  }): IWorkerHttpHandler;
+}
+export interface S3TransferManagerRuntimeDependencies {
+  joinStreams: typeof joinStreams;
+  destroyStreams: typeof destroyStreams;
+  WorkerHttpHandler: WorkerHttpHandlerConstructor;
+  createEmptyReadable: () => any;
+  defaultWorkerCount: () => number;
+}
+export interface S3TransferManagerConstructor {
+  new (config?: S3TransferManagerConfig): IS3TransferManager;
+}
+export declare const bindS3TransferManager: (
+  deps: S3TransferManagerRuntimeDependencies,
+) => S3TransferManagerConstructor;
+export declare const internalEventHandler: {
+  afterInitialGetObject(): Promise<void>;
+};

package/dist-types/ts3.4/submodules/transfer-manager/chunker.d.ts

@@ -0,0 +1,10 @@
+export interface RawDataPart {
+  partNumber: number;
+  data: Uint8Array;
+  lastPart?: boolean;
+}
+export declare const byteLength: (input: any) => number | undefined;
+export declare const getChunk: (
+  data: any,
+  partSize: number,
+) => AsyncGenerator<RawDataPart, void, undefined>;

package/dist-types/ts3.4/submodules/transfer-manager/directory-transfer-utils.d.ts

@@ -0,0 +1,23 @@
+import { _Object as S3Object } from "@aws-sdk/client-s3";
+import { CannedFailurePolicy, DirectoryTransferFailureContext, FailurePolicy } from "./types";
+export declare class Semaphore {
+  private readonly limit;
+  private inFlightRequests;
+  private waiters;
+  constructor(limit: number);
+  acquire(count?: number): Promise<void>;
+  release(count?: number): void;
+}
+export declare function validateDirectory(dirPath: string): Promise<string>;
+export declare function handleFailure(
+  policy: FailurePolicy,
+  context: DirectoryTransferFailureContext,
+  abortController: AbortController,
+): Promise<CannedFailurePolicy>;
+export declare function createDestinationDirectory(dirPath: string): Promise<string>;
+export declare function isFolderObject(object: S3Object): boolean;
+export declare function deriveLocalPath(destination: string, key: string): string;
+export declare function createLocalParentDirectories(
+  destination: string,
+  localPath: string,
+): Promise<void>;

package/dist-types/ts3.4/submodules/transfer-manager/event-listener-types.d.ts

@@ -0,0 +1,12 @@
+export type EventListenerFunction<E = Event> = (event: Event & E) => void;
+export type EventListener<E = Event> = EventListenerFunction<E> | EventListenerObject<E>;
+export type EventListenerObject<E = Event> = {
+  handleEvent: EventListenerFunction<E>;
+};
+export type AddEventListenerOptions = {
+  once?: boolean;
+  signal?: AbortSignal;
+};
+export type RemoveEventListenerOptions = {
+  capture?: boolean;
+};

package/dist-types/ts3.4/submodules/transfer-manager/file-manager.d.ts

@@ -0,0 +1,9 @@
+export declare class FileManager {
+  private cleanupHandlers;
+  createTempFile(destination: string, totalSize: number): Promise<string>;
+  atomicRename(tempFilePath: string, destination: string): Promise<void>;
+  deleteTempFile(tempFilePath: string): Promise<void>;
+  registerCleanupHandler(tempFilePath: string): void;
+  unregisterCleanupHandler(tempFilePath: string): void;
+  private generateUniqueId;
+}

package/dist-types/ts3.4/submodules/transfer-manager/index.browser.d.ts

@@ -0,0 +1,3 @@
+declare const S3TransferManager_base: import("./S3TransferManager").S3TransferManagerConstructor;
+export declare class S3TransferManager extends S3TransferManager_base {}
+export { DownloadToFileRequest, DownloadToFileResponse, IS3TransferManager } from "./types";

package/dist-types/ts3.4/submodules/transfer-manager/index.d.ts

@@ -0,0 +1,3 @@
+declare const S3TransferManager_base: import("./S3TransferManager").S3TransferManagerConstructor;
+export declare class S3TransferManager extends S3TransferManager_base {}
+export { DownloadToFileRequest, DownloadToFileResponse, IS3TransferManager } from "./types";

package/dist-types/ts3.4/submodules/transfer-manager/join-streams.browser.d.ts

@@ -0,0 +1,13 @@
+import { StreamingBlobPayloadOutputTypes } from "@smithy/types";
+import { JoinStreamIterationEvents } from "./types";
+export declare function joinStreams(
+  streams: Promise<StreamingBlobPayloadOutputTypes>[],
+  eventListeners?: JoinStreamIterationEvents,
+): Promise<StreamingBlobPayloadOutputTypes>;
+export declare function iterateStreams(
+  promises: Promise<StreamingBlobPayloadOutputTypes>[],
+  eventListeners?: JoinStreamIterationEvents,
+): AsyncIterable<Uint8Array, void, void>;
+export declare function destroyStreams(
+  promises: Promise<StreamingBlobPayloadOutputTypes>[],
+): Promise<void>;

package/dist-types/ts3.4/submodules/transfer-manager/join-streams.d.ts

@@ -0,0 +1,13 @@
+import { StreamingBlobPayloadOutputTypes } from "@smithy/types";
+import { JoinStreamIterationEvents } from "./types";
+export declare function joinStreams(
+  streams: Promise<StreamingBlobPayloadOutputTypes>[],
+  eventListeners?: JoinStreamIterationEvents,
+): Promise<StreamingBlobPayloadOutputTypes>;
+export declare function iterateStreams(
+  promises: Promise<StreamingBlobPayloadOutputTypes>[],
+  eventListeners?: JoinStreamIterationEvents,
+): AsyncIterable<StreamingBlobPayloadOutputTypes, void, void>;
+export declare function destroyStreams(
+  promises: Promise<StreamingBlobPayloadOutputTypes>[],
+): Promise<void>;

package/dist-types/ts3.4/submodules/transfer-manager/log-level.d.ts

@@ -0,0 +1,13 @@
+import { Logger } from "@smithy/types";
+export declare class LogLevel implements Logger {
+  private minimum;
+  private logger;
+  private static LEVELS;
+  private readonly level;
+  constructor(minimum: keyof Logger, logger?: Logger);
+  trace(...args: any[]): void;
+  debug(...args: any[]): void;
+  info(...args: any[]): void;
+  warn(...args: any[]): void;
+  error(...args: any[]): void;
+}

package/dist-types/ts3.4/submodules/transfer-manager/ordered-part-queue.d.ts

@@ -0,0 +1,18 @@
+export interface QueuedPart {
+  buffer: ArrayBuffer;
+  byteLength: number;
+}
+export declare class OrderedPartQueue {
+  private pending;
+  private nextExpected;
+  private readonly totalParts;
+  private waitingConsumer;
+  private error;
+  constructor(totalParts: number);
+  enqueue(rangeIndex: number, buffer: ArrayBuffer, byteLength: number): void;
+  dequeue(): Promise<QueuedPart | null>;
+  setError(error: unknown): void;
+  hasError(): boolean;
+  getError(): unknown;
+  readonly pendingCount: number;
+}