@chidchanun/bcp 0.1.25 → 0.1.26

package/packages/server/src/storage-s3.ts

@@ -0,0 +1,1159 @@
+import {
+    createHash,
+} from "node:crypto";
+import {
+    Readable,
+} from "node:stream";
+
+import {
+    DeleteObjectCommand,
+    GetObjectCommand,
+    HeadObjectCommand,
+    PutObjectCommand,
+    S3Client,
+} from "@aws-sdk/client-s3";
+import {
+    Upload,
+} from "@aws-sdk/lib-storage";
+
+import {
+    normalizeStorageKey,
+    StorageError,
+    type StorageAdapter,
+    type StorageObjectMetadata,
+    type StoragePutOptions,
+    type StoragePutStreamOptions,
+    type StorageReadableStream,
+    type StorageReadOptions,
+    type StorageWriteValue,
+} from "./storage.js";
+
+const MIN_MULTIPART_PART_BYTES =
+    5 * 1024 * 1024;
+
+export interface S3StorageMultipartOptions {
+    partSize?: number;
+    queueSize?: number;
+}
+
+export interface S3StorageOptions {
+    bucket: string;
+    region: string;
+    endpoint?: string;
+    forcePathStyle?: boolean;
+    prefix?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    sessionToken?: string;
+    multipart?: S3StorageMultipartOptions;
+    client?: S3Client;
+}
+
+export interface S3StorageAdapter
+extends StorageAdapter {
+    readonly client: S3Client;
+    destroy(): void;
+}
+
+export function createS3Storage(
+    options: S3StorageOptions
+): S3StorageAdapter {
+    const bucket =
+        assertNonEmptyString(
+            options.bucket,
+            "bucket"
+        );
+    const region =
+        assertNonEmptyString(
+            options.region,
+            "region"
+        );
+    const prefix =
+        normalizePrefix(
+            options.prefix
+        );
+    const multipart =
+        normalizeMultipartOptions(
+            options.multipart
+        );
+    const ownsClient =
+        options.client === undefined;
+    const client =
+        options.client ??
+        new S3Client({
+            region,
+            ...(options.endpoint === undefined
+                ? {}
+                : {
+                    endpoint:
+                        assertNonEmptyString(
+                            options.endpoint,
+                            "endpoint"
+                        ),
+                }),
+            forcePathStyle:
+                options.forcePathStyle ??
+                false,
+            ...createCredentials(
+                options
+            ),
+        });
+
+    const providerKey =
+        (key: string) => {
+            const normalizedKey =
+                normalizeStorageKey(
+                    key
+                );
+
+            return {
+                normalizedKey,
+                providerKey:
+                    prefix.length === 0
+                        ? normalizedKey
+                        : `${prefix}/${normalizedKey}`,
+            };
+        };
+
+    const statObject =
+        async (
+            key: string
+        ): Promise<StorageObjectMetadata | null> => {
+            const resolved =
+                providerKey(
+                    key
+                );
+
+            try {
+                const response =
+                    await client.send(
+                        new HeadObjectCommand({
+                            Bucket:
+                                bucket,
+                            Key:
+                                resolved.providerKey,
+                        })
+                    );
+
+                return createS3Metadata(
+                    resolved.normalizedKey,
+                    response.ContentLength ??
+                        0,
+                    response.ContentType ??
+                        "application/octet-stream",
+                    response.LastModified ??
+                        new Date(0),
+                    response.ETag,
+                    response.Metadata
+                        ?.bcp_sha256
+                );
+            } catch (error) {
+                if (
+                    isS3Status(
+                        error,
+                        404
+                    )
+                ) {
+                    return null;
+                }
+
+                throw error;
+            }
+        };
+
+    const readStream =
+        async (
+            key: string,
+            readOptions:
+                StorageReadOptions = {}
+        ): Promise<StorageReadableStream> => {
+            const resolved =
+                providerKey(
+                    key
+                );
+            const metadata =
+                await statObject(
+                    resolved.normalizedKey
+                );
+
+            if (!metadata) {
+                throw objectNotFound(
+                    resolved.normalizedKey
+                );
+            }
+
+            const range =
+                normalizeRemoteRange(
+                    resolved.normalizedKey,
+                    metadata.size,
+                    readOptions
+                );
+
+            try {
+                const response =
+                    await client.send(
+                        new GetObjectCommand({
+                            Bucket:
+                                bucket,
+                            Key:
+                                resolved.providerKey,
+                            ...(range === null
+                                ? {}
+                                : {
+                                    Range:
+                                        `bytes=${range.start}-${range.end}`,
+                                }),
+                        })
+                    );
+
+                if (!response.Body) {
+                    return emptyStorageStream();
+                }
+
+                return ownStorageStream(
+                    response.Body
+                        .transformToWebStream() as
+                        ReadableStream<
+                            Uint8Array
+                        >
+                );
+            } catch (error) {
+                if (
+                    isS3Status(
+                        error,
+                        404
+                    )
+                ) {
+                    throw objectNotFound(
+                        resolved.normalizedKey
+                    );
+                }
+
+                if (
+                    isS3Status(
+                        error,
+                        416
+                    )
+                ) {
+                    throw rangeNotSatisfiable(
+                        resolved.normalizedKey,
+                        metadata.size
+                    );
+                }
+
+                throw error;
+            }
+        };
+
+    return {
+        client,
+
+        capabilities: {
+            streamingRead: true,
+            streamingWrite: true,
+            ranges: true,
+            signedUrls: false,
+            listing: false,
+        },
+
+        async put(
+            key,
+            value,
+            putOptions:
+                StoragePutOptions = {}
+        ) {
+            const resolved =
+                providerKey(
+                    key
+                );
+            const bytes =
+                await toStorageBytes(
+                    value
+                );
+            const checksumSha256 =
+                createHash(
+                    "sha256"
+                )
+                    .update(
+                        bytes
+                    )
+                    .digest(
+                        "hex"
+                    );
+            const contentType =
+                normalizeContentType(
+                    putOptions.contentType ??
+                    (
+                        value instanceof Blob
+                            ? value.type
+                            : ""
+                    )
+                );
+
+            try {
+                await client.send(
+                    new PutObjectCommand({
+                        Bucket:
+                            bucket,
+                        Key:
+                            resolved.providerKey,
+                        Body:
+                            bytes,
+                        ContentLength:
+                            bytes.byteLength,
+                        ContentType:
+                            contentType,
+                        Metadata: {
+                            bcp_sha256:
+                                checksumSha256,
+                        },
+                        ...(putOptions.overwrite
+                            ? {}
+                            : {
+                                IfNoneMatch:
+                                    "*",
+                            }),
+                    })
+                );
+            } catch (error) {
+                const mapped =
+                    mapWriteError(
+                        error,
+                        resolved.normalizedKey
+                    );
+
+                if (mapped) {
+                    throw mapped;
+                }
+
+                throw error;
+            }
+
+            return requireStoredMetadata(
+                statObject,
+                resolved.normalizedKey
+            );
+        },
+
+        async putStream(
+            key,
+            stream,
+            putOptions:
+                StoragePutStreamOptions = {}
+        ) {
+            const resolved =
+                providerKey(
+                    key
+                );
+
+            if (
+                !putOptions.overwrite &&
+                await statObject(
+                    resolved.normalizedKey
+                ) !== null
+            ) {
+                throw objectExists(
+                    resolved.normalizedKey
+                );
+            }
+
+            const contentType =
+                normalizeContentType(
+                    putOptions.contentType ??
+                    ""
+                );
+            const body =
+                createLimitedNodeReadable(
+                    stream,
+                    putOptions.maxBytes,
+                    putOptions.signal
+                );
+            const upload =
+                new Upload({
+                    client,
+                    params: {
+                        Bucket:
+                            bucket,
+                        Key:
+                            resolved.providerKey,
+                        Body:
+                            body,
+                        ContentType:
+                            contentType,
+                    },
+                    leavePartsOnError:
+                        false,
+                    ...(multipart.partSize === undefined
+                        ? {}
+                        : {
+                            partSize:
+                                multipart.partSize,
+                        }),
+                    ...(multipart.queueSize === undefined
+                        ? {}
+                        : {
+                            queueSize:
+                                multipart.queueSize,
+                        }),
+                });
+
+            const onAbort =
+                () => {
+                    void upload.abort();
+                };
+
+            putOptions.signal
+                ?.addEventListener(
+                    "abort",
+                    onAbort,
+                    {
+                        once: true,
+                    }
+                );
+
+            try {
+                putOptions.signal
+                    ?.throwIfAborted();
+                await upload.done();
+                putOptions.signal
+                    ?.throwIfAborted();
+            } catch (error) {
+                if (
+                    putOptions.signal
+                        ?.aborted
+                ) {
+                    throw putOptions.signal
+                        .reason ??
+                        error;
+                }
+
+                const mapped =
+                    mapWriteError(
+                        error,
+                        resolved.normalizedKey
+                    );
+
+                if (mapped) {
+                    throw mapped;
+                }
+
+                throw error;
+            } finally {
+                putOptions.signal
+                    ?.removeEventListener(
+                        "abort",
+                        onAbort
+                    );
+            }
+
+            return requireStoredMetadata(
+                statObject,
+                resolved.normalizedKey
+            );
+        },
+
+        stat:
+            statObject,
+
+        async read(
+            key,
+            readOptions:
+                StorageReadOptions = {}
+        ) {
+            return collectStorageStream(
+                await readStream(
+                    key,
+                    readOptions
+                )
+            );
+        },
+
+        readStream,
+
+        async exists(key) {
+            return (
+                await statObject(
+                    key
+                )
+            ) !== null;
+        },
+
+        async delete(key) {
+            const resolved =
+                providerKey(
+                    key
+                );
+
+            if (
+                await statObject(
+                    resolved.normalizedKey
+                ) === null
+            ) {
+                return false;
+            }
+
+            await client.send(
+                new DeleteObjectCommand({
+                    Bucket:
+                        bucket,
+                    Key:
+                        resolved.providerKey,
+                })
+            );
+
+            return true;
+        },
+
+        destroy() {
+            if (ownsClient) {
+                client.destroy();
+            }
+        },
+    };
+}
+
+function createCredentials(
+    options: S3StorageOptions
+): {
+    credentials: {
+        accessKeyId: string;
+        secretAccessKey: string;
+        sessionToken?: string;
+    };
+} | Record<string, never> {
+    const hasAccessKey =
+        typeof options.accessKeyId ===
+            "string" &&
+        options.accessKeyId.trim().length >
+            0;
+    const hasSecret =
+        typeof options.secretAccessKey ===
+            "string" &&
+        options.secretAccessKey.trim().length >
+            0;
+
+    if (
+        hasAccessKey !== hasSecret
+    ) {
+        throw new TypeError(
+            "BCP Framework: S3 accessKeyId and secretAccessKey must be provided together."
+        );
+    }
+
+    if (!hasAccessKey) {
+        return {};
+    }
+
+    return {
+        credentials: {
+            accessKeyId:
+                options.accessKeyId!
+                    .trim(),
+            secretAccessKey:
+                options.secretAccessKey!
+                    .trim(),
+            ...(typeof options.sessionToken ===
+                "string" &&
+            options.sessionToken.trim().length >
+                0
+                ? {
+                    sessionToken:
+                        options.sessionToken.trim(),
+                }
+                : {}),
+        },
+    };
+}
+
+function normalizeMultipartOptions(
+    value:
+        S3StorageMultipartOptions |
+        undefined
+): S3StorageMultipartOptions {
+    if (!value) {
+        return {};
+    }
+
+    if (
+        value.partSize !== undefined &&
+        (
+            !Number.isSafeInteger(
+                value.partSize
+            ) ||
+            value.partSize <
+                MIN_MULTIPART_PART_BYTES
+        )
+    ) {
+        throw new TypeError(
+            `BCP Framework: S3 multipart partSize must be at least ${MIN_MULTIPART_PART_BYTES} bytes.`
+        );
+    }
+
+    if (
+        value.queueSize !== undefined &&
+        (
+            !Number.isSafeInteger(
+                value.queueSize
+            ) ||
+            value.queueSize < 1
+        )
+    ) {
+        throw new TypeError(
+            "BCP Framework: S3 multipart queueSize must be a positive safe integer."
+        );
+    }
+
+    return {
+        ...(value.partSize === undefined
+            ? {}
+            : {
+                partSize:
+                    value.partSize,
+            }),
+        ...(value.queueSize === undefined
+            ? {}
+            : {
+                queueSize:
+                    value.queueSize,
+            }),
+    };
+}
+
+function normalizePrefix(
+    value: string | undefined
+): string {
+    if (value === undefined) {
+        return "";
+    }
+
+    const trimmed =
+        value
+            .trim()
+            .replace(
+                /^\/+|\/+$/g,
+                ""
+            );
+
+    if (trimmed.length === 0) {
+        return "";
+    }
+
+    return normalizeStorageKey(
+        trimmed
+    );
+}
+
+async function toStorageBytes(
+    value: StorageWriteValue
+): Promise<Uint8Array> {
+    if (
+        typeof value ===
+        "string"
+    ) {
+        return Uint8Array.from(
+            Buffer.from(
+                value,
+                "utf8"
+            )
+        );
+    }
+
+    if (
+        value instanceof Blob
+    ) {
+        return new Uint8Array(
+            await value.arrayBuffer()
+        );
+    }
+
+    if (
+        value instanceof ArrayBuffer
+    ) {
+        return new Uint8Array(
+            value
+        );
+    }
+
+    if (
+        ArrayBuffer.isView(
+            value
+        )
+    ) {
+        return Uint8Array.from(
+            new Uint8Array(
+                value.buffer,
+                value.byteOffset,
+                value.byteLength
+            )
+        );
+    }
+
+    throw new TypeError(
+        "BCP Framework: unsupported S3 storage write value."
+    );
+}
+
+function createLimitedNodeReadable(
+    stream:
+        ReadableStream<Uint8Array>,
+    maxBytes: number | undefined,
+    signal: AbortSignal | undefined
+): Readable {
+    const normalizedLimit =
+        normalizeOptionalByteLimit(
+            maxBytes
+        );
+
+    return Readable.from(
+        (async function* () {
+            const reader =
+                stream.getReader();
+            let totalBytes =
+                0;
+            const onAbort =
+                () => {
+                    void reader.cancel(
+                        signal?.reason
+                    );
+                };
+
+            signal?.addEventListener(
+                "abort",
+                onAbort,
+                {
+                    once: true,
+                }
+            );
+
+            try {
+                while (true) {
+                    signal?.throwIfAborted();
+
+                    const result =
+                        await reader.read();
+
+                    if (result.done) {
+                        break;
+                    }
+
+                    const chunk =
+                        result.value;
+                    const nextTotal =
+                        totalBytes +
+                        chunk.byteLength;
+
+                    if (
+                        normalizedLimit !== undefined &&
+                        nextTotal >
+                            normalizedLimit
+                    ) {
+                        throw new StorageError(
+                            "STREAM_TOO_LARGE",
+                            `BCP Framework: storage stream exceeded ${normalizedLimit} bytes.`,
+                            413
+                        );
+                    }
+
+                    totalBytes =
+                        nextTotal;
+
+                    yield Buffer.from(
+                        chunk
+                    );
+                }
+            } catch (error) {
+                try {
+                    await reader.cancel(
+                        error
+                    );
+                } catch {
+                    // Preserve the source/upload failure.
+                }
+
+                throw error;
+            } finally {
+                signal?.removeEventListener(
+                    "abort",
+                    onAbort
+                );
+                reader.releaseLock();
+            }
+        })()
+    );
+}
+
+async function collectStorageStream(
+    stream: StorageReadableStream
+): Promise<Uint8Array<ArrayBuffer>> {
+    const reader =
+        stream.getReader();
+    const chunks:
+        Uint8Array<ArrayBuffer>[] = [];
+    let totalBytes =
+        0;
+
+    try {
+        while (true) {
+            const result =
+                await reader.read();
+
+            if (result.done) {
+                break;
+            }
+
+            const chunk =
+                result.value;
+            const owned =
+                new Uint8Array(
+                    chunk.byteLength
+                );
+            owned.set(
+                chunk
+            );
+            chunks.push(
+                owned
+            );
+            totalBytes +=
+                owned.byteLength;
+        }
+    } finally {
+        reader.releaseLock();
+    }
+
+    const result =
+        new Uint8Array(
+            totalBytes
+        );
+    let offset =
+        0;
+
+    for (const chunk of chunks) {
+        result.set(
+            chunk,
+            offset
+        );
+        offset +=
+            chunk.byteLength;
+    }
+
+    return result;
+}
+
+function ownStorageStream(
+    source:
+        ReadableStream<Uint8Array>
+): StorageReadableStream {
+    const reader =
+        source.getReader();
+
+    return new ReadableStream<
+        Uint8Array<ArrayBuffer>
+    >({
+        async pull(controller) {
+            try {
+                const result =
+                    await reader.read();
+
+                if (result.done) {
+                    reader.releaseLock();
+                    controller.close();
+                    return;
+                }
+
+                const owned =
+                    new Uint8Array(
+                        result.value.byteLength
+                    );
+                owned.set(
+                    result.value
+                );
+                controller.enqueue(
+                    owned
+                );
+            } catch (error) {
+                try {
+                    reader.releaseLock();
+                } catch {
+                    // Ignore lock cleanup after failure.
+                }
+                controller.error(
+                    error
+                );
+            }
+        },
+
+        async cancel(reason) {
+            try {
+                await reader.cancel(
+                    reason
+                );
+            } finally {
+                try {
+                    reader.releaseLock();
+                } catch {
+                    // Ignore duplicate release.
+                }
+            }
+        },
+    });
+}
+
+function emptyStorageStream(): StorageReadableStream {
+    return new ReadableStream<
+        Uint8Array<ArrayBuffer>
+    >({
+        start(controller) {
+            controller.close();
+        },
+    });
+}
+
+function normalizeRemoteRange(
+    key: string,
+    size: number,
+    options: StorageReadOptions
+): {
+    start: number;
+    end: number;
+} | null {
+    if (
+        options.start === undefined &&
+        options.end === undefined
+    ) {
+        return null;
+    }
+
+    const start =
+        options.start ??
+        0;
+    const end =
+        options.end ??
+        size - 1;
+
+    if (
+        !Number.isSafeInteger(
+            start
+        ) ||
+        !Number.isSafeInteger(
+            end
+        ) ||
+        start < 0 ||
+        end < start ||
+        start >= size ||
+        end >= size
+    ) {
+        throw rangeNotSatisfiable(
+            key,
+            size
+        );
+    }
+
+    return {
+        start,
+        end,
+    };
+}
+
+function createS3Metadata(
+    key: string,
+    size: number,
+    contentType: string,
+    lastModified: Date,
+    etag: string | undefined,
+    checksumSha256: string | undefined
+): StorageObjectMetadata {
+    const normalizedChecksum =
+        typeof checksumSha256 ===
+            "string" &&
+        /^[a-f0-9]{64}$/i.test(
+            checksumSha256
+        )
+            ? checksumSha256
+                .toLowerCase()
+            : undefined;
+    const normalizedEtag =
+        typeof etag === "string" &&
+        etag.trim().length > 0
+            ? etag.trim()
+            : `W/"${size}-${Math.trunc(
+                lastModified.getTime()
+            )}"`;
+
+    return {
+        key,
+        size,
+        contentType:
+            normalizeContentType(
+                contentType
+            ),
+        lastModified:
+            new Date(
+                lastModified.getTime()
+            ),
+        etag:
+            normalizedEtag,
+        ...(normalizedChecksum === undefined
+            ? {}
+            : {
+                checksumSha256:
+                    normalizedChecksum,
+            }),
+    };
+}
+
+async function requireStoredMetadata(
+    statObject:
+        (
+            key: string
+        ) => Promise<
+            StorageObjectMetadata |
+            null
+        >,
+    key: string
+): Promise<StorageObjectMetadata> {
+    const metadata =
+        await statObject(
+            key
+        );
+
+    if (!metadata) {
+        throw new Error(
+            `BCP Framework: S3 object "${key}" was written but could not be read back with HeadObject.`
+        );
+    }
+
+    return metadata;
+}
+
+function mapWriteError(
+    error: unknown,
+    key: string
+): StorageError | null {
+    if (error instanceof StorageError) {
+        return error;
+    }
+
+    if (
+        isS3Status(
+            error,
+            409
+        ) ||
+        isS3Status(
+            error,
+            412
+        )
+    ) {
+        return objectExists(
+            key
+        );
+    }
+
+    return null;
+}
+
+function objectNotFound(
+    key: string
+): StorageError {
+    return new StorageError(
+        "OBJECT_NOT_FOUND",
+        `BCP Framework: storage object "${key}" was not found.`,
+        404
+    );
+}
+
+function objectExists(
+    key: string
+): StorageError {
+    return new StorageError(
+        "OBJECT_EXISTS",
+        `BCP Framework: storage object "${key}" already exists.`,
+        409
+    );
+}
+
+function rangeNotSatisfiable(
+    key: string,
+    size: number
+): StorageError {
+    return new StorageError(
+        "RANGE_NOT_SATISFIABLE",
+        `BCP Framework: requested storage range for "${key}" is invalid for ${size} bytes.`,
+        416
+    );
+}
+
+function isS3Status(
+    error: unknown,
+    status: number
+): boolean {
+    return (
+        error instanceof Error &&
+        "$metadata" in error &&
+        (
+            error as Error & {
+                $metadata?: {
+                    httpStatusCode?: number;
+                };
+            }
+        ).$metadata
+            ?.httpStatusCode ===
+            status
+    );
+}
+
+function normalizeContentType(
+    value: string
+): string {
+    const normalized =
+        value
+            .trim()
+            .toLowerCase();
+
+    if (
+        normalized.length === 0 ||
+        /[\r\n]/.test(
+            normalized
+        )
+    ) {
+        return "application/octet-stream";
+    }
+
+    return normalized;
+}
+
+function normalizeOptionalByteLimit(
+    value: number | undefined
+): number | undefined {
+    if (value === undefined) {
+        return undefined;
+    }
+
+    if (
+        !Number.isSafeInteger(
+            value
+        ) ||
+        value < 0
+    ) {
+        throw new TypeError(
+            "BCP Framework: S3 storage stream maxBytes must be a non-negative safe integer."
+        );
+    }
+
+    return value;
+}
+
+function assertNonEmptyString(
+    value: string,
+    label: string
+): string {
+    if (
+        typeof value !== "string" ||
+        value.trim().length === 0
+    ) {
+        throw new TypeError(
+            `BCP Framework: S3 storage ${label} must be a non-empty string.`
+        );
+    }
+
+    return value.trim();
+}