@chidchanun/bcp 0.1.26 → 0.1.28

package/packages/server/src/storage-s3-ecosystem.ts

@@ -0,0 +1,947 @@
+import {
+    CopyObjectCommand,
+    DeleteObjectsCommand,
+    GetObjectCommand,
+    HeadObjectCommand,
+    ListObjectsV2Command,
+    PutObjectCommand,
+} from "@aws-sdk/client-s3";
+import {
+    getSignedUrl,
+} from "@aws-sdk/s3-request-presigner";
+
+import {
+    createS3Storage as createBaseS3Storage,
+    type S3StorageAdapter,
+    type S3StorageOptions,
+} from "./storage-s3.js";
+import {
+    normalizeStorageKey,
+    StorageError,
+    type StorageObjectMetadata,
+} from "./storage.js";
+import {
+    normalizeStorageListLimit,
+    normalizeStorageListPrefix,
+    normalizeStorageSignedUrlExpiry,
+    normalizeStorageUserMetadata,
+    type StorageCopyOptions,
+    type StorageDeleteManyFailure,
+    type StorageDeleteManyResult,
+    type StorageEcosystemAdapter,
+    type StorageEcosystemPutOptions,
+    type StorageEcosystemPutStreamOptions,
+    type StorageListOptions,
+    type StorageListResult,
+    type StorageUserMetadata,
+} from "./storage-ecosystem.js";
+
+const S3_DELETE_BATCH_SIZE =
+    1000;
+
+export type S3StorageEcosystemAdapter =
+    Omit<
+        S3StorageAdapter,
+        "put" |
+        "putStream"
+    > &
+    StorageEcosystemAdapter;
+
+export function createS3Storage(
+    options: S3StorageOptions
+): S3StorageEcosystemAdapter {
+    const base =
+        createBaseS3Storage(
+            options
+        );
+    const adapter =
+        base as unknown as
+        S3StorageEcosystemAdapter;
+    const bucket =
+        assertNonEmptyString(
+            options.bucket,
+            "bucket"
+        );
+    const prefix =
+        normalizeS3Prefix(
+            options.prefix
+        );
+    const basePut =
+        base.put.bind(
+            base
+        );
+    const basePutStream =
+        base.putStream?.bind(
+            base
+        );
+    const baseDelete =
+        base.delete.bind(
+            base
+        );
+
+    adapter.capabilities = {
+        ...base.capabilities,
+        listing: true,
+        signedUrls: true,
+    };
+
+    adapter.put =
+        async (
+            key,
+            value,
+            putOptions:
+                StorageEcosystemPutOptions = {}
+        ) => {
+            const stored =
+                await basePut(
+                    key,
+                    value,
+                    putOptions
+                );
+
+            if (
+                putOptions.metadata !==
+                undefined
+            ) {
+                await setS3Metadata(
+                    adapter,
+                    bucket,
+                    prefix,
+                    stored.key,
+                    putOptions.metadata
+                );
+            }
+
+            return (
+                await adapter.stat(
+                    stored.key
+                )
+            ) ?? stored;
+        };
+
+    if (basePutStream) {
+        adapter.putStream =
+            async (
+                key,
+                stream,
+                putOptions:
+                    StorageEcosystemPutStreamOptions = {}
+            ) => {
+                const stored =
+                    await basePutStream(
+                        key,
+                        stream,
+                        putOptions
+                    );
+
+                if (
+                    putOptions.metadata !==
+                    undefined
+                ) {
+                    await setS3Metadata(
+                        adapter,
+                        bucket,
+                        prefix,
+                        stored.key,
+                        putOptions.metadata
+                    );
+                }
+
+                return (
+                    await adapter.stat(
+                        stored.key
+                    )
+                ) ?? stored;
+            };
+    }
+
+    adapter.getMetadata =
+        (key) =>
+            getS3Metadata(
+                adapter,
+                bucket,
+                prefix,
+                key
+            );
+
+    adapter.setMetadata =
+        (
+            key,
+            metadata
+        ) =>
+            setS3Metadata(
+                adapter,
+                bucket,
+                prefix,
+                key,
+                metadata
+            );
+
+    adapter.list =
+        (listOptions = {}) =>
+            listS3Objects(
+                adapter,
+                bucket,
+                prefix,
+                listOptions
+            );
+
+    adapter.copy =
+        (
+            sourceKey,
+            destinationKey,
+            copyOptions = {}
+        ) =>
+            copyS3Object(
+                adapter,
+                bucket,
+                prefix,
+                sourceKey,
+                destinationKey,
+                copyOptions
+            );
+
+    adapter.move =
+        async (
+            sourceKey,
+            destinationKey,
+            copyOptions = {}
+        ) => {
+            const source =
+                normalizeStorageKey(
+                    sourceKey
+                );
+            const destination =
+                normalizeStorageKey(
+                    destinationKey
+                );
+
+            if (source === destination) {
+                return requireS3Object(
+                    adapter,
+                    source
+                );
+            }
+
+            const stored =
+                await copyS3Object(
+                    adapter,
+                    bucket,
+                    prefix,
+                    source,
+                    destination,
+                    copyOptions
+                );
+
+            await baseDelete(
+                source
+            );
+
+            return stored;
+        };
+
+    adapter.deleteMany =
+        (keys) =>
+            deleteS3Objects(
+                adapter,
+                bucket,
+                prefix,
+                keys
+            );
+
+    adapter.createSignedReadUrl =
+        async (
+            key,
+            signedOptions = {}
+        ) => {
+            const resolved =
+                resolveS3Key(
+                    prefix,
+                    key
+                );
+
+            return getSignedUrl(
+                adapter.client,
+                new GetObjectCommand({
+                    Bucket:
+                        bucket,
+                    Key:
+                        resolved.providerKey,
+                }),
+                {
+                    expiresIn:
+                        normalizeStorageSignedUrlExpiry(
+                            signedOptions.expiresIn
+                        ),
+                }
+            );
+        };
+
+    adapter.createSignedWriteUrl =
+        async (
+            key,
+            signedOptions = {}
+        ) => {
+            const resolved =
+                resolveS3Key(
+                    prefix,
+                    key
+                );
+            const metadata =
+                normalizeStorageUserMetadata(
+                    signedOptions.metadata
+                );
+
+            return getSignedUrl(
+                adapter.client,
+                new PutObjectCommand({
+                    Bucket:
+                        bucket,
+                    Key:
+                        resolved.providerKey,
+                    ...(signedOptions.contentType
+                        ? {
+                            ContentType:
+                                normalizeContentType(
+                                    signedOptions.contentType
+                                ),
+                        }
+                        : {}),
+                    ...(Object.keys(
+                        metadata
+                    ).length > 0
+                        ? {
+                            Metadata:
+                                metadata,
+                        }
+                        : {}),
+                }),
+                {
+                    expiresIn:
+                        normalizeStorageSignedUrlExpiry(
+                            signedOptions.expiresIn
+                        ),
+                }
+            );
+        };
+
+    return adapter;
+}
+
+async function listS3Objects(
+    adapter: S3StorageEcosystemAdapter,
+    bucket: string,
+    prefix: string,
+    options: StorageListOptions
+): Promise<StorageListResult> {
+    const logicalPrefix =
+        normalizeStorageListPrefix(
+            options.prefix
+        );
+    const providerPrefix =
+        combineS3Prefix(
+            prefix,
+            logicalPrefix
+        );
+    const response =
+        await adapter.client.send(
+            new ListObjectsV2Command({
+                Bucket:
+                    bucket,
+                ...(providerPrefix
+                    ? {
+                        Prefix:
+                            providerPrefix,
+                    }
+                    : {}),
+                MaxKeys:
+                    normalizeStorageListLimit(
+                        options.limit
+                    ),
+                ...(options.cursor
+                    ? {
+                        ContinuationToken:
+                            options.cursor,
+                    }
+                    : {}),
+            })
+        );
+    const objects:
+        StorageObjectMetadata[] = [];
+
+    for (
+        const object
+        of response.Contents ?? []
+    ) {
+        if (!object.Key) {
+            continue;
+        }
+
+        const logicalKey =
+            toLogicalS3Key(
+                prefix,
+                object.Key
+            );
+
+        if (!logicalKey) {
+            continue;
+        }
+
+        const size =
+            object.Size ?? 0;
+        const lastModified =
+            object.LastModified ??
+            new Date(0);
+
+        objects.push({
+            key:
+                logicalKey,
+            size,
+            contentType:
+                "application/octet-stream",
+            lastModified:
+                new Date(
+                    lastModified.getTime()
+                ),
+            etag:
+                object.ETag ??
+                `W/"${size}-${Math.trunc(
+                    lastModified.getTime()
+                )}"`,
+        });
+    }
+
+    return {
+        objects,
+        ...(response.IsTruncated &&
+        response.NextContinuationToken
+            ? {
+                cursor:
+                    response.NextContinuationToken,
+            }
+            : {}),
+    };
+}
+
+async function copyS3Object(
+    adapter: S3StorageEcosystemAdapter,
+    bucket: string,
+    prefix: string,
+    sourceKey: string,
+    destinationKey: string,
+    options: StorageCopyOptions
+): Promise<StorageObjectMetadata> {
+    const source =
+        resolveS3Key(
+            prefix,
+            sourceKey
+        );
+    const destination =
+        resolveS3Key(
+            prefix,
+            destinationKey
+        );
+    const sourceMetadata =
+        await requireS3Object(
+            adapter,
+            source.normalizedKey
+        );
+
+    if (
+        source.normalizedKey ===
+        destination.normalizedKey
+    ) {
+        return sourceMetadata;
+    }
+
+    if (
+        !options.overwrite &&
+        await adapter.exists(
+            destination.normalizedKey
+        )
+    ) {
+        throw new StorageError(
+            "OBJECT_EXISTS",
+            `BCP Framework: storage object "${destination.normalizedKey}" already exists.`,
+            409
+        );
+    }
+
+    await adapter.client.send(
+        new CopyObjectCommand({
+            Bucket:
+                bucket,
+            Key:
+                destination.providerKey,
+            CopySource:
+                createCopySource(
+                    bucket,
+                    source.providerKey
+                ),
+            MetadataDirective:
+                "COPY",
+        })
+    );
+
+    return requireS3Object(
+        adapter,
+        destination.normalizedKey
+    );
+}
+
+async function getS3Metadata(
+    adapter: S3StorageEcosystemAdapter,
+    bucket: string,
+    prefix: string,
+    key: string
+): Promise<StorageUserMetadata> {
+    const resolved =
+        resolveS3Key(
+            prefix,
+            key
+        );
+
+    try {
+        const response =
+            await adapter.client.send(
+                new HeadObjectCommand({
+                    Bucket:
+                        bucket,
+                    Key:
+                        resolved.providerKey,
+                })
+            );
+        const metadata:
+            StorageUserMetadata = {};
+
+        for (
+            const [rawKey, rawValue]
+            of Object.entries(
+                response.Metadata ?? {}
+            )
+        ) {
+            if (
+                rawKey.toLowerCase() ===
+                    "bcp_sha256" ||
+                rawValue === undefined
+            ) {
+                continue;
+            }
+
+            metadata[
+                rawKey.toLowerCase()
+            ] = rawValue;
+        }
+
+        return normalizeStorageUserMetadata(
+            metadata
+        );
+    } catch (error) {
+        if (
+            isS3Status(
+                error,
+                404
+            )
+        ) {
+            throw objectNotFound(
+                resolved.normalizedKey
+            );
+        }
+
+        throw error;
+    }
+}
+
+async function setS3Metadata(
+    adapter: S3StorageEcosystemAdapter,
+    bucket: string,
+    prefix: string,
+    key: string,
+    metadata: StorageUserMetadata
+): Promise<StorageUserMetadata> {
+    const resolved =
+        resolveS3Key(
+            prefix,
+            key
+        );
+    let head;
+
+    try {
+        head =
+            await adapter.client.send(
+                new HeadObjectCommand({
+                    Bucket:
+                        bucket,
+                    Key:
+                        resolved.providerKey,
+                })
+            );
+    } catch (error) {
+        if (
+            isS3Status(
+                error,
+                404
+            )
+        ) {
+            throw objectNotFound(
+                resolved.normalizedKey
+            );
+        }
+
+        throw error;
+    }
+
+    const userMetadata =
+        normalizeStorageUserMetadata(
+            metadata
+        );
+    const replacement:
+        Record<string, string> = {
+        ...userMetadata,
+    };
+    const checksumSha256 =
+        head.Metadata
+            ?.bcp_sha256;
+
+    if (checksumSha256) {
+        replacement.bcp_sha256 =
+            checksumSha256;
+    }
+
+    await adapter.client.send(
+        new CopyObjectCommand({
+            Bucket:
+                bucket,
+            Key:
+                resolved.providerKey,
+            CopySource:
+                createCopySource(
+                    bucket,
+                    resolved.providerKey
+                ),
+            MetadataDirective:
+                "REPLACE",
+            Metadata:
+                replacement,
+            ContentType:
+                head.ContentType ??
+                "application/octet-stream",
+        })
+    );
+
+    return {
+        ...userMetadata,
+    };
+}
+
+async function deleteS3Objects(
+    adapter: S3StorageEcosystemAdapter,
+    bucket: string,
+    prefix: string,
+    keys: readonly string[]
+): Promise<StorageDeleteManyResult> {
+    const normalizedKeys =
+        [
+            ...new Set(
+                keys.map(
+                    (key) =>
+                        normalizeStorageKey(
+                            key
+                        )
+                )
+            ),
+        ];
+    const deleted:
+        string[] = [];
+    const failed:
+        StorageDeleteManyFailure[] = [];
+
+    for (
+        let offset = 0;
+        offset < normalizedKeys.length;
+        offset += S3_DELETE_BATCH_SIZE
+    ) {
+        const batch =
+            normalizedKeys.slice(
+                offset,
+                offset +
+                    S3_DELETE_BATCH_SIZE
+            );
+        const byProviderKey =
+            new Map<string, string>();
+
+        for (const key of batch) {
+            byProviderKey.set(
+                resolveS3Key(
+                    prefix,
+                    key
+                ).providerKey,
+                key
+            );
+        }
+
+        const response =
+            await adapter.client.send(
+                new DeleteObjectsCommand({
+                    Bucket:
+                        bucket,
+                    Delete: {
+                        Quiet: false,
+                        Objects:
+                            [
+                                ...byProviderKey.keys(),
+                            ].map(
+                                (providerKey) => ({
+                                    Key:
+                                        providerKey,
+                                })
+                            ),
+                    },
+                })
+            );
+        const failedKeys =
+            new Set<string>();
+
+        for (
+            const error
+            of response.Errors ?? []
+        ) {
+            if (!error.Key) {
+                continue;
+            }
+
+            const logicalKey =
+                byProviderKey.get(
+                    error.Key
+                );
+
+            if (!logicalKey) {
+                continue;
+            }
+
+            failedKeys.add(
+                logicalKey
+            );
+            failed.push({
+                key:
+                    logicalKey,
+                message:
+                    error.Message ??
+                    error.Code ??
+                    "S3 delete failed.",
+            });
+        }
+
+        for (const key of batch) {
+            if (
+                !failedKeys.has(
+                    key
+                )
+            ) {
+                deleted.push(
+                    key
+                );
+            }
+        }
+    }
+
+    return {
+        deleted,
+        missing: [],
+        failed,
+    };
+}
+
+async function requireS3Object(
+    adapter: S3StorageAdapter,
+    key: string
+): Promise<StorageObjectMetadata> {
+    const metadata =
+        await adapter.stat(
+            key
+        );
+
+    if (!metadata) {
+        throw objectNotFound(
+            key
+        );
+    }
+
+    return metadata;
+}
+
+function resolveS3Key(
+    prefix: string,
+    key: string
+): {
+    normalizedKey: string;
+    providerKey: string;
+} {
+    const normalizedKey =
+        normalizeStorageKey(
+            key
+        );
+
+    return {
+        normalizedKey,
+        providerKey:
+            prefix
+                ? `${prefix}/${normalizedKey}`
+                : normalizedKey,
+    };
+}
+
+function normalizeS3Prefix(
+    value: string | undefined
+): string {
+    if (value === undefined) {
+        return "";
+    }
+
+    const normalized =
+        value
+            .trim()
+            .replace(
+                /\\/g,
+                "/"
+            )
+            .replace(
+                /^\/+|\/+$/g,
+                ""
+            );
+
+    return normalized
+        ? normalizeStorageKey(
+            normalized
+        )
+        : "";
+}
+
+function combineS3Prefix(
+    prefix: string,
+    logicalPrefix: string
+): string {
+    if (!prefix) {
+        return logicalPrefix;
+    }
+
+    return logicalPrefix
+        ? `${prefix}/${logicalPrefix}`
+        : `${prefix}/`;
+}
+
+function toLogicalS3Key(
+    prefix: string,
+    providerKey: string
+): string | null {
+    const value =
+        prefix
+            ? (
+                providerKey.startsWith(
+                    `${prefix}/`
+                )
+                    ? providerKey.slice(
+                        prefix.length + 1
+                    )
+                    : ""
+            )
+            : providerKey;
+
+    if (!value) {
+        return null;
+    }
+
+    try {
+        return normalizeStorageKey(
+            value
+        );
+    } catch {
+        return null;
+    }
+}
+
+function createCopySource(
+    bucket: string,
+    providerKey: string
+): string {
+    return `/${encodeURIComponent(
+        bucket
+    )}/${providerKey
+        .split("/")
+        .map(
+            encodeURIComponent
+        )
+        .join("/")}`;
+}
+
+function normalizeContentType(
+    value: string
+): string {
+    const normalized =
+        value
+            .trim()
+            .toLowerCase();
+
+    return (
+        !normalized ||
+        /[\r\n]/.test(
+            normalized
+        )
+    )
+        ? "application/octet-stream"
+        : normalized;
+}
+
+function objectNotFound(
+    key: string
+): StorageError {
+    return new StorageError(
+        "OBJECT_NOT_FOUND",
+        `BCP Framework: storage object "${key}" was not found.`,
+        404
+    );
+}
+
+function isS3Status(
+    error: unknown,
+    status: number
+): boolean {
+    if (
+        !error ||
+        typeof error !==
+            "object"
+    ) {
+        return false;
+    }
+
+    const metadata =
+        "$metadata" in error
+            ? (
+                error as {
+                    $metadata?: {
+                        httpStatusCode?: number;
+                    };
+                }
+            ).$metadata
+            : undefined;
+
+    return metadata
+        ?.httpStatusCode ===
+        status;
+}
+
+function assertNonEmptyString(
+    value: string,
+    label: string
+): string {
+    if (
+        typeof value !== "string" ||
+        value.trim().length === 0
+    ) {
+        throw new TypeError(
+            `BCP Framework: S3 storage ${label} must be a non-empty string.`
+        );
+    }
+
+    return value.trim();
+}