@chidchanun/bcp 0.1.25 → 0.1.26

package/packages/server/src/storage.ts

@@ -20,12 +20,15 @@ import {
 
 const METADATA_DIRECTORY =
     ".bcp-storage-meta";
+const STREAM_CHUNK_BYTES =
+    64 * 1024;
 
 export type StorageErrorCode =
     | "INVALID_KEY"
     | "OBJECT_NOT_FOUND"
     | "OBJECT_EXISTS"
-    | "RANGE_NOT_SATISFIABLE";
+    | "RANGE_NOT_SATISFIABLE"
+    | "STREAM_TOO_LARGE";
 
 export class StorageError extends Error {
     readonly code:
@@ -62,18 +65,49 @@ export interface StoragePutOptions {
     overwrite?: boolean;
 }
 
+export interface StoragePutStreamOptions
+extends StoragePutOptions {
+    maxBytes?: number;
+    signal?: AbortSignal;
+}
+
 export interface StorageReadOptions {
     start?: number;
     end?: number;
 }
 
+export interface StorageAdapterCapabilities {
+    streamingRead: boolean;
+    streamingWrite: boolean;
+    ranges: boolean;
+    signedUrls: boolean;
+    listing: boolean;
+}
+
+export type StorageReadableStream =
+    ReadableStream<
+        Uint8Array<ArrayBuffer>
+    >;
+
 export interface StorageAdapter {
+    capabilities?:
+        Partial<
+            StorageAdapterCapabilities
+        >;
+
     put(
         key: string,
         value: StorageWriteValue,
         options?: StoragePutOptions
     ): Promise<StorageObjectMetadata>;
 
+    putStream?(
+        key: string,
+        stream:
+            ReadableStream<Uint8Array>,
+        options?: StoragePutStreamOptions
+    ): Promise<StorageObjectMetadata>;
+
     stat(
         key: string
     ): Promise<StorageObjectMetadata | null>;
@@ -83,6 +117,11 @@ export interface StorageAdapter {
         options?: StorageReadOptions
     ): Promise<Uint8Array>;
 
+    readStream?(
+        key: string,
+        options?: StorageReadOptions
+    ): Promise<StorageReadableStream>;
+
     exists(
         key: string
     ): Promise<boolean>;
@@ -137,7 +176,433 @@ export function createLocalStorage(
             METADATA_DIRECTORY
         );
 
+    const statObject =
+        async (
+            key: string
+        ): Promise<StorageObjectMetadata | null> => {
+            const normalizedKey =
+                normalizeStorageKey(
+                    key
+                );
+            const destination =
+                resolveStoragePath(
+                    rootDirectory,
+                    normalizedKey
+                );
+
+            let fileStat;
+
+            try {
+                fileStat =
+                    await stat(
+                        destination
+                    );
+            } catch (error) {
+                if (
+                    isNodeErrorCode(
+                        error,
+                        "ENOENT"
+                    )
+                ) {
+                    return null;
+                }
+
+                throw error;
+            }
+
+            if (!fileStat.isFile()) {
+                return null;
+            }
+
+            const record =
+                await readMetadataRecord(
+                    metadataDirectory,
+                    normalizedKey
+                );
+
+            return createObjectMetadata(
+                normalizedKey,
+                fileStat.size,
+                fileStat.mtime,
+                record?.contentType ??
+                    "application/octet-stream",
+                record?.checksumSha256
+            );
+        };
+
+    const putStream =
+        async (
+            key: string,
+            stream:
+                ReadableStream<Uint8Array>,
+            putOptions:
+                StoragePutStreamOptions = {}
+        ): Promise<StorageObjectMetadata> => {
+            const normalizedKey =
+                normalizeStorageKey(
+                    key
+                );
+            const destination =
+                resolveStoragePath(
+                    rootDirectory,
+                    normalizedKey
+                );
+            const contentType =
+                normalizeContentType(
+                    putOptions.contentType ??
+                    ""
+                );
+            const maxBytes =
+                normalizeOptionalByteLimit(
+                    putOptions.maxBytes,
+                    "stream maxBytes"
+                );
+
+            await mkdir(
+                path.dirname(
+                    destination
+                ),
+                {
+                    recursive: true,
+                }
+            );
+
+            const reader =
+                stream.getReader();
+            let handle:
+                Awaited<
+                    ReturnType<typeof open>
+                > | null =
+                null;
+            let destinationOpened =
+                false;
+            let completed =
+                false;
+
+            try {
+                putOptions.signal
+                    ?.throwIfAborted();
+
+                try {
+                    handle =
+                        await open(
+                            destination,
+                            putOptions.overwrite
+                                ? "w"
+                                : "wx"
+                        );
+                    destinationOpened =
+                        true;
+                } catch (error) {
+                    if (
+                        isNodeErrorCode(
+                            error,
+                            "EEXIST"
+                        )
+                    ) {
+                        throw new StorageError(
+                            "OBJECT_EXISTS",
+                            `BCP Framework: storage object "${normalizedKey}" already exists.`,
+                            409
+                        );
+                    }
+
+                    throw error;
+                }
+
+                const hash =
+                    createHash(
+                        "sha256"
+                    );
+                let totalBytes =
+                    0;
+
+                while (true) {
+                    putOptions.signal
+                        ?.throwIfAborted();
+
+                    const result =
+                        await reader.read();
+
+                    if (result.done) {
+                        break;
+                    }
+
+                    const chunk =
+                        result.value;
+
+                    if (
+                        !(chunk instanceof
+                            Uint8Array)
+                    ) {
+                        throw new TypeError(
+                            "BCP Framework: storage streams must emit Uint8Array chunks."
+                        );
+                    }
+
+                    const nextTotal =
+                        totalBytes +
+                        chunk.byteLength;
+
+                    if (
+                        maxBytes !== undefined &&
+                        nextTotal > maxBytes
+                    ) {
+                        throw new StorageError(
+                            "STREAM_TOO_LARGE",
+                            `BCP Framework: storage stream exceeded ${maxBytes} bytes.`,
+                            413
+                        );
+                    }
+
+                    await writeAll(
+                        handle,
+                        chunk
+                    );
+                    hash.update(
+                        chunk
+                    );
+                    totalBytes =
+                        nextTotal;
+                }
+
+                await handle.close();
+                handle =
+                    null;
+
+                const checksumSha256 =
+                    hash.digest(
+                        "hex"
+                    );
+
+                await writeMetadataRecord(
+                    metadataDirectory,
+                    {
+                        version: 1,
+                        key:
+                            normalizedKey,
+                        contentType,
+                        checksumSha256,
+                    }
+                );
+
+                const fileStat =
+                    await stat(
+                        destination
+                    );
+
+                completed =
+                    true;
+
+                return createObjectMetadata(
+                    normalizedKey,
+                    fileStat.size,
+                    fileStat.mtime,
+                    contentType,
+                    checksumSha256
+                );
+            } catch (error) {
+                try {
+                    await reader.cancel(
+                        error
+                    );
+                } catch {
+                    // Ignore source cancellation failures.
+                }
+
+                throw error;
+            } finally {
+                reader.releaseLock();
+
+                if (handle) {
+                    try {
+                        await handle.close();
+                    } catch {
+                        // Preserve the original failure.
+                    }
+                }
+
+                if (
+                    destinationOpened &&
+                    !completed
+                ) {
+                    await rm(
+                        destination,
+                        {
+                            force: true,
+                        }
+                    );
+                }
+            }
+        };
+
+    const readStream =
+        async (
+            key: string,
+            readOptions:
+                StorageReadOptions = {}
+        ): Promise<StorageReadableStream> => {
+            const normalizedKey =
+                normalizeStorageKey(
+                    key
+                );
+            const destination =
+                resolveStoragePath(
+                    rootDirectory,
+                    normalizedKey
+                );
+            const metadata =
+                await statObject(
+                    normalizedKey
+                );
+
+            if (!metadata) {
+                throw new StorageError(
+                    "OBJECT_NOT_FOUND",
+                    `BCP Framework: storage object "${normalizedKey}" was not found.`,
+                    404
+                );
+            }
+
+            const range =
+                normalizeReadRange(
+                    metadata.size,
+                    readOptions
+                );
+
+            if (
+                metadata.size === 0 &&
+                range === null
+            ) {
+                return new ReadableStream<
+                    Uint8Array<ArrayBuffer>
+                >({
+                    start(controller) {
+                        controller.close();
+                    },
+                });
+            }
+
+            const start =
+                range?.start ??
+                0;
+            const end =
+                range?.end ??
+                metadata.size - 1;
+            const handle =
+                await open(
+                    destination,
+                    "r"
+                );
+            let position =
+                start;
+            let closed =
+                false;
+
+            const closeHandle =
+                async () => {
+                    if (closed) {
+                        return;
+                    }
+
+                    closed =
+                        true;
+                    await handle.close();
+                };
+
+            return new ReadableStream<
+                Uint8Array<ArrayBuffer>
+            >({
+                async pull(controller) {
+                    try {
+                        if (
+                            position > end
+                        ) {
+                            await closeHandle();
+                            controller.close();
+                            return;
+                        }
+
+                        const remaining =
+                            end -
+                            position +
+                            1;
+                        const requested =
+                            Math.min(
+                                STREAM_CHUNK_BYTES,
+                                remaining
+                            );
+                        const buffer =
+                            Buffer.allocUnsafe(
+                                requested
+                            );
+                        const result =
+                            await handle.read(
+                                buffer,
+                                0,
+                                requested,
+                                position
+                            );
+
+                        if (
+                            result.bytesRead === 0
+                        ) {
+                            await closeHandle();
+                            controller.close();
+                            return;
+                        }
+
+                        const chunk =
+                            new Uint8Array(
+                                result.bytesRead
+                            );
+                        chunk.set(
+                            buffer.subarray(
+                                0,
+                                result.bytesRead
+                            )
+                        );
+                        position +=
+                            result.bytesRead;
+
+                        controller.enqueue(
+                            chunk
+                        );
+
+                        if (
+                            position > end
+                        ) {
+                            await closeHandle();
+                            controller.close();
+                        }
+                    } catch (error) {
+                        try {
+                            await closeHandle();
+                        } catch {
+                            // Preserve the read failure.
+                        }
+
+                        controller.error(
+                            error
+                        );
+                    }
+                },
+
+                async cancel() {
+                    await closeHandle();
+                },
+            });
+        };
+
     return {
+        capabilities: {
+            streamingRead: true,
+            streamingWrite: true,
+            ranges: true,
+            signedUrls: false,
+            listing: false,
+        },
+
         async put(
             key,
             value,
@@ -234,56 +699,10 @@ export function createLocalStorage(
             );
         },
 
-        async stat(key) {
-            const normalizedKey =
-                normalizeStorageKey(
-                    key
-                );
-            const destination =
-                resolveStoragePath(
-                    rootDirectory,
-                    normalizedKey
-                );
-
-            let fileStat;
-
-            try {
-                fileStat =
-                    await stat(
-                        destination
-                    );
-            } catch (error) {
-                if (
-                    isNodeErrorCode(
-                        error,
-                        "ENOENT"
-                    )
-                ) {
-                    return null;
-                }
-
-                throw error;
-            }
+        putStream,
 
-            if (!fileStat.isFile()) {
-                return null;
-            }
-
-            const record =
-                await readMetadataRecord(
-                    metadataDirectory,
-                    normalizedKey
-                );
-
-            return createObjectMetadata(
-                normalizedKey,
-                fileStat.size,
-                fileStat.mtime,
-                record?.contentType ??
-                    "application/octet-stream",
-                record?.checksumSha256
-            );
-        },
+        stat:
+            statObject,
 
         async read(
             key,
@@ -299,7 +718,7 @@ export function createLocalStorage(
                     normalizedKey
                 );
             const metadata =
-                await this.stat(
+                await statObject(
                     normalizedKey
                 );
 
@@ -359,9 +778,11 @@ export function createLocalStorage(
             }
         },
 
+        readStream,
+
         async exists(key) {
             return (
-                await this.stat(
+                await statObject(
                     key
                 )
             ) !== null;
@@ -410,6 +831,113 @@ export function createLocalStorage(
     };
 }
 
+export function getStorageCapabilities(
+    storage: StorageAdapter
+): StorageAdapterCapabilities {
+    return {
+        streamingRead:
+            storage.capabilities
+                ?.streamingRead ??
+            typeof storage.readStream ===
+                "function",
+        streamingWrite:
+            storage.capabilities
+                ?.streamingWrite ??
+            typeof storage.putStream ===
+                "function",
+        ranges:
+            storage.capabilities
+                ?.ranges ??
+            true,
+        signedUrls:
+            storage.capabilities
+                ?.signedUrls ??
+            false,
+        listing:
+            storage.capabilities
+                ?.listing ??
+            false,
+    };
+}
+
+export async function putStorageStream(
+    storage: StorageAdapter,
+    key: string,
+    stream:
+        ReadableStream<Uint8Array>,
+    options:
+        StoragePutStreamOptions = {}
+): Promise<StorageObjectMetadata> {
+    if (storage.putStream) {
+        return storage.putStream(
+            key,
+            stream,
+            options
+        );
+    }
+
+    const bytes =
+        await collectStorageStream(
+            stream,
+            options
+        );
+
+    return storage.put(
+        key,
+        bytes,
+        {
+            contentType:
+                options.contentType,
+            overwrite:
+                options.overwrite,
+        }
+    );
+}
+
+export async function readStorageStream(
+    storage: StorageAdapter,
+    key: string,
+    options:
+        StorageReadOptions = {}
+): Promise<StorageReadableStream> {
+    if (storage.readStream) {
+        return storage.readStream(
+            key,
+            options
+        );
+    }
+
+    const bytes =
+        await storage.read(
+            key,
+            options
+        );
+    const chunk =
+        new Uint8Array(
+            bytes.byteLength
+        );
+
+    chunk.set(
+        bytes
+    );
+
+    return new ReadableStream<
+        Uint8Array<ArrayBuffer>
+    >({
+        start(controller) {
+            if (
+                chunk.byteLength > 0
+            ) {
+                controller.enqueue(
+                    chunk
+                );
+            }
+
+            controller.close();
+        },
+    });
+}
+
 export async function storeUploadedFile(
     file: File,
     options: StoreUploadedFileOptions
@@ -428,15 +956,19 @@ export async function storeUploadedFile(
                 options.key
             );
     const metadata =
-        await options.storage.put(
+        await putStorageStream(
+            options.storage,
             key,
-            file,
+            file.stream(),
             {
                 overwrite:
                     options.overwrite,
                 contentType:
                     file.type ||
                     "application/octet-stream",
+                maxBytes:
+                    options.constraints
+                        ?.maxBytes,
             }
         );
 
@@ -583,6 +1115,160 @@ async function toStorageBytes(
     );
 }
 
+async function collectStorageStream(
+    stream:
+        ReadableStream<Uint8Array>,
+    options:
+        Pick<
+            StoragePutStreamOptions,
+            "maxBytes" |
+            "signal"
+        >
+): Promise<Uint8Array<ArrayBuffer>> {
+    const maxBytes =
+        normalizeOptionalByteLimit(
+            options.maxBytes,
+            "stream maxBytes"
+        );
+    const chunks:
+        Uint8Array<ArrayBuffer>[] = [];
+    let totalBytes =
+        0;
+    const reader =
+        stream.getReader();
+
+    try {
+        while (true) {
+            options.signal
+                ?.throwIfAborted();
+
+            const result =
+                await reader.read();
+
+            if (result.done) {
+                break;
+            }
+
+            const chunk =
+                result.value;
+            const nextTotal =
+                totalBytes +
+                chunk.byteLength;
+
+            if (
+                maxBytes !== undefined &&
+                nextTotal > maxBytes
+            ) {
+                throw new StorageError(
+                    "STREAM_TOO_LARGE",
+                    `BCP Framework: storage stream exceeded ${maxBytes} bytes.`,
+                    413
+                );
+            }
+
+            const ownedChunk =
+                new Uint8Array(
+                    chunk.byteLength
+                );
+            ownedChunk.set(
+                chunk
+            );
+            chunks.push(
+                ownedChunk
+            );
+            totalBytes =
+                nextTotal;
+        }
+    } catch (error) {
+        try {
+            await reader.cancel(
+                error
+            );
+        } catch {
+            // Ignore source cancellation failures.
+        }
+
+        throw error;
+    } finally {
+        reader.releaseLock();
+    }
+
+    const bytes =
+        new Uint8Array(
+            totalBytes
+        );
+    let offset =
+        0;
+
+    for (const chunk of chunks) {
+        bytes.set(
+            chunk,
+            offset
+        );
+        offset +=
+            chunk.byteLength;
+    }
+
+    return bytes;
+}
+
+async function writeAll(
+    handle:
+        Awaited<
+            ReturnType<typeof open>
+        >,
+    bytes: Uint8Array
+): Promise<void> {
+    let offset =
+        0;
+
+    while (
+        offset < bytes.byteLength
+    ) {
+        const result =
+            await handle.write(
+                bytes,
+                offset,
+                bytes.byteLength -
+                    offset,
+                null
+            );
+
+        if (
+            result.bytesWritten <= 0
+        ) {
+            throw new Error(
+                "BCP Framework: storage stream write made no progress."
+            );
+        }
+
+        offset +=
+            result.bytesWritten;
+    }
+}
+
+function normalizeOptionalByteLimit(
+    value: number | undefined,
+    label: string
+): number | undefined {
+    if (value === undefined) {
+        return undefined;
+    }
+
+    if (
+        !Number.isSafeInteger(
+            value
+        ) ||
+        value < 0
+    ) {
+        throw new TypeError(
+            `BCP Framework: storage ${label} must be a non-negative safe integer.`
+        );
+    }
+
+    return value;
+}
+
 function normalizeReadRange(
     size: number,
     options: StorageReadOptions