@ultimat3/storage 29.4.0 → 30.0.0

package/src/driver-memory.ts

@@ -9,14 +9,17 @@
 import {
   type Clock,
   finiteCount,
+  isUltimateError,
   NotImplementedError,
   type Page,
   pageOf,
   systemClock,
 } from '@ultimat3/core';
+import { pumpBounded } from './byte-pump';
 import {
   assertListOptions,
   assertPutOptions,
+  type CopyOptions,
   DEFAULT_CONTENT_TYPE,
   etagOf,
   type ListOptions,
@@ -29,11 +32,15 @@ import {
   type StorageObject,
   type StorageRead,
   sha256Base64,
+  signedByteBound,
   toBytes,
 } from './driver';
 import type { LocalDriverOptions } from './driver-local';
 import { keyInTheWay } from './driver-memory-conflict';
-import { checksumMismatch, getTooLarge, keyConflict, objectNotFound } from './errors';
+import { memoryPartStore } from './driver-memory-multipart';
+import { checksumMismatch, getTooLarge, keyConflict, objectNotFound, putFailed } from './errors';
+import { objectChanged } from './errors-promote';
+import { withStagedMultipart } from './multipart-staged';
 import {
   assertObjectLockOptions,
   isLocked,
@@ -42,7 +49,15 @@ import {
   requestedLock,
 } from './object-lock';
 import { assertSafeKey } from './path';
-import { assertGetOptions, type GetOptions, servedRange } from './range';
+import { assertPutStream, type PutStreamOptions, streamLimit } from './put-stream';
+import {
+  assertGetOptions,
+  assertStreamRange,
+  type ByteRange,
+  type GetOptions,
+  servedRange,
+  streamedRange,
+} from './range';
 import type { SignedUrlVerification } from './signed-url';
 import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
 import { resolveSigningSecret } from './signing-secret';
@@ -121,167 +136,214 @@ export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): M
     if (blocking !== undefined) throw keyConflict(registered, key, blocking);
   };
 
+  /** Both writes end here, with no await between the checks and the write. */
+  const commit = (safe: string, bytes: Uint8Array, putOptions?: PutOptions): StorageObject => {
+    const object: StorageObject = {
+      key: safe,
+      size: bytes.byteLength,
+      contentType: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE,
+      etag: etagOf(bytes),
+      lastModified: clock.now(),
+      ...(putOptions?.cacheControl === undefined ? {} : { cacheControl: putOptions.cacheControl }),
+      ...(putOptions?.metadata === undefined ? {} : { metadata: putOptions.metadata }),
+    };
+    // Checked AFTER the body is read: the map is the queue the local disk needs a `keyedQueue` for.
+    refuseConflict(safe);
+    refuseOverwrite(safe, 'overwrite');
+    // A copy in, as a file write is: the caller's buffer and metadata are the caller's to reuse.
+    stored.set(safe, {
+      bytes: bytes.slice(),
+      object: snapshot(object),
+      lock: requestedLock(putOptions),
+    });
+    return snapshot(object);
+  };
+
   const found = (key: string): Stored => {
     const hit = stored.get(key);
     if (hit === undefined) throw objectNotFound(registered, key);
     return hit;
   };
 
-  return {
-    name: DRIVER_NAME,
+  return withStagedMultipart(
+    {
+      name: DRIVER_NAME,
 
-    get signedUrlBase(): string {
-      return baseUrl;
-    },
+      get signedUrlBase(): string {
+        return baseUrl;
+      },
 
-    registerAs(diskName: string): void {
-      registered = diskName;
-      if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
-    },
+      registerAs(diskName: string): void {
+        registered = diskName;
+        if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
+      },
 
-    objects(): ReadonlyMap<string, Uint8Array> {
-      return new Map([...stored].map(([key, value]) => [key, value.bytes.slice()]));
-    },
+      objects(): ReadonlyMap<string, Uint8Array> {
+        return new Map([...stored].map(([key, value]) => [key, value.bytes.slice()]));
+      },
 
-    async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
-      const safe = assertSafeKey(key);
-      // The same refusal both shipped disks make: a `put()` that succeeds in a test and throws in
-      // production is a gap an app meets on the worst day.
-      if (putOptions?.serverSideEncryption !== undefined) {
-        throw new NotImplementedError({
-          cause:
-            'server-side encryption on the memory driver (it holds bytes in a Map) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule',
-          fix: 'put(key, body)',
+      async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
+        const safe = assertSafeKey(key);
+        // The same refusal both shipped disks make: a `put()` that succeeds in a test and throws in
+        // production is a gap an app meets on the worst day.
+        if (putOptions?.serverSideEncryption !== undefined) {
+          throw new NotImplementedError({
+            cause:
+              'server-side encryption on the memory driver (it holds bytes in a Map) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule',
+            fix: 'put(key, body)',
+          });
+        }
+        assertPutOptions(DRIVER_NAME, putOptions);
+        assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
+        const bytes = await toBytes(body, {
+          driver: DRIVER_NAME,
+          key: safe,
+          maxBytes: maxPutBytes,
         });
-      }
-      assertPutOptions(DRIVER_NAME, putOptions);
-      assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
-      const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
-      const claimed = putOptions?.checksum;
-      if (claimed !== undefined) {
-        const actual = sha256Base64(bytes);
-        if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
-      }
-      const object: StorageObject = {
-        key: safe,
-        size: bytes.byteLength,
-        contentType: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE,
-        etag: etagOf(bytes),
-        lastModified: clock.now(),
-        ...(putOptions?.cacheControl === undefined
-          ? {}
-          : { cacheControl: putOptions.cacheControl }),
-        ...(putOptions?.metadata === undefined ? {} : { metadata: putOptions.metadata }),
-      };
-      // Checked AFTER the body is read, with no await between check and write: the map is the
-      // queue the local disk needs a `keyedQueue` for.
-      refuseConflict(safe);
-      refuseOverwrite(safe, 'overwrite');
-      // A copy in, as a file write is: the caller's buffer and metadata are the caller's to reuse.
-      stored.set(safe, {
-        bytes: bytes.slice(),
-        object: snapshot(object),
-        lock: requestedLock(putOptions),
-      });
-      return snapshot(object);
-    },
+        const claimed = putOptions?.checksum;
+        if (claimed !== undefined) {
+          const actual = sha256Base64(bytes);
+          if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
+        }
+        return commit(safe, bytes, putOptions);
+      },
 
-    async stat(key: string): Promise<StorageObject | undefined> {
-      const hit = stored.get(assertSafeKey(key));
-      return hit && snapshot(hit.object);
-    },
+      /** A `Map` holds bytes, so "unbuffered" has no meaning here. The contract is the rest of it. */
+      async putStream(
+        key: string,
+        body: ReadableStream<Uint8Array>,
+        streamOptions: PutStreamOptions,
+      ): Promise<StorageObject> {
+        const safe = assertSafeKey(key);
+        const maxBytes = assertPutStream(DRIVER_NAME, body, streamOptions);
+        refuseOverwrite(safe, 'overwrite');
+        const chunks: Uint8Array[] = [];
+        const limit = streamLimit(DRIVER_NAME, safe, maxBytes);
+        await pumpBounded(body, limit, (chunk) => {
+          chunks.push(chunk);
+        }).catch((error: unknown) => {
+          if (isUltimateError(error)) throw error;
+          throw putFailed(
+            registered,
+            safe,
+            error,
+            'await disk.putStream(key, stream, { maxBytes })   # with a fresh stream',
+          );
+        });
+        return commit(safe, new Uint8Array(Bun.concatArrayBuffers(chunks)), streamOptions);
+      },
 
-    async get(key: string, getOptions?: GetOptions): Promise<StorageRead> {
-      const safe = assertSafeKey(key);
-      assertGetOptions(DRIVER_NAME, getOptions);
-      const hit = found(safe);
-      const range = getOptions?.range;
-      if (range !== undefined) {
-        const served = servedRange(DRIVER_NAME, safe, range, hit.bytes.byteLength, maxGetBytes);
-        const bytes = hit.bytes.slice(served[0], served[1] + 1);
-        return { object: snapshot(hit.object), bytes, range: served };
-      }
-      // The local disk's refusal, so a suite on this disk meets the ceiling production has.
-      if (hit.bytes.byteLength > maxGetBytes) {
-        throw getTooLarge(DRIVER_NAME, safe, hit.bytes.byteLength, maxGetBytes);
-      }
-      return { object: snapshot(hit.object), bytes: hit.bytes.slice() };
-    },
+      async stat(key: string): Promise<StorageObject | undefined> {
+        const hit = stored.get(assertSafeKey(key));
+        return hit && snapshot(hit.object);
+      },
 
-    async stream(key: string): Promise<ReadableStream<Uint8Array>> {
-      const hit = found(assertSafeKey(key));
-      return new Blob([hit.bytes.slice()]).stream();
-    },
+      async get(key: string, getOptions?: GetOptions): Promise<StorageRead> {
+        const safe = assertSafeKey(key);
+        assertGetOptions(DRIVER_NAME, getOptions);
+        const hit = found(safe);
+        const range = getOptions?.range;
+        if (range !== undefined) {
+          const served = servedRange(DRIVER_NAME, safe, range, hit.bytes.byteLength, maxGetBytes);
+          const bytes = hit.bytes.slice(served[0], served[1] + 1);
+          return { object: snapshot(hit.object), bytes, range: served };
+        }
+        // The local disk's refusal, so a suite on this disk meets the ceiling production has.
+        if (hit.bytes.byteLength > maxGetBytes) {
+          throw getTooLarge(DRIVER_NAME, safe, hit.bytes.byteLength, maxGetBytes);
+        }
+        return { object: snapshot(hit.object), bytes: hit.bytes.slice() };
+      },
 
-    async copy(from: string, to: string): Promise<StorageObject> {
-      // Both keys before either lookup, as the other two disks do: an unsafe destination is the
-      // refusal whether or not the source exists.
-      const [from_, destination] = [assertSafeKey(from), assertSafeKey(to)];
-      const source = found(from_);
-      refuseConflict(destination);
-      refuseOverwrite(destination, 'copy onto');
-      const object: StorageObject = {
-        ...source.object,
-        key: destination,
-        lastModified: clock.now(),
-      };
-      stored.set(destination, { bytes: source.bytes.slice(), object: snapshot(object) });
-      return snapshot(object);
-    },
+      async stream(key: string): Promise<ReadableStream<Uint8Array>> {
+        const hit = found(assertSafeKey(key));
+        return new Blob([hit.bytes.slice()]).stream();
+      },
 
-    async delete(key: string): Promise<void> {
-      const safe = assertSafeKey(key);
-      const lock = lockAt(safe);
-      if (lock !== undefined) {
-        throw new ObjectLockedError({ disk: registered, key: safe, action: 'delete', lock });
-      }
-      stored.delete(safe);
-    },
+      async streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>> {
+        const safe = assertSafeKey(key);
+        assertStreamRange(DRIVER_NAME, range);
+        const hit = found(safe);
+        const [first, last] = streamedRange(DRIVER_NAME, safe, range, hit.bytes.byteLength);
+        return new Blob([hit.bytes.slice(first, last + 1)]).stream();
+      },
 
-    async retentionOf(key: string): Promise<ObjectLock> {
-      const lock = found(assertSafeKey(key)).lock;
-      return lock === undefined ? { legalHold: false } : structuredClone(lock);
-    },
+      async copy(from: string, to: string, options?: CopyOptions): Promise<StorageObject> {
+        // Both keys before either lookup, as the other two disks do: an unsafe destination is the
+        // refusal whether or not the source exists.
+        const [from_, destination] = [assertSafeKey(from), assertSafeKey(to)];
+        const source = found(from_);
+        if (options?.ifMatch !== undefined && source.object.etag !== options.ifMatch) {
+          throw objectChanged(from_, options.ifMatch, source.object.etag);
+        }
+        refuseConflict(destination);
+        refuseOverwrite(destination, 'copy onto');
+        const object: StorageObject = {
+          ...source.object,
+          key: destination,
+          lastModified: clock.now(),
+        };
+        stored.set(destination, { bytes: source.bytes.slice(), object: snapshot(object) });
+        return snapshot(object);
+      },
 
-    async exists(key: string): Promise<boolean> {
-      return stored.has(assertSafeKey(key));
-    },
+      async delete(key: string): Promise<void> {
+        const safe = assertSafeKey(key);
+        const lock = lockAt(safe);
+        if (lock !== undefined) {
+          throw new ObjectLockedError({ disk: registered, key: safe, action: 'delete', lock });
+        }
+        stored.delete(safe);
+      },
 
-    async list(listOptions?: ListOptions): Promise<Page<StorageListEntry>> {
-      const prefix = listOptions?.prefix ?? '';
-      assertListOptions(listOptions);
-      const limit = resolveListLimit(listOptions?.limit);
-      const cursor = listOptions?.cursor ?? undefined;
-      // Code-unit order and "the cursor is the last key of the page before" — the local disk's own.
-      const keys = [...stored.keys()]
-        .filter((key) => key.startsWith(prefix) && (cursor === undefined || key > cursor))
-        .sort();
-      const page = keys.slice(0, limit);
-      const objects = page.flatMap((key) => {
-        const hit = stored.get(key);
-        return hit === undefined ? [] : [snapshot(hit.object)];
-      });
-      return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
-    },
+      async retentionOf(key: string): Promise<ObjectLock> {
+        const lock = found(assertSafeKey(key)).lock;
+        return lock === undefined ? { legalHold: false } : structuredClone(lock);
+      },
 
-    async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
-      return buildSignedUrl({
-        secret,
-        key: assertSafeKey(key),
-        method: urlOptions?.method,
-        expiresInMs: urlOptions?.expiresInMs,
-        maxBytes: urlOptions?.maxBytes,
-        contentType: urlOptions?.contentType,
-        baseUrl,
-        clock,
-      });
-    },
+      async exists(key: string): Promise<boolean> {
+        return stored.has(assertSafeKey(key));
+      },
 
-    async verifySigned(input: {
-      readonly url: string;
-      readonly clock?: Clock | undefined;
-    }): Promise<SignedUrlVerification> {
-      return verifySignedUrl({ url: input.url, secret, baseUrl, clock: input.clock ?? clock });
+      async list(listOptions?: ListOptions): Promise<Page<StorageListEntry>> {
+        const prefix = listOptions?.prefix ?? '';
+        assertListOptions(listOptions);
+        const limit = resolveListLimit(listOptions?.limit);
+        const cursor = listOptions?.cursor ?? undefined;
+        // Code-unit order and "the cursor is the last key of the page before" — the local disk's own.
+        const keys = [...stored.keys()]
+          .filter((key) => key.startsWith(prefix) && (cursor === undefined || key > cursor))
+          .sort();
+        const page = keys.slice(0, limit);
+        const objects = page.flatMap((key) => {
+          const hit = stored.get(key);
+          return hit === undefined ? [] : [snapshot(hit.object)];
+        });
+        return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
+      },
+
+      async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {
+        return buildSignedUrl({
+          secret,
+          key: assertSafeKey(key),
+          method: urlOptions?.method,
+          expiresInMs: urlOptions?.expiresInMs,
+          maxBytes: signedByteBound(DRIVER_NAME, urlOptions),
+          contentType: urlOptions?.contentType,
+          part: urlOptions?.part,
+          baseUrl,
+          clock,
+        });
+      },
+
+      async verifySigned(input: {
+        readonly url: string;
+        readonly clock?: Clock | undefined;
+      }): Promise<SignedUrlVerification> {
+        return verifySignedUrl({ url: input.url, secret, baseUrl, clock: input.clock ?? clock });
+      },
     },
-  };
+    memoryPartStore(),
+    clock,
+  );
 }

package/src/driver-s3-client.ts

@@ -14,6 +14,8 @@ export interface S3FileLike {
   exists(): Promise<boolean>;
   delete(): Promise<void>;
   stream(): ReadableStream<Uint8Array>;
+  /** Bun's `S3File.slice`: `end` exclusive. Its `stream()` sends one `Range:`, measured on 1.4.4. */
+  slice(begin: number, end: number): { stream(): ReadableStream<Uint8Array> };
   stat(): Promise<S3StatLike>;
   presign(options: { method?: string; expiresIn?: number; type?: string }): string;
 }
@@ -92,3 +94,10 @@ export function buildClient(options: S3DriverOptions): S3ClientLike {
     ...clientTarget(options),
   });
 }
+
+/**
+ * A missing `LastModified` is ABSENT, never `new Date(0)`: epoch 0 is older than every window, so
+ * `sweepOrphans` deleted an upload on the strength of a field the provider never sent.
+ */
+export const dated = (value: string | Date | undefined): { readonly lastModified?: Date } =>
+  value === undefined ? {} : { lastModified: value instanceof Date ? value : new Date(value) };

package/src/driver-s3-copy.ts

@@ -0,0 +1,35 @@
+// Single responsibility: the s3 copy that names a revision. `Bun.S3Client`'s `write` takes no
+// condition, and `x-amz-copy-source-if-match` is what lets the provider refuse an overwrite in the
+// one moment this process cannot see, so a revision-bound copy is a signed CopyObject.
+
+import { type S3Refusal, type S3WireOptions, sendSigned, xmlField } from './driver-s3-signed';
+
+/**
+ * The provider's refusal, or `undefined` when the copy landed. A 200 is not yet a verdict: S3 sends
+ * its headers before a long copy ends, so a failure can arrive as an `<Error>` inside a 200's body.
+ */
+export async function signedCopy(
+  options: S3WireOptions,
+  source: string,
+  destination: string,
+  ifMatch: string,
+): Promise<S3Refusal | undefined> {
+  const path = source.split('/').map(encodeURIComponent).join('/');
+  const result = await sendSigned(options, {
+    method: 'PUT',
+    key: destination,
+    headers: {
+      'x-amz-copy-source': `/${encodeURIComponent(options.bucket)}/${path}`,
+      'x-amz-copy-source-if-match': ifMatch,
+    },
+  });
+  if (!result.ok) return result.refusal;
+  const body = await result.response.text();
+  if (!body.includes('<Error>')) return undefined;
+  return {
+    name: 'S3Error',
+    code: xmlField(body, 'Code') ?? 'InternalError',
+    message: xmlField(body, 'Message') ?? 'the copy failed after it was accepted',
+    statusCode: 500,
+  };
+}

package/src/driver-s3-lock.ts

@@ -25,12 +25,19 @@ export function needsSignedPut(options: PutOptions | undefined): boolean {
 
 /** Every header the signed PUT sends beyond the signer's own, screened before a byte moves. */
 export function putHeaders(bytes: Uint8Array, options: PutOptions): Record<string, string> {
-  const headers: Record<string, string> = {
-    'content-type': options.contentType ?? DEFAULT_CONTENT_TYPE,
+  return {
+    ...objectHeaders(options),
     // Required by S3 on any PUT that sets a retention, and an integrity check on every other one:
     // a body that changed in flight is `BadDigest`, never a stored object.
     'content-md5': new Bun.CryptoHasher('md5').update(bytes).digest('base64'),
   };
+}
+
+/** What describes the object rather than one body: a PUT's headers, and a multipart Create's. */
+export function objectHeaders(options: PutOptions): Record<string, string> {
+  const headers: Record<string, string> = {
+    'content-type': options.contentType ?? DEFAULT_CONTENT_TYPE,
+  };
   if (options.cacheControl !== undefined) {
     assert(
       HEADER_VALUE.test(options.cacheControl),

package/src/driver-s3-multipart-session.ts

@@ -0,0 +1,87 @@
+// Single responsibility: the s3 disk's `MultipartCapability` — the browser's parts go straight to
+// the bucket on URLs core's signer presigns, and every other step is one signed call. No bytes
+// pass through this process, so the web role never carries them.
+
+import type { StorageDriver, StorageObject } from './driver';
+import {
+  abortS3Upload,
+  completeS3Upload,
+  createS3Upload,
+  listS3Parts,
+  listS3Uploads,
+} from './driver-s3-multipart';
+import type { S3Refusal, S3WireOptions } from './driver-s3-signed';
+import { objectNotFound } from './errors';
+import { partsInvalid, uploadUnknown } from './errors-multipart';
+import type { MultipartCapability } from './multipart';
+import { assertSafeKey } from './path';
+
+export interface S3MultipartInput {
+  readonly wire: S3WireOptions;
+  /** The disk: it presigns the part URLs and stats the completed object. */
+  readonly driver: StorageDriver;
+  /** The disk's coded write refusal for a key. */
+  readonly refusedWrite: (key: string) => (refusal: unknown) => never;
+}
+
+/** S3's answer for an id that is not an open upload: never opened, completed or aborted. */
+const isUnknownUpload = (refusal: S3Refusal): boolean => refusal.code === 'NoSuchUpload';
+/** S3's refusals of a Complete whose parts do not match what it holds. */
+const PART_REFUSALS: ReadonlySet<string> = new Set([
+  'InvalidPart',
+  'InvalidPartOrder',
+  'EntityTooSmall',
+]);
+
+export function s3Multipart(input: S3MultipartInput): MultipartCapability {
+  const { wire, driver } = input;
+  const target = (key: string, uploadId?: string) => ({
+    wire,
+    key,
+    refused: (refusal: S3Refusal): never => {
+      if (uploadId !== undefined && isUnknownUpload(refusal)) {
+        throw uploadUnknown(driver.name, key, uploadId);
+      }
+      if (uploadId !== undefined && PART_REFUSALS.has(refusal.code)) {
+        throw partsInvalid(key, uploadId, refusal.message);
+      }
+      return input.refusedWrite(key)(refusal);
+    },
+  });
+
+  return {
+    create: (key, { contentType }) =>
+      createS3Upload(target(assertSafeKey(key)), { 'content-type': contentType }),
+
+    signPart: (key, uploadId, partNumber, { contentLength, expiresInMs }) =>
+      driver.signedUrl(key, {
+        method: 'PUT',
+        contentLength,
+        expiresInMs,
+        part: { uploadId, partNumber },
+      }),
+
+    listParts: (key, uploadId) => listS3Parts(target(assertSafeKey(key), uploadId), uploadId),
+
+    async complete(key, uploadId, parts): Promise<StorageObject> {
+      const safe = assertSafeKey(key);
+      await completeS3Upload(target(safe, uploadId), uploadId, parts);
+      const object = await driver.stat(safe);
+      if (object === undefined) throw objectNotFound(driver.name, safe);
+      return object;
+    },
+
+    async abort(key, uploadId): Promise<void> {
+      const safe = assertSafeKey(key);
+      const refusal = await abortS3Upload(target(safe), uploadId);
+      // Gone already is the state asked for; anything else is a refusal the caller must hear.
+      if (refusal !== undefined && !isUnknownUpload(refusal)) input.refusedWrite(safe)(refusal);
+    },
+
+    listUploads: (prefix) =>
+      listS3Uploads(
+        { wire, refused: (refusal) => input.refusedWrite(prefix ?? '')(refusal) },
+        prefix,
+      ),
+  };
+}