@ultimat3/storage 23.0.0 → 25.0.0

package/src/driver-s3.ts

@@ -4,11 +4,22 @@
 // The client is built lazily on first use so importing this module never opens a socket, and
 // credentials arrive as env var NAMES: a literal key in app.config.ts is a key in git.
 
-import { ConfigInvalidError, EnvMissingError, finiteCount, stringField } from '@ultimat3/core';
 import {
+  type Clock,
+  finiteCount,
+  isUltimateError,
+  NotImplementedError,
+  type Page,
+  pageOf,
+  renderFixLiteral,
+  renderFixShellArg,
+  systemClock,
+} from '@ultimat3/core';
+import {
+  assertListOptions,
+  assertPutOptions,
   DEFAULT_CONTENT_TYPE,
   type ListOptions,
-  type ListPage,
   type PutOptions,
   resolveListLimit,
   type SignedUrlOptions,
@@ -20,61 +31,28 @@ import {
   sha256Base64,
   toBytes,
 } from './driver';
+import { isAbsentObject } from './driver-s3-absent';
+import { buildClient, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
+import { needsSignedPut, readObjectLock, signedPut } from './driver-s3-lock';
 import { regionMismatch } from './driver-s3-region';
+import type { S3FetchLike } from './driver-s3-signed';
 import {
   checksumMismatch,
   deleteFailed,
+  getTooLarge,
+  isStorageError,
   listFailed,
   objectNotFound,
-  storageNotImplemented,
+  putFailed,
+  readFailed,
 } from './errors';
+import { assertObjectLockOptions, type ObjectLock } from './object-lock';
 import { assertSafeKey } from './path';
 import { DEFAULT_SIGNED_URL_TTL_MS } from './signed-url';
 import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
 
 const DRIVER_NAME = 's3';
 
-/** Structural view of `Bun.S3Client` — typing it here keeps `bun-types` out of the contract. */
-export interface S3FileLike {
-  /** `S3FileLike` is in the union because Bun's own `write` takes an `S3File` — that is `copy`. */
-  write(data: Uint8Array | Blob | S3FileLike, options?: { type?: string }): Promise<number>;
-  arrayBuffer(): Promise<ArrayBuffer>;
-  exists(): Promise<boolean>;
-  delete(): Promise<void>;
-  stream(): ReadableStream<Uint8Array>;
-  stat(): Promise<S3StatLike>;
-  presign(options: { method?: string; expiresIn?: number; type?: string }): string;
-}
-
-export interface S3StatLike {
-  readonly size: number;
-  readonly type?: string | undefined;
-  readonly etag?: string | undefined;
-  readonly lastModified?: string | Date | undefined;
-}
-
-export interface S3ListEntryLike {
-  readonly key?: string | undefined;
-  readonly size?: number | undefined;
-  readonly eTag?: string | undefined;
-  readonly lastModified?: string | Date | undefined;
-}
-
-export interface S3ListResultLike {
-  readonly contents?: readonly S3ListEntryLike[] | undefined;
-  readonly isTruncated?: boolean | undefined;
-  readonly nextContinuationToken?: string | undefined;
-}
-
-export interface S3ClientLike {
-  file(key: string): S3FileLike;
-  list(input: {
-    prefix?: string;
-    maxKeys?: number;
-    continuationToken?: string;
-  }): Promise<S3ListResultLike>;
-}
-
 export interface S3DriverOptions {
   readonly bucket: string;
   /**
@@ -95,6 +73,15 @@ export interface S3DriverOptions {
   readonly env?: Readonly<Record<string, string | undefined>> | undefined;
   /** Injected in tests; production constructs `Bun.S3Client`. */
   readonly client?: S3ClientLike | undefined;
+  /**
+   * Injected in tests; production uses the global `fetch`. Carries the SIGNED requests the Bun
+   * client has no option for — a `put()` with `metadata`, `cacheControl`, `retention` or
+   * `legalHold`, and `retentionOf()` — signed by core's `signAwsRequest` against the same
+   * endpoint, region and credentials.
+   */
+  readonly fetch?: S3FetchLike | undefined;
+  /** Signs the signed requests and screens a `retainUntil`. Default `systemClock`. */
+  readonly clock?: Clock | undefined;
   /**
    * Ceiling on ONE server-side `put()`, because `put()` buffers the whole body. Defaults to the
    * upload policy's ceiling — the same number for the same fact. Raise it for a disk that really
@@ -102,91 +89,20 @@ export interface S3DriverOptions {
    * and S3's single-PUT limit is 5GB regardless of what this says.
    */
   readonly maxPutBytes?: number | undefined;
+  /**
+   * Ceiling on ONE `get()`, because `get()` buffers the whole object — and on this disk nothing
+   * bounds what a presigned PUT stored (`SignedUrlOptions.maxBytes`). Defaults to `maxPutBytes`;
+   * an object past it is read with `stream()`.
+   */
+  readonly maxGetBytes?: number | undefined;
 }
 
-interface S3ClientConstructor {
-  new (options: Record<string, unknown>): S3ClientLike;
-}
-
-function requireEnv(
-  env: Readonly<Record<string, string | undefined>>,
-  name: string,
-  partner: string,
-): string {
-  const value = env[name];
-  if (value === undefined || value === '') {
-    throw new EnvMissingError({
-      cause: `${name} is not set, so the s3 disk cannot authenticate`,
-      fix: `set ${name} and ${partner} in .env (or the container's secret store), then re-run`,
-      meta: { missing: name },
-    });
-  }
-  return value;
-}
-
-function buildClient(options: S3DriverOptions): S3ClientLike {
-  if (options.client !== undefined) return options.client;
-  if (options.bucket === '') {
-    throw new ConfigInvalidError({
-      cause: 's3 disk was defined without a bucket',
-      fix: 'set storage.disks.<name>.bucket in app.config.ts',
-    });
-  }
-  const env = options.env ?? process.env;
-  const idVar = options.accessKeyIdEnv ?? 'S3_ACCESS_KEY_ID';
-  const secretVar = options.secretAccessKeyEnv ?? 'S3_SECRET_ACCESS_KEY';
-  const Client = (Bun as unknown as { S3Client?: S3ClientConstructor }).S3Client;
-  if (Client === undefined) {
-    throw new ConfigInvalidError({
-      cause: 'Bun.S3Client is unavailable in this runtime',
-      fix: 'upgrade the runtime: bun upgrade   # the s3 disk needs bun >= 1.3',
-    });
-  }
-  const tokenVar = options.sessionTokenEnv;
-  const sessionToken = tokenVar === undefined ? undefined : env[tokenVar];
-  // Bun's flag is the inverse: path style means "not virtual hosted".
-  const pathStyle = options.forcePathStyle;
-  return new Client({
-    bucket: options.bucket,
-    accessKeyId: requireEnv(env, idVar, secretVar),
-    secretAccessKey: requireEnv(env, secretVar, idVar),
-    ...(options.region === undefined ? {} : { region: options.region }),
-    ...(options.endpoint === undefined ? {} : { endpoint: options.endpoint }),
-    ...(pathStyle === undefined ? {} : { virtualHostedStyle: !pathStyle }),
-    ...(sessionToken === undefined ? {} : { sessionToken }),
-  });
-}
-
-const toDate = (value: string | Date | undefined): Date =>
-  value === undefined ? new Date(0) : value instanceof Date ? value : new Date(value);
-
-/** One numeric field off a value that may fight being read — `stringField`'s missing twin. */
-function numberField(value: unknown, key: string): number | undefined {
-  if (typeof value !== 'object' || value === null) return undefined;
-  try {
-    const held = (value as Record<string, unknown>)[key];
-    return typeof held === 'number' ? held : undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-/** The provider codes that mean "there is nothing at that key", and nothing wider. */
-const ABSENT_OBJECT_CODES: ReadonlySet<string> = new Set(['NoSuchKey', 'NotFound', 'ENOENT']);
-
 /**
- * The ONE delete failure the contract calls success. `AccessDenied`, `SlowDown`, an expired
- * credential and a reset connection are none of them, and the previous `.catch(() => undefined)`
- * reported all four as deleted — which is how an erasure sweep certifies data it never removed.
- *
- * Read structurally: an `S3Error` is `name: 'S3Error'` with a `code` the service returned, and
- * every field of a value this process did not build is a getter that can throw.
+ * A missing `LastModified` is ABSENT, never `new Date(0)`: epoch 0 is older than every window, so
+ * `sweepOrphans` deleted an upload on the strength of a field the provider never sent.
  */
-function isAbsentObject(error: unknown): boolean {
-  const code = stringField(error, 'code');
-  if (code !== undefined && ABSENT_OBJECT_CODES.has(code)) return true;
-  return numberField(error, 'statusCode') === 404 || numberField(error, 'status') === 404;
-}
+const dated = (value: string | Date | undefined): { readonly lastModified?: Date } =>
+  value === undefined ? {} : { lastModified: value instanceof Date ? value : new Date(value) };
 
 /**
  * Everything `put()` may be handed that this driver cannot honour, refused before a byte moves.
@@ -194,25 +110,22 @@ function isAbsentObject(error: unknown): boolean {
  * `PutOptions` so that "this disk cannot prove per-object encryption" is something an engineer
  * meets while writing the call, not while answering a security review.
  */
-function refuseUnsupportedPut(bucket: string, key: string, putOptions?: PutOptions): void {
-  if (putOptions?.metadata !== undefined || putOptions?.cacheControl !== undefined) {
-    const uri = `s3://${bucket}/${key}`;
-    throw storageNotImplemented(
-      'user metadata and cache-control on the s3 driver (Bun exposes no header hook yet)',
-      `drop metadata/cacheControl from put(), or set them out of band: ` +
-        `aws s3 cp ${uri} ${uri} --metadata-directive REPLACE`,
-    );
-  }
+function refuseUnsupportedPut(bucket: string, putOptions?: PutOptions): void {
   const sse = putOptions?.serverSideEncryption;
   if (sse === undefined) return;
-  const rule =
-    sse.algorithm === 'aws:kms'
-      ? `{"SSEAlgorithm":"aws:kms","KMSMasterKeyID":"${sse.kmsKeyId ?? '<key-arn>'}"}`
-      : '{"SSEAlgorithm":"AES256"}';
-  throw storageNotImplemented(
-    'per-object server-side encryption on the s3 driver (Bun.S3Client exposes acl, storageClass and type, and nothing for x-amz-server-side-encryption)',
-    `set it bucket-wide, then drop serverSideEncryption from put(): aws s3api put-bucket-encryption --bucket ${bucket} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":${rule},"BucketKeyEnabled":true}]}'`,
-  );
+  // The key id is the caller's own option, spliced into single-quoted JSON: `renderFixLiteral`
+  // escapes the `"` and `\` JSON needs, and a `'` — the one byte live inside single quotes — makes
+  // it the placeholder rather than ending the argument (plan 101 row S12).
+  const keyId = sse.kmsKeyId?.includes("'") === false ? sse.kmsKeyId : undefined;
+  const bucketArg = renderFixShellArg(bucket, "'<bucket>'");
+  throw new NotImplementedError({
+    cause:
+      'per-object server-side encryption on the s3 driver (Bun.S3Client exposes acl, storageClass and type, and nothing for x-amz-server-side-encryption) is not implemented by this driver — set it bucket-wide with the command in fix, then drop serverSideEncryption from put()',
+    fix:
+      sse.algorithm === 'aws:kms'
+        ? `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"aws:kms","KMSMasterKeyID":${renderFixLiteral(keyId, '"<key-arn>"')}},"BucketKeyEnabled":true}]}'`
+        : `aws s3api put-bucket-encryption --bucket ${bucketArg} --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}]}'`,
+  });
 }
 
 export function s3Driver(options: S3DriverOptions): StorageDriver {
@@ -224,6 +137,16 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
     options.maxPutBytes === undefined ? DEFAULT_MAX_UPLOAD_BYTES : options.maxPutBytes,
     1,
   );
+  const maxGetBytes = finiteCount(
+    'the s3 driver',
+    'maxGetBytes',
+    options.maxGetBytes === undefined ? maxPutBytes : options.maxGetBytes,
+    1,
+  );
+  // The key `defineStorage` registered this disk under: a not-found `fix:` is `disk('<it>')…`,
+  // and the driver kind there is `X_STORAGE_DISK_UNKNOWN` on any app that named its disk.
+  let diskName = DRIVER_NAME;
+  const clock = options.clock ?? systemClock;
   let client: S3ClientLike | undefined;
   const conn = (): S3ClientLike => {
     client ??= buildClient(options);
@@ -240,28 +163,68 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
     const mismatch = regionMismatch(options.bucket, error);
     if (mismatch !== undefined) throw mismatch;
   };
-  const misconfigured = (error: unknown): never => {
-    throwIfMisconfigured(error);
-    throw error;
-  };
+  /**
+   * A write the provider refused, coded: a denied `s3:PutObject`, a throttle, an expired
+   * credential. The bare `S3Error` used to escape `put()` and `copy()` exactly as it once escaped
+   * `list()` — no code, no fix, an anonymous 500.
+   */
+  const refusedWrite =
+    (key: string) =>
+    (error: unknown): never => {
+      // A refusal this process already coded — a screened header, a missing credential — is its
+      // own answer, never re-read as the provider refusing the write.
+      if (isUltimateError(error)) throw error;
+      throwIfMisconfigured(error);
+      throw putFailed(
+        DRIVER_NAME,
+        key,
+        error,
+        `aws s3api get-bucket-policy --bucket ${renderFixShellArg(options.bucket, "'<bucket>'")}`,
+      );
+    };
+
+  /**
+   * A read the provider refused, coded — `refusedWrite`'s twin. A 404 is "not found" wherever it
+   * arrives: an object deleted between `exists()` and the read is absent, not unreadable.
+   */
+  const refusedRead =
+    (key: string) =>
+    (error: unknown): never => {
+      if (isStorageError(error)) throw error;
+      if (isAbsentObject(error)) throw objectNotFound(diskName, key);
+      throwIfMisconfigured(error);
+      throw readFailed(
+        DRIVER_NAME,
+        key,
+        error,
+        `aws s3api head-object --bucket ${renderFixShellArg(options.bucket, "'<bucket>'")} --key ${renderFixShellArg(key, "'<key>'")}`,
+      );
+    };
+  const present = (key: string): Promise<boolean> =>
+    conn().file(key).exists().catch(refusedRead(key));
 
   const statObject = async (key: string): Promise<StorageObject> => {
-    const stat = await conn().file(key).stat();
+    const stat = await conn().file(key).stat().catch(refusedRead(key));
     return {
       key,
       size: stat.size,
       contentType: stat.type ?? DEFAULT_CONTENT_TYPE,
       etag: stat.etag ?? '',
-      lastModified: toDate(stat.lastModified),
+      ...dated(stat.lastModified),
     };
   };
 
   return {
     name: DRIVER_NAME,
+    registerAs(name: string): void {
+      diskName = name;
+    },
 
     async put(key: string, body: StorageBody, putOptions?: PutOptions): Promise<StorageObject> {
       const safe = assertSafeKey(key);
-      refuseUnsupportedPut(options.bucket, safe, putOptions);
+      refuseUnsupportedPut(options.bucket, putOptions);
+      assertPutOptions(DRIVER_NAME, putOptions);
+      assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
       // Buffered on purpose: size and checksum must be known before the object exists — so this
       // path is for objects that FIT IN MEMORY, and `maxPutBytes` is what makes that a contract
       // rather than a hope. User uploads never come through here: they go direct to the bucket
@@ -272,25 +235,49 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
         const actual = sha256Base64(bytes);
         if (claimed !== actual) throw checksumMismatch(safe, claimed, actual);
       }
+      if (putOptions !== undefined && needsSignedPut(putOptions)) {
+        // Off the Bun client only for what it cannot send: `S3File.write` takes a type, an ACL and
+        // a storage class and no header. One request either way — never a PUT then a second call.
+        const refusal = await signedPut(options, safe, bytes, putOptions).catch(refusedWrite(safe));
+        if (refusal !== undefined) refusedWrite(safe)(refusal);
+        // Stored — the PUT succeeded with them — so reported, as the local disk reports its own.
+        return {
+          ...(await statObject(safe)),
+          ...(putOptions.cacheControl === undefined
+            ? {}
+            : { cacheControl: putOptions.cacheControl }),
+          ...(putOptions.metadata === undefined ? {} : { metadata: { ...putOptions.metadata } }),
+        };
+      }
       await conn()
         .file(safe)
         .write(bytes, { type: putOptions?.contentType ?? DEFAULT_CONTENT_TYPE })
-        .catch(misconfigured);
+        .catch(refusedWrite(safe));
       return statObject(safe);
     },
 
+    async stat(key: string): Promise<StorageObject | undefined> {
+      const safe = assertSafeKey(key);
+      return (await present(safe)) ? statObject(safe) : undefined;
+    },
+
     async get(key: string): Promise<StorageRead> {
       const safe = assertSafeKey(key);
       const file = conn().file(safe);
-      if (!(await file.exists())) throw objectNotFound(DRIVER_NAME, safe);
-      const bytes = new Uint8Array(await file.arrayBuffer());
-      return { object: await statObject(safe), bytes };
+      if (!(await present(safe))) throw objectNotFound(diskName, safe);
+      // The HEAD comes FIRST, and it is the whole point: a client PUT straight into the bucket is
+      // bounded by nothing on this disk, so `arrayBuffer()` on whatever is there was heap growth
+      // the uploader chose. Refused on the provider's own size before a byte is read.
+      const object = await statObject(safe);
+      if (object.size > maxGetBytes) throw getTooLarge(DRIVER_NAME, safe, object.size, maxGetBytes);
+      const bytes = new Uint8Array(await file.arrayBuffer().catch(refusedRead(safe)));
+      return { object, bytes };
     },
 
     async stream(key: string): Promise<ReadableStream<Uint8Array>> {
       const safe = assertSafeKey(key);
       const file = conn().file(safe);
-      if (!(await file.exists())) throw objectNotFound(DRIVER_NAME, safe);
+      if (!(await present(safe))) throw objectNotFound(diskName, safe);
       return file.stream();
     },
 
@@ -305,12 +292,12 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
       const source = assertSafeKey(from);
       const destination = assertSafeKey(to);
       const file = conn().file(source);
-      if (!(await file.exists())) throw objectNotFound(DRIVER_NAME, source);
-      const stat = await file.stat();
+      if (!(await present(source))) throw objectNotFound(diskName, source);
+      const stat = await file.stat().catch(refusedRead(source));
       await conn()
         .file(destination)
         .write(file, { type: stat.type ?? DEFAULT_CONTENT_TYPE })
-        .catch(misconfigured);
+        .catch(refusedWrite(destination));
       return statObject(destination);
     },
 
@@ -327,19 +314,36 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
           DRIVER_NAME,
           safe,
           error,
-          `grant s3:DeleteObject on this prefix to the app's role, then reproduce with the provider's own words: aws s3api delete-object --bucket ${options.bucket} --key ${safe}`,
+          // `isSafeKey` admits `$(…)`, a backtick and `;` — legal key bytes, live shell syntax.
+          `grant s3:DeleteObject on this prefix to the app's role, then reproduce with the provider's own words: aws s3api delete-object --bucket ${renderFixShellArg(options.bucket, "'<bucket>'")} --key ${renderFixShellArg(safe, "'<key>'")}`,
         );
       }
     },
 
     async exists(key: string): Promise<boolean> {
-      return conn().file(assertSafeKey(key)).exists();
+      return present(assertSafeKey(key));
+    },
+
+    /** Signed `GET ?retention` and `?legal-hold`, in parallel. A bucket with no lock reports none. */
+    async retentionOf(key: string): Promise<ObjectLock> {
+      const safe = assertSafeKey(key);
+      const read = await readObjectLock(options, safe).catch(refusedRead(safe));
+      if (read.kind === 'lock') return read.lock;
+      if (isAbsentObject(read.refusal)) throw objectNotFound(diskName, safe);
+      throwIfMisconfigured(read.refusal);
+      throw readFailed(
+        DRIVER_NAME,
+        safe,
+        read.refusal,
+        `aws s3api get-object-retention --bucket ${renderFixShellArg(options.bucket, "'<bucket>'")} --key ${renderFixShellArg(safe, "'<key>'")}   # reproduces the refusal; grant s3:GetObjectRetention and s3:GetObjectLegalHold to the app's role`,
+      );
     },
 
-    async list(listOptions?: ListOptions): Promise<ListPage> {
+    async list(listOptions?: ListOptions): Promise<Page<StorageListEntry>> {
       const prefix = listOptions?.prefix ?? '';
       // Refused at the seam both drivers share, before the provider is asked: `maxKeys: 0` used to
       // go straight through, while the local disk answered a complete-looking empty page.
+      assertListOptions(listOptions);
       const maxKeys = resolveListLimit(listOptions?.limit);
       // `conn()` OUTSIDE the try: a missing credential or an absent `Bun.S3Client` is this disk
       // misconfigured, and it already answers with its own code and its own fix.
@@ -349,7 +353,9 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
         result = await client.list({
           maxKeys,
           ...(listOptions?.prefix === undefined ? {} : { prefix: listOptions.prefix }),
-          ...(listOptions?.cursor === undefined ? {} : { continuationToken: listOptions.cursor }),
+          ...(typeof listOptions?.cursor === 'string'
+            ? { continuationToken: listOptions.cursor }
+            : {}),
         });
       } catch (error) {
         // A bare `S3Error` used to escape here, uncoded: no `X_*`, no `fix`, no `--json` shape, and
@@ -360,7 +366,11 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
           DRIVER_NAME,
           prefix,
           error,
-          `grant s3:ListBucket on this bucket to the app's role, then reproduce with the provider's own words: aws s3api list-objects-v2 --bucket ${options.bucket}${prefix === '' ? '' : ` --prefix ${prefix}`}`,
+          // Two whole lines rather than a ternary INSIDE one: each substitution is then a screen
+          // the fix-shell-arg guard can read as one.
+          prefix === ''
+            ? `grant s3:ListBucket on this bucket to the app's role, then reproduce with the provider's own words: aws s3api list-objects-v2 --bucket ${renderFixShellArg(options.bucket, "'<bucket>'")}`
+            : `grant s3:ListBucket on this bucket to the app's role, then reproduce with the provider's own words: aws s3api list-objects-v2 --bucket ${renderFixShellArg(options.bucket, "'<bucket>'")} --prefix ${renderFixShellArg(prefix, "'<prefix>'")}`,
         );
       }
       const objects: StorageListEntry[] = [];
@@ -375,13 +385,16 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
           key: entry.key,
           size: entry.size ?? 0,
           etag: entry.eTag ?? '',
-          lastModified: toDate(entry.lastModified),
+          ...dated(entry.lastModified),
         });
       }
-      const cursor = result.nextContinuationToken;
-      return result.isTruncated === true && cursor !== undefined
-        ? { objects, truncated: true, cursor }
-        : { objects, truncated: false };
+      // `IsTruncated` decides and `NextContinuationToken` is the cursor: a token beside
+      // `IsTruncated: false` is not a next page, and a truncated answer with no token has none to
+      // offer — both are the last page, never a cursor that fetches an empty one.
+      return pageOf(
+        objects,
+        result.isTruncated === true ? (result.nextContinuationToken ?? null) : null,
+      );
     },
 
     /**