@ultimat3/storage 23.0.0 → 25.0.0

package/src/driver-memory.ts

@@ -1,38 +1,56 @@
 // Single responsibility: the disk a TEST holds — a `StorageDriver` over one `Map`, answering every
-// question `localDriver` answers and refusing what it refuses, with nothing on a filesystem to
-// create or clean up. Not a deployment's disk: a restart is every object gone.
+// question `localDriver` answers and refusing what it refuses (a key that is another key's path
+// included), with nothing on a filesystem to create or clean up. Not a deployment's disk: a
+// restart is every object gone.
 //
 // It exists because the alternative was written by hand in every suite that needed a bucket: a
 // fake with three methods and five `unsupported()` stubs, each a slightly different contract.
 
-import { type Clock, finiteCount, systemClock } from '@ultimat3/core';
 import {
+  type Clock,
+  finiteCount,
+  NotImplementedError,
+  type Page,
+  pageOf,
+  systemClock,
+} from '@ultimat3/core';
+import {
+  assertListOptions,
+  assertPutOptions,
   DEFAULT_CONTENT_TYPE,
   etagOf,
   type ListOptions,
-  type ListPage,
   type PutOptions,
   resolveListLimit,
   type SignedUrlOptions,
   type StorageBody,
   type StorageDriver,
+  type StorageListEntry,
   type StorageObject,
   type StorageRead,
   sha256Base64,
   toBytes,
 } from './driver';
 import type { LocalDriverOptions } from './driver-local';
-import { resolveSigningSecret } from './driver-local';
-import { checksumMismatch, objectNotFound, storageNotImplemented } from './errors';
+import { keyInTheWay } from './driver-memory-conflict';
+import { checksumMismatch, getTooLarge, keyConflict, objectNotFound } from './errors';
+import {
+  assertObjectLockOptions,
+  isLocked,
+  type ObjectLock,
+  ObjectLockedError,
+  requestedLock,
+} from './object-lock';
 import { assertSafeKey } from './path';
 import type { SignedUrlVerification } from './signed-url';
 import { buildSignedUrl, signedUrlBaseFor, verifySignedUrl } from './signed-url';
+import { resolveSigningSecret } from './signing-secret';
 import { DEFAULT_MAX_UPLOAD_BYTES } from './upload';
 
 const DRIVER_NAME = 'memory';
 
 /** `localDriver`'s options, minus the directory there is none of. */
-export type MemoryDriverOptions = Omit<LocalDriverOptions, 'root'>;
+export type MemoryStorageDriverOptions = Omit<LocalDriverOptions, 'root'>;
 
 export interface MemoryStorageDriver extends StorageDriver {
   /**
@@ -45,6 +63,8 @@ export interface MemoryStorageDriver extends StorageDriver {
 interface Stored {
   readonly bytes: Uint8Array;
   readonly object: StorageObject;
+  /** The local disk's sidecar lock, held here instead. */
+  readonly lock?: ObjectLock | undefined;
 }
 
 /**
@@ -54,25 +74,55 @@ interface Stored {
  */
 const snapshot = (object: StorageObject): StorageObject => ({
   ...object,
-  lastModified: new Date(object.lastModified.getTime()),
+  ...(object.lastModified === undefined
+    ? {}
+    : { lastModified: new Date(object.lastModified.getTime()) }),
   ...(object.metadata === undefined ? {} : { metadata: { ...object.metadata } }),
 });
 
-export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDriver {
+export function memoryStorageDriver(options: MemoryStorageDriverOptions = {}): MemoryStorageDriver {
   const maxPutBytes = finiteCount(
     'the memory disk driver',
     'maxPutBytes',
     options.maxPutBytes === undefined ? DEFAULT_MAX_UPLOAD_BYTES : options.maxPutBytes,
     1,
   );
+  const maxGetBytes = finiteCount(
+    'the memory disk driver',
+    'maxGetBytes',
+    options.maxGetBytes === undefined ? maxPutBytes : options.maxGetBytes,
+    1,
+  );
   const clock = options.clock ?? systemClock;
   let baseUrl = options.baseUrl ?? signedUrlBaseFor(DRIVER_NAME);
   const secret = resolveSigningSecret(DRIVER_NAME, options);
   const stored = new Map<string, Stored>();
+  // The name a registry gave this disk, so a miss names `disk('<it>')` — the call the author wrote.
+  let registered: string = DRIVER_NAME;
+
+  /** The lock that holds NOW, or `undefined` — the local disk's rule, read off the map. */
+  const lockAt = (key: string): ObjectLock | undefined => {
+    const lock = stored.get(key)?.lock;
+    return isLocked(lock, clock.now()) ? lock : undefined;
+  };
+  const refuseOverwrite = (key: string, action: 'overwrite' | 'copy onto'): void => {
+    const lock = lockAt(key);
+    if (lock !== undefined) throw new ObjectLockedError({ disk: registered, key, action, lock });
+  };
+
+  /**
+   * The local disk's path rule: a key cannot be another key's directory (`X_STORAGE_KEY_CONFLICT`).
+   * Emulated, not inherited — a `Map` holds both — because a suite on this disk stands in for the
+   * dev disk, and a key layout that passes here and fails in `x dev` is a gap found late.
+   */
+  const refuseConflict = (key: string): void => {
+    const blocking = keyInTheWay(stored, key);
+    if (blocking !== undefined) throw keyConflict(registered, key, blocking);
+  };
 
   const found = (key: string): Stored => {
     const hit = stored.get(key);
-    if (hit === undefined) throw objectNotFound(DRIVER_NAME, key);
+    if (hit === undefined) throw objectNotFound(registered, key);
     return hit;
   };
 
@@ -84,6 +134,7 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
     },
 
     registerAs(diskName: string): void {
+      registered = diskName;
       if (options.baseUrl === undefined) baseUrl = signedUrlBaseFor(diskName);
     },
 
@@ -96,11 +147,14 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
       // The same refusal both shipped disks make: a `put()` that succeeds in a test and throws in
       // production is a gap an app meets on the worst day.
       if (putOptions?.serverSideEncryption !== undefined) {
-        throw storageNotImplemented(
-          'server-side encryption on the memory driver (it holds bytes in a Map)',
-          'drop serverSideEncryption from put(), and encrypt the disk itself — an s3Driver over a bucket with a default KMS rule',
-        );
+        throw new NotImplementedError({
+          cause:
+            'server-side encryption on the memory driver (it holds bytes in a Map) is not implemented by this driver — drop serverSideEncryption from put() and encrypt the disk itself: an s3Driver over a bucket with a default KMS rule',
+          fix: 'put(key, body)',
+        });
       }
+      assertPutOptions(DRIVER_NAME, putOptions);
+      assertObjectLockOptions(DRIVER_NAME, putOptions, clock);
       const bytes = await toBytes(body, { driver: DRIVER_NAME, key: safe, maxBytes: maxPutBytes });
       const claimed = putOptions?.checksum;
       if (claimed !== undefined) {
@@ -118,13 +172,31 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
           : { cacheControl: putOptions.cacheControl }),
         ...(putOptions?.metadata === undefined ? {} : { metadata: putOptions.metadata }),
       };
+      // Checked AFTER the body is read, with no await between check and write: the map is the
+      // queue the local disk needs a `keyedQueue` for.
+      refuseConflict(safe);
+      refuseOverwrite(safe, 'overwrite');
       // A copy in, as a file write is: the caller's buffer and metadata are the caller's to reuse.
-      stored.set(safe, { bytes: bytes.slice(), object: snapshot(object) });
+      stored.set(safe, {
+        bytes: bytes.slice(),
+        object: snapshot(object),
+        lock: requestedLock(putOptions),
+      });
       return snapshot(object);
     },
 
+    async stat(key: string): Promise<StorageObject | undefined> {
+      const hit = stored.get(assertSafeKey(key));
+      return hit && snapshot(hit.object);
+    },
+
     async get(key: string): Promise<StorageRead> {
-      const hit = found(assertSafeKey(key));
+      const safe = assertSafeKey(key);
+      const hit = found(safe);
+      // The local disk's refusal, so a suite on this disk meets the ceiling production has.
+      if (hit.bytes.byteLength > maxGetBytes) {
+        throw getTooLarge(DRIVER_NAME, safe, hit.bytes.byteLength, maxGetBytes);
+      }
       return { object: snapshot(hit.object), bytes: hit.bytes.slice() };
     },
 
@@ -134,8 +206,12 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
     },
 
     async copy(from: string, to: string): Promise<StorageObject> {
-      const source = found(assertSafeKey(from));
-      const destination = assertSafeKey(to);
+      // Both keys before either lookup, as the other two disks do: an unsafe destination is the
+      // refusal whether or not the source exists.
+      const [from_, destination] = [assertSafeKey(from), assertSafeKey(to)];
+      const source = found(from_);
+      refuseConflict(destination);
+      refuseOverwrite(destination, 'copy onto');
       const object: StorageObject = {
         ...source.object,
         key: destination,
@@ -146,17 +222,28 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
     },
 
     async delete(key: string): Promise<void> {
-      stored.delete(assertSafeKey(key));
+      const safe = assertSafeKey(key);
+      const lock = lockAt(safe);
+      if (lock !== undefined) {
+        throw new ObjectLockedError({ disk: registered, key: safe, action: 'delete', lock });
+      }
+      stored.delete(safe);
+    },
+
+    async retentionOf(key: string): Promise<ObjectLock> {
+      const lock = found(assertSafeKey(key)).lock;
+      return lock === undefined ? { legalHold: false } : structuredClone(lock);
     },
 
     async exists(key: string): Promise<boolean> {
       return stored.has(assertSafeKey(key));
     },
 
-    async list(listOptions?: ListOptions): Promise<ListPage> {
+    async list(listOptions?: ListOptions): Promise<Page<StorageListEntry>> {
       const prefix = listOptions?.prefix ?? '';
+      assertListOptions(listOptions);
       const limit = resolveListLimit(listOptions?.limit);
-      const cursor = listOptions?.cursor;
+      const cursor = listOptions?.cursor ?? undefined;
       // Code-unit order and "the cursor is the last key of the page before" — the local disk's own.
       const keys = [...stored.keys()]
         .filter((key) => key.startsWith(prefix) && (cursor === undefined || key > cursor))
@@ -166,10 +253,7 @@ export function memoryDriver(options: MemoryDriverOptions = {}): MemoryStorageDr
         const hit = stored.get(key);
         return hit === undefined ? [] : [snapshot(hit.object)];
       });
-      const last = page.at(-1);
-      return keys.length > page.length && last !== undefined
-        ? { objects, truncated: true, cursor: last }
-        : { objects, truncated: false };
+      return pageOf(objects, keys.length > page.length ? (page.at(-1) ?? null) : null);
     },
 
     async signedUrl(key: string, urlOptions?: SignedUrlOptions): Promise<string> {

package/src/driver-s3-absent.ts

@@ -0,0 +1,32 @@
+// Single responsibility: does a provider failure mean "there is nothing at that key"? Split from
+// `driver-s3.ts` at its line ceiling; the one reader is that driver's delete and read paths.
+
+import { stringField } from '@ultimat3/core';
+
+/** One numeric field off a value that may fight being read — `stringField`'s missing twin. */
+function numberField(value: unknown, key: string): number | undefined {
+  if (typeof value !== 'object' || value === null) return undefined;
+  try {
+    const held = (value as Record<string, unknown>)[key];
+    return typeof held === 'number' ? held : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** The provider codes that mean "there is nothing at that key", and nothing wider. */
+const ABSENT_OBJECT_CODES: ReadonlySet<string> = new Set(['NoSuchKey', 'NotFound', 'ENOENT']);
+
+/**
+ * The ONE delete failure the contract calls success. `AccessDenied`, `SlowDown`, an expired
+ * credential and a reset connection are none of them, and the previous `.catch(() => undefined)`
+ * reported all four as deleted — which is how an erasure sweep certifies data it never removed.
+ *
+ * Read structurally: an `S3Error` is `name: 'S3Error'` with a `code` the service returned, and
+ * every field of a value this process did not build is a getter that can throw.
+ */
+export function isAbsentObject(error: unknown): boolean {
+  const code = stringField(error, 'code');
+  if (code !== undefined && ABSENT_OBJECT_CODES.has(code)) return true;
+  return numberField(error, 'statusCode') === 404 || numberField(error, 'status') === 404;
+}

package/src/driver-s3-client.ts

@@ -0,0 +1,94 @@
+// Single responsibility: the s3 disk's view of `Bun.S3Client` — the structural types its methods
+// are called through (so `bun-types` stays out of the public contract and a test injects a fake),
+// and the one place the real client is constructed, lazily, from the disk's options.
+
+import { ConfigInvalidError } from '@ultimat3/core';
+import type { S3DriverOptions } from './driver-s3';
+import { requireEnv, resolveS3Target } from './driver-s3-signed';
+
+/** Structural view of `Bun.S3Client` — typing it here keeps `bun-types` out of the contract. */
+export interface S3FileLike {
+  /** `S3FileLike` is in the union because Bun's own `write` takes an `S3File` — that is `copy`. */
+  write(data: Uint8Array | Blob | S3FileLike, options?: { type?: string }): Promise<number>;
+  arrayBuffer(): Promise<ArrayBuffer>;
+  exists(): Promise<boolean>;
+  delete(): Promise<void>;
+  stream(): ReadableStream<Uint8Array>;
+  stat(): Promise<S3StatLike>;
+  presign(options: { method?: string; expiresIn?: number; type?: string }): string;
+}
+
+export interface S3StatLike {
+  readonly size: number;
+  readonly type?: string | undefined;
+  readonly etag?: string | undefined;
+  readonly lastModified?: string | Date | undefined;
+}
+
+export interface S3ListEntryLike {
+  readonly key?: string | undefined;
+  readonly size?: number | undefined;
+  readonly eTag?: string | undefined;
+  readonly lastModified?: string | Date | undefined;
+}
+
+export interface S3ListResultLike {
+  readonly contents?: readonly S3ListEntryLike[] | undefined;
+  readonly isTruncated?: boolean | undefined;
+  readonly nextContinuationToken?: string | undefined;
+}
+
+export interface S3ClientLike {
+  file(key: string): S3FileLike;
+  list(input: {
+    prefix?: string;
+    maxKeys?: number;
+    continuationToken?: string;
+  }): Promise<S3ListResultLike>;
+}
+
+interface S3ClientConstructor {
+  new (options: Record<string, unknown>): S3ClientLike;
+}
+
+/**
+ * Everything but the key pair `Bun.S3Client` is built with — the SAME `resolveS3Target` the signed
+ * path reads, handed over explicitly so Bun never consults a second table.
+ */
+export function clientTarget(options: S3DriverOptions): Record<string, unknown> {
+  const target = resolveS3Target(options);
+  // Bun's flag is the inverse: path style means "not virtual hosted".
+  const pathStyle = options.forcePathStyle;
+  return {
+    bucket: options.bucket,
+    ...(target.region === undefined ? {} : { region: target.region }),
+    ...(target.endpoint === undefined ? {} : { endpoint: target.endpoint }),
+    ...(pathStyle === undefined ? {} : { virtualHostedStyle: !pathStyle }),
+    ...(target.sessionToken === undefined ? {} : { sessionToken: target.sessionToken }),
+  };
+}
+
+export function buildClient(options: S3DriverOptions): S3ClientLike {
+  if (options.client !== undefined) return options.client;
+  if (options.bucket === '') {
+    throw new ConfigInvalidError({
+      cause: 's3 disk was defined without a bucket',
+      fix: 'set storage.disks.<name>.bucket in app.config.ts',
+    });
+  }
+  const env = options.env ?? process.env;
+  const idVar = options.accessKeyIdEnv ?? 'S3_ACCESS_KEY_ID';
+  const secretVar = options.secretAccessKeyEnv ?? 'S3_SECRET_ACCESS_KEY';
+  const Client = (Bun as unknown as { S3Client?: S3ClientConstructor }).S3Client;
+  if (Client === undefined) {
+    throw new ConfigInvalidError({
+      cause: 'Bun.S3Client is unavailable in this runtime',
+      fix: 'upgrade the runtime: bun upgrade   # the s3 disk needs bun >= 1.3',
+    });
+  }
+  return new Client({
+    accessKeyId: requireEnv(env, idVar, secretVar),
+    secretAccessKey: requireEnv(env, secretVar, idVar),
+    ...clientTarget(options),
+  });
+}

package/src/driver-s3-lock.ts

@@ -0,0 +1,131 @@
+// Single responsibility: the two s3 calls `Bun.S3Client` cannot make — a PUT carrying user
+// metadata, `Cache-Control` and Object Lock headers, and the `?retention` / `?legal-hold` reads
+// behind `retentionOf` — over the signed wire in `driver-s3-signed.ts`, answered in this package's
+// vocabulary. A plain `put()` never comes here: it stays on the Bun client.
+
+import { assert, describeValue } from '@ultimat3/core';
+import { DEFAULT_CONTENT_TYPE, type PutOptions } from './driver';
+import { type S3Refusal, type S3WireOptions, sendSigned, xmlField } from './driver-s3-signed';
+import { isRetentionMode, type ObjectLock } from './object-lock';
+
+/** RFC 9110 `token` — the only bytes an `x-amz-meta-<name>` header name may carry. */
+const HEADER_TOKEN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/;
+/** Printable ASCII: S3 stores a metadata value as a header, and a CR or a non-ASCII byte is not one. */
+const HEADER_VALUE = /^[\x20-\x7e]*$/;
+
+/** The options only the signed PUT can carry. Any one of them routes the put off the Bun client. */
+export function needsSignedPut(options: PutOptions | undefined): boolean {
+  return (
+    options?.metadata !== undefined ||
+    options?.cacheControl !== undefined ||
+    options?.retention !== undefined ||
+    options?.legalHold !== undefined
+  );
+}
+
+/** Every header the signed PUT sends beyond the signer's own, screened before a byte moves. */
+export function putHeaders(bytes: Uint8Array, options: PutOptions): Record<string, string> {
+  const headers: Record<string, string> = {
+    'content-type': options.contentType ?? DEFAULT_CONTENT_TYPE,
+    // Required by S3 on any PUT that sets a retention, and an integrity check on every other one:
+    // a body that changed in flight is `BadDigest`, never a stored object.
+    'content-md5': new Bun.CryptoHasher('md5').update(bytes).digest('base64'),
+  };
+  if (options.cacheControl !== undefined) {
+    assert(
+      HEADER_VALUE.test(options.cacheControl),
+      `put() on the s3 disk was given a cacheControl that cannot travel as a header (${describeValue(options.cacheControl)})`,
+      "put(key, body, { cacheControl: 'public, max-age=3600' }) — printable ASCII, no line breaks",
+    );
+    headers['cache-control'] = options.cacheControl;
+  }
+  for (const [name, value] of Object.entries(options.metadata ?? {})) {
+    assert(
+      HEADER_TOKEN.test(name) && HEADER_VALUE.test(value),
+      `put() on the s3 disk was given metadata that cannot travel as an x-amz-meta-* header (name ${describeValue(name)}, value ${describeValue(value)})`,
+      'name each field with letters, digits and hyphens, and keep values printable ASCII — encodeURIComponent(value) anything else',
+    );
+    headers[`x-amz-meta-${name.toLowerCase()}`] = value;
+  }
+  if (options.retention !== undefined) {
+    headers['x-amz-object-lock-mode'] = options.retention.mode;
+    headers['x-amz-object-lock-retain-until-date'] = options.retention.retainUntil.toISOString();
+  }
+  if (options.legalHold !== undefined) {
+    headers['x-amz-object-lock-legal-hold'] = options.legalHold ? 'ON' : 'OFF';
+  }
+  return headers;
+}
+
+/** The signed PUT. `undefined` on success; the provider's refusal otherwise, for the driver to code. */
+export async function signedPut(
+  wire: S3WireOptions,
+  key: string,
+  bytes: Uint8Array,
+  options: PutOptions,
+): Promise<S3Refusal | undefined> {
+  const headers = putHeaders(bytes, options);
+  const result = await sendSigned(wire, { method: 'PUT', key, headers, body: bytes });
+  return result.ok ? undefined : result.refusal;
+}
+
+/**
+ * "Nothing locks this", as the provider says it: no configuration on the object, or a bucket made
+ * without Object Lock at all. Asked BEFORE the absent-object test, because the first answers 404.
+ */
+function isUnlockedAnswer(refusal: S3Refusal): boolean {
+  if (refusal.code === 'NoSuchObjectLockConfiguration') return true;
+  if (refusal.code === 'ObjectLockConfigurationNotFoundError') return true;
+  return refusal.code === 'InvalidRequest' && /object lock/i.test(refusal.message);
+}
+
+export type LockRead =
+  | { readonly kind: 'lock'; readonly lock: ObjectLock }
+  | { readonly kind: 'refused'; readonly refusal: S3Refusal };
+
+async function readSubresource(
+  wire: S3WireOptions,
+  key: string,
+  subresource: 'retention' | 'legal-hold',
+): Promise<{ readonly xml: string } | { readonly unlocked: true } | S3Refusal> {
+  const result = await sendSigned(wire, { method: 'GET', key, subresource });
+  if (result.ok) return { xml: await result.response.text() };
+  return isUnlockedAnswer(result.refusal) ? { unlocked: true } : result.refusal;
+}
+
+const unreadable = (key: string, what: string): S3Refusal => ({
+  name: 'S3Error',
+  code: 'UnreadableAnswer',
+  message: `the provider's ${what} for "${key}" did not parse`,
+  statusCode: 200,
+});
+
+/**
+ * Both halves of the lock, read in parallel. A refusal of either is the answer — a caller that
+ * asked "may I delete this?" must not hear "nothing locks it" because one read was denied.
+ */
+export async function readObjectLock(wire: S3WireOptions, key: string): Promise<LockRead> {
+  const [retention, hold] = await Promise.all([
+    readSubresource(wire, key, 'retention'),
+    readSubresource(wire, key, 'legal-hold'),
+  ]);
+  if ('code' in retention) return { kind: 'refused', refusal: retention };
+  if ('code' in hold) return { kind: 'refused', refusal: hold };
+  let lock: ObjectLock = { legalHold: false };
+  if ('xml' in retention) {
+    const mode = xmlField(retention.xml, 'Mode');
+    const until = new Date(xmlField(retention.xml, 'RetainUntilDate') ?? '');
+    if (!isRetentionMode(mode) || !Number.isFinite(until.getTime())) {
+      return { kind: 'refused', refusal: unreadable(key, 'retention') };
+    }
+    lock = { retention: { mode, retainUntil: until }, legalHold: false };
+  }
+  if ('xml' in hold) {
+    const status = xmlField(hold.xml, 'Status');
+    if (status !== 'ON' && status !== 'OFF') {
+      return { kind: 'refused', refusal: unreadable(key, 'legal hold') };
+    }
+    lock = { ...lock, legalHold: status === 'ON' };
+  }
+  return { kind: 'lock', lock };
+}