@ultimat3/storage 29.3.2 → 30.0.0

package/src/grant.ts

@@ -10,6 +10,7 @@ import type { AttachmentTarget } from './attachment';
 import { attachmentKey, pendingKey, quarantineKey, uploadName } from './attachment';
 import type { StorageDriver } from './driver';
 import { contentTypeNotAllowed, tooLarge } from './errors';
+import { uploadSizeUndeclared } from './errors-stream';
 import { DEFAULT_SIGNED_URL_TTL_MS } from './signed-url';
 import type { UploadPolicy } from './upload';
 import { normalizeContentType, uploadPolicy } from './upload';
@@ -19,10 +20,11 @@ export interface UploadRequest {
   readonly filename: string;
   readonly contentType: string;
   /**
-   * Optional, and never trusted: it only buys the client an early refusal. The bytes are counted
-   * again by `acceptSignedUpload`, which is the authority, so a lie here changes nothing.
+   * The file's size in bytes. Required, because it is what the URL binds: an s3 disk signs it as the
+   * exact `Content-Length`, and the local disks sign it as the ceiling `acceptSignedUpload` counts
+   * against. A lie here shrinks the client's own upload; it cannot pass the policy's `maxBytes`.
    */
-  readonly size?: number | undefined;
+  readonly size: number;
 }
 
 export interface UploadGrant {
@@ -32,6 +34,7 @@ export interface UploadGrant {
   readonly method: 'PUT';
   /** Normalised. The client MUST send exactly this as `Content-Type` or the accept refuses. */
   readonly contentType: string;
+  /** What the URL accepts: the declared `size`, already held to the policy. */
   readonly maxBytes: number;
   /** Epoch ms. The grant's own view of the window; the signature is what actually enforces it. */
   readonly expiresAt: number;
@@ -57,12 +60,21 @@ export interface GrantUploadInput {
   readonly uploadId?: (() => string) | undefined;
 }
 
+/** What every grant decides before it signs: the key, the type and size the policy accepted. */
+export interface GrantedUpload {
+  readonly key: string;
+  readonly contentType: string;
+  readonly size: number;
+  readonly expiresInMs: number;
+  readonly now: number;
+}
+
 /**
- * Refuses before it signs. A URL only exists for a request the policy already accepted, so an
- * over-limit or disallowed upload costs one round trip instead of a full transfer — and the
- * signature then carries the same two constraints, so the client cannot widen either of them.
+ * The refusals both grants share — `grantUpload`'s one PUT and `grantMultipart`'s parts — so a
+ * request one refuses the other refuses too: the key derived, never the client's; the type held to
+ * the allowlist; the size to the policy; the window screened.
  */
-export async function grantUpload(input: GrantUploadInput): Promise<UploadGrant> {
+export function decideGrant(caller: string, input: GrantUploadInput): GrantedUpload {
   const policy = input.policy ?? uploadPolicy();
   const clock = input.clock ?? systemClock;
   const declared = normalizeContentType(input.request.contentType);
@@ -78,33 +90,38 @@ export async function grantUpload(input: GrantUploadInput): Promise<UploadGrant>
     throw contentTypeNotAllowed(key, declared, policy.allowedContentTypes);
   }
   const size = input.request.size;
-  if (size !== undefined && Number.isSafeInteger(size) && size > policy.maxBytes) {
-    throw tooLarge(key, size, policy.maxBytes);
-  }
+  // Off the wire, so the type promises nothing here. An empty file has nothing to upload.
+  if (!Number.isSafeInteger(size) || size < 1) throw uploadSizeUndeclared(key, size);
+  if (size > policy.maxBytes) throw tooLarge(key, size, policy.maxBytes);
 
   // Refused here as well as in the presigner one call down: `expiresAt: now + NaN` is `NaN` in the
-  // grant this function RETURNS, and the message a caller can act on names `grantUpload`'s
-  // own option rather than `buildSignedUrl`'s.
-  // `=== undefined`, never `??`, for the reason `buildSignedUrl` states: `??` coalesces on `null`
-  // too, so an explicit one took the default instead of this refusal.
+  // grant this function RETURNS, and the message a caller can act on names the grant's own option
+  // rather than `buildSignedUrl`'s. `=== undefined`, never `??`, for the reason `buildSignedUrl`
+  // states: `??` coalesces on `null` too, so an explicit one took the default instead.
   const expiresInMs = finiteCount(
-    'grantUpload',
+    caller,
     'expiresInMs',
     input.expiresInMs === undefined ? DEFAULT_SIGNED_URL_TTL_MS : input.expiresInMs,
     1,
   );
+  return { key, contentType: declared, size, expiresInMs, now: clock.now().getTime() };
+}
+
+/**
+ * Refuses before it signs. A URL only exists for a request the policy already accepted, so an
+ * over-limit or disallowed upload costs one round trip instead of a full transfer. The signature
+ * then carries the size on every disk, so the client cannot widen it: at the provider's PUT on s3
+ * (403 on any other length), and at `acceptSignedUpload` on the local disks. The content type is
+ * signed only on a disk that signs its own URLs; on s3, `promoteAttachment` sniffs it.
+ */
+export async function grantUpload(input: GrantUploadInput): Promise<UploadGrant> {
+  const { key, contentType, size, expiresInMs, now } = decideGrant('grantUpload', input);
   const url = await input.disk.signedUrl(key, {
     method: 'PUT',
-    maxBytes: policy.maxBytes,
-    contentType: declared,
+    maxBytes: size,
+    contentLength: size,
+    contentType,
     expiresInMs,
   });
-  return {
-    key,
-    url,
-    method: 'PUT',
-    contentType: declared,
-    maxBytes: policy.maxBytes,
-    expiresAt: clock.now().getTime() + expiresInMs,
-  };
+  return { key, url, method: 'PUT', contentType, maxBytes: size, expiresAt: now + expiresInMs };
 }

package/src/image-warm.ts

@@ -0,0 +1,84 @@
+// Single responsibility: write a stored image's variants AHEAD of the first request, under the exact
+// keys the media route looks up, so no visitor pays the decode. The job an upload enqueues calls
+// this; the request path then answers `exists` + `get` and never reaches the pipeline.
+
+import {
+  assert,
+  finiteCount,
+  IMAGE_MIME_TYPES,
+  isUltimateError,
+  probeImage,
+  transformImageBytes,
+} from '@ultimat3/core';
+import type { StorageDriver } from './driver';
+import { type VariantFormat, variantKey, variantRevision } from './image';
+
+export interface WarmVariantsInput {
+  readonly disk: StorageDriver;
+  /** The SOURCE object's key, as the media route is asked for it. */
+  readonly key: string;
+  /** Whole pixels. Widths above the source's are skipped: the route never stores an upscale. */
+  readonly widths: readonly number[];
+  /** `avif` is accepted by the type and fails per variant with core's `X_IMAGE_UNSUPPORTED`. */
+  readonly formats: readonly VariantFormat[];
+}
+
+/** A variant that was not written: its key, and the failure's code and cause. */
+export interface WarmFailure {
+  readonly key: string;
+  /** `X_IMAGE_UNSUPPORTED` for an `avif`, a storage code for a refused `put`. */
+  readonly code: string;
+  readonly cause: string;
+}
+
+export interface WarmVariantsResult {
+  /** The variant keys written, in width-then-format order. */
+  readonly written: readonly string[];
+  /** One entry per variant that could not be made or stored; the others were still written. */
+  readonly failed: readonly WarmFailure[];
+  /** Widths above the source's intrinsic width: never minted, never stored. */
+  readonly skipped: readonly number[];
+}
+
+/**
+ * Every `widths` x `formats` variant of `key`, keyed as `variantKey(key, { width, format,
+ * revision })` with the revision read off THIS read's etag — the one the bytes were made from, so
+ * an overwrite racing the warm-up can only leave keys its new etag never names. The transform spec
+ * is the builtin image driver's (`width` + `format`, default quality), so a warmed variant is the
+ * object the route would have stored itself.
+ *
+ * The source is READ and PROBED once, and the probe holds it to core's `MAX_IMAGE_PIXELS` before
+ * any pixel is decoded; each variant is then one pipeline pass over the bytes already in hand.
+ */
+export async function warmVariants(input: WarmVariantsInput): Promise<WarmVariantsResult> {
+  for (const width of input.widths) finiteCount('warmVariants', 'widths', width, 1);
+  const source = await input.disk.get(input.key);
+  const revision = variantRevision(source.object.etag);
+  assert(
+    revision !== undefined,
+    `warmVariants('${input.key}'): the source etag "${source.object.etag}" carries no 8-hex revision, so no variant of it has a key the media route would read`,
+    `x doctor --json   # the disk driver answers no content etag; variants of ${input.key} are served, never stored`,
+  );
+  // A header read: past `MAX_IMAGE_PIXELS` it is core's `X_IMAGE_TOO_LARGE` before any decode.
+  const info = probeImage(source.bytes);
+  const written: string[] = [];
+  const failed: WarmFailure[] = [];
+  const skipped = input.widths.filter((width) => width > info.width);
+  for (const width of input.widths.filter((one) => one <= info.width)) {
+    for (const format of input.formats) {
+      const key = variantKey(input.key, { width, format, revision });
+      try {
+        const bytes = await transformImageBytes(source.bytes, { width, format });
+        await input.disk.put(key, bytes, {
+          contentType: IMAGE_MIME_TYPES[probeImage(bytes).format],
+        });
+        written.push(key);
+      } catch (error) {
+        // Coded failures are per-variant answers; anything uncoded is a defect and is not swallowed.
+        if (!isUltimateError(error)) throw error;
+        failed.push({ key, code: error.code, cause: error.cause });
+      }
+    }
+  }
+  return { written, failed, skipped };
+}

package/src/image.ts

@@ -4,6 +4,7 @@
 // bytes, `transformImage()` returns exactly the size `fitDimensions()` already promised.
 
 import {
+  assert,
   assertFiniteImageQuality,
   blurDataUrl,
   finiteCount,
@@ -13,6 +14,7 @@ import {
   transformImageBytes,
 } from '@ultimat3/core';
 import { assertSafeKey } from './path';
+import { normalizeContentType } from './upload';
 
 /**
  * The formats a stored VARIANT can be minted in — a strict subset of the ones `@ultimat3/core`
@@ -41,6 +43,50 @@ export type VariantFormat = (typeof VARIANT_FORMATS)[number];
 export const isVariantFormat = (format: string): format is VariantFormat =>
   (VARIANT_FORMATS as readonly string[]).includes(format);
 
+/** The source encodings core can write back, so a variant with no `?f=` keeps its source's. */
+const SOURCE_VARIANT_FORMATS: ReadonlyMap<string, VariantFormat> = new Map([
+  ['image/png', 'png'],
+  ['image/jpeg', 'jpeg'],
+  ['image/webp', 'webp'],
+]);
+
+/**
+ * The format a variant of this stored type is written in when nobody named one. `undefined` (a
+ * `gif`) means no key can state what the transform will answer: such a variant is served, never
+ * stored. Without this `variantKey` read the absent format as `webp` while the driver kept the
+ * source's, so `?w=320` and `?w=320&f=webp` shared one key and the first request chose the bytes.
+ */
+export const variantFormatOf = (contentType: string): VariantFormat | undefined =>
+  SOURCE_VARIANT_FORMATS.get(normalizeContentType(contentType));
+
+const REVISION = /^[0-9a-f]{8}$/;
+/** The `@…r<8 hex>.<ext>` only `variantKey({ revision })` writes. The last `@` is the variant's. */
+const REVISIONED_VARIANT = /^(.+)@(?:[^@/]*-)?r([0-9a-f]{8})\.(?:avif|webp|jpg|png)$/;
+
+/** The source and revision a stored variant's key names, or `undefined` for any other key. */
+export function variantSourceOf(
+  key: string,
+): { readonly source: string; readonly revision: string } | undefined {
+  const match = REVISIONED_VARIANT.exec(key);
+  return match?.[1] === undefined || match[2] === undefined
+    ? undefined
+    : { source: match[1], revision: match[2] };
+}
+
+/**
+ * The part of a source's etag a variant key carries, so an overwritten source misses every variant
+ * of its old bytes. Hex only, because a provider's etag arrives quoted and a multipart one carries
+ * `-<parts>`. `undefined` when the etag is unknowable (`''`) or too short to tell two revisions
+ * apart: that variant has no honest key, so it is served, never stored.
+ */
+export function variantRevision(etag: string): string | undefined {
+  const revision = etag
+    .toLowerCase()
+    .replace(/[^0-9a-f]/g, '')
+    .slice(0, 8);
+  return REVISION.test(revision) ? revision : undefined;
+}
+
 /** `cover` fills the box and crops the overflow; `contain` fits inside it, no crop. */
 export type ImageFit = 'cover' | 'contain';
 
@@ -51,6 +97,8 @@ export interface ImageTransform {
   /** 1-100. Omitted means the format default (`DEFAULT_QUALITY`). */
   readonly quality?: number | undefined;
   readonly fit?: ImageFit | undefined;
+  /** `variantRevision(source.etag)`: which bytes this variant was made from. Absent, unkeyed. */
+  readonly revision?: string | undefined;
 }
 
 export interface ImageSize {
@@ -116,6 +164,15 @@ export function variantKey(sourceKey: string, transform: ImageTransform): string
     transform.quality === undefined ? DEFAULT_QUALITY : transform.quality,
   );
   if (quality !== DEFAULT_QUALITY) parts.push(`q${quality}`);
+  const { revision } = transform;
+  if (revision !== undefined) {
+    assert(
+      REVISION.test(revision),
+      'variantKey was given a revision that is not 8 hex digits, so it would mint a key nothing can derive again',
+      'variantKey(key, { width, revision: variantRevision(object.etag) })',
+    );
+    parts.push(`r${revision}`);
+  }
   if (parts.length === 0) parts.push('full');
   return assertSafeKey(`${safe}@${parts.join('-')}.${FORMAT_EXTENSIONS[format]}`);
 }

package/src/index.ts

@@ -3,8 +3,11 @@
 
 export type { AcceptSignedUploadInput, SignedRequestInput } from './accept';
 export { acceptSignedUpload, readSignedObject, signedUploadConstraints } from './accept';
+export type { AcceptSignedPartInput } from './accept-part';
+export { acceptSignedPart } from './accept-part';
 export type {
   AttachmentTarget,
+  DeleteVariantsInput,
   PromoteAttachmentInput,
   ReleaseQuarantineInput,
   SweepFailure,
@@ -14,6 +17,7 @@ export type {
 export {
   attachmentKey,
   attachmentPrefix,
+  deleteVariants,
   isPendingKey,
   isQuarantinedKey,
   PENDING_SEGMENT,
@@ -28,8 +32,11 @@ export {
   uploadExtension,
   uploadName,
 } from './attachment';
+export type { PumpLimit } from './byte-pump';
+export { pumpBounded } from './byte-pump';
 export type {
   ByteLimit,
+  CopyOptions,
   ListOptions,
   PutOptions,
   ServerSideEncryption,
@@ -50,6 +57,7 @@ export {
   etagOf,
   resolveListLimit,
   sha256Base64,
+  signedByteBound,
   toBytes,
 } from './driver';
 export type { LocalDriverOptions } from './driver-local';
@@ -96,9 +104,33 @@ export {
   uploadFailed,
   xmlBodyUnreadable,
 } from './errors';
+export { multipartUnsupported, partsInvalid, uploadUnknown } from './errors-multipart';
 export { alreadyPromoted, quarantined } from './errors-promote';
+export { rangeTooLarge, rangeUnsatisfiable } from './errors-range';
+export type {
+  AbortResult,
+  AbortStaleUploadsInput,
+  ExpireObjectsInput,
+  ExpireResult,
+  ObjectPurgeTargetInput,
+} from './expire';
+export { abortStaleUploads, expireObjects, objectPurgeTarget } from './expire';
 export type { GrantUploadInput, UploadGrant, UploadRequest } from './grant';
 export { grantUpload } from './grant';
+export type {
+  CompleteUploadInput,
+  GrantMultipartInput,
+  MultipartGrant,
+  SignPartInput,
+  UploadSessionInput,
+} from './grant-multipart';
+export {
+  abortUpload,
+  completeUpload,
+  grantMultipart,
+  listParts,
+  signPart,
+} from './grant-multipart';
 export type {
   ImageFit,
   ImageSize,
@@ -119,8 +151,21 @@ export {
   srcsetDescriptors,
   transformImage,
   VARIANT_FORMATS,
+  variantFormatOf,
   variantKey,
+  variantRevision,
+  variantSourceOf,
 } from './image';
+export type { WarmFailure, WarmVariantsInput, WarmVariantsResult } from './image-warm';
+export { warmVariants } from './image-warm';
+export type {
+  CompletedPart,
+  MultipartCapability,
+  MultipartSession,
+  SignPartOptions,
+  UploadedPart,
+} from './multipart';
+export { MAX_PART_BYTES, MAX_PARTS, MIN_PART_BYTES, matchParts, partBytesFor } from './multipart';
 export type {
   LockedAction,
   ObjectLock,
@@ -129,7 +174,6 @@ export type {
   RetentionMode,
 } from './object-lock';
 export { isLocked, ObjectLockedError, RETENTION_MODES } from './object-lock';
-
 export {
   assertSafeKey,
   isSafeKey,
@@ -144,7 +188,16 @@ export {
   orgPrefix,
   scopedKey,
 } from './path';
+export type { PutStreamOptions } from './put-stream';
+export { assertPutStream, streamLimit } from './put-stream';
+/**
+ * `get(key, { range })`. A third-party driver screens and clamps through these two, or it is a
+ * fourth answer to what a range means.
+ */
+export type { ByteRange, GetOptions } from './range';
+export { assertGetOptions, assertStreamRange, servedRange, streamedRange } from './range';
 export type {
+  SignedPart,
   SignedUrlConstraints,
   SignedUrlFailure,
   SignedUrlInput,
@@ -171,15 +224,31 @@ export {
 } from './signing-secret';
 export type { Storage, StorageConfig } from './storage';
 export { definedStorage, defineStorage, disk, resetStorage, storage } from './storage';
-export type { UploadCandidate, UploadPolicy, UploadPolicyInit, ValidatedUpload } from './upload';
+export type {
+  StorageSubjectRef,
+  StorageSubjectTarget,
+  StorageSubjectTargetOptions,
+} from './subject-target';
+export { STORAGE_SUBJECT_COLUMNS, storageSubjectTarget } from './subject-target';
+export type {
+  StoredHeadCandidate,
+  UploadCandidate,
+  UploadPolicy,
+  UploadPolicyInit,
+  ValidatedHead,
+  ValidatedUpload,
+} from './upload';
 export {
   contentTypeMatches,
   DEFAULT_MAX_UPLOAD_BYTES,
   DOCUMENT_CONTENT_TYPES,
   IMAGE_CONTENT_TYPES,
+  MAX_LEADING_WHITESPACE,
   normalizeContentType,
+  SNIFF_WINDOW_BYTES,
   sniffContentType,
   uploadPolicy,
+  validateStoredHead,
   validateUpload,
 } from './upload';
 export type {
@@ -196,3 +265,13 @@ export {
   uploadFile,
   xhrSignedPut,
 } from './upload-client';
+export type {
+  MultipartUploadClient,
+  MultipartUploadInput,
+  UploadSessionRef,
+} from './upload-client-multipart';
+export {
+  DEFAULT_PART_THRESHOLD_BYTES,
+  PART_CONCURRENCY,
+  uploadInParts,
+} from './upload-client-multipart';

package/src/multipart-staged.ts

@@ -0,0 +1,127 @@
+// Single responsibility: an upload in parts on a disk that signs its own URLs (`local`, `memory`):
+// the session rules over a `PartStore` the disk supplies. Same contract as s3's, so dev and tests
+// behave as production: a part URL is signed, a part lands only in an open session, a Complete
+// assembles exactly the parts named and drops the session, and an abort leaves nothing.
+
+import type { Clock } from '@ultimat3/core';
+import { etagOf, type StorageDriver, type StorageObject } from './driver';
+import { uploadUnknown } from './errors-multipart';
+import type { MultipartCapability, MultipartSession, UploadedPart } from './multipart';
+import { assertSafeKey } from './path';
+
+/** What a staged session records at `create`. `initiated` is ISO, so a file store can hold it. */
+export interface StagedSession {
+  readonly key: string;
+  readonly contentType: string;
+  readonly initiated: string;
+}
+
+/** Where a disk keeps open sessions and their parts: a directory, or a `Map`. */
+export interface PartStore {
+  open(uploadId: string, session: StagedSession): Promise<void>;
+  session(uploadId: string): Promise<StagedSession | undefined>;
+  writePart(uploadId: string, part: UploadedPart, bytes: Uint8Array): Promise<void>;
+  /** Ascending by part number. */
+  parts(uploadId: string): Promise<readonly UploadedPart[]>;
+  /** The named parts' bytes, end to end, read as the stream is pulled. */
+  read(uploadId: string, parts: readonly UploadedPart[]): ReadableStream<Uint8Array>;
+  drop(uploadId: string): Promise<void>;
+  sessions(): Promise<readonly (readonly [string, StagedSession])[]>;
+}
+
+export interface StagedMultipartOptions {
+  readonly store: PartStore;
+  readonly clock: Clock;
+  /** The disk the parts belong to: it signs their URLs and takes the Complete's stream. */
+  readonly driver: StorageDriver;
+}
+
+/**
+ * The disk with `multipart` attached IN PLACE, never spread: a spread reads `signedUrlBase` once,
+ * and `registerAs` re-points it after construction.
+ */
+export function withStagedMultipart<D extends StorageDriver>(
+  driver: D,
+  store: PartStore,
+  clock: Clock,
+): D {
+  return Object.assign(driver, { multipart: stagedMultipart({ store, clock, driver }) });
+}
+
+export function stagedMultipart(options: StagedMultipartOptions): MultipartCapability {
+  const { store } = options;
+  /** The session, held to the key it was opened for: an id is never a capability on its own. */
+  const openSession = async (key: string, uploadId: string): Promise<StagedSession> => {
+    const session = await store.session(uploadId);
+    if (session === undefined || session.key !== key) {
+      throw uploadUnknown(options.driver.name, key, uploadId);
+    }
+    return session;
+  };
+
+  return {
+    async create(key, { contentType }): Promise<string> {
+      const safe = assertSafeKey(key);
+      const uploadId = crypto.randomUUID();
+      await store.open(uploadId, {
+        key: safe,
+        contentType,
+        initiated: options.clock.now().toISOString(),
+      });
+      return uploadId;
+    },
+
+    signPart: (key, uploadId, partNumber, { contentLength, expiresInMs }) =>
+      options.driver.signedUrl(key, {
+        method: 'PUT',
+        contentLength,
+        maxBytes: contentLength,
+        expiresInMs,
+        part: { uploadId, partNumber },
+      }),
+
+    async listParts(key, uploadId): Promise<readonly UploadedPart[]> {
+      await openSession(assertSafeKey(key), uploadId);
+      return store.parts(uploadId);
+    },
+
+    async receivePart(key, uploadId, partNumber, bytes): Promise<UploadedPart> {
+      await openSession(assertSafeKey(key), uploadId);
+      const part = { partNumber, etag: etagOf(bytes), size: bytes.byteLength };
+      await store.writePart(uploadId, part, bytes);
+      return part;
+    },
+
+    async complete(key, uploadId, parts): Promise<StorageObject> {
+      const safe = assertSafeKey(key);
+      const session = await openSession(safe, uploadId);
+      const total = parts.reduce((sum, part) => sum + part.size, 0);
+      const object = await options.driver.putStream(safe, store.read(uploadId, parts), {
+        maxBytes: Math.max(total, 1),
+        contentType: session.contentType,
+      });
+      await store.drop(uploadId);
+      return object;
+    },
+
+    async abort(key, uploadId): Promise<void> {
+      const safe = assertSafeKey(key);
+      const session = await store.session(uploadId);
+      if (session?.key === safe) await store.drop(uploadId);
+    },
+
+    async listUploads(prefix): Promise<readonly MultipartSession[]> {
+      const open = await store.sessions();
+      return open
+        .filter(([, session]) => prefix === undefined || session.key.startsWith(prefix))
+        .map(([uploadId, session]) => {
+          const initiated = new Date(session.initiated);
+          return {
+            key: session.key,
+            uploadId,
+            ...(Number.isNaN(initiated.getTime()) ? {} : { initiated }),
+          };
+        });
+    },
+  };
+}