@ultimat3/storage 29.4.0 → 30.1.0

package/src/subject-target.ts

@@ -0,0 +1,82 @@
+// `storageSubjectTarget(disk, keysOf)`: the subject's stored objects as a subject target a
+// `defineSubject({ targets })` lists — structural, so `@ultimat3/jobs` imports nothing from here.
+// Which keys are the person's is the app's (`keysOf`); the export lists each object, the erase
+// deletes them. A key the app does not name is not found: there is no listing by owner.
+
+import type { StorageDriver } from './driver';
+
+/** The subject as a target is handed it: the key asked for, and the subject's row read fresh. */
+export interface StorageSubjectRef {
+  readonly key: string;
+  readonly row: Readonly<Record<string, unknown>>;
+}
+
+export interface StorageSubjectTargetOptions {
+  /** The step key and export folder. Default `storage`. Unique among one subject's targets. */
+  readonly name?: string;
+}
+
+type Cell = string | number | boolean | null;
+
+export interface StorageSubjectTarget {
+  readonly name: string;
+  readonly columns: readonly string[];
+  exportPage(
+    subject: StorageSubjectRef,
+    cursor: string | null,
+    size: number,
+  ): Promise<{
+    readonly rows: readonly Readonly<Record<string, Cell>>[];
+    readonly cursor: string | null;
+  }>;
+  erase(subject: StorageSubjectRef, size: number): Promise<number>;
+}
+
+export const STORAGE_SUBJECT_COLUMNS = ['key', 'size', 'contentType', 'lastModified'] as const;
+
+/**
+ * `disk` is a THUNK, read per call, as an export's sink is: a subject is declared at import and
+ * the app's disks exist after `defineStorage()`.
+ */
+export function storageSubjectTarget(
+  disk: () => StorageDriver,
+  /** The person's keys — a list the app builds, or lists from a prefix it owns. */
+  keysOf: (subject: StorageSubjectRef) => readonly string[] | Promise<readonly string[]>,
+  options: StorageSubjectTargetOptions = {},
+): StorageSubjectTarget {
+  return {
+    name: options.name ?? 'storage',
+    columns: STORAGE_SUBJECT_COLUMNS,
+    async exportPage(subject, cursor, size) {
+      const keys = await keysOf(subject);
+      const start = cursor === null ? 0 : Number(cursor);
+      const page = keys.slice(start, start + size);
+      const rows: Record<string, Cell>[] = [];
+      for (const key of page) {
+        const found = await disk().stat(key);
+        if (found === undefined) continue;
+        rows.push({
+          key,
+          size: found.size,
+          contentType: found.contentType,
+          // An instant, UTC, its zone in the string — never formatted for a locale here. Absent
+          // when the provider sent none, and never invented.
+          lastModified: found.lastModified?.toISOString() ?? null,
+        });
+      }
+      const next = start + page.length;
+      return { rows, cursor: next < keys.length ? String(next) : null };
+    },
+    async erase(subject, size) {
+      // Only what still exists counts, so a replayed batch answers short and the loop ends.
+      let removed = 0;
+      for (const key of await keysOf(subject)) {
+        if (removed >= size) break;
+        if (!(await disk().exists(key))) continue;
+        await disk().delete(key);
+        removed += 1;
+      }
+      return removed;
+    },
+  };
+}

package/src/upload-client-multipart.ts

@@ -0,0 +1,156 @@
+// Single responsibility: the browser half of an upload in parts — grant (or resume), PUT each part
+// at its own signed URL with bounded concurrency and core's retry, then complete. Browser-safe as
+// `upload-client.ts` is: no `Bun.*`, no `node:`, no driver import, and every network call is either
+// the app's typed action or the one `SignedPut` seam.
+
+import { finiteCount, retry } from '@ultimat3/core';
+import { uploadFailed } from './errors';
+import type { UploadRequest } from './grant';
+import type { MultipartGrant } from './grant-multipart';
+import type { CompletedPart, UploadedPart } from './multipart';
+import type { SignedPut, UploadProgress, UploadSource } from './upload-client';
+
+/** One open session, as each of the app's calls names it. */
+export interface UploadSessionRef {
+  readonly key: string;
+  readonly uploadId: string;
+}
+
+/**
+ * The app's typed action calls, each wrapping one server function of `grant-multipart.ts` under the
+ * app's own policy — the reason `grant` is the app's in `uploadFile`.
+ */
+export interface MultipartUploadClient {
+  readonly grant: (request: UploadRequest) => Promise<MultipartGrant>;
+  readonly signPart: (
+    input: UploadSessionRef & { readonly partNumber: number; readonly size: number },
+  ) => Promise<string>;
+  readonly listParts: (input: UploadSessionRef) => Promise<readonly UploadedPart[]>;
+  /** Etags are read on the server (`completeUpload`), so no bucket CORS has to expose `ETag`. */
+  readonly complete: (
+    input: UploadSessionRef & { readonly parts: readonly CompletedPart[] },
+  ) => Promise<unknown>;
+  /** Called when the caller's `signal` aborts: a cancelled upload leaves no parts billing. */
+  readonly abort: (input: UploadSessionRef) => Promise<unknown>;
+  /** At or under this, one PUT as before. Default 64 MiB. */
+  readonly partThresholdBytes?: number | undefined;
+  /**
+   * A grant from an earlier, interrupted call for the same file: its parts are listed first and only
+   * the missing ones are sent. Persisting it is the caller's — this file keeps no browser storage.
+   */
+  readonly resume?: MultipartGrant | undefined;
+  /** Handed the grant before any part moves, so the caller can persist it for `resume`. */
+  readonly onGrant?: ((grant: MultipartGrant) => void) | undefined;
+  /** Test seam for the retry waits. Defaults to a timer. */
+  readonly sleep?: ((ms: number) => Promise<void>) | undefined;
+}
+
+export const DEFAULT_PART_THRESHOLD_BYTES = 64 * 1024 * 1024;
+/** Parts in flight at once: enough to fill a phone's uplink, few enough to keep its memory. */
+export const PART_CONCURRENCY = 4;
+const PART_RETRY = { attempts: 4, base: 500, max: 8_000, jitter: 'full' } as const;
+
+export const partThresholdOf = (client: MultipartUploadClient): number =>
+  finiteCount(
+    'uploadFile',
+    'multipart.partThresholdBytes',
+    client.partThresholdBytes === undefined
+      ? DEFAULT_PART_THRESHOLD_BYTES
+      : client.partThresholdBytes,
+    1,
+  );
+
+export interface MultipartUploadInput {
+  readonly file: UploadSource;
+  readonly client: MultipartUploadClient;
+  readonly put: SignedPut;
+  readonly onProgress?: ((progress: UploadProgress) => void) | undefined;
+  readonly signal?: AbortSignal | undefined;
+}
+
+/** A grant for this file, or the resumed one; a resume for another size is a different file. */
+async function sessionFor(input: MultipartUploadInput): Promise<MultipartGrant> {
+  const { file, client } = input;
+  const resumed = client.resume;
+  if (resumed !== undefined && resumed.size === file.size) return resumed;
+  const grant = await client.grant({
+    filename: file.name,
+    contentType: file.type,
+    size: file.size,
+  });
+  client.onGrant?.(grant);
+  return grant;
+}
+
+/** Send every part not yet on the disk, then complete. Answers the grant the object landed under. */
+export async function uploadInParts(input: MultipartUploadInput): Promise<MultipartGrant> {
+  const { file, client, put, signal } = input;
+  const grant = await sessionFor(input);
+  const session = { key: grant.key, uploadId: grant.uploadId };
+  const total = file.size;
+  const count = Math.ceil(total / grant.partBytes);
+  const landed = client.resume === grant ? await client.listParts(session) : [];
+  const have = new Set(
+    landed
+      .filter(
+        (part) =>
+          part.size === Math.min(grant.partBytes, total - (part.partNumber - 1) * grant.partBytes),
+      )
+      .map((part) => part.partNumber),
+  );
+  let confirmed = landed
+    .filter((part) => have.has(part.partNumber))
+    .reduce((sum, part) => sum + part.size, 0);
+  input.onProgress?.({ loaded: confirmed, total, ratio: total > 0 ? confirmed / total : 0 });
+
+  const missing: number[] = [];
+  for (let number = 1; number <= count; number += 1) if (!have.has(number)) missing.push(number);
+
+  const cancelled = () => uploadFailed(grant.key, 0, 'the upload was aborted');
+  // A rejected wait ends core's retry loop with that error: a cancelled upload stops retrying.
+  const waitUnlessAborted = (ms: number): Promise<void> =>
+    signal?.aborted === true
+      ? Promise.reject(cancelled())
+      : (client.sleep ?? ((wait) => new Promise<void>((done) => setTimeout(done, wait))))(ms);
+
+  const sendPart = async (partNumber: number): Promise<void> => {
+    const start = (partNumber - 1) * grant.partBytes;
+    const body = file.slice(start, Math.min(start + grant.partBytes, total));
+    await retry(
+      async () => {
+        const url = await client.signPart({ ...session, partNumber, size: body.size });
+        await put({
+          url,
+          contentType: grant.contentType,
+          body,
+          ...(signal === undefined ? {} : { signal }),
+        });
+      },
+      PART_RETRY,
+      { sleep: waitUnlessAborted },
+    );
+    confirmed += body.size;
+    input.onProgress?.({ loaded: confirmed, total, ratio: total > 0 ? confirmed / total : 0 });
+  };
+
+  const worker = async (): Promise<void> => {
+    for (let next = missing.shift(); next !== undefined; next = missing.shift()) {
+      if (signal?.aborted === true) return;
+      await sendPart(next);
+    }
+  };
+  try {
+    await Promise.all(Array.from({ length: Math.min(PART_CONCURRENCY, missing.length) }, worker));
+    if (signal?.aborted === true) throw cancelled();
+  } catch (error) {
+    // Cancelled is abandoned: the parts are dropped now rather than billing until a sweep. Any
+    // other failure keeps the session open, so the same grant passed as `resume` sends only what
+    // is missing. A refused abort is the sweep's to reclaim (`abortStaleUploads`), and the
+    // caller hears the failure that stopped the upload, not the clean-up's.
+    if (signal?.aborted === true) await client.abort(session).catch(() => undefined);
+    throw error;
+  }
+  const parts = Array.from({ length: count }, (_, at) => ({ partNumber: at + 1 }));
+  await client.complete({ ...session, parts });
+  return grant;
+}

package/src/upload-client.ts

@@ -12,6 +12,11 @@
 import { clientTransport } from '@ultimat3/core';
 import { tooLarge, uploadFailed } from './errors';
 import type { UploadGrant, UploadRequest } from './grant';
+import {
+  type MultipartUploadClient,
+  partThresholdOf,
+  uploadInParts,
+} from './upload-client-multipart';
 
 export interface UploadProgress {
   readonly loaded: number;
@@ -144,6 +149,11 @@ export interface UploadFileInput {
    * package has no way to know which route it was projected onto.
    */
   readonly grant: (request: UploadRequest) => Promise<UploadGrant>;
+  /**
+   * The parts form, beside `grant` — which answers one signed PUT and cannot carry parts. A file
+   * over `partThresholdBytes` (64 MiB) goes up in parts, resumable; at or under it, `grant` as before.
+   */
+  readonly multipart?: MultipartUploadClient | undefined;
   readonly onProgress?: ((progress: UploadProgress) => void) | undefined;
   readonly signal?: AbortSignal | undefined;
   readonly put?: SignedPut | undefined;
@@ -162,6 +172,17 @@ export interface UploadedFile {
  * that was always going to be refused.
  */
 export async function uploadFile(input: UploadFileInput): Promise<UploadedFile> {
+  const multipart = input.multipart;
+  if (multipart !== undefined && input.file.size > partThresholdOf(multipart)) {
+    const grant = await uploadInParts({
+      file: input.file,
+      client: multipart,
+      put: input.put ?? defaultSignedPut(),
+      ...(input.onProgress === undefined ? {} : { onProgress: input.onProgress }),
+      ...(input.signal === undefined ? {} : { signal: input.signal }),
+    });
+    return { key: grant.key, contentType: grant.contentType, size: input.file.size };
+  }
   const grant = await input.grant({
     filename: input.file.name,
     contentType: input.file.type,

package/src/upload.ts

@@ -17,11 +17,21 @@ import {
   tooLarge,
   xmlBodyUnreadable,
 } from './errors';
+import { leadingWhitespacePastLimit, xmlBodyPastWindow } from './errors-sniff';
 import { HEIF_CONTAINER, ISO_BMFF_CONTAINER, ISO_BMFF_TYPES, sniffIsoBmff } from './iso-bmff';
 import { assertSafeKey } from './path';
 
 export const DEFAULT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
 
+/**
+ * How much of a stored object promotion reads to decide its type. Every signature fits, and
+ * `MAX_LEADING_WHITESPACE` is what stops a body hiding its markup past the end of it.
+ */
+export const SNIFF_WINDOW_BYTES = 64 * 1024;
+
+/** Bytes of whitespace a body may open with. No real file opens with more. */
+export const MAX_LEADING_WHITESPACE = 4096;
+
 /**
  * `image/svg+xml` is deliberately ABSENT, and this is the default `uploadPolicy()` allowlist. An
  * SVG is a script document: served back from the app's own origin under its declared type it runs
@@ -127,10 +137,11 @@ const matches = (bytes: Uint8Array, rule: MagicRule): boolean =>
     part.pattern.every((byte, index) => bytes[part.offset + index] === byte),
   );
 
-function sniffText(bytes: Uint8Array): string | undefined {
+/** `truncated`: the bytes are a head, so a character cut at their end is not a bad sequence. */
+function sniffText(bytes: Uint8Array, truncated: boolean): string | undefined {
   let text: string;
   try {
-    text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+    text = new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: truncated });
   } catch {
     return undefined;
   }
@@ -139,7 +150,9 @@ function sniffText(bytes: Uint8Array): string | undefined {
     const printable = code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d;
     if (!printable) return undefined;
   }
-  const head = text.slice(0, 512).trimStart().toLowerCase();
+  // Trimmed BEFORE it is cut. The other way round, 512 bytes of padding left an empty head, and
+  // XHTML under `application/xml` was accepted as the plain text it was not.
+  const head = text.trimStart().slice(0, 512).toLowerCase();
   if (
     head.startsWith('<!doctype html') ||
     head.startsWith('<html') ||
@@ -195,14 +208,28 @@ function activeMarkupType(text: string): 'text/html' | 'image/svg+xml' | undefin
   return scanned.includes(SVG_NAMESPACE) ? 'image/svg+xml' : undefined;
 }
 
-/** `undefined` means "no rule recognised it", never "it is fine". */
-export function sniffContentType(bytes: Uint8Array): string | undefined {
+function sniffBytes(bytes: Uint8Array, truncated: boolean): string | undefined {
   for (const rule of MAGIC_RULES) {
     if (matches(bytes, rule)) return rule.type;
   }
   // Not a `MagicRule`: every ISO base media file opens with the same `ftyp` box, and the one rule
   // that matched it labelled AVIF, HEIC, MOV and M4A all `video/mp4`. The major brand decides.
-  return sniffIsoBmff(bytes) ?? sniffText(bytes);
+  return sniffIsoBmff(bytes) ?? sniffText(bytes, truncated);
+}
+
+/** `undefined` means "no rule recognised it", never "it is fine". */
+export const sniffContentType = (bytes: Uint8Array): string | undefined => sniffBytes(bytes, false);
+
+/**
+ * Whether the first `MAX_LEADING_WHITESPACE + 1` bytes are all whitespace. Decoded leniently, and
+ * with `stream` so a wide space cut at the end of the slice is not a replacement character: the
+ * question is asked of bytes the strict sniff may go on to give up on.
+ */
+function opensPastWhitespaceLimit(bytes: Uint8Array): boolean {
+  if (bytes.byteLength <= MAX_LEADING_WHITESPACE) return false;
+  const opening = bytes.subarray(0, MAX_LEADING_WHITESPACE + 1);
+  const text = new TextDecoder('utf-8', { ignoreBOM: true }).decode(opening, { stream: true });
+  return text.trimStart() === '';
 }
 
 // A `Map`, not an object literal: `base` is the transport's own `Content-Type` header by the time
@@ -274,23 +301,24 @@ const isXmlType = (type: string): boolean =>
   type === 'application/xml' || type === 'text/xml' || type.endsWith('+xml');
 
 /**
- * Throws the first violated constraint, in this order: key, size, type, checksum. The key comes
- * before the size because a key nothing may store makes the other three moot, and which
- * constraint a rejected upload reports is what the client retries on — `upload.test.ts` pins it.
+ * Whether these bytes may be what `declared` says: the one type rule, over a whole body or a head.
+ * `truncated` says the object goes on past `bytes`, which only matters to the two questions a head
+ * cannot answer in full: a character cut at its end, and whether an XML document runs a script.
  */
-export function validateUpload(
-  candidate: UploadCandidate,
-  policy: UploadPolicy = uploadPolicy(),
-): ValidatedUpload {
-  const key = assertSafeKey(candidate.key);
-  const size = candidate.bytes.byteLength;
-  if (size > policy.maxBytes) throw tooLarge(key, size, policy.maxBytes);
-
-  const declared = normalizeContentType(candidate.declaredContentType);
-  if (!policy.allowedContentTypes.includes(declared)) {
-    throw contentTypeNotAllowed(key, declared, policy.allowedContentTypes);
+function assertTypeAgrees(
+  key: string,
+  declared: string,
+  bytes: Uint8Array,
+  truncated: boolean,
+): void {
+  if (opensPastWhitespaceLimit(bytes)) {
+    throw leadingWhitespacePastLimit(key, declared, MAX_LEADING_WHITESPACE);
+  }
+  // `activeMarkupType` must read the whole document, because XML runs a script anywhere in it.
+  if (truncated && isXmlType(declared)) {
+    throw xmlBodyPastWindow(key, declared, SNIFF_WINDOW_BYTES);
   }
-  const sniffed = sniffContentType(candidate.bytes);
+  const sniffed = sniffBytes(bytes, truncated);
   if (sniffed === undefined) {
     // `undefined` is "no rule recognised it", never "it is fine" — `sniffContentType`'s own doc
     // says so and this branch read it the other way, so any body the sniffer bailed on (one
@@ -306,6 +334,26 @@ export function validateUpload(
   } else if (!contentTypeMatches(declared, sniffed)) {
     throw contentTypeMismatch(key, declared, sniffed);
   }
+}
+
+/**
+ * Throws the first violated constraint, in this order: key, size, type, checksum. The key comes
+ * before the size because a key nothing may store makes the other three moot, and which
+ * constraint a rejected upload reports is what the client retries on — `upload.test.ts` pins it.
+ */
+export function validateUpload(
+  candidate: UploadCandidate,
+  policy: UploadPolicy = uploadPolicy(),
+): ValidatedUpload {
+  const key = assertSafeKey(candidate.key);
+  const size = candidate.bytes.byteLength;
+  if (size > policy.maxBytes) throw tooLarge(key, size, policy.maxBytes);
+
+  const declared = normalizeContentType(candidate.declaredContentType);
+  if (!policy.allowedContentTypes.includes(declared)) {
+    throw contentTypeNotAllowed(key, declared, policy.allowedContentTypes);
+  }
+  assertTypeAgrees(key, declared, candidate.bytes, false);
 
   const checksum = sha256Base64(candidate.bytes);
   const claimed = candidate.checksum;
@@ -315,3 +363,41 @@ export function validateUpload(
   }
   return { key, contentType: declared, bytes: candidate.bytes, size, checksum };
 }
+
+export interface StoredHeadCandidate {
+  readonly key: string;
+  /** The type the object was stored under. On a provider disk, whatever the uploader sent. */
+  readonly declaredContentType: string;
+  /** The object's first bytes: `disk.get(key, { range: [0, SNIFF_WINDOW_BYTES - 1] })`. */
+  readonly head: Uint8Array;
+  /** The WHOLE object's size, which is how a head knows it is one. */
+  readonly size: number;
+}
+
+export interface ValidatedHead {
+  readonly key: string;
+  readonly contentType: string;
+  readonly size: number;
+}
+
+/**
+ * `validateUpload` for bytes already on a disk, from a head and a size, so promotion neither
+ * buffers an object nor refuses one past `maxGetBytes`. Same order, no checksum: a whole-body hash
+ * is what this exists not to need. An XML type over the window is refused here and only here, as
+ * the server-side path reads all of it.
+ */
+export function validateStoredHead(
+  candidate: StoredHeadCandidate,
+  policy: UploadPolicy = uploadPolicy(),
+): ValidatedHead {
+  const key = assertSafeKey(candidate.key);
+  const { size } = candidate;
+  if (size > policy.maxBytes) throw tooLarge(key, size, policy.maxBytes);
+
+  const declared = normalizeContentType(candidate.declaredContentType);
+  if (!policy.allowedContentTypes.includes(declared)) {
+    throw contentTypeNotAllowed(key, declared, policy.allowedContentTypes);
+  }
+  assertTypeAgrees(key, declared, candidate.head, size > candidate.head.byteLength);
+  return { key, contentType: declared, size };
+}