@12-apps/storage 1.0.0

package/src/server/create-api-storage.ts

@@ -0,0 +1,221 @@
+import { ACCEPTED_CONTENT_TYPES } from '../content-types';
+import { DEFAULT_KEY_PREFIX } from '../keys';
+import { megabytes } from '../limits';
+import { defaultStorageMessages, type StorageMessages } from '../problems';
+import { CATALOG_RENDITIONS, type RenditionSpec } from '../renditions';
+import { imageSources, objectUrl, versionedObjectUrl, type ImageSources } from '../urls';
+import { inlineImageSchema, objectKeySchema, type InlineImage } from '../wire';
+import type { StorageDriver } from './driver';
+import type { ImagePipeline } from './pipeline/port';
+import {
+  deleteIfOrphaned,
+  deleteReplaced,
+  discardMinted,
+  objectKeysFor,
+  type ReclaimDeps,
+  type StorageLogger,
+  type StorageReferenceProbe,
+} from './reclaim';
+import { storageRoutes, type StorageRoute } from './routes';
+import { storeImage, storeInlineImage, type StoreImageDeps } from './store-image';
+
+/**
+ * `createApiStorage(config)` — the ONE thing a backend host mounts.
+ *
+ * Everything the upload system IS lives behind it: the endpoint, the streaming
+ * cap, the magic-number check, the image pipeline, the key grammar, the rendition
+ * set, the URL builder and the orphan reclaim. The host supplies config only —
+ * where bytes live, who is calling, which of its tables can pin an object, and
+ * the ceiling.
+ *
+ * ## Nothing here defaults to a value only one deployment can survive
+ *
+ * `driver`, `maxBytes`, `imagePipeline`, `unscopedKeys` and `references` are all
+ * REQUIRED, and each for the same reason: the safe value is host-specific, so a
+ * default would be a decision made by whoever typed the first host, silently
+ * inherited by every one after it. A defaulted driver writes to a container's
+ * ephemeral disk in production and reports success. A defaulted ceiling is
+ * whatever number this file happens to carry. A defaulted pipeline decides
+ * whether crops exist at all. A defaulted `unscopedKeys` decides whether one
+ * tenant's reclaim can reach another tenant's bytes. A defaulted `references`
+ * decides whether a restored row still has its photo.
+ *
+ * `references` joined that list last, and it is the one worth explaining: it DID
+ * default, to `[]`, and `[]` is not the absence of a decision — it means *reclaim
+ * everything immediately*. Two fail-open defaults in this series became findings
+ * before this one was noticed, and the argument the other four are made of applies
+ * to it verbatim. `logger` still defaults, and that is a different case: a reclaim
+ * can only report, so the worst a defaulted logger costs is silence. A defaulted
+ * `references` DESTROYS.
+ */
+
+export interface ApiStorageConfig {
+  /** Where bytes live. See `./drivers/local-disk` and `@12-apps/storage/s3`. */
+  driver: StorageDriver;
+  /**
+   * The upload ceiling, in bytes. There is exactly one number: pass the same
+   * value to any tool schema advertising it, or better, read it back off
+   * {@link ApiStorage.limits} so the two cannot drift.
+   */
+  maxBytes: number;
+  /** How bytes are downscaled and cropped. `passthroughImagePipeline()` opts out. */
+  imagePipeline: ImagePipeline;
+  /**
+   * Whether a key with NO scope segment may be touched by a scoped caller.
+   *
+   * `'accept'` for a host whose existing rows predate the scope segment — such a
+   * key carries no tenant, so only `references` stands between one tenant and
+   * another's objects, and a host choosing it is saying it has those probes.
+   * `'reject'` for a fresh host: every key it will ever mint is scoped, so a key
+   * without a scope can only be someone else's or nobody's.
+   */
+  unscopedKeys: 'accept' | 'reject';
+  /**
+   * The host tables that can still need an object after the row stopped pointing
+   * at it — a duplicated row sharing one photo, a soft-deleted row whose "restore"
+   * is expected to bring it back, a draft or a pending approval holding a key.
+   *
+   * REQUIRED, and the fifth knob for the same reason as the other four. This one
+   * used to default to `[]`, which is not a neutral default: it means *reclaim
+   * everything immediately*. An adopter wired the mount, got working uploads and
+   * shipped; weeks later a store owner restored a product from the recycle bin and
+   * the row came back with its photo gone, because the write that replaced the
+   * photo had correctly found no probe claiming the old key. Nothing logged
+   * anything — the reclaim did exactly what it was told.
+   *
+   * Pass `[]` to mean it: nothing in this host copies a key, so a replaced object
+   * is safe to delete at once. Written out, by a host that has checked, rather than
+   * inherited from whoever typed the first mount.
+   */
+  references: readonly StorageReferenceProbe[];
+  /** Where a best-effort reclaim's failures go. Default: no-op, and say so. */
+  logger?: StorageLogger;
+  /** The derived sizes to cut. Default {@link CATALOG_RENDITIONS}. */
+  renditions?: readonly RenditionSpec[];
+  /** Top-level key segment. Default `products`. */
+  keyPrefix?: string;
+  /** pt-BR refusal copy overrides. */
+  messages?: Partial<StorageMessages>;
+}
+
+/** What the mount reports about itself — the anti-drift surface. */
+export interface StorageLimits {
+  maxBytes: number;
+  /** Formatted for a message a store owner reads (`8 MB`). */
+  maxBytesLabel: string;
+  contentTypes: readonly string[];
+  driver: string;
+  pipeline: string;
+  /** Whether this mount cuts crops at all — false on a passthrough pipeline. */
+  cutsRenditions: boolean;
+}
+
+export interface ApiStorage {
+  routes: readonly StorageRoute[];
+  limits: StorageLimits;
+  /** Store raw bytes (the browser's path, and any host write holding bytes). */
+  storeImage(input: { bytes: Uint8Array; contentType: string; scope: string }): Promise<string>;
+  /** Store base64 bytes carried inside a write body — an agent's only path. */
+  storeInlineImage(image: InlineImage, scope: string): Promise<string>;
+  /**
+   * Schemas built from THIS mount's own config, for a host's write bodies and tool
+   * definitions — `inlineImage`'s base64 ceiling IS `maxBytes`, and `objectKey`'s
+   * grammar IS `keyPrefix`. Read them off the mount so neither can advertise
+   * something the endpoint does not enforce.
+   */
+  schemas: {
+    inlineImage: ReturnType<typeof inlineImageSchema>;
+    /**
+     * For the `imageKey` field beside it. `z.string()` there accepts an absolute
+     * URL, which `objectUrl` renders verbatim into an `<img src>` — see
+     * {@link objectKeySchema}.
+     */
+    objectKey: ReturnType<typeof objectKeySchema>;
+  };
+  /** Resolve a stored key for display, through the active driver. */
+  urls: {
+    objectUrl(key: string | null): string | null;
+    imageSources(key: string | null): ImageSources | null;
+    versionedObjectUrl(key: string | null, version: string | null): string | null;
+  };
+  /** Reclaiming bytes nothing points at any more. Never throws. */
+  reclaim: {
+    deleteIfOrphaned(scope: string, key: string | null | undefined): Promise<void>;
+    deleteReplaced(
+      scope: string,
+      before: readonly (string | null | undefined)[],
+      after: readonly (string | null | undefined)[],
+    ): Promise<void>;
+    discardMinted(scope: string, keys: readonly (string | null | undefined)[]): Promise<void>;
+    /** Every object key a stored photo occupies — the key plus its crops. */
+    objectKeysFor(key: string): string[];
+  };
+}
+
+/**
+ * A logger that drops everything, and the only default in this file.
+ *
+ * A reclaim's failures are the one thing here that must not be raised — the write
+ * it followed already succeeded — so they can only be reported. A host that
+ * passes nothing is choosing not to hear about a bucket that has started refusing
+ * deletes; that is a legitimate choice for a test and a bad one in production,
+ * and it is stated here rather than hidden behind a `console` call that a server
+ * logger would not pick up anyway.
+ */
+const SILENT: StorageLogger = { error: () => undefined };
+
+export function createApiStorage(config: ApiStorageConfig): ApiStorage {
+  const specs = config.renditions ?? CATALOG_RENDITIONS;
+  const keyPrefix = config.keyPrefix ?? DEFAULT_KEY_PREFIX;
+  const messages: StorageMessages = {
+    ...defaultStorageMessages({ limit: megabytes(config.maxBytes) }),
+    ...config.messages,
+  };
+  const writeDeps: StoreImageDeps = {
+    driver: config.driver,
+    pipeline: config.imagePipeline,
+    messages,
+    maxBytes: config.maxBytes,
+    specs,
+    keyPrefix,
+  };
+  const reclaimDeps: ReclaimDeps = {
+    driver: config.driver,
+    probes: config.references,
+    logger: config.logger ?? SILENT,
+    unscopedKeys: config.unscopedKeys,
+    specs,
+    keyPrefix,
+  };
+  const resolve = (key: string): string => config.driver.publicUrl(key);
+
+  return {
+    routes: storageRoutes(writeDeps),
+    limits: {
+      maxBytes: config.maxBytes,
+      maxBytesLabel: megabytes(config.maxBytes),
+      contentTypes: ACCEPTED_CONTENT_TYPES,
+      driver: config.driver.name,
+      pipeline: config.imagePipeline.name,
+      cutsRenditions: config.imagePipeline.cutsRenditions,
+    },
+    storeImage: (input) => storeImage(writeDeps, input),
+    storeInlineImage: (image, scope) => storeInlineImage(writeDeps, image, scope),
+    schemas: {
+      inlineImage: inlineImageSchema(config.maxBytes),
+      objectKey: objectKeySchema(keyPrefix),
+    },
+    urls: {
+      objectUrl: (key) => objectUrl(key, resolve),
+      imageSources: (key) => imageSources(key, resolve, { specs, prefix: keyPrefix }),
+      versionedObjectUrl: (key, version) => versionedObjectUrl(key, version, resolve),
+    },
+    reclaim: {
+      deleteIfOrphaned: (scope, key) => deleteIfOrphaned(reclaimDeps, scope, key),
+      deleteReplaced: (scope, before, after) =>
+        deleteReplaced(reclaimDeps, scope, before, after),
+      discardMinted: (scope, keys) => discardMinted(reclaimDeps, scope, keys),
+      objectKeysFor: (key) => objectKeysFor(key, specs, keyPrefix),
+    },
+  };
+}

package/src/server/driver.ts

@@ -0,0 +1,43 @@
+/**
+ * The DRIVER port: the only thing the rest of the package knows about where
+ * bytes live.
+ *
+ * Three operations, and a fourth that is deliberately optional:
+ *
+ *   - `put`    — store bytes at a key, world-readable, FROM THE SERVER. The only
+ *                way anything is ever written; there is no address a driver can
+ *                hand a browser to PUT at, which is what makes the CORS class of
+ *                failure described in the root entry unreachable rather than
+ *                merely avoided.
+ *   - `delete` — remove an object. Deleting one that is already gone is SUCCESS:
+ *                the caller is reconciling "nothing references this key any
+ *                more", and an absent object satisfies that.
+ *   - `publicUrl` — a stored key as something a browser can load. Synchronous,
+ *                because it is called during render.
+ *   - `read`   — OPTIONAL, and its presence is what tells the serve route which
+ *                of its two behaviours applies. A driver that keeps objects on
+ *                the app server's own disk implements it and the route streams
+ *                bytes; a bucket-backed driver does not, and the route redirects
+ *                to `publicUrl` instead. That used to be an environment flag
+ *                read inside the route, which is the same fact stated somewhere
+ *                it could disagree with the driver actually in use.
+ *
+ * A second vendor is a second factory satisfying this interface plus a config
+ * entry naming it — no host code, and nothing in the upload path changes.
+ */
+
+/** An object read back from a driver that holds its own bytes. */
+export interface StoredObject {
+  bytes: Uint8Array;
+  contentType: string;
+}
+
+export interface StorageDriver {
+  /** Which driver this is, for log lines and for the mount's own reporting. */
+  readonly name: string;
+  put(key: string, bytes: Uint8Array, contentType: string): Promise<void>;
+  delete(key: string): Promise<void>;
+  publicUrl(key: string): string;
+  /** Present only on drivers whose objects the app server can serve itself. */
+  read?(key: string): Promise<StoredObject | null>;
+}

package/src/server/drivers/local-disk.ts

@@ -0,0 +1,104 @@
+import { mkdir, readFile, rm, rmdir, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import { CONTENT_TYPE_BY_EXTENSION } from '../../content-types';
+import type { StorageDriver, StoredObject } from '../driver';
+
+/**
+ * Objects on the app server's OWN DISK — the development driver, and the one
+ * that needs no account, no bucket and no credentials.
+ *
+ * `root` is REQUIRED and has no default on purpose. `<cwd>/.uploads` is a fine
+ * development value and a silent disaster in production: a container's disk is
+ * ephemeral, so an app that fell back to it would report every upload as a
+ * success and lose the objects at the next deploy — with nothing in any log to
+ * say so. A host that wants the development default writes it in its own config,
+ * where the choice is visible and reviewable.
+ *
+ * `publicPathPrefix` is where the serve route is mounted, so a stored key can be
+ * turned into a URL on this same origin. One definition of both, so a deployment
+ * cannot write objects to one place and serve them from another.
+ */
+export interface LocalDiskDriverConfig {
+  /** Absolute path of the directory objects live under. */
+  root: string;
+  /** Path the serve route is mounted at, e.g. `/api/uploads/local`. */
+  publicPathPrefix: string;
+}
+
+/** A key segment safe to use as a path component. */
+const SAFE_SEGMENT = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
+
+/**
+ * Absolute path for a stored object key under `root`.
+ *
+ * Keys reaching here are server-minted, so the segment check is a backstop
+ * rather than the validation of an untrusted key — the serve route validates its
+ * caller-supplied path itself, in more detail, before it ever builds one. It
+ * throws rather than returning null because there is no legitimate caller that
+ * can produce a traversal segment: one here would mean a key was assembled
+ * somewhere it should not have been.
+ */
+export function localObjectPath(root: string, key: string): string {
+  const segments = key.split('/');
+  if (!segments.every((segment) => SAFE_SEGMENT.test(segment))) {
+    throw new Error(`Refusing to resolve an unsafe object key: ${key}`);
+  }
+  return path.join(root, ...segments);
+}
+
+/** `full.webp` → `image/webp`; an unknown extension → a generic type. */
+function contentTypeOfPath(filePath: string): string {
+  const extension = path.extname(filePath).replace('.', '').toLowerCase();
+  return CONTENT_TYPE_BY_EXTENSION[extension] ?? 'application/octet-stream';
+}
+
+/**
+ * Remove the object at `filePath`, then prune the directory it was the last
+ * member of.
+ *
+ * A photo stored with its crops is six files in a directory of its own, so
+ * reclaiming one leaves that directory behind, empty, once the last file goes.
+ * A bucket has no such thing — a key's slashes are just characters — so the
+ * leftover is purely an artifact of this driver, and pruning it keeps the root a
+ * truthful picture of what is stored. `rmdir` on a directory that still holds a
+ * sibling fails, which IS the check: no listing, and no race between two
+ * concurrent reclaims.
+ */
+async function removeAndPrune(root: string, filePath: string): Promise<void> {
+  await rm(filePath, { force: true });
+  const parent = path.dirname(filePath);
+  if (path.resolve(parent) === path.resolve(root)) return;
+  try {
+    await rmdir(parent);
+  } catch {
+    // Not empty, or already gone. Either way there is nothing to clean up.
+  }
+}
+
+export function createLocalDiskDriver(config: LocalDiskDriverConfig): StorageDriver {
+  const prefix = config.publicPathPrefix.replace(/\/$/, '');
+  return {
+    name: 'local-disk',
+    put: async (key, bytes) => {
+      const filePath = localObjectPath(config.root, key);
+      await mkdir(path.dirname(filePath), { recursive: true });
+      await writeFile(filePath, bytes);
+    },
+    delete: async (key) => {
+      await removeAndPrune(config.root, localObjectPath(config.root, key));
+    },
+    publicUrl: (key) => `${prefix}/${key}`,
+    read: async (key): Promise<StoredObject | null> => {
+      const filePath = localObjectPath(config.root, key);
+      try {
+        // `stat` first, so a directory whose name happens to match a key reads
+        // as absent rather than as an unreadable object.
+        if (!(await stat(filePath)).isFile()) return null;
+        return { bytes: new Uint8Array(await readFile(filePath)), contentType: contentTypeOfPath(filePath) };
+      } catch {
+        return null;
+      }
+    },
+  };
+}

package/src/server/index.ts

@@ -0,0 +1,61 @@
+/**
+ * `@12-apps/storage/server` — the backend half.
+ *
+ * One factory, {@link createApiStorage}, plus the ports it is configured with and
+ * the implementations the package ships for them. Nothing here reads
+ * `process.env`: every deployment-shaped decision is an argument, which is what
+ * makes the same mount correct in a test, on a laptop and in production.
+ *
+ * The S3-compatible driver lives behind `@12-apps/storage/s3` instead of here, so
+ * a host on local disk never resolves the AWS SDK — the same arrangement `hono`
+ * has.
+ */
+
+export {
+  createApiStorage,
+  type ApiStorage,
+  type ApiStorageConfig,
+  type StorageLimits,
+} from './create-api-storage';
+
+export type { StorageDriver, StoredObject } from './driver';
+
+export {
+  createLocalDiskDriver,
+  localObjectPath,
+  type LocalDiskDriverConfig,
+} from './drivers/local-disk';
+
+export {
+  assertRemoteUrlConfig,
+  remoteObjectUrl,
+  type RemoteObjectUrlConfig,
+} from './remote-url';
+
+export { passthroughImagePipeline } from './pipeline/passthrough';
+export { createSharpImagePipeline, type SharpPipelineConfig } from './pipeline/sharp';
+export type {
+  ImagePipeline,
+  ProcessResult,
+  ProcessedImage,
+} from './pipeline/port';
+export type {
+  Rgba,
+  SharpImage,
+  SharpMetadata,
+  SharpModule,
+  SharpResizeOptions,
+} from './pipeline/sharp-module';
+
+export { readBodyCapped } from './read-body';
+
+export type { StorageLogger, StorageReferenceProbe } from './reclaim';
+
+export { STORAGE_PATHS } from '../paths';
+
+export {
+  type StorageActor,
+  type StorageRoute,
+  type StorageRouteContext,
+  type StorageRouteResponse,
+} from './routes';

package/src/server/pipeline/passthrough.ts

@@ -0,0 +1,28 @@
+import { verifyImageBytes } from '../../payload';
+import type { ImagePipeline, ProcessResult } from './port';
+
+/**
+ * Store the bytes as they arrived: no downscale, no re-encode, no crops.
+ *
+ * The pipeline for a host that will not take a native dependency. It is not a
+ * degraded mode so much as the behaviour every catalog image had before crops
+ * existed: one object, drawn `contain`, and a key whose shape says truthfully
+ * that there are no crops to ask for.
+ *
+ * What it still does is verify: the magic-number check is what stops an upload
+ * surface being a way to park arbitrary content at a world-readable URL on the
+ * store's own domain, and that is not the resizer's job to provide.
+ */
+export function passthroughImagePipeline(): ImagePipeline {
+  return {
+    name: 'passthrough',
+    cutsRenditions: false,
+    process: async (bytes, declaredContentType): Promise<ProcessResult> => {
+      const problem = verifyImageBytes(bytes, declaredContentType);
+      return problem
+        ? { ok: false, problem }
+        : { ok: true, image: { bytes, contentType: declaredContentType } };
+    },
+    cut: async () => [],
+  };
+}

package/src/server/pipeline/port.ts

@@ -0,0 +1,59 @@
+import type { RenditionOutput, RenditionSpec } from '../../renditions';
+import type { StorageProblem } from '../../problems';
+
+/**
+ * The IMAGE PIPELINE port — how bytes are downscaled, re-encoded and cropped.
+ *
+ * It is a port rather than a dependency because image processing means a NATIVE
+ * module, and a native module is build surface every consumer of this package
+ * would pay for on every deploy whether or not it wants renditions. A host that
+ * has `sharp` passes `createSharpImagePipeline({ sharp })` and gets the whole
+ * behaviour; a host that does not passes {@link passthroughImagePipeline} and
+ * gets uploads with no crops, which is exactly what every catalog image was
+ * before crops existed. Neither choice is a default: `imagePipeline` is required
+ * config, because "which one am I running?" is not a question a host should have
+ * to discover from the shape of a key in production.
+ *
+ * The two operations are independent on purpose, and the writer runs them
+ * together: cutting crops from the ORIGINAL bytes rather than from the
+ * downscaled object avoids resampling a 1280px intermediate down to a 320px
+ * card, and on a large photo the two are the whole cost of the request.
+ */
+
+/** What the pipeline decided the uncropped object should be. */
+export interface ProcessedImage {
+  bytes: Uint8Array;
+  /** The type the object is STORED as — the extension is minted from it. */
+  contentType: string;
+}
+
+export type ProcessResult =
+  | { ok: true; image: ProcessedImage }
+  | { ok: false; problem: StorageProblem };
+
+export interface ImagePipeline {
+  /** A name for the mount's own reporting. */
+  readonly name: string;
+  /**
+   * Whether this pipeline cuts crops at all. Stated rather than inferred: a
+   * consumer deciding what to draw needs to know before the first upload, and
+   * "call it and see" is not an answer a host can put in a health check.
+   */
+  readonly cutsRenditions: boolean;
+  /**
+   * The uncropped object to store. NEVER throws: a decode failure is reported as
+   * a problem, because the caller is a write that needs to answer with a status
+   * rather than a stack trace.
+   */
+  process(bytes: Uint8Array, declaredContentType: string): Promise<ProcessResult>;
+  /**
+   * Cut every requested size, or return an EMPTY list.
+   *
+   * Empty is a first-class answer and the writer reads it as "this photo has no
+   * crops", which is what routes it to the flat key shape. Never throws, and
+   * never returns a PARTIAL set: a browser handed a key whose shape promises
+   * five objects would draw the broken-image glyph for the ones that are missing,
+   * so one size failing drops the four that worked.
+   */
+  cut(bytes: Uint8Array, specs: readonly RenditionSpec[]): Promise<RenditionOutput[]>;
+}

package/src/server/pipeline/sharp-module.ts

@@ -0,0 +1,57 @@
+/**
+ * The slice of `sharp`'s API this package uses, described STRUCTURALLY.
+ *
+ * The pipeline takes the module as an argument instead of importing it, so
+ * `sharp` is neither a dependency nor a peer of this package: a host that wants
+ * renditions passes the module it already has, and every other consumer installs
+ * nothing. Describing the surface here rather than importing sharp's types keeps
+ * that true for the type-checker too — a package that imported the types would
+ * make `@types/sharp` a hard requirement of compiling it.
+ *
+ * `sharp`'s own `Sharp` interface is wider than this in every direction, so the
+ * real module satisfies it by structural width subtyping; nothing here needs to
+ * be kept in step with sharp's version except when this file starts calling
+ * something new.
+ */
+
+/** An RGBA colour, in the shape sharp's `background` option takes. */
+export interface Rgba {
+  r: number;
+  g: number;
+  b: number;
+  alpha: number;
+}
+
+export interface SharpMetadata {
+  width?: number | undefined;
+  height?: number | undefined;
+  /** Frames in an animation. 1 (or absent) for a still. */
+  pages?: number | undefined;
+}
+
+export interface SharpResizeOptions {
+  width?: number | undefined;
+  height?: number | undefined;
+  fit?: 'inside' | 'contain' | undefined;
+  background?: Rgba | undefined;
+  withoutEnlargement?: boolean | undefined;
+}
+
+export interface SharpImage {
+  metadata(): Promise<SharpMetadata>;
+  resize(options: SharpResizeOptions): SharpImage;
+  extract(region: { left: number; top: number; width: number; height: number }): SharpImage;
+  ensureAlpha(): SharpImage;
+  raw(): SharpImage;
+  trim(options: { background: Rgba; threshold: number }): SharpImage;
+  png(): SharpImage;
+  webp(options: { quality: number }): SharpImage;
+  toFormat(format: string): SharpImage;
+  toBuffer(): Promise<Uint8Array>;
+}
+
+/** `sharp(bytes, { animated })`. */
+export type SharpModule = (
+  input: Uint8Array,
+  options?: { animated?: boolean },
+) => SharpImage;