@crombolo/server 0.0.0-stage → 0.9.1

package/src/media-files.ts

@@ -0,0 +1,276 @@
+/**
+ * Uploaded files, as Umbraco keeps them: a temporary upload first
+ * (`TemporaryFile`, kept for a day), then, when a value that names it is saved, a
+ * placed file served at `/media/<folder>/<name>`.
+ *
+ * Where the bytes actually live is the `MediaStore`'s business (see
+ * media-store.ts) — a disk, an S3 bucket, an Azure container. This is the layer
+ * that knows the *naming*: which key a `/media/…` URL means, and that a temporary
+ * upload is `.temp/<id>/…` with its metadata beside it.
+ *
+ * Temporary uploads go to the store rather than to local disk, so a load-balanced
+ * set can place an upload that a different node received.
+ */
+import { basename, extname } from 'node:path'
+import {
+  contentTypeFor,
+  fileSystemMediaStore,
+  type MediaStore,
+  type StoredFile,
+} from './media-store.ts'
+
+export interface TemporaryFile {
+  id: string
+  fileName: string
+  availableUntil: Date
+  size: number
+}
+
+export interface PlacedFile {
+  /** The URL path, e.g. `/media/ab12cd34/photo.jpg`. */
+  src: string
+  bytes: number
+  /** Lowercase, without the dot. */
+  extension: string
+  width?: number
+  height?: number
+}
+
+const TEMP = '.temp'
+const DAY = 24 * 60 * 60 * 1000
+
+/** `My Photo (1).JPG` → `my-photo-1.jpg`: safe on every file system and in a URL. */
+export function safeFileName(name: string): string {
+  const base = basename(name.replaceAll('\\', '/'))
+  const extension = extname(base).toLowerCase()
+  const stem = base
+    .slice(0, base.length - extname(base).length)
+    .toLowerCase()
+    .normalize('NFKD')
+    .replace(/[̀-ͯ]/g, '')
+    .replace(/[^a-z0-9._-]+/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^[-.]+|[-.]+$/g, '')
+  return `${stem || 'file'}${extension.replace(/[^a-z0-9.]/g, '')}`
+}
+
+const isUuid = (id: string) =>
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)
+
+export interface MediaFileStoreOptions {
+  lifetimeMs?: number
+}
+
+export class MediaFileStore {
+  readonly store: MediaStore
+  #lifetimeMs: number
+
+  /** Takes a store, or a directory for the file system store a plain path implies. */
+  constructor(store: MediaStore | string, options: MediaFileStoreOptions = {}) {
+    this.store = typeof store === 'string' ? fileSystemMediaStore(store) : store
+    this.#lifetimeMs = options.lifetimeMs ?? DAY
+  }
+
+  #tempKey(id: string, name: string) {
+    return `${TEMP}/${id.toLowerCase()}/${name}`
+  }
+
+  async saveTemporary(id: string, file: File, now = new Date()): Promise<TemporaryFile> {
+    if (!isUuid(id)) throw new Error('A temporary file id must be a uuid.')
+    // A second upload under the same id replaces the first, including its name.
+    await this.deleteTemporary(id)
+    const fileName = safeFileName(file.name)
+    // From the name, never from `file.type`: that is the multipart part's own
+    // `Content-Type` header, so the uploader picks it. A bucket-backed store keeps
+    // what it is given and hands it back on read, which would let `photo.jpg` be
+    // served as `text/html` from this origin — past the extension checks, because
+    // the name really is a .jpg. The file system store re-derives on read and was
+    // never exposed; deriving here makes all three agree.
+    await this.store.put(this.#tempKey(id, fileName), file, {
+      contentType: contentTypeFor(fileName),
+    })
+    const record = {
+      id: id.toLowerCase(),
+      fileName,
+      availableUntil: new Date(now.getTime() + this.#lifetimeMs),
+      size: file.size,
+    }
+    await this.store.put(
+      this.#tempKey(id, 'meta.json'),
+      new TextEncoder().encode(JSON.stringify(record)),
+      { contentType: 'application/json' },
+    )
+    return record
+  }
+
+  async temporary(id: string, now = new Date()): Promise<TemporaryFile | undefined> {
+    if (!isUuid(id)) return undefined
+    const meta = await this.store.get(this.#tempKey(id, 'meta.json'))
+    if (!meta) return undefined
+    let raw: TemporaryFile & { availableUntil: string }
+    try {
+      raw = JSON.parse(new TextDecoder().decode(await meta.bytes()))
+    } catch {
+      return undefined
+    }
+    const record = { ...raw, availableUntil: new Date(raw.availableUntil) }
+    if (record.availableUntil.getTime() < now.getTime()) {
+      await this.deleteTemporary(id)
+      return undefined
+    }
+    return record
+  }
+
+  /** A temporary upload's name and contents, for uploads that are read rather than placed. */
+  async readTemporary(id: string): Promise<{ fileName: string; bytes: Uint8Array } | undefined> {
+    const record = await this.temporary(id)
+    if (!record) return undefined
+    const stored = await this.store.get(this.#tempKey(id, record.fileName))
+    if (!stored) return undefined
+    return { fileName: record.fileName, bytes: await stored.bytes() }
+  }
+
+  async deleteTemporary(id: string): Promise<boolean> {
+    if (!isUuid(id)) return false
+    const keys = await this.store.list(`${TEMP}/${id.toLowerCase()}/`)
+    if (keys.length === 0) return false
+    for (const key of keys) await this.store.delete(key)
+    return true
+  }
+
+  /** Moves a temporary upload into the media root; undefined when it is gone or expired. */
+  async place(id: string): Promise<PlacedFile | undefined> {
+    const temporary = await this.temporary(id)
+    if (!temporary) return undefined
+    const source = await this.store.get(this.#tempKey(id, temporary.fileName))
+    if (!source) return undefined
+    const bytes = await source.bytes()
+    let folder: string
+    do folder = crypto.randomUUID().replaceAll('-', '').slice(0, 8)
+    while ((await this.store.list(`${folder}/`)).length > 0)
+    const key = `${folder}/${temporary.fileName}`
+    await this.store.put(key, bytes, { contentType: source.contentType })
+    await this.deleteTemporary(id)
+    const placed: PlacedFile = {
+      src: `/media/${key}`,
+      bytes: bytes.byteLength,
+      extension: extname(temporary.fileName).slice(1).toLowerCase(),
+    }
+    try {
+      const metadata = await new Bun.Image(bytes).metadata()
+      placed.width = metadata.width
+      placed.height = metadata.height
+    } catch {
+      // Not an image Bun can decode (a PDF, an SVG): no dimensions.
+    }
+    return placed
+  }
+
+  /** The store key a `/media/…` URL path names; undefined when it names nothing. */
+  key(urlPath: string): string | undefined {
+    if (!urlPath.startsWith('/media/')) return undefined
+    let key: string
+    try {
+      key = decodeURIComponent(urlPath.slice('/media/'.length))
+    } catch {
+      return undefined
+    }
+    // A placed file is always `<folder>/<name>`; the reserved folders are not
+    // reachable through a URL, whatever the store would allow.
+    const parts = key.split('/')
+    if (parts.length < 2 || parts.some((part) => part === '' || part.startsWith('.')))
+      return undefined
+    return key
+  }
+
+  /** The file a `/media/…` URL path names, or undefined. */
+  async open(urlPath: string): Promise<StoredFile | undefined> {
+    const key = this.key(urlPath)
+    return key ? this.store.get(key) : undefined
+  }
+
+  /** Where the browser should go for a `/media/…` path, when the store serves it itself. */
+  publicUrl(urlPath: string): string | undefined {
+    const key = this.key(urlPath)
+    return key ? this.store.publicUrl?.(key) : undefined
+  }
+
+  /** Deletes a placed file. */
+  async remove(src: string): Promise<void> {
+    const key = this.key(src)
+    if (key) await this.store.delete(key)
+  }
+
+  /** Drops temporary uploads past their lifetime; returns how many went. */
+  async cleanupExpired(now = new Date()): Promise<number> {
+    const ids = new Set<string>()
+    for (const key of await this.store.list(`${TEMP}/`)) {
+      const id = key.split('/')[1]
+      if (id) ids.add(id)
+    }
+    let removed = 0
+    for (const id of ids) {
+      if (await this.temporary(id, now)) continue
+      // Reading an expired upload already drops it; this clears whatever is left,
+      // such as a folder whose metadata went missing.
+      await this.deleteTemporary(id)
+      removed += 1
+    }
+    return removed
+  }
+}
+
+/**
+ * The media store a command should use: whatever the site configured, else the
+ * environment's, else the directory on this disk. The same resolution the
+ * server does at boot, for a process that is not the server.
+ */
+export async function mediaStoreFor(config: {
+  mediaStore?: MediaStore | undefined
+  mediaDir: string
+}): Promise<MediaStore> {
+  if (config.mediaStore) return config.mediaStore
+  const { mediaStoreFromEnvironment } = await import('./media-store.ts')
+  return mediaStoreFromEnvironment(config.mediaDir)
+}
+
+/**
+ * Reads the media files a bundle wants to carry.
+ *
+ * A key the store has not got is reported rather than guessed at: the writer
+ * then marks that blob as not carried, so the manifest says what is true and
+ * the destination reports it missing instead of silently serving nothing.
+ */
+export async function readBlobs(
+  store: MediaStore,
+  keys: readonly string[],
+): Promise<{ bytes: Map<string, Uint8Array>; missing: string[] }> {
+  const bytes = new Map<string, Uint8Array>()
+  const missing: string[] = []
+  for (const key of keys) {
+    const file = await store.get(key)
+    if (!file) {
+      missing.push(key)
+      continue
+    }
+    bytes.set(key, await file.bytes())
+  }
+  return { bytes, missing }
+}
+
+/** Puts a bundle's media files into this environment's store, under their own keys. */
+export async function placeBlobs(
+  store: MediaStore,
+  blobs: ReadonlyMap<string, string>,
+): Promise<{ placed: number; bytes: number }> {
+  let placed = 0
+  let bytes = 0
+  for (const [key, path] of blobs) {
+    const file = Bun.file(path)
+    const content = new Uint8Array(await file.arrayBuffer())
+    await store.put(key, content, { contentType: contentTypeFor(key) })
+    placed += 1
+    bytes += content.length
+  }
+  return { placed, bytes }
+}

package/src/media-store-azure.ts

@@ -0,0 +1,218 @@
+/**
+ * Media in an Azure Blob Storage container.
+ *
+ * Azure has no client built into Bun, and `@azure/storage-blob` is a large
+ * dependency for four verbs, so this speaks the REST API directly. Two ways in:
+ *
+ * - **A SAS token.** Appended to every request; nothing to sign. Simplest, and
+ *   what a deployment that already issues SAS tokens should use.
+ * - **An account key.** Signed per request with Shared Key authorisation, which
+ *   is an HMAC-SHA256 over a canonical form of the request that Azure specifies
+ *   exactly — hence the deliberate ordering below, where a stray newline or a
+ *   header out of order fails with 403 and no explanation.
+ *
+ * Serving: by default the server streams the bytes, keeping a private container
+ * private. `publicUrlPrefix` redirects browsers to a CDN; a container with public
+ * read access can point it at the container URL itself.
+ */
+import { contentTypeFor, isSafeMediaKey, type MediaStore, type StoredFile } from './media-store.ts'
+
+export interface AzureMediaStoreOptions {
+  account: string
+  container: string
+  /** The account key, for Shared Key authorisation. */
+  accountKey?: string
+  /** A SAS token instead of a key; a leading `?` is optional. */
+  sasToken?: string
+  /** Override for Azurite or a sovereign cloud; default is the public endpoint. */
+  endpoint?: string
+  /** A folder inside the container. */
+  prefix?: string
+  /** A CDN or public container URL; browsers are redirected there. */
+  publicUrlPrefix?: string
+}
+
+const VERSION = '2021-08-06'
+
+/**
+ * Azure's `StringToSign` for Shared Key: the verb, thirteen fixed headers, then
+ * the canonicalised `x-ms-*` headers and the canonicalised resource. Every field
+ * is present even when empty, which is why this is a list and not a template.
+ */
+async function sharedKeyAuthorization(
+  account: string,
+  accountKey: string,
+  method: string,
+  url: URL,
+  headers: Headers,
+  contentLength: number,
+): Promise<string> {
+  const canonicalHeaders = [...headers.keys()]
+    .filter((name) => name.startsWith('x-ms-'))
+    .sort()
+    .map((name) => `${name}:${(headers.get(name) ?? '').trim()}`)
+    .join('\n')
+  const canonicalResource = [
+    `/${account}${url.pathname}`,
+    ...[...url.searchParams.keys()]
+      .sort()
+      .map((name) => `${name.toLowerCase()}:${url.searchParams.getAll(name).sort().join(',')}`),
+  ].join('\n')
+  const stringToSign = [
+    method,
+    headers.get('content-encoding') ?? '',
+    headers.get('content-language') ?? '',
+    // Azure wants an empty string for a zero-length body, not "0".
+    contentLength === 0 ? '' : String(contentLength),
+    headers.get('content-md5') ?? '',
+    headers.get('content-type') ?? '',
+    '', // Date, superseded by x-ms-date
+    '', // If-Modified-Since
+    '', // If-Match
+    '', // If-None-Match
+    '', // If-Unmodified-Since
+    '', // Range
+    canonicalHeaders,
+    canonicalResource,
+  ].join('\n')
+
+  const key = await crypto.subtle.importKey(
+    'raw',
+    Uint8Array.from(atob(accountKey), (c) => c.charCodeAt(0)),
+    { name: 'HMAC', hash: 'SHA-256' },
+    false,
+    ['sign'],
+  )
+  const signature = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(stringToSign))
+  return `SharedKey ${account}:${btoa(String.fromCharCode(...new Uint8Array(signature)))}`
+}
+
+export function azureMediaStore(options: AzureMediaStoreOptions): MediaStore {
+  if (!options.account) throw new Error('An Azure media store needs an account.')
+  if (!options.container) throw new Error('An Azure media store needs a container.')
+  if (!options.accountKey && !options.sasToken)
+    throw new Error('An Azure media store needs either an account key or a SAS token.')
+  const origin = (options.endpoint ?? `https://${options.account}.blob.core.windows.net`).replace(
+    /\/$/,
+    '',
+  )
+  const prefix = options.prefix ? `${options.prefix.replace(/^\/+|\/+$/g, '')}/` : ''
+  const sas = options.sasToken?.replace(/^\?/, '')
+  const publicPrefix = options.publicUrlPrefix?.replace(/\/$/, '')
+
+  const blobUrl = (key: string, query: Record<string, string> = {}) => {
+    const url = new URL(
+      `${origin}/${options.container}/${encodeURI(`${prefix}${key}`).replaceAll('#', '%23')}`,
+    )
+    for (const [name, value] of Object.entries(query)) url.searchParams.set(name, value)
+    return url
+  }
+
+  /** Adds the SAS token last, so it is never part of what Shared Key signs. */
+  const send = async (
+    method: string,
+    url: URL,
+    init: { body?: Uint8Array<ArrayBuffer>; headers?: Record<string, string> } = {},
+  ): Promise<Response> => {
+    const headers = new Headers(init.headers)
+    headers.set('x-ms-version', VERSION)
+    headers.set('x-ms-date', new Date().toUTCString())
+    const length = init.body?.byteLength ?? 0
+    if (options.accountKey) {
+      headers.set(
+        'authorization',
+        await sharedKeyAuthorization(
+          options.account,
+          options.accountKey,
+          method,
+          url,
+          headers,
+          length,
+        ),
+      )
+    }
+    const signed = new URL(url)
+    if (sas)
+      for (const [name, value] of new URLSearchParams(sas)) signed.searchParams.set(name, value)
+    return fetch(signed, { method, headers, body: init.body })
+  }
+
+  return {
+    description: `azure://${options.account}/${options.container}${prefix ? `/${prefix.slice(0, -1)}` : ''}`,
+
+    async get(key): Promise<StoredFile | undefined> {
+      if (!isSafeMediaKey(key)) return undefined
+      const url = blobUrl(key)
+      const head = await send('HEAD', url)
+      if (head.status === 404) return undefined
+      if (!head.ok) throw new Error(`Azure HEAD ${key} failed with ${head.status}.`)
+      return {
+        key,
+        size: Number(head.headers.get('content-length') ?? 0),
+        etag: (head.headers.get('etag') ?? '').replaceAll('"', ''),
+        contentType: head.headers.get('content-type') || contentTypeFor(key),
+        bytes: async () => {
+          const response = await send('GET', url)
+          if (!response.ok) throw new Error(`Azure GET ${key} failed with ${response.status}.`)
+          return new Uint8Array(await response.arrayBuffer())
+        },
+      }
+    },
+
+    async put(key, body, putOptions) {
+      if (!isSafeMediaKey(key)) throw new Error(`"${key}" is not a usable media key.`)
+      // Copied into a plain ArrayBuffer so `fetch` and the signature agree on its length.
+      const bytes = new Uint8Array(body instanceof File ? await body.arrayBuffer() : body)
+      const response = await send('PUT', blobUrl(key), {
+        body: bytes,
+        headers: {
+          'x-ms-blob-type': 'BlockBlob',
+          'content-type': putOptions?.contentType ?? contentTypeFor(key),
+        },
+      })
+      if (!response.ok)
+        throw new Error(`Azure PUT ${key} failed with ${response.status}: ${await response.text()}`)
+    },
+
+    async delete(key) {
+      if (!isSafeMediaKey(key)) return
+      const response = await send('DELETE', blobUrl(key))
+      if (!response.ok && response.status !== 404)
+        throw new Error(`Azure DELETE ${key} failed with ${response.status}.`)
+    },
+
+    async list(keyPrefix) {
+      const found: string[] = []
+      let marker: string | undefined
+      do {
+        const url = blobUrl('', {
+          restype: 'container',
+          comp: 'list',
+          prefix: `${prefix}${keyPrefix}`,
+        })
+        // blobUrl appends the key to the container path; listing is the container itself.
+        url.pathname = `/${options.container}`
+        if (marker) url.searchParams.set('marker', marker)
+        const response = await send('GET', url)
+        if (!response.ok) throw new Error(`Azure list failed with ${response.status}.`)
+        const xml = await response.text()
+        for (const match of xml.matchAll(/<Name>([^<]*)<\/Name>/g)) {
+          const name = decodeXmlText(match[1] ?? '')
+          if (name.startsWith(prefix)) found.push(name.slice(prefix.length))
+        }
+        marker = /<NextMarker>([^<]*)<\/NextMarker>/.exec(xml)?.[1] || undefined
+      } while (marker)
+      return found
+    },
+
+    publicUrl: publicPrefix ? (key) => `${publicPrefix}/${key}` : undefined,
+  }
+}
+
+const decodeXmlText = (value: string) =>
+  value
+    .replaceAll('&lt;', '<')
+    .replaceAll('&gt;', '>')
+    .replaceAll('&quot;', '"')
+    .replaceAll('&apos;', "'")
+    .replaceAll('&amp;', '&')

package/src/media-store-s3.ts

@@ -0,0 +1,113 @@
+/**
+ * Media in an S3 bucket, through Bun's own S3 client — so this needs no
+ * dependency, and works against anything that speaks S3: AWS, Cloudflare R2,
+ * DigitalOcean Spaces, MinIO, Backblaze B2.
+ *
+ * Credentials come from the options, else from the environment Bun's client
+ * already reads (`AWS_ACCESS_KEY_ID` and friends), so a machine with an instance
+ * role needs none configured.
+ *
+ * Serving: by default the server streams the bytes, which keeps the bucket
+ * private. `publicUrlPrefix` sends browsers to a CDN instead, and
+ * `presignSeconds` sends them to a signed bucket URL — pick one, not both.
+ */
+import { contentTypeFor, isSafeMediaKey, type MediaStore, type StoredFile } from './media-store.ts'
+
+export interface S3MediaStoreOptions {
+  bucket: string
+  region?: string
+  /** For S3-compatible services: `https://<account>.r2.cloudflarestorage.com`. */
+  endpoint?: string
+  accessKeyId?: string
+  secretAccessKey?: string
+  sessionToken?: string
+  /** A folder inside the bucket, when media shares it with something else. */
+  prefix?: string
+  /** A CDN in front of the bucket; browsers are redirected there. */
+  publicUrlPrefix?: string
+  /** Redirect browsers to a presigned URL valid for this long. */
+  presignSeconds?: number
+  /** The ACL to write objects with; omit for a private bucket. */
+  acl?: 'private' | 'public-read'
+}
+
+export function s3MediaStore(options: S3MediaStoreOptions): MediaStore {
+  if (!options.bucket) throw new Error('An S3 media store needs a bucket.')
+  const client = new Bun.S3Client({
+    bucket: options.bucket,
+    region: options.region,
+    endpoint: options.endpoint,
+    accessKeyId: options.accessKeyId,
+    secretAccessKey: options.secretAccessKey,
+    sessionToken: options.sessionToken,
+  })
+  const prefix = options.prefix ? `${options.prefix.replace(/^\/+|\/+$/g, '')}/` : ''
+  const publicPrefix = options.publicUrlPrefix?.replace(/\/$/, '')
+  const objectKey = (key: string) => `${prefix}${key}`
+
+  return {
+    description: `s3://${options.bucket}${prefix ? `/${prefix.slice(0, -1)}` : ''}`,
+
+    async get(key): Promise<StoredFile | undefined> {
+      if (!isSafeMediaKey(key)) return undefined
+      const file = client.file(objectKey(key))
+      try {
+        const stat = await file.stat()
+        return {
+          key,
+          size: stat.size,
+          // S3's ETag is the content hash for a single-part upload, and a
+          // composite for a multipart one; either way it changes with the bytes.
+          etag: stat.etag.replaceAll('"', ''),
+          contentType: stat.type || contentTypeFor(key),
+          bytes: async () => new Uint8Array(await file.arrayBuffer()),
+        }
+      } catch {
+        // A missing object is a thrown NoSuchKey, not a null stat.
+        return undefined
+      }
+    },
+
+    async put(key, body, putOptions) {
+      if (!isSafeMediaKey(key)) throw new Error(`"${key}" is not a usable media key.`)
+      await client.write(objectKey(key), body, {
+        type: putOptions?.contentType ?? contentTypeFor(key),
+        acl: options.acl,
+      })
+    },
+
+    async delete(key) {
+      if (!isSafeMediaKey(key)) return
+      // A delete of something absent succeeds in S3, which is what we want.
+      await client.delete(objectKey(key))
+    },
+
+    async list(keyPrefix) {
+      const found: string[] = []
+      let continuationToken: string | undefined
+      do {
+        const page = await client.list({
+          prefix: `${prefix}${keyPrefix}`,
+          continuationToken,
+          maxKeys: 1000,
+        })
+        for (const object of page.contents ?? [])
+          if (object.key.startsWith(prefix)) found.push(object.key.slice(prefix.length))
+        continuationToken = page.isTruncated ? page.nextContinuationToken : undefined
+      } while (continuationToken)
+      return found
+    },
+
+    publicUrl:
+      publicPrefix || options.presignSeconds
+        ? (key) => {
+            if (!isSafeMediaKey(key)) return undefined
+            if (publicPrefix) return `${publicPrefix}/${key}`
+            return client.presign(objectKey(key), {
+              method: 'GET',
+              expiresIn: options.presignSeconds,
+            })
+          }
+        : undefined,
+  }
+}