@ultimabase/files 0.0.0-stage → 0.1.0

package/src/upload.ts

@@ -0,0 +1,282 @@
+// An upload (D-152): its field, its size and its type checked, its pending
+// `FilesBlob` row inserted as its caller, then its object stored. When it
+// fails, its objects and its rows are deleted.
+import { randomUUID } from 'node:crypto'
+import { fileTypeFromBuffer } from 'file-type'
+import type { Identity, JsonValue } from 'ultimabase'
+import { isMaster, PublicError } from 'ultimabase'
+import type { FileInput, FileOptions, Inline } from './field.ts'
+import { defaultMaxSize } from './field.ts'
+import { bytesOf } from './memory.ts'
+import { FilesBlob, hour, M } from './model.ts'
+import type { FileBody } from './storage.ts'
+import { settled } from './storage.ts'
+import { runningOf } from './values.ts'
+
+/** What an upload says of its file. */
+export interface UploadOptions {
+	/** The field the file is for: `Post.cover`. */
+	readonly for: string
+	readonly name: string
+	/** The declared `Content-Type`, read only when the bytes say none. */
+	readonly type?: string | undefined
+	readonly as?: Identity | undefined
+}
+
+/** How many first bytes the type is read from, as `file-type` reads them. */
+const sampleSize = 4100
+/** An inline value's JSON, at most. */
+const inlineSize = 2048
+
+const invalid = (path: string, message: string): PublicError =>
+	new PublicError('VALIDATION_FAILED', 'Invalid upload.', {
+		details: { issues: [{ path: [path], message }] },
+	})
+
+function fieldOf(target: string): FileOptions {
+	const [model = '', field = ''] = target.split('.')
+	const options = runningOf().fields.get(model)?.get(field)
+	if (options === undefined) {
+		const message =
+			target === ''
+				? 'Expected a file field: Model.field.'
+				: `${target} is not a file field.`
+		throw invalid('for', message)
+	}
+	return options
+}
+
+function concat(chunks: readonly Uint8Array[], length: number): Uint8Array {
+	const bytes = new Uint8Array(length)
+	let at = 0
+	for (const chunk of chunks) {
+		bytes.set(chunk, at)
+		at += chunk.length
+	}
+	return bytes
+}
+
+/** The first bytes of a body, and the body to store, whole again. */
+async function sampled(
+	body: FileBody,
+): Promise<{ readonly sample: Uint8Array; readonly body: FileBody }> {
+	if (body instanceof Uint8Array) return { sample: body, body }
+	const reader = body.getReader()
+	const chunks: Uint8Array[] = []
+	let length = 0
+	while (length < sampleSize) {
+		const { done, value } = await reader.read()
+		if (done) {
+			const whole = concat(chunks, length)
+			return { sample: whole, body: whole }
+		}
+		chunks.push(value)
+		length += value.length
+	}
+	const sample = concat(chunks, length)
+	const rest = new ReadableStream<Uint8Array>({
+		start: (controller) => {
+			controller.enqueue(sample)
+		},
+		pull: async (controller) => {
+			const { done, value } = await reader.read()
+			if (done) controller.close()
+			else controller.enqueue(value)
+		},
+		cancel: (reason) => reader.cancel(reason),
+	})
+	return { sample, body: rest }
+}
+
+const matches = (accept: readonly string[], type: string): boolean =>
+	accept.some((pattern) =>
+		pattern.endsWith('/*')
+			? type.startsWith(pattern.slice(0, -1))
+			: pattern === type,
+	)
+
+/** The type a file is stored with, and whether it is served as a download. */
+async function typeOf(
+	sample: Uint8Array,
+	declared: string | undefined,
+	accept: readonly string[] | undefined,
+): Promise<{ readonly type: string; readonly attachment: boolean }> {
+	const detected = (await fileTypeFromBuffer(sample))?.mime
+	if (
+		detected !== undefined &&
+		(accept === undefined || matches(accept, detected))
+	) {
+		return { type: detected, attachment: detected === 'image/svg+xml' }
+	}
+	const [media = ''] = (declared ?? '').split(';')
+	const type = media.trim().toLowerCase()
+	if (detected === undefined && accept === undefined) {
+		return {
+			type: type === '' ? 'application/octet-stream' : type,
+			attachment: true,
+		}
+	}
+	// An undetected type is taken as declared only when accept lists it.
+	if (detected === undefined && accept?.includes(type) === true) {
+		return { type, attachment: true }
+	}
+	throw new PublicError(
+		'UNSUPPORTED_MEDIA_TYPE',
+		`${detected ?? (type === '' ? 'An undetected type' : type)} is not accepted.`,
+	)
+}
+
+/** A variant's own type, served as a download unless its bytes say it. */
+async function attachmentOf(body: Uint8Array, type: string): Promise<boolean> {
+	const detected = (await fileTypeFromBuffer(body))?.mime
+	return detected !== type || type === 'image/svg+xml'
+}
+
+async function inlineValue(
+	key: string,
+	given: Inline,
+	file: FileInput,
+): Promise<JsonValue> {
+	const run = typeof given === 'function' ? given : given.run
+	let value: unknown = await run(file)
+	if (typeof given !== 'function') {
+		const result = await given.schema['~standard'].validate(value)
+		if (result.issues !== undefined) {
+			throw new TypeError(`The inline ${key} does not match its schema.`)
+		}
+		value = result.value
+	}
+	const json = JSON.stringify(value)
+	if (
+		json === undefined ||
+		new TextEncoder().encode(json).length > inlineSize
+	) {
+		throw new TypeError(`The inline ${key} is not JSON of 2 KB at most.`)
+	}
+	// oxlint-disable-next-line typescript/no-unsafe-type-assertion -- JSON.parse gives JSON.
+	return JSON.parse(json) as JsonValue
+}
+
+const uploaderOf = (caller: Identity): string | null =>
+	isMaster(caller) ? 'master' : caller.kind === 'user' ? caller.user.id : null
+
+/** What a buffered upload made: its inline values and its variants. */
+async function madeOf(
+	options: FileOptions,
+	file: FileInput,
+): Promise<{
+	readonly inline: { readonly [key: string]: JsonValue }
+	readonly variants: readonly (readonly [string, Uint8Array, string])[]
+}> {
+	const inline = Object.entries(options.inline ?? {}).map(
+		async ([key, given]) => [key, await inlineValue(key, given, file)] as const,
+	)
+	const variants = Object.entries(options.variants ?? {}).map(
+		async ([key, variant]) => {
+			const { body, type } = await variant(file)
+			return [key, body, type] as const
+		},
+	)
+	const [values, made] = await Promise.all([
+		Promise.all(inline),
+		Promise.all(variants),
+	])
+	return { inline: Object.fromEntries(values), variants: made }
+}
+
+/**
+ * Stores a file for `options.for`, as `POST /files` does, and gives its id,
+ * which a write of that field takes. `size` is the body's length in bytes.
+ */
+export async function upload(
+	body: FileBody,
+	size: number,
+	options: UploadOptions,
+): Promise<{ readonly id: string }> {
+	const { name, as: caller = { kind: 'anonymous' } } = options
+	const field = fieldOf(options.for)
+	if (name === '') throw invalid('name', 'Expected a file name.')
+	const maxSize = field.maxSize ?? defaultMaxSize
+	if (size > maxSize) {
+		throw new PublicError(
+			'PAYLOAD_TOO_LARGE',
+			`The file is over ${maxSize} bytes.`,
+		)
+	}
+	const buffered = field.inline !== undefined || field.variants !== undefined
+	const read = buffered
+		? { sample: await bytesOf(body), body: undefined }
+		: await sampled(body)
+	const { type, attachment } = await typeOf(
+		read.sample,
+		options.type,
+		field.accept,
+	)
+	const { storage, logger } = runningOf()
+	const prefix = field.public === true ? 'public' : 'private'
+	const keyOf = () => `${prefix}/${randomUUID()}`
+	const uploadedBy = uploaderOf(caller)
+	const deleteAfter = new Date(Date.now() + hour)
+	const blob = { field: options.for, uploadedBy, state: 'pending', deleteAfter }
+	const main = await FilesBlob.create(
+		{ data: { ...blob, key: keyOf(), name, type, size } },
+		{ as: caller },
+	)
+	const keys = [main.key]
+	const ids = [main.id]
+	try {
+		if (read.body !== undefined) {
+			await storage.put(main.key, read.body, { type, size, attachment })
+			return { id: main.id }
+		}
+		const bytes = read.sample
+		if (bytes.length !== size) {
+			throw invalid('size', `The file has ${bytes.length} bytes, not ${size}.`)
+		}
+		const made = await madeOf(field, { body: bytes, type, name })
+		const variants = await settled(
+			made.variants.map(async ([key, variantBody, variantType]) => {
+				const row = await FilesBlob.create(
+					{
+						data: {
+							...blob,
+							key: keyOf(),
+							name: key,
+							type: variantType,
+							size: variantBody.length,
+							variantOf: main.id,
+						},
+					},
+					M,
+				)
+				keys.push(row.key)
+				ids.push(row.id)
+				return { row, body: variantBody }
+			}),
+		)
+		await settled([
+			storage.put(main.key, bytes, { type, size, attachment }),
+			...variants.map(async ({ row, body: variantBody }) =>
+				storage.put(row.key, variantBody, {
+					type: row.type,
+					size: row.size,
+					attachment: await attachmentOf(variantBody, row.type),
+				}),
+			),
+		])
+		await FilesBlob.update(
+			{ where: { id: main.id }, data: { inline: made.inline } },
+			M,
+		)
+		return { id: main.id }
+	} catch (error) {
+		// The sweep deletes what this cannot: the rows expire within the hour.
+		try {
+			await Promise.all(keys.map((key) => storage.delete(key)))
+			await FilesBlob.deleteMany({ where: { id: { in: ids } } }, M)
+		} catch (cleanup) {
+			logger.warn({ err: cleanup }, 'A failed upload was not cleaned up.')
+		}
+		throw error
+	}
+}

package/src/values.ts

@@ -0,0 +1,213 @@
+// A file field's values (D-152): the copy of each file's metadata the row
+// stores, the value code reads, and the ids a write takes until the plugin's
+// hooks resolve them.
+import type { CallerRule, JsonValue, Logger } from 'ultimabase'
+import type { FileOptions } from './field.ts'
+import type { FileStorage } from './storage.ts'
+
+/** A variant as a client reads it. */
+export interface FileVariant {
+	readonly type: string
+	readonly size: number
+	readonly url: string
+}
+
+/** What a file field's value holds. */
+export interface FileValue<
+	I extends object = { readonly [key: string]: JsonValue | null },
+	V extends object = { readonly [key: string]: FileVariant | null },
+> {
+	readonly id: string
+	readonly name: string
+	readonly type: string
+	readonly size: number
+	/** Stable for a public file, presigned for one hour for a private one. */
+	readonly url: string
+	readonly inline: I
+	readonly variants: V
+}
+
+/** The copy of a variant's metadata the row stores. */
+export interface StoredVariant {
+	readonly key: string
+	readonly type: string
+	readonly size: number
+}
+
+/** The copy of a file's metadata the row stores; `FilesBlob` is the truth. */
+export interface StoredFile {
+	readonly id: string
+	readonly key: string
+	readonly name: string
+	readonly type: string
+	readonly size: number
+	readonly inline: { readonly [key: string]: JsonValue }
+	readonly variants: { readonly [key: string]: StoredVariant }
+}
+
+/** What a field's options say of its value's keys. */
+export interface Keys {
+	readonly inline: readonly string[]
+	readonly variants: readonly string[]
+}
+
+/** How a write changes a field's files, before the hooks resolve it. */
+export type Operation = 'set' | 'push' | 'pushUnique' | 'remove'
+
+/** The ids a write gives, which the plugin's hooks resolve to files. */
+export class Pending {
+	readonly operation: Operation
+	readonly ids: readonly string[]
+	constructor(operation: Operation, ids: readonly string[]) {
+		this.operation = operation
+		this.ids = ids
+	}
+}
+
+/** What the running application gives the plugin. */
+export interface Running {
+	readonly storage: FileStorage
+	/** Who may upload; Master always may. */
+	readonly upload: CallerRule
+	/** Each model's file fields, with their options. */
+	readonly fields: ReadonlyMap<string, ReadonlyMap<string, FileOptions>>
+	readonly logger: Logger
+}
+
+// The running application's: one runs at a time in a process.
+let running: Running | undefined
+
+export function useRunning(given: Running | undefined): void {
+	running = given
+}
+
+export function runningOf(): Running {
+	if (running === undefined) throw new Error('filesPlugin has not started.')
+	return running
+}
+
+/** Each value code holds, with the copy the row stores. */
+const copies = new WeakMap<object, StoredFile>()
+
+/** The value code reads of a file, its URLs signed now. */
+export function fileValue(stored: StoredFile, keys: Keys): FileValue {
+	const { storage } = runningOf()
+	const variants: { [key: string]: FileVariant | null } = {}
+	for (const key of keys.variants) {
+		const variant = stored.variants[key]
+		variants[key] =
+			variant === undefined
+				? null
+				: Object.freeze({
+						type: variant.type,
+						size: variant.size,
+						url: storage.url(variant.key),
+					})
+	}
+	const inline: { [key: string]: JsonValue | null } = {}
+	for (const key of keys.inline) inline[key] = stored.inline[key] ?? null
+	const value: FileValue = Object.freeze({
+		id: stored.id,
+		name: stored.name,
+		type: stored.type,
+		size: stored.size,
+		url: storage.url(stored.key),
+		inline: Object.freeze(inline),
+		variants: Object.freeze(variants),
+	})
+	copies.set(value, stored)
+	return value
+}
+
+/** Whether a value is a file the plugin resolved. */
+export const isFile = (value: unknown): value is FileValue =>
+	typeof value === 'object' && value !== null && copies.has(value)
+
+/** The copy the row stores of a value code holds. */
+export function copyOf(value: unknown, where: string): StoredFile {
+	const copy =
+		typeof value === 'object' && value !== null ? copies.get(value) : undefined
+	if (copy === undefined) {
+		throw new TypeError(
+			`${where}: a file field stores the files filesPlugin resolved, written as ids.`,
+		)
+	}
+	return copy
+}
+
+const isRecord = (value: unknown): value is { [key: string]: unknown } =>
+	typeof value === 'object' && value !== null && !Array.isArray(value)
+
+const isString = (value: unknown): value is string => typeof value === 'string'
+const isSize = (value: unknown): value is number =>
+	typeof value === 'number' && Number.isInteger(value) && value >= 0
+
+function variantsOf(
+	value: unknown,
+): { readonly [key: string]: StoredVariant } | undefined {
+	if (!isRecord(value)) return undefined
+	const variants: { [key: string]: StoredVariant } = {}
+	for (const [name, variant] of Object.entries(value)) {
+		if (!isRecord(variant)) return undefined
+		const { key, type, size } = variant
+		if (!isString(key) || !isString(type) || !isSize(size)) return undefined
+		variants[name] = { key, type, size }
+	}
+	return variants
+}
+
+/** A stored copy, read back from the row; any other value is a bug. */
+export function storedFile(value: unknown, where: string): StoredFile {
+	if (isRecord(value)) {
+		const { id, key, name, type, size, inline } = value
+		const variants = variantsOf(value['variants'])
+		if (
+			isString(id) &&
+			isString(key) &&
+			isString(name) &&
+			isString(type) &&
+			isSize(size) &&
+			isRecord(inline) &&
+			variants !== undefined
+		) {
+			// oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the row's JSON column holds JSON.
+			const json = inline as { readonly [key: string]: JsonValue }
+			return { id, key, name, type, size, inline: json, variants }
+		}
+	}
+	throw new TypeError(`${where}: the row holds a file that is not one.`)
+}
+
+/** A copy as the row's JSON column stores it. */
+export const storedJson = (copy: StoredFile): JsonValue => ({
+	id: copy.id,
+	key: copy.key,
+	name: copy.name,
+	type: copy.type,
+	size: copy.size,
+	inline: copy.inline,
+	variants: Object.fromEntries(
+		Object.entries(copy.variants).map(([name, { key, type, size }]) => [
+			name,
+			{ key, type, size },
+		]),
+	),
+})
+
+/** A value as a client reads it. */
+export const fileJson = (value: FileValue): JsonValue => ({
+	id: value.id,
+	name: value.name,
+	type: value.type,
+	size: value.size,
+	url: value.url,
+	inline: { ...value.inline },
+	variants: Object.fromEntries(
+		Object.entries(value.variants).map(([name, variant]) => [
+			name,
+			variant === null
+				? null
+				: { type: variant.type, size: variant.size, url: variant.url },
+		]),
+	),
+})