@ultimabase/files 0.0.0-stage → 0.1.0

package/src/field.ts

@@ -0,0 +1,236 @@
+// `file(options)` (D-152): a field of one file or a list of them, stored as a
+// JSON copy of their metadata. A write gives ids, which the plugin's hooks
+// resolve; a read gives each file's URL, signed through the running storage.
+import type { Field, JsonValue } from 'ultimabase'
+import type { StandardSchema } from './standard.ts'
+import { defineFieldType } from 'ultimabase'
+import { shapeOf } from './shape.ts'
+import type { FileValue, FileVariant, Keys, Operation } from './values.ts'
+import {
+	copyOf,
+	fileJson,
+	fileValue,
+	Pending,
+	storedFile,
+	storedJson,
+} from './values.ts'
+
+/** What `inline` and `variants` are made from, at upload. */
+export interface FileInput {
+	readonly body: Uint8Array
+	readonly type: string
+	readonly name: string
+}
+
+/** Computes one inline value: JSON of 2 KB at most. */
+export type InlineRun = (file: FileInput) => JsonValue | Promise<JsonValue>
+/** An inline value; with its `schema`, the transports type it. */
+export type Inline =
+	InlineRun | { readonly run: InlineRun; readonly schema: StandardSchema }
+
+/** Another file made from the upload: a thumbnail. */
+export type Variant = (file: FileInput) => MadeVariant | Promise<MadeVariant>
+export interface MadeVariant {
+	readonly body: Uint8Array
+	readonly type: string
+}
+
+/** What `file()` takes (D-152). */
+export interface FileOptions {
+	readonly description?: string
+	/** A single file that may be `null`. */
+	readonly optional?: boolean
+	/** A list of files, `[]` when empty. */
+	readonly list?: boolean
+	/** In bytes, 50 MB by default. */
+	readonly maxSize?: number
+	/** MIME types or patterns: `['image/*', 'application/pdf']`; any by default. */
+	readonly accept?: readonly string[]
+	/** Served to everyone at a stable URL; presigned for one hour otherwise. */
+	readonly public?: boolean
+	readonly inline?: { readonly [key: string]: Inline }
+	readonly variants?: { readonly [key: string]: Variant }
+}
+
+type InlineOf<T> = T extends {
+	readonly schema: StandardSchema<unknown, infer O>
+}
+	? O
+	: T extends (...args: never[]) => infer R
+		? Awaited<R>
+		: never
+type Inlines<O> = O extends { readonly inline: infer I }
+	? { readonly [K in keyof I]: InlineOf<I[K]> | null }
+	: { readonly [K in never]: never }
+type Variants<O> = O extends { readonly variants: infer V }
+	? { readonly [K in keyof V]: FileVariant | null }
+	: { readonly [K in never]: never }
+/** The value of a field with options `O`. */
+export type FileOf<O> = FileValue<Inlines<O>, Variants<O>>
+type ValueOf<O> = O extends { readonly list: true }
+	? readonly FileOf<O>[]
+	: FileOf<O>
+type Ids = string | readonly string[]
+/** A list's write: the new list, or one operation, as kernel lists (D-149). */
+type ListInput =
+	| readonly string[]
+	| {
+			readonly [K in Operation]: { readonly [P in K]: Ids } & {
+				readonly [P in Exclude<Operation, K>]?: never
+			}
+	  }[Operation]
+type InputOf<O> = O extends { readonly list: true } ? ListInput : string
+
+/** A file field: read as files, written as ids. */
+export type FileField<O> = Field<
+	ValueOf<O>,
+	O,
+	'files',
+	never,
+	InputOf<O>,
+	true,
+	Field<JsonValue, O, never, never, JsonValue, true>
+>
+
+/** 50 MB. */
+export const defaultMaxSize: number = 50 * 1024 * 1024
+const operations: readonly Operation[] = ['set', 'push', 'pushUnique', 'remove']
+
+export const keysOf = (options: FileOptions): Keys => ({
+	inline: Object.keys(options.inline ?? {}),
+	variants: Object.keys(options.variants ?? {}),
+})
+
+/** Ids, each once; `undefined` when the value is not a list of them. */
+function idsOf(value: unknown, unique: boolean): readonly string[] | undefined {
+	const ids: unknown[] = Array.isArray(value) ? value : [value]
+	if (!ids.every((id): id is string => typeof id === 'string' && id !== '')) {
+		return undefined
+	}
+	if (unique && new Set(ids).size !== ids.length) return undefined
+	return ids
+}
+
+/** The ids a write gives, as the hooks resolve them. */
+function pendingOf(value: unknown, list: boolean): Pending | string {
+	if (!list) {
+		return typeof value === 'string' && value !== ''
+			? new Pending('set', [value])
+			: 'Expected a file id.'
+	}
+	const expected =
+		'Expected a list of file ids, each once, or one of set, push, pushUnique, remove.'
+	if (Array.isArray(value)) {
+		const ids = idsOf(value, true)
+		return ids === undefined ? expected : new Pending('set', ids)
+	}
+	if (typeof value !== 'object' || value === null) return expected
+	const given = Object.entries(value).filter(([, ids]) => ids !== undefined)
+	const [entry] = given
+	const operation = operations.find((name) => name === entry?.[0])
+	if (given.length !== 1 || operation === undefined) return expected
+	const ids = idsOf(entry?.[1], operation === 'set' || operation === 'push')
+	return ids === undefined ? expected : new Pending(operation, ids)
+}
+
+const schemaOf = (options: FileOptions): StandardSchema<unknown, Pending> => ({
+	'~standard': {
+		version: 1,
+		vendor: 'ultimabase-files',
+		validate: (value) => {
+			const pending = pendingOf(value, options.list === true)
+			return typeof pending === 'string'
+				? { issues: [{ message: pending }] }
+				: { value: pending }
+		},
+	},
+})
+
+type Value = FileValue | readonly FileValue[] | Pending
+
+const isList = (value: Value): value is readonly FileValue[] =>
+	Array.isArray(value)
+
+const fileType = defineFieldType({
+	plugin: 'files',
+	options: [
+		'optional',
+		'list',
+		'maxSize',
+		'accept',
+		'public',
+		'inline',
+		'variants',
+	],
+	stored: (f, options: FileOptions) =>
+		options.list === true
+			? f.json({ default: [] })
+			: options.optional === true
+				? f.json({ optional: true })
+				: f.json(),
+	decode: (stored, options): Value => {
+		const keys = keysOf(options)
+		const where = 'A file field'
+		return Array.isArray(stored)
+			? stored.map((item) => fileValue(storedFile(item, where), keys))
+			: fileValue(storedFile(stored, where), keys)
+	},
+	encode: (value: Value): JsonValue => {
+		const where = 'A file field'
+		if (value instanceof Pending) copyOf(value, where)
+		return isList(value)
+			? value.map((item) => storedJson(copyOf(item, where)))
+			: storedJson(copyOf(value, where))
+	},
+	schema: (options: FileOptions): StandardSchema<unknown, Value> =>
+		schemaOf(options),
+	toJSON: (value): JsonValue => {
+		if (value instanceof Pending) return null
+		return isList(value) ? value.map((item) => fileJson(item)) : fileJson(value)
+	},
+	shape: (options) => shapeOf(options),
+})
+
+function checked(options: FileOptions): void {
+	const { list, optional, maxSize, accept } = options
+	if (list === true && optional === true) {
+		throw new Error('A file list is empty, never null: list or optional.')
+	}
+	if (maxSize !== undefined && (!Number.isInteger(maxSize) || maxSize <= 0)) {
+		throw new Error('A file field’s maxSize is a number of bytes.')
+	}
+	if (accept?.some((type) => !/^[\w.+-]+\/(\*|[\w.+-]+)$/.test(type))) {
+		throw new Error(
+			'A file field’s accept lists MIME types or patterns: image/png, image/*.',
+		)
+	}
+	for (const [key, inline] of Object.entries(options.inline ?? {})) {
+		if (typeof inline !== 'function' && typeof inline?.run !== 'function') {
+			throw new Error(`A file field’s inline ${key} is a function.`)
+		}
+	}
+	for (const [key, variant] of Object.entries(options.variants ?? {})) {
+		if (typeof variant !== 'function') {
+			throw new Error(`A file field’s variant ${key} is a function.`)
+		}
+	}
+}
+
+/**
+ * A field of files (D-152): written as ids from `POST /files` or
+ * `files.upload`, read with each file's URL.
+ */
+export function file(): FileField<{}>
+// Neither a default for `O` nor inference from where the field goes: either
+// would leave the functions of `inline` and `variants` untyped.
+export function file<const O extends FileOptions>(
+	options: O,
+): FileField<NoInfer<O>>
+export function file(options?: FileOptions): FileField<FileOptions> {
+	const given: FileOptions = options ?? {}
+	checked(given)
+	const field =
+		given.list === true ? fileType({ ...given, default: [] }) : fileType(given)
+	// oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the field's options give its value and its input.
+	return field as unknown as FileField<FileOptions>
+}

package/src/index.ts

@@ -0,0 +1,22 @@
+// `@ultimabase/files` (D-152): file fields stored in an S3-compatible
+// storage, uploaded with `POST /files`, and never an orphan.
+export type {
+	FileField,
+	FileInput,
+	FileOf,
+	FileOptions,
+	Inline,
+	InlineRun,
+	MadeVariant,
+	Variant,
+} from './field.ts'
+export type { Checked } from './lifecycle.ts'
+export type { MemoryFiles, MemoryObject } from './memory.ts'
+export type { Files, FilesOptions, UploadArgs } from './plugin.ts'
+export type { S3Options } from './s3.ts'
+export type { FileBody, FileStorage, PutOptions } from './storage.ts'
+export type { FileValue, FileVariant } from './values.ts'
+export { file } from './field.ts'
+export { memoryFiles } from './memory.ts'
+export { files, filesPlugin } from './plugin.ts'
+export { s3 } from './s3.ts'

package/src/lifecycle.ts

@@ -0,0 +1,270 @@
+// A model's files through its writes (D-152): ids resolved to files before
+// the write, files attached and released in its transaction, released
+// objects deleted after its commit, and the sweep for what a crash left.
+import type { Caller } from 'ultimabase'
+import { isMaster, PublicError } from 'ultimabase'
+import type { FileOptions } from './field.ts'
+import { keysOf } from './field.ts'
+import { FilesBlob, M } from './model.ts'
+import { settled } from './storage.ts'
+import type { FileValue, StoredVariant } from './values.ts'
+import { fileValue, isFile, Pending, runningOf, storedFile } from './values.ts'
+
+type Blob = typeof FilesBlob extends {
+	findMany(...args: never[]): Promise<(infer R)[]>
+}
+	? R
+	: never
+type BlobWhere = NonNullable<Parameters<typeof FilesBlob.findMany>[0]>['where']
+
+/** What a hook gives of the write. */
+interface Facts {
+	readonly name: string
+	readonly caller: Caller
+	readonly record: object
+}
+
+const fieldsOf = (model: string): ReadonlyMap<string, FileOptions> =>
+	runningOf().fields.get(model) ?? new Map<string, FileOptions>()
+
+const invalid = (field: string, message: string): PublicError =>
+	new PublicError('VALIDATION_FAILED', 'Invalid data.', {
+		details: { issues: [{ path: [field], message }] },
+	})
+
+/** The files a field's value holds. */
+function heldOf(value: unknown): readonly FileValue[] {
+	const items: readonly unknown[] = Array.isArray(value) ? value : [value]
+	return items.filter(isFile)
+}
+
+const idsOf = (record: object, field: string): readonly string[] =>
+	heldOf(Reflect.get(record, field)).map(({ id }) => id)
+
+const idOf = (record: object): string => String(Reflect.get(record, 'id'))
+
+/** Whether `caller` may attach a pending blob: its uploader, or Master. */
+function usable(blob: Blob, target: string, caller: Caller): boolean {
+	const { state, field, variantOf, deleteAfter, uploadedBy } = blob
+	return (
+		state === 'pending' &&
+		field === target &&
+		variantOf === null &&
+		deleteAfter !== null &&
+		deleteAfter.getTime() > Date.now() &&
+		(isMaster(caller) ||
+			uploadedBy === null ||
+			(caller.kind === 'user' && uploadedBy === caller.user.id))
+	)
+}
+
+/** The pending files of `ids`, each one the caller may attach to `target`. */
+async function pendingFiles(
+	target: string,
+	options: FileOptions,
+	ids: readonly string[],
+	caller: Caller,
+): Promise<ReadonlyMap<string, FileValue>> {
+	if (ids.length === 0) return new Map()
+	const field = target.slice(target.indexOf('.') + 1)
+	const where = { id: { in: ids } }
+	const blobs = await FilesBlob.findMany({ where }, M)
+	const missing = ids.find(
+		(id) =>
+			!blobs.some((blob) => blob.id === id && usable(blob, target, caller)),
+	)
+	if (missing !== undefined) {
+		throw invalid(field, `${missing} is not a file uploaded for ${target}.`)
+	}
+	const made = await FilesBlob.findMany({ where: { variantOf: where.id } }, M)
+	const keys = keysOf(options)
+	return new Map(
+		blobs.map((blob) => {
+			const variants: { [key: string]: StoredVariant } = {}
+			for (const { variantOf, name, key, type, size } of made) {
+				if (variantOf === blob.id) variants[name] = { key, type, size }
+			}
+			const { id, key, name, type, size, inline } = blob
+			const copy = { id, key, name, type, size, inline: inline ?? {}, variants }
+			return [id, fileValue(storedFile(copy, 'FilesBlob'), keys)] as const
+		}),
+	)
+}
+
+/** A field's files after a write's operation on the ones it holds. */
+async function filesOf(
+	target: string,
+	options: FileOptions,
+	pending: Pending,
+	held: readonly FileValue[],
+	caller: Caller,
+): Promise<readonly FileValue[]> {
+	const { operation, ids } = pending
+	const holds = new Map(held.map((file) => [file.id, file]))
+	if (operation === 'remove') return held.filter(({ id }) => !ids.includes(id))
+	const field = target.slice(target.indexOf('.') + 1)
+	const taken = ids.find((id) => holds.has(id))
+	if (operation === 'push' && taken !== undefined) {
+		throw invalid(field, `The list holds ${taken} already.`)
+	}
+	const given =
+		operation === 'pushUnique'
+			? [...new Set(ids)].filter((id) => !holds.has(id))
+			: ids
+	const fresh = given.filter((id) => !holds.has(id))
+	const found = await pendingFiles(target, options, fresh, caller)
+	const listed = given.flatMap((id) => {
+		const file = holds.get(id) ?? found.get(id)
+		return file === undefined ? [] : [file]
+	})
+	return operation === 'set' ? listed : [...held, ...listed]
+}
+
+/**
+ * `beforeCreate` and `beforeUpdate`: each file field's ids become its
+ * files, computed on the row the update locked.
+ */
+export async function resolve(facts: {
+	readonly name: string
+	readonly caller: Caller
+	readonly data: object
+	readonly record?: object
+}): Promise<void> {
+	const { name, caller, data, record } = facts
+	for (const [field, options] of fieldsOf(name)) {
+		const value: unknown = Reflect.get(data, field)
+		if (!(value instanceof Pending)) continue
+		const held = record === undefined ? [] : heldOf(Reflect.get(record, field))
+		const target = `${name}.${field}`
+		const files = await filesOf(target, options, value, held, caller)
+		Reflect.set(data, field, options.list === true ? files : files[0])
+	}
+}
+
+const attached = (name: string, field: string, record: object): string =>
+	`${name}.${field}:${idOf(record)}`
+
+/**
+ * `afterCreate` and `afterUpdate`: the files the row gained are attached,
+ * with their variants, and the ones it lost released.
+ */
+export async function attach(
+	facts: Facts & { readonly previous?: object },
+): Promise<void> {
+	const { name, record, previous } = facts
+	for (const field of fieldsOf(name).keys()) {
+		const now = idsOf(record, field)
+		const before = previous === undefined ? [] : idsOf(previous, field)
+		const gained = now.filter((id) => !before.includes(id))
+		const lost = before.filter((id) => !now.includes(id))
+		const attachedTo = attached(name, field, record)
+		const data = { state: 'attached', attachedTo, deleteAfter: null }
+		if (gained.length > 0) {
+			const ids = { in: gained }
+			const where = {
+				id: ids,
+				state: 'pending',
+				deleteAfter: { gt: new Date() },
+			}
+			const { count } = await FilesBlob.updateMany({ where, data }, M)
+			// The sweep released it, or another row attached it, since.
+			if (count !== gained.length) {
+				throw invalid(field, 'A file expired or was attached elsewhere.')
+			}
+			await FilesBlob.updateMany({ where: { variantOf: ids }, data }, M)
+		}
+		if (lost.length > 0) {
+			const where = { OR: [{ id: { in: lost } }, { variantOf: { in: lost } }] }
+			await FilesBlob.updateMany({ where, data: { state: 'released' } }, M)
+		}
+	}
+}
+
+const heldBy = (name: string, record: object): NonNullable<BlobWhere> => ({
+	attachedTo: {
+		in: [...fieldsOf(name).keys()].map((field) =>
+			attached(name, field, record),
+		),
+	},
+})
+
+/** `afterDelete`: every file of the row, released. */
+export async function release({ name, record }: Facts): Promise<void> {
+	const where = heldBy(name, record)
+	await FilesBlob.updateMany({ where, data: { state: 'released' } }, M)
+}
+
+/**
+ * Deletes the released blobs `where` names: their objects, then their rows.
+ * A failure keeps the rows, which the sweep deletes later.
+ */
+async function purge(where: NonNullable<BlobWhere>): Promise<void> {
+	const { storage, logger } = runningOf()
+	try {
+		const blobs = await FilesBlob.findMany(
+			{ where: { AND: [where, { state: 'released' }] } },
+			M,
+		)
+		if (blobs.length === 0) return
+		await settled(blobs.map(({ key }) => storage.delete(key)))
+		const ids = blobs.map(({ id }) => id)
+		await FilesBlob.deleteMany({ where: { id: { in: ids } } }, M)
+	} catch (error) {
+		logger.warn({ err: error }, 'Released files were not deleted yet.')
+	}
+}
+
+/** `afterUpdateCommit`: the objects of the files the row lost, deleted. */
+export async function deleteLost(
+	facts: Facts & { readonly previous: object },
+): Promise<void> {
+	const { name, record, previous } = facts
+	const lost = [...fieldsOf(name).keys()].flatMap((field) => {
+		const now = idsOf(record, field)
+		return idsOf(previous, field).filter((id) => !now.includes(id))
+	})
+	if (lost.length === 0) return
+	await purge({ OR: [{ id: { in: lost } }, { variantOf: { in: lost } }] })
+}
+
+/** `afterDeleteCommit`: the objects of a deleted row's files, deleted. */
+export async function deleteHeld({ name, record }: Facts): Promise<void> {
+	await purge(heldBy(name, record))
+}
+
+/**
+ * The sweep: expired pending blobs are released, so no write attaches them,
+ * then every released blob is deleted, what a crash left included.
+ */
+export async function sweep(): Promise<void> {
+	const where = { state: 'pending', deleteAfter: { lt: new Date() } }
+	await FilesBlob.updateMany({ where, data: { state: 'released' } }, M)
+	await purge({})
+}
+
+/** What `files.check` found. */
+export interface Checked {
+	/** Objects without a row, deleted with `delete: true`. */
+	readonly objects: readonly string[]
+	/** Rows whose object is missing: an upload under way, or a lost object. */
+	readonly rows: readonly string[]
+}
+
+/** Compares the storage's keys with `FilesBlob`. */
+export async function check(options?: {
+	readonly delete?: boolean
+}): Promise<Checked> {
+	const { storage } = runningOf()
+	// The keys first: an upload inserts its row before its object, so an
+	// object listed without a row is never one under way.
+	const keys = await storage.keys()
+	const blobs = await FilesBlob.findMany({ select: { key: true } }, M)
+	const held = new Set(blobs.map(({ key }) => key))
+	const stored = new Set(keys)
+	const objects = keys.filter((key) => !held.has(key))
+	const rows = blobs.map(({ key }) => key).filter((key) => !stored.has(key))
+	if (options?.delete === true) {
+		await settled(objects.map((key) => storage.delete(key)))
+	}
+	return { objects, rows }
+}

package/src/memory.ts

@@ -0,0 +1,46 @@
+// `memoryFiles()` (D-152): a storage in memory, which tests use, so that no
+// test writes to a real storage.
+import type { FileBody, FileStorage } from './storage.ts'
+import { isPublic } from './storage.ts'
+
+/** An object `memoryFiles()` holds. */
+export interface MemoryObject {
+	readonly body: Uint8Array
+	readonly type: string
+	readonly attachment: boolean
+}
+
+/** A storage in memory, its objects readable by tests. */
+export interface MemoryFiles extends FileStorage {
+	readonly objects: ReadonlyMap<string, MemoryObject>
+}
+
+/** The bytes of a body, read to its end. */
+export async function bytesOf(body: FileBody): Promise<Uint8Array> {
+	if (body instanceof Uint8Array) return body
+	return new Uint8Array(await new Response(body).arrayBuffer())
+}
+
+/** A storage in memory: its URLs are `memory://files/<key>`, never served. */
+export function memoryFiles(): MemoryFiles {
+	const objects = new Map<string, MemoryObject>()
+	return {
+		objects,
+		start: () => {},
+		put: async (key, body, { type, size, attachment }) => {
+			const bytes = await bytesOf(body)
+			if (bytes.length !== size) {
+				throw new Error(
+					`memoryFiles: ${key} has ${bytes.length} bytes, not ${size}.`,
+				)
+			}
+			objects.set(key, { body: bytes, type, attachment })
+		},
+		delete: async (key) => {
+			objects.delete(key)
+		},
+		keys: async () => [...objects.keys()],
+		url: (key) =>
+			`memory://files/${key}${isPublic(key) ? '' : '?expires=3600'}`,
+	}
+}

package/src/model.ts

@@ -0,0 +1,67 @@
+// `FilesBlob` (D-152): one row for each object in the storage, a file or one
+// of its variants. Its row is the truth; a file field stores a copy.
+import type { Field, ModelOf, ScalarOptions } from 'ultimabase'
+import { defineModel, master } from 'ultimabase'
+import { runningOf } from './values.ts'
+
+export const M: { readonly as: typeof master } = { as: master }
+
+/** One hour, a pending blob's life. */
+export const hour: number = 60 * 60 * 1000
+
+/** A blob's state: uploaded, held by a row, or to delete. */
+export type BlobState = 'pending' | 'attached' | 'released'
+
+type Optional<V> = Field<V | null, { readonly optional: true }>
+
+type BlobFields = {
+	/** `public/…` or `private/…`, the object's key in the storage. */
+	readonly key: Field<string, { readonly unique: true }>
+	/** The file's name; a variant's own, `thumb`. */
+	readonly name: Field<string, ScalarOptions<string>>
+	readonly type: Field<string, ScalarOptions<string>>
+	readonly size: Field<number, { readonly integer: true }>
+	/** `Post.cover`. */
+	readonly field: Field<string, ScalarOptions<string>>
+	/** The uploading user's id, `master`, or `null` for an anonymous caller. */
+	readonly uploadedBy: Optional<string>
+	readonly state: Field<string, ScalarOptions<string>>
+	/** `Post.cover:<id>`, once attached. */
+	readonly attachedTo: Optional<string>
+	/** A variant's file. */
+	readonly variantOf: Optional<string>
+	readonly inline: Field<
+		unknown,
+		{ readonly optional: true },
+		never,
+		never,
+		unknown,
+		true
+	>
+	/** When a pending blob expires. */
+	readonly deleteAfter: Optional<Date>
+}
+
+export const FilesBlob: ModelOf<'FilesBlob', BlobFields, []> = defineModel({
+	name: 'FilesBlob',
+	fields: (f) => ({
+		key: f.string({ unique: true }),
+		name: f.string(),
+		type: f.string(),
+		size: f.number({ integer: true }),
+		field: f.string(),
+		uploadedBy: f.string({ optional: true }),
+		state: f.string(),
+		attachedTo: f.string({ optional: true }),
+		variantOf: f.string({ optional: true }),
+		inline: f.json({ optional: true }),
+		deleteAfter: f.dateTime({ optional: true }),
+	}),
+	indexes: [
+		{ fields: ['state', 'deleteAfter'] },
+		{ fields: ['variantOf'] },
+		{ fields: ['attachedTo'] },
+	],
+	// An upload creates its row as its caller: the kernel gives the rule the
+	// whole caller, and Master always passes. The rest stays Master's.
+}).with({ access: { create: ({ caller }) => runningOf().upload({ caller }) } })