@ultimabase/files 0.0.0-stage → 0.1.1

package/src/plugin.ts

@@ -0,0 +1,144 @@
+// `filesPlugin({ storage, access })` (D-152): the plugin of the `file` field
+// type, which runs its hooks on every model holding a file field, owns
+// `FilesBlob`, sweeps what a crash left and serves `POST /files`.
+import type { Http } from '@ultimabase/http'
+import { httpPlugin } from '@ultimabase/http'
+import type {
+	CallerRule,
+	ExposedModel,
+	Identity,
+	PluginOf,
+	PluginTypes,
+} from 'ultimabase'
+import { definePlugin } from 'ultimabase'
+import manifest from '../package.json' with { type: 'json' }
+import type { FileOptions } from './field.ts'
+import type { Checked } from './lifecycle.ts'
+import {
+	attach,
+	check,
+	deleteHeld,
+	deleteLost,
+	release,
+	resolve,
+	sweep,
+} from './lifecycle.ts'
+import { FilesBlob } from './model.ts'
+import type { FileBody, FileStorage } from './storage.ts'
+import { uploads } from './routes.ts'
+import { upload } from './upload.ts'
+import { runningOf, useRunning } from './values.ts'
+
+/** What `filesPlugin` takes (D-152). */
+export interface FilesOptions {
+	readonly storage: FileStorage
+	readonly access?: {
+		/** A signed-in user by default; Master always may. */
+		readonly upload?: CallerRule
+	}
+}
+
+/** The files plugin's types (D-57). */
+export interface Files extends PluginTypes {
+	readonly name: 'files'
+	readonly appOptions: FilesOptions
+	readonly requires: Http
+}
+
+/** Ten minutes, between two sweeps. */
+const sweepEvery = 10 * 60 * 1000
+
+/** Each model's file fields, with the options `file()` checked. */
+function fieldsOf(
+	models: readonly ExposedModel[],
+): ReadonlyMap<string, ReadonlyMap<string, FileOptions>> {
+	const fields = new Map<string, ReadonlyMap<string, FileOptions>>()
+	for (const model of models) {
+		const own = Object.entries(model.fieldTypes).flatMap(([field, type]) =>
+			type.plugin === 'files'
+				? [[field, type.options as FileOptions] as const]
+				: [],
+		)
+		if (own.length > 0) fields.set(model.schema.name, new Map(own))
+	}
+	return fields
+}
+
+const signedIn: CallerRule = ({ caller }) => caller.kind === 'user'
+
+// The sweep's timer, while the application runs.
+let timer: NodeJS.Timeout | undefined
+
+/** Sweeps, then again every ten minutes; a failed sweep is logged. */
+async function sweeping(): Promise<void> {
+	try {
+		await sweep()
+	} catch (error) {
+		runningOf().logger.warn({ err: error }, 'The files sweep failed.')
+	}
+}
+
+const definition = definePlugin<Files>({
+	name: 'files',
+	manifest,
+	requires: { http: httpPlugin },
+	models: [FilesBlob],
+	api: ({ plugins }) => {
+		plugins.http.mount(uploads(), { bodyLimit: false })
+	},
+	hooks: {
+		beforeCreate: (facts) => resolve(facts),
+		beforeUpdate: (facts) => resolve(facts),
+		afterCreate: (facts) => attach(facts),
+		afterUpdate: (facts) => attach(facts),
+		afterDelete: (facts) => release(facts),
+		afterUpdateCommit: (facts) => deleteLost(facts),
+		afterDeleteCommit: (facts) => deleteHeld(facts),
+	},
+	start: async ({ options, models, logger }) => {
+		const { storage, access } = options
+		storage.start()
+		const fields = fieldsOf(models)
+		useRunning({ storage, upload: access?.upload ?? signedIn, fields, logger })
+		await sweeping()
+		timer = setInterval(() => void sweeping(), sweepEvery)
+		timer.unref()
+	},
+	stop: () => {
+		clearInterval(timer)
+		timer = undefined
+		useRunning(undefined)
+	},
+})
+
+/** Files for the application's file fields; it requires `httpPlugin()`. */
+export const filesPlugin: PluginOf<Files> = definition.plugin
+
+/** What `files.upload` takes. */
+export interface UploadArgs {
+	/** The field the file is for: `Post.cover`. */
+	readonly for: string
+	readonly name: string
+	readonly as?: Identity
+}
+
+/** The running files plugin, from server code (D-152). */
+export const files: {
+	/** Stores a file as `POST /files` does, and gives its id. */
+	upload(
+		body: Uint8Array | Blob,
+		args: UploadArgs,
+	): Promise<{ readonly id: string }>
+	/** Compares the storage's keys with `FilesBlob`. */
+	check(options?: { readonly delete?: boolean }): Promise<Checked>
+} = {
+	upload: (body, args) => {
+		const given: FileBody = body instanceof Uint8Array ? body : body.stream()
+		const type = body instanceof Uint8Array ? undefined : body.type
+		return upload(given, body instanceof Uint8Array ? body.length : body.size, {
+			...args,
+			type,
+		})
+	},
+	check: (options) => check(options),
+}

package/src/routes.ts

@@ -0,0 +1,42 @@
+// `POST /files?for=Post.cover&name=photo.jpg` (D-152): the file as the raw
+// body, which its field's `maxSize` limits, not the HTTP plugin (D-153).
+import type { Routes } from '@ultimabase/http'
+import { routes } from '@ultimabase/http'
+import { PublicError } from 'ultimabase'
+import { upload } from './upload.ts'
+
+/** An upload's own refusals, which the HTTP plugin does not know. */
+const statuses: ReadonlyMap<string, 413 | 415> = new Map([
+	['PAYLOAD_TOO_LARGE', 413],
+	['UNSUPPORTED_MEDIA_TYPE', 415],
+] as const)
+
+/** The upload route, which the plugin mounts without the body limit. */
+export const uploads = (): Routes =>
+	routes().post('/files', async (c) => {
+		const length = c.req.header('content-length')
+		if (length === undefined) {
+			return c.json(
+				{ code: 'LENGTH_REQUIRED', message: 'Content-Length is required.' },
+				411,
+			)
+		}
+		try {
+			const { id } = await upload(
+				c.req.raw.body ?? new Uint8Array(),
+				Number(length),
+				{
+					for: c.req.query('for') ?? '',
+					name: c.req.query('name') ?? '',
+					type: c.req.header('content-type'),
+					as: c.var.caller,
+				},
+			)
+			return c.json({ id }, 201)
+		} catch (error) {
+			if (!(error instanceof PublicError)) throw error
+			const status = statuses.get(error.code)
+			if (status === undefined) throw error
+			return c.json({ code: error.code, message: error.message }, status)
+		}
+	})

package/src/s3.ts

@@ -0,0 +1,141 @@
+// `s3()` (D-152): any S3-compatible storage (AWS, DigitalOcean Spaces,
+// MinIO, SeaweedFS), path-style, each request presigned with SigV4.
+import type { Signer } from './sigv4.ts'
+import { presign } from './sigv4.ts'
+import type { FileStorage } from './storage.ts'
+import { isPublic } from './storage.ts'
+
+/** What `s3()` takes; the first five are required, read when the application starts. */
+export interface S3Options {
+	readonly bucket: string | undefined
+	readonly region: string | undefined
+	/** `https://fra1.digitaloceanspaces.com`. */
+	readonly endpoint: string | undefined
+	readonly accessKeyId: string | undefined
+	readonly secretAccessKey: string | undefined
+	/** Sends `x-amz-acl: public-read` with a public file, which Spaces needs. */
+	readonly publicAcl?: boolean
+	/** The base of a public file's URL, a CDN; the storage's own otherwise. */
+	readonly publicUrl?: string
+}
+
+const required = [
+	'bucket',
+	'region',
+	'endpoint',
+	'accessKeyId',
+	'secretAccessKey',
+] as const
+
+type Config = Signer & { readonly bucket: string; readonly base: string }
+
+const trimmed = (url: string): string => url.replace(/\/+$/, '')
+
+/** The options checked: a missing one stops startup, naming it. */
+function configOf(options: S3Options): Config {
+	const missing = required.filter((key) => !options[key])
+	if (missing.length > 0) {
+		throw new Error(`s3() needs ${missing.join(', ')}: undefined or empty.`)
+	}
+	const { bucket = '', region = '', endpoint = '' } = options
+	const { accessKeyId = '', secretAccessKey = '' } = options
+	const base = `${trimmed(endpoint)}/${encodeURIComponent(bucket)}`
+	return { bucket, region, accessKeyId, secretAccessKey, base }
+}
+
+/** Sent unchanged when the storage answers otherwise. */
+async function answered(
+	response: Response,
+	what: string,
+	expected: readonly number[] = [200],
+): Promise<Response> {
+	if (expected.includes(response.status)) return response
+	const text = await response.text()
+	throw new Error(`s3: ${what} failed with ${response.status}: ${text}`)
+}
+
+const entities: { readonly [entity: string]: string } = {
+	amp: '&',
+	lt: '<',
+	gt: '>',
+	quot: '"',
+	apos: "'",
+}
+const decoded = (text: string): string =>
+	text.replaceAll(
+		/&(amp|lt|gt|quot|apos);/g,
+		(entity, name: string) => entities[name] ?? entity,
+	)
+
+/** The keys of one `ListObjectsV2` page, and the token of the next. */
+function pageOf(xml: string): {
+	readonly keys: readonly string[]
+	readonly next: string | undefined
+} {
+	const keys = [...xml.matchAll(/<Key>([^<]*)<\/Key>/g)].map(([, key = '']) =>
+		decoded(key),
+	)
+	const truncated = xml.includes('<IsTruncated>true</IsTruncated>')
+	const token = /<NextContinuationToken>([^<]*)</.exec(xml)?.[1]
+	return { keys, next: truncated && token ? decoded(token) : undefined }
+}
+
+/** An S3-compatible storage; its options are checked when the application starts. */
+export function s3(options: S3Options): FileStorage {
+	let config: Config | undefined
+	const configured = (): Config => (config ??= configOf(options))
+	const objectUrl = (key: string) => new URL(`${configured().base}/${key}`)
+	const signed = (
+		method: string,
+		url: URL,
+		expires: number,
+		headers: { readonly [name: string]: string } = {},
+	) => presign(configured(), { method, url, expires, headers })
+	return {
+		start: () => {
+			configured()
+		},
+		put: async (key, body, { type, size, attachment }) => {
+			const headers: Record<string, string> = { 'content-type': type }
+			if (attachment) headers['content-disposition'] = 'attachment'
+			if (isPublic(key) && options.publicAcl === true) {
+				headers['x-amz-acl'] = 'public-read'
+			}
+			const url = signed('PUT', objectUrl(key), 900, headers)
+			const response = await fetch(url, {
+				method: 'PUT',
+				headers: { ...headers, 'content-length': String(size) },
+				body,
+				duplex: 'half',
+			})
+			await answered(response, `PUT ${key}`)
+		},
+		delete: async (key) => {
+			const url = signed('DELETE', objectUrl(key), 900)
+			const response = await fetch(url, { method: 'DELETE' })
+			await answered(response, `DELETE ${key}`, [200, 204, 404])
+		},
+		keys: async () => {
+			const keys: string[] = []
+			let next: string | undefined
+			do {
+				const list = new URL(configured().base)
+				list.searchParams.set('list-type', '2')
+				if (next !== undefined)
+					list.searchParams.set('continuation-token', next)
+				const response = await fetch(signed('GET', list, 900))
+				const page = pageOf(await (await answered(response, 'list')).text())
+				keys.push(...page.keys)
+				next = page.next
+			} while (next !== undefined)
+			return keys
+		},
+		url: (key) => {
+			if (!isPublic(key)) return signed('GET', objectUrl(key), 3600)
+			const { publicUrl } = options
+			return publicUrl === undefined
+				? objectUrl(key).href
+				: `${trimmed(publicUrl)}/${key}`
+		},
+	}
+}

package/src/shape.ts

@@ -0,0 +1,83 @@
+// A file field's shape (D-152): the JSON Schema of what clients read, which
+// REST and GraphQL type it with. It validates nothing: the plugin made the
+// value.
+import type { StandardSchema } from './standard.ts'
+import type { FileOptions } from './field.ts'
+
+type Node = { readonly [key: string]: unknown }
+type Exported = {
+	readonly jsonSchema: {
+		readonly input: () => Node
+		readonly output: () => Node
+	}
+}
+
+const isNode = (value: unknown): value is Node =>
+	typeof value === 'object' && value !== null && !Array.isArray(value)
+
+const string = { type: 'string' }
+const size = { type: 'integer', minimum: 0 }
+const nullable = (node: Node): Node =>
+	Object.keys(node).length === 0 ? node : { anyOf: [node, { type: 'null' }] }
+
+const closed = (properties: { readonly [key: string]: Node }): Node => ({
+	type: 'object',
+	properties,
+	required: Object.keys(properties),
+	additionalProperties: false,
+})
+
+/** An inline value's JSON Schema, from its schema's; any JSON without one. */
+function inlineOf(inline: unknown): Node {
+	if (!isNode(inline) || !isNode(inline['schema'])) return {}
+	const standard = inline['schema']['~standard']
+	const exporter = isNode(standard) ? standard['jsonSchema'] : undefined
+	const output = isNode(exporter) ? exporter['output'] : undefined
+	if (typeof output !== 'function') return {}
+	const node: unknown = Reflect.apply(output, exporter, [
+		{ target: 'draft-2020-12' },
+	])
+	if (!isNode(node)) return {}
+	const { $schema: _, ...rest } = node
+	return rest
+}
+
+function fileNode(options: FileOptions): Node {
+	const inline = Object.entries(options.inline ?? {}).map(
+		([key, given]) => [key, nullable(inlineOf(given))] as const,
+	)
+	const variant = closed({ type: string, size, url: string })
+	const variants = Object.keys(options.variants ?? {}).map(
+		(key) => [key, nullable(variant)] as const,
+	)
+	return closed({
+		id: string,
+		name: string,
+		type: string,
+		size,
+		url: string,
+		inline: closed(Object.fromEntries(inline)),
+		variants: closed(Object.fromEntries(variants)),
+	})
+}
+
+function nodeOf(options: FileOptions): Node {
+	const node = fileNode(options)
+	if (options.list === true) return { type: 'array', items: node }
+	return options.optional === true ? nullable(node) : node
+}
+
+/** What a field with `options` gives clients, as a Standard Schema. */
+export const shapeOf = (
+	options: FileOptions,
+): StandardSchema & { readonly '~standard': Exported } => ({
+	'~standard': {
+		version: 1,
+		vendor: 'ultimabase-files',
+		validate: (value) => ({ value }),
+		jsonSchema: {
+			input: () => nodeOf(options),
+			output: () => nodeOf(options),
+		},
+	},
+})

package/src/sigv4.ts

@@ -0,0 +1,81 @@
+// AWS Signature Version 4, query presigning (D-152): on `node:crypto`, which
+// is synchronous, so a read signs its URLs while it decodes. The payload is
+// unsigned, as presigned URLs are.
+import { createHash, createHmac } from 'node:crypto'
+
+export interface Signer {
+	readonly region: string
+	readonly accessKeyId: string
+	readonly secretAccessKey: string
+}
+
+export interface Presigned {
+	readonly method: string
+	readonly url: URL
+	/** The headers the request sends and signs, `host` added. */
+	readonly headers: { readonly [name: string]: string }
+	/** In seconds. */
+	readonly expires: number
+}
+
+/** RFC 3986, as SigV4 encodes: everything but `A-Za-z0-9-_.~`. */
+const encoded = (value: string): string =>
+	encodeURIComponent(value).replaceAll(
+		/[!'()*]/g,
+		(c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`,
+	)
+
+const hmac = (key: string | Buffer, value: string): Buffer =>
+	createHmac('sha256', key).update(value).digest()
+
+/** `20130524T000000Z`. */
+const stampOf = (now: Date): string =>
+	now
+		.toISOString()
+		.replaceAll(/[-:]/g, '')
+		.replace(/\.\d{3}/, '')
+
+/** The URL that runs the request until it expires, signed now. */
+export function presign(signer: Signer, request: Presigned): string {
+	const { method, url, expires } = request
+	const stamp = stampOf(new Date())
+	const day = stamp.slice(0, 8)
+	const scope = `${day}/${signer.region}/s3/aws4_request`
+	const headers = new Map(
+		Object.entries({ ...request.headers, host: url.host }).map(
+			([name, value]) => [name.toLowerCase(), value.trim()],
+		),
+	)
+	const names = [...headers.keys()].toSorted()
+	const signed = names.join(';')
+	const query = new URLSearchParams(url.search)
+	query.set('X-Amz-Algorithm', 'AWS4-HMAC-SHA256')
+	query.set('X-Amz-Credential', `${signer.accessKeyId}/${scope}`)
+	query.set('X-Amz-Date', stamp)
+	query.set('X-Amz-Expires', String(expires))
+	query.set('X-Amz-SignedHeaders', signed)
+	const canonicalQuery = [...query]
+		.map(([key, value]) => [encoded(key), encoded(value)])
+		.toSorted(([a = ''], [b = '']) => (a < b ? -1 : a > b ? 1 : 0))
+		.map(([key, value]) => `${key}=${value}`)
+		.join('&')
+	const path = url.pathname
+		.split('/')
+		.map((part) => encoded(decodeURIComponent(part)))
+	const canonical = [
+		method,
+		path.join('/'),
+		canonicalQuery,
+		names.map((name) => `${name}:${headers.get(name)}\n`).join(''),
+		signed,
+		'UNSIGNED-PAYLOAD',
+	].join('\n')
+	const digest = createHash('sha256').update(canonical).digest('hex')
+	const toSign = ['AWS4-HMAC-SHA256', stamp, scope, digest].join('\n')
+	const key = ['s3', 'aws4_request'].reduce(
+		hmac,
+		hmac(hmac(`AWS4${signer.secretAccessKey}`, day), signer.region),
+	)
+	const signature = createHmac('sha256', key).update(toSign).digest('hex')
+	return `${url.origin}${path.join('/')}?${canonicalQuery}&X-Amz-Signature=${signature}`
+}

package/src/standard.ts

@@ -0,0 +1,15 @@
+// Standard Schema (standardschema.dev), the part this package reads. Its
+// specification asks for the interface to be copied, not imported.
+export interface StandardSchema<I = unknown, O = I> {
+	readonly '~standard': {
+		readonly version: 1
+		readonly vendor: string
+		readonly validate: (
+			value: unknown,
+		) => SchemaResult<O> | Promise<SchemaResult<O>>
+		readonly types?: { readonly input: I; readonly output: O } | undefined
+	}
+}
+type SchemaResult<O> =
+	| { readonly value: O; readonly issues?: undefined }
+	| { readonly issues: readonly { readonly message: string }[] }

package/src/storage.ts

@@ -0,0 +1,46 @@
+// What `filesPlugin` stores files in (D-152): an S3-compatible storage, or
+// memory for tests. A key starts with `public/` or `private/`, which says
+// how the object is served.
+
+/** A file's bytes: streamed from a request, or held in memory. */
+export type FileBody = ReadableStream<Uint8Array> | Uint8Array
+
+/** How an object is stored and served. */
+export interface PutOptions {
+	readonly type: string
+	/** The body's length in bytes, which the storage checks. */
+	readonly size: number
+	/** Served as a download, so a browser never runs it: SVG, undetected types. */
+	readonly attachment: boolean
+}
+
+/** A storage adapter: `s3()` or `memoryFiles()`. */
+export interface FileStorage {
+	/** Checks its options when the application starts. */
+	start(): void
+	put(key: string, body: FileBody, options: PutOptions): Promise<void>
+	/** Deletes an object; a missing one is not an error. */
+	delete(key: string): Promise<void>
+	/** Every key it stores. */
+	keys(): Promise<readonly string[]>
+	/** A public object's stable URL, a private one's presigned for one hour. */
+	url(key: string): string
+}
+
+/** Whether a key is served to everyone. */
+export const isPublic = (key: string): boolean => key.startsWith('public/')
+
+/**
+ * Each promise ended, then the first failure thrown: a put still running
+ * would store an object after its cleanup.
+ */
+export async function settled<T>(
+	promises: readonly Promise<T>[],
+): Promise<T[]> {
+	const results = await Promise.allSettled(promises)
+	const failed = results.find((result) => result.status === 'rejected')
+	if (failed !== undefined) throw failed.reason
+	return results.flatMap((result) =>
+		result.status === 'fulfilled' ? [result.value] : [],
+	)
+}