@ultimabase/postgres 0.0.0-stage → 0.1.0

package/src/migrations.ts

@@ -0,0 +1,275 @@
+// How a PostgreSQL database follows the models (D-105): the snapshot its
+// tables match in `ultimabase_schema`, the files `migrate deploy` applied in
+// `ultimabase_migrations`, and every change under one advisory lock.
+import type { Kysely, QueryExecutorProvider, Transaction } from 'kysely'
+import { sql } from 'kysely'
+import { parse } from 'pg-connection-string'
+import type { Migration, Run, Stored } from 'ultimabase'
+import { isLocalHost, MigrationFailed, Refused } from 'ultimabase'
+import { id } from './plan-parts.ts'
+import { parseSnapshot } from './snapshot.ts'
+
+/** The tables UltimaBase keeps its own records in. */
+export const tables: readonly string[] = [
+	'ultimabase_schema',
+	'ultimabase_migrations',
+]
+
+const bookkeeping = [
+	sql`create table if not exists ultimabase_schema (id integer primary key check (id = 1), snapshot jsonb not null, written_by text not null, written_at timestamptz not null)`,
+	sql`create table if not exists ultimabase_migrations (name text primary key, checksum text not null, schema_hash text, applied_at timestamptz not null)`,
+]
+
+/**
+ * How long a statement waits for a table another session holds: past it,
+ * the statement fails rather than queue every query on that table behind it.
+ */
+const lockTimeout = '5s'
+
+/**
+ * Runs `run` in a transaction that holds the schema's lock, which it waits
+ * for as long as another sync or deploy holds it.
+ */
+async function locked(
+	db: Kysely<object>,
+	run: (trx: Transaction<object>) => Promise<void>,
+): Promise<void> {
+	await db.transaction().execute(async (trx) => {
+		await sql`select pg_advisory_xact_lock(hashtext(current_schema()))`.execute(
+			trx,
+		)
+		// A file may set its own, with `set local lock_timeout`.
+		await sql.raw(`set local lock_timeout = '${lockTimeout}'`).execute(trx)
+		for (const statement of bookkeeping) await statement.execute(trx)
+		await run(trx)
+	})
+}
+
+async function store(
+	trx: Transaction<object>,
+	to: unknown,
+	by: Stored['by'],
+): Promise<void> {
+	await sql`insert into ultimabase_schema (id, snapshot, written_by, written_at)
+		values (1, ${JSON.stringify(to)}::jsonb, ${by}, now())
+		on conflict (id) do update set snapshot = excluded.snapshot,
+		written_by = excluded.written_by, written_at = excluded.written_at`.execute(trx)
+}
+
+const isApplied = (value: unknown): value is Stored['applied'] =>
+	Array.isArray(value) &&
+	value.every(
+		(item: unknown) =>
+			typeof item === 'object' &&
+			item !== null &&
+			'name' in item &&
+			typeof item.name === 'string' &&
+			'checksum' in item &&
+			typeof item.checksum === 'string' &&
+			(!('schema' in item) || typeof item.schema === 'string'),
+	)
+
+/**
+ * The stored snapshot and the applied files, in one query. A deploy that
+ * stopped before it stored its snapshot leaves files and no snapshot.
+ */
+export async function stored(db: Kysely<object>): Promise<Stored | undefined> {
+	try {
+		const { rows } = await sql<{
+			snapshot: unknown
+			written_by: unknown
+			applied: unknown
+		}>`select
+			(select snapshot from ultimabase_schema) as snapshot,
+			(select written_by from ultimabase_schema) as written_by,
+			(select coalesce(json_agg(json_strip_nulls(json_build_object('name', name, 'checksum', checksum, 'schema', schema_hash)) order by name), '[]')
+				from ultimabase_migrations) as applied`.execute(db)
+		const [row] = rows
+		const { snapshot, written_by: by, applied } = row ?? {}
+		if (!isApplied(applied)) throw new Error('ultimabase_migrations is broken.')
+		if (snapshot === null && applied.length === 0) return undefined
+		return {
+			snapshot: snapshot ?? undefined,
+			by: by === 'sync' ? 'sync' : 'deploy',
+			applied,
+		}
+	} catch (error) {
+		if (error instanceof Error && 'code' in error && error.code === '42P01') {
+			return undefined
+		}
+		throw error
+	}
+}
+
+/** Refuses tables that no snapshot describes (D-105). */
+async function refuseTables(
+	trx: Transaction<object>,
+	to: unknown,
+): Promise<void> {
+	const names = Object.keys(parseSnapshot(to).tables)
+	const { rows } = await sql<{ table_name: string }>`
+		select table_name from information_schema.tables
+		where table_schema = current_schema() and table_name = any(${names})
+		order by table_name`.execute(trx)
+	if (rows.length === 0) return
+	const list = rows.map((row) => row.table_name).join(', ')
+	throw new Refused(
+		`The database already holds ${list}, and no UltimaBase snapshot says what they are: sync only builds an empty database.`,
+	)
+}
+
+/** What to do when the database refuses with `code`, after its reason. */
+function hint(code: unknown, transaction: boolean): string {
+	if (code === '55P03') {
+		return ': another query held a table it changes past lock_timeout'
+	}
+	if (code !== '25001') return ''
+	return transaction
+		? ': put it in its own file, whose second line is `-- ultimabase no transaction`'
+		: ': a file without a transaction holds one statement: put each in its own file'
+}
+
+/** Runs `text`, and names `at` when the database refuses it. */
+async function executed(
+	db: Kysely<object>,
+	text: string,
+	at: string,
+	transaction = true,
+): Promise<void> {
+	try {
+		await sql.raw(text).execute(db)
+	} catch (error) {
+		const reason = error instanceof Error ? error.message : String(error)
+		const code: unknown =
+			error instanceof Error ? Reflect.get(error, 'code') : undefined
+		throw new MigrationFailed(
+			at,
+			`${reason}${hint(code, transaction)}`,
+			code === '55P03',
+		)
+	}
+}
+
+async function sync(
+	db: Kysely<object>,
+	run: Extract<Run, { kind: 'sync' }>,
+): Promise<void> {
+	await locked(db, async (trx) => {
+		const { rows } = await sql<{ done: boolean; same: boolean | null }>`
+			select snapshot = ${JSON.stringify(run.to)}::jsonb as done,
+			snapshot = ${run.from === undefined ? null : JSON.stringify(run.from)}::jsonb as same
+			from ultimabase_schema`.execute(trx)
+		const [row] = rows
+		// Another process synced the same models first.
+		if (row?.done === true) return
+		if (row === undefined ? run.from !== undefined : row.same !== true) {
+			throw new Refused(
+				'The database changed while UltimaBase synced it: start again.',
+			)
+		}
+		if (row === undefined) await refuseTables(trx, run.to)
+		for (const statement of run.statements) {
+			await executed(trx, statement.sql, statement.sql)
+		}
+		await store(trx, run.to, 'sync')
+	})
+}
+
+/** Whether another deploy applied `file` first. */
+async function isDone(db: Kysely<object>, file: Migration): Promise<boolean> {
+	const { rows } = await sql`
+		select 1 from ultimabase_migrations where name = ${file.name}`.execute(db)
+	return rows.length > 0
+}
+
+const done = (db: Kysely<object>, file: Migration) =>
+	sql`insert into ultimabase_migrations (name, checksum, schema_hash, applied_at)
+		values (${file.name}, ${file.checksum}, ${file.schema ?? null}, now())`.execute(
+		db,
+	)
+
+/**
+ * Runs a file outside a transaction, under the schema's lock taken for the
+ * session. It waits for locks as long as it needs: `create index
+ * concurrently` waits for running transactions without blocking any, and
+ * one that fails leaves an invalid index.
+ */
+async function outside(db: Kysely<object>, file: Migration): Promise<void> {
+	await db.connection().execute(async (session) => {
+		await sql`select pg_advisory_lock(hashtext(current_schema()))`.execute(
+			session,
+		)
+		try {
+			for (const statement of bookkeeping) await statement.execute(session)
+			if (await isDone(session, file)) return
+			await executed(session, file.sql, file.name, false)
+			await done(session, file)
+		} finally {
+			await sql`select pg_advisory_unlock(hashtext(current_schema()))`.execute(
+				session,
+			)
+		}
+	})
+}
+
+async function deploy(
+	db: Kysely<object>,
+	run: Extract<Run, { kind: 'deploy' }>,
+): Promise<void> {
+	for (const file of run.files) {
+		if (!file.transaction) {
+			await outside(db, file)
+			continue
+		}
+		await locked(db, async (trx) => {
+			if (await isDone(trx, file)) return
+			await executed(trx, file.sql, file.name)
+			await done(trx, file)
+		})
+	}
+	await locked(db, async (trx) => store(trx, run.to, 'deploy'))
+}
+
+export async function apply(db: Kysely<object>, run: Run): Promise<void> {
+	await (run.kind === 'sync' ? sync(db, run) : deploy(db, run))
+}
+
+/** The current schema's tables but an extension's, in name order. */
+export async function tablesOf(db: QueryExecutorProvider): Promise<string[]> {
+	const { rows } = await sql<{ table_name: string }>`
+		select relname as table_name from pg_class
+		where relnamespace = current_schema()::regnamespace
+		and relkind in ('r', 'p') and not relispartition
+		and not exists (select from pg_depend where classid = 'pg_class'::regclass
+			and objid = pg_class.oid and deptype = 'e')
+		order by relname`.execute(db)
+	return rows.map((row) => row.table_name)
+}
+
+/**
+ * Drops every table of the current schema but an extension's, such as
+ * PostGIS's `spatial_ref_sys`; returns the models' tables.
+ */
+export async function reset(db: Kysely<object>): Promise<readonly string[]> {
+	const dropped: string[] = []
+	await locked(db, async (trx) => {
+		const names = await tablesOf(trx)
+		const drop = `drop table ${names.map(id).join(', ')} cascade`
+		await executed(trx, drop, drop)
+		dropped.push(...names.filter((name) => !tables.includes(name)))
+	})
+	return dropped
+}
+
+/**
+ * The host `url` connects to when it is not this machine, read as `pg`
+ * reads it: no host, or a socket's directory, is this machine.
+ */
+export function remoteOf(url: string): string | undefined {
+	const host = parse(url).host || process.env['PGHOST'] || ''
+	// pg reads an IPv6 host without the brackets a URL's hostname keeps.
+	const hostname = host.includes(':') ? `[${host}]` : host
+	return host === '' || host.startsWith('/') || isLocalHost(hostname)
+		? undefined
+		: host
+}

package/src/names.ts

@@ -0,0 +1,217 @@
+// Storage names (D-63, D-98): snake_case tables and columns that `dbName`
+// overrides, the join tables of many-to-many relations, and constraint
+// names that fit PostgreSQL's 63 bytes.
+import { createHash } from 'node:crypto'
+import type { ScalarType, Schema, SchemaField, SchemaModel } from 'ultimabase'
+
+/**
+ * What a column holds: `id` and a `one` key hold uuids, a list the array of
+ * its elements' type (D-127), an integer field `integer` (D-132).
+ */
+export type Kind =
+	ScalarType | 'id' | 'integer' | 'text[]' | 'double precision[]' | 'integer[]'
+
+export type Unique = {
+	readonly name: string
+	readonly fields: readonly string[]
+}
+
+export type Table = {
+	readonly name: string
+	readonly model: SchemaModel
+	readonly fields: ReadonlyMap<string, SchemaField>
+	/** The primary key, unique fields, unique indexes, one-to-one keys. */
+	readonly uniques: readonly Unique[]
+}
+
+/**
+ * A many-to-many side: the relation its join table is named after, the join
+ * table, the column of this side's ids and of the other's, and the tables
+ * those ids belong to.
+ */
+export type Join = {
+	readonly relation: string
+	readonly table: string
+	readonly self: string
+	readonly other: string
+	readonly owner: string
+	readonly target: string
+}
+
+const system: ReadonlyMap<string, readonly [string, Kind]> = new Map([
+	['id', ['id', 'id']],
+	['createdAt', ['created_at', 'dateTime']],
+	['updatedAt', ['updated_at', 'dateTime']],
+])
+
+export function snake(name: string): string {
+	return name
+		.replaceAll(/([a-z0-9])([A-Z])/g, '$1_$2')
+		.replaceAll(/([A-Z])([A-Z][a-z])/g, '$1_$2')
+		.toLowerCase()
+}
+
+/** A name PostgreSQL keeps whole: a long one ends with its hash. */
+export function constraint(name: string): string {
+	if (Buffer.byteLength(name) <= 63) return name
+	let head = name
+	while (Buffer.byteLength(head) > 54) head = head.slice(0, -1)
+	const hash = createHash('sha256').update(name).digest('hex').slice(0, 8)
+	return `${head}_${hash}`
+}
+
+/** The relation field a foreign key belongs to, and the model it links. */
+export type Relation = {
+	readonly model: string
+	readonly field: string
+	readonly target: string
+}
+
+export class Catalog {
+	readonly tables: ReadonlyMap<string, Table>
+	readonly joins: ReadonlyMap<string, Join>
+	/** Constraint name → the model and fields a refusal reports. */
+	readonly uniques: ReadonlyMap<string, { model: string; fields: string[] }>
+	/** Foreign key name → the relation a refusal reports (D-99). */
+	readonly foreignKeys: ReadonlyMap<string, Relation>
+	/** The text search configuration of `_search` (D-148). */
+	readonly language: string
+
+	constructor(schema: Schema, language = 'simple') {
+		this.language = language
+		const tables = new Map<string, Table>()
+		for (const model of schema.models) {
+			const name = model.dbName ?? snake(model.name)
+			const fields = new Map(model.fields.map((field) => [field.name, field]))
+			tables.set(model.name, { name, model, fields, uniques: [] })
+		}
+		this.tables = tables
+		const uniques = new Map<string, { model: string; fields: string[] }>()
+		for (const table of tables.values()) {
+			const sets: (readonly string[])[] = [['id']]
+			for (const field of table.model.fields) {
+				const stored =
+					field.kind === 'scalar' || (field.kind === 'one' && field.stored)
+				if (stored && field.unique) sets.push([field.name])
+			}
+			for (const index of table.model.indexes) {
+				if (index.unique) sets.push(index.fields)
+			}
+			const named = sets.map((fields, at) => ({
+				name: constraint(
+					at === 0
+						? `${table.name}_pkey`
+						: `${table.name}_${fields.map((field) => this.column(table, field)).join('_')}_key`,
+				),
+				fields,
+			}))
+			tables.set(table.model.name, { ...table, uniques: named })
+			for (const { name, fields } of named) {
+				uniques.set(name, { model: table.model.name, fields: [...fields] })
+			}
+		}
+		this.uniques = uniques
+		this.joins = this.joinsOf()
+		this.foreignKeys = this.foreignKeysOf()
+	}
+
+	table(model: string): Table {
+		const table = this.tables.get(model)
+		if (table === undefined) throw new Error(`Unknown model ${model}.`)
+		return table
+	}
+
+	/** The column of a system field, a scalar or a stored `one`. */
+	column(table: Table, name: string): string {
+		const known = system.get(name)
+		if (known !== undefined) return known[0]
+		const field = table.fields.get(name)
+		if (field?.kind === 'scalar') return field.dbName ?? snake(name)
+		if (field?.kind === 'one' && field.stored) {
+			return field.dbName ?? `${snake(name)}_id`
+		}
+		throw new Error(`${table.model.name}.${name} has no column.`)
+	}
+
+	kind(table: Table, name: string): Kind {
+		const known = system.get(name)
+		if (known !== undefined) return known[1]
+		const field = table.fields.get(name)
+		if (field?.kind !== 'scalar') return 'id'
+		if (field.integer === true) return field.list ? 'integer[]' : 'integer'
+		if (!field.list) return field.type
+		return field.type === 'number' ? 'double precision[]' : 'text[]'
+	}
+
+	join(model: string, field: string): Join {
+		const join = this.joins.get(`${model}.${field}`)
+		if (join === undefined) throw new Error(`${model}.${field} has no join.`)
+		return join
+	}
+
+	/** Named as the snapshot names them: a `one` key, a join's column. */
+	private foreignKeysOf(): Map<string, Relation> {
+		const keys = new Map<string, Relation>()
+		for (const table of this.tables.values()) {
+			const model = table.model.name
+			for (const field of table.model.fields) {
+				let key: string
+				if (field.kind === 'one' && field.stored) {
+					key = `${table.name}_${this.column(table, field.name)}_fkey`
+				} else if (
+					field.kind === 'many' &&
+					field.join &&
+					field.from === undefined
+				) {
+					const join = this.join(model, field.name)
+					key = `${join.table}_${join.other}_fkey`
+				} else continue
+				const target = field.target
+				keys.set(constraint(key), { model, field: field.name, target })
+			}
+		}
+		return keys
+	}
+
+	/**
+	 * Both sides of each many-to-many, named after the side whose table sorts
+	 * first, so a model renamed with its `dbName` keeps its join (D-98).
+	 */
+	private joinsOf(): Map<string, Join> {
+		const joins = new Map<string, Join>()
+		for (const table of this.tables.values()) {
+			for (const field of table.model.fields) {
+				// A view has no join of its own (D-151).
+				if (field.kind !== 'many' || !field.join || field.from !== undefined)
+					continue
+				const target = this.table(field.target)
+				const self = table === target
+				const first = self
+					? field.name < field.inverse
+					: table.name < target.name
+				if (!first) continue
+				const name = `${table.name}_${snake(field.name)}`
+				const mine = self ? `${snake(field.inverse)}_id` : `${table.name}_id`
+				const theirs = self ? `${snake(field.name)}_id` : `${target.name}_id`
+				const relation = `${table.model.name}.${field.name}`
+				joins.set(relation, {
+					relation,
+					table: name,
+					self: mine,
+					other: theirs,
+					owner: table.name,
+					target: target.name,
+				})
+				joins.set(`${target.model.name}.${field.inverse}`, {
+					relation,
+					table: name,
+					self: theirs,
+					other: mine,
+					owner: target.name,
+					target: table.name,
+				})
+			}
+		}
+		return joins
+	}
+}

package/src/notify.ts

@@ -0,0 +1,101 @@
+// What a write changed, told by its own statements (D-154): each part's rows
+// go to `pg_notify` by chunks that fit its 8000 bytes, or as `null` past
+// `feedIds`. The notifications leave with the commit; a rollback drops them.
+import { sql } from 'kysely'
+import type { Changed } from 'ultimabase'
+import { feedFields, feedIds } from 'ultimabase'
+import type { Sql } from './filter.ts'
+
+/** What the feed says of a part: its head, then an item per row. */
+export type Noted = {
+	/** The JSON array's start: kind, model, then the field or fields. */
+	readonly head: string
+	readonly item: Sql
+	/** How many items a notification holds. */
+	readonly per: number
+}
+
+/** A part of rows, told by their ids: 150 of 40 bytes fit beside a head. */
+export function rows(
+	kind: 'insert' | 'update' | 'delete',
+	model: string,
+	fields?: readonly string[],
+): Noted {
+	const listed =
+		fields === undefined || JSON.stringify(fields).length > feedFields
+			? null
+			: fields
+	const head = JSON.stringify([kind, model, listed]).slice(0, -1)
+	return { head: `${head},`, item: sql.id('id'), per: 150 }
+}
+
+/** A part of a join table, told by its pairs, the model's id first. */
+export function pairs(
+	kind: 'link' | 'unlink',
+	model: string,
+	field: string,
+	self: string,
+	other: string,
+): Noted {
+	const head = JSON.stringify([kind, model, field]).slice(0, -1)
+	const item = sql`json_build_array(${sql.id(self)}, ${sql.id(other)})`
+	return { head: `${head},`, item, per: 75 }
+}
+
+/** The CTE that tells `channel` what each of `parts`, by its CTE, wrote. */
+export function notifying(
+	channel: string,
+	parts: readonly (readonly [Sql, Noted])[],
+): Sql {
+	const to = sql.lit(channel)
+	const most = sql.lit(feedIds)
+	const each = parts.map(([from, noted]) => {
+		const head = sql.lit(noted.head)
+		const count = sql`(select count(*) from ${from})`
+		const chunked = sql`select ${noted.item} as "item", (row_number() over () - 1) / ${sql.lit(noted.per)} as "k" from ${from}`
+		return sql`select pg_notify(${to}, ${head} || json_agg("s"."item")::text || ']') from (${chunked}) as "s" where ${count} <= ${most} group by "s"."k" union all select pg_notify(${to}, ${head} || 'null]') where ${count} > ${most}`
+	})
+	return sql`"#feed" as materialized (${sql.join(each, sql` union all `)})`
+}
+
+const kinds = new Set(['insert', 'update', 'delete', 'link', 'unlink'])
+
+const strings = (value: unknown): value is string[] =>
+	Array.isArray(value) && value.every((item) => typeof item === 'string')
+
+const pairList = (value: unknown): value is [string, string][] =>
+	Array.isArray(value) &&
+	value.every((item) => strings(item) && item.length === 2)
+
+/** A notification's payload as the change it tells, if it is one. */
+export function changedOf(payload: string | undefined): Changed | undefined {
+	let parsed: unknown
+	try {
+		parsed = JSON.parse(payload ?? '')
+	} catch {
+		return undefined
+	}
+	if (!Array.isArray(parsed) || parsed.length !== 4) return undefined
+	const kind: unknown = parsed[0]
+	const model: unknown = parsed[1]
+	const target: unknown = parsed[2]
+	const listed: unknown = parsed[3]
+	if (typeof kind !== 'string' || !kinds.has(kind)) return undefined
+	if (typeof model !== 'string') return undefined
+	if (kind === 'link' || kind === 'unlink') {
+		if (typeof target !== 'string') return undefined
+		if (listed !== null && !pairList(listed)) return undefined
+		return { kind, model, field: target, ...(listed ? { pairs: listed } : {}) }
+	}
+	if (kind !== 'insert' && kind !== 'update' && kind !== 'delete') {
+		return undefined
+	}
+	if (target !== null && !strings(target)) return undefined
+	if (listed !== null && !strings(listed)) return undefined
+	return {
+		kind,
+		model,
+		...(target ? { fields: target } : {}),
+		...(listed ? { ids: listed } : {}),
+	}
+}

package/src/numbers.ts

@@ -0,0 +1,64 @@
+// Number and list updates (D-132, D-149): computed by the UPDATE on the row
+// as it finds it, and a number outside its column's type refused as the
+// kernel's error.
+import type { Arithmetic, ListUpdate, Write } from 'ultimabase'
+import { OutOfRange } from 'ultimabase'
+import type { Sql } from './filter.ts'
+import { value } from './filter.ts'
+import type { Catalog, Kind, Table } from './names.ts'
+import { sql } from 'kysely'
+
+// An integer divided by an integer truncates toward zero, as D-132 says.
+const operators = { increment: '+', decrement: '-', multiply: '*', divide: '/' }
+
+/**
+ * A list update on `column`: `pushUnique` appends each element the list
+ * lacks once, in its first position's order; `remove` keeps the others in
+ * theirs.
+ */
+function listed(
+	column: Sql,
+	{ operation, elements }: ListUpdate,
+	kind: Kind,
+): Sql {
+	const given = value(kind, elements)
+	if (operation === 'push') return sql`${column} || ${given}`
+	if (operation === 'pushUnique') {
+		return sql`${column} || array(select e from unnest(${given}) with ordinality as o(e, i) where e <> all(${column}) group by e order by min(i))`
+	}
+	return sql`array(select e from unnest(${column}) with ordinality as o(e, i) where e <> all(${given}) order by i)`
+}
+
+/** An update's computed assignments, `null` counting as 0. */
+export function computed(
+	catalog: Catalog,
+	table: Table,
+	compute: { readonly [field: string]: Arithmetic | ListUpdate } = {},
+): Sql[] {
+	return Object.entries(compute).map(([field, operation]) => {
+		const column = sql.id(catalog.column(table, field))
+		const kind = catalog.kind(table, field)
+		if ('elements' in operation) {
+			return sql`${column} = ${listed(column, operation, kind)}`
+		}
+		const operand = value(kind, operation.by)
+		return sql`${column} = coalesce(${column}, 0) ${sql.raw(operators[operation.operation])} ${operand}`
+	})
+}
+
+/**
+ * PostgreSQL's numeric overflow, 22003, names no column: the error names
+ * the write's first computed number.
+ */
+export function overflowed(given: Write, error: unknown): unknown {
+	const code =
+		error instanceof Error && 'code' in error ? error.code : undefined
+	if (code !== '22003') return error
+	for (const change of given.changes) {
+		if (change.kind !== 'update') continue
+		const computes = Object.entries(change.compute ?? {})
+		const [field] = computes.find(([, operation]) => 'by' in operation) ?? []
+		if (field !== undefined) return new OutOfRange(change.model, field)
+	}
+	return error
+}