@ultimabase/postgres 0.0.0-stage → 0.1.0

package/src/adapter.ts

@@ -0,0 +1,349 @@
+// The PostgreSQL adapter (D-47): Kysely builds the statements, `pg` runs
+// them. `log` lets the tests count the statements (D-98); `sql` runs the
+// application's own, and a test file may run in a schema of its own (D-137).
+import type {
+	LogEvent,
+	QueryExecutorProvider,
+	Transaction as Open,
+} from 'kysely'
+import { Kysely, PostgresDialect, sql } from 'kysely'
+import { Pool } from 'pg'
+import { CheckFailed, Refused } from 'ultimabase'
+import { joining, scratchable } from 'ultimabase/internal'
+import type {
+	Adapter,
+	CountRead,
+	Feed,
+	Logger,
+	Read,
+	Row,
+	Schema,
+	Transaction,
+	Write,
+} from 'ultimabase'
+import { aggregate } from './aggregate.ts'
+import { decodeRows } from './decode.ts'
+import { apply, remoteOf, reset, stored } from './migrations.ts'
+import { Catalog } from './names.ts'
+import { overflowed } from './numbers.ts'
+import { fillsOf } from './fills.ts'
+import type { Listener } from './listen.ts'
+import { channelOf, listening } from './listen.ts'
+import { plan } from './plan.ts'
+import { pooled } from './pool.ts'
+import { compileCount, compileFind } from './read.ts'
+import {
+	connecting,
+	dropped,
+	emptied,
+	scratchName,
+	trigrams,
+} from './scratch.ts'
+import { parseSnapshot, snapshotOf } from './snapshot.ts'
+import type { Statement } from './write.ts'
+import { compileWrite, translate } from './write.ts'
+
+export type Options = {
+	readonly url: string | undefined
+	/** How many connections the pool opens at most (D-47). */
+	readonly connections?: number
+	/** The milliseconds a query waits for a connection, or to open one. */
+	readonly wait?: number
+	readonly log?: (event: LogEvent) => void
+	/** The text search configuration of every `_search`, `'simple'` by default (D-148). */
+	readonly searchLanguage?: string
+}
+
+/** What `postgres()` returns: an adapter that also runs raw SQL (D-137). */
+export type Postgres = Adapter & {
+	/**
+	 * Runs one statement, in the open transaction if any, and returns its
+	 * rows; each `${value}` is a parameter. No rule or hook runs.
+	 */
+	readonly sql: <R extends object = Record<string, unknown>>(
+		strings: TemplateStringsArray,
+		...values: readonly unknown[]
+	) => Promise<R[]>
+}
+
+/** What the callback a rollback ends throws, which `rollback()` swallows. */
+const rolledBack = new Error('The transaction was rolled back.')
+
+/** `key` of a statement's first row, if it returned one. */
+function firstOf(rows: readonly unknown[], key: string): unknown {
+	const [first] = rows
+	if (typeof first !== 'object' || first === null) return undefined
+	const value: unknown = Reflect.get(first, key)
+	return value
+}
+
+async function find(
+	catalog: Catalog,
+	db: QueryExecutorProvider,
+	read: Read,
+): Promise<Row[]> {
+	const { rows } = await compileFind(catalog, read).execute(db)
+	return decodeRows(catalog, read.model, firstOf(rows, 'rows'), read)
+}
+
+async function count(
+	catalog: Catalog,
+	db: QueryExecutorProvider,
+	read: CountRead,
+): Promise<number> {
+	const { rows } = await compileCount(catalog, read).execute(db)
+	const value = firstOf(rows, 'count')
+	if (typeof value !== 'number') throw new Error('The count is not a number.')
+	return value
+}
+
+/** Runs a write's statements; the one that reads returns the rows. */
+async function run(
+	catalog: Catalog,
+	db: QueryExecutorProvider,
+	given: Write,
+	statements: readonly Statement[],
+): Promise<Row[]> {
+	let rows: unknown
+	try {
+		for (const statement of statements) {
+			const [first] = (await statement.sql.execute(db)).rows
+			for (let at = 0; at < statement.checks; at += 1) {
+				if (first?.[`c${at}`] !== true) throw new CheckFailed(at)
+			}
+			if (first !== undefined && 'rows' in first) rows = first['rows']
+		}
+	} catch (error) {
+		throw translate(catalog, overflowed(given, error))
+	}
+	const read = given.read
+	// A `before` read of a model the write deletes nothing of reads none.
+	if (read === undefined || rows === undefined) return []
+	return decodeRows(catalog, read.model, rows, read)
+}
+
+async function write(
+	catalog: Catalog,
+	db: QueryExecutorProvider,
+	given: Write,
+	channel: string | undefined,
+): Promise<Row[]> {
+	return run(catalog, db, given, compileWrite(catalog, given, channel))
+}
+
+/** Refuses a server below PostgreSQL 15 (D-50). */
+export function requireVersion(version: string | undefined): void {
+	if (!(Number(version) >= 150000)) {
+		throw new Refused(`UltimaBase needs PostgreSQL 15 or later.`)
+	}
+}
+
+/** What a case-insensitive filter misses when `lower('É') <> 'é'` (D-140). */
+export function foldWarning(ctype: string): string {
+	return `PostgreSQL folds case under its ${ctype} locale for ASCII only: a case-insensitive filter matches É and é in memory() but not here. Create the database with a UTF-8 locale, such as en_US.UTF-8.`
+}
+
+export function createAdapter(options: Options): Postgres {
+	const { url, connections = 10, wait = 10_000, searchLanguage } = options
+	const remote = url ? remoteOf(url) : undefined
+	let state:
+		| {
+				db: Kysely<object>
+				catalog: Catalog
+				/** The feed's channel and listener, when it has a taker (D-154). */
+				channel?: string
+				listener?: Listener
+		  }
+		| undefined
+	/** A test file's schema, and whether a start created it. */
+	let scratch: { readonly name: string; made: boolean } | undefined
+	/** Each open transaction's own, for `sql` to join. */
+	const opens = new WeakMap<Transaction, Open<object>>()
+	const started = () => {
+		if (state === undefined) throw new Error('postgres() is not started.')
+		return state
+	}
+	const adapter: Postgres = {
+		start: async (schema: Schema, logger: Logger, feed?: Feed) => {
+			if (!url) {
+				throw new Refused(
+					'postgres() was given no url: is the variable it reads set?',
+				)
+			}
+			if (!Number.isInteger(connections) || connections < 1) {
+				throw new Refused(
+					`postgres() takes connections as a whole number, 1 or more, not ${connections}.`,
+				)
+			}
+			const catalog = new Catalog(schema, searchLanguage)
+			// Refuses two things stored under one name before it connects.
+			const { tables } = snapshotOf(catalog)
+			const trigram = Object.values(tables).some(
+				(table) => Object.keys(table.trigrams).length > 0,
+			)
+			const extensions =
+				scratch !== undefined && trigram ? [await trigrams(url)] : []
+			const config = connecting(url, scratch?.name, extensions)
+			const pool = new Pool({
+				...config,
+				max: connections,
+				connectionTimeoutMillis: wait,
+			})
+			// A connection the server ends would crash the process unheard. An
+			// idle one the pool drops, and the next query opens another; one a
+			// transaction holds fails its next statement, and the pool drops it
+			// once given back.
+			pool.on('error', () => undefined)
+			pool.on('connect', (client) => {
+				client.on('error', () => undefined)
+				// Compiling a nested read costs a hundred times more than running
+				// it; queued, this runs before the client's first query (D-47).
+				client.query('set jit = off').catch(() => undefined)
+			})
+			const dialect = new PostgresDialect({
+				pool: pooled(url, pool, connections, wait),
+			})
+			const db = new Kysely<object>({
+				dialect,
+				...(options.log === undefined ? {} : { log: options.log }),
+			})
+			try {
+				const { rows } = await sql<{
+					version: string
+					folds: boolean
+					ctype: string
+					language: boolean
+					schema: string
+				}>`select current_setting('server_version_num') as version, current_schema() as schema,
+					lower('É') = 'é' as folds,
+					(select datctype from pg_database where datname = current_database()) as ctype,
+					exists (select from pg_ts_config as c join pg_namespace as n on n.oid = c.cfgnamespace where (c.cfgname = ${catalog.language} and pg_ts_config_is_visible(c.oid)) or n.nspname || '.' || c.cfgname = ${catalog.language}) as language`.execute(
+					db,
+				)
+				const [server] = rows
+				requireVersion(server?.version)
+				if (server?.language === false) {
+					throw new Refused(
+						`postgres() searches in '${catalog.language}', a text search configuration this server does not have.`,
+					)
+				}
+				if (server !== undefined && !server.folds) {
+					logger.warn(foldWarning(server.ctype))
+				}
+				if (scratch !== undefined) {
+					scratch.made = true
+					await emptied(db, scratch.name)
+				}
+				const channel = channelOf(scratch?.name ?? server?.schema ?? '')
+				state =
+					feed === undefined
+						? { db, catalog }
+						: {
+								db,
+								catalog,
+								channel,
+								listener: await listening(config, channel, feed, logger),
+							}
+			} catch (error) {
+				await db.destroy()
+				throw error
+			}
+		},
+		stop: async () => {
+			const stopping = state
+			state = undefined
+			await stopping?.listener?.stop()
+			await stopping?.db.destroy()
+		},
+		find: async (read) => find(started().catalog, started().db, read),
+		count: async (read) => count(started().catalog, started().db, read),
+		aggregate: async (read) => aggregate(started().catalog, started().db, read),
+		write: async (given) => {
+			const { catalog, db, channel } = started()
+			const statements = compileWrite(catalog, given, channel)
+			// One statement is atomic alone: no BEGIN, no COMMIT (D-99).
+			if (statements.length <= 1) return run(catalog, db, given, statements)
+			try {
+				return await db
+					.transaction()
+					.execute(async (trx) => run(catalog, trx, given, statements))
+			} catch (error) {
+				// A deferred foreign key checks at the COMMIT (D-99).
+				throw translate(catalog, error)
+			}
+		},
+		migrations: {
+			snapshot: (schema) => snapshotOf(new Catalog(schema, searchLanguage)),
+			plan: (from, schema) => {
+				const catalog = new Catalog(schema, searchLanguage)
+				return plan(
+					from === undefined ? undefined : parseSnapshot(from),
+					snapshotOf(catalog),
+					fillsOf(catalog),
+				)
+			},
+			stored: async () => stored(started().db),
+			apply: async (given) => apply(started().db, given),
+			reset: async () => reset(started().db),
+			...(remote === undefined ? {} : { remote }),
+		},
+		begin: async (): Promise<Transaction> => {
+			const { catalog, db, channel } = started()
+			const opened = Promise.withResolvers<Open<object>>()
+			const ending = Promise.withResolvers<boolean>()
+			// Kysely gives the connection back however its transaction ends, a
+			// failed ROLLBACK included; a COMMIT that fails rolls back.
+			const ended = db.transaction().execute(async (trx) => {
+				opened.resolve(trx)
+				if (!(await ending.promise)) throw rolledBack
+			})
+			// A BEGIN that fails rejects `opened` too.
+			void ended.catch(opened.reject)
+			const trx = await opened.promise
+			const end = async (commit: boolean) => {
+				ending.resolve(commit)
+				await ended
+			}
+			const tx: Transaction = {
+				find: async (read) => find(catalog, trx, read),
+				count: async (read) => count(catalog, trx, read),
+				aggregate: async (read) => aggregate(catalog, trx, read),
+				write: async (given) => write(catalog, trx, given, channel),
+				commit: async () => {
+					try {
+						await end(true)
+					} catch (error) {
+						throw translate(catalog, error)
+					}
+				},
+				rollback: async () => {
+					try {
+						await end(false)
+					} catch (error) {
+						if (error !== rolledBack) throw error
+					}
+				},
+			}
+			opens.set(tx, trx)
+			return tx
+		},
+		sql: async <R extends object>(
+			strings: TemplateStringsArray,
+			...values: readonly unknown[]
+		) =>
+			joining(adapter, async (tx) => {
+				const db = (tx && opens.get(tx)) ?? started().db
+				return (await sql<R>(strings, ...values).execute(db)).rows
+			}),
+	}
+	return scratchable(adapter, () => {
+		scratch = { name: scratchName(), made: false }
+		const own = scratch
+		return {
+			drop: async () => {
+				scratch = undefined
+				if (own.made && url) await dropped(url, own.name)
+			},
+		}
+	})
+}

package/src/aggregate.ts

@@ -0,0 +1,168 @@
+// An aggregate as one statement (D-143), in groups (D-146): an inner select
+// gives each row's keys and aggregated columns, the outer one groups, keeps
+// those `having` passes (D-149), sorts and pages them. Each group's values
+// are a jsonb array: a sum and an average as float8, a date in UTC as a read
+// returns it, a decimal as text, exact (D-145).
+import type { QueryExecutorProvider } from 'kysely'
+import { sql } from 'kysely'
+import type { Aggregate, AggregateRead, Filter, GroupKey } from 'ultimabase'
+import type { Sql } from './filter.ts'
+import { all, any, isFilters } from './filter.ts'
+import type { Catalog, Kind, Table } from './names.ts'
+import { Compiler, sorted } from './read.ts'
+
+/** A value as jsonb keeps it: a decimal as text, a date in UTC. */
+function shown(kind: Kind | undefined, item: Sql): Sql {
+	if (kind === 'decimal') return sql`cast(${item} as text)`
+	return kind === 'dateTime' ? sql`(${item} at time zone 'UTC')` : item
+}
+
+/** The aggregate of `column`, as PostgreSQL computes it. */
+function computed({ operation, field }: Aggregate, column: Sql): Sql {
+	if (field === undefined) return sql`count(*)`
+	return sql`${sql.raw(operation)}(${column})`
+}
+
+/** The aggregate as jsonb keeps it; a sum and an average as float8. */
+function value(kind: Kind | undefined, asked: Aggregate, column: Sql): Sql {
+	const result = computed(asked, column)
+	const { operation, field } = asked
+	if (field === undefined || operation === 'count') return result
+	if (kind === 'decimal' && operation !== 'avg') return shown(kind, result)
+	if (operation === 'sum' || operation === 'avg') {
+		return sql`cast(${result} as float8)`
+	}
+	return shown(kind, result)
+}
+
+/** The kind of a key's value: its field's, on the model `through` reaches. */
+function kindOf(catalog: Catalog, table: Table, key: GroupKey): Kind {
+	let at = table
+	for (const step of key.through ?? []) {
+		const field = at.fields.get(step.relation)
+		if (field === undefined || field.kind === 'scalar') {
+			throw new Error(`${at.model.name}.${step.relation} is not a relation.`)
+		}
+		at = catalog.table(field.target)
+	}
+	return catalog.kind(at, key.field)
+}
+
+/** Each aggregate's kind, `undefined` for a count of rows. */
+function kinds(
+	catalog: Catalog,
+	table: Table,
+	read: AggregateRead,
+): (Kind | undefined)[] {
+	return read.aggregates.map(({ field }) =>
+		field === undefined ? undefined : catalog.kind(table, field),
+	)
+}
+
+export function compileAggregate(catalog: Catalog, read: AggregateRead): Sql {
+	const compiler = new Compiler(catalog)
+	const { filters } = compiler
+	const alias = filters.alias()
+	const grouped = filters.alias()
+	const table = catalog.table(read.model)
+	const by = read.by ?? []
+	const inner = [
+		...by.map(
+			(key, at) =>
+				sql`${compiler.key(table, alias, key)} as ${sql.id(`k${at}`)}`,
+		),
+		...read.aggregates.flatMap(({ field }, at) =>
+			field === undefined
+				? []
+				: [sql`${filters.column(table, alias, field)} as ${sql.id(`c${at}`)}`],
+		),
+	]
+	const keys = by.map((_, at) => sql.id(grouped, `k${at}`))
+	const types = kinds(catalog, table, read)
+	const aggregates = read.aggregates.map((each, at) => {
+		const column = sql.id(grouped, `c${at}`)
+		return { kind: types[at], each, column }
+	})
+	const values = [
+		...by.map((key, at) =>
+			shown(kindOf(catalog, table, key), sql.id(grouped, `k${at}`)),
+		),
+		...aggregates.map(({ kind, each, column }) => value(kind, each, column)),
+	]
+	// PostgreSQL takes at most 100 arguments per call.
+	const arrays: Sql[] = []
+	for (let at = 0; at < values.length; at += 100) {
+		arrays.push(sql`jsonb_build_array(${sql.join(values.slice(at, at + 100))})`)
+	}
+	const where = filters.where(read.model, alias, read.where)
+	const rows = sql`(select ${sql.join([sql`1`, ...inner])} from ${sql.id(table.name)} as ${sql.id(alias)} where ${where}) as ${sql.id(grouped)}`
+	const group = by.length === 0 ? sql`` : sql` group by ${sql.join(keys)}`
+	// A key or an aggregate, by its position.
+	const at = (index: number): { readonly item: Sql; readonly kind: Kind } => {
+		const found = aggregates[index - by.length]
+		const key = by[index]
+		if (found === undefined) {
+			if (key === undefined) throw new Error(`No value at ${index}.`)
+			const kind = kindOf(catalog, table, key)
+			return { item: sql.id(grouped, `k${index}`), kind }
+		}
+		const { operation } = found.each
+		const kind =
+			operation === 'min' || operation === 'max'
+				? (found.kind ?? 'number')
+				: operation === 'sum' && found.kind === 'decimal'
+					? 'decimal'
+					: 'number'
+		return { item: computed(found.each, found.column), kind }
+	}
+	const kept = (filter: Filter): Sql =>
+		all(Object.entries(filter).map(([key, part]) => condition(key, part)))
+	function condition(key: string, part: Filter[string]): Sql {
+		if (part === undefined) return sql`true`
+		if (!isFilters(part)) {
+			const { item, kind } = at(Number(key))
+			return filters.compare(item, kind, part)
+		}
+		const list = part.map((each) => kept(each))
+		if (key === 'AND') return all(list)
+		if (key === 'OR') return any(list)
+		return all(list.map((each) => sql`(not ${each})`))
+	}
+	const having =
+		read.having === undefined ? sql`` : sql` having ${kept(read.having)}`
+	const sorts = (read.orderBy ?? []).map((each) =>
+		sorted(at(each.at).item, each),
+	)
+	const order = sorts.length === 0 ? sql`` : sql` order by ${sql.join(sorts)}`
+	const limit = read.take === undefined ? sql`` : sql` limit ${read.take}`
+	const offset = read.skip === undefined ? sql`` : sql` offset ${read.skip}`
+	return sql`select ${sql.join(arrays, sql` || `)} as "values" from ${rows}${group}${having}${order}${limit}${offset}`
+}
+
+/** Each group's values, a date back as a `Date`. */
+export async function aggregate(
+	catalog: Catalog,
+	db: QueryExecutorProvider,
+	read: AggregateRead,
+): Promise<unknown[][]> {
+	const { rows } = await compileAggregate(catalog, read).execute(db)
+	const table = catalog.table(read.model)
+	const by = read.by ?? []
+	const types = [
+		...by.map((key) => kindOf(catalog, table, key)),
+		...kinds(catalog, table, read),
+	]
+	return rows.map((row) => {
+		const values: unknown =
+			typeof row === 'object' && row !== null
+				? Reflect.get(row, 'values')
+				: undefined
+		if (!Array.isArray(values)) throw new Error('The values are not a list.')
+		return types.map((kind, index): unknown => {
+			const item: unknown = values[index]
+			return typeof item === 'string' && kind === 'dateTime'
+				? new Date(`${item}Z`)
+				: item
+		})
+	})
+}

package/src/decode.ts

@@ -0,0 +1,46 @@
+// What a read's jsonb holds, back as the rows the kernel expects (D-73): a
+// date arrives as text in UTC without its zone, everything else as written.
+import type { Nested, Row } from 'ultimabase'
+import { isRow } from 'ultimabase/internal'
+import type { Catalog } from './names.ts'
+
+const dates = new Set(['createdAt', 'updatedAt'])
+
+function decodeRow(
+	catalog: Catalog,
+	model: string,
+	value: unknown,
+	nested: Nested,
+): Row {
+	if (!isRow(value)) throw new Error(`A ${model} row has no id.`)
+	const table = catalog.table(model)
+	const row: { id: string; [field: string]: unknown } = { ...value }
+	for (const [name, item] of Object.entries(value)) {
+		const field = table.fields.get(name)
+		const inner = nested.include?.[name]
+		if (
+			typeof item === 'string' &&
+			(dates.has(name) ||
+				(field?.kind === 'scalar' && field.type === 'dateTime'))
+		) {
+			row[name] = new Date(`${item}Z`)
+		} else if (inner !== undefined && field?.kind === 'many') {
+			row[name] = decodeRows(catalog, field.target, item, inner)
+		} else if (inner !== undefined && field?.kind === 'one' && item !== null) {
+			row[name] = decodeRow(catalog, field.target, item, inner)
+		}
+	}
+	return row
+}
+
+/** The rows of a jsonb array of `model`, with the relations `nested` includes. */
+export function decodeRows(
+	catalog: Catalog,
+	model: string,
+	value: unknown,
+	nested: Nested,
+): Row[] {
+	if (!Array.isArray(value))
+		throw new Error(`The ${model} rows are not a list.`)
+	return value.map((item: unknown) => decodeRow(catalog, model, item, nested))
+}

package/src/fills.ts

@@ -0,0 +1,52 @@
+// The fixed defaults a column added to a table with rows fills them with
+// (D-105); a list's is `[]` when it has none (D-120, D-127).
+import type { Catalog, Kind } from './names.ts'
+
+export const literal = (value: string): string =>
+	`'${value.replaceAll("'", "''")}'`
+
+/** A fixed default as SQL, and as the log shows it: `'draft'`. */
+export type Fill = { readonly sql: string; readonly shown: string }
+
+/** Each table's columns with a fixed default (D-105). */
+export type Fills = {
+	readonly [table: string]: { readonly [column: string]: Fill }
+}
+
+function fillOf(kind: Kind, value: unknown): Fill {
+	if (kind.endsWith('[]') && Array.isArray(value)) {
+		const items = value.map(
+			(item) => `"${String(item).replaceAll(/[\\"]/g, '\\$&')}"`,
+		)
+		const text = literal(`{${items.join(',')}}`)
+		return { sql: `${text}::${kind}`, shown: text }
+	}
+	if (kind === 'json') {
+		const text = literal(JSON.stringify(value))
+		return { sql: `${text}::jsonb`, shown: text }
+	}
+	if (value instanceof Date) {
+		const text = literal(value.toISOString())
+		return { sql: `${text}::timestamptz`, shown: text }
+	}
+	const text = typeof value === 'string' ? literal(value) : String(value)
+	return { sql: text, shown: text }
+}
+
+/** The columns whose field has a fixed default, which rows take when added. */
+export function fillsOf(catalog: Catalog): Fills {
+	const fills: Record<string, Record<string, Fill>> = {}
+	for (const table of catalog.tables.values()) {
+		const columns: Record<string, Fill> = {}
+		for (const field of table.model.fields) {
+			if (field.kind !== 'scalar') continue
+			// A list's rows hold `[]` when nothing else is given (D-120).
+			const given = field.list ? (field.default ?? []) : field.default
+			if (given === undefined) continue
+			const kind = catalog.kind(table, field.name)
+			columns[catalog.column(table, field.name)] = fillOf(kind, given)
+		}
+		fills[table.name] = columns
+	}
+	return fills
+}