@fabrika/platform-node 0.3.3 → 0.4.0

package/src/database-code.ts

@@ -0,0 +1,164 @@
+// The catalog half of ADR-0017's migration rules: the runtime witness. Postgres has already resolved every name the
+// schema stores, so this reads the catalog instead of the SQL text: the runner runs it inside each migration's
+// transaction, before its commit, and `@fabrika/db`'s codegen runs it at every start. It refuses what escapes the
+// schema and any database object that no migration declares, such as one created at runtime.
+
+import type { DatabaseObject } from './migration-classify'
+
+const IDENTIFIER = /^[a-z_][a-z0-9_]*$/
+const encoder = new TextEncoder()
+
+/** A text literal with no quote and no backslash in it, whatever the session's `standard_conforming_strings`. */
+function hexText(value: string): string {
+	const hex = Array.from(encoder.encode(value), (byte) => byte.toString(16).padStart(2, '0')).join('')
+	return `pg_catalog.convert_from(pg_catalog.decode('${hex}', 'hex'), 'UTF8')`
+}
+
+/** The manifest as a `text[]` of `<kind> <name>`, the names `objects` gives the schema's objects. */
+function declaredArray(declared: readonly DatabaseObject[]): string {
+	if (declared.length === 0) return 'ARRAY[]::pg_catalog.text[]'
+	return `ARRAY[${declared.map((object) => hexText(`${object.kind} ${object.name}`)).join(', ')}]::pg_catalog.text[]`
+}
+
+/** A catalog's `regclass`, qualified, since the query may run with the application's schema first on the path. */
+const catalog = (table: string): string => `'pg_catalog.${table}'::pg_catalog.regclass`
+
+/**
+ * One row (`violation`) per escape and per undeclared object in `schema`. Escapes: a function that is `SECURITY
+ * DEFINER`, has a `SET` clause or a language other than `sql` and `plpgsql`; an event trigger on a function of the
+ * schema; and any dependency of an object of the schema on a type, function, relation, operator, operator class or
+ * collation outside the schema and `pg_catalog`; a routine that shadows a `pg_catalog` name or has more than one
+ * signature; a sequence other than an identity column's. Undeclared: a function, procedure or aggregate, operator,
+ * trigger (a foreign key's excepted), rule, view, policy or domain that `declared` does not name. Every name is
+ * qualified with `pg_catalog`, the query has no backslash, and the manifest is
+ * hex text, so the session's `search_path` and `standard_conforming_strings` cannot change what it finds.
+ */
+export function databaseCodeQuery(schema: string, declared: readonly DatabaseObject[]): string {
+	if (!IDENTIFIER.test(schema)) throw new Error(`Invalid application schema: ${schema}`)
+	// A constant, not a subquery, so the planner sizes each scan of the schema and does not reach for JIT.
+	const namespace = `'${schema}'::pg_catalog.regnamespace::pg_catalog.oid`
+	const trusted = `(SELECT oid FROM pg_catalog.pg_namespace WHERE nspname IN ('${schema}', 'pg_catalog'))`
+	const outside = (kind: string, table: string, name: string, namespaceColumn: string): string =>
+		`SELECT d.object || ' depends on the ${kind} ' || s.nspname || '.' || r.${name}
+	FROM dependencies d
+	JOIN pg_catalog.${table} r ON d.refclassid = ${catalog(table)} AND r.oid = d.refobjid
+	JOIN pg_catalog.pg_namespace s ON s.oid = r.${namespaceColumn}
+	WHERE r.${namespaceColumn} NOT IN ${trusted}`
+	return `
+WITH objects AS (
+	SELECT CASE p.prokind WHEN 'a' THEN 'aggregate ' WHEN 'p' THEN 'procedure ' ELSE 'function ' END || p.proname AS object
+	FROM pg_catalog.pg_proc p
+	WHERE p.pronamespace = ${namespace} AND NOT EXISTS (
+		SELECT 1 FROM pg_catalog.pg_depend d WHERE d.classid = ${catalog('pg_proc')} AND d.objid = p.oid AND d.deptype = 'i'
+	)
+	UNION ALL
+	SELECT 'operator ' || o.oprname FROM pg_catalog.pg_operator o WHERE o.oprnamespace = ${namespace}
+	UNION ALL
+	SELECT 'trigger ' || c.relname || '.' || t.tgname
+	FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid = t.tgrelid
+	WHERE c.relnamespace = ${namespace} AND NOT t.tgisinternal
+	UNION ALL
+	SELECT 'rule ' || c.relname || '.' || r.rulename
+	FROM pg_catalog.pg_rewrite r JOIN pg_catalog.pg_class c ON c.oid = r.ev_class
+	WHERE c.relnamespace = ${namespace} AND r.rulename <> '_RETURN'
+	UNION ALL
+	SELECT CASE c.relkind WHEN 'm' THEN 'materialized view ' ELSE 'view ' END || c.relname
+	FROM pg_catalog.pg_class c WHERE c.relnamespace = ${namespace} AND c.relkind IN ('v', 'm')
+	UNION ALL
+	SELECT 'policy ' || c.relname || '.' || p.polname
+	FROM pg_catalog.pg_policy p JOIN pg_catalog.pg_class c ON c.oid = p.polrelid
+	WHERE c.relnamespace = ${namespace}
+	UNION ALL
+	SELECT 'domain ' || t.typname FROM pg_catalog.pg_type t WHERE t.typnamespace = ${namespace} AND t.typtype = 'd'
+),
+dependents AS (
+	SELECT ${catalog('pg_class')} AS classid, c.oid AS objid,
+		CASE c.relkind WHEN 'i' THEN 'index ' WHEN 'v' THEN 'view ' WHEN 'm' THEN 'materialized view ' WHEN 'S' THEN 'sequence ' ELSE 'table ' END
+			|| c.relname AS object
+	FROM pg_catalog.pg_class c WHERE c.relnamespace = ${namespace}
+	UNION ALL
+	SELECT ${catalog('pg_proc')}, p.oid, 'function ' || p.proname FROM pg_catalog.pg_proc p WHERE p.pronamespace = ${namespace}
+	UNION ALL
+	SELECT ${catalog('pg_type')}, t.oid, 'type ' || t.typname FROM pg_catalog.pg_type t WHERE t.typnamespace = ${namespace}
+	UNION ALL
+	SELECT ${catalog('pg_operator')}, o.oid, 'operator ' || o.oprname FROM pg_catalog.pg_operator o WHERE o.oprnamespace = ${namespace}
+	UNION ALL
+	SELECT ${catalog('pg_constraint')}, k.oid, 'constraint ' || k.conname FROM pg_catalog.pg_constraint k WHERE k.connamespace = ${namespace}
+	UNION ALL
+	SELECT ${catalog('pg_attrdef')}, d.oid, 'default ' || c.relname || '.' || a.attname
+	FROM pg_catalog.pg_attrdef d
+	JOIN pg_catalog.pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum
+	JOIN pg_catalog.pg_class c ON c.oid = d.adrelid
+	WHERE c.relnamespace = ${namespace}
+	UNION ALL
+	SELECT ${catalog('pg_trigger')}, t.oid, 'trigger ' || c.relname || '.' || t.tgname
+	FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid = t.tgrelid WHERE c.relnamespace = ${namespace}
+	UNION ALL
+	SELECT ${catalog('pg_rewrite')}, r.oid, CASE WHEN r.rulename = '_RETURN' THEN 'view ' || c.relname ELSE 'rule ' || c.relname || '.' || r.rulename END
+	FROM pg_catalog.pg_rewrite r JOIN pg_catalog.pg_class c ON c.oid = r.ev_class WHERE c.relnamespace = ${namespace}
+	UNION ALL
+	SELECT ${catalog('pg_policy')}, p.oid, 'policy ' || c.relname || '.' || p.polname
+	FROM pg_catalog.pg_policy p JOIN pg_catalog.pg_class c ON c.oid = p.polrelid WHERE c.relnamespace = ${namespace}
+),
+dependencies AS (
+	SELECT DISTINCT o.object, d.refclassid, d.refobjid
+	FROM pg_catalog.pg_depend d JOIN dependents o ON o.classid = d.classid AND o.objid = d.objid
+),
+violations AS (
+	SELECT 'function ' || p.proname || ' is SECURITY DEFINER' AS violation
+	FROM pg_catalog.pg_proc p WHERE p.pronamespace = ${namespace} AND p.prosecdef
+	UNION ALL
+	SELECT 'function ' || p.proname || ' has a SET clause' FROM pg_catalog.pg_proc p WHERE p.pronamespace = ${namespace} AND p.proconfig IS NOT NULL
+	UNION ALL
+	SELECT 'function ' || p.proname || ' is in the language ' || l.lanname
+	FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_language l ON l.oid = p.prolang
+	WHERE p.pronamespace = ${namespace} AND l.lanname NOT IN ('sql', 'plpgsql')
+	UNION ALL
+	SELECT DISTINCT 'function ' || p.proname || ' shadows a pg_catalog function'
+	FROM pg_catalog.pg_proc p
+	JOIN pg_catalog.pg_proc b ON b.proname = p.proname
+	JOIN pg_catalog.pg_namespace n ON n.oid = b.pronamespace AND n.nspname = 'pg_catalog'
+	WHERE p.pronamespace = ${namespace}
+	UNION ALL
+	SELECT 'routine ' || p.proname || ' has more than one signature'
+	FROM pg_catalog.pg_proc p WHERE p.pronamespace = ${namespace} GROUP BY p.proname HAVING pg_catalog.count(*) > 1
+	UNION ALL
+	SELECT 'sequence ' || c.relname || ' (only an identity column''s; use GENERATED ALWAYS AS IDENTITY)'
+	FROM pg_catalog.pg_class c
+	WHERE c.relnamespace = ${namespace} AND c.relkind = 'S' AND NOT EXISTS (
+		SELECT 1 FROM pg_catalog.pg_depend d
+		WHERE d.classid = ${catalog('pg_class')} AND d.objid = c.oid AND d.refclassid = ${catalog('pg_class')} AND d.refobjsubid > 0 AND d.deptype = 'i'
+	)
+	UNION ALL
+	SELECT 'event trigger ' || e.evtname FROM pg_catalog.pg_event_trigger e JOIN pg_catalog.pg_proc p ON p.oid = e.evtfoid
+	WHERE p.pronamespace = ${namespace}
+	UNION ALL
+	${outside('type', 'pg_type', 'typname', 'typnamespace')}
+	UNION ALL
+	${outside('function', 'pg_proc', 'proname', 'pronamespace')}
+	UNION ALL
+	${outside('relation', 'pg_class', 'relname', 'relnamespace')}
+	UNION ALL
+	${outside('operator', 'pg_operator', 'oprname', 'oprnamespace')}
+	UNION ALL
+	${outside('operator class', 'pg_opclass', 'opcname', 'opcnamespace')}
+	UNION ALL
+	${outside('collation', 'pg_collation', 'collname', 'collnamespace')}
+	UNION ALL
+	SELECT o.object || ' (no migration declares it)' FROM objects o WHERE o.object <> ALL (${declaredArray(declared)})
+)
+SELECT DISTINCT violation FROM violations ORDER BY violation`
+}
+
+/** The message a `databaseCodeCheck` failure carries before its list of violations. */
+export const DATABASE_CODE_FAILURE = 'fabrika-migration-database-code: '
+
+/** Fails the transaction it runs in when `schema` escapes or holds an object `declared` does not name. */
+export function databaseCodeCheck(schema: string, declared: readonly DatabaseObject[]): string {
+	return `DO $check$
+DECLARE found text;
+BEGIN
+	SELECT pg_catalog.string_agg(violation, ', ' ORDER BY violation) INTO found FROM (${databaseCodeQuery(schema, declared)}) AS code;
+	IF found IS NOT NULL THEN RAISE EXCEPTION '${DATABASE_CODE_FAILURE}%', found; END IF;
+END $check$`
+}

package/src/file-table-ddl.ts

@@ -0,0 +1,96 @@
+// The SQL of a file collection's table that three readers must agree on: `@fabrika/db`'s `fileTableSql` and
+// `fileScanSql` write it, the migration classifier admits exactly the scan migration, and `@fabrika/check` recognizes
+// a table that can hold the scan states. Pure: tokens in, answers out.
+
+import { type SqlToken, sqlTokens, topLevelStatements, upperWord } from './migration-sql'
+import { nameText } from './sql-names'
+
+/**
+ * A file row's states. `pending`: its object is still being written. `quarantined`: written to a scanning collection
+ * and not yet given a clean verdict. `live`: served. `rejected`: refused by the scan and its object is gone.
+ */
+export const FILE_STATUSES = ['pending', 'live', 'quarantined', 'rejected'] as const
+
+export const FILE_TABLE_PREFIX = 'fabrika_files_'
+
+/** The columns the scan job keeps: the scanned object's version, and the retry count and time of a file without a verdict. */
+export const FILE_SCAN_COLUMNS = ['object_version TEXT', 'scan_attempts INTEGER NOT NULL DEFAULT 0', 'scan_after TIMESTAMPTZ'] as const
+
+const STATUS_CHECK = `CHECK (status IN (${FILE_STATUSES.map((status) => `'${status}'`).join(', ')}))`
+
+/** The named status check of every state. */
+export function fileStatusCheck(table: string): string {
+	return `CONSTRAINT ${table}_status ${STATUS_CHECK}`
+}
+
+/** The index the scan job selects its batch by. */
+export function fileScanIndex(table: string): string {
+	return `CREATE INDEX ${table}_scan ON ${table} (scan_after) WHERE status = 'quarantined'`
+}
+
+/**
+ * The name Postgres gave the unnamed status check of a table an earlier `fileTableSql` created:
+ * `<table>_status_check`, the table's name cut to 50 characters first (`makeObjectName`).
+ */
+export function legacyStatusCheckName(table: string): string {
+	return `${table.slice(0, 63 - '_status_check'.length)}_status_check`
+}
+
+/** The statement of `fileScanSql` that widens a table an earlier `fileTableSql` created: its check and its columns. */
+export function fileScanAlter(table: string): string {
+	const columns = FILE_SCAN_COLUMNS.map((column) => `ADD COLUMN ${column}`).join(', ')
+	return `ALTER TABLE ${table} DROP CONSTRAINT IF EXISTS ${legacyStatusCheckName(table)}, ADD ${fileStatusCheck(table)}, ${columns}`
+}
+
+const tokenKey = (token: SqlToken): string =>
+	token.kind === 'word'
+		? `w:${token.text.toLowerCase()}`
+		: token.kind === 'punct'
+		? `p:${token.char}`
+		: token.kind === 'string'
+		? `s:${token.text ?? ''}`
+		: `${token.kind}:`
+
+const keys = (tokens: readonly SqlToken[]): string[] => tokens.map(tokenKey)
+
+const equalKeys = (actual: readonly string[], expected: readonly string[]): boolean =>
+	actual.length === expected.length && expected.every((key, index) => actual[index] === key)
+
+/** Whether `needle`'s tokens appear in `haystack`, in a row. */
+function containsRun(haystack: readonly string[], needle: readonly string[]): boolean {
+	for (let start = 0; start + needle.length <= haystack.length; start++) {
+		if (needle.every((key, offset) => haystack[start + offset] === key)) return true
+	}
+	return false
+}
+
+/** The file table a statement is exactly `fileScanAlter` of, or `undefined`. */
+export function fileScanAlterTable(tokens: readonly SqlToken[]): string | undefined {
+	const table = nameText(tokens[2])
+	if (upperWord(tokens[0]) !== 'ALTER' || table?.startsWith(FILE_TABLE_PREFIX) !== true || tokens[2]?.kind !== 'word') return undefined
+	return equalKeys(keys(tokens), keys(sqlTokens(fileScanAlter(table)))) ? table : undefined
+}
+
+/** Whether a `CREATE TABLE` of `table` declares the check of every state and the scan columns. */
+function createsScanReady(tokens: readonly SqlToken[], table: string): boolean {
+	const statement = keys(tokens)
+	return containsRun(statement, keys(sqlTokens(fileStatusCheck(table))))
+		&& FILE_SCAN_COLUMNS.every((column) => containsRun(statement, keys(sqlTokens(column))))
+}
+
+/**
+ * Whether the migrations, in order, leave `table` able to hold the scan states: created with the check of every state
+ * and the scan columns, or widened by `fileScanAlter`. `undefined` when none of them creates it.
+ */
+export function fileTableScanReady(migrations: readonly string[], table: string): boolean | undefined {
+	let state: boolean | undefined
+	for (const sql of migrations) {
+		for (const { tokens } of topLevelStatements(sql)) {
+			const create = upperWord(tokens[0]) === 'CREATE' && upperWord(tokens[1]) === 'TABLE'
+			const named = upperWord(tokens[2]) === 'IF' ? tokens[5] : tokens[2]
+			if (create && nameText(named) === table) state = createsScanReady(tokens, table)
+			else if (state !== undefined && fileScanAlterTable(tokens) === table) state = true
+		}
+	}
+	return state
+}

package/src/index.ts

@@ -18,8 +18,24 @@
 // Migrations for what this package owns (the jobs table, the Postgres lock table) live in
 // `migrations/` — DDL is per-dialect even when the statement over it is portable.
 
+export {
+	APPLICATION_MIGRATION_BUNDLE,
+	APPLICATION_MIGRATION_LEDGER,
+	applicationDatabaseRoles,
+	applicationMigrationLockKey,
+	applicationRuntimeUrl,
+	DEFAULT_MIGRATION_LOCK_TIMEOUT_MS,
+	DEFAULT_MIGRATION_STATEMENT_TIMEOUT_MS,
+	definePostgresApplicationMigrations,
+	dropApplicationDatabase,
+	type PostgresApplicationMigrations,
+	type PostgresApplicationMigrationsConfig,
+} from './application-postgres-migrations'
+export type { ApplicationDatabaseRoles } from './application-postgres-roles'
+export { applicationDatabaseSchema } from './application-schema'
 export { FileSystemAssetServer, type FileSystemAssetServerOptions } from './asset-server-fs'
 export { S3BlobStore, type S3BlobStoreOptions } from './blob-s3'
+export { databaseCodeQuery } from './database-code'
 export {
 	decodeJob,
 	type Job,
@@ -28,19 +44,24 @@ export {
 	PostgresJobQueue,
 	type PostgresJobQueueOptions,
 } from './job-queue-postgres'
+export type { DatabaseObject, DatabaseObjectKind } from './migration-classify'
 export { rewritePlaceholders, type RewrittenSql } from './placeholders'
 export {
 	applyPostgresMigrations,
 	definePostgresMigrationPlan,
+	migrationChecksum,
 	platformNodePostgresMigrationBundle,
+	type PostgresApplicationMigrationSettings,
 	type PostgresMigration,
 	type PostgresMigrationBundle,
+	PostgresMigrationError,
 	postgresMigrationFilenames,
 	postgresMigrationIdentity,
 	type PostgresMigrationPlan,
 	readPostgresMigrationBundle,
 	type ReadPostgresMigrationBundleOptions,
 } from './postgres-migrations'
+export { createTestProvisioner, type TestProvisioner } from './postgres-test-provisioner'
 export {
 	definePostgresServiceMigrations,
 	postgresMigrationResultMessage,