@owlmeans/postgres-resource 0.1.15

package/src/utils/diff.ts

@@ -0,0 +1,363 @@
+import type { BasicContext } from '@owlmeans/context'
+import type { ResourceRecord } from '@owlmeans/resource'
+
+import { PgIndexMethod } from '../consts.js'
+import { PostgresPlaceholderError } from '../errors.js'
+import type {
+  ColumnSpec, DdlPlan, DdlStatement, LiveTable, PgIndexSpec, PostgresResource, TableSpec
+} from '../types.js'
+import { qualify, quoteIdent } from './name.js'
+
+const literal = (value: string | number | boolean | null): string => {
+  if (value === null) return 'NULL'
+  if (typeof value === 'number') return `${value}`
+  if (typeof value === 'boolean') return value ? 'TRUE' : 'FALSE'
+
+  return `'${value.replace(/'/g, "''")}'`
+}
+
+const defaultExpression = (column: ColumnSpec): string | undefined => {
+  if (column.defaultRaw != null) return column.defaultRaw
+  if (column.defaultLiteral !== undefined) return literal(column.defaultLiteral)
+
+  return undefined
+}
+
+/**
+ * Postgres echoes defaults back with an explicit cast — a declared `'active'` comes back
+ * as `'active'::text`. Strip a trailing cast when the literal underneath still matches, so
+ * an unchanged default doesn't look like drift on every boot.
+ */
+const normalizeDefault = (expression: string | null | undefined): string | null => {
+  if (expression == null) return null
+
+  return expression.trim().replace(/::[a-z0-9_ ."\[\]]+$/i, '').trim().toLowerCase()
+}
+
+const normalizeDefinition = (definition: string): string =>
+  definition.replace(/\s+/g, ' ').replace(/ ?, ?/g, ',').trim().toLowerCase()
+
+const columnList = (columns: string[]): string => columns.map(quoteIdent).join(', ')
+
+const indexStatement = (spec: TableSpec, entry: PgIndexSpec): string => {
+  const target = entry.expression != null
+    ? `(${entry.expression})`
+    : `(${columnList(entry.columns as string[])})`
+  const where = entry.where != null ? ` WHERE ${entry.where}` : ''
+
+  return `CREATE ${entry.unique === true ? 'UNIQUE ' : ''}INDEX ${quoteIdent(entry.name!)}`
+    + ` ON ${spec.qualified} USING ${entry.method ?? PgIndexMethod.BTree} ${target}${where}`
+}
+
+const columnDefinition = (column: ColumnSpec): string => {
+  const parts = [quoteIdent(column.column), column.sqlType]
+  if (column.notNull) parts.push('NOT NULL')
+  const expression = defaultExpression(column)
+  if (expression != null) parts.push(`DEFAULT ${expression}`)
+
+  return parts.join(' ')
+}
+
+/**
+ * Foreign keys are planned separately from the rest of the table, and applied only once
+ * every resource has initialized: a key points at a table another resource owns, and that
+ * resource's own `init()` may not have run yet.
+ *
+ * @throws {PostgresPlaceholderError} when a target can't be resolved.
+ */
+export const planForeignKeys = (
+  spec: TableSpec, context?: BasicContext<any>
+): DdlStatement[] => spec.references.map(reference => {
+  const column = reference.property != null ? spec.byProperty[reference.property] : undefined
+  if (column == null) {
+    throw new PostgresPlaceholderError(`fk-unknown-property:${reference.property}`)
+  }
+
+  let targetSchema = reference.schema ?? spec.schema
+  let targetTable = reference.table
+  if (reference.resource != null) {
+    if (context == null) {
+      throw new PostgresPlaceholderError(`fk-no-context:${reference.resource}`)
+    }
+    let target: PostgresResource<ResourceRecord>
+    try {
+      target = context.resource<PostgresResource<ResourceRecord>>(reference.resource)
+    } catch {
+      throw new PostgresPlaceholderError(`fk-unknown-resource:${reference.resource}`)
+    }
+    if (target.table == null) {
+      throw new PostgresPlaceholderError(`fk-uninitialized:${reference.resource}`)
+    }
+    targetSchema = target.table.schema
+    targetTable = target.table.table
+  }
+  if (targetTable == null) {
+    throw new PostgresPlaceholderError(`fk-no-target:${reference.property}`)
+  }
+
+  const actions = [
+    reference.onDelete != null ? ` ON DELETE ${reference.onDelete}` : '',
+    reference.onUpdate != null ? ` ON UPDATE ${reference.onUpdate}` : ''
+  ].join('')
+
+  return {
+    kind: 'add-foreign-key' as const,
+    target: reference.name!,
+    sql: `ALTER TABLE ${spec.qualified} ADD CONSTRAINT ${quoteIdent(reference.name!)}`
+      + ` FOREIGN KEY (${quoteIdent(column.column)})`
+      + ` REFERENCES ${qualify(targetSchema, targetTable)} (${quoteIdent(reference.column!)})${actions}`
+  }
+})
+
+/**
+ * Compare the compiled specification against the live table and produce the ordered DDL
+ * that converges one onto the other.
+ *
+ * Statement order is load bearing:
+ *  - `DROP COLUMN` is last, because a `USING` cast or a backfill may legitimately read a
+ *    column that is about to disappear.
+ *  - A primary key constraint is never dropped: it cascades into every referencing key and
+ *    can't be undone inside the same transaction once dependents exist.
+ *
+ * Foreign keys are not part of the plan at all — see {@link planForeignKeys}.
+ */
+export const planSync = (
+  spec: TableSpec, live: LiveTable, additive: boolean = false
+): DdlPlan => {
+  if (!live.exists) {
+    const constraints: string[] = []
+    if (spec.primaryKey.length > 0) {
+      constraints.push(`CONSTRAINT ${quoteIdent(`${spec.table}_pkey`)} PRIMARY KEY (${columnList(spec.primaryKey)})`)
+    }
+    for (const unique of spec.uniques) {
+      constraints.push(`CONSTRAINT ${quoteIdent(unique.name!)} UNIQUE (${columnList(unique.columns)})`)
+    }
+    for (const check of spec.checks) {
+      constraints.push(`CONSTRAINT ${quoteIdent(check.name!)} CHECK (${check.expression})`)
+    }
+
+    const statements: DdlStatement[] = [{
+      kind: 'create-table',
+      target: spec.table,
+      sql: `CREATE TABLE ${spec.qualified} (\n  `
+        + [...spec.columns.map(columnDefinition), ...constraints].join(',\n  ')
+        + '\n)'
+    }]
+    for (const entry of spec.indexes) {
+      statements.push({ kind: 'create-index', target: entry.name!, sql: indexStatement(spec, entry) })
+    }
+
+    return { fresh: true, statements }
+  }
+
+  const statements: DdlStatement[] = []
+  const byName: Record<string, typeof live.columns[number]> = {}
+  for (const column of live.columns) {
+    byName[column.name] = column
+  }
+
+  const managed = spec.columns.filter(column => column.managed)
+
+  /** 1. Columns the table doesn't have yet — always added nullable so existing rows survive. */
+  for (const column of managed) {
+    if (byName[column.column] != null) {
+      continue
+    }
+    const expression = defaultExpression(column)
+    statements.push({
+      kind: 'add-column',
+      target: column.column,
+      sql: `ALTER TABLE ${spec.qualified} ADD COLUMN ${quoteIdent(column.column)} ${column.sqlType}`
+        + (expression != null ? ` DEFAULT ${expression}` : '')
+    })
+    if (column.notNull) {
+      if (expression == null) {
+        statements.push({
+          kind: 'backfill',
+          target: column.column,
+          sql: `UPDATE ${spec.qualified} SET ${quoteIdent(column.column)} = ${fallback(column)}`
+            + ` WHERE ${quoteIdent(column.column)} IS NULL`
+        })
+      }
+      statements.push({
+        kind: 'set-not-null',
+        target: column.column,
+        sql: `ALTER TABLE ${spec.qualified} ALTER COLUMN ${quoteIdent(column.column)} SET NOT NULL`
+      })
+    }
+  }
+
+  /** 2-4. Type, nullability and default drift on columns that already exist. */
+  for (const column of managed) {
+    const existing = byName[column.column]
+    if (existing == null) {
+      continue
+    }
+
+    if (!additive && existing.type !== column.sqlType) {
+      const using = column.using ?? `${quoteIdent(column.column)}::${column.sqlType}`
+      statements.push({
+        kind: 'alter-type',
+        target: column.column,
+        sql: `ALTER TABLE ${spec.qualified} ALTER COLUMN ${quoteIdent(column.column)}`
+          + ` TYPE ${column.sqlType} USING ${using}`
+      })
+    }
+
+    const expression = defaultExpression(column)
+    if (normalizeDefault(expression) !== normalizeDefault(existing.defaultExpr)) {
+      statements.push(expression != null
+        ? {
+          kind: 'set-default',
+          target: column.column,
+          sql: `ALTER TABLE ${spec.qualified} ALTER COLUMN ${quoteIdent(column.column)}`
+            + ` SET DEFAULT ${expression}`
+        }
+        : {
+          kind: 'set-default',
+          target: column.column,
+          sql: `ALTER TABLE ${spec.qualified} ALTER COLUMN ${quoteIdent(column.column)} DROP DEFAULT`
+        })
+    }
+
+    if (column.notNull !== existing.notNull) {
+      if (column.notNull) {
+        statements.push({
+          kind: 'backfill',
+          target: column.column,
+          sql: `UPDATE ${spec.qualified} SET ${quoteIdent(column.column)}`
+            + ` = ${expression ?? fallback(column)} WHERE ${quoteIdent(column.column)} IS NULL`
+        })
+      }
+      statements.push({
+        kind: column.notNull ? 'set-not-null' : 'drop-not-null',
+        target: column.column,
+        sql: `ALTER TABLE ${spec.qualified} ALTER COLUMN ${quoteIdent(column.column)}`
+          + ` ${column.notNull ? 'SET' : 'DROP'} NOT NULL`
+      })
+    }
+  }
+
+  /** 5. Unique and check constraints. Primary keys are deliberately excluded. */
+  const desiredConstraints: Record<string, string> = {}
+  for (const unique of spec.uniques) {
+    desiredConstraints[unique.name!] = `UNIQUE (${columnList(unique.columns)})`
+  }
+  for (const check of spec.checks) {
+    desiredConstraints[check.name!] = `CHECK ((${check.expression}))`
+  }
+  const liveConstraints: Record<string, string> = {}
+  for (const constraint of live.constraints) {
+    if (constraint.type === 'p' || constraint.type === 'f') {
+      continue
+    }
+    liveConstraints[constraint.name] = constraint.definition
+  }
+
+  for (const [name, definition] of Object.entries(liveConstraints)) {
+    const desired = desiredConstraints[name]
+    if (desired != null && normalizeDefinition(desired) === normalizeDefinition(definition)) {
+      continue
+    }
+    if (desired == null && additive) {
+      continue
+    }
+    if (desired == null && !isGenerated(name, spec)) {
+      /** A constraint nobody declared was added by hand — leave it alone. */
+      continue
+    }
+    statements.push({
+      kind: 'drop-constraint',
+      target: name,
+      sql: `ALTER TABLE ${spec.qualified} DROP CONSTRAINT ${quoteIdent(name)}`
+    })
+  }
+  for (const [name, definition] of Object.entries(desiredConstraints)) {
+    const existing = liveConstraints[name]
+    if (existing != null && normalizeDefinition(existing) === normalizeDefinition(definition)) {
+      continue
+    }
+    if (existing != null && additive) {
+      continue
+    }
+    statements.push({
+      kind: 'add-constraint',
+      target: name,
+      sql: `ALTER TABLE ${spec.qualified} ADD CONSTRAINT ${quoteIdent(name)} ${definition}`
+    })
+  }
+
+  /** 6. Indexes. Constraint backed indexes are owned by their constraint, not by us. */
+  const constraintBacked = live.constraints.map(constraint => constraint.name)
+  const desiredIndexes: Record<string, string> = {}
+  for (const entry of spec.indexes) {
+    desiredIndexes[entry.name!] = indexStatement(spec, entry)
+  }
+  for (const existing of live.indexes) {
+    if (constraintBacked.includes(existing.name)) {
+      continue
+    }
+    const desired = desiredIndexes[existing.name]
+    if (desired != null && normalizeDefinition(desired) === normalizeDefinition(existing.definition)) {
+      continue
+    }
+    if (desired == null && (additive || !isGenerated(existing.name, spec))) {
+      continue
+    }
+    statements.push({
+      kind: 'drop-index',
+      target: existing.name,
+      sql: `DROP INDEX ${qualify(spec.schema, existing.name)}`
+    })
+  }
+  for (const [name, statement] of Object.entries(desiredIndexes)) {
+    const existing = live.indexes.find(entry => entry.name === name)
+    if (existing != null && normalizeDefinition(existing.definition) === normalizeDefinition(statement)) {
+      continue
+    }
+    if (constraintBacked.includes(name)) {
+      continue
+    }
+    statements.push({ kind: 'create-index', target: name, sql: statement })
+  }
+
+  /** 7. Drops, last and only in full mode. */
+  if (!additive) {
+    const keep = [...managed.map(column => column.column), ...spec.unmanaged]
+    for (const existing of live.columns) {
+      if (keep.includes(existing.name) || existing.generated !== '') {
+        continue
+      }
+      statements.push({
+        kind: 'drop-column',
+        target: existing.name,
+        destructive: true,
+        sql: `ALTER TABLE ${spec.qualified} DROP COLUMN ${quoteIdent(existing.name)}`
+      })
+    }
+  }
+
+  return { fresh: false, statements }
+}
+
+/** A name this package would have generated is a name this package may drop. */
+const isGenerated = (name: string, spec: TableSpec): boolean =>
+  name.startsWith(`${spec.table}_`)
+
+/** Value used to fill an existing row before a column becomes `NOT NULL`. */
+const fallback = (column: ColumnSpec): string => {
+  if (column.array) return `'{}'::${column.sqlType}`
+  if (column.jsonb) return `'{}'::jsonb`
+  switch (column.jsonType) {
+    case 'number':
+    case 'integer':
+    case 'bigint':
+      return '0'
+    case 'boolean':
+      return 'FALSE'
+    case 'date':
+      return 'now()'
+    default:
+      return `''::${column.sqlType}`
+  }
+}

package/src/utils/index.ts

@@ -0,0 +1,11 @@
+export * from './criteria.js'
+export * from './diff.js'
+export * from './introspect.js'
+export * from './life-cycle.js'
+export * from './marshal.js'
+export * from './migrations.js'
+export * from './name.js'
+export * from './schema.js'
+export * from './sql.js'
+export * from './sync.js'
+export * from './table.js'

package/src/utils/introspect.ts

@@ -0,0 +1,87 @@
+import type { PoolClient } from 'pg'
+
+import type { LiveColumn, LiveConstraint, LiveIndex, LiveTable } from '../types.js'
+
+/**
+ * `pg_attribute` + `format_type` rather than `information_schema.columns`, because it
+ * returns one canonical type string with the typmod already baked in
+ * (`character varying(320)`). `information_schema` splits the same information across
+ * four columns and reports every array as `ARRAY`, which no comparison can use.
+ */
+const COLUMNS = `
+  SELECT a.attname                            AS name,
+         format_type(a.atttypid, a.atttypmod) AS type,
+         a.attnotnull                         AS not_null,
+         pg_get_expr(d.adbin, d.adrelid)      AS default_expr,
+         a.attidentity                        AS identity,
+         a.attgenerated                       AS generated,
+         a.attnum                             AS ordinal
+    FROM pg_attribute a
+    LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
+   WHERE a.attrelid = to_regclass($1) AND a.attnum > 0 AND NOT a.attisdropped
+   ORDER BY a.attnum
+`
+
+const INDEXES = `
+  SELECT indexname AS name, indexdef AS definition
+    FROM pg_indexes
+   WHERE schemaname = $1 AND tablename = $2
+`
+
+const CONSTRAINTS = `
+  SELECT c.conname AS name, c.contype AS type, pg_get_constraintdef(c.oid) AS definition
+    FROM pg_constraint c
+   WHERE c.conrelid = to_regclass($1)
+`
+
+/** Read the table exactly as Postgres currently holds it. */
+export const introspectTable = async (
+  client: PoolClient, schema: string, table: string, qualified: string
+): Promise<LiveTable> => {
+  const exists = await client.query<{ present: boolean }>(
+    'SELECT to_regclass($1) IS NOT NULL AS present', [qualified]
+  )
+  if (exists.rows[0]?.present !== true) {
+    return { exists: false, columns: [], indexes: [], constraints: [] }
+  }
+
+  /**
+   * Sequential, not `Promise.all`: this is one checked out connection, and a connection
+   * runs one statement at a time. Overlapping them only queues them behind each other —
+   * and node-postgres deprecated tolerating it.
+   */
+  const columns = await client.query<LiveColumn>(COLUMNS, [qualified])
+  const indexes = await client.query<LiveIndex>(INDEXES, [schema, table])
+  const constraints = await client.query<LiveConstraint>(CONSTRAINTS, [qualified])
+
+  return {
+    exists: true,
+    columns: columns.rows.map(row => ({
+      name: row.name,
+      type: row.type,
+      notNull: (row as unknown as { not_null: boolean }).not_null,
+      defaultExpr: (row as unknown as { default_expr: string | null }).default_expr,
+      identity: row.identity ?? '',
+      generated: row.generated ?? '',
+      ordinal: row.ordinal
+    })),
+    indexes: indexes.rows,
+    constraints: constraints.rows
+  }
+}
+
+/** Count the rows a destructive step would lose, so the boot log can say how much. */
+export const countNonNull = async (
+  client: PoolClient, qualified: string, column: string
+): Promise<number> => {
+  try {
+    const result = await client.query<{ count: string }>(
+      `SELECT count(*)::text AS count FROM ${qualified} WHERE "${column.replace(/"/g, '""')}" IS NOT NULL`
+    )
+
+    return parseInt(result.rows[0]?.count ?? '0', 10)
+  } catch {
+    /** Never let a diagnostic read break reconciliation. */
+    return -1
+  }
+}

package/src/utils/life-cycle.ts

@@ -0,0 +1,155 @@
+import type { BasicContext } from '@owlmeans/context'
+import { MigrationStage, runMigrations } from '@owlmeans/resource'
+import type { DbConfig, MigrationReport, ResourceRecord } from '@owlmeans/resource'
+import type { PoolClient } from 'pg'
+
+import { DEF_MIGRATIONS_TABLE, PgAutoSync } from '../consts.js'
+import { getDeclaration } from '../declarations.js'
+import { describePgError, pgErrorToResourceError } from '../errors.js'
+import type {
+  PgRuntimeTable, PostgresDb, PostgresMeta, PostgresResource, TableSpec
+} from '../types.js'
+import { planForeignKeys, planSync } from './diff.js'
+import { introspectTable } from './introspect.js'
+import { makeMigrationStore } from './migrations.js'
+import { pgTableName } from './name.js'
+import { schemaToTableSpec } from './schema.js'
+import { refOf, resolvePlaceholders } from './sql.js'
+import { acquireLock, applyPlan, ensureSchema, releaseLock } from './sync.js'
+import { specToTable } from './table.js'
+
+export interface TableInit {
+  spec: TableSpec
+  /** The Drizzle table CRUD is built against. */
+  entity: PgRuntimeTable
+  reports: MigrationReport[]
+}
+
+/**
+ * Bring a resource's table to the shape its schema declares, then hand back everything the
+ * resource needs to query it. The Postgres counterpart of mongo's `initializeCollection`.
+ *
+ * The whole sequence runs inside a session level advisory lock keyed on the qualified table
+ * name, so replicas booting simultaneously converge one at a time instead of racing each
+ * other's DDL — a race mongo's converge-on-boot leaves open.
+ *
+ * Order is deliberate:
+ *
+ *  1. probe for the table
+ *  2. absent → *baseline* every registered migration; present → run the `pre` ones
+ *  3. reconcile structure against the schema
+ *  4. present → run the `post` migrations
+ *  5. queue foreign keys for after every resource has initialized
+ *
+ * Step 2 is what stops the two mechanisms colliding. On a fresh deployment the table is
+ * created directly to its final shape, so migrations that would have produced that shape
+ * are recorded as satisfied rather than replayed against a table that already matches.
+ */
+export const initializeTable = async (
+  db: PostgresDb, config: DbConfig, resource: PostgresResource<ResourceRecord>,
+  context: BasicContext<any>, defer: (task: () => Promise<void>) => void
+): Promise<TableInit> => {
+  const declaration = getDeclaration(resource.alias)
+  const meta = (config.meta ?? {}) as PostgresMeta
+  const mode = meta.autoSync ?? PgAutoSync.Full
+
+  const spec = schemaToTableSpec(
+    resource.alias, resource.schema ?? declaration.schema, db.schema,
+    pgTableName(config, resource), mode !== PgAutoSync.Off, declaration.indexes
+  )
+  const entity = specToTable(spec)
+  const reports: MigrationReport[] = []
+
+  const client = await db.pool.connect()
+  try {
+    await acquireLock(client, spec.qualified)
+    await ensureSchema(client, spec.schema)
+
+    const resolve = (text: string): string => resolvePlaceholders(text, context, spec)
+    const ref = (alias?: string): string => refOf(context, spec, alias)
+    const store = makeMigrationStore(client, spec, DEF_MIGRATIONS_TABLE, resolve, ref)
+
+    const fresh = !(await introspectTable(client, spec.schema, spec.table, spec.qualified)).exists
+
+    reports.push(await runMigrations(resource.alias, declaration.migrations, store, fresh
+      ? { baseline: true }
+      : { stage: MigrationStage.Pre }))
+
+    if (spec.autoSync) {
+      /** Re-read: a `pre` migration may have reshaped exactly what the diff is about to compare. */
+      const live = await introspectTable(client, spec.schema, spec.table, spec.qualified)
+      const plan = planSync(spec, live, mode === PgAutoSync.Additive)
+      await applyPlan(client, spec, plan)
+    }
+
+    if (!fresh) {
+      reports.push(await runMigrations(
+        resource.alias, declaration.migrations, store, { stage: MigrationStage.Post }
+      ))
+    }
+
+    for (const report of reports) {
+      if (report.applied.length > 0) {
+        console.log(
+          `@owlmeans/postgres-resource: ${spec.qualified} applied ${report.stage} migrations —`
+          + ` ${report.applied.join(', ')}`
+        )
+      }
+    }
+  } finally {
+    await releaseLock(client, spec.qualified)
+    client.release()
+  }
+
+  if (spec.references.length > 0) {
+    defer(async () => { await applyForeignKeys(db, spec, context) })
+  }
+
+  return { spec, entity, reports }
+}
+
+/**
+ * Add the foreign keys the schema declares, skipping any Postgres already holds.
+ *
+ * Constraints are compared by name only. A key whose *definition* drifted has to be dropped
+ * by hand or in a migration: `ALTER TABLE ... DROP CONSTRAINT` on a foreign key is not a
+ * change worth making on a boot nobody is watching.
+ */
+export const applyForeignKeys = async (
+  db: PostgresDb, spec: TableSpec, context: BasicContext<any>
+): Promise<void> => {
+  const client = await db.pool.connect()
+  try {
+    const statements = planForeignKeys(spec, context)
+    const live = await introspectTable(client, spec.schema, spec.table, spec.qualified)
+    if (!live.exists) {
+      return
+    }
+    const present = live.constraints.map(constraint => constraint.name)
+
+    for (const statement of statements) {
+      if (present.includes(statement.target)) {
+        continue
+      }
+      await apply(client, spec, statement.sql)
+    }
+  } finally {
+    client.release()
+  }
+}
+
+const apply = async (client: PoolClient, spec: TableSpec, statement: string): Promise<void> => {
+  try {
+    await client.query(statement)
+  } catch (error) {
+    const translated = pgErrorToResourceError(error)
+    /**
+     * A foreign key failing means live rows point at nothing. That's a data problem the
+     * boot can't fix, and refusing to start is better than starting with the constraint
+     * quietly missing.
+     */
+    throw Object.assign(translated, {
+      message: `${spec.qualified}: ${describePgError(error)} — statement: ${statement}`
+    })
+  }
+}