@owlmeans/postgres-resource 0.1.15

package/src/utils/schema.ts

@@ -0,0 +1,426 @@
+import { UnsupportedArgumentError } from '@owlmeans/resource'
+import type { AnySchema } from 'ajv'
+
+import {
+  DEF_ID_DEFAULT, DEF_JSON_TYPE, DEF_SQL_TYPE, ID_FIELD, PG_KEYWORD, PgIndexMethod
+} from '../consts.js'
+import type {
+  ColumnJsonType, ColumnSpec, PgCheckSpec, PgIndexSpec, PgPropertyOverride, PgRootOverride,
+  PgReferenceSpec, PgUniqueSpec, TableSpec
+} from '../types.js'
+import { pgIdentifier, qualify, quoteIdent } from './name.js'
+
+interface RawProperty {
+  type?: string | string[]
+  format?: string
+  enum?: unknown[]
+  items?: RawProperty
+  maxLength?: number
+  maximum?: number
+  minimum?: number
+  default?: unknown
+  nullable?: boolean
+  secure?: boolean
+  [PG_KEYWORD]?: PgPropertyOverride
+}
+
+interface RawSchema {
+  properties?: Record<string, RawProperty>
+  required?: string[]
+  allOf?: RawSchema[]
+  [PG_KEYWORD]?: PgRootOverride
+}
+
+const INT32_MAX = 2147483647
+
+/**
+ * Normalize a Postgres type name to the spelling `format_type()` reports, so the drift
+ * comparison against a live table is an exact string match instead of a fuzzy one.
+ */
+export const toFormatType = (type: string): string => {
+  const trimmed = type.trim().toLowerCase()
+  const match = /^([a-z0-9_ ]+?)\s*(\(([^)]*)\))?\s*(\[\])?$/.exec(trimmed)
+  if (match == null) {
+    return trimmed
+  }
+  const [, base, , args, array] = match
+  const aliases: Record<string, string> = {
+    int: 'integer', int4: 'integer', int2: 'smallint', int8: 'bigint', serial: 'integer',
+    bool: 'boolean', float4: 'real', float8: 'double precision', decimal: 'numeric',
+    varchar: 'character varying', char: 'character', bpchar: 'character',
+    timestamptz: 'timestamp with time zone', timetz: 'time with time zone',
+    timestamp: 'timestamp without time zone', time: 'time without time zone'
+  }
+  const resolved = aliases[base.trim()] ?? base.trim()
+  /** Postgres prints multi-argument typmods without spaces: `numeric(12,2)`. */
+  const suffix = args != null ? `(${args.split(',').map(part => part.trim()).join(',')})` : ''
+
+  return `${resolved}${suffix}${array ?? ''}`
+}
+
+const readOverride = (property: RawProperty): PgPropertyOverride => property[PG_KEYWORD] ?? {}
+
+/**
+ * Nullability is decided in exactly one place and threaded down from there. Mongo's mapper
+ * recomputes it per branch and lost it twice — on `date-time` and on optional objects.
+ */
+const isNullable = (property: RawProperty, isRequired: boolean): boolean => {
+  const override = readOverride(property)
+  if (override.nullable != null) {
+    return override.nullable
+  }
+  if (property.nullable === true) {
+    return true
+  }
+  if (Array.isArray(property.type) && property.type.includes('null')) {
+    return true
+  }
+
+  return !isRequired
+}
+
+const baseType = (property: RawProperty): string | undefined =>
+  Array.isArray(property.type) ? property.type.find(type => type !== 'null') : property.type
+
+const scalarSqlType = (property: RawProperty): { sqlType: string, jsonType: ColumnJsonType } => {
+  const override = readOverride(property)
+  const type = baseType(property)
+
+  switch (type) {
+    case 'string':
+      switch (property.format) {
+        case 'date-time': return { sqlType: 'timestamp with time zone', jsonType: 'date' }
+        case 'date': return { sqlType: 'date', jsonType: 'date' }
+        case 'time': return { sqlType: 'time without time zone', jsonType: 'string' }
+        case 'uuid': return { sqlType: 'uuid', jsonType: 'string' }
+        case 'binary': return { sqlType: 'bytea', jsonType: 'binary' }
+        default: break
+      }
+      return property.maxLength != null
+        ? { sqlType: `character varying(${property.maxLength})`, jsonType: 'string' }
+        : { sqlType: DEF_SQL_TYPE, jsonType: 'string' }
+    case 'integer':
+      return property.maximum != null && property.maximum > INT32_MAX
+        ? { sqlType: 'bigint', jsonType: 'bigint' }
+        : { sqlType: 'integer', jsonType: 'integer' }
+    case 'number':
+      return override.precision != null
+        ? { sqlType: `numeric(${override.precision},${override.scale ?? 0})`, jsonType: 'number' }
+        : { sqlType: 'double precision', jsonType: 'number' }
+    case 'boolean':
+      return { sqlType: 'boolean', jsonType: 'boolean' }
+    case 'object':
+      /**
+       * The framework's date convention — `{ type: 'object', format: 'date-time' }`. Mongo's
+       * mapper broke twice on exactly this branch by dropping nullability, so nullability is
+       * decided by the caller, never here.
+       */
+      return property.format === 'date-time'
+        ? { sqlType: 'timestamp with time zone', jsonType: 'date' }
+        : { sqlType: DEF_JSON_TYPE, jsonType: 'object' }
+    default:
+      return { sqlType: DEF_JSON_TYPE, jsonType: 'unknown' }
+  }
+}
+
+const arraySqlType = (property: RawProperty): { sqlType: string, jsonType: ColumnJsonType, array: boolean } => {
+  const items = property.items
+  const itemType = items != null ? baseType(items) : undefined
+  /** Only scalar element types become Postgres arrays; anything else stays jsonb. */
+  if (items != null && itemType != null && ['string', 'integer', 'number', 'boolean'].includes(itemType)
+    && items.format == null && items.enum == null) {
+    const scalar = scalarSqlType(items)
+    return { sqlType: `${scalar.sqlType}[]`, jsonType: 'array', array: true }
+  }
+
+  return { sqlType: DEF_JSON_TYPE, jsonType: 'array', array: false }
+}
+
+const compileColumn = (
+  name: string, property: RawProperty, required: string[]
+): ColumnSpec => {
+  const override = readOverride(property)
+  const secure = property.secure === true
+  const type = baseType(property)
+  const nullable = isNullable(property, required.includes(name))
+
+  let sqlType: string
+  let jsonType: ColumnJsonType
+  let array = false
+
+  if (override.type != null) {
+    sqlType = override.type
+    jsonType = secure ? 'string' : scalarSqlType(property).jsonType
+    array = override.array === true || override.type.endsWith('[]')
+  } else if (secure) {
+    /**
+     * A secure field is stored as ciphertext, so its declared JSON type says nothing about
+     * its column. It's always unbounded text: `maxLength` would truncate the ciphertext,
+     * an enum CHECK would never match it, and jsonb would fail to parse it.
+     */
+    sqlType = DEF_SQL_TYPE
+    jsonType = 'string'
+  } else if (type === 'array') {
+    const resolved = arraySqlType(property)
+    sqlType = resolved.sqlType
+    jsonType = resolved.jsonType
+    array = resolved.array
+  } else {
+    const resolved = scalarSqlType(property)
+    sqlType = resolved.sqlType
+    jsonType = resolved.jsonType
+  }
+
+  if (override.length != null) {
+    sqlType = `character varying(${override.length})`
+  }
+  if (override.array === true && !array && !sqlType.endsWith('[]')) {
+    sqlType = `${sqlType}[]`
+    array = true
+  }
+  if (override.jsonb === true) {
+    sqlType = DEF_JSON_TYPE
+    jsonType = jsonType === 'array' ? 'array' : 'object'
+    array = false
+  }
+
+  const formatted = toFormatType(sqlType)
+  const isJsonb = formatted === 'jsonb' || formatted === 'json'
+
+  const column: ColumnSpec = {
+    property: name,
+    /** No implicit snake_casing — a silent rename is a silent DROP plus ADD. */
+    column: pgIdentifier(override.column ?? name),
+    sqlType: formatted,
+    jsonType,
+    notNull: override.nullable === false ? true : !nullable,
+    primaryKey: override.primaryKey === true,
+    secure,
+    jsonb: isJsonb,
+    array,
+    managed: override.managed !== false,
+    ...(override.using != null ? { using: override.using } : {}),
+    ...(override.comment != null ? { comment: override.comment } : {})
+  }
+
+  if (override.defaultRaw != null) {
+    column.defaultRaw = override.defaultRaw
+  } else if (override.default !== undefined) {
+    column.defaultLiteral = override.default
+  } else if (property.default !== undefined && !isJsonb && !array) {
+    /** Non-scalar defaults stay in `getDefaults()` only — they'd need casting in DDL. */
+    column.defaultLiteral = property.default as string | number | boolean | null
+  }
+
+  return column
+}
+
+const collectProperties = (schema: RawSchema): { properties: Record<string, RawProperty>, required: string[] } => {
+  const properties: Record<string, RawProperty> = { ...schema.properties }
+  const required: string[] = [...(schema.required ?? [])]
+  /** `allOf` composition is used across the framework's schemas — flatten it like mongo does. */
+  for (const part of schema.allOf ?? []) {
+    const nested = collectProperties(part)
+    Object.assign(properties, nested.properties)
+    required.push(...nested.required)
+  }
+
+  return { properties, required }
+}
+
+/**
+ * Compile a resource's AJV schema into the table specification that drives both DDL
+ * emission and drift detection.
+ *
+ * `extraIndexes` carries what `resource.index()` declared. It's compiled here rather than
+ * merged afterwards so chained declarations get the same property-to-column mapping and
+ * the same generated names as schema-borne ones.
+ *
+ * @throws {UnsupportedArgumentError} when an override asks for something unrepresentable.
+ */
+export const schemaToTableSpec = (
+  alias: string, schema: AnySchema | undefined, pgSchema: string, table: string,
+  autoSync: boolean, extraIndexes: PgIndexSpec[] = []
+): TableSpec => {
+  const raw = (schema ?? {}) as RawSchema
+  const root = raw[PG_KEYWORD] ?? {}
+  const { properties, required } = collectProperties(raw)
+
+  const columns: ColumnSpec[] = []
+  const uniques: PgUniqueSpec[] = [...(root.unique ?? [])]
+  const checks: PgCheckSpec[] = [...(root.checks ?? [])]
+  const indexes: PgIndexSpec[] = [...(root.indexes ?? []), ...extraIndexes]
+  const references: PgReferenceSpec[] = []
+
+  const resolvedTable = pgIdentifier(root.table ?? table)
+  const resolvedSchema = pgIdentifier(root.schema ?? pgSchema)
+
+  for (const [name, property] of Object.entries(properties)) {
+    const column = compileColumn(name, property, required)
+    columns.push(column)
+
+    const override = readOverride(property)
+
+    if (override.unique != null && override.unique !== false) {
+      if (column.secure) {
+        /**
+         * Encryption uses a managed nonce, so two identical plaintexts encrypt to different
+         * ciphertexts — a unique constraint on the column would never collide, silently.
+         */
+        throw new UnsupportedArgumentError(`secure-unique:${name}`)
+      }
+      uniques.push({
+        columns: [name],
+        ...(typeof override.unique === 'string' ? { name: override.unique } : {})
+      })
+    }
+
+    if (override.index != null && override.index !== false) {
+      if (column.secure) {
+        console.warn(
+          `@owlmeans/postgres-resource: index on secure column "${alias}.${name}" — encryption`
+          + ' is non-deterministic, so equality lookups through it will never match.'
+        )
+      }
+      const specs = override.index === true
+        ? [{ columns: [name] }]
+        : Array.isArray(override.index) ? override.index : [override.index]
+      indexes.push(...specs.map(spec => ({ ...spec, columns: spec.columns ?? [name] })))
+    }
+
+    if (override.references != null) {
+      references.push({
+        ...override.references,
+        property: name,
+        column: override.references.column ?? ID_FIELD,
+        name: pgIdentifier(override.references.name ?? `${resolvedTable}_${column.column}_fkey`)
+      })
+    }
+
+    if (override.check != null) {
+      checks.push({
+        name: `${resolvedTable}_${column.column}_check`,
+        expression: override.check.replace(/\{\{col\}\}/g, quoteIdent(column.column))
+      })
+    }
+
+    /** A string enum becomes a CHECK — a native PG enum can't drop values and can't be altered in a transaction. */
+    if (property.enum != null && !column.secure && override.type == null
+      && baseType(property) === 'string' && property.enum.every(value => typeof value === 'string')) {
+      const values = (property.enum as string[]).map(value => `'${value.replace(/'/g, "''")}'`).join(', ')
+      const predicate = column.notNull
+        ? `${quoteIdent(column.column)} IN (${values})`
+        : `${quoteIdent(column.column)} IS NULL OR ${quoteIdent(column.column)} IN (${values})`
+      checks.push({ name: `${resolvedTable}_${column.column}_enum`, expression: predicate })
+    }
+  }
+
+  const byProperty: Record<string, ColumnSpec> = {}
+  const byColumn: Record<string, ColumnSpec> = {}
+  for (const column of columns) {
+    byProperty[column.property] = column
+    byColumn[column.column] = column
+  }
+
+  let primaryKey: string[] = []
+  if (root.primaryKey != null && root.primaryKey.length > 0) {
+    primaryKey = root.primaryKey.map(property => {
+      const column = byProperty[property]
+      if (column == null) {
+        throw new UnsupportedArgumentError(`primary-key:${property}`)
+      }
+      column.notNull = true
+
+      return column.column
+    })
+  } else {
+    const explicit = columns.filter(column => column.primaryKey)
+    if (explicit.length > 0) {
+      primaryKey = explicit.map(column => {
+        column.notNull = true
+        return column.column
+      })
+    } else {
+      /**
+       * Every OwlMeans record is addressed by `id`, so the column is synthesized when the
+       * schema doesn't declare it — `create()` and `load()` both assume it exists.
+       */
+      let id = byProperty[ID_FIELD]
+      if (id == null) {
+        id = {
+          property: ID_FIELD, column: ID_FIELD, sqlType: DEF_SQL_TYPE, jsonType: 'string',
+          notNull: true, primaryKey: true, secure: false, jsonb: false, array: false,
+          managed: true, defaultRaw: DEF_ID_DEFAULT
+        }
+        columns.unshift(id)
+        byProperty[ID_FIELD] = id
+        byColumn[ID_FIELD] = id
+      } else {
+        id.notNull = true
+        id.primaryKey = true
+        if (id.defaultRaw == null && id.defaultLiteral === undefined) {
+          id.defaultRaw = id.sqlType === 'uuid' ? 'gen_random_uuid()' : DEF_ID_DEFAULT
+        }
+      }
+      primaryKey = [id.column]
+    }
+  }
+
+  /** Index and unique specs address properties; map them onto physical columns once, here. */
+  const mapColumns = (names: string[]): string[] => names.map(name => {
+    const column = byProperty[name]
+    if (column == null) {
+      throw new UnsupportedArgumentError(`unknown-property:${name}`)
+    }
+
+    return column.column
+  })
+
+  const resolvedIndexes: PgIndexSpec[] = indexes.map(spec => {
+    const list = spec.columns == null
+      ? []
+      : mapColumns(Array.isArray(spec.columns) ? spec.columns : [spec.columns])
+    const suffix = spec.unique === true ? 'uq' : 'idx'
+
+    return {
+      ...spec,
+      columns: list,
+      method: spec.method ?? PgIndexMethod.BTree,
+      name: pgIdentifier(spec.name ?? `${resolvedTable}_${list.join('_')}_${suffix}`)
+    }
+  })
+
+  const resolvedUniques: PgUniqueSpec[] = uniques.map(spec => {
+    const list = mapColumns(spec.columns)
+
+    return { columns: list, name: pgIdentifier(spec.name ?? `${resolvedTable}_${list.join('_')}_key`) }
+  })
+
+  return {
+    alias,
+    schema: resolvedSchema,
+    table: resolvedTable,
+    qualified: qualify(resolvedSchema, resolvedTable),
+    columns,
+    byProperty,
+    byColumn,
+    primaryKey,
+    uniques: resolvedUniques,
+    checks: checks.map(check => ({ ...check, name: pgIdentifier(check.name ?? `${resolvedTable}_check`) })),
+    indexes: resolvedIndexes,
+    references,
+    unmanaged: [
+      ...(root.unmanaged ?? []).map(name => byProperty[name]?.column ?? pgIdentifier(name)),
+      ...columns.filter(column => !column.managed).map(column => column.column)
+    ],
+    autoSync: root.autoSync ?? autoSync,
+    ...(root.comment != null ? { comment: root.comment } : {})
+  }
+}
+
+/**
+ * AJV keyword registration for consumers running in strict mode. The mapper reads the raw
+ * schema object and never validates through AJV, so this is purely to stop strict mode
+ * rejecting a schema that carries `pg` overrides.
+ */
+export const pgKeyword = { keyword: PG_KEYWORD, valid: true }

package/src/utils/sync.ts

@@ -0,0 +1,106 @@
+import type { PoolClient } from 'pg'
+
+import { PgErrorCode } from '../consts.js'
+import { PostgresCastRequired, PostgresSyncError, describePgError, pgErrorToResourceError } from '../errors.js'
+import type { DdlPlan, TableSpec } from '../types.js'
+import { countNonNull } from './introspect.js'
+import { advisoryKey, quoteIdent } from './name.js'
+
+/**
+ * Codes that mean "the cast is legal, the data isn't": a `text` column holding `'a'` retyped to
+ * `integer`, a value too wide for the new length, a number past the new range. Postgres refuses
+ * each of these rather than truncating, and the remedy is the same one a refused cast needs — so
+ * they are only read as a cast problem here, inside DDL. On the CRUD path the identical code means
+ * a caller passed a bad value, which is a different bug with a different fix.
+ */
+const CAST_DATA_CODES: string[] = [
+  PgErrorCode.InvalidTextRepresentation,
+  PgErrorCode.StringDataRightTruncation,
+  PgErrorCode.NumericValueOutOfRange
+]
+
+/**
+ * Serialize initialization across replicas. A session level lock, not `xact`, because
+ * migrations run in their own transactions inside the critical section.
+ */
+export const acquireLock = async (client: PoolClient, qualified: string): Promise<void> => {
+  const [first, second] = advisoryKey(qualified)
+  await client.query('SELECT pg_advisory_lock($1, $2)', [first, second])
+}
+
+export const releaseLock = async (client: PoolClient, qualified: string): Promise<void> => {
+  const [first, second] = advisoryKey(qualified)
+  try {
+    await client.query('SELECT pg_advisory_unlock($1, $2)', [first, second])
+  } catch {
+    /** Releasing the session drops the lock anyway — never mask the original failure. */
+  }
+}
+
+export const ensureSchema = async (client: PoolClient, schema: string): Promise<void> => {
+  try {
+    await client.query(`CREATE SCHEMA IF NOT EXISTS ${quoteIdent(schema)}`)
+  } catch (error) {
+    throw pgErrorToResourceError(error)
+  }
+}
+
+/**
+ * Apply a reconciliation plan in a single transaction.
+ *
+ * Postgres DDL is transactional, so a plan that fails partway leaves the table exactly as
+ * it was. That property is what makes converging without confirmation prompts tolerable —
+ * the table is either fully converged or untouched, never half migrated.
+ *
+ * @throws {PostgresCastRequired} Postgres refused an automatic cast.
+ * @throws {PostgresSyncError} any other statement failed — the message names the statement.
+ */
+export const applyPlan = async (
+  client: PoolClient, spec: TableSpec, plan: DdlPlan
+): Promise<DdlPlan> => {
+  if (plan.statements.length < 1) {
+    return plan
+  }
+
+  /** Read what a drop would cost before the transaction opens, so the log can say it. */
+  for (const statement of plan.statements) {
+    if (statement.destructive === true) {
+      statement.affected = await countNonNull(client, spec.qualified, statement.target)
+    }
+  }
+
+  await client.query('BEGIN')
+  let current = ''
+  try {
+    for (const statement of plan.statements) {
+      current = statement.sql
+      if (statement.destructive === true && (statement.affected ?? 0) > 0) {
+        console.warn(
+          `@owlmeans/postgres-resource: dropping ${spec.qualified}.${statement.target} —`
+          + ` ${statement.affected} row(s) hold a value. Declare it under \`pg.unmanaged\``
+          + ' or set `pg: { managed: false }` to keep it.'
+        )
+      }
+      await client.query(statement.sql)
+    }
+    await client.query('COMMIT')
+  } catch (error) {
+    await client.query('ROLLBACK').catch(() => undefined)
+    const translated = pgErrorToResourceError(error)
+    const code = (error as { code?: string } | null)?.code
+    if (translated instanceof PostgresCastRequired || (code != null && CAST_DATA_CODES.includes(code))) {
+      const failure = new PostgresCastRequired(
+        `${spec.qualified}: ${describePgError(error)} — statement: ${current}.`
+        + ' Declare `pg: { using: \'<expr>\' }` on the property, or perform the change in a'
+        + ' `pre` migration so reconciliation observes no drift.'
+      )
+      failure.cause = error
+      throw failure
+    }
+    const failure = new PostgresSyncError(`${spec.qualified}: ${describePgError(error)} — statement: ${current}`)
+    failure.cause = error
+    throw failure
+  }
+
+  return plan
+}

package/src/utils/table.ts

@@ -0,0 +1,76 @@
+import { sql } from 'drizzle-orm'
+import { customType, index, pgSchema, primaryKey, uniqueIndex } from 'drizzle-orm/pg-core'
+
+import { PgIndexMethod } from '../consts.js'
+import type { PgRuntimeTable, TableSpec } from '../types.js'
+
+/**
+ * One universal column builder.
+ *
+ * Drizzle's typed builders (`text()`, `integer()`, ...) exist to drive compile time
+ * inference from a statically declared schema. Our tables are compiled at runtime from
+ * JSON Schema, so there's nothing to infer and a per-type switch would buy nothing but a
+ * mapping to keep in sync. `customType` lets the already-canonical `sqlType` string
+ * through verbatim, and `getSQLType()` reads it straight back for the drift cross-check.
+ *
+ * Marshalling is deliberately identity here — coercion lives in `marshal.ts`, which is
+ * driven by the same spec and also serves the raw-SQL path that never touches Drizzle.
+ */
+const pgRaw = customType<{ data: unknown, driverData: unknown, config: { sqlType: string } }>({
+  dataType: config => config?.sqlType ?? 'text'
+})
+
+/**
+ * Build the Drizzle table used for CRUD query construction. Structure DDL is emitted from
+ * the {@link TableSpec} directly — Drizzle exposes no runtime DDL generator (that lives in
+ * drizzle-kit, which would be a second owner of the same tables).
+ */
+export const specToTable = (spec: TableSpec): PgRuntimeTable => {
+  const namespace = pgSchema(spec.schema)
+  const columns: Record<string, any> = {}
+
+  for (const column of spec.columns) {
+    let builder: any = pgRaw(column.column, { sqlType: column.sqlType })
+    if (column.notNull) {
+      builder = builder.notNull()
+    }
+    if (column.defaultRaw != null) {
+      builder = builder.default(sql.raw(column.defaultRaw))
+    } else if (column.defaultLiteral !== undefined) {
+      builder = builder.default(column.defaultLiteral)
+    }
+    /** Composite keys are declared in the extra-config callback instead. */
+    if (column.primaryKey && spec.primaryKey.length === 1) {
+      builder = builder.primaryKey()
+    }
+    columns[column.property] = builder
+  }
+
+  return namespace.table(spec.table, columns, (table: any) => [
+    ...spec.indexes
+      .filter(entry => entry.columns != null && entry.columns.length > 0 && entry.expression == null)
+      .map(entry => {
+        const properties = (entry.columns as string[])
+          .map(name => spec.byColumn[name]?.property)
+          .filter((name): name is string => name != null && table[name] != null)
+        const builder = entry.unique === true ? uniqueIndex(entry.name!) : index(entry.name!)
+
+        /**
+         * Both `using` and `primaryKey` want a non-empty tuple of statically known columns.
+         * Nothing here is statically known — the columns were compiled a few lines above —
+         * so the tuple shape is asserted rather than proven.
+         */
+        return builder.using(
+          entry.method ?? PgIndexMethod.BTree,
+          ...properties.map(name => table[name]) as [any, ...any[]]
+        )
+      }),
+    ...(spec.primaryKey.length > 1
+      ? [primaryKey({
+        columns: spec.primaryKey
+          .map(name => table[spec.byColumn[name]?.property ?? name])
+          .filter((column: unknown) => column != null) as [any, ...any[]]
+      })]
+      : [])
+  ]) as PgRuntimeTable
+}