@bakery-framework/orm 2.1.2 → 2.2.0

package/src/adapters/pgsql.ts

@@ -1,8 +1,41 @@
 import { Case, Try } from '@bakery-framework/core/utils'
 import { SQL } from 'bun'
+import { encodePgArray, isArrayValue } from '../array-value'
 import { type PoolOptions, withPoolOptions } from '../pool'
 import type * as SyncTypes from '../sync/types'
 import { createExecutor, isOpenConnection, SQLAdapter } from './base'
+import { pgTypeProblem } from './live-types'
+import type { ScriptDialect } from '../migrate/script'
+/** `pg_advisory_lock` key for migration runs: "bakerymg" in ASCII, 0x62616b6572796d67. */
+const MIGRATION_LOCK_KEY = '7089065371914300775'
+
+/**
+ * Tables, partitioned tables, views, materialized views and foreign tables of
+ * the current schema, a row per column. `array_agg(…::text)` because an
+ * enum's labels are of type `name`, and Bun returns an array of an unfamiliar
+ * type as its raw `{…}` string.
+ */
+const PG_LIVE_COLUMNS = `
+  SELECT c.relname AS table_name,
+         c.relkind AS kind,
+         a.attname AS column_name,
+         CASE WHEN t.typtype = 'd'
+              THEN format_type(t.typbasetype, t.typtypmod)
+              ELSE format_type(a.atttypid, a.atttypmod) END AS type,
+         NOT a.attnotnull AS nullable,
+         COALESCE(a.attnum = ANY (pk.conkey), false) AS is_primary,
+         CASE WHEN t.typtype = 'e' THEN (
+           SELECT array_agg(e.enumlabel::text ORDER BY e.enumsortorder)
+           FROM pg_enum e WHERE e.enumtypid = t.oid
+         ) END AS enum_values
+  FROM pg_class c
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped
+  JOIN pg_type t ON t.oid = a.atttypid
+  LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid AND pk.contype = 'p'
+  WHERE n.nspname = current_schema()
+    AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
+  ORDER BY c.relname, a.attnum`
 
 interface PGSQLParserState {
   inSingleQuote: boolean
@@ -16,6 +49,55 @@ export class PGAdapter extends SQLAdapter {
   protected readonly sql: SQL
   override readonly quoteChar: string = '"'
 
+  override get scriptDialect(): ScriptDialect {
+    return 'pgsql'
+  }
+
+  /**
+   * A session advisory lock. `pg_advisory_lock` waits for the other run to
+   * release it, so a second deploy blocks and then finds nothing pending. The
+   * key is the ASCII of "bakerymg".
+   */
+  override async takeMigrationLock(): Promise<() => Promise<void>> {
+    await this.sql.unsafe(`SELECT pg_advisory_lock(${MIGRATION_LOCK_KEY})`)
+    return async () => {
+      await this.sql.unsafe(`SELECT pg_advisory_unlock(${MIGRATION_LOCK_KEY})`)
+    }
+  }
+
+  /**
+   * From `pg_catalog`, which shows every relation to every role, where
+   * `information_schema` shows only what the role holds a privilege on. A
+   * domain is resolved to its base type; an enum brings its labels.
+   */
+  override async liveColumns(): Promise<SQLAdapter.LiveColumn[]> {
+    const rows = (await this.sql.unsafe(PG_LIVE_COLUMNS)) as {
+      table_name: string
+      kind: string
+      column_name: string
+      type: string
+      nullable: boolean
+      is_primary: boolean
+      enum_values: string[] | null
+    }[]
+    return rows.map(row => ({
+      table: row.table_name,
+      column: row.column_name,
+      type: row.type.toLowerCase(),
+      nullable: Boolean(row.nullable),
+      primary: Boolean(row.is_primary),
+      view: row.kind === 'v' || row.kind === 'm',
+      ...(row.enum_values ? { enumValues: row.enum_values } : {}),
+    }))
+  }
+
+  override columnTypeProblem(
+    declared: SyncTypes.ColumnConstraint,
+    live: SQLAdapter.LiveColumn,
+  ): string | null {
+    return pgTypeProblem(declared, live)
+  }
+
   constructor(connectionTarget?: string | URL | SQL, pool: PoolOptions = {}) {
     const target =
       typeof connectionTarget === 'string' || connectionTarget instanceof URL
@@ -130,12 +212,42 @@ export class PGAdapter extends SQLAdapter {
     return result
   }
 
+  /**
+   * Plain arrays where Bun returned typed ones. Measured: an `int4[]` column
+   * comes back as an `Int32Array` (and `float4[]` as a `Float32Array`) once
+   * the query has a bound parameter, and as an ordinary array when it has
+   * none, so the same column changed type with the WHERE clause, and
+   * `JSON.stringify` wrote an Int32Array as `{"0":1,"1":2}`. A `Uint8Array`
+   * is left alone: that is a `bytea`, read as a Buffer.
+   */
+  private static plainArrays(rows: unknown): unknown {
+    if (!Array.isArray(rows)) return rows
+    for (const row of rows) {
+      if (!row || typeof row !== 'object') continue
+      for (const key in row) {
+        const value = (row as Record<string, unknown>)[key]
+        if (
+          ArrayBuffer.isView(value) &&
+          !(value instanceof Uint8Array) &&
+          !(value instanceof DataView)
+        ) {
+          ;(row as Record<string, unknown>)[key] = Array.from(
+            value as unknown as ArrayLike<unknown>,
+          )
+        }
+      }
+    }
+    return rows
+  }
+
   readonly execute: SQLAdapter.Executor = createExecutor(
     async (sqlText: string, params: unknown[] = []) =>
-      (await this.sql.unsafe(
-        PGAdapter.normalizePostgresSQL(sqlText, params),
-        params,
-      )) as any,
+      PGAdapter.plainArrays(
+        await this.sql.unsafe(
+          PGAdapter.normalizePostgresSQL(sqlText, params),
+          params,
+        ),
+      ) as any,
     async (
       sqlText: string,
       params: unknown[] = [],
@@ -175,7 +287,14 @@ export class PGAdapter extends SQLAdapter {
       return { lastInsertRowid, changes }
     },
     this.driver,
-    { explain: error => this.explainError(error) },
+    {
+      explain: error => this.explainError(error),
+      // A `DB.array()` value as the array literal Postgres casts from text.
+      bind: params =>
+        params.map(param =>
+          isArrayValue(param) ? encodePgArray(param.values) : param,
+        ),
+    },
   )
 
   async hasCol(table: string, column: string): Promise<boolean> {

package/src/adapters/sqlite.ts

@@ -6,6 +6,7 @@ import { Case, Try } from '@bakery-framework/core/utils'
 import { SQL } from 'bun'
 import type * as SyncTypes from '../sync/types'
 import { createExecutor, SQLAdapter } from './base'
+import { sqliteTypeProblem } from './live-types'
 
 // Convention 4: logging is data, declared in a table rather than formatted at
 // the call site.
@@ -579,6 +580,41 @@ export class SQLiteAdapter extends SQLAdapter {
     return false
   }
 
+  /** `sqlite_master` and `pragma_table_info`, SQLite's whole catalog. */
+  override async liveColumns(): Promise<SQLAdapter.LiveColumn[]> {
+    const rows = (await this.sql.unsafe(
+      `SELECT m.name AS table_name, m.type AS kind, p.name AS column_name,
+              p.type AS type, p."notnull" AS not_null, p.pk AS pk
+       FROM sqlite_master m JOIN pragma_table_info(m.name) p
+       WHERE m.type IN ('table', 'view') AND m.name NOT LIKE 'sqlite_%'
+       ORDER BY m.name, p.cid`,
+    )) as {
+      table_name: string
+      kind: string
+      column_name: string
+      type: string | null
+      not_null: number
+      pk: number
+    }[]
+    return rows.map(row => ({
+      table: row.table_name,
+      column: row.column_name,
+      type: (row.type ?? '').toLowerCase(),
+      // An INTEGER PRIMARY KEY is the rowid and cannot hold NULL, whatever
+      // `notnull` says.
+      nullable: !Number(row.not_null) && !Number(row.pk),
+      primary: Number(row.pk) > 0,
+      view: row.kind === 'view',
+    }))
+  }
+
+  override columnTypeProblem(
+    declared: SyncTypes.ColumnConstraint,
+    live: SQLAdapter.LiveColumn,
+  ): string | null {
+    return sqliteTypeProblem(declared, live)
+  }
+
   async getConstraints(): Promise<SyncTypes.DBConstraints> {
     const tables = (await this.query(
       'SELECT sql,name,type FROM sqlite_master' +
@@ -596,9 +632,9 @@ export class SQLiteAdapter extends SQLAdapter {
       // records. This one survived it: `getConstraints` runs over every table
       // the database reports, so a name carrying an apostrophe closed the
       // literal and the rest of the statement went with it.
-      const cols = (await this.query(
-        'SELECT * FROM pragma_table_info(?)',
-      ).all(table.name)) as any[]
+      const cols = (await this.query('SELECT * FROM pragma_table_info(?)').all(
+        table.name,
+      )) as any[]
 
       if (table.type === 'view') {
         const match = table.sql.match(/AS\s+(.*)/is)

package/src/array-value.ts

@@ -0,0 +1,48 @@
+/**
+ * A Postgres array, bound as one parameter: what `DB.array([...])` makes.
+ *
+ * Needed because Bun cannot bind a JavaScript array to a Postgres array
+ * parameter. Measured: into `text[]` it fails with `22P02 malformed array
+ * literal: "x,y"`, into `int4[]` with `08P01 insufficient data left in
+ * message`, explicit `::int4[]` cast or not, and the same in `@>`, `&&` and
+ * `= ANY(?)`. An array literal sent as text binds in all of those places with
+ * no cast, for text, int4, int8, jsonb and uuid elements alike.
+ *
+ * A wrapper rather than encoding every JavaScript array: a `jsonb` column
+ * handed a plain array stores a JSON array today, and an array literal sent
+ * there would arrive as a malformed JSON string instead. Only the query can
+ * say which one it means, since the query layer carries no column types.
+ */
+export class ArrayValue<T = unknown> {
+  constructor(readonly values: readonly T[]) {}
+}
+
+export function isArrayValue(value: unknown): value is ArrayValue {
+  return value instanceof ArrayValue
+}
+
+/**
+ * The text of a Postgres array literal for `values`: `{"a","b c",NULL}`.
+ *
+ * Every element is double-quoted, with backslashes and double quotes escaped,
+ * so a comma, a brace or a space inside one is only text. NULL is bare.
+ * Numbers and booleans are quoted too, which Postgres reads the same as bare
+ * when it casts the literal to the column's element type. A nested array is a
+ * nested literal, a Date its ISO string, and any other object its JSON, which
+ * is how a `jsonb[]` element is written.
+ */
+export function encodePgArray(values: readonly unknown[]): string {
+  return `{${values.map(encodeElement).join(',')}}`
+}
+
+function encodeElement(value: unknown): string {
+  if (value === null || value === undefined) return 'NULL'
+  if (Array.isArray(value)) return encodePgArray(value)
+  const text =
+    value instanceof Date
+      ? value.toISOString()
+      : typeof value === 'object'
+        ? JSON.stringify(value)
+        : String(value)
+  return `"${text.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`
+}

package/src/field.ts

@@ -35,6 +35,35 @@ function column<T, N extends boolean = false, O extends boolean = false>(
 /** `T` when the column is NOT NULL, `T | null` when it is nullable. */
 type Nullable<T, N extends boolean> = N extends true ? T | null : T
 
+/** The element types `Field.Array` takes. See there for why these and not `uuid`. */
+type ArrayElement =
+  | 'text'
+  | 'varchar'
+  | 'integer'
+  | 'smallint'
+  | 'bigint'
+  | 'numeric'
+  | 'real'
+  | 'double precision'
+  | 'boolean'
+  | 'timestamptz'
+  | 'jsonb'
+
+/** What Bun returns for one element of an array of `E`. */
+type ElementOf<E extends ArrayElement> = E extends
+  | 'integer'
+  | 'smallint'
+  | 'real'
+  | 'double precision'
+  ? number
+  : E extends 'boolean'
+    ? boolean
+    : E extends 'timestamptz'
+      ? Date
+      : E extends 'jsonb'
+        ? unknown
+        : string
+
 /**
  * Optional on insert when there is a default, or the column is nullable.
  *
@@ -295,6 +324,75 @@ export const Field = {
     (nullable: true): TableDef<unknown, true, true>
   },
 
+  /**
+   * A column whose SQL type the migration files give, for a type this
+   * vocabulary has no spelling for: `numeric(5,2)`, `timestamptz`, a `bigint`
+   * key, `text[]`. **Migrations mode only**: classic `db:sync` refuses a schema
+   * that uses it, since it cannot create or compare a type it does not know.
+   *
+   * The type argument is what the driver hands back, and it is the caller's
+   * to get right. Measured with Bun on Postgres: `numeric` and `bigint` come
+   * back as strings, `timestamptz` and `date` as a `Date`, `uuid` as a string.
+   * `db:sync` checks the SQL type against the database, not the type
+   * argument against the SQL type.
+   *
+   *     weight: Field.Sql<string>('numeric(5,2)'),
+   *     closedAt: Field.Sql<Date>('timestamptz', { nullable: true }),
+   *     createdAt: Field.Sql<Date>('timestamptz', { optional: true }),
+   *
+   * `optional` says the database fills the column when an insert leaves it
+   * out (a `DEFAULT`, an identity), so inserts may omit it.
+   */
+  Sql: ((
+    sqlType: string,
+    options: { nullable?: true; optional?: true } = {},
+  ) => {
+    const def: Record<string, unknown> = { type: 'sql', sqlType }
+    if (options.nullable) def.nullable = true
+    return def
+  }) as {
+    <T>(sqlType: string): TableDef<T, false, false>
+    <T>(sqlType: string, options: { optional: true }): TableDef<T, false, true>
+    <T>(
+      sqlType: string,
+      options: { nullable: true; optional?: true },
+    ): TableDef<T | null, true, true>
+  },
+
+  /**
+   * A Postgres array column, typed by its element: `Field.Array('text')` is
+   * `text[]` and `string[]`. `Field.Sql` with the element's type, so it
+   * belongs to migrations mode too, and `db:sync` checks it as `text[]`.
+   *
+   * The element types are the ones whose arrays Bun returns as the type
+   * says, measured: `bigint` and `numeric` elements come back as strings,
+   * `timestamptz` ones as a `Date`. `uuid` is not offered, because Bun hands
+   * back a `uuid[]` as its raw `{…}` string; store such ids as `text[]`.
+   *
+   * Write one with `DB.array([...])`; a plain array cannot be bound. And keep
+   * an `integer[]` free of NULL elements (a CHECK does it): once a query has a
+   * parameter, Bun's driver throws on reading one, as it does on any 2-D
+   * array, inside the driver where nothing here can catch it first.
+   */
+  Array: (<E extends ArrayElement>(
+    element: E,
+    options: { nullable?: true; optional?: true } = {},
+  ) => {
+    const def: Record<string, unknown> = { type: 'sql', sqlType: `${element}[]` }
+    if (options.nullable) def.nullable = true
+    return def
+  }) as {
+    <E extends ArrayElement>(element: E): TableDef<ElementOf<E>[], false, false>
+    <E extends ArrayElement>(
+      element: E,
+      options: { optional: true },
+    ): TableDef<ElementOf<E>[], false, true>
+    <E extends ArrayElement>(
+      element: E,
+      options: { nullable: true; optional?: true },
+    ): TableDef<ElementOf<E>[] | null, true, true>
+  },
+
   /** True/false: `BOOLEAN` on Postgres, `TINYINT(1)` on MySQL. */
   Bool: <D extends boolean | null | undefined = undefined>(d?: D) =>
     column<

package/src/migrate/check.ts

@@ -0,0 +1,108 @@
+import { Case } from '@bakery-framework/core/utils'
+import type { SQLAdapter } from '../adapters/base'
+import type * as SyncTypes from '../sync/types'
+
+/** One way the declared tables and the database disagree. */
+export interface SchemaMismatch {
+  /** The table as declared. */
+  table: string
+  /** The column as declared, when the mismatch is a column's. */
+  column?: string
+  problem: string
+}
+
+/**
+ * Compare the declared tables with the database, column by column, reading
+ * the catalog and nothing else: what `db:sync` does in migrations mode, where
+ * the migration files are the schema and the declarations only type queries.
+ *
+ * It asks one question of each declared column: would a query typed by this
+ * declaration be right about the column? So a table or column the database
+ * lacks, a type the declaration does not accept (each adapter's rules, see
+ * `adapters/live-types.ts`), a disagreement about NULL, and a primary key in
+ * the wrong place are mismatches. A table or column the database has and the
+ * declarations leave out is not: queries simply never name it.
+ *
+ * A view's columns are not held to their nullability, since catalogs report
+ * every view column as nullable whatever the query beneath it can return.
+ */
+export async function checkSchema(
+  adapter: SQLAdapter,
+  declared: SyncTypes.DBConstraints,
+): Promise<SchemaMismatch[]> {
+  const live = new Map<string, Map<string, SQLAdapter.LiveColumn>>()
+  for (const column of await adapter.liveColumns()) {
+    let columns = live.get(column.table)
+    if (!columns) live.set(column.table, (columns = new Map()))
+    columns.set(column.column, column)
+  }
+
+  const mismatches: SchemaMismatch[] = []
+  for (const [table, constraints] of Object.entries(declared)) {
+    const sqlTable = Case.snake(table)
+    const columns = live.get(sqlTable)
+    if (!columns) {
+      mismatches.push({
+        table,
+        problem: `${sqlTable} is declared, and the database has no table or view by that name.`,
+      })
+      continue
+    }
+
+    for (const [column, def] of Object.entries(constraints)) {
+      // `_view`, `_oldTable` and `_transform` sit beside the columns.
+      if (column.startsWith('_') || !def || typeof def !== 'object') continue
+      const constraint = def as SyncTypes.ColumnConstraint
+      const sqlColumn = Case.snake(column)
+      const where = `${sqlTable}.${sqlColumn}`
+      const liveColumn = columns.get(sqlColumn)
+      if (!liveColumn) {
+        mismatches.push({
+          table,
+          column,
+          problem: `${where} is declared, and the table has no such column.`,
+        })
+        continue
+      }
+
+      const typeProblem = adapter.columnTypeProblem(constraint, liveColumn)
+      if (typeProblem) {
+        mismatches.push({ table, column, problem: `${where}: ${typeProblem}.` })
+      }
+      if (liveColumn.view) continue
+
+      const nullable = Boolean(constraint.nullable) && !constraint.primary
+      if (nullable !== liveColumn.nullable) {
+        mismatches.push({
+          table,
+          column,
+          problem: nullable
+            ? `${where}: declared nullable, and the database has it NOT NULL.`
+            : `${where}: declared NOT NULL, and the database allows NULL.`,
+        })
+      }
+      if (constraint.primary && !liveColumn.primary) {
+        mismatches.push({
+          table,
+          column,
+          problem: `${where}: declared the primary key, and it is not the database's.`,
+        })
+      }
+    }
+  }
+  return mismatches
+}
+
+/** The `Field.Sql` columns of a schema, as `table.column`, for classic sync to refuse. */
+export function sqlColumnsIn(declared: SyncTypes.DBConstraints): string[] {
+  const found: string[] = []
+  for (const [table, constraints] of Object.entries(declared)) {
+    for (const [column, def] of Object.entries(constraints)) {
+      if (column.startsWith('_') || !def || typeof def !== 'object') continue
+      if ((def as SyncTypes.ColumnConstraint).type === 'sql') {
+        found.push(`${table}.${column}`)
+      }
+    }
+  }
+  return found
+}

package/src/migrate/files.ts

@@ -0,0 +1,111 @@
+import { readdir } from 'node:fs/promises'
+import { join } from 'node:path'
+
+/**
+ * A migration file: SQL, named for the order it runs in, run once.
+ *
+ * `0001_rooms.sql`, `0002_campus_check.sql`: a number, an underscore, a name.
+ * Files run in the order of their numbers (so `9_x` comes before `10_y`), and
+ * by name where two share a number, which is what two branches that each
+ * added `0007_…` produce when they merge.
+ */
+export interface MigrationFile {
+  /** The file name, which is also the file's key in the ledger. */
+  name: string
+  /** The number before the underscore, as a bigint so no prefix can overflow. */
+  order: bigint
+  path: string
+  /** The text with a byte-order mark and CRLF line endings removed. */
+  text: string
+  /** SHA-256 of `text`, hex. What the ledger compares on every run. */
+  checksum: string
+  /**
+   * False when the file's first line is `-- bakery:no-transaction`, for a
+   * statement no transaction can hold (`CREATE INDEX CONCURRENTLY`). Such a
+   * file may hold only that one statement.
+   */
+  transaction: boolean
+}
+
+/** `<digits>_<name>.sql`. */
+const MIGRATION_NAME = /^(\d+)_[A-Za-z0-9._-]+\.sql$/
+
+const NO_TRANSACTION = /^--\s*bakery:\s*no-transaction\s*$/i
+
+export type ReadResult =
+  | { ok: true; files: MigrationFile[] }
+  | { ok: false; problems: string[] }
+
+/**
+ * Every migration in `dir`, in the order they run.
+ *
+ * A `.sql` file whose name does not fit is refused rather than skipped: a
+ * misnamed migration that is silently ignored is a schema change that never
+ * happens, discovered in production.
+ */
+export async function readMigrations(dir: string): Promise<ReadResult> {
+  let names: string[]
+  try {
+    names = await readdir(dir)
+  } catch (error: any) {
+    if (error?.code === 'ENOENT') {
+      return { ok: false, problems: [`${dir} does not exist.`] }
+    }
+    throw error
+  }
+
+  const problems: string[] = []
+  const files: MigrationFile[] = []
+  for (const name of names) {
+    if (!name.toLowerCase().endsWith('.sql')) continue
+    const match = MIGRATION_NAME.exec(name)
+    if (!match) {
+      problems.push(
+        `${name} is not named <number>_<name>.sql, so it has no place in the order.`,
+      )
+      continue
+    }
+    const path = join(dir, name)
+    const text = normalize(await Bun.file(path).text())
+    files.push({
+      name,
+      order: BigInt(match[1]!),
+      path,
+      text,
+      checksum: checksumOf(text),
+      transaction: !NO_TRANSACTION.test(firstLine(text)),
+    })
+  }
+
+  if (problems.length) return { ok: false, problems }
+  files.sort(compareMigrations)
+  return { ok: true, files }
+}
+
+/** Run order: by number, then by name. */
+export function compareMigrations(
+  a: Pick<MigrationFile, 'order' | 'name'>,
+  b: Pick<MigrationFile, 'order' | 'name'>,
+): number {
+  if (a.order !== b.order) return a.order < b.order ? -1 : 1
+  return a.name < b.name ? -1 : a.name > b.name ? 1 : 0
+}
+
+/**
+ * The text the checksum is taken over. A byte-order mark and CRLF line endings
+ * are an editor's or a checkout's, not the migration's: a repository cloned on
+ * Windows with `core.autocrlf` must not make every applied file look edited.
+ */
+export function normalize(text: string): string {
+  return text.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n')
+}
+
+export function checksumOf(text: string): string {
+  return new Bun.CryptoHasher('sha256').update(text).digest('hex')
+}
+
+function firstLine(text: string): string {
+  const trimmed = text.trimStart()
+  const end = trimmed.indexOf('\n')
+  return (end === -1 ? trimmed : trimmed.slice(0, end)).trim()
+}