@bakery-framework/orm 2.1.2 → 2.2.0

package/src/migrate/index.ts

@@ -0,0 +1,168 @@
+import { resolve } from 'node:path'
+import { Logger, messageLogger } from '@bakery-framework/core/logger'
+import { resolveAdapter, type SQLAdapter } from '../adapters'
+import { type MigrationFile, readMigrations } from './files'
+import {
+  applyMigrations,
+  ensureLedger,
+  MigrationError,
+  planMigrations,
+  readApplied,
+} from './runner'
+
+export * from './files'
+export * from './runner'
+export {
+  type ScriptDialect,
+  scanStatements,
+  transactionControl,
+} from './script'
+
+const logger = new Logger('db-migrate')
+
+// prettier-ignore
+export const migrateMsgs = {
+  OFF: 'E Migrations are off. Set %ymigrations%* in server.config.ts to the folder that holds them.',
+  PROBLEM: 'E {problem}',
+  REFUSED:
+    'E Nothing was applied. Fix the %y{count}%* problem(s) above, then run %ydb:migrate%* again.',
+  UP_TO_DATE: 'I Up to date: %y{count}%* migration(s) have run.',
+  PENDING: 'I %y{count}%* migration(s) to apply from %y{dir}%*:',
+  PENDING_ITEM: 'I   %y{name}%*',
+  APPLIED: 'I Applied %y{name}%* in %y{ms}%* ms.',
+  DONE: 'I %gApplied {count} migration(s).%*',
+  FAILED: 'E {message}',
+} as const
+
+export const MESSAGES = messageLogger(logger, migrateMsgs)
+
+const HELP = `
+Usage: bun run db:migrate [--status] [--help]
+
+Applies the SQL files in the migrations folder (\`migrations\` in
+server.config.ts) that have not run yet, in order, each in a transaction where
+the database can roll DDL back (Postgres and SQLite; not MySQL).
+
+Flags:
+  --status   List what would run, and apply nothing.
+  --help     This.
+
+The connection is DB_MIGRATE_URL when it is set, so migrations can run as a
+role that owns the schema while the app connects as one that does not. Without
+it, DB_URL.
+`
+
+export interface MigrateResult {
+  applied: string[]
+  pending: string[]
+  problems: string[]
+}
+
+/**
+ * Read the folder, compare it with the ledger and, when `apply` is set and
+ * nothing is wrong, apply what is pending. The migration lock is held from the
+ * ledger read to the last file, so a second run started meanwhile waits and
+ * then finds nothing to do.
+ */
+export async function runMigrations(
+  adapter: SQLAdapter,
+  dir: string,
+  options: {
+    apply: boolean
+    onApplied?: (file: MigrationFile, durationMs: number) => void
+  },
+): Promise<MigrateResult> {
+  const read = await readMigrations(dir)
+  if (!read.ok) return { applied: [], pending: [], problems: read.problems }
+
+  const release = await adapter.takeMigrationLock()
+  try {
+    await ensureLedger(adapter)
+    const plan = planMigrations(read.files, await readApplied(adapter), {
+      transactionalDDL: adapter.transactionalDDL,
+      dialect: adapter.scriptDialect,
+    })
+    const pending = plan.pending.map(file => file.name)
+    if (plan.problems.length || !options.apply) {
+      return { applied: [], pending, problems: plan.problems }
+    }
+
+    const applied: string[] = []
+    await applyMigrations(adapter, plan.pending, (file, durationMs) => {
+      applied.push(file.name)
+      options.onApplied?.(file, durationMs)
+    })
+    return { applied, pending, problems: [] }
+  } finally {
+    await release()
+  }
+}
+
+/**
+ * The connection `db:migrate` uses: `DB_MIGRATE_URL`, then the app's own.
+ * Opened with a pool of one, because the migration lock belongs to a session
+ * and every statement has to run on the session that holds it.
+ */
+export async function openMigrationAdapter(): Promise<SQLAdapter> {
+  const url =
+    process.env.DB_MIGRATE_URL ||
+    process.env.DB_URL ||
+    process.env.DATABASE_URL ||
+    ''
+  return await resolveAdapter(url).open(url || undefined, { max: 1 })
+}
+
+/** `bun run db:migrate`. Returns the exit code rather than exiting. */
+export async function migrate(
+  argv: string[] = process.argv.slice(2),
+  cwd: string = process.cwd(),
+): Promise<number> {
+  if (argv.includes('--help') || argv.includes('-h')) {
+    // Usage text is program output, not a log line, as in `db:sync --help`.
+    console.log(HELP)
+    return 0
+  }
+
+  const { initConfig } = await import('@bakery-framework/core/core/config')
+  const config = await initConfig()
+  if (!config.migrations) {
+    MESSAGES.OFF()
+    return 1
+  }
+
+  const dir = resolve(cwd, config.migrations)
+  const apply = !argv.includes('--status')
+  const adapter = await openMigrationAdapter()
+  try {
+    const result = await runMigrations(adapter, dir, {
+      apply,
+      onApplied: (file, ms) => MESSAGES.APPLIED({ name: file.name, ms }),
+    })
+
+    if (result.problems.length) {
+      for (const problem of result.problems) MESSAGES.PROBLEM({ problem })
+      MESSAGES.REFUSED({ count: result.problems.length })
+      return 1
+    }
+    if (!result.pending.length) {
+      const ran = (await readApplied(adapter)).length
+      MESSAGES.UP_TO_DATE({ count: ran })
+      return 0
+    }
+    if (!apply) {
+      MESSAGES.PENDING({ count: result.pending.length, dir })
+      for (const name of result.pending) MESSAGES.PENDING_ITEM({ name })
+      return 0
+    }
+    MESSAGES.DONE({ count: result.applied.length })
+    return 0
+  } catch (error) {
+    if (!(error instanceof MigrationError)) throw error
+    MESSAGES.FAILED({ message: error.message })
+    return 1
+  } finally {
+    await adapter.close()
+  }
+}
+
+if (import.meta.main) process.exit(await migrate())

package/src/migrate/runner.ts

@@ -0,0 +1,208 @@
+import type { SQLAdapter } from '../adapters/base'
+import { compareMigrations, type MigrationFile } from './files'
+import {
+  type ScriptDialect,
+  scanStatements,
+  transactionControl,
+} from './script'
+
+/**
+ * Applied migrations, one row per file.
+ *
+ * Its own table, not a kind of row in `__bakery_schema`: that ledger holds
+ * snapshots of a declared schema, which `db:rollback` replays by altering and
+ * dropping, and a migration is a file that ran. `sync/ledger.ts` lists this
+ * name among the tables classic sync never offers to drop.
+ */
+export const MIGRATIONS_TABLE = '__bakery_migrations'
+
+export interface AppliedMigration {
+  name: string
+  checksum: string
+  /** Unix seconds. */
+  appliedAt: number
+}
+
+/** What a run will apply, and anything that stops it from applying at all. */
+export interface MigrationPlan {
+  pending: MigrationFile[]
+  /** Empty when the run may go ahead. One problem anywhere stops every file. */
+  problems: string[]
+}
+
+export interface PlanOptions {
+  /** `adapter.transactionalDDL`: whether each file runs inside a transaction. */
+  transactionalDDL: boolean
+  dialect: ScriptDialect
+}
+
+/**
+ * Compare the folder with the ledger.
+ *
+ * Fails closed: an applied file that changed or went missing, or a new file
+ * that sorts before one that has run, refuses the whole run. Each of those
+ * means the database and the folder no longer tell the same story, and
+ * applying the rest on top would build on whichever one is wrong.
+ */
+export function planMigrations(
+  files: MigrationFile[],
+  applied: AppliedMigration[],
+  options: PlanOptions,
+): MigrationPlan {
+  const problems: string[] = []
+  const byName = new Map(files.map(file => [file.name, file]))
+  const ran = new Set(applied.map(row => row.name))
+
+  for (const row of applied) {
+    const file = byName.get(row.name)
+    const on = new Date(row.appliedAt * 1000).toISOString().slice(0, 10)
+    if (!file) {
+      problems.push(
+        `${row.name} ran on ${on} and is no longer in the folder. Put it back: the folder has to hold every migration that has run.`,
+      )
+    } else if (file.checksum !== row.checksum) {
+      problems.push(
+        `${row.name} changed after it ran on ${on}. Undo the edit, and write the change as a new migration.`,
+      )
+    }
+  }
+
+  const pending = files.filter(file => !ran.has(file.name))
+  const last = files.filter(file => ran.has(file.name)).at(-1)
+  for (const file of pending) {
+    if (last && compareMigrations(file, last) < 0) {
+      problems.push(
+        `${file.name} sorts before ${last.name}, which has already run. Renumber it to sort after ${last.name}.`,
+      )
+    }
+    problems.push(...fileProblems(file, options))
+  }
+
+  return { pending, problems }
+}
+
+/** What in one file's text would make the runner's handling of it unsafe. */
+function fileProblems(file: MigrationFile, options: PlanOptions): string[] {
+  // MySQL runs every file statement by statement with nothing to roll back,
+  // so there is no transaction for a COMMIT to escape and no reason to hold a
+  // no-transaction file to one statement.
+  if (!options.transactionalDDL) return []
+
+  if (!file.transaction) {
+    const count = scanStatements(file.text, options.dialect).length
+    return count === 1
+      ? []
+      : [
+          `${file.name} runs outside a transaction, so it may hold one statement, and it holds ${count}. Move the others into a file of their own.`,
+        ]
+  }
+
+  return transactionControl(file.text, options.dialect).map(
+    statement =>
+      `${file.name}, line ${statement.line}: ${statement.head} would end the transaction the file runs in and commit part of it. Remove it; each file is committed whole.`,
+  )
+}
+
+/** A migration that failed, and whether anything of it stayed applied. */
+export class MigrationError extends Error {
+  constructor(
+    readonly file: MigrationFile,
+    readonly rolledBack: boolean,
+    options: { cause: unknown },
+  ) {
+    const reason =
+      options.cause instanceof Error
+        ? options.cause.message
+        : String(options.cause)
+    super(
+      `${file.name} failed: ${reason}. ${
+        rolledBack
+          ? 'Nothing in it was applied.'
+          : 'It ran without a transaction, so statements before the failure stay applied. Check the database before running it again.'
+      }`,
+      options,
+    )
+    this.name = 'MigrationError'
+  }
+}
+
+/** Create the ledger table when it is missing. */
+export async function ensureLedger(adapter: SQLAdapter): Promise<void> {
+  const q = (name: string) => adapter.quote(name)
+  // Written out rather than through `colDef`: these four type names mean the
+  // same thing on all three dialects, and `applied_at` is a BIGINT because
+  // MySQL's INT is 32 bits and runs out in 2038.
+  await adapter
+    .query(
+      `CREATE TABLE IF NOT EXISTS ${q(MIGRATIONS_TABLE)} (` +
+        `${q('name')} VARCHAR(255) NOT NULL, ` +
+        `${q('checksum')} VARCHAR(64) NOT NULL, ` +
+        `${q('applied_at')} BIGINT NOT NULL, ` +
+        `${q('duration_ms')} INTEGER NOT NULL, ` +
+        `PRIMARY KEY (${q('name')}))`,
+    )
+    .run()
+}
+
+export async function readApplied(
+  adapter: SQLAdapter,
+): Promise<AppliedMigration[]> {
+  const q = (name: string) => adapter.quote(name)
+  const rows = (await adapter
+    .query(
+      `SELECT ${q('name')}, ${q('checksum')}, ${q('applied_at')} FROM ${q(MIGRATIONS_TABLE)}`,
+    )
+    .all()) as Record<string, unknown>[]
+  return rows.map(row => ({
+    name: String(row.name),
+    checksum: String(row.checksum),
+    appliedAt: Number(row.applied_at ?? row.appliedAt),
+  }))
+}
+
+async function recordApplied(
+  adapter: SQLAdapter,
+  file: MigrationFile,
+  durationMs: number,
+): Promise<void> {
+  const q = (name: string) => adapter.quote(name)
+  await adapter
+    .query(
+      `INSERT INTO ${q(MIGRATIONS_TABLE)} ` +
+        `(${q('name')}, ${q('checksum')}, ${q('applied_at')}, ${q('duration_ms')}) VALUES (?, ?, ?, ?)`,
+    )
+    .run(file.name, file.checksum, Math.floor(Date.now() / 1000), durationMs)
+}
+
+/**
+ * Apply `pending` in order, each file in a transaction of its own with its
+ * ledger row inside it, where the dialect can roll DDL back. Stops at the first
+ * failure: the files after it may depend on it.
+ */
+export async function applyMigrations(
+  adapter: SQLAdapter,
+  pending: MigrationFile[],
+  onApplied?: (file: MigrationFile, durationMs: number) => void,
+): Promise<void> {
+  for (const file of pending) {
+    const inTransaction = adapter.transactionalDDL && file.transaction
+    const started = performance.now()
+    let durationMs = 0
+    try {
+      if (inTransaction) {
+        await adapter.transaction(async tx => {
+          await tx.executeScript(file.text)
+          durationMs = Math.round(performance.now() - started)
+          await recordApplied(tx, file, durationMs)
+        })
+      } else {
+        await adapter.executeScript(file.text)
+        durationMs = Math.round(performance.now() - started)
+        await recordApplied(adapter, file, durationMs)
+      }
+    } catch (cause) {
+      throw new MigrationError(file, inTransaction, { cause })
+    }
+    onApplied?.(file, durationMs)
+  }
+}

package/src/migrate/script.ts

@@ -0,0 +1,228 @@
+/**
+ * Just enough of a SQL lexer to read a migration file's statements without
+ * running them: where each one starts, what its first words are, and how many
+ * there are.
+ *
+ * It exists for two refusals. A migration runs inside the runner's own
+ * transaction, so a `COMMIT` in the file would commit part of it and leave the
+ * rest, and its ledger row, outside (measured on Postgres: the table created
+ * before a `COMMIT` survived the rollback of a later failure). And a file that
+ * opts out of the transaction must hold one statement, because a failure
+ * halfway through anything longer cannot be undone.
+ *
+ * The file itself goes to the driver as written. Splitting it here and sending
+ * the pieces would mean trusting this scanner with every construct a dialect
+ * has; reading it only to refuse means a construct it misreads costs a false
+ * refusal, never a corrupted migration.
+ */
+
+export type ScriptDialect = 'pgsql' | 'sqlite' | 'mysql'
+
+export interface ScriptStatement {
+  /** The statement's first one or two words, upper-cased: `COMMIT`, `START TRANSACTION`. */
+  head: string
+  /** 1-based line the statement starts on, for messages. */
+  line: number
+}
+
+/** Words that open a statement which ends or opens a transaction. */
+const TRANSACTION_CONTROL = new Set([
+  'BEGIN',
+  'COMMIT',
+  'END',
+  'ROLLBACK',
+  'ABORT',
+  'SAVEPOINT',
+  'RELEASE',
+  'START TRANSACTION',
+  'PREPARE TRANSACTION',
+])
+
+const WORD = /[A-Za-z_][A-Za-z0-9_$]*/y
+const DOLLAR_TAG = /\$([A-Za-z_][A-Za-z0-9_]*)?\$/y
+
+/**
+ * The statements in `text`, in order.
+ *
+ * Handles what a migration actually contains: `--` and `/* *\/` comments
+ * (nested on Postgres), single-quoted strings with doubled quotes and the
+ * `E'…'` backslash form, double-quoted and backticked identifiers, Postgres
+ * dollar-quoted bodies (`$$ … $$`, `$fn$ … $fn$`), and the `BEGIN … END` body
+ * of a trigger, which holds semicolons that do not end the statement.
+ */
+export function scanStatements(
+  text: string,
+  dialect: ScriptDialect,
+): ScriptStatement[] {
+  const statements: ScriptStatement[] = []
+  let line = 1
+  let i = 0
+
+  // The statement being read: its first words, and how deep it is inside a
+  // BEGIN … END body (above 0 once a CREATE statement's BEGIN is seen).
+  let words: string[] = []
+  let startLine = 0
+  let bodyDepth = 0
+
+  const finish = () => {
+    if (words.length) {
+      statements.push({ head: headOf(words), line: startLine })
+    }
+    words = []
+    bodyDepth = 0
+  }
+
+  const advance = (to: number) => {
+    for (let k = i; k < to; k++) if (text.charCodeAt(k) === 10) line++
+    i = to
+  }
+
+  while (i < text.length) {
+    const ch = text[i]!
+
+    if (
+      (ch === '-' && text[i + 1] === '-') ||
+      (ch === '#' && dialect === 'mysql')
+    ) {
+      const end = text.indexOf('\n', i)
+      advance(end === -1 ? text.length : end)
+      continue
+    }
+
+    if (ch === '/' && text[i + 1] === '*') {
+      advance(blockCommentEnd(text, i, dialect === 'pgsql'))
+      continue
+    }
+
+    if (/\s/.test(ch)) {
+      advance(i + 1)
+      continue
+    }
+
+    if (ch === ';') {
+      advance(i + 1)
+      if (bodyDepth === 0) finish()
+      continue
+    }
+
+    if (!words.length) startLine = line
+
+    if (ch === "'" || ((ch === 'E' || ch === 'e') && text[i + 1] === "'")) {
+      const backslashes = ch !== "'" || dialect === 'mysql'
+      const open = ch === "'" ? i : i + 1
+      advance(quotedEnd(text, open, "'", backslashes))
+      words.push('')
+      continue
+    }
+
+    if (ch === '"' || (ch === '`' && dialect === 'mysql')) {
+      advance(quotedEnd(text, i, ch, false))
+      words.push('')
+      continue
+    }
+
+    if (ch === '$' && dialect === 'pgsql') {
+      DOLLAR_TAG.lastIndex = i
+      const tag = DOLLAR_TAG.exec(text)
+      if (tag) {
+        const close = text.indexOf(tag[0], i + tag[0].length)
+        advance(close === -1 ? text.length : close + tag[0].length)
+        words.push('')
+        continue
+      }
+    }
+
+    WORD.lastIndex = i
+    const word = WORD.exec(text)
+    if (word) {
+      const upper = word[0].toUpperCase()
+      advance(i + word[0].length)
+      if (words.length < 8) words.push(upper)
+
+      // A BEGIN inside a CREATE statement opens a body with statements of its
+      // own: a SQLite trigger's, a Postgres `BEGIN ATOMIC` function's. A CASE
+      // inside the body has an END of its own. A CREATE statement never holds
+      // a transaction's BEGIN, so a plain `BEGIN;` still reads as one.
+      if (words[0] === 'CREATE' && words.length > 1) {
+        if (upper === 'BEGIN' || (upper === 'CASE' && bodyDepth > 0)) {
+          bodyDepth++
+        } else if (upper === 'END' && bodyDepth > 0) {
+          bodyDepth--
+        }
+      }
+      continue
+    }
+
+    advance(i + 1)
+    if (words.length < 8) words.push(ch)
+  }
+
+  finish()
+  return statements
+}
+
+/** The statements in `text` that would end or open a transaction. */
+export function transactionControl(
+  text: string,
+  dialect: ScriptDialect,
+): ScriptStatement[] {
+  return scanStatements(text, dialect).filter(s =>
+    TRANSACTION_CONTROL.has(s.head),
+  )
+}
+
+function headOf(words: string[]): string {
+  const [first = '', second = ''] = words
+  if ((first === 'START' || first === 'PREPARE') && second === 'TRANSACTION') {
+    return `${first} TRANSACTION`
+  }
+  return first
+}
+
+/** Index just past the comment opening at `start`. */
+function blockCommentEnd(text: string, start: number, nests: boolean): number {
+  let depth = 0
+  let i = start
+  while (i < text.length) {
+    if (text[i] === '/' && text[i + 1] === '*') {
+      depth++
+      i += 2
+      if (!nests && depth > 1) depth = 1
+      continue
+    }
+    if (text[i] === '*' && text[i + 1] === '/') {
+      depth--
+      i += 2
+      if (depth === 0) return i
+      continue
+    }
+    i++
+  }
+  return text.length
+}
+
+/** Index just past the quoted run opening at `start`. */
+function quotedEnd(
+  text: string,
+  start: number,
+  quote: string,
+  backslashes: boolean,
+): number {
+  let i = start + 1
+  while (i < text.length) {
+    const ch = text[i]
+    if (backslashes && ch === '\\') {
+      i += 2
+      continue
+    }
+    if (ch === quote) {
+      if (text[i + 1] === quote) {
+        i += 2
+        continue
+      }
+      return i + 1
+    }
+    i++
+  }
+  return text.length
+}

package/src/orm/query.ts

@@ -4,6 +4,7 @@ import { getActiveDb, txStorage } from '../connection'
 import type { AppDBSchema as DBSchema } from '../schema-registry'
 import { ColumnRef as schemaColumnRef, OperatorRef as schemaOperatorRef, SQLFunctionRef as schemaSQLFunctionRef, SQL_FUNCTIONS, WindowRef as schemaWindowRef, col as schemaCol, evalOperands, isSafeIdentifier, nullComparison, qId, qRaw } from '../schema-util'
 import { Mutation } from './mutation'
+import { ArrayValue } from '../array-value'
 
 export namespace DB {
   export type MapOf<T> = Record<string, T>
@@ -370,6 +371,31 @@ export namespace DB {
     return new schemaOperatorRef('BETWEEN', [minVal, maxVal])
   }
 
+  /**
+   * A Postgres array value, for an insert, an update or a comparison:
+   * `Insert.into('roles').values({ permissions: DB.array(['grades.post']) })`.
+   * Bun cannot bind a JavaScript array to a Postgres array, so a plain one
+   * fails; see `array-value.ts`. Postgres only: the other adapters refuse it.
+   */
+  export function array<T>(values: readonly T[]): ArrayValue<T> {
+    return new ArrayValue(values)
+  }
+
+  /** `column @> values`: the array column holds every one of `values`. Postgres only. */
+  export function contains<T>(values: readonly T[]): schemaOperatorRef<T[]> {
+    return new schemaOperatorRef('@>', new ArrayValue(values) as any)
+  }
+
+  /** `column && values`: the array column holds at least one of `values`. Postgres only. */
+  export function overlaps<T>(values: readonly T[]): schemaOperatorRef<T[]> {
+    return new schemaOperatorRef('&&', new ArrayValue(values) as any)
+  }
+
+  /** `column @> ARRAY[value]`: the array column holds `value`. Postgres only. */
+  export function has<T>(value: T): schemaOperatorRef<T[]> {
+    return new schemaOperatorRef('@>', new ArrayValue([value]) as any)
+  }
+
   export type SelectValue<S extends TableSchemas, J extends string> =
     | ColumnString<S, J>
     | SQLFunctionRef<ColumnString<S, J> | '*'>
@@ -1213,6 +1239,10 @@ export namespace DB {
     static isNull = isNull
     static isNotNull = isNotNull
     static between = between
+    static array = array
+    static contains = contains
+    static overlaps = overlaps
+    static has = has
 
     /**
      * Both columns, the joined table and the alias go through the convention-8

package/src/schema-util.ts

@@ -1,6 +1,7 @@
 import { Case } from '@bakery-framework/core/utils'
 import { is, throws } from '@bakery-framework/core/utils/common'
 import { quoteIdentifier } from './adapters/base'
+import { isArrayValue } from './array-value'
 import { getActiveDb } from './connection'
 import type * as SyncTypes from './sync/types'
 
@@ -320,6 +321,13 @@ export function evalOperands(
 
   if (where === null) return 'NULL'
 
+  // A whole Postgres array, bound as one parameter; the adapter encodes it.
+  // Before the object branch below, which would read it as `{table: column}`.
+  if (isArrayValue(where)) {
+    params.push(where)
+    return '?'
+  }
+
   if (typeof where === 'object' && where !== null) {
     if (where instanceof WindowRef) {
       // The function half only. `spec` was built by the query builder out of