@pikku/migrator-sql 0.12.2 → 0.12.4

package/src/schema-sql.test.ts

@@ -0,0 +1,92 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+
+import {
+  splitStatements,
+  tableCreationSql,
+  tablesInSourceOrder,
+} from './schema-sql.js'
+
+test('a semicolon inside a string literal is not a statement boundary', () => {
+  const sql = `CREATE TABLE a (id TEXT, note TEXT DEFAULT 'one; two');
+CREATE TABLE b (id TEXT);`
+
+  assert.deepEqual(splitStatements(sql), [
+    `CREATE TABLE a (id TEXT, note TEXT DEFAULT 'one; two');`,
+    'CREATE TABLE b (id TEXT);',
+  ])
+})
+
+test('a doubled quote inside a literal does not end it', () => {
+  const sql = `CREATE TABLE a (note TEXT DEFAULT 'it''s; fine');`
+  assert.deepEqual(splitStatements(sql), [sql])
+})
+
+test('semicolons in comments and dollar-quoted bodies are ignored', () => {
+  const sql = `-- a comment; with a semicolon
+CREATE TABLE a (id TEXT);
+/* another; one */
+CREATE FUNCTION f() RETURNS void AS $$ BEGIN; END; $$ LANGUAGE plpgsql;`
+
+  assert.deepEqual(splitStatements(sql), [
+    `-- a comment; with a semicolon
+CREATE TABLE a (id TEXT);`,
+    `/* another; one */
+CREATE FUNCTION f() RETURNS void AS $$ BEGIN; END; $$ LANGUAGE plpgsql;`,
+  ])
+})
+
+test('a trailing statement with no semicolon still counts', () => {
+  assert.deepEqual(splitStatements('CREATE TABLE a (id TEXT)'), [
+    'CREATE TABLE a (id TEXT)',
+  ])
+})
+
+test("a table's own SQL comes back with its indexes and constraints", () => {
+  const sql = `CREATE TABLE "user" ("id" text primary key);
+CREATE TABLE "two_factor" ("id" text primary key, "user_id" text not null references "user" ("id"));
+CREATE UNIQUE INDEX "two_factor_user_id_idx" ON "two_factor" ("user_id");
+CREATE INDEX "user_id_idx" ON "user" ("id");
+ALTER TABLE "two_factor" ADD CONSTRAINT "two_factor_secret_check" CHECK (length("id") > 0);`
+
+  assert.deepEqual(tableCreationSql(sql, 'two_factor'), [
+    `CREATE TABLE "two_factor" ("id" text primary key, "user_id" text not null references "user" ("id"));`,
+    `CREATE UNIQUE INDEX "two_factor_user_id_idx" ON "two_factor" ("user_id");`,
+    `ALTER TABLE "two_factor" ADD CONSTRAINT "two_factor_secret_check" CHECK (length("id") > 0);`,
+  ])
+})
+
+test('quoting, casing and a schema qualifier all name the same table', () => {
+  const sql = 'create table if not exists public."Two_Factor" (id text);'
+  assert.deepEqual(tableCreationSql(sql, '"two_factor"'), [sql])
+  assert.deepEqual(tableCreationSql(sql, 'app.two_factor'), [sql])
+})
+
+test('a table the SQL never creates yields nothing to copy', () => {
+  const sql = `CREATE TABLE "user" ("id" text primary key);
+CREATE INDEX "orphan_idx" ON "orders" ("id");`
+
+  assert.deepEqual(tableCreationSql(sql, 'orders'), [])
+})
+
+test('a referencing table is created after the one it references, not alphabetically', () => {
+  const sql = `create table "channels" ("channel_id" text primary key);
+create table "channel_subscriptions" ("channel_id" text not null references "channels" ("channel_id"));`
+
+  assert.deepEqual(
+    tablesInSourceOrder(sql, ['channel_subscriptions', 'channels']),
+    ['channels', 'channel_subscriptions']
+  )
+})
+
+test('a table the source does not create is left at the end', () => {
+  const sql = 'create table "a" ("id" text);'
+  assert.deepEqual(tablesInSourceOrder(sql, ['unknown', 'a']), ['a', 'unknown'])
+})
+
+test('source order survives a schema qualifier on either side', () => {
+  const sql = `create table app."b" ("id" text);
+create table "a" ("id" text);`
+
+  assert.deepEqual(tablesInSourceOrder(sql, ['a', 'app.b']), ['app.b', 'a'])
+})

package/src/schema-sql.ts

@@ -0,0 +1,179 @@
+/**
+ * Reading a schema source's own SQL back out, one table at a time.
+ *
+ * A source hands `db generate` both what must exist (`tables`) and what creates
+ * it (`sql`). The column map is a comparison surface — it answers "is this
+ * table there, and does it have these columns" — and it is deliberately lossy:
+ * primary keys, foreign keys, uniqueness, check constraints and indexes are all
+ * absent from it. So the moment the generator has to *create* something, the
+ * only honest source is the SQL, and this is what pulls the relevant part of it
+ * out.
+ */
+
+/**
+ * Split a SQL script into its top-level statements, semicolons included.
+ *
+ * A naive `split(';')` is wrong in a way that only shows up later: a semicolon
+ * inside a string literal, a quoted identifier or a comment is not a statement
+ * boundary, and cutting there yields two fragments that are each valid-looking
+ * and neither of which does what the original did.
+ */
+export function splitStatements(sql: string): string[] {
+  const statements: string[] = []
+  let start = 0
+  let i = 0
+
+  const closeQuote = (quote: string) => {
+    i++
+    while (i < sql.length) {
+      if (sql[i] === '\\' && quote === "'") {
+        i += 2
+        continue
+      }
+      if (sql[i] === quote) {
+        // A doubled quote is an escaped one, not the end of the literal.
+        if (sql[i + 1] === quote) {
+          i += 2
+          continue
+        }
+        i++
+        return
+      }
+      i++
+    }
+  }
+
+  while (i < sql.length) {
+    const ch = sql[i]!
+
+    if (ch === "'" || ch === '"' || ch === '`') {
+      closeQuote(ch)
+      continue
+    }
+
+    if (ch === '-' && sql[i + 1] === '-') {
+      const end = sql.indexOf('\n', i)
+      i = end === -1 ? sql.length : end + 1
+      continue
+    }
+
+    if (ch === '/' && sql[i + 1] === '*') {
+      const end = sql.indexOf('*/', i + 2)
+      i = end === -1 ? sql.length : end + 2
+      continue
+    }
+
+    // Postgres dollar quoting: everything between `$tag$` and its twin is a
+    // literal, and a function body written that way is full of semicolons.
+    if (ch === '$') {
+      const tag = /^\$[A-Za-z_0-9]*\$/.exec(sql.slice(i))
+      if (tag) {
+        const end = sql.indexOf(tag[0], i + tag[0].length)
+        i = end === -1 ? sql.length : end + tag[0].length
+        continue
+      }
+    }
+
+    if (ch === ';') {
+      const statement = sql.slice(start, i + 1).trim()
+      if (statement.length > 1) statements.push(statement)
+      start = i + 1
+    }
+
+    i++
+  }
+
+  // A script whose last statement has no trailing semicolon still ran it.
+  const tail = sql.slice(start).trim()
+  if (tail.length > 0) statements.push(tail)
+
+  return statements
+}
+
+/**
+ * Reduce a written table name to the form two sources can be compared on.
+ *
+ * The same table is `two_factor` to one writer, `"two_factor"` to Kysely and
+ * `public.two_factor` to Postgres introspection. Dropping the schema qualifier,
+ * the quoting and the case is the only shape all three agree on.
+ */
+export function bareTableName(name: string): string {
+  const last = name.split('.').pop() ?? name
+  return last.replace(/^["'`[]|["'`\]]$/g, '').toLowerCase()
+}
+
+const CREATE_TABLE =
+  /^CREATE\s+(?:TEMP(?:ORARY)?\s+|UNLOGGED\s+)*TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([^\s(]+)/i
+
+const CREATE_INDEX =
+  /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?\S+\s+ON\s+(?:ONLY\s+)?([^\s(]+)/i
+
+const ALTER_TABLE = /^ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?([^\s(]+)/i
+
+/**
+ * Every statement in `sql` that builds `table`, in the order the source wrote
+ * them.
+ *
+ * The `CREATE TABLE` and its indexes, plus any `ALTER TABLE` the source uses to
+ * hang a constraint on it afterwards — which together are what the column list
+ * cannot express. Source order is preserved because it is load-bearing: an
+ * index cannot precede its table, and a table cannot precede one it references.
+ *
+ * An empty result means the source's SQL does not visibly create the table.
+ * That is a real answer rather than a failure — a source may create a table
+ * from something other than a literal `CREATE TABLE` — and the caller is
+ * expected to fall back rather than emit nothing.
+ */
+export function tableCreationSql(sql: string, table: string): string[] {
+  const wanted = bareTableName(table)
+  const statements: string[] = []
+  let creates = false
+
+  for (const statement of splitStatements(sql)) {
+    const create = CREATE_TABLE.exec(statement)
+    if (create && bareTableName(create[1]!) === wanted) {
+      creates = true
+      statements.push(statement)
+      continue
+    }
+
+    const index = CREATE_INDEX.exec(statement)
+    if (index && bareTableName(index[1]!) === wanted) {
+      statements.push(statement)
+      continue
+    }
+
+    const alter = ALTER_TABLE.exec(statement)
+    if (alter && bareTableName(alter[1]!) === wanted) {
+      statements.push(statement)
+    }
+  }
+
+  // Indexes and alters without the table they belong to would fail on their own,
+  // and their presence says the table came from somewhere this cannot read.
+  return creates ? statements : []
+}
+
+/**
+ * `tables`, in the order `sql` creates them.
+ *
+ * A diff hands its missing tables back in whatever order it walked them, which
+ * is usually alphabetical and is never the order they can be created in: a
+ * table that references another has to come after it, and `channel_subscriptions`
+ * sorts before `channels`. The source's own SQL already has a workable order —
+ * it was applied in it — so that is the order used, with anything the source
+ * does not visibly create left at the end for the caller to render its own way.
+ */
+export function tablesInSourceOrder(sql: string, tables: string[]): string[] {
+  const position = new Map<string, number>()
+  let index = 0
+  for (const statement of splitStatements(sql)) {
+    const create = CREATE_TABLE.exec(statement)
+    if (create) {
+      const name = bareTableName(create[1]!)
+      if (!position.has(name)) position.set(name, index++)
+    }
+  }
+  const rank = (table: string) => position.get(bareTableName(table)) ?? Number.MAX_SAFE_INTEGER
+  return [...tables].sort((a, b) => rank(a) - rank(b))
+}

package/src/sql-migrator.ts

@@ -0,0 +1,219 @@
+import { createHash } from 'node:crypto'
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { assertSnakeCaseIdentifiers } from './migration-identifiers.js'
+
+/**
+ * The migrator's own bookkeeping table, which belongs to no dialect and to no
+ * project's schema.
+ *
+ * Every introspector hides it for that reason. Leaving it visible on one
+ * dialect and not the other is not cosmetic: a schema source exported from a
+ * database that has been migrated would publish `sql_migrations` as one of its
+ * own tables, and the consumer — which also has it — would then read the source
+ * as partially covered and emit column deltas instead of the source's own SQL,
+ * silently dropping its primary keys, indexes and constraints.
+ */
+export const MIGRATION_TRACKING_TABLE = 'sql_migrations'
+
+export class MigrationDriftError extends Error {
+  constructor(
+    public readonly file: string,
+    public readonly recordedHash: string,
+    public readonly currentHash: string | null,
+    public readonly appliedAt: string,
+    migrationsDir: string
+  ) {
+    const onDisk =
+      currentHash === null
+        ? 'file missing on disk'
+        : `sha256:${currentHash.slice(0, 8)}…`
+    super(
+      `[PKU-DB-DRIFT] ${migrationsDir}/${file}\n\n` +
+        `Migration content has changed since it was applied.\n` +
+        `  recorded:  sha256:${recordedHash.slice(0, 8)}…  applied ${appliedAt}\n` +
+        `  on disk:   ${onDisk}\n\n` +
+        `If this edit was intentional, write a new forward migration to revert the change.\n` +
+        `Production migrations are immutable.`
+    )
+    this.name = 'MigrationDriftError'
+  }
+}
+
+export interface MigrateResult {
+  applied: string[]
+  skipped: string[]
+}
+
+export interface AppliedMigration {
+  name: string
+  hash: string
+  applied_at: string
+}
+
+/**
+ * Provider-agnostic migration executor. Implement this for each DB dialect.
+ * Each method maps to a single DB operation; all file I/O and hashing lives
+ * in the shared `migrate()` function above.
+ */
+export interface MigrationExecutor {
+  ensureTrackingTable(): Promise<void>
+  getApplied(): Promise<AppliedMigration[]>
+  runMigration(sql: string, name: string, hash: string): Promise<void>
+  /**
+   * Record a migration as applied without running its SQL.
+   *
+   * For a database that already contains what the migration describes, because
+   * something created those tables before anyone wrote them down. Only ever
+   * called once the caller has confirmed that is actually true — recording a
+   * migration whose tables are absent leaves a database permanently behind with
+   * no pending migration to reveal it.
+   */
+  recordMigration(name: string, hash: string): Promise<void>
+}
+
+function sha256(bytes: Buffer): string {
+  return createHash('sha256').update(bytes).digest('hex')
+}
+
+/**
+ * Apply pending migrations from `migrationsDir/*.sql` using the supplied
+ * executor. Hashes raw file bytes on apply; subsequent runs re-hash and bail
+ * with `MigrationDriftError` if any applied file has changed on disk.
+ */
+/**
+ * The migrations on disk, or none.
+ *
+ * A project that has never generated a migration has no directory to read, and
+ * that is the ordinary first-run state rather than a failure — it is precisely
+ * the project `db generate` exists to serve.
+ */
+const migrationFiles = (migrationsDir: string): string[] =>
+  existsSync(migrationsDir)
+    ? readdirSync(migrationsDir)
+        .filter((f) => f.endsWith('.sql'))
+        .sort()
+    : []
+
+/**
+ * Re-hash every applied migration and bail if one has changed on disk.
+ *
+ * Applies to baselining as much as to migrating: recording a file as applied
+ * only means anything if the file is still the one that was applied.
+ */
+function assertNoDrift(
+  applied: AppliedMigration[],
+  migrationsDir: string
+): void {
+  for (const row of applied) {
+    let currentHash: string | null = null
+    try {
+      currentHash = sha256(readFileSync(join(migrationsDir, row.name)))
+    } catch {
+      currentHash = null
+    }
+    if (currentHash !== row.hash) {
+      throw new MigrationDriftError(
+        row.name,
+        row.hash,
+        currentHash,
+        row.applied_at,
+        migrationsDir
+      )
+    }
+  }
+}
+
+/**
+ * Every migration on disk, read once.
+ *
+ * The identifier check reads all of them and not just the pending ones, so that
+ * whether a migration is rejected depends on the files alone. A camelCase
+ * column that had already been applied somewhere would otherwise pass on that
+ * machine and fail on a fresh checkout, which is the opposite of deterministic.
+ */
+const readMigrations = (
+  migrationsDir: string
+): Array<{ name: string; sql: string }> =>
+  migrationFiles(migrationsDir).map((name) => ({
+    name,
+    sql: readFileSync(join(migrationsDir, name), 'utf8'),
+  }))
+
+export async function migrate(
+  executor: MigrationExecutor,
+  migrationsDir: string
+): Promise<MigrateResult> {
+  assertSnakeCaseIdentifiers(readMigrations(migrationsDir))
+  await executor.ensureTrackingTable()
+  const applied = await executor.getApplied()
+  assertNoDrift(applied, migrationsDir)
+  const appliedNames = new Set(applied.map((r) => r.name))
+
+  const result: MigrateResult = { applied: [], skipped: [] }
+
+  for (const name of migrationFiles(migrationsDir)) {
+    if (appliedNames.has(name)) {
+      result.skipped.push(name)
+      continue
+    }
+    const raw = readFileSync(join(migrationsDir, name))
+    const hash = sha256(raw)
+    await executor.runMigration(raw.toString('utf8'), name, hash)
+    result.applied.push(name)
+  }
+
+  return result
+}
+
+/**
+ * Record every pending migration as applied, without running any of it.
+ *
+ * The escape hatch for a database that already has the tables a migration
+ * creates — the shape you get when a runtime bootstrapped its own schema at
+ * boot and the migration writing it down was authored afterwards. Running that
+ * migration would fail on every existing deployment; skipping it forever would
+ * leave the history lying. Recording it says what is true.
+ *
+ * Deliberately unconditional here. Whether the database really does match is a
+ * question about schemas, not migration files, so the caller answers it first
+ * and this only runs once it has.
+ */
+/**
+ * The migrations on disk that the database has not recorded.
+ *
+ * Deliberately not derived by the caller: "pending" has to mean the same set
+ * `migrate` is about to apply, and that is filename order over `*.sql` minus
+ * what is recorded — not whatever a directory listing happens to return.
+ */
+export function pendingMigrations(
+  migrationsDir: string,
+  applied: AppliedMigration[]
+): string[] {
+  const appliedNames = new Set(applied.map((row) => row.name))
+  return migrationFiles(migrationsDir).filter(
+    (name) => !appliedNames.has(name)
+  )
+}
+
+export async function baselineMigrations(
+  executor: MigrationExecutor,
+  migrationsDir: string
+): Promise<string[]> {
+  await executor.ensureTrackingTable()
+  const applied = await executor.getApplied()
+  assertNoDrift(applied, migrationsDir)
+
+  const appliedNames = new Set(applied.map((r) => r.name))
+  const recorded: string[] = []
+  for (const name of migrationFiles(migrationsDir)) {
+    if (appliedNames.has(name)) continue
+    await executor.recordMigration(
+      name,
+      sha256(readFileSync(join(migrationsDir, name)))
+    )
+    recorded.push(name)
+  }
+  return recorded
+}

package/src/sqlite/index.ts

@@ -0,0 +1,8 @@
+export { SqliteMigrationExecutor, dropTrackingTable } from './sqlite-migrator.js'
+export { loadSqliteRuntime } from './sqlite-runtime.js'
+export type {
+  SqliteRuntime,
+  SyncSqliteChanges,
+  SyncSqliteDatabase,
+  SyncSqliteStatement,
+} from './sqlite-runtime.js'

package/src/sqlite/sqlite-migrator.ts

@@ -0,0 +1,49 @@
+import type { MigrationExecutor, AppliedMigration } from '../sql-migrator.js'
+import { MIGRATION_TRACKING_TABLE as TRACKING_TABLE } from '../sql-migrator.js'
+import type { SyncSqliteDatabase } from './sqlite-runtime.js'
+
+export class SqliteMigrationExecutor implements MigrationExecutor {
+  constructor(private readonly db: SyncSqliteDatabase) {}
+
+  async ensureTrackingTable(): Promise<void> {
+    this.db.exec(
+      `CREATE TABLE IF NOT EXISTS ${TRACKING_TABLE} (
+         name       TEXT PRIMARY KEY,
+         hash       TEXT NOT NULL,
+         applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+       )`
+    )
+  }
+
+  async getApplied(): Promise<AppliedMigration[]> {
+    return this.db
+      .prepare(
+        `SELECT name, hash, applied_at FROM ${TRACKING_TABLE} ORDER BY name`
+      )
+      .all() as unknown as AppliedMigration[]
+  }
+
+  async recordMigration(name: string, hash: string): Promise<void> {
+    this.db
+      .prepare(`INSERT INTO ${TRACKING_TABLE} (name, hash) VALUES (?, ?)`)
+      .run(name, hash)
+  }
+
+  async runMigration(sql: string, name: string, hash: string): Promise<void> {
+    this.db.exec('BEGIN')
+    try {
+      this.db.exec(sql)
+      this.db
+        .prepare(`INSERT INTO ${TRACKING_TABLE} (name, hash) VALUES (?, ?)`)
+        .run(name, hash)
+      this.db.exec('COMMIT')
+    } catch (err) {
+      this.db.exec('ROLLBACK')
+      throw err
+    }
+  }
+}
+
+export function dropTrackingTable(db: SyncSqliteDatabase): void {
+  db.exec(`DROP TABLE IF EXISTS ${TRACKING_TABLE}`)
+}

package/src/sqlite/sqlite-runtime-bun.ts

@@ -0,0 +1,83 @@
+import { Database, type SQLQueryBindings } from 'bun:sqlite'
+import type {
+  SqliteRuntime,
+  SyncSqliteChanges,
+  SyncSqliteDatabase,
+  SyncSqliteStatement,
+} from './sqlite-runtime.js'
+
+class BunSqliteStatement implements SyncSqliteStatement {
+  readonly reader: boolean
+
+  constructor(
+    private readonly stmt: ReturnType<Database['prepare']>,
+    reader: boolean
+  ) {
+    this.reader = reader
+  }
+
+  all(...parameters: unknown[]): unknown[] {
+    return this.stmt.all(...(parameters as SQLQueryBindings[])) as unknown[]
+  }
+
+  get(...parameters: unknown[]): unknown | null {
+    return (
+      (this.stmt.get(...(parameters as SQLQueryBindings[])) as unknown) ?? null
+    )
+  }
+
+  iterate(...parameters: unknown[]): IterableIterator<unknown> {
+    return this.stmt.iterate(
+      ...(parameters as SQLQueryBindings[])
+    ) as IterableIterator<unknown>
+  }
+
+  run(...parameters: unknown[]): SyncSqliteChanges {
+    const result = this.stmt.run(...(parameters as SQLQueryBindings[]))
+    return {
+      changes: result.changes,
+      lastInsertRowid: result.lastInsertRowid,
+    }
+  }
+}
+
+class BunSqliteDatabase implements SyncSqliteDatabase {
+  constructor(private readonly db: Database) {}
+
+  exec(sql: string): void {
+    // bun:sqlite throws "no valid SQL statement" on comment-only/empty input
+    // (e.g. a placeholder dev-seed.sql); node:sqlite silently no-ops. Match node's
+    // tolerance by skipping when nothing executable remains after stripping
+    // comments. The original `sql` is still exec'd verbatim when non-empty.
+    const executable = sql
+      .replace(/--[^\n]*/g, '')
+      .replace(/\/\*[\s\S]*?\*\//g, '')
+      .trim()
+    if (executable.length === 0) return
+    this.db.exec(sql)
+  }
+
+  prepare(sql: string): SyncSqliteStatement {
+    return new BunSqliteStatement(this.db.prepare(sql), isReaderSql(sql))
+  }
+
+  close(): void {
+    this.db.close()
+  }
+}
+
+function isReaderSql(sql: string): boolean {
+  const normalized = sql.trimStart().toUpperCase()
+  return (
+    normalized.startsWith('SELECT') ||
+    normalized.startsWith('WITH') ||
+    normalized.startsWith('PRAGMA') ||
+    normalized.startsWith('EXPLAIN')
+  )
+}
+
+export const bunSqliteRuntime: SqliteRuntime = {
+  open(filename) {
+    return new BunSqliteDatabase(new Database(filename))
+  },
+}