@pikku/migrator-sql 0.12.2 → 0.12.3

package/src/schema-sql.ts

@@ -0,0 +1,155 @@
+/**
+ * Reading a schema source's own SQL back out, one table at a time.
+ *
+ * A source hands `db generate` both what must exist (`tables`) and what creates
+ * it (`sql`). The column map is a comparison surface — it answers "is this
+ * table there, and does it have these columns" — and it is deliberately lossy:
+ * primary keys, foreign keys, uniqueness, check constraints and indexes are all
+ * absent from it. So the moment the generator has to *create* something, the
+ * only honest source is the SQL, and this is what pulls the relevant part of it
+ * out.
+ */
+
+/**
+ * Split a SQL script into its top-level statements, semicolons included.
+ *
+ * A naive `split(';')` is wrong in a way that only shows up later: a semicolon
+ * inside a string literal, a quoted identifier or a comment is not a statement
+ * boundary, and cutting there yields two fragments that are each valid-looking
+ * and neither of which does what the original did.
+ */
+export function splitStatements(sql: string): string[] {
+  const statements: string[] = []
+  let start = 0
+  let i = 0
+
+  const closeQuote = (quote: string) => {
+    i++
+    while (i < sql.length) {
+      if (sql[i] === '\\' && quote === "'") {
+        i += 2
+        continue
+      }
+      if (sql[i] === quote) {
+        // A doubled quote is an escaped one, not the end of the literal.
+        if (sql[i + 1] === quote) {
+          i += 2
+          continue
+        }
+        i++
+        return
+      }
+      i++
+    }
+  }
+
+  while (i < sql.length) {
+    const ch = sql[i]!
+
+    if (ch === "'" || ch === '"' || ch === '`') {
+      closeQuote(ch)
+      continue
+    }
+
+    if (ch === '-' && sql[i + 1] === '-') {
+      const end = sql.indexOf('\n', i)
+      i = end === -1 ? sql.length : end + 1
+      continue
+    }
+
+    if (ch === '/' && sql[i + 1] === '*') {
+      const end = sql.indexOf('*/', i + 2)
+      i = end === -1 ? sql.length : end + 2
+      continue
+    }
+
+    // Postgres dollar quoting: everything between `$tag$` and its twin is a
+    // literal, and a function body written that way is full of semicolons.
+    if (ch === '$') {
+      const tag = /^\$[A-Za-z_0-9]*\$/.exec(sql.slice(i))
+      if (tag) {
+        const end = sql.indexOf(tag[0], i + tag[0].length)
+        i = end === -1 ? sql.length : end + tag[0].length
+        continue
+      }
+    }
+
+    if (ch === ';') {
+      const statement = sql.slice(start, i + 1).trim()
+      if (statement.length > 1) statements.push(statement)
+      start = i + 1
+    }
+
+    i++
+  }
+
+  // A script whose last statement has no trailing semicolon still ran it.
+  const tail = sql.slice(start).trim()
+  if (tail.length > 0) statements.push(tail)
+
+  return statements
+}
+
+/**
+ * Reduce a written table name to the form two sources can be compared on.
+ *
+ * The same table is `two_factor` to one writer, `"two_factor"` to Kysely and
+ * `public.two_factor` to Postgres introspection. Dropping the schema qualifier,
+ * the quoting and the case is the only shape all three agree on.
+ */
+export function bareTableName(name: string): string {
+  const last = name.split('.').pop() ?? name
+  return last.replace(/^["'`[]|["'`\]]$/g, '').toLowerCase()
+}
+
+const CREATE_TABLE =
+  /^CREATE\s+(?:TEMP(?:ORARY)?\s+|UNLOGGED\s+)*TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([^\s(]+)/i
+
+const CREATE_INDEX =
+  /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?\S+\s+ON\s+(?:ONLY\s+)?([^\s(]+)/i
+
+const ALTER_TABLE = /^ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?([^\s(]+)/i
+
+/**
+ * Every statement in `sql` that builds `table`, in the order the source wrote
+ * them.
+ *
+ * The `CREATE TABLE` and its indexes, plus any `ALTER TABLE` the source uses to
+ * hang a constraint on it afterwards — which together are what the column list
+ * cannot express. Source order is preserved because it is load-bearing: an
+ * index cannot precede its table, and a table cannot precede one it references.
+ *
+ * An empty result means the source's SQL does not visibly create the table.
+ * That is a real answer rather than a failure — a source may create a table
+ * from something other than a literal `CREATE TABLE` — and the caller is
+ * expected to fall back rather than emit nothing.
+ */
+export function tableCreationSql(sql: string, table: string): string[] {
+  const wanted = bareTableName(table)
+  const statements: string[] = []
+  let creates = false
+
+  for (const statement of splitStatements(sql)) {
+    const create = CREATE_TABLE.exec(statement)
+    if (create && bareTableName(create[1]!) === wanted) {
+      creates = true
+      statements.push(statement)
+      continue
+    }
+
+    const index = CREATE_INDEX.exec(statement)
+    if (index && bareTableName(index[1]!) === wanted) {
+      statements.push(statement)
+      continue
+    }
+
+    const alter = ALTER_TABLE.exec(statement)
+    if (alter && bareTableName(alter[1]!) === wanted) {
+      statements.push(statement)
+    }
+  }
+
+  // Indexes and alters without the table they belong to would fail on their own,
+  // and their presence says the table came from somewhere this cannot read.
+  return creates ? statements : []
+}

package/src/sql-migrator.ts

@@ -0,0 +1,219 @@
+import { createHash } from 'node:crypto'
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { assertSnakeCaseIdentifiers } from './migration-identifiers.js'
+
+/**
+ * The migrator's own bookkeeping table, which belongs to no dialect and to no
+ * project's schema.
+ *
+ * Every introspector hides it for that reason. Leaving it visible on one
+ * dialect and not the other is not cosmetic: a schema source exported from a
+ * database that has been migrated would publish `sql_migrations` as one of its
+ * own tables, and the consumer — which also has it — would then read the source
+ * as partially covered and emit column deltas instead of the source's own SQL,
+ * silently dropping its primary keys, indexes and constraints.
+ */
+export const MIGRATION_TRACKING_TABLE = 'sql_migrations'
+
+export class MigrationDriftError extends Error {
+  constructor(
+    public readonly file: string,
+    public readonly recordedHash: string,
+    public readonly currentHash: string | null,
+    public readonly appliedAt: string,
+    migrationsDir: string
+  ) {
+    const onDisk =
+      currentHash === null
+        ? 'file missing on disk'
+        : `sha256:${currentHash.slice(0, 8)}…`
+    super(
+      `[PKU-DB-DRIFT] ${migrationsDir}/${file}\n\n` +
+        `Migration content has changed since it was applied.\n` +
+        `  recorded:  sha256:${recordedHash.slice(0, 8)}…  applied ${appliedAt}\n` +
+        `  on disk:   ${onDisk}\n\n` +
+        `If this edit was intentional, write a new forward migration to revert the change.\n` +
+        `Production migrations are immutable.`
+    )
+    this.name = 'MigrationDriftError'
+  }
+}
+
+export interface MigrateResult {
+  applied: string[]
+  skipped: string[]
+}
+
+export interface AppliedMigration {
+  name: string
+  hash: string
+  applied_at: string
+}
+
+/**
+ * Provider-agnostic migration executor. Implement this for each DB dialect.
+ * Each method maps to a single DB operation; all file I/O and hashing lives
+ * in the shared `migrate()` function above.
+ */
+export interface MigrationExecutor {
+  ensureTrackingTable(): Promise<void>
+  getApplied(): Promise<AppliedMigration[]>
+  runMigration(sql: string, name: string, hash: string): Promise<void>
+  /**
+   * Record a migration as applied without running its SQL.
+   *
+   * For a database that already contains what the migration describes, because
+   * something created those tables before anyone wrote them down. Only ever
+   * called once the caller has confirmed that is actually true — recording a
+   * migration whose tables are absent leaves a database permanently behind with
+   * no pending migration to reveal it.
+   */
+  recordMigration(name: string, hash: string): Promise<void>
+}
+
+function sha256(bytes: Buffer): string {
+  return createHash('sha256').update(bytes).digest('hex')
+}
+
+/**
+ * Apply pending migrations from `migrationsDir/*.sql` using the supplied
+ * executor. Hashes raw file bytes on apply; subsequent runs re-hash and bail
+ * with `MigrationDriftError` if any applied file has changed on disk.
+ */
+/**
+ * The migrations on disk, or none.
+ *
+ * A project that has never generated a migration has no directory to read, and
+ * that is the ordinary first-run state rather than a failure — it is precisely
+ * the project `db generate` exists to serve.
+ */
+const migrationFiles = (migrationsDir: string): string[] =>
+  existsSync(migrationsDir)
+    ? readdirSync(migrationsDir)
+        .filter((f) => f.endsWith('.sql'))
+        .sort()
+    : []
+
+/**
+ * Re-hash every applied migration and bail if one has changed on disk.
+ *
+ * Applies to baselining as much as to migrating: recording a file as applied
+ * only means anything if the file is still the one that was applied.
+ */
+function assertNoDrift(
+  applied: AppliedMigration[],
+  migrationsDir: string
+): void {
+  for (const row of applied) {
+    let currentHash: string | null = null
+    try {
+      currentHash = sha256(readFileSync(join(migrationsDir, row.name)))
+    } catch {
+      currentHash = null
+    }
+    if (currentHash !== row.hash) {
+      throw new MigrationDriftError(
+        row.name,
+        row.hash,
+        currentHash,
+        row.applied_at,
+        migrationsDir
+      )
+    }
+  }
+}
+
+/**
+ * Every migration on disk, read once.
+ *
+ * The identifier check reads all of them and not just the pending ones, so that
+ * whether a migration is rejected depends on the files alone. A camelCase
+ * column that had already been applied somewhere would otherwise pass on that
+ * machine and fail on a fresh checkout, which is the opposite of deterministic.
+ */
+const readMigrations = (
+  migrationsDir: string
+): Array<{ name: string; sql: string }> =>
+  migrationFiles(migrationsDir).map((name) => ({
+    name,
+    sql: readFileSync(join(migrationsDir, name), 'utf8'),
+  }))
+
+export async function migrate(
+  executor: MigrationExecutor,
+  migrationsDir: string
+): Promise<MigrateResult> {
+  assertSnakeCaseIdentifiers(readMigrations(migrationsDir))
+  await executor.ensureTrackingTable()
+  const applied = await executor.getApplied()
+  assertNoDrift(applied, migrationsDir)
+  const appliedNames = new Set(applied.map((r) => r.name))
+
+  const result: MigrateResult = { applied: [], skipped: [] }
+
+  for (const name of migrationFiles(migrationsDir)) {
+    if (appliedNames.has(name)) {
+      result.skipped.push(name)
+      continue
+    }
+    const raw = readFileSync(join(migrationsDir, name))
+    const hash = sha256(raw)
+    await executor.runMigration(raw.toString('utf8'), name, hash)
+    result.applied.push(name)
+  }
+
+  return result
+}
+
+/**
+ * Record every pending migration as applied, without running any of it.
+ *
+ * The escape hatch for a database that already has the tables a migration
+ * creates — the shape you get when a runtime bootstrapped its own schema at
+ * boot and the migration writing it down was authored afterwards. Running that
+ * migration would fail on every existing deployment; skipping it forever would
+ * leave the history lying. Recording it says what is true.
+ *
+ * Deliberately unconditional here. Whether the database really does match is a
+ * question about schemas, not migration files, so the caller answers it first
+ * and this only runs once it has.
+ */
+/**
+ * The migrations on disk that the database has not recorded.
+ *
+ * Deliberately not derived by the caller: "pending" has to mean the same set
+ * `migrate` is about to apply, and that is filename order over `*.sql` minus
+ * what is recorded — not whatever a directory listing happens to return.
+ */
+export function pendingMigrations(
+  migrationsDir: string,
+  applied: AppliedMigration[]
+): string[] {
+  const appliedNames = new Set(applied.map((row) => row.name))
+  return migrationFiles(migrationsDir).filter(
+    (name) => !appliedNames.has(name)
+  )
+}
+
+export async function baselineMigrations(
+  executor: MigrationExecutor,
+  migrationsDir: string
+): Promise<string[]> {
+  await executor.ensureTrackingTable()
+  const applied = await executor.getApplied()
+  assertNoDrift(applied, migrationsDir)
+
+  const appliedNames = new Set(applied.map((r) => r.name))
+  const recorded: string[] = []
+  for (const name of migrationFiles(migrationsDir)) {
+    if (appliedNames.has(name)) continue
+    await executor.recordMigration(
+      name,
+      sha256(readFileSync(join(migrationsDir, name)))
+    )
+    recorded.push(name)
+  }
+  return recorded
+}

package/src/sqlite/index.ts

@@ -0,0 +1,8 @@
+export { SqliteMigrationExecutor, dropTrackingTable } from './sqlite-migrator.js'
+export { loadSqliteRuntime } from './sqlite-runtime.js'
+export type {
+  SqliteRuntime,
+  SyncSqliteChanges,
+  SyncSqliteDatabase,
+  SyncSqliteStatement,
+} from './sqlite-runtime.js'

package/src/sqlite/sqlite-migrator.ts

@@ -0,0 +1,49 @@
+import type { MigrationExecutor, AppliedMigration } from '../sql-migrator.js'
+import { MIGRATION_TRACKING_TABLE as TRACKING_TABLE } from '../sql-migrator.js'
+import type { SyncSqliteDatabase } from './sqlite-runtime.js'
+
+export class SqliteMigrationExecutor implements MigrationExecutor {
+  constructor(private readonly db: SyncSqliteDatabase) {}
+
+  async ensureTrackingTable(): Promise<void> {
+    this.db.exec(
+      `CREATE TABLE IF NOT EXISTS ${TRACKING_TABLE} (
+         name       TEXT PRIMARY KEY,
+         hash       TEXT NOT NULL,
+         applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+       )`
+    )
+  }
+
+  async getApplied(): Promise<AppliedMigration[]> {
+    return this.db
+      .prepare(
+        `SELECT name, hash, applied_at FROM ${TRACKING_TABLE} ORDER BY name`
+      )
+      .all() as unknown as AppliedMigration[]
+  }
+
+  async recordMigration(name: string, hash: string): Promise<void> {
+    this.db
+      .prepare(`INSERT INTO ${TRACKING_TABLE} (name, hash) VALUES (?, ?)`)
+      .run(name, hash)
+  }
+
+  async runMigration(sql: string, name: string, hash: string): Promise<void> {
+    this.db.exec('BEGIN')
+    try {
+      this.db.exec(sql)
+      this.db
+        .prepare(`INSERT INTO ${TRACKING_TABLE} (name, hash) VALUES (?, ?)`)
+        .run(name, hash)
+      this.db.exec('COMMIT')
+    } catch (err) {
+      this.db.exec('ROLLBACK')
+      throw err
+    }
+  }
+}
+
+export function dropTrackingTable(db: SyncSqliteDatabase): void {
+  db.exec(`DROP TABLE IF EXISTS ${TRACKING_TABLE}`)
+}

package/src/sqlite/sqlite-runtime-bun.ts

@@ -0,0 +1,83 @@
+import { Database, type SQLQueryBindings } from 'bun:sqlite'
+import type {
+  SqliteRuntime,
+  SyncSqliteChanges,
+  SyncSqliteDatabase,
+  SyncSqliteStatement,
+} from './sqlite-runtime.js'
+
+class BunSqliteStatement implements SyncSqliteStatement {
+  readonly reader: boolean
+
+  constructor(
+    private readonly stmt: ReturnType<Database['prepare']>,
+    reader: boolean
+  ) {
+    this.reader = reader
+  }
+
+  all(...parameters: unknown[]): unknown[] {
+    return this.stmt.all(...(parameters as SQLQueryBindings[])) as unknown[]
+  }
+
+  get(...parameters: unknown[]): unknown | null {
+    return (
+      (this.stmt.get(...(parameters as SQLQueryBindings[])) as unknown) ?? null
+    )
+  }
+
+  iterate(...parameters: unknown[]): IterableIterator<unknown> {
+    return this.stmt.iterate(
+      ...(parameters as SQLQueryBindings[])
+    ) as IterableIterator<unknown>
+  }
+
+  run(...parameters: unknown[]): SyncSqliteChanges {
+    const result = this.stmt.run(...(parameters as SQLQueryBindings[]))
+    return {
+      changes: result.changes,
+      lastInsertRowid: result.lastInsertRowid,
+    }
+  }
+}
+
+class BunSqliteDatabase implements SyncSqliteDatabase {
+  constructor(private readonly db: Database) {}
+
+  exec(sql: string): void {
+    // bun:sqlite throws "no valid SQL statement" on comment-only/empty input
+    // (e.g. a placeholder dev-seed.sql); node:sqlite silently no-ops. Match node's
+    // tolerance by skipping when nothing executable remains after stripping
+    // comments. The original `sql` is still exec'd verbatim when non-empty.
+    const executable = sql
+      .replace(/--[^\n]*/g, '')
+      .replace(/\/\*[\s\S]*?\*\//g, '')
+      .trim()
+    if (executable.length === 0) return
+    this.db.exec(sql)
+  }
+
+  prepare(sql: string): SyncSqliteStatement {
+    return new BunSqliteStatement(this.db.prepare(sql), isReaderSql(sql))
+  }
+
+  close(): void {
+    this.db.close()
+  }
+}
+
+function isReaderSql(sql: string): boolean {
+  const normalized = sql.trimStart().toUpperCase()
+  return (
+    normalized.startsWith('SELECT') ||
+    normalized.startsWith('WITH') ||
+    normalized.startsWith('PRAGMA') ||
+    normalized.startsWith('EXPLAIN')
+  )
+}
+
+export const bunSqliteRuntime: SqliteRuntime = {
+  open(filename) {
+    return new BunSqliteDatabase(new Database(filename))
+  },
+}

package/src/sqlite/sqlite-runtime-node.ts

@@ -0,0 +1,118 @@
+import type {
+  SqliteRuntime,
+  SyncSqliteChanges,
+  SyncSqliteDatabase,
+  SyncSqliteStatement,
+} from './sqlite-runtime.js'
+
+interface NodeSqliteStatementShape {
+  reader?: boolean
+  all(...parameters: unknown[]): unknown[]
+  get(...parameters: unknown[]): unknown
+  iterate(...parameters: unknown[]): IterableIterator<unknown>
+  run(...parameters: unknown[]): {
+    changes: number | bigint
+    lastInsertRowid: number | bigint
+  }
+}
+
+interface NodeSqliteDatabaseShape {
+  exec(sql: string): void
+  prepare(sql: string): NodeSqliteStatementShape
+  close(): void
+}
+
+interface NodeSqliteModule {
+  DatabaseSync: new (filename?: string) => NodeSqliteDatabaseShape
+}
+
+class NodeSqliteStatement implements SyncSqliteStatement {
+  readonly reader: boolean
+
+  constructor(
+    private readonly stmt: NodeSqliteStatementShape,
+    sql: string
+  ) {
+    // node:sqlite StatementSync does not have a .reader property
+    // (that's a better-sqlite3 API). Fall back to SQL inspection when absent.
+    if (stmt.reader !== undefined) {
+      this.reader = Boolean(stmt.reader)
+    } else {
+      const upper = sql.trimStart().toUpperCase()
+      this.reader =
+        upper.startsWith('SELECT') ||
+        upper.startsWith('WITH') ||
+        upper.startsWith('PRAGMA') ||
+        upper.startsWith('EXPLAIN') ||
+        upper.startsWith('VALUES') ||
+        /\bRETURNING\b/.test(upper)
+    }
+  }
+
+  all(...parameters: unknown[]): unknown[] {
+    return this.stmt.all(...parameters) as unknown[]
+  }
+
+  get(...parameters: unknown[]): unknown | null {
+    return (this.stmt.get(...parameters) as unknown) ?? null
+  }
+
+  iterate(...parameters: unknown[]): IterableIterator<unknown> {
+    return this.stmt.iterate(...parameters) as IterableIterator<unknown>
+  }
+
+  run(...parameters: unknown[]): SyncSqliteChanges {
+    const result = this.stmt.run(...parameters)
+    return {
+      changes: result.changes,
+      lastInsertRowid: result.lastInsertRowid,
+    }
+  }
+}
+
+class NodeSqliteDatabase implements SyncSqliteDatabase {
+  constructor(private readonly db: NodeSqliteDatabaseShape) {}
+
+  exec(sql: string): void {
+    this.db.exec(sql)
+  }
+
+  prepare(sql: string): SyncSqliteStatement {
+    return new NodeSqliteStatement(this.db.prepare(sql), sql)
+  }
+
+  close(): void {
+    this.db.close()
+  }
+}
+
+async function importNodeSqlite(): Promise<NodeSqliteModule> {
+  const dynamicImport = new Function(
+    'return import("node:sqlite")'
+  ) as () => Promise<NodeSqliteModule>
+  try {
+    return await dynamicImport()
+  } catch (error: any) {
+    // node ships node:sqlite unflagged only from 24. The raw failure is
+    // ERR_UNKNOWN_BUILTIN_MODULE, which reads like a broken install rather than
+    // a runtime that is simply too old — so say which it is and how to get past it.
+    if (error?.code === 'ERR_UNKNOWN_BUILTIN_MODULE') {
+      throw new Error(
+        `This needs node:sqlite, which Node ${process.versions.node} does not provide ` +
+          `(it is unflagged from Node 24). Either upgrade Node, or run the CLI on bun, ` +
+          `which has it: \`bunx --bun pikku <command>\`.`,
+        { cause: error }
+      )
+    }
+    throw error
+  }
+}
+
+export async function createNodeSqliteRuntime(): Promise<SqliteRuntime> {
+  const { DatabaseSync } = await importNodeSqlite()
+  return {
+    open(filename) {
+      return new NodeSqliteDatabase(new DatabaseSync(filename))
+    },
+  }
+}

package/src/sqlite/sqlite-runtime.ts

@@ -0,0 +1,41 @@
+export interface SyncSqliteChanges {
+  changes: number | bigint
+  lastInsertRowid: number | bigint
+}
+
+export interface SyncSqliteStatement {
+  reader: boolean
+  all(...parameters: unknown[]): unknown[]
+  get(...parameters: unknown[]): unknown | null
+  iterate(...parameters: unknown[]): IterableIterator<unknown>
+  run(...parameters: unknown[]): SyncSqliteChanges
+}
+
+export interface SyncSqliteDatabase {
+  exec(sql: string): void
+  prepare(sql: string): SyncSqliteStatement
+  close(): void
+}
+
+export interface SqliteRuntime {
+  open(filename: string): SyncSqliteDatabase
+}
+
+let runtimePromise: Promise<SqliteRuntime> | undefined
+
+export async function loadSqliteRuntime(): Promise<SqliteRuntime> {
+  runtimePromise ??= (async () => {
+    const isBunRuntime =
+      typeof (globalThis as { Bun?: unknown }).Bun !== 'undefined'
+
+    if (isBunRuntime) {
+      const { bunSqliteRuntime } = await import('./sqlite-runtime-bun.js')
+      return bunSqliteRuntime
+    }
+
+    const { createNodeSqliteRuntime } = await import('./sqlite-runtime-node.js')
+    return createNodeSqliteRuntime()
+  })()
+
+  return runtimePromise
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "types": ["node", "bun"],
+    "module": "Node18",
+    "outDir": "dist",
+    "target": "esnext",
+    "declaration": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["**/*.test.ts", "node_modules"]
+}