@bakery-framework/orm 1.0.0

package/src/sync/history.ts

@@ -0,0 +1,169 @@
+import '@bakery-framework/core/core/init'
+
+import { Logger } from '@bakery-framework/core/logger'
+import { type LedgerEntry, readLedgerEntries } from './ledger'
+import type * as SyncTypes from './types'
+
+const logger = new Logger('db-history')
+
+/**
+ * What changed between two applied schemas, by name only.
+ *
+ * Names, not types — the same restraint `shapesMatch` uses and for the same
+ * reason: a type comparison here would need dialect normalisation, which is the
+ * thing the ledger exists to avoid depending on. A summary that says "column
+ * changed" when a Postgres default merely re-rendered itself would be worse
+ * than saying nothing.
+ */
+export interface HistoryDiff {
+  tablesAdded: string[]
+  tablesRemoved: string[]
+  columnsAdded: string[]
+  columnsRemoved: string[]
+}
+
+const meta = (k: string) => k.startsWith('_')
+const namesOf = (o: unknown) => Object.keys(o ?? {}).filter(k => !meta(k))
+
+export function diffEntries(
+  older: SyncTypes.DBConstraints | null,
+  newer: SyncTypes.DBConstraints,
+): HistoryDiff {
+  const out: HistoryDiff = {
+    tablesAdded: [],
+    tablesRemoved: [],
+    columnsAdded: [],
+    columnsRemoved: [],
+  }
+  const before = namesOf(older ?? {})
+  const after = namesOf(newer)
+  out.tablesAdded = after.filter(t => !before.includes(t))
+  out.tablesRemoved = before.filter(t => !after.includes(t))
+
+  // Only tables present in both: a column of a table that was added whole is
+  // already accounted for by `tablesAdded`, and listing it again would make a
+  // one-table migration read like dozens of changes.
+  for (const table of after.filter(t => before.includes(t))) {
+    const b = namesOf((older as any)?.[table])
+    const a = namesOf((newer as any)[table])
+    for (const c of a.filter(c => !b.includes(c)))
+      out.columnsAdded.push(`${table}.${c}`)
+    for (const c of b.filter(c => !a.includes(c)))
+      out.columnsRemoved.push(`${table}.${c}`)
+  }
+  return out
+}
+
+export function isEmptyDiff(d: HistoryDiff): boolean {
+  return (
+    !d.tablesAdded.length &&
+    !d.tablesRemoved.length &&
+    !d.columnsAdded.length &&
+    !d.columnsRemoved.length
+  )
+}
+
+/** `applied_at` is stored as whole seconds — see `writeLedger`. */
+export function formatWhen(appliedAt: number): string {
+  if (!Number.isFinite(appliedAt) || appliedAt <= 0) return 'unknown'
+  return new Date(appliedAt * 1000).toISOString().replace('T', ' ').slice(0, 19)
+}
+
+export function formatEntry(
+  entry: LedgerEntry,
+  previous: LedgerEntry | undefined,
+  isCurrent: boolean,
+): string[] {
+  const lines: string[] = []
+  const marker = isCurrent ? ' (current)' : ''
+  const tables = namesOf(entry.constraints).length
+  lines.push(
+    `  #${entry.id}  ${formatWhen(entry.appliedAt)} UTC  ${tables} table${
+      tables === 1 ? '' : 's'
+    }${marker}`,
+  )
+
+  const diff = diffEntries(previous?.constraints ?? null, entry.constraints)
+  if (!previous) {
+    lines.push('      initial schema')
+  } else if (isEmptyDiff(diff)) {
+    // Reachable and not a bug: a sync that only changed a column's *type* or an
+    // index moves no names. Saying so is better than printing an empty block.
+    lines.push('      no table or column names changed')
+  } else {
+    const show = (label: string, items: string[]) => {
+      if (items.length) lines.push(`      ${label} ${items.join(', ')}`)
+    }
+    show('+ tables ', diff.tablesAdded)
+    show('- tables ', diff.tablesRemoved)
+    show('+ columns', diff.columnsAdded)
+    show('- columns', diff.columnsRemoved)
+  }
+  // Only worth saying on rows that predate the payload carrying indexes,
+  // because those are exactly the rows `db:rollback` will refuse.
+  if (entry.indexes === undefined) {
+    lines.push('      (no index record — written before ledger v2)')
+  }
+  return lines
+}
+
+export class HistoryService {
+  protected constructor() {}
+
+  static helpRequested(argv: string[] = process.argv.slice(2)): boolean {
+    return argv.includes('--help') || argv.includes('-h')
+  }
+
+  static printHelp(): void {
+    // Program output, not a log line — the same call the other commands' usage
+    // text makes, and one of the two documented `console` exceptions.
+    console.log(`
+Usage: bun run db:history
+
+Lists every schema Bakery has applied to this database, newest first, with
+what changed between each one and the one before it.
+
+Read-only. Nothing here writes to the database.
+`)
+  }
+
+  static async run(): Promise<void> {
+    if (HistoryService.helpRequested()) return HistoryService.printHelp()
+
+    const { initConfig } = await import('@bakery-framework/core/core/config')
+    const { closeDB, connection, initDB } = await import('../connection')
+    await initConfig()
+    await initDB()
+
+    const entries = await readLedgerEntries(connection)
+    if (!entries.length) {
+      // The level is `logger.log`'s second argument. A leading 'I ' is the
+      // `messageLogger` table syntax and would be printed verbatim here.
+      logger.log(
+        'No schema history yet. The ledger fills up as %ydb:sync%* applies changes.',
+        'info',
+      )
+      await closeDB()
+      return
+    }
+
+    logger.log(
+      `%g${entries.length}%* applied schema${entries.length === 1 ? '' : 's'}, newest first:`,
+      'info',
+    )
+    // Newest first, and each row is compared against the row *after* it in the
+    // list, which is the one that came before it in time.
+    entries.forEach((entry, i) => {
+      for (const line of formatEntry(entry, entries[i + 1], i === 0)) {
+        logger.log(line, 'info')
+      }
+    })
+
+    await closeDB()
+  }
+}
+
+if (import.meta.main) {
+  await HistoryService.run()
+  process.exit(0)
+}

package/src/sync/index.ts

@@ -0,0 +1,113 @@
+import '@bakery-framework/core/core/init'
+
+import { Logger, messageLogger } from '@bakery-framework/core/logger'
+import { closeDB, connection, initDB } from '../connection'
+import { loadSchema, schemaFromConfig } from './load'
+
+const logger = new Logger('db-sync')
+
+const syncMsgs = {
+  INVALID_SCHEMA: 'W %yschema.ts is invalid or corrupt. Treating as new.%*',
+  NO_DBINFO: 'W %yDBInfo namespace not found in schema.ts!%*',
+  FOREIGN_TARGET:
+    'E %rForeign key target is not a primary key or unique%*: {refs}. SQL requires the referenced column to be a PRIMARY KEY or carry a UNIQUE index. MySQL and Postgres refuse the CREATE; SQLite accepts it and then fails every insert with "foreign key mismatch". Add unique() on the target column.',
+  FOREIGN_UNSUPPORTED:
+    'E %rforeign() is declared but not implemented%*: {names}. No adapter emits FOREIGN KEY DDL, so it would be created as a plain index and then re-diffed on every sync. Use index() on the column and enforce the reference in your application.',
+  SCHEMA_NOT_FOUND:
+    'E %rConfigured schema path not found%*: {path}. %yschema%* in server.config.ts must name a file or an orm/ folder that exists; remove it to auto-detect. Generating one from the database? Create the (empty) file first.',
+} as const
+
+const MESSAGES = messageLogger(logger, syncMsgs)
+
+export class SyncService {
+  protected constructor() {}
+
+  /**
+   * Usage text, and whether `--help` was asked for.
+   *
+   * Separated from `run()` so the answer is available before anything is
+   * opened. It used to be the last check in `run()`, after `initConfig`,
+   * `initDB`, `loadSchema` and both fatal guards — so the one flag whose whole
+   * job is to explain the others exited 1 on an unreachable database or a
+   * single `foreign()` declaration, and creating `bakery/server.db` as a side
+   * effect of asking for help.
+   */
+  static helpRequested(argv: string[] = process.argv.slice(2)): boolean {
+    return argv.includes('--help') || argv.includes('-h')
+  }
+
+  static printHelp(): void {
+    // CLI usage text goes to stdout verbatim — it is program output, not a
+    // log line, so it deliberately bypasses the structured logger.
+    console.log(`
+Usage: bun run db:sync [--choose=db|ts] [--dry-run] [--force-sync] [--help]
+
+Flags:
+  --choose=db     Generate schema.ts from the database (DB wins)
+  --choose=ts     Apply schema.ts to the database (TS wins, default)
+  --dry-run       Preview planned changes without applying them
+  --force-sync    In production, allow destructive changes
+  --no-ledger     Diff against live introspection, ignoring the recorded schema
+  --help, -h      Show this help message
+`)
+  }
+
+  static async run() {
+    // Before initConfig/initDB/loadSchema: help must not depend on a working
+    // connection, a loadable schema, or the absence of a `foreign()`.
+    if (SyncService.helpRequested()) return SyncService.printHelp()
+
+    const { initConfig } = await import('@bakery-framework/core/core/config')
+    const config = await initConfig()
+    await initDB()
+    // `schema` in server.config.ts when the app sets one; otherwise prefers an
+    // orm/ folder and falls back to a single schema.ts.
+    const loaded = await loadSchema(process.cwd(), schemaFromConfig(config))
+    const schemaPath = loaded.targetPath
+    const constraints = loaded.constraints
+    const tsIndexes = loaded.indexes
+
+    // A configured path that does not exist is a config error, not an empty
+    // project. Continuing would sync against no schema and then generate a new
+    // one at the wrong location — with the app's real model still sitting
+    // where the typo missed it.
+    if (loaded.missing) {
+      MESSAGES.SCHEMA_NOT_FOUND({ path: loaded.missing })
+      await closeDB()
+      return process.exit(1)
+    }
+
+    if (loaded.unreferenceable?.length) {
+      MESSAGES.FOREIGN_TARGET({ refs: loaded.unreferenceable.join(', ') })
+      await closeDB()
+      return process.exit(1)
+    }
+
+    // `foreign()` used to abort here, because no adapter emitted FOREIGN KEY
+    // DDL and the declaration would have become a plain index — referential
+    // integrity in appearance only. All three adapters now emit and read back
+    // real foreign keys, so the guard is gone.
+    //
+    // `findUnsupportedForeignKeys` is kept and still exported *from
+    // `sync/load`* — it is what a future adapter without support would use to
+    // refuse rather than pretend. It is no longer imported here, which is the
+    // distinction: the function has a reason to exist, the dead import did not.
+
+    if (loaded.layout === 'none' && (await Bun.file(schemaPath).exists())) {
+      MESSAGES.NO_DBINFO()
+    }
+
+    await connection.syncSchema(
+      constraints,
+      tsIndexes,
+      schemaPath,
+      loaded.layout,
+    )
+    await closeDB()
+  }
+}
+
+if (import.meta.main) {
+  await SyncService.run()
+  process.exit(0)
+}

package/src/sync/ledger.ts

@@ -0,0 +1,335 @@
+import { Case } from '@bakery-framework/core/utils'
+import type { SQLAdapter } from '../adapters/base'
+import type * as SyncTypes from './types'
+
+/**
+ * A record, inside the database, of the schema Bakery last applied to it.
+ *
+ * **Why this exists.** Sync has always compared your TypeScript schema against
+ * live introspection, which means reconstructing *intent* from whatever the
+ * dialect reports back. Nearly every sync bug this project has had is that
+ * reconstruction going wrong: Postgres re-rendering `''` as
+ * `''::character varying`, MySQL reporting `information_schema` columns
+ * uppercase, `tinyint(1)` versus `data_type`, `EXTRACT` rewritten to
+ * `date_part` on older servers, an empty-string default parsed into the number
+ * zero. Each one produced a column that differed from itself forever, rebuilding
+ * the table on every single sync.
+ *
+ * Comparing against what we *wrote down* removes that whole class. Two JSON
+ * objects, no dialect spellings, no normalisation — and it can carry things
+ * introspection cannot report reliably, such as a `VARCHAR` width.
+ *
+ * **Why it is not simply trusted.** Introspection cannot lie about what exists;
+ * a ledger can. If someone alters the database outside Bakery, the ledger is
+ * stale, and migrating from a stale premise is worse than a spurious rebuild —
+ * it can drop a column somebody added. So the ledger is used only when its
+ * *shape* still matches the live database, and shape is checked by comparing
+ * table and column names, which needs no normalisation and therefore cannot
+ * suffer the bugs above. Anything else falls back to introspection.
+ *
+ * One consequence to know: **MySQL commits DDL implicitly**, so the schema
+ * change and the ledger write cannot be one atomic unit there. A crash between
+ * them leaves the ledger behind, which the shape check then catches — it fails
+ * safe, toward introspection.
+ */
+export const LEDGER_TABLE = '__bakery_schema'
+
+/** Append-only: each successful sync adds a row, so history comes free. */
+async function ensureTable(adapter: SQLAdapter): Promise<void> {
+  const q = (s: string) => adapter.quote(s)
+  const id = adapter.colDef({
+    type: 'integer',
+    autoIncrement: true,
+    primary: true,
+  })
+  const at = adapter.colDef({ type: 'integer' })
+  const payload = adapter.colDef({ type: 'string' })
+  await adapter
+    .query(
+      `CREATE TABLE IF NOT EXISTS ${q(LEDGER_TABLE)} (` +
+        `${q('id')} ${id}, ${q('applied_at')} ${at}, ${q('payload')} ${payload})`,
+    )
+    .run()
+}
+
+/** One applied schema, as stored. */
+export interface LedgerEntry {
+  id: number
+  appliedAt: number
+  constraints: SyncTypes.DBConstraints
+  /**
+   * Absent on rows written before the payload carried indexes — see
+   * {@link parsePayload}. `undefined` means "not recorded", which is not the
+   * same as `{}` ("recorded, and there were none"), and `db:rollback` refuses
+   * the difference rather than guessing.
+   */
+  indexes?: SyncTypes.DBIndexes
+}
+
+/**
+ * A stored payload, in either shape it can have.
+ *
+ * v1 rows are a bare `DBConstraints` object; v2 wraps constraints alongside the
+ * indexes that were applied with them. The version is detected by the `v` key
+ * rather than by a column, so existing rows keep working with no migration of
+ * the migration table — which would be a fine joke and a bad idea.
+ *
+ * Indexes were missing from v1 because the ledger only ever fed the *diff*,
+ * which reads constraints. `db:rollback` replays a stored schema as a target,
+ * and a target with no indexes means "drop every index" — so the payload had to
+ * grow before rollback could be honest.
+ */
+function parsePayload(
+  raw: unknown,
+): Omit<LedgerEntry, 'id' | 'appliedAt'> | null {
+  if (typeof raw !== 'string') return null
+  let parsed: any
+  try {
+    parsed = JSON.parse(raw)
+  } catch {
+    return null
+  }
+  if (!parsed || typeof parsed !== 'object') return null
+  if (
+    parsed.v === 2 &&
+    parsed.constraints &&
+    typeof parsed.constraints === 'object'
+  ) {
+    return {
+      constraints: parsed.constraints,
+      indexes:
+        parsed.indexes && typeof parsed.indexes === 'object'
+          ? parsed.indexes
+          : undefined,
+    }
+  }
+  // v1: the object *is* the constraints. No `indexes` key to report.
+  return 'v' in parsed ? null : { constraints: parsed }
+}
+
+/**
+ * Every applied schema, newest first.
+ *
+ * Never throws, for the same reason {@link readLedger} does not: an absent
+ * table is the normal state of a database Bakery has not synced, and a row that
+ * will not parse must not take a read-only command down. Unparseable rows are
+ * skipped rather than nulled into the list, so a caller counting entries counts
+ * usable ones.
+ */
+export async function readLedgerEntries(
+  adapter: SQLAdapter,
+): Promise<LedgerEntry[]> {
+  try {
+    const q = (s: string) => adapter.quote(s)
+    const rows = (await adapter
+      .query(
+        `SELECT ${q('id')}, ${q('applied_at')}, ${q('payload')} ` +
+          `FROM ${q(LEDGER_TABLE)} ORDER BY ${q('id')} DESC`,
+      )
+      .all()) as any[]
+    const out: LedgerEntry[] = []
+    for (const row of rows ?? []) {
+      const payload = parsePayload(row?.payload)
+      if (!payload) continue
+      out.push({
+        id: Number(row.id),
+        appliedAt: Number(row.applied_at ?? row.appliedAt ?? 0),
+        ...payload,
+      })
+    }
+    return out
+  } catch {
+    return []
+  }
+}
+
+/**
+ * The most recently applied schema, or `null` when there is none.
+ *
+ * Never throws: an unreadable or absent ledger is the normal state for a
+ * database Bakery has not synced yet, and for one it has, a corrupt row must
+ * degrade to introspection rather than take `db:sync` down.
+ */
+export async function readLedger(
+  adapter: SQLAdapter,
+): Promise<SyncTypes.DBConstraints | null> {
+  const entries = await readLedgerEntries(adapter)
+  return entries[0]?.constraints ?? null
+}
+
+/** Record what was just applied. Best-effort: a sync that worked still worked. */
+export async function writeLedger(
+  adapter: SQLAdapter,
+  constraints: SyncTypes.DBConstraints,
+  indexes: SyncTypes.DBIndexes = {},
+): Promise<boolean> {
+  try {
+    await ensureTable(adapter)
+    const q = (s: string) => adapter.quote(s)
+    const payload = JSON.stringify({
+      v: 2,
+      constraints: stripLedger(constraints),
+      indexes,
+    })
+    await adapter
+      .query(
+        `INSERT INTO ${q(LEDGER_TABLE)} (${q('applied_at')}, ${q('payload')}) VALUES (?, ?)`,
+      )
+      .run(Math.floor(Date.now() / 1000), payload)
+    return true
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Every spelling of the ledger's name that can appear in a constraints object.
+ *
+ * `getConstraints()` camelCases table names, so the table created as
+ * `__bakery_schema` comes back as `bakerySchema` — matching only the literal
+ * name silently stripped nothing, and the ledger showed up in its own diff as
+ * a table the schema did not declare. Which then made the shape check fail
+ * forever and the ledger never get used at all.
+ */
+const LEDGER_ALIASES = new Set([
+  LEDGER_TABLE,
+  Case.camel(LEDGER_TABLE),
+  Case.snake(LEDGER_TABLE),
+])
+
+/** The ledger must never appear in a diff, or sync would try to manage itself. */
+export function stripLedger(
+  constraints: SyncTypes.DBConstraints,
+): SyncTypes.DBConstraints {
+  const hit = Object.keys(constraints).filter(k => LEDGER_ALIASES.has(k))
+  if (!hit.length) return constraints
+  const out = { ...constraints }
+  for (const k of hit) delete (out as any)[k]
+  return out
+}
+
+/**
+ * Does the ledger still describe the same set of tables and columns as the
+ * live database?
+ *
+ * **Names only, deliberately.** Comparing types or defaults here would need the
+ * same normalisation the ledger exists to avoid, and a disagreement there is
+ * exactly what the ledger is more trustworthy about. Names are names in every
+ * dialect, so this check cannot itself be wrong in the way the others were.
+ */
+export function shapesMatch(
+  ledger: SyncTypes.DBConstraints,
+  live: SyncTypes.DBConstraints,
+): { ok: true } | { ok: false; reason: string } {
+  const meta = (k: string) => k.startsWith('_')
+  const tablesOf = (c: SyncTypes.DBConstraints) =>
+    Object.keys(c)
+      .filter(t => !meta(t))
+      .sort()
+  const colsOf = (t: any) =>
+    Object.keys(t ?? {})
+      .filter(c => !meta(c))
+      .sort()
+
+  const a = tablesOf(ledger)
+  const b = tablesOf(live)
+  if (a.join() !== b.join()) {
+    const added = b.filter(t => !a.includes(t))
+    const gone = a.filter(t => !b.includes(t))
+    return {
+      ok: false,
+      reason: `tables differ (${added.length ? `+${added.join(', ')}` : ''}${
+        added.length && gone.length ? '; ' : ''
+      }${gone.length ? `-${gone.join(', ')}` : ''})`,
+    }
+  }
+
+  for (const t of a) {
+    const lc = colsOf((ledger as any)[t])
+    const dc = colsOf((live as any)[t])
+    if (lc.join() !== dc.join()) {
+      return { ok: false, reason: `columns of ${t} differ` }
+    }
+  }
+  return { ok: true }
+}
+
+/**
+ * The state sync should diff against: the ledger when it is still true of the
+ * database, introspection otherwise.
+ *
+ * Returning the *source* as well as the constraints is what lets the caller say
+ * out loud which one it used — a sync that quietly changed its mind about where
+ * truth lives would be very hard to debug later.
+ */
+export async function resolveCurrentState(
+  adapter: SQLAdapter,
+  options: { ignoreLedger?: boolean } = {},
+): Promise<{
+  constraints: SyncTypes.DBConstraints
+  source: 'ledger' | 'introspection'
+  reason?: string
+}> {
+  const live = stripLedger(await adapter.getConstraints())
+  // `--no-ledger`, and it exists because the ledger can be *wrong about types*
+  // in a way `shapesMatch` cannot see. That check compares names only, on
+  // purpose — comparing types would need the dialect normalisation the ledger
+  // exists to avoid — so a ledger claiming `VARCHAR(64)` where the column is
+  // really `TEXT` matches its shape and wins the diff. That happens whenever a
+  // sync legitimately concluded "no change needed" and the *rule* it used later
+  // got stricter, which is exactly what adding width to the diff did.
+  //
+  // Without an escape hatch the only recovery is deleting the ledger table by
+  // hand, and the symptom is `db:sync` insisting a database is perfectly synced
+  // against a schema it does not match.
+  if (options.ignoreLedger) {
+    return {
+      constraints: live,
+      source: 'introspection',
+      reason: 'ledger ignored (--no-ledger)',
+    }
+  }
+  const ledger = await readLedger(adapter)
+  if (!ledger)
+    return {
+      constraints: live,
+      source: 'introspection',
+      reason: 'no ledger yet',
+    }
+
+  const match = shapesMatch(ledger, live)
+  if (!match.ok) {
+    return { constraints: live, source: 'introspection', reason: match.reason }
+  }
+  return { constraints: ledger, source: 'ledger' }
+}
+
+/**
+ * Has the database stopped matching what Bakery last applied?
+ *
+ * The same comparison {@link resolveCurrentState} makes, exposed on its own so
+ * a caller can *report* it. Sync already handles drift correctly — it quietly
+ * falls back to introspection — but quietly is the problem: a column added by
+ * hand in production is indistinguishable, from the outside, from a normal run.
+ * This is how you find out.
+ *
+ * `null` when there is nothing to say: no ledger yet (a database Bakery has not
+ * synced is not "drifted"), or the shape still matches.
+ *
+ * Never throws. It runs at boot, and a database that cannot answer must not be
+ * the reason a server fails to start — the sync path will raise anything that
+ * genuinely matters.
+ */
+export async function detectDrift(
+  adapter: SQLAdapter,
+): Promise<{ reason: string } | null> {
+  try {
+    const ledger = await readLedger(adapter)
+    if (!ledger) return null
+    const live = stripLedger(await adapter.getConstraints())
+    const match = shapesMatch(ledger, live)
+    return match.ok ? null : { reason: match.reason }
+  } catch {
+    return null
+  }
+}