@sigitex/outlaw 1.0.0

package/src/cowboyMigration/CowboyMigrator.ts

@@ -0,0 +1,437 @@
+import { newline, print } from "@sigitex/print"
+import type { Connection } from "../api"
+import { Format } from "../framework"
+import { Reflector } from "../reflection"
+import type {
+  BuildTable,
+  BuildView,
+  IndexData,
+  SchemaMembers,
+  Schema,
+  TableData,
+  ViewData,
+} from "../schemaBuilder"
+import { generateCreateTable } from "../schemaGenerator"
+import { generateCreateIndex } from "../schemaGenerator/generateCreateIndex"
+import { generateCreateView } from "../schemaGenerator/generateCreateView"
+import { Compare } from "./Compare"
+import type { SchemaHack } from "./cowboyMigration.types"
+
+type Migratable = TableData | ViewData | IndexData
+type MigratableType = "table" | "view" | "index"
+
+const MIGRATION_TABLE = "cowboy_migration"
+
+export class CowboyMigrator {
+  private readonly connection: Connection
+  private readonly schema: Schema<SchemaMembers>
+  private readonly hacks: SchemaHack[]
+  private readonly reflect: Reflector
+
+  constructor(
+    connection: Connection,
+    schema: Schema<SchemaMembers>,
+    hacks: SchemaHack[],
+  ) {
+    this.connection = connection
+    this.schema = schema
+    this.hacks = hacks
+    this.reflect = new Reflector(connection)
+  }
+
+  async migrate() {
+    console.log("🤠 checking for migrations")
+
+    const hasMigrationTable = await this.reflect.hasTable(MIGRATION_TABLE)
+    if (!hasMigrationTable) {
+      await this.connection.script([
+        "PRAGMA defer_foreign_keys = on",
+        createMigrationTable(),
+      ])
+      this.reflect.invalidate()
+    }
+
+    // phase 1: execute hacks
+    await this.executeHacks()
+
+    // phase 2: schema convergence
+    const statements: string[] = ["PRAGMA defer_foreign_keys = on"]
+    for (const { $meta: table } of Object.values<BuildTable<unknown>>(
+      this.schema.tables,
+    )) {
+      const isMissingTable = await this.reflect.isMissingTable(table.name)
+      if (isMissingTable) {
+        console.log(`🤠 creating table: ${table.name}`)
+        statements.push(generateCreateTable(table).trim())
+        statements.push(insertMigration("table", table))
+      } else {
+        const oldTable = await this.getTableData(table.name)
+        const equal = Compare.table(oldTable, table)
+        if (!equal) {
+          const migration = migrateTable(oldTable, table, this.hacks)
+          if (migration.length > 0) {
+            console.log(`🤠 migrating table: ${table.name}`)
+            statements.push(...migration)
+            statements.push(updateMigration("table", table))
+          }
+        }
+      }
+    }
+    if (statements.length > 1) {
+      await this.connection.script(statements)
+      this.reflect.invalidate()
+    }
+
+    // phase 3: views and indexes (after tables are converged)
+    await this.convergeViewsAndIndexes()
+  }
+
+  private async convergeViewsAndIndexes() {
+    const statements: string[] = []
+
+    // Collect current schema view/index names for stale detection
+    const schemaViewNames = new Set<string>()
+    const schemaIndexNames = new Set<string>()
+
+    // Views
+    for (const member of Object.values(this.schema.views)) {
+      const view = (member as BuildView).$meta
+      schemaViewNames.add(view.name)
+      const exists = await this.hasMigration(view.name)
+      if (!exists) {
+        console.log(`🤠 creating view: ${view.name}`)
+        statements.push(generateCreateView(view).trim())
+        statements.push(insertMigration("view", view))
+      } else {
+        const oldData = await this.getMigrationData(view.name)
+        if (oldData !== JSON.stringify(view)) {
+          console.log(`🤠 recreating view: ${view.name}`)
+          statements.push(
+            print(["drop view if exists ", Format.name(view.name)]),
+          )
+          statements.push(generateCreateView(view).trim())
+          statements.push(updateMigration("view", view))
+        }
+      }
+    }
+
+    // Indexes
+    for (const member of Object.values(this.schema.indexes)) {
+      const index = (member as { $meta: IndexData }).$meta
+      schemaIndexNames.add(index.name)
+      const exists = await this.hasMigration(index.name)
+      if (!exists) {
+        console.log(`🤠 creating index: ${index.name}`)
+        statements.push(generateCreateIndex(index).trim())
+        statements.push(insertMigration("index", index))
+      } else {
+        const oldData = await this.getMigrationData(index.name)
+        if (oldData !== JSON.stringify(index)) {
+          console.log(`🤠 recreating index: ${index.name}`)
+          statements.push(
+            print(["drop index if exists ", Format.name(index.name)]),
+          )
+          statements.push(generateCreateIndex(index).trim())
+          statements.push(updateMigration("index", index))
+        }
+      }
+    }
+
+    // Drop stale views/indexes that are tracked but no longer in schema
+    const tracked = await this.getTrackedNonTables()
+    for (const { name, type } of tracked) {
+      if (type === "view" && !schemaViewNames.has(name)) {
+        console.log(`🤠 dropping stale view: ${name}`)
+        statements.push(print(["drop view if exists ", Format.name(name)]))
+        statements.push(deleteMigration(name))
+      }
+      if (type === "index" && !schemaIndexNames.has(name)) {
+        console.log(`🤠 dropping stale index: ${name}`)
+        statements.push(print(["drop index if exists ", Format.name(name)]))
+        statements.push(deleteMigration(name))
+      }
+    }
+
+    if (statements.length > 0) {
+      await this.connection.script(statements)
+    }
+  }
+
+  private async executeHacks() {
+    if (this.hacks.length === 0) {
+      return
+    }
+
+    const statements: string[] = ["PRAGMA defer_foreign_keys = on"]
+    let dirty = false
+
+    for (const hack of this.hacks) {
+      switch (hack.type) {
+        case "droppedTable": {
+          if (await this.reflect.hasTable(hack.tableName)) {
+            console.log(`🤠 dropping table: ${hack.tableName}`)
+            statements.push(print(["drop table ", Format.name(hack.tableName)]))
+            statements.push(deleteMigration(hack.tableName))
+            dirty = true
+          }
+          break
+        }
+        case "renamedTable": {
+          if (await this.reflect.hasTable(hack.fromTable)) {
+            if (await this.reflect.hasTable(hack.toTable)) {
+              // auto-recovery: previous hack-less migration created an empty
+              // table under the new name — drop it so we can rename the old one
+              console.log(`🤠 dropping stale table: ${hack.toTable}`)
+              statements.push(print(["drop table ", Format.name(hack.toTable)]))
+              statements.push(deleteMigration(hack.toTable))
+            }
+            console.log(
+              `🤠 renaming table: ${hack.fromTable} → ${hack.toTable}`,
+            )
+            statements.push(
+              print([
+                "alter table ",
+                Format.name(hack.fromTable),
+                " rename to ",
+                Format.name(hack.toTable),
+              ]),
+            )
+            statements.push(renameMigration(hack.fromTable, hack.toTable))
+            dirty = true
+          }
+          break
+        }
+        case "droppedColumn": {
+          if (await this.reflect.hasTable(hack.tableName)) {
+            const tableData = await this.getTableData(hack.tableName)
+            const columnExists = tableData.columns.some(
+              ({ name }) => name === hack.columnName,
+            )
+            if (columnExists) {
+              console.log(
+                `🤠 dropping column: ${hack.tableName}.${hack.columnName}`,
+              )
+              statements.push(
+                print([
+                  "alter table ",
+                  Format.name(hack.tableName),
+                  " drop column ",
+                  Format.name(hack.columnName),
+                ]),
+              )
+              const updated: TableData = {
+                ...tableData,
+                columns: tableData.columns.filter(
+                  ({ name }) => name !== hack.columnName,
+                ),
+              }
+              statements.push(updateMigration("table", updated))
+              dirty = true
+            }
+          }
+          break
+        }
+        case "renamedColumn": {
+          if (await this.reflect.hasTable(hack.fromTable)) {
+            const tableData = await this.getTableData(hack.fromTable)
+            const columnExists = tableData.columns.some(
+              ({ name }) => name === hack.fromColumn,
+            )
+            if (columnExists) {
+              console.log(
+                `🤠 renaming column: ${hack.fromTable}.${hack.fromColumn} → ${hack.toColumn}`,
+              )
+              statements.push(
+                print([
+                  "alter table ",
+                  Format.name(hack.fromTable),
+                  " rename column ",
+                  Format.name(hack.fromColumn),
+                  " to ",
+                  Format.name(hack.toColumn),
+                ]),
+              )
+              const updated: TableData = {
+                ...tableData,
+                columns: tableData.columns.map((col) =>
+                  col.name === hack.fromColumn
+                    ? { ...col, name: hack.toColumn }
+                    : col,
+                ),
+              }
+              statements.push(updateMigration("table", updated))
+              dirty = true
+            }
+          }
+          break
+        }
+      }
+    }
+
+    if (dirty) {
+      await this.connection.script(statements)
+      this.reflect.invalidate()
+    }
+  }
+
+  private async getTableData(tableName: string) {
+    const results = await this.connection.query<{ data: string }>(`
+      select data from ${MIGRATION_TABLE}
+      where name = ${Format.text(tableName)}
+    `)
+    return JSON.parse(results[0].data) as TableData
+  }
+
+  private async hasMigration(name: string) {
+    const results = await this.connection.query<{ name: string }>(`
+      select name from ${MIGRATION_TABLE}
+      where name = ${Format.text(name)}
+    `)
+    return results.length > 0
+  }
+
+  private async getMigrationData(name: string) {
+    const results = await this.connection.query<{ data: string }>(`
+      select data from ${MIGRATION_TABLE}
+      where name = ${Format.text(name)}
+    `)
+    return results[0]?.data
+  }
+
+  private async getTrackedNonTables() {
+    return this.connection.query<{ name: string; type: string }>(`
+      select name, type from ${MIGRATION_TABLE}
+      where type != 'table'
+    `)
+  }
+}
+
+function createMigrationTable() {
+  return `
+    create table ${MIGRATION_TABLE} (
+      id integer primary key autoincrement,
+      timestamp integer not null,
+      type string not null,
+      name string not null unique,
+      data string not null
+    )
+  `
+}
+
+function insertMigration(type: MigratableType, data: Migratable) {
+  const json = JSON.stringify(data)
+  return `
+    insert into ${MIGRATION_TABLE} (
+      timestamp,
+      type,
+      name,
+      data
+    )
+    values (
+      ${Format.NOW},
+      '${type}',
+      ${Format.text(data.name)},
+      ${Format.text(json)}
+    )
+  `
+}
+
+function updateMigration(_type: MigratableType, data: Migratable) {
+  const json = JSON.stringify(data)
+  return `
+    update ${MIGRATION_TABLE}
+    set timestamp = ${Format.NOW},
+        data = ${Format.text(json)}
+    where name = ${Format.text(data.name)}
+  `
+}
+
+function deleteMigration(tableName: string) {
+  return `
+    delete from ${MIGRATION_TABLE}
+    where name = ${Format.text(tableName)}
+  `
+}
+
+function renameMigration(fromName: string, toName: string) {
+  return `
+    update ${MIGRATION_TABLE}
+    set timestamp = ${Format.NOW},
+        name = ${Format.text(toName)}
+    where name = ${Format.text(fromName)}
+  `
+}
+
+function migrateTable(
+  oldTable: TableData,
+  newTable: TableData,
+  hacks: SchemaHack[],
+): string[] {
+  const interimName = `__interim_${newTable.name}`
+  const renames = hacks
+    .filter((hack) => hack.type === "renamedColumn")
+    .filter(({ fromTable }) => fromTable === newTable.name)
+  // oxlint-disable-next-line unicorn/no-array-reduce
+  const mappings = newTable.columns.reduce<{ from: string; to: string }[]>(
+    (mappings, column) => {
+      const oldColumnExists = oldTable.columns.some(
+        ({ name }) => name === column.name,
+      )
+      const rename = renames.find((rename) => rename.toColumn === column.name)
+      const renamesOldColumn =
+        !!rename &&
+        oldTable.columns.some(({ name }) => name === rename.fromColumn)
+      if (!oldColumnExists && !renamesOldColumn) {
+        return mappings
+      }
+      return [
+        // biome-ignore lint/performance/noAccumulatingSpread: small n
+        ...mappings,
+        {
+          from: renamesOldColumn ? rename.fromColumn : column.name,
+          to: column.name,
+        },
+      ]
+    },
+    [],
+  )
+  const orphaned = oldTable.columns.filter((col) => {
+    const isMapped = mappings.some((m) => m.from === col.name)
+    const isDropped = hacks.some(
+      (h) =>
+        h.type === "droppedColumn" &&
+        h.tableName === newTable.name &&
+        h.columnName === col.name,
+    )
+    return !isMapped && !isDropped
+  })
+  if (orphaned.length > 0) {
+    const names = orphaned.map((c) => c.name).join(", ")
+    console.error(
+      `🤠 refusing to migrate ${newTable.name}: columns [${names}] would be lost — add renamed.column() or dropped.column() hacks`,
+    )
+    return []
+  }
+  return [
+    generateCreateTable({ ...newTable, name: interimName }).trim(),
+    print([
+      "insert into ",
+      Format.name(interimName),
+      " (",
+      mappings.map((mapping, m) => [m > 0 && ", ", Format.name(mapping.to)]),
+      ")",
+      newline,
+      "select ",
+      mappings.map((mapping, m) => [m > 0 && ", ", Format.name(mapping.from)]),
+      " from ",
+      Format.name(oldTable.name),
+    ]),
+    print(["drop table ", Format.name(oldTable.name)]),
+    print([
+      "alter table ",
+      Format.name(interimName),
+      " rename to ",
+      Format.name(newTable.name),
+    ]),
+  ]
+}

package/src/cowboyMigration/CowboySeeder.ts

@@ -0,0 +1,245 @@
+// oxlint-disable typescript/no-explicit-any
+import { join, print } from "@sigitex/print"
+import stableStringify from "fast-json-stable-stringify"
+import { createHash } from "node:crypto"
+import type { Connection } from "../api"
+import { Format } from "../framework"
+import { Mappings } from "../queryBuilder/Mappings"
+import { generateDelete, generateInsert } from "../queryGenerator"
+import { Reflector } from "../reflection"
+import { text } from "../schemaBuilder/columnBuilders"
+import { Fixture, type Fixtures } from "../schemaBuilder/createFixture"
+import { createTable } from "../schemaBuilder/createTable"
+import type { RefBy } from "../schemaBuilder/schemaBuilder.types"
+import { generateCreateTable } from "../schemaGenerator"
+
+const SEED_TABLE = "cowboy_seed"
+
+const seedTable = createTable(SEED_TABLE, {
+  name: text.primaryKey,
+  hash: text.notNull,
+})
+
+export class CowboySeeder {
+  private readonly connection: Connection
+  private readonly seeds: Fixtures
+  private readonly fixtures: Fixtures
+  private readonly runFixtures: boolean
+  private readonly reflect: Reflector
+
+  constructor(
+    connection: Connection,
+    seeds: Fixtures,
+    fixtures: Fixtures,
+    runFixtures: boolean,
+  ) {
+    this.connection = connection
+    this.seeds = seeds
+    this.fixtures = fixtures
+    this.runFixtures = runFixtures
+    this.reflect = new Reflector(connection)
+  }
+
+  async seed() {
+    console.log("🌱 checking for seed changes")
+    const statements: string[] = []
+
+    const hasSeedTable = await this.reflect.hasTable(SEED_TABLE)
+    if (!hasSeedTable) {
+      statements.push(generateCreateTable(seedTable.$meta).trim())
+    }
+
+    const existingHashes = hasSeedTable
+      ? await this.connection.query<{ name: string; hash: string }>(
+          `select name, hash from ${SEED_TABLE}`,
+        )
+      : []
+
+    const hashMap = new Map(existingHashes.map((r) => [r.name, r.hash]))
+    const combined = this.combineByTable()
+
+    for (const [tableName, { table, rows }] of combined) {
+      const hash = computeHash(rows)
+      const existing = hashMap.get(tableName)
+      if (existing === hash) {
+        continue
+      }
+
+      console.log(`🌱 reseeding: ${tableName}`)
+      statements.push(...generateReseed(tableName, table, rows))
+      statements.push(upsertHash(tableName, hash))
+    }
+
+    if (statements.length > 0) {
+      statements.unshift("PRAGMA defer_foreign_keys = on")
+      await this.connection.script(statements)
+    }
+  }
+
+  private combineByTable() {
+    const combined = new Map<
+      string,
+      { table: Fixture<any>["table"]; rows: Record<string, unknown>[] }
+    >()
+    const allFixtures = {
+      ...this.seeds,
+      ...(this.runFixtures ? this.fixtures : {}),
+    }
+
+    for (const fixture of Object.values(allFixtures)) {
+      const tableName: string = fixture.table.$meta.name
+      const existing = combined.get(tableName)
+      if (existing) {
+        existing.rows.push(...fixture.rows)
+      } else {
+        combined.set(tableName, {
+          table: fixture.table,
+          rows: [...fixture.rows],
+        })
+      }
+    }
+    return combined
+  }
+}
+
+function computeHash(rows: Record<string, unknown>[]) {
+  const sanitized = rows.map((row) => {
+    const out: Record<string, unknown> = {}
+    for (const key in row) {
+      const value = row[key]
+      if (Fixture.isRefBy(value)) {
+        out[key] =
+          `ref:${value.table.$meta.name}.${String(value.column)}=${value.value}`
+      } else {
+        out[key] = value
+      }
+    }
+    return out
+  })
+  const serialized = stableStringify(sanitized)
+  return createHash("sha256").update(serialized).digest("hex")
+}
+
+function generateReseed(
+  tableName: string,
+  table: Fixture<any>["table"],
+  rows: Record<string, unknown>[],
+): string[] {
+  const tableData = table.$meta
+  const autoincrementCol = tableData.columns.find(
+    (c: any) => c.primaryKey?.autoincrement,
+  )?.name
+
+  const deleteSQL = generateDelete({ table: tableName }).trim()
+  if (rows.length === 0) {
+    return [deleteSQL]
+  }
+
+  const strippedRows = autoincrementCol
+    ? rows.map((row) => stripAutoincrement(row, autoincrementCol))
+    : rows
+
+  const processedRows = strippedRows.map((row) => {
+    const mapped: Record<string, unknown> = {}
+    for (const key in row) {
+      const value = row[key]
+      if (Fixture.isRefBy(value)) {
+        continue
+      }
+      mapped[key] = value
+    }
+    return Mappings.row(tableData, mapped)
+  })
+
+  const hasRefBy = strippedRows.some((row) =>
+    Object.values(row).some(Fixture.isRefBy),
+  )
+
+  if (!hasRefBy) {
+    const columns = Object.keys(strippedRows[0])
+    const insertSQL = generateInsert({
+      table: tableName,
+      columns,
+      rows: processedRows,
+    }).trim()
+    return [deleteSQL, insertSQL]
+  }
+
+  return [
+    deleteSQL,
+    ...generateRefByInserts(tableName, tableData, strippedRows),
+  ]
+}
+
+function generateRefByInserts(
+  tableName: string,
+  tableData: Fixture<any>["table"]["$meta"],
+  rows: Record<string, unknown>[],
+): string[] {
+  const statements: string[] = []
+  for (const row of rows) {
+    const columns: string[] = []
+    const values: string[] = []
+
+    for (const key in row) {
+      columns.push(key)
+      const value = row[key]
+      if (Fixture.isRefBy(value)) {
+        values.push(refBySubquery(value))
+      } else {
+        const mapped = Mappings.row(tableData, { [key]: value })
+        values.push(Format.value(mapped[key]))
+      }
+    }
+
+    statements.push(
+      print([
+        "insert into ",
+        Format.name(tableName),
+        " (",
+        join(", ", columns, Format.name),
+        ") values (",
+        values.join(", "),
+        ")",
+      ]),
+    )
+  }
+  return statements
+}
+
+function refBySubquery(ref: RefBy) {
+  const refTableName = ref.table.$meta.name
+  const pk = ref.table.$meta.columns.find((c: any) => c.primaryKey)
+  const pkColumn = pk ? pk.name : "id"
+  return print([
+    "(select ",
+    Format.name(pkColumn),
+    " from ",
+    Format.name(refTableName),
+    " where ",
+    Format.name(ref.column as string),
+    " = ",
+    Format.value(ref.value),
+    ")",
+  ])
+}
+
+function stripAutoincrement(row: Record<string, unknown>, col: string) {
+  if (row[col] !== 0) {
+    return row
+  }
+  const { [col]: _, ...rest } = row
+  return rest
+}
+
+function upsertHash(tableName: string, hash: string) {
+  return print([
+    "insert or replace into ",
+    SEED_TABLE,
+    " (name, hash) values (",
+    Format.text(tableName),
+    ", ",
+    Format.text(hash),
+    ")",
+  ])
+}

package/src/cowboyMigration/cowboyMigration.types.ts

@@ -0,0 +1,29 @@
+export type SchemaHack =
+| RenamedTable
+| RenamedColumn
+| DroppedTable
+| DroppedColumn
+
+export type RenamedTable = {
+  readonly type: "renamedTable"
+  readonly fromTable: string
+  readonly toTable: string
+}
+
+export type RenamedColumn = {
+  readonly type: "renamedColumn"
+  readonly fromTable: string
+  readonly fromColumn: string
+  readonly toColumn: string
+}
+
+export type DroppedTable = {
+  readonly type: "droppedTable"
+  readonly tableName: string
+}
+
+export type DroppedColumn = {
+  readonly type: "droppedColumn"
+  readonly tableName: string
+  readonly columnName: string
+}