@crombolo/data 0.9.1 → 0.10.0

package/src/bundle-tables/chain.ts

@@ -0,0 +1,105 @@
+/**
+ * A bundle's compiled steps as a migration chain, run by the same `migrate`,
+ * under the same lock and into the same ledger as core's — only its state key
+ * and its ledger rows' `bundle` differ. And purging one: dropping exactly the
+ * tables under its prefix, found in the database rather than the declaration,
+ * because a purged bundle is one the site no longer wires.
+ */
+import type { Db } from '../database.ts'
+import { deleteKeyValue, type Migration, MigrationPlan, recordInLedger } from '../migrations.ts'
+import { createTablesCapability } from './capability.ts'
+import { renderStep } from './ddl.ts'
+import { guardStatement } from './guard.ts'
+import { bundlePrefix } from './names.ts'
+import type { CompiledStep, CompiledTables } from './schema.ts'
+
+function migrationOf(step: CompiledStep): Migration {
+  return {
+    from: step.from,
+    to: step.to,
+    name: step.name,
+    kind: step.kind,
+    release: step.release,
+    contracts: step.contracts,
+    checksum: step.checksum,
+    async up(db) {
+      for (const sql of renderStep(db.dialect, step)) await db.exec(sql)
+    },
+    backfill: step.backfill
+      ? (db) =>
+          (step.backfill as NonNullable<CompiledStep['backfill']>)(
+            createTablesCapability(db, step.after),
+          )
+      : undefined,
+  }
+}
+
+/** The chain `migrate` walks. Throws when the declaration has problems: compile first. */
+export function bundlePlan(compiled: CompiledTables): MigrationPlan {
+  if (compiled.problems.length > 0)
+    throw new Error(
+      `The bundle "${compiled.id}"'s tables cannot be planned:\n- ${compiled.problems.join('\n- ')}`,
+    )
+  return new MigrationPlan(compiled.steps.map(migrationOf), {
+    stateKey: compiled.stateKey,
+    initialState: compiled.initialState,
+    bundle: compiled.id,
+  })
+}
+
+/**
+ * Creates a bundle's tables directly, step by step, without its ledger or its
+ * state. Only for proving what core's own migrations do around a bundle's
+ * tables: an install is always `migrate`.
+ */
+export async function applyBundleStepsUnledgered(db: Db, compiled: CompiledTables): Promise<void> {
+  for (const migration of bundlePlan(compiled).migrations)
+    await db.transaction(async (tx) => {
+      await migration.up(tx)
+      await migration.backfill?.(tx)
+    })
+}
+
+/** Every table in the database under a bundle's prefix. */
+export async function bundleTablesIn(db: Db, id: string): Promise<string[]> {
+  const prefix = bundlePrefix(id)
+  const rows =
+    db.dialect.name === 'sqlite'
+      ? await db.query<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table'")
+      : await db.query<{ name: string }>(
+          "SELECT table_name AS name FROM information_schema.tables WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'",
+        )
+  // Compared as text rather than with LIKE, where `_` is a wildcard.
+  return rows
+    .map((row) => row.name)
+    .filter((name) => name.startsWith(prefix))
+    .sort()
+}
+
+/**
+ * Drops every table under the bundle's prefix and forgets its state, so adding
+ * it again starts its chain from the beginning. Ledgered as a `purge`.
+ */
+export async function purgeBundleTables(
+  db: Db,
+  id: string,
+  options: { by?: string; stateKey: string },
+): Promise<string[]> {
+  const prefix = bundlePrefix(id)
+  const tables = await bundleTablesIn(db, id)
+  const started = performance.now()
+  await db.transaction(async (tx) => {
+    for (const table of tables)
+      await tx.exec(guardStatement(id, prefix, `DROP TABLE ${tx.dialect.quote(table)}`))
+  })
+  await deleteKeyValue(db, options.stateKey)
+  await recordInLedger(db, {
+    name: `purge ${id}`,
+    kind: 'purge',
+    durationMs: Math.round(performance.now() - started),
+    appliedBy: options.by ?? null,
+    note: tables.length > 0 ? `dropped ${tables.join(', ')}` : 'no tables',
+    bundle: id,
+  })
+  return tables
+}

package/src/bundle-tables/ddl.ts

@@ -0,0 +1,124 @@
+/**
+ * A step's operations as SQL, per dialect. Every identifier comes from the
+ * resolved schema and is quoted here, every default is a literal the builder
+ * already checked, and every statement passes the guard before it is returned.
+ */
+import type { Dialect } from '../dialect.ts'
+import type { ColumnSpec, Operation } from './builder.ts'
+import { guardStatement } from './guard.ts'
+import type { ResolvedSchema } from './schema.ts'
+
+/** The column type, from the dialect seam. Text compares case-sensitively on both. */
+export function columnType(dialect: Dialect, spec: ColumnSpec): string {
+  const t = dialect.types
+  switch (spec.type) {
+    case 'uuid':
+      return t.uuid
+    case 'varchar':
+      return t.varcharExact(spec.length ?? 1)
+    case 'text':
+      return t.text
+    case 'integer':
+      return t.integer
+    case 'bigint':
+      return t.bigint
+    case 'boolean':
+      return t.boolean
+    case 'timestamp':
+      return t.timestamp
+    case 'json':
+      return t.json
+  }
+}
+
+function literal(dialect: Dialect, value: string | number | boolean): string {
+  if (typeof value === 'boolean')
+    return dialect.name === 'postgres' ? (value ? 'TRUE' : 'FALSE') : value ? '1' : '0'
+  if (typeof value === 'number') return String(value)
+  return `'${value.replaceAll("'", "''")}'`
+}
+
+function columnDefinition(dialect: Dialect, name: string, spec: ColumnSpec): string {
+  const parts = [dialect.quote(name), columnType(dialect, spec)]
+  if (!spec.nullable) parts.push('NOT NULL')
+  if (spec.default !== undefined) parts.push(`DEFAULT ${literal(dialect, spec.default)}`)
+  return parts.join(' ')
+}
+
+/** The statements one operation runs, against the schema as it stood before it. */
+export function renderOperation(
+  dialect: Dialect,
+  before: ResolvedSchema,
+  after: ResolvedSchema,
+  op: Operation,
+): string[] {
+  const q = (identifier: string) => dialect.quote(identifier)
+  const guard = (sql: string) => guardStatement(before.id, before.prefix, sql)
+  const table = after.tables.get(op.table) ?? before.tables.get(op.table)
+  if (!table) throw new Error(`no table "${op.table}" to render ${op.op} against`)
+  const statements: string[] = []
+  const indexFor = (columns: readonly string[]) => {
+    const index = (after.tables.get(op.table) ?? table).indexes.get(columns.join(','))
+    if (!index) throw new Error(`no index on "${op.table}" (${columns.join(',')})`)
+    return index
+  }
+  const createIndex = (columns: readonly string[]) => {
+    const index = indexFor(columns)
+    statements.push(
+      `CREATE ${index.unique ? 'UNIQUE ' : ''}INDEX ${q(index.name)} ON ${q(table.physical)} (${index.columns.map(q).join(', ')})`,
+    )
+  }
+
+  switch (op.op) {
+    case 'createTable': {
+      const columns = Object.entries(op.columns).map(([name, spec]) =>
+        columnDefinition(dialect, name, spec),
+      )
+      const key = Object.entries(op.columns).find(([, spec]) => spec.primaryKey)?.[0] as string
+      statements.push(
+        `CREATE TABLE ${q(table.physical)} (${columns.join(', ')}, PRIMARY KEY (${q(key)}))`,
+      )
+      for (const [name, spec] of Object.entries(op.columns))
+        if (spec.indexed && !spec.primaryKey) createIndex([name])
+      break
+    }
+    case 'addColumn':
+      statements.push(
+        `ALTER TABLE ${q(table.physical)} ADD COLUMN ${columnDefinition(dialect, op.column, op.spec)}`,
+      )
+      if (op.spec.indexed) createIndex([op.column])
+      break
+    case 'addIndex':
+      createIndex(op.columns)
+      break
+    case 'widenColumn':
+      // SQLite stores by affinity and enforces no length, so a wider type is
+      // already what it holds; only Postgres has anything to change.
+      if (dialect.name === 'postgres')
+        statements.push(
+          `ALTER TABLE ${q(table.physical)} ALTER COLUMN ${q(op.column)} TYPE ${columnType(dialect, op.spec)}`,
+        )
+      break
+    case 'dropTable':
+      statements.push(`DROP TABLE ${q(table.physical)}`)
+      break
+    case 'dropColumn':
+      statements.push(`ALTER TABLE ${q(table.physical)} DROP COLUMN ${q(op.column)}`)
+      break
+    case 'dropIndex': {
+      const index = before.tables.get(op.table)?.indexes.get(op.columns.join(','))
+      if (!index) throw new Error(`no index on "${op.table}" (${op.columns.join(',')})`)
+      statements.push(`DROP INDEX ${q(index.name)}`)
+      break
+    }
+  }
+  return statements.map(guard)
+}
+
+/** Every statement a step runs, in order. */
+export function renderStep(
+  dialect: Dialect,
+  step: { before: ResolvedSchema; after: ResolvedSchema; operations: readonly Operation[] },
+): string[] {
+  return step.operations.flatMap((op) => renderOperation(dialect, step.before, step.after, op))
+}

package/src/bundle-tables/guard.ts

@@ -0,0 +1,116 @@
+/**
+ * The one check every statement a bundle's builder or capability emits passes.
+ * `docs/bundles.md`, Part 4.
+ *
+ * It only ever sees SQL this module wrote, which is regular enough to check
+ * token by token: every identifier is double-quoted and lowercase, every value
+ * is a `?` placeholder or a literal the builder rendered, and there is exactly
+ * one statement. What it asserts is that every table or index the statement
+ * names carries the bundle's own prefix. A statement that fails is refused
+ * before it reaches the database, so a bug in the code above it is a refusal
+ * rather than a query against a core table.
+ */
+
+/** A refusal: the statement never ran. */
+export class BundleStatementError extends Error {
+  constructor(
+    readonly bundle: string,
+    readonly reason: string,
+  ) {
+    super(`Bundle "${bundle}": a statement was refused before it ran: ${reason}.`)
+  }
+}
+
+/** Keywords after which the next identifier names a table or an index. */
+const OBJECT_POSITIONS = new Set([
+  'FROM',
+  'INTO',
+  'UPDATE',
+  'TABLE',
+  'JOIN',
+  'INDEX',
+  'ON',
+  'REFERENCES',
+])
+
+/** Keywords that may sit between a position keyword and the object it names. */
+const FILLERS = new Set(['IF', 'NOT', 'EXISTS', 'ONLY'])
+
+type Token =
+  | { kind: 'word'; value: string }
+  | { kind: 'identifier'; value: string }
+  | { kind: 'other'; value: string }
+
+const QUOTED = /^"([a-z][a-z0-9_]*)"/
+const WORD = /^[A-Za-z_][A-Za-z0-9_]*/
+const STRING = /^'(?:[^']|'')*'/
+const NUMBER = /^-?\d+(?:\.\d+)?/
+const PUNCTUATION = /^(?:<=|>=|<>|[(),=<>*?.])/
+
+function tokenize(bundle: string, sql: string): Token[] {
+  const tokens: Token[] = []
+  let rest = sql
+  for (;;) {
+    rest = rest.replace(/^\s+/, '')
+    if (rest === '') return tokens
+    if (rest.startsWith('--') || rest.startsWith('/*'))
+      throw new BundleStatementError(bundle, 'it contains a comment')
+    if (rest.startsWith(';')) throw new BundleStatementError(bundle, 'it contains a ";"')
+    if (rest.startsWith('"')) {
+      const match = QUOTED.exec(rest)
+      if (!match)
+        throw new BundleStatementError(bundle, 'it quotes an identifier no bundle may write')
+      tokens.push({ kind: 'identifier', value: match[1] as string })
+      rest = rest.slice(match[0].length)
+      continue
+    }
+    const string = STRING.exec(rest)
+    if (string) {
+      tokens.push({ kind: 'other', value: string[0] })
+      rest = rest.slice(string[0].length)
+      continue
+    }
+    const word = WORD.exec(rest)
+    if (word) {
+      tokens.push({ kind: 'word', value: word[0].toUpperCase() })
+      rest = rest.slice(word[0].length)
+      continue
+    }
+    const other = NUMBER.exec(rest) ?? PUNCTUATION.exec(rest)
+    if (!other) throw new BundleStatementError(bundle, `it contains "${rest[0]}"`)
+    tokens.push({ kind: 'other', value: other[0] })
+    rest = rest.slice(other[0].length)
+  }
+}
+
+/** The tables and indexes a statement names, as the guard sees them. */
+export function objectsNamed(bundle: string, sql: string): string[] {
+  const tokens = tokenize(bundle, sql)
+  const named: string[] = []
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i] as Token
+    if (token.kind !== 'word' || !OBJECT_POSITIONS.has(token.value)) continue
+    // `ON CONFLICT` and `ON DELETE` are not objects; only an index's `ON <table>` is.
+    let j = i + 1
+    while (tokens[j]?.kind === 'word' && FILLERS.has((tokens[j] as Token).value)) j++
+    const next = tokens[j]
+    if (token.value === 'ON' && next?.kind !== 'identifier') continue
+    if (next?.kind !== 'identifier')
+      throw new BundleStatementError(bundle, `"${token.value}" is not followed by a quoted name`)
+    named.push(next.value)
+  }
+  return named
+}
+
+/**
+ * Refuses a statement unless every table and index it names carries `prefix`,
+ * and it names at least one.
+ */
+export function guardStatement(bundle: string, prefix: string, sql: string): string {
+  const named = objectsNamed(bundle, sql)
+  if (named.length === 0) throw new BundleStatementError(bundle, 'it names no table')
+  for (const name of named)
+    if (!name.startsWith(prefix))
+      throw new BundleStatementError(bundle, `it names "${name}", which is not this bundle's`)
+  return sql
+}

package/src/bundle-tables/index.ts

@@ -0,0 +1,7 @@
+export * from './builder.ts'
+export * from './capability.ts'
+export * from './chain.ts'
+export * from './ddl.ts'
+export * from './guard.ts'
+export * from './names.ts'
+export * from './schema.ts'

package/src/bundle-tables/names.ts

@@ -0,0 +1,58 @@
+/**
+ * The names a bundle's tables are stored under. `docs/bundles.md`, Part 4.
+ *
+ * Every physical name is `bundle_<id>__<name>`, with the id's hyphens written
+ * as underscores. An id may not contain `--` and a logical name may not contain
+ * `__`, so the double underscore is an unambiguous separator and no two bundles
+ * can produce the same physical name. Core reserves the `bundle_` prefix, so a
+ * match on a bundle's own prefix can only ever be that bundle's table.
+ */
+
+/** Postgres truncates a longer identifier silently, so a longer one is refused. */
+export const MAX_IDENTIFIER = 63
+
+/** Table and column names, as a bundle writes them. */
+export const LOGICAL_NAME = /^[a-z][a-z0-9_]*$/
+
+/** The prefix every one of this bundle's objects carries. */
+export function bundlePrefix(id: string): string {
+  return `bundle_${id.replaceAll('-', '_')}__`
+}
+
+export const physicalTable = (id: string, table: string): string => `${bundlePrefix(id)}${table}`
+
+/**
+ * An index's name, generated rather than chosen, from what it covers. The
+ * `ix_` that follows the prefix is a namespace a table may not enter, because
+ * Postgres keeps tables and indexes in one.
+ */
+export function indexName(id: string, table: string, columns: readonly string[]): string {
+  const hash = new Bun.CryptoHasher('sha256').update(`${table}(${columns.join(',')})`).digest('hex')
+  return `${bundlePrefix(id)}ix_${hash.slice(0, 12)}`
+}
+
+/** Why an id cannot own tables, or undefined when it can. */
+export function bundleIdProblem(id: string): string | undefined {
+  if (id.includes('--'))
+    return `the id "${id}" contains "--", which would make its table names ambiguous`
+  if (id.endsWith('-'))
+    return `the id "${id}" ends with "-", which would make its table names ambiguous`
+  return undefined
+}
+
+/** Why a table or column name is not allowed, or undefined when it is. */
+export function logicalNameProblem(kind: 'table' | 'column', name: string): string | undefined {
+  if (typeof name !== 'string' || !LOGICAL_NAME.test(name))
+    return `the ${kind} name "${String(name)}" is not lowercase letters, digits and underscores starting with a letter`
+  if (name.includes('__')) return `the ${kind} name "${name}" contains "__", the separator`
+  if (kind === 'table' && name.startsWith('ix_'))
+    return `the table name "${name}" starts with "ix_", which index names use`
+  return undefined
+}
+
+/** Why a physical name is too long for Postgres, or undefined when it fits. */
+export function lengthProblem(physical: string): string | undefined {
+  return physical.length > MAX_IDENTIFIER
+    ? `"${physical}" is ${physical.length} characters, past the ${MAX_IDENTIFIER} Postgres keeps`
+    : undefined
+}