@zmdb/migrations 1.0.0-beta.1

package/src/introspect/drift.ts

@@ -0,0 +1,157 @@
+import {
+  type ChangeOp,
+  type ColumnSnapshot,
+  type SchemaSnapshot,
+  type SqlDialect,
+  type TableSnapshot,
+} from '@zmdb/sql';
+
+import { diff } from '../index.js';
+import { tableSelected, type CatalogForeignKeySnapshot, type CatalogIndexSnapshot } from './common.js';
+
+export interface DriftOptions {
+  /**
+   * Table-name globs to omit. When absent, the migration ledger is omitted.
+   * Supplying the list replaces that default, matching introspection selection.
+   */
+  readonly exclude?: readonly string[];
+  /** The selected database product whose migration rules compare the snapshots. */
+  readonly dialect: SqlDialect;
+  /**
+   * Database-owned catalog normalization. MySQL-family introspectors set this
+   * because their servers may create an undeclared btree index for a foreign key.
+   */
+  readonly omitForeignKeySupportIndexes?: boolean;
+}
+
+export interface DriftReport {
+  /** Operations that transform the declared snapshot into the live database shape. */
+  readonly onlyInDatabase: readonly ChangeOp[];
+  /** Operations that transform the live database snapshot into the declared shape. */
+  readonly onlyInDeclarations: readonly ChangeOp[];
+  readonly clean: boolean;
+}
+
+type DriftColumnSnapshot = ColumnSnapshot & {
+  readonly catalogType?: string;
+  readonly default?: string;
+};
+
+interface DriftTableSnapshot extends Omit<TableSnapshot, 'columns' | 'foreignKeys'> {
+  readonly columns: readonly DriftColumnSnapshot[];
+  readonly foreignKeys?: readonly CatalogForeignKeySnapshot[];
+  readonly indexes?: readonly CatalogIndexSnapshot[];
+}
+
+interface DriftComparableSnapshot extends Omit<SchemaSnapshot, 'tables'> {
+  readonly tables: readonly DriftTableSnapshot[];
+}
+
+interface NormalizedDriftTableSnapshot extends Omit<TableSnapshot, 'foreignKeys'> {
+  readonly foreignKeys: readonly CatalogForeignKeySnapshot[];
+  readonly indexes?: readonly CatalogIndexSnapshot[];
+}
+
+interface NormalizedDriftSnapshot extends Omit<SchemaSnapshot, 'tables'> {
+  readonly tables: readonly NormalizedDriftTableSnapshot[];
+}
+
+type SnapshotRole = 'live' | 'declared';
+
+function normalizeColumn(column: DriftColumnSnapshot): ColumnSnapshot {
+  return {
+    name: column.name,
+    type: column.type,
+    nullable: column.nullable,
+    primaryKey: column.primaryKey,
+    ...(column.length === undefined ? {} : { length: column.length }),
+    ...(column.unique === undefined ? {} : { unique: column.unique }),
+  };
+}
+
+function sameColumns(index: CatalogIndexSnapshot, foreignKey: CatalogForeignKeySnapshot): boolean {
+  if (index.columns.length !== foreignKey.columns.length) return false;
+  return index.columns.every((column, position) => {
+    const expected = foreignKey.columns[position];
+    return typeof column === 'string' && column === expected;
+  });
+}
+
+function isMySqlForeignKeySupportingIndex(
+  index: CatalogIndexSnapshot,
+  foreignKeys: readonly CatalogForeignKeySnapshot[],
+): boolean {
+  if (index.unique || index.where !== undefined) return false;
+  if (index.method !== undefined && index.method.toLowerCase() !== 'btree') return false;
+
+  return foreignKeys.some(
+    foreignKey =>
+      (index.name === foreignKey.name || index.name === `${foreignKey.name}_idx`) && sameColumns(index, foreignKey),
+  );
+}
+
+function normalizeIndexes(
+  table: DriftTableSnapshot,
+  role: SnapshotRole,
+  omitForeignKeySupportIndexes: boolean,
+): readonly CatalogIndexSnapshot[] | undefined {
+  if (table.indexes === undefined) return undefined;
+  if (role !== 'live' || !omitForeignKeySupportIndexes) return table.indexes;
+  const foreignKeys = table.foreignKeys ?? [];
+  return table.indexes.filter(index => !isMySqlForeignKeySupportingIndex(index, foreignKeys));
+}
+
+/**
+ * Internal normalization boundary used before both calls to the migration diff.
+ *
+ * Exported from the introspection subpath as the vendor-neutral normalization
+ * seam used by independently shipped database introspectors.
+ */
+export function normalizeDriftSnapshot(
+  snapshot: SchemaSnapshot,
+  role: SnapshotRole,
+  options: Pick<DriftOptions, 'exclude' | 'omitForeignKeySupportIndexes'> = {},
+): NormalizedDriftSnapshot {
+  const comparable: DriftComparableSnapshot = snapshot;
+  const selection = options.exclude === undefined ? {} : { exclude: options.exclude };
+  const tables = comparable.tables
+    .filter(table => tableSelected(table.name, selection))
+    .map(table => {
+      const indexes = normalizeIndexes(table, role, options.omitForeignKeySupportIndexes ?? false);
+      const normalized: NormalizedDriftTableSnapshot = {
+        name: table.name,
+        // Catalog spellings and default expressions are evidence, not drift:
+        // aliases have already collapsed into `type`, and servers rewrite defaults.
+        columns: table.columns.map(normalizeColumn),
+        primaryKey: table.primaryKey,
+        foreignKeys: table.foreignKeys ?? [],
+        ...(table.tableOptions === undefined ? {} : { tableOptions: table.tableOptions }),
+        ...(indexes === undefined ? {} : { indexes }),
+      };
+      return normalized;
+    })
+    .toSorted((left, right) => left.name.localeCompare(right.name));
+
+  return {
+    version: 1,
+    tables,
+    extensions: snapshot.extensions.toSorted((left, right) => left.name.localeCompare(right.name)),
+  };
+}
+
+/**
+ * Compare an introspected snapshot and declared snapshot through the migration
+ * comparator in both directions.
+ */
+export function detectDrift(live: SchemaSnapshot, declared: SchemaSnapshot, options: DriftOptions): DriftReport {
+  const normalizedLive = normalizeDriftSnapshot(live, 'live', options);
+  const normalizedDeclared = normalizeDriftSnapshot(declared, 'declared', options);
+  const onlyInDatabase = diff(normalizedDeclared, normalizedLive, { dialect: options.dialect });
+  const onlyInDeclarations = diff(normalizedLive, normalizedDeclared, { dialect: options.dialect });
+
+  return {
+    onlyInDatabase,
+    onlyInDeclarations,
+    clean: onlyInDatabase.length === 0 && onlyInDeclarations.length === 0,
+  };
+}

package/src/introspect/index.ts

@@ -0,0 +1,33 @@
+import { type Introspector, type SqlDialect } from '@zmdb/sql';
+
+export { type IntrospectionDriver, type Introspector, type IntrospectOptions } from '@zmdb/sql';
+export { detectDrift, type DriftOptions, type DriftReport } from './drift.js';
+export {
+  action,
+  CatalogRowError,
+  deterministicForeignKeyName,
+  flagField,
+  integerField,
+  nullableIntegerField,
+  nullableTextField,
+  query,
+  sortByName,
+  sortWarnings,
+  splitSqlList,
+  tableSelected,
+  textField,
+  type CatalogColumnSnapshot,
+  type CatalogForeignKeySnapshot,
+  type CatalogIndexColumn,
+  type CatalogIndexSnapshot,
+  type CatalogSchemaSnapshot,
+  type CatalogTableSnapshot,
+  type CatalogWarning,
+  type ReferentialAction,
+} from './common.js';
+export { normalizeDriftSnapshot } from './drift.js';
+
+/** Resolve the catalog reader carried by an explicitly selected database package. */
+export function createIntrospector<Name extends string>(dialect: SqlDialect<Name>): Introspector<Name> {
+  return dialect.introspector;
+}

package/src/operations/check.ts

@@ -0,0 +1,309 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { readMigrations } from '../file-io.js';
+import {
+  diff,
+  snapshot,
+  type ColumnSnapshot,
+  type ExtensionSnapshot,
+  type ExtensionType,
+  type ForeignKeySnapshot,
+  type ReferentialAction,
+  type SchemaSnapshot,
+  type TableSnapshot,
+} from '../index.js';
+import { detectDrift } from '../introspect/index.js';
+import { migrationTarget, requiredIntrospector, type MigrationProject } from '../project.js';
+import { EMBEDDED_WITH_DOWN_MARKER, embeddedOutputPath, renderEmbeddedModule } from './embed.js';
+
+const EMPTY_SNAPSHOT: SchemaSnapshot = { version: 1, tables: [], extensions: [] };
+
+export type CheckFindingKind =
+  | 'uncommitted-schema'
+  | 'duplicate-version'
+  | 'snapshot-version'
+  | 'missing-down'
+  | 'stale-embedded'
+  | 'drift';
+
+export interface CheckFinding {
+  readonly kind: CheckFindingKind;
+  readonly message: string;
+  readonly subject: string;
+}
+
+export interface CheckResult {
+  readonly findings: readonly CheckFinding[];
+  readonly skipped: readonly { readonly kind: 'drift'; readonly reason: string }[];
+}
+
+export async function checkProject(project: MigrationProject): Promise<CheckResult> {
+  const findings: CheckFinding[] = [];
+  const skipped: { readonly kind: 'drift'; readonly reason: string }[] = [];
+  const snapshotPath = join(project.outDir, 'snapshot.json');
+  const stored = await storedSnapshot(snapshotPath);
+  const declared = snapshot(project.schemas);
+
+  if ('newerVersion' in stored) {
+    findings.push({
+      kind: 'snapshot-version',
+      message: `snapshot version ${String(stored.newerVersion)} is newer than this build's version 1`,
+      subject: snapshotPath,
+    });
+  } else if (diff(stored.snapshot, declared, { dialect: migrationTarget(project) }).length > 0) {
+    findings.push({
+      kind: 'uncommitted-schema',
+      message: 'declarations differ from the stored snapshot; run zmdb generate',
+      subject: snapshotPath,
+    });
+  }
+
+  const migrations = await readMigrations(project.outDir);
+  const byVersion = new Map<number, string[]>();
+  for (const migration of migrations) {
+    const paths = byVersion.get(migration.version) ?? [];
+    paths.push(migration.path);
+    byVersion.set(migration.version, paths);
+    if (!migration.hasDown) {
+      findings.push({
+        kind: 'missing-down',
+        message: `migration ${String(migration.version)} ${migration.name} has no -- zmdb:down section`,
+        subject: migration.path,
+      });
+    }
+  }
+  for (const [version, paths] of byVersion) {
+    if (paths.length < 2) continue;
+    findings.push({
+      kind: 'duplicate-version',
+      message: `migration version ${String(version)} appears in ${String(paths.length)} files`,
+      subject: paths.join(', '),
+    });
+  }
+
+  if (project.dialect.migrations.embedded) {
+    const embeddedPath = embeddedOutputPath(project);
+    const embedded = await optionalFile(embeddedPath);
+    if (embedded !== undefined) {
+      const withDown = embedded.startsWith(
+        `// Generated by \`zmdb embed\`. Do not edit.\n${EMBEDDED_WITH_DOWN_MARKER}\n`,
+      );
+      const expected = await renderEmbeddedModule(project, withDown ? { withDown: true } : {});
+      if (embedded !== expected.source) {
+        findings.push({
+          kind: 'stale-embedded',
+          message: 'embedded migration module differs from the configured migration files; run zmdb embed',
+          subject: embeddedPath,
+        });
+      }
+    }
+  }
+
+  if (project.driver === undefined) {
+    skipped.push({ kind: 'drift', reason: `config ${project.configPath} has no driver` });
+  } else if (!('newerVersion' in stored)) {
+    const introspector = requiredIntrospector(project);
+    const live = await introspector.snapshot(project.driver, project.introspect);
+    const report = detectDrift(
+      introspector.normalizeForDrift(live, 'live'),
+      introspector.normalizeForDrift(stored.snapshot, 'declared'),
+      { dialect: project.dialect },
+    );
+    if (!report.clean) {
+      findings.push({
+        kind: 'drift',
+        message:
+          `live database differs from the stored snapshot: ` +
+          `${String(report.onlyInDatabase.length)} database-only and ` +
+          `${String(report.onlyInDeclarations.length)} declaration-only operations`,
+        subject: project.configPath,
+      });
+    }
+  }
+
+  return { findings, skipped };
+}
+
+async function optionalFile(path: string): Promise<string | undefined> {
+  try {
+    return await readFile(path, 'utf8');
+  } catch (error) {
+    if (errorCode(error) === 'ENOENT') return undefined;
+    throw error;
+  }
+}
+
+async function storedSnapshot(
+  path: string,
+): Promise<{ readonly snapshot: SchemaSnapshot } | { readonly newerVersion: number }> {
+  let source: string;
+  try {
+    source = await readFile(path, 'utf8');
+  } catch (error) {
+    if (errorCode(error) === 'ENOENT') return { snapshot: EMPTY_SNAPSHOT };
+    throw error;
+  }
+  const parsed: unknown = JSON.parse(source);
+  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+    throw new TypeError(`stored snapshot ${path} must be a JSON object`);
+  }
+  const version: unknown = Reflect.get(parsed, 'version');
+  if (typeof version !== 'number' || !Number.isSafeInteger(version)) {
+    throw new TypeError(`stored snapshot ${path} has no safe integer version`);
+  }
+  if (version > 1) return { newerVersion: version };
+  if (version !== 1) throw new TypeError(`stored snapshot ${path} uses unknown version ${String(version)}`);
+  return { snapshot: snapshotV1(parsed, path) };
+}
+
+function snapshotV1(value: object, path: string): SchemaSnapshot {
+  const stored = record(value, `stored snapshot ${path}`);
+  const tableValues = array(stored.tables, `stored snapshot ${path} tables`);
+  const extensionValues = array(stored.extensions, `stored snapshot ${path} extensions`);
+  return {
+    version: 1,
+    tables: tableValues.map((table, index) => tableSnapshot(table, path, index)),
+    extensions: extensionValues.map((extension, index) => extensionSnapshot(extension, path, index)),
+  };
+}
+
+function tableSnapshot(value: unknown, path: string, index: number): TableSnapshot {
+  const table = record(value, `stored snapshot ${path} table ${String(index)}`);
+  const name = text(table.name, `stored snapshot ${path} table ${String(index)} name`);
+  const columns = array(table.columns, `stored snapshot ${path} table ${name} columns`).map((column, columnIndex) =>
+    columnSnapshot(column, path, name, columnIndex),
+  );
+  const primaryKey =
+    table.primaryKey === undefined
+      ? columns.filter(column => column.primaryKey).map(column => column.name)
+      : stringArray(table.primaryKey, `stored snapshot ${path} table ${name} primaryKey`);
+  const foreignKeys =
+    table.foreignKeys === undefined
+      ? []
+      : array(table.foreignKeys, `stored snapshot ${path} table ${name} foreignKeys`).map((foreignKey, keyIndex) =>
+          foreignKeySnapshot(foreignKey, path, name, keyIndex),
+        );
+  return { name, columns, primaryKey, foreignKeys };
+}
+
+function columnSnapshot(value: unknown, path: string, table: string, index: number): ColumnSnapshot {
+  const column = record(value, `stored snapshot ${path} table ${table} column ${String(index)}`);
+  const length = column.length;
+  if (length !== undefined && (typeof length !== 'number' || !Number.isSafeInteger(length) || length < 1)) {
+    throw new TypeError(`stored snapshot ${path} table ${table} column ${String(index)} length must be positive`);
+  }
+  return {
+    name: text(column.name, `stored snapshot ${path} table ${table} column ${String(index)} name`),
+    type: columnType(column.type, path, table, index),
+    nullable: flag(column.nullable, `stored snapshot ${path} table ${table} column ${String(index)} nullable`),
+    primaryKey: flag(column.primaryKey, `stored snapshot ${path} table ${table} column ${String(index)} primaryKey`),
+    ...(length === undefined ? {} : { length }),
+  };
+}
+
+function columnType(value: unknown, path: string, table: string, index: number): string | ExtensionType {
+  if (typeof value === 'string') return value;
+  const type = record(value, `stored snapshot ${path} table ${table} column ${String(index)} type`);
+  const args =
+    type.args === undefined
+      ? undefined
+      : array(type.args, `stored snapshot ${path} table ${table} column ${String(index)} type args`).map(
+          (argument, argumentIndex) => {
+            if (typeof argument === 'string' || typeof argument === 'number') return argument;
+            throw new TypeError(
+              `stored snapshot ${path} table ${table} column ${String(index)} type arg ${String(argumentIndex)} ` +
+                'must be a string or number',
+            );
+          },
+        );
+  return {
+    extension: text(type.extension, `stored snapshot ${path} table ${table} column ${String(index)} type extension`),
+    name: text(type.name, `stored snapshot ${path} table ${table} column ${String(index)} type name`),
+    ...(args === undefined ? {} : { args }),
+  };
+}
+
+function foreignKeySnapshot(value: unknown, path: string, table: string, index: number): ForeignKeySnapshot {
+  const foreignKey = record(value, `stored snapshot ${path} table ${table} foreign key ${String(index)}`);
+  return {
+    name: text(foreignKey.name, `stored snapshot ${path} table ${table} foreign key ${String(index)} name`),
+    columns: stringArray(
+      foreignKey.columns,
+      `stored snapshot ${path} table ${table} foreign key ${String(index)} columns`,
+    ),
+    targetTable: text(
+      foreignKey.targetTable,
+      `stored snapshot ${path} table ${table} foreign key ${String(index)} targetTable`,
+    ),
+    targetColumns: stringArray(
+      foreignKey.targetColumns,
+      `stored snapshot ${path} table ${table} foreign key ${String(index)} targetColumns`,
+    ),
+    onDelete: referentialAction(
+      foreignKey.onDelete,
+      `stored snapshot ${path} table ${table} foreign key ${String(index)} onDelete`,
+    ),
+    onUpdate: referentialAction(
+      foreignKey.onUpdate,
+      `stored snapshot ${path} table ${table} foreign key ${String(index)} onUpdate`,
+    ),
+  };
+}
+
+function extensionSnapshot(value: unknown, path: string, index: number): ExtensionSnapshot {
+  const extension = record(value, `stored snapshot ${path} extension ${String(index)}`);
+  const schema = extension.schema;
+  if (schema !== undefined && typeof schema !== 'string') {
+    throw new TypeError(`stored snapshot ${path} extension ${String(index)} schema must be a string`);
+  }
+  return {
+    name: text(extension.name, `stored snapshot ${path} extension ${String(index)} name`),
+    ...(schema === undefined ? {} : { schema }),
+  };
+}
+
+function referentialAction(value: unknown, label: string): ReferentialAction {
+  if (
+    value === 'cascade' ||
+    value === 'restrict' ||
+    value === 'set null' ||
+    value === 'set default' ||
+    value === 'no action'
+  ) {
+    return value;
+  }
+  throw new TypeError(`${label} must be a referential action`);
+}
+
+function record(value: unknown, label: string): Readonly<Record<string, unknown>> {
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    throw new TypeError(`${label} must be an object`);
+  }
+  return Object.fromEntries(Object.entries(value));
+}
+
+function array(value: unknown, label: string): readonly unknown[] {
+  if (!Array.isArray(value)) throw new TypeError(`${label} must be an array`);
+  return value;
+}
+
+function stringArray(value: unknown, label: string): readonly string[] {
+  return array(value, label).map((item, index) => text(item, `${label} item ${String(index)}`));
+}
+
+function text(value: unknown, label: string): string {
+  if (typeof value !== 'string') throw new TypeError(`${label} must be a string`);
+  return value;
+}
+
+function flag(value: unknown, label: string): boolean {
+  if (typeof value !== 'boolean') throw new TypeError(`${label} must be a boolean`);
+  return value;
+}
+
+function errorCode(error: unknown): string | undefined {
+  if (typeof error !== 'object' || error === null) return undefined;
+  const code = Reflect.get(error, 'code');
+  return typeof code === 'string' ? code : undefined;
+}

package/src/operations/embed.ts

@@ -0,0 +1,87 @@
+import { dirname, join, resolve } from 'node:path';
+
+import { dialectName } from '@zmdb/sql';
+
+import { readMigrations, writeTextAtomically } from '../file-io.js';
+import { MigrationProjectError, requiredSourceFormatter, type MigrationProject } from '../project.js';
+
+export const EMBEDDED_WITH_DOWN_MARKER = '// Includes down sections for development tooling.';
+
+export interface EmbedOptions {
+  readonly out?: string;
+  readonly withDown?: boolean;
+}
+
+export interface EmbeddedModuleResult {
+  readonly file: string;
+  readonly source: string;
+  readonly migrations: readonly {
+    readonly version: number;
+    readonly name: string;
+    readonly checksum: string;
+  }[];
+}
+
+export interface EmbedResult {
+  readonly file: string;
+  readonly migrations: EmbeddedModuleResult['migrations'];
+}
+
+export function embeddedOutputPath(project: MigrationProject, requested?: string): string {
+  return requested === undefined
+    ? join(project.outDir, 'embedded.ts')
+    : resolve(dirname(project.configPath), requested);
+}
+
+export async function renderEmbeddedModule(
+  project: MigrationProject,
+  options: EmbedOptions = {},
+): Promise<EmbeddedModuleResult> {
+  const file = embeddedOutputPath(project, options.out);
+  const migrations = await Promise.all(
+    (await readMigrations(project.outDir)).map(async migration => ({
+      version: migration.version,
+      name: migration.name,
+      up: migration.up,
+      ...(options.withDown === true ? { down: migration.down } : {}),
+      checksum: await checksum(migration.up),
+    })),
+  );
+  const arrayType =
+    options.withDown === true
+      ? 'readonly (EmbeddedMigration & { readonly down: string })[]'
+      : 'readonly EmbeddedMigration[]';
+  const marker = options.withDown === true ? `${EMBEDDED_WITH_DOWN_MARKER}\n` : '';
+  const raw =
+    '// Generated by `zmdb embed`. Do not edit.\n' +
+    marker +
+    "import type { EmbeddedMigration } from '@zmdb/migrations/embedded';\n\n" +
+    `export const migrations: ${arrayType} = ${JSON.stringify(migrations, null, 2)} as const;\n`;
+  return {
+    file,
+    source: await requiredSourceFormatter(project)(file, raw),
+    migrations: migrations.map(migration => ({
+      version: migration.version,
+      name: migration.name,
+      checksum: migration.checksum,
+    })),
+  };
+}
+
+export async function embedMigrations(project: MigrationProject, options: EmbedOptions = {}): Promise<EmbedResult> {
+  if (!project.dialect.migrations.embedded) {
+    throw new MigrationProjectError(
+      `embedded migrations are not supported by ${dialectName(project.dialect)} in ${project.configPath}`,
+    );
+  }
+  const rendered = await renderEmbeddedModule(project, options);
+  await writeTextAtomically(rendered.file, rendered.source);
+  return { file: rendered.file, migrations: rendered.migrations };
+}
+
+async function checksum(source: string): Promise<string> {
+  const bytes = new TextEncoder().encode(source);
+  const digest = new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', bytes));
+  const hex = Array.from(digest, byte => byte.toString(16).padStart(2, '0')).join('');
+  return `sha256:${hex}`;
+}

package/src/operations/export.ts

@@ -0,0 +1,21 @@
+import { diff, emitUp, snapshot, type ChangeOp, type SchemaSnapshot } from '../index.js';
+import { migrationTarget, type MigrationProject } from '../project.js';
+
+const EMPTY_SNAPSHOT: SchemaSnapshot = { version: 1, tables: [], extensions: [] };
+
+export interface ExportResult {
+  readonly ops: readonly ChangeOp[];
+  readonly statements: readonly string[];
+}
+
+/** Reflect declarations and hand the resulting plan directly to the DDL emitter. */
+export function exportSchema(project: MigrationProject): ExportResult {
+  const target = migrationTarget(project);
+  const next = snapshot(project.schemas);
+  const ops = diff(EMPTY_SNAPSHOT, next, { dialect: target });
+  target.migrations.validatePlan({ before: EMPTY_SNAPSHOT, after: next, operations: ops });
+  return {
+    ops,
+    statements: ops.map(operation => emitUp(operation, target)),
+  };
+}