@metaobjectsdev/migrate-ts 0.8.1 → 0.9.0

package/src/snapshot/serialize.ts

@@ -0,0 +1,81 @@
+// src/snapshot/serialize.ts
+import type { SchemaSnapshot } from "../types.js";
+
+/**
+ * On-disk format version for the committed schema snapshot. Bump when the
+ * SchemaSnapshot descriptor gains a field (a DDL-coverage feature); add the
+ * matching upgrade branch in parseSnapshot at the same time.
+ */
+export const SNAPSHOT_FORMAT_VERSION = 2;
+
+interface SnapshotFile {
+  formatVersion: number;
+  snapshot: SchemaSnapshot;
+}
+
+function sortByName<T extends { name: string }>(arr: readonly T[]): T[] {
+  return [...arr].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+}
+
+/** Sort arrays by name so serialization is order-independent. */
+function canonicalize(s: SchemaSnapshot): SchemaSnapshot {
+  return {
+    tables: sortByName(s.tables).map((t) => ({
+      ...t,
+      columns: sortByName(t.columns),
+      indexes: sortByName(t.indexes),
+      foreignKeys: sortByName(t.foreignKeys),
+      checks: sortByName(t.checks ?? []),
+    })),
+    views: sortByName(s.views),
+    ...(s.meta ? { meta: s.meta } : {}),
+  };
+}
+
+/** JSON.stringify with object keys sorted recursively (arrays left as-is). */
+function stableStringify(value: unknown): string {
+  return JSON.stringify(
+    value,
+    (_key, v) => {
+      if (v && typeof v === "object" && !Array.isArray(v)) {
+        const obj = v as Record<string, unknown>;
+        return Object.fromEntries(Object.keys(obj).sort().map((k) => [k, obj[k]]));
+      }
+      return v;
+    },
+    2,
+  );
+}
+
+export function serializeSnapshot(snapshot: SchemaSnapshot): string {
+  const file: SnapshotFile = {
+    formatVersion: SNAPSHOT_FORMAT_VERSION,
+    snapshot: canonicalize(snapshot),
+  };
+  return stableStringify(file) + "\n";
+}
+
+export function parseSnapshot(text: string): SchemaSnapshot {
+  const file = JSON.parse(text) as SnapshotFile;
+  if (typeof file.formatVersion !== "number") {
+    throw new Error("snapshot file is missing a numeric 'formatVersion'");
+  }
+  if (file.formatVersion > SNAPSHOT_FORMAT_VERSION) {
+    throw new Error(
+      `snapshot formatVersion ${file.formatVersion} is newer than supported ` +
+        `${SNAPSHOT_FORMAT_VERSION}; upgrade @metaobjectsdev/migrate-ts`,
+    );
+  }
+  if (file.snapshot === null || typeof file.snapshot !== "object") {
+    throw new Error("snapshot file is missing a 'snapshot' object");
+  }
+  if (file.formatVersion < 2) {
+    // v1 → v2: the table descriptor gained `checks`; default older snapshots to [].
+    for (const t of file.snapshot.tables) {
+      if ((t as { checks?: unknown }).checks === undefined) {
+        (t as { checks: unknown[] }).checks = [];
+      }
+    }
+  }
+  return file.snapshot;
+}

package/src/snapshot/store.ts

@@ -0,0 +1,33 @@
+// src/snapshot/store.ts
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import type { Dialect, SchemaSnapshot } from "../types.js";
+import { parseSnapshot, serializeSnapshot } from "./serialize.js";
+
+/**
+ * Committed reference-snapshot path for a dialect, e.g.
+ * `<migrationsDir>/.schema.postgres.json`. d1 shares sqlite's schema, so both
+ * map to `.schema.sqlite.json`.
+ */
+export function snapshotPath(migrationsDir: string, dialect: Dialect): string {
+  const d = dialect === "d1" ? "sqlite" : dialect;
+  return join(migrationsDir, `.schema.${d}.json`);
+}
+
+/** Read + parse the snapshot, or null if the file is absent. */
+export async function readSnapshot(path: string): Promise<SchemaSnapshot | null> {
+  let text: string;
+  try {
+    text = await readFile(path, "utf8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw err;
+  }
+  return parseSnapshot(text);
+}
+
+/** Serialize + write the snapshot, creating the parent directory if needed. */
+export async function writeSnapshot(path: string, snapshot: SchemaSnapshot): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, serializeSnapshot(snapshot), "utf8");
+}

package/src/sql/split-statements.ts

@@ -0,0 +1,115 @@
+/**
+ * Split a SQL script into its top-level statements on `;` boundaries, ignoring
+ * separators that fall inside string literals, quoted identifiers, comments, or
+ * dollar-quoted blocks. The single source of truth for statement splitting —
+ * used both when applying migration files ({@link ../apply/apply.ts}) and when
+ * post-processing emitted DDL for the D1 target ({@link ../emit/d1-safety-pass.ts}).
+ *
+ * Returned statements are trimmed and DO NOT include the trailing `;`
+ * separator; callers that need a terminator re-add one.
+ */
+export function splitSqlStatements(text: string): string[] {
+  const statements: string[] = [];
+  let start = 0;
+  let i = 0;
+  const n = text.length;
+
+  while (i < n) {
+    const ch = text[i];
+    const next = text[i + 1];
+
+    // -- line comment: skip to end of line (or input).
+    if (ch === "-" && next === "-") {
+      i += 2;
+      while (i < n && text[i] !== "\n") i++;
+      continue;
+    }
+
+    // /* block comment */: skip to closing delimiter (or input).
+    if (ch === "/" && next === "*") {
+      i += 2;
+      while (i < n && !(text[i] === "*" && text[i + 1] === "/")) i++;
+      i += 2; // consume the closing */ (clamped by the while below)
+      continue;
+    }
+
+    // Single-quoted literal: consume to the closing quote, treating '' as escape.
+    if (ch === "'") {
+      i++;
+      while (i < n) {
+        if (text[i] === "'" && text[i + 1] === "'") {
+          i += 2; // embedded ''
+          continue;
+        }
+        if (text[i] === "'") {
+          i++;
+          break;
+        }
+        i++;
+      }
+      continue;
+    }
+
+    // Double-quoted identifier: consume to the closing quote, treating "" as escape.
+    if (ch === '"') {
+      i++;
+      while (i < n) {
+        if (text[i] === '"' && text[i + 1] === '"') {
+          i += 2; // embedded ""
+          continue;
+        }
+        if (text[i] === '"') {
+          i++;
+          break;
+        }
+        i++;
+      }
+      continue;
+    }
+
+    // Dollar-quoted string: $tag$ … $tag$ with tag = [A-Za-z0-9_]*.
+    if (ch === "$") {
+      const open = matchDollarTag(text, i);
+      if (open !== null) {
+        const tag = text.slice(i, open); // includes both $ delimiters
+        i = open;
+        // Scan for the matching closing tag.
+        while (i < n) {
+          if (text[i] === "$" && text.startsWith(tag, i)) {
+            i += tag.length;
+            break;
+          }
+          i++;
+        }
+        continue;
+      }
+    }
+
+    // Statement separator (only reached when in no special state).
+    if (ch === ";") {
+      const stmt = text.slice(start, i).trim();
+      if (stmt.length > 0) statements.push(stmt);
+      i++;
+      start = i;
+      continue;
+    }
+
+    i++;
+  }
+
+  const tail = text.slice(start).trim();
+  if (tail.length > 0) statements.push(tail);
+  return statements;
+}
+
+/**
+ * If a dollar-quote tag opens at `pos` (text[pos] === "$"), return the index
+ * just past the opening `$tag$` delimiter; otherwise null. The tag body is
+ * `[A-Za-z0-9_]*` and must be terminated by a second `$`.
+ */
+function matchDollarTag(text: string, pos: number): number | null {
+  let j = pos + 1;
+  while (j < text.length && /[A-Za-z0-9_]/.test(text[j] as string)) j++;
+  if (text[j] === "$") return j + 1;
+  return null;
+}

package/src/sql-type.ts

@@ -14,6 +14,7 @@ export type SqlType =
   | { kind: "boolean" }
   | { kind: "timestamp"; withTimezone: boolean }
   | { kind: "date" }
+  | { kind: "time" }
   | { kind: "json" }
   | { kind: "blob" }
   | { kind: "uuid" };
@@ -36,6 +37,7 @@ export function sqlTypeEquals(a: SqlType, b: SqlType): boolean {
     case "real4":
     case "boolean":
     case "date":
+    case "time":
     case "json":
     case "blob":
     case "uuid":
@@ -85,6 +87,7 @@ export function isWidening(from: SqlType, to: SqlType): boolean {
     case "real4":
     case "boolean":
     case "date":
+    case "time":
     case "json":
     case "blob":
     case "uuid":

package/src/types.ts

@@ -31,6 +31,7 @@ export interface TableDescriptor {
   columns: ColumnDescriptor[];
   indexes: IndexDescriptor[];
   foreignKeys: FkDescriptor[];
+  checks: CheckDescriptor[];
   primaryKey: string[];              // column names; [] if none
   /**
    * Human-readable description threaded from entity `@description`.
@@ -67,6 +68,13 @@ export interface IndexDescriptor {
   unique: boolean;
 }
 
+export interface CheckDescriptor {
+  /** Constraint name, e.g. `<table>_<column>_chk`. Diff/identity key. */
+  name: string;
+  /** The boolean SQL expression, e.g. `status IN ('OPEN','CLOSED')`. */
+  expression: string;
+}
+
 export interface FkDescriptor {
   name: string;
   columns: string[];
@@ -83,11 +91,17 @@ export interface ViewDescriptor {
   /** Same semantics as TableDescriptor.schema. */
   schema?: string;
   /**
-   * View body: everything between `CREATE VIEW <name> AS` and the trailing `;`
-   * (the SELECT clause through the FROM/WHERE/GROUP-BY tail). Populated by
-   * `buildExpectedSchema` from projection metadata; omitted by introspect
-   * (body-level comparison isn't implemented yet — diff matches by name only,
-   * so a body change does NOT trigger replace-view today).
+   * View definition SQL.
+   *
+   * On the EXPECTED side (`buildExpectedSchema` / `buildExpectedViews`) this is
+   * the view body — the SELECT clause through the FROM/WHERE/GROUP-BY tail.
+   *
+   * On the ACTUAL side (`introspect`) this is whatever the DB catalog stores:
+   * sqlite's `sqlite_master.sql` is the full `CREATE VIEW <name> AS <body>`
+   * statement, while Postgres' `information_schema.views.view_definition` is the
+   * body only. `diff`'s view-body comparator normalizes both sides (strips any
+   * leading `CREATE VIEW ... AS`, collapses whitespace) before comparing, so a
+   * body change triggers a `replace-view`.
    */
   sql?: string;
 }
@@ -109,10 +123,10 @@ export interface ViewDescriptor {
  */
 export type Change =
   | { kind: "create-table"; table: TableDescriptor; schema?: string; status: ChangeStatus }
-  | { kind: "drop-table"; table: string; schema?: string; status: ChangeStatus }
+  | { kind: "drop-table"; table: string; schema?: string; restore?: TableDescriptor; status: ChangeStatus }
   | { kind: "rename-table"; from: string; to: string; schema?: string; status: ChangeStatus }
   | { kind: "add-column"; table: string; schema?: string; column: ColumnDescriptor; status: ChangeStatus }
-  | { kind: "drop-column"; table: string; schema?: string; column: string; status: ChangeStatus }
+  | { kind: "drop-column"; table: string; schema?: string; column: string; restore?: ColumnDescriptor; status: ChangeStatus }
   | { kind: "rename-column"; table: string; schema?: string; from: string; to: string; status: ChangeStatus }
   | { kind: "change-column-type"; table: string; schema?: string; column: string;
       from: SqlType; to: SqlType; status: ChangeStatus }
@@ -121,9 +135,11 @@ export type Change =
   | { kind: "change-column-default"; table: string; schema?: string; column: string;
       from?: ColumnDefault; to?: ColumnDefault; status: ChangeStatus }
   | { kind: "add-index"; table: string; schema?: string; index: IndexDescriptor; status: ChangeStatus }
-  | { kind: "drop-index"; table: string; schema?: string; index: string; status: ChangeStatus }
+  | { kind: "drop-index"; table: string; schema?: string; index: string; restore?: IndexDescriptor; status: ChangeStatus }
   | { kind: "add-fk"; table: string; schema?: string; fk: FkDescriptor; status: ChangeStatus }
-  | { kind: "drop-fk"; table: string; schema?: string; fk: string; status: ChangeStatus }
+  | { kind: "drop-fk"; table: string; schema?: string; fk: string; restore?: FkDescriptor; status: ChangeStatus }
+  | { kind: "add-check"; table: string; schema?: string; check: CheckDescriptor; status: ChangeStatus }
+  | { kind: "drop-check"; table: string; schema?: string; check: string; restore?: CheckDescriptor; status: ChangeStatus }
   // Declared for v0.3, never produced in v0.1:
   | { kind: "create-view"; view: ViewDescriptor; schema?: string; status: ChangeStatus }
   | { kind: "drop-view"; view: string; schema?: string; status: ChangeStatus }
@@ -147,6 +163,7 @@ export interface AllowOptions {
   typeChange?: boolean;
   dropIndex?: boolean;
   dropFk?: boolean;
+  dropCheck?: boolean;
   /** Existing data must satisfy NOT NULL; diff cannot verify this. */
   nullableToNotNull?: boolean;
 }

package/src/verify/replay.ts

@@ -0,0 +1,43 @@
+// src/verify/replay.ts
+import type { Kysely } from "kysely";
+import { applyPending } from "../apply/apply.js";
+import { MIGRATIONS_TABLE } from "../apply/ledger.js";
+import { introspect } from "../introspect/index.js";
+import { driftAgainstSnapshot, type DriftClassification } from "../drift/classify.js";
+import type { Dialect, SchemaSnapshot } from "../types.js";
+
+export interface VerifyReplayArgs {
+  /** A FRESH, throwaway database. Replay applies every migration into it from empty. */
+  db: Kysely<Record<string, unknown>>;
+  dialect: Extract<Dialect, "postgres" | "sqlite">;
+  /** Directory holding the committed `<timestamp>-<slug>/up.sql` migrations. */
+  migrationsDir: string;
+  /** The committed snapshot the migrations are expected to reproduce. */
+  snapshot: SchemaSnapshot;
+}
+
+export interface VerifyReplayResult extends DriftClassification {
+  /** True when the replayed schema matches the snapshot (no drift, no unmanaged). */
+  ok: boolean;
+}
+
+/**
+ * Replay all committed migrations into a fresh database, introspect the result,
+ * and compare it to the committed snapshot. A non-empty `drift`/`unmanaged` means
+ * the migrations-as-applied diverge from the snapshot — e.g. a hand-edited up.sql
+ * that changed structure the metadata-derived snapshot doesn't know about. The
+ * ledger sidecar table is excluded from the comparison.
+ */
+export async function verifyReplay(args: VerifyReplayArgs): Promise<VerifyReplayResult> {
+  await applyPending(args.db, args.migrationsDir, { dryRun: false, dialect: args.dialect });
+  const introspected = await introspect(args.db, args.dialect);
+  const actual: SchemaSnapshot = {
+    ...introspected,
+    tables: introspected.tables.filter((t) => t.name !== MIGRATIONS_TABLE),
+  };
+  const classification = await driftAgainstSnapshot(args.snapshot, actual, args.dialect);
+  return {
+    ...classification,
+    ok: classification.drift.length === 0 && classification.unmanaged.length === 0,
+  };
+}

package/src/view-sql-compare.ts

@@ -0,0 +1,46 @@
+// view-sql-compare.ts — the single shared comparator for view definition SQL.
+//
+// View definition SQL arrives in two shapes across the pipeline:
+//   - EXPECTED (buildExpectedViews): the body only — `SELECT ... FROM ...`.
+//   - ACTUAL (introspect): sqlite's sqlite_master.sql is the full
+//     `CREATE VIEW <name> AS <body>`; Postgres' view_definition is body-only.
+//   - The CLI's readExistingViewSql synthesizes `CREATE VIEW <name> AS <body>`
+//     for Postgres so it matches what emitViewDdl produces.
+//
+// normalizeViewSql reduces any of these to a comparable canonical form so the
+// diff (expected vs introspected) and the CLI (emitted DDL vs existing DDL) use
+// ONE comparison. It strips a leading `CREATE [OR REPLACE] VIEW <name> AS`,
+// collapses runs of whitespace to a single space, drops a trailing `;`, and
+// lower-cases — view-body drift should be classified by structure, not by
+// incidental whitespace/case/wrapper differences.
+//
+// CAVEAT (accepted tradeoff): lower-casing makes keyword/identifier comparison
+// case-insensitive but can mask a difference that lives ONLY in a case-sensitive
+// string literal in the body (e.g. `WHERE status = 'Active'` vs `'active'`) —
+// such a change would NOT be flagged as drift. Acceptable for generated
+// aggregate/passthrough projections (no literals); revisit if hand-authored
+// views with case-sensitive literals become a drift concern. The name regex
+// matches one whitespace/`(`-free token, so a quoted view name containing a
+// space would not strip cleanly — also a non-issue for generated identifiers.
+
+const CREATE_VIEW_PREFIX =
+  /^\s*create\s+(?:or\s+replace\s+)?(?:temp(?:orary)?\s+)?view\s+(?:if\s+not\s+exists\s+)?[^\s(]+(?:\s*\([^)]*\))?\s+as\s+/i;
+
+/**
+ * Collapse a view definition (full `CREATE VIEW ... AS body` OR a bare body)
+ * to a canonical, comparable string. Whitespace-, case-, and wrapper-insensitive.
+ */
+export function normalizeViewSql(sql: string): string {
+  return sql
+    .replace(CREATE_VIEW_PREFIX, "")
+    .replace(/\s+/g, " ")
+    .replace(/;\s*$/, "")
+    .trim()
+    .toLowerCase();
+}
+
+/** True when two view definitions are equivalent after normalization. */
+export function viewSqlEquals(a: string | undefined, b: string | undefined): boolean {
+  if (a === undefined || b === undefined) return false;
+  return normalizeViewSql(a) === normalizeViewSql(b);
+}