@intentius/chant-lexicon-sql 0.124.45 → 0.124.48

package/src/migration-replay.test.ts

@@ -0,0 +1,173 @@
+/**
+ * Replaying a migration directory (#4400), against a fake server: the
+ * statements it sends, the catalog names it records, and the round trip into
+ * the kind reader's outputs. The emulator runs in `migration-replay.e2e.test.ts`.
+ */
+
+import { describe, expect, test } from "vitest";
+import { LINK_READERS } from "@intentius/chant/workspace/kind-link-readers";
+import { REPLAY_FILE } from "@intentius/chant/workspace/migration-dirs";
+import type { WorkspaceTree } from "@intentius/chant/workspace/tree";
+import { applySteps, gooseUpStatements, readCatalogOutputs, renderSnapshot, replayDirectory, replaySteps, splitStatements, type ReplayDb } from "./migration-replay";
+
+function memoryTree(files: Record<string, string>): WorkspaceTree {
+  const dirs = new Set([""]);
+  for (const f of Object.keys(files)) {
+    const parts = f.split("/");
+    for (let i = 1; i < parts.length; i++) dirs.add(parts.slice(0, i).join("/"));
+  }
+  return {
+    label: "",
+    stat: (p) => (p in files ? "file" : dirs.has(p) ? "dir" : undefined),
+    list(p) {
+      if (!dirs.has(p)) return undefined;
+      const prefix = p === "" ? "" : `${p}/`;
+      const seen = new Map<string, "file" | "dir">();
+      for (const f of Object.keys(files)) {
+        if (!f.startsWith(prefix)) continue;
+        const rest = f.slice(prefix.length);
+        seen.set(rest.split("/")[0]!, rest.includes("/") ? "dir" : "file");
+      }
+      return [...seen].map(([name, type]) => ({ name, type }));
+    },
+    read(p) {
+      if (!(p in files)) throw new Error(`no such file ${p}`);
+      return files[p]!;
+    },
+  };
+}
+
+/** A server that records what it is sent and answers the catalog query with `catalog`. */
+function fakeDb(dialect: "postgres" | "clickhouse", catalog: () => Record<string, unknown>[], failOn?: RegExp): ReplayDb & { sent: string[] } {
+  const sent: string[] = [];
+  return {
+    dialect,
+    sent,
+    async run(sql) {
+      if (failOn?.test(sql)) throw new Error("relation does not exist");
+      sent.push(sql);
+    },
+    rows: async () => catalog(),
+    close: async () => {},
+  };
+}
+
+describe("splitting statements", () => {
+  test("semicolons inside strings, identifiers, comments and dollar quotes do not end a statement", () => {
+    const sql = `create table a (x text default 'a;b');\n-- c; d\ncreate function f() returns int as $$ begin return 1; end $$ language plpgsql;\n/* x; */ select "a;b" from a;\n;;`;
+    expect(splitStatements(sql)).toEqual([
+      "create table a (x text default 'a;b')",
+      "-- c; d\ncreate function f() returns int as $$ begin return 1; end $$ language plpgsql",
+      '/* x; */ select "a;b" from a',
+    ]);
+  });
+
+  test("a final statement without a semicolon is kept, and a comment alone is dropped", () => {
+    expect(splitStatements("select 1;\n-- done\n")).toEqual(["select 1"]);
+    expect(splitStatements("select 1;\nselect 2")).toEqual(["select 1", "select 2"]);
+  });
+});
+
+describe("goose Up statements", () => {
+  test("only the Up section runs, and StatementBegin to StatementEnd is one statement", () => {
+    const text = [
+      "-- +goose Up",
+      "create table a (id int);",
+      "-- +goose StatementBegin",
+      "create function f() returns int as $$ begin return 1; end; $$ language plpgsql;",
+      "select 1;",
+      "-- +goose StatementEnd",
+      "alter table a add c int;",
+      "-- +goose Down",
+      "drop table a;",
+    ].join("\n");
+    expect(gooseUpStatements(text)).toEqual(["create table a (id int)", "create function f() returns int as $$ begin return 1; end; $$ language plpgsql;\nselect 1;", "alter table a add c int"]);
+  });
+});
+
+describe("the steps per kind", () => {
+  test("golang-migrate runs the up files in version order, never the down files", () => {
+    const { steps } = replaySteps("golang-migrate", memoryTree({ "10_b.up.sql": "create table b (id int);", "2_a.up.sql": "create table a (id int);", "2_a.down.sql": "drop table a;" }), "");
+    expect(steps.map((s) => [s.migration.file, s.statements])).toEqual([["2_a.up.sql", ["create table a (id int)"]], ["10_b.up.sql", ["create table b (id int)"]]]);
+  });
+
+  test("Flyway substitutes placeholders, runs versioned then repeatable, and skips undo and callbacks", () => {
+    const { steps } = replaySteps(
+      "flyway",
+      memoryTree({
+        "flyway.conf": "flyway.placeholders.t=orders\n",
+        "V1__a.sql": "create table ${t} (id int);",
+        "R__v.sql": "create or replace view v as select 1;",
+        "U1__a.sql": "drop table orders;",
+        "beforeMigrate.sql": "select 1;",
+      }),
+      "",
+    );
+    expect(steps.map((s) => s.statements)).toEqual([["create table orders (id int)"], ["create or replace view v as select 1"]]);
+  });
+
+  test("a Flyway placeholder with no value fails naming the file and line", () => {
+    expect(() => replaySteps("flyway", memoryTree({ "V1__a.sql": "select 1;\ncreate table ${nope} (id int);" }), "")).toThrow(/V1__a\.sql:2: no value for the placeholder \$\{nope\}/);
+  });
+});
+
+describe("replaying", () => {
+  test("a failing statement names the migration file", async () => {
+    const { steps } = replaySteps("golang-migrate", memoryTree({ "1_a.up.sql": "create table a (id int);", "2_b.up.sql": "alter table missing add c int;" }), "");
+    await expect(applySteps(fakeDb("postgres", () => [], /missing/), steps)).rejects.toThrow(/2_b\.up\.sql: statement 1 failed: relation does not exist/);
+  });
+
+  test("the catalog rows become <schema>.<table> and <schema>.<table>.<column>", async () => {
+    const db = fakeDb("clickhouse", () => [
+      { schema: "app", relation: "events", col: "ts" },
+      { schema: "app", relation: "events", col: "kind" },
+      { schema: "app", relation: "empty", col: null },
+    ]);
+    expect(await readCatalogOutputs(db)).toEqual(["app.events", "app.events.ts", "app.events.kind", "app.empty"]);
+  });
+
+  test("a server that already has tables is refused", async () => {
+    await expect(replayDirectory(fakeDb("postgres", () => [{ schema: "public", relation: "old", col: "id" }]), "goose", memoryTree({}), "")).rejects.toThrow(/already has public\.old/);
+  });
+
+  test("a dialect mismatch is refused before anything runs", async () => {
+    const db = fakeDb("clickhouse", () => []);
+    await expect(replayDirectory(db, "goose", memoryTree({ "1_a.sql": "-- +goose Up\nselect 1;" }), "")).rejects.toThrow(/migrations are postgres and the server is clickhouse/);
+    expect(db.sent).toEqual([]);
+  });
+
+  test("the snapshot a replay writes is what the kind reader reads back as outputs, until a migration changes", async () => {
+    const files: Record<string, string> = {
+      "1_users.sql": "-- +goose Up\ncreate table users (id int, email text);\n-- +goose Down\ndrop table users;\n",
+      "2_orders.sql": "-- +goose Up\ncreate table orders (id int, user_id int);\n",
+    };
+    let created = false;
+    const db = fakeDb("postgres", () =>
+      created
+        ? [
+            { schema: "public", relation: "users", col: "id" },
+            { schema: "public", relation: "users", col: "email" },
+            { schema: "public", relation: "orders", col: "id" },
+            { schema: "public", relation: "orders", col: "user_id" },
+          ]
+        : [],
+    );
+    const original = db.run;
+    db.run = async (sql) => {
+      await original(sql);
+      created = true;
+    };
+    const snapshot = await replayDirectory(db, "goose", memoryTree(files), "");
+    expect(db.sent).toEqual(["create table users (id int, email text)", "create table orders (id int, user_id int)"]);
+    expect(snapshot.outputs).toEqual(["public.users", "public.users.id", "public.users.email", "public.orders", "public.orders.id", "public.orders.user_id"]);
+
+    const withSnapshot = { ...files, [REPLAY_FILE]: renderSnapshot(snapshot) };
+    const read = (f: Record<string, string>) => LINK_READERS["goose-dir"]({ member: { name: "db" } as never, dir: "", fields: null, tree: memoryTree(f) });
+    expect(read(withSnapshot).outputs!.map((o) => o.name)).toEqual(snapshot.outputs);
+    expect(read(withSnapshot).gaps).toEqual([]);
+
+    const dropped = read({ ...withSnapshot, "3_drop.sql": "-- +goose Up\nalter table orders drop column user_id;\n" });
+    expect(dropped.outputs).toBeUndefined();
+    expect(dropped.gaps[0]).toMatchObject({ outputs: true, message: expect.stringContaining("changed since") });
+  });
+});

package/src/migration-replay.ts

@@ -0,0 +1,289 @@
+/**
+ * Replay a migration directory into an empty server and read the catalog
+ * (#4400). The workspace kinds for goose, golang-migrate,
+ * clickhouse-migrations and Flyway list a directory's migrations without
+ * running anything (`packages/core/src/workspace/migration-dirs.ts`); the
+ * tables and columns they produce come from here.
+ *
+ *     chant sql replay <dir> --kind <kind> --env <env> [--dir-field migrationsDir=<sub>] [--check]
+ *
+ * The replay applies each migration's up statements in the order the kind's
+ * reader lists them, then reads the tables, views and columns the server has
+ * and writes them as `chant.replay.json` beside the migrations, named by the
+ * contract of #3931: `<schema>.<table>` and `<schema>.<table>.<column>` for
+ * Postgres, `<database>.<table>` and `<database>.<table>.<column>` for
+ * ClickHouse. The file also holds a digest of the files it ran, so the
+ * reader knows when a migration changed after it. `--check` replays and
+ * compares instead of writing, exiting 1 when the file differs, the way
+ * `chant sql types --check` does.
+ *
+ * What it runs per kind:
+ * - goose: the `-- +goose Up` section of each file; `StatementBegin` to
+ *   `StatementEnd` is one statement.
+ * - golang-migrate and clickhouse-migrations: the whole up file.
+ * - Flyway: the SQL files of the locations, versioned in version order and
+ *   then repeatable ones, with placeholders substituted. Baseline and undo
+ *   files are not run.
+ *
+ * The server must hold no table, view or column of its own: the replay
+ * refuses a server that does, since it cannot tell what the migrations made.
+ * It runs the migrations' SQL against the server `--env` binds, so point it
+ * at a throwaway server or emulator.
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { join, relative, resolve } from "node:path";
+import type { CommandGroupContext } from "@intentius/chant/cli/command-group";
+import { LINK_READERS, type KindReadMigration, type KindReaderInput } from "@intentius/chant/workspace/kind-link-readers";
+import { migrationDigest, REPLAY_FILE, replayFiles, type ReplaySnapshot } from "@intentius/chant/workspace/migration-dirs";
+import { findFlywayConfig, readFlywayConfig, substitute } from "@intentius/chant/workspace/flyway-layout";
+import { workingTree, type WorkspaceTree } from "@intentius/chant/workspace/tree";
+
+export const REPLAY_KINDS = ["goose", "golang-migrate", "clickhouse-migrations", "flyway"] as const;
+export type ReplayKind = (typeof REPLAY_KINDS)[number];
+
+const READER = { goose: "goose-dir", "golang-migrate": "golang-migrate-dir", "clickhouse-migrations": "clickhouse-migrations-dir", flyway: "flyway-dir" } as const;
+
+/** One server the replay runs on. */
+export interface ReplayDb {
+  dialect: "postgres" | "clickhouse";
+  run(sql: string): Promise<void>;
+  rows(sql: string): Promise<Record<string, unknown>[]>;
+  close(): Promise<void>;
+}
+
+/**
+ * Split SQL text into statements at `;`. Single-quoted strings, quoted
+ * identifiers, `--` and `/* *\/` comments and Postgres dollar-quoted bodies
+ * hold semicolons without ending the statement. A statement of only
+ * comments is dropped.
+ */
+export function splitStatements(text: string): string[] {
+  const out: string[] = [];
+  let start = 0;
+  let i = 0;
+  const flush = (end: number) => {
+    const s = text.slice(start, end).trim();
+    if (s !== "" && s.replace(/--[^\n]*/g, "").replace(/\/\*[\s\S]*?\*\//g, "").trim() !== "") out.push(s);
+    start = end + 1;
+  };
+  while (i < text.length) {
+    const c = text[i]!;
+    const two = text.slice(i, i + 2);
+    if (two === "--") {
+      const nl = text.indexOf("\n", i);
+      i = nl < 0 ? text.length : nl + 1;
+    } else if (two === "/*") {
+      const end = text.indexOf("*/", i + 2);
+      i = end < 0 ? text.length : end + 2;
+    } else if (c === "'" || c === '"' || c === "`") {
+      i++;
+      while (i < text.length) {
+        if (text[i] === "\\" && c !== '"') i += 2;
+        else if (text[i] === c) {
+          if (text[i + 1] === c) i += 2;
+          else break;
+        } else i++;
+      }
+      i++;
+    } else if (c === "$") {
+      const tag = /^\$[A-Za-z_]*\$/.exec(text.slice(i))?.[0];
+      if (tag) {
+        const end = text.indexOf(tag, i + tag.length);
+        i = end < 0 ? text.length : end + tag.length;
+      } else i++;
+    } else if (c === ";") {
+      flush(i);
+      i++;
+    } else i++;
+  }
+  flush(text.length);
+  return out;
+}
+
+/** goose's Up statements: the lines between `-- +goose Up` and `-- +goose Down`; a StatementBegin to StatementEnd block is one statement. */
+export function gooseUpStatements(text: string): string[] {
+  const out: string[] = [];
+  let section: "none" | "up" | "down" = "none";
+  let block: string[] | undefined;
+  let plain: string[] = [];
+  const annotation = (l: string) => {
+    const t = l.trim();
+    return t.startsWith("--") && t.includes("+goose") ? t.replace("+goose", "").replace(/^--\s*/, "").trim().toLowerCase() : undefined;
+  };
+  const flushPlain = () => {
+    out.push(...splitStatements(plain.join("\n")));
+    plain = [];
+  };
+  for (const line of text.split(/\r?\n/)) {
+    const a = annotation(line);
+    if (a === "up") section = "up";
+    else if (a === "down") {
+      flushPlain();
+      section = "down";
+    } else if (a === "statementbegin" && section === "up") {
+      flushPlain();
+      block = [];
+    } else if (a === "statementend" && section === "up" && block) {
+      const s = block.join("\n").trim();
+      if (s !== "") out.push(s);
+      block = undefined;
+    } else if (section === "up" && a === undefined) (block ?? plain).push(line);
+  }
+  flushPlain();
+  return out;
+}
+
+export interface ReplayStep {
+  migration: KindReadMigration;
+  statements: string[];
+}
+
+/** The statements to run, per migration, in order. Throws when a Flyway placeholder has no value. */
+export function replaySteps(kind: ReplayKind, tree: WorkspaceTree, dir: string, fields: KindReaderInput["fields"] = null): { steps: ReplayStep[]; files: string[]; dialect: string } {
+  const read = LINK_READERS[READER[kind]]({ member: { name: "replay" } as never, dir, fields, tree });
+  const migrations = (read.migrations ?? []).filter((m) => m.type === "versioned" || m.type === "repeatable");
+  const config = kind === "flyway" ? readFlywayConfig(tree, findFlywayConfig(tree, dir), dir) : undefined;
+  const steps = migrations.map((migration) => {
+    const text = tree.read(migration.file);
+    const statements = kind === "goose" ? gooseUpStatements(text) : splitStatements(config ? substitute(text, migration.file, config) : text);
+    return { migration, statements };
+  });
+  const extra = config?.file !== undefined ? [config.file] : [];
+  return { steps, files: replayFiles(tree, read.migrations ?? [], extra), dialect: read.migrations?.[0]?.dialect ?? "postgres" };
+}
+
+/** Apply each migration's statements; a failure names the migration file and the statement. */
+export async function applySteps(db: ReplayDb, steps: readonly ReplayStep[]): Promise<void> {
+  for (const { migration, statements } of steps) {
+    for (const [i, sql] of statements.entries()) {
+      try {
+        await db.run(sql);
+      } catch (err) {
+        throw new Error(`${migration.file}: statement ${String(i + 1)} failed: ${(err as Error).message}\n${sql.slice(0, 200)}`);
+      }
+    }
+  }
+}
+
+const POSTGRES_CATALOG = `
+select n.nspname as schema, c.relname as relation, a.attname as col
+from pg_class c
+join pg_namespace n on n.oid = c.relnamespace
+left join pg_attribute a on a.attrelid = c.oid and a.attnum > 0 and not a.attisdropped
+where c.relkind in ('r', 'p', 'v', 'm', 'f')
+  and n.nspname not in ('pg_catalog', 'information_schema')
+  and n.nspname not like 'pg\\_toast%' and n.nspname not like 'pg\\_temp%'
+order by 1, 2, a.attnum`;
+
+const CLICKHOUSE_CATALOG = `
+select t.database as schema, t.name as relation, c.name as col
+from system.tables t
+left join system.columns c on c.database = t.database and c.table = t.name
+where t.database not in ('system', 'INFORMATION_SCHEMA', 'information_schema')
+  and t.is_temporary = 0
+order by t.database, t.name, c.position`;
+
+/** The tables, views and columns the server has, as workspace output names (#3931). */
+export async function readCatalogOutputs(db: ReplayDb): Promise<string[]> {
+  const rows = await db.rows(db.dialect === "postgres" ? POSTGRES_CATALOG : CLICKHOUSE_CATALOG);
+  const out: string[] = [];
+  const seen = new Set<string>();
+  for (const r of rows) {
+    const rel = `${String(r.schema)}.${String(r.relation)}`;
+    if (!seen.has(rel)) {
+      seen.add(rel);
+      out.push(rel);
+    }
+    if (r.col !== null && r.col !== undefined) out.push(`${rel}.${String(r.col)}`);
+  }
+  return out;
+}
+
+/** Replay a directory into `db`, which must hold nothing, and return the snapshot to record. */
+export async function replayDirectory(db: ReplayDb, kind: ReplayKind, tree: WorkspaceTree, dir: string, fields: KindReaderInput["fields"] = null): Promise<ReplaySnapshot> {
+  const before = await readCatalogOutputs(db);
+  if (before.length > 0) throw new Error(`the server already has ${before[0]!.split(".").slice(0, 2).join(".")}${before.length > 1 ? ` and ${String(before.length - 1)} more names` : ""}; replay needs a server with no tables or views of its own, so the tables it finds are the migrations' (use a throwaway server or emulator)`);
+  const { steps, files, dialect } = replaySteps(kind, tree, dir, fields);
+  if (dialect !== db.dialect) throw new Error(`the migrations are ${dialect} and the server is ${db.dialect}; set the member's dialect field or bind the matching server`);
+  await applySteps(db, steps);
+  return { schema: 1, dialect, digest: migrationDigest((f) => tree.read(f), files), outputs: await readCatalogOutputs(db) };
+}
+
+export const renderSnapshot = (s: ReplaySnapshot): string => `${JSON.stringify(s, null, 2)}\n`;
+
+const USAGE = "usage: chant sql replay <dir> --kind goose|golang-migrate|clickhouse-migrations|flyway --env <env> [--migrations-dir <sub>] [--dialect postgres|clickhouse] [--check]";
+
+/** Open the server `--env` binds, of the migrations' dialect. */
+async function openBound(dialect: "postgres" | "clickhouse", environment: string): Promise<ReplayDb> {
+  if (dialect === "postgres") {
+    const { bindPostgres } = await import("./postgres/live/bind");
+    const { client } = await bindPostgres({ environment });
+    return { dialect, run: async (sql) => void (await client.query(sql)), rows: (sql) => client.query(sql), close: () => client.end() };
+  }
+  const { bindClickHouse } = await import("./clickhouse/live/bind");
+  const { clickhouseQuery } = await import("./clickhouse/http");
+  const target = await bindClickHouse({ environment });
+  return { dialect, run: async (sql) => void (await clickhouseQuery(target.endpoint, sql)), rows: (sql) => clickhouseQuery(target.endpoint, sql), async close() {} };
+}
+
+export async function runReplay(ctx: CommandGroupContext): Promise<number> {
+  const args = ctx.rawArgs;
+  let kind: string | undefined;
+  let env: string | undefined;
+  let sub: string | undefined;
+  let dialect: string | undefined;
+  const check = args.includes("--check");
+  const positional: string[] = [];
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i]!;
+    if (a === "--kind") kind = args[++i];
+    else if (a === "--env") env = args[++i];
+    else if (a === "--migrations-dir") sub = args[++i];
+    else if (a === "--dialect") dialect = args[++i];
+    else if (a === "--check") continue;
+    else if (a.startsWith("--")) {
+      console.error(`unknown argument ${a}\n${USAGE}`);
+      return 1;
+    } else positional.push(a);
+  }
+  if (positional.length !== 1 || !kind || !(REPLAY_KINDS as readonly string[]).includes(kind) || !env) {
+    console.error(USAGE);
+    return 1;
+  }
+  const abs = resolve(process.cwd(), positional[0]!);
+  const tree = workingTree(abs);
+  const fields = { ...(sub ? { migrationsDir: sub } : {}), ...(dialect ? { dialect } : {}) } as KindReaderInput["fields"];
+  const { steps, dialect: found } = replaySteps(kind as ReplayKind, tree, "", fields);
+  if (steps.length === 0) {
+    console.error(`chant sql replay: ${relative(process.cwd(), abs) || "."} lists no ${kind} migrations`);
+    return 1;
+  }
+  const db = await openBound(found as "postgres" | "clickhouse", env);
+  let snapshot: ReplaySnapshot;
+  try {
+    snapshot = await replayDirectory(db, kind as ReplayKind, tree, "", fields);
+  } catch (err) {
+    console.error(`chant sql replay: ${(err as Error).message}`);
+    return 1;
+  } finally {
+    await db.close();
+  }
+  const file = join(abs, REPLAY_FILE);
+  const text = renderSnapshot(snapshot);
+  const current = (() => {
+    try {
+      return readFileSync(file, "utf8");
+    } catch {
+      return undefined;
+    }
+  })();
+  if (check) {
+    if (current === text) return 0;
+    console.error(`chant sql replay --check: ${relative(process.cwd(), file)} ${current === undefined ? "is missing" : "differs from a replay of the migrations"}; run chant sql replay and commit it`);
+    return 1;
+  }
+  writeFileSync(file, text);
+  console.log(`recorded ${String(snapshot.outputs.length)} names from ${String(steps.length)} migrations in ${relative(process.cwd(), file)}`);
+  return 0;
+}

package/src/op/__snapshots__/watch-pipeline.test.ts.snap

@@ -20,7 +20,7 @@ jobs:
       CLICKHOUSE_READER_USER: chant_reader
       CLICKHOUSE_READER_PASSWORD: '\${{ secrets.CLICKHOUSE_READER_PASSWORD }}'
     steps:
-      - uses: https://code.forgejo.org/actions/checkout@v7
+      - uses: https://code.forgejo.org/actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - run: npm ci
       - run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
       - run: chant run schema-watch
@@ -52,7 +52,7 @@ jobs:
       CLICKHOUSE_READER_USER: chant_reader
       CLICKHOUSE_READER_PASSWORD: '\${{ secrets.CLICKHOUSE_READER_PASSWORD }}'
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - run: npm ci
       - run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
       - run: chant run schema-watch

package/src/op/activities/postgres-migration.ts

@@ -109,6 +109,8 @@ async function bound<T>(args: PostgresMigrationArgs, signal: AbortSignal | undef
       objects,
       column: args.column,
       ...(args.using !== undefined ? { using: args.using } : {}),
+      ...(args.reverse !== undefined ? { reverse: args.reverse } : {}),
+      ...(args.defaultAtContract ? { defaultAtContract: true } : {}),
       ...(marker ? { marker } : {}),
       ...(major !== undefined ? { major } : {}),
       batchSize: args.batchSize ?? 1000,
@@ -158,6 +160,8 @@ export async function postgresMigrationPlan(args: PostgresMigrationArgs, signal?
       column: run.column,
       ...(run.marker ? { marker: run.marker } : {}),
       ...(run.using !== undefined ? { using: run.using } : {}),
+      ...(run.reverse !== undefined ? { reverse: run.reverse } : {}),
+      ...(run.defaultAtContract ? { defaultAtContract: true } : {}),
       ...(run.major !== undefined ? { major: run.major } : {}),
     });
     const n = o.names;

package/src/op/activity-contracts.ts

@@ -124,6 +124,8 @@ const migrationArgs = z.strictObject({
   buildPath: z.string(),
   environment: z.string().optional(),
   using: z.string().optional(),
+  reverse: z.string().optional(),
+  defaultAtContract: z.boolean().optional(),
   retain: z.string().optional(),
   batchSize: z.number().int().positive().optional(),
   replicationLag: z.union([z.literal(false), z.strictObject({ max: z.string().optional(), wait: z.string().optional() })]).optional(),

package/src/plugin.ts

@@ -18,7 +18,7 @@ import { completions } from "./lsp/completions";
 import { hover } from "./lsp/hover";
 import { sqlMcpResources, sqlMcpTools } from "./mcp";
 import { sqlConfigSchema } from "./config";
-import { SqlFileParser, dialectOption } from "./import-parser";
+import { SqlFileParser, dialectOption, sourceOption } from "./import-parser";
 import { sqlTemplateGenerator } from "./import-generator";
 import { sqlCommands } from "./clickhouse/plan/commands";
 import { sqlDeepNormalizationHooks } from "./clickhouse/plan/deep";
@@ -29,6 +29,7 @@ import { CLICKHOUSE_EMULATOR } from "./op/activities/clickhouse-emulator";
 import { POSTGRES_EMULATOR } from "./op/activities/postgres-emulator";
 import { sqlSkills } from "./skill-defs";
 import { detectTemplate } from "./detect";
+import { looksLikePostgres } from "./migrate/from-postgres/detect";
 import { sqlWorkspaceOutputs } from "./workspace-outputs";
 import { CDC_TEMPLATE, DEFAULT_TEMPLATE, EVENTS_TEMPLATE, POSTGRES_EVENTS_TEMPLATE, POSTGRES_TEMPLATE, POSTGRES_TENANT_TEMPLATE } from "./init-templates";
 import { compositeCatalog } from "./composites/catalog";
@@ -113,6 +114,11 @@ export const sqlPlugin: LexiconPlugin = {
       { name: "aggregate", isTag: true, description: "A Postgres CREATE AGGREGATE, parsed into an aggregate entity identified by its argument types" },
       { name: "trigger", isTag: true, description: "A Postgres CREATE TRIGGER, parsed into a trigger entity on its table" },
       { name: "eventTrigger", isTag: true, description: "A Postgres CREATE EVENT TRIGGER, parsed into the database's event trigger entity" },
+      { name: "operator", isTag: true, description: "A Postgres CREATE OPERATOR, parsed into an operator entity identified by its name and operand types" },
+      { name: "statistics", isTag: true, description: "A Postgres CREATE STATISTICS, parsed into an extended statistics entity on its table" },
+      { name: "foreignDataWrapper", isTag: true, description: "A Postgres CREATE FOREIGN DATA WRAPPER, parsed into the database's wrapper entity" },
+      { name: "foreignServer", isTag: true, description: "A Postgres CREATE SERVER, parsed into the database's foreign server entity (user mappings are not declared)" },
+      { name: "publication", isTag: true, description: "A Postgres CREATE PUBLICATION, parsed into the database's publication entity" },
       { name: "collation", isTag: true, description: "A Postgres CREATE COLLATION, parsed into a collation entity" },
       { name: "cast", isTag: true, description: "A Postgres CREATE CAST, parsed into a cast entity named by its two types" },
       { name: "policy", isTag: true, description: "A Postgres CREATE POLICY, parsed into a row-level security policy on its table, or a ClickHouse CREATE ROW POLICY" },
@@ -166,7 +172,31 @@ export const sqlPlugin: LexiconPlugin = {
 
   /** `chant import schema.sql`: a file of Postgres or ClickHouse DDL (`./import-parser.ts`). */
   templateParser(options) {
-    return new SqlFileParser(dialectOption(options), { skipUndeclarable: options?.skipUndeclarable === true });
+    const from = sourceOption(options);
+    return new SqlFileParser(dialectOption(options), { skipUndeclarable: options?.skipUndeclarable === true, ...(from ? { from } : {}) });
+  },
+
+  /** `chant import --validate` (#4398): create the imported DDL on the pinned ClickHouse emulator. */
+  async validateImport(ir) {
+    const { validateOnEmulator } = await import("./sqlserver/validate");
+    return validateOnEmulator(ir);
+  },
+
+  /**
+   * `chant migrate --from sql/postgres --to sql/clickhouse` (#4399): the
+   * Postgres tables a CDC pipeline mirrors, as ClickHouse tables or
+   * `CdcMirror` calls (`./migrate/from-postgres/index.ts`).
+   */
+  migrationSource(from, to) {
+    if (from !== "sql/postgres" || (to !== undefined && to !== "sql/clickhouse")) return undefined;
+    return {
+      detect: looksLikePostgres,
+      directory: true,
+      async transform(content, opts) {
+        const { transform } = await import("./migrate/from-postgres/index");
+        return transform(content, opts);
+      },
+    };
   },
 
   parserOptions() {
@@ -176,6 +206,11 @@ export const sqlPlugin: LexiconPlugin = {
         type: "string" as const,
         description: "the DDL's dialect, clickhouse or postgres; read off its statements when omitted",
       },
+      {
+        name: "from",
+        type: "string" as const,
+        description: "read the file as a SQL Server schema (sqlserver) and translate it to ClickHouse; each judgment call is a needs-review diagnostic for --report and --strict",
+      },
       {
         name: "skipUndeclarable",
         type: "boolean" as const,

package/src/postgres/apply/apply.ts

@@ -553,6 +553,9 @@ async function applyAccess(
   }
 }
 
+/** The kinds that belong to the database, not a schema. */
+const UNPLACED_KINDS: ReadonlySet<CanonicalKind> = new Set(["eventTrigger", "foreignDataWrapper", "foreignServer", "publication"]);
+
 /** The address a live object is keyed by in the diff. */
 function liveCanonicalAddress(o: LivePgObject): { kind: CanonicalKind; schema?: string; name: string; signature?: string } {
   const kind = (
@@ -571,6 +574,11 @@ function liveCanonicalAddress(o: LivePgObject): { kind: CanonicalKind; schema?:
       "Postgres::Collation": "collation",
       "Postgres::Cast": "cast",
       "Postgres::EventTrigger": "eventTrigger",
+      "Postgres::Operator": "operator",
+      "Postgres::Statistics": "statistics",
+      "Postgres::ForeignDataWrapper": "foreignDataWrapper",
+      "Postgres::ForeignServer": "foreignServer",
+      "Postgres::Publication": "publication",
       "Postgres::Extension": "extension",
       "Postgres::Function": "function",
       "Postgres::Procedure": "procedure",
@@ -583,7 +591,7 @@ function liveCanonicalAddress(o: LivePgObject): { kind: CanonicalKind; schema?:
   )[o.type];
   return {
     kind,
-    ...(o.schema !== undefined && kind !== "schema" && kind !== "extension" && kind !== "role" && kind !== "cast" && kind !== "eventTrigger" ? { schema: o.schema } : {}),
+    ...(o.schema !== undefined && kind !== "schema" && kind !== "extension" && kind !== "role" && kind !== "cast" && !UNPLACED_KINDS.has(kind) ? { schema: o.schema } : {}),
     name: o.name,
     ...(o.signature !== undefined ? { signature: o.signature } : {}),
   };