@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/migration-replay.e2e.test.ts

@@ -0,0 +1,155 @@
+/**
+ * Goose and golang-migrate directories replayed into a Postgres emulator
+ * (#4400): the replay records the tables and columns, a dbt member links to
+ * one of them, `chant workspace check` passes, and a migration that drops the
+ * linked column fails it after the next replay. Also the `--check` of the
+ * recorded file.
+ *
+ * Needs Docker; skips cleanly without it. One throwaway `postgres` container,
+ * a fresh database for each replay. Not part of the unit suite.
+ */
+
+import { spawnSync } from "node:child_process";
+import { readFileSync, symlinkSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { cleanScratch, declaration, REPO, repo } from "../../../packages/core/src/workspace/__fixtures__/contract-repo";
+import { workingTree } from "@intentius/chant/workspace/tree";
+import { REPLAY_FILE } from "@intentius/chant/workspace/migration-dirs";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "./postgres/testing/server";
+import { renderSnapshot, replayDirectory, type ReplayDb, type ReplayKind } from "./migration-replay";
+
+const enabled = await dockerAvailable();
+const TIMEOUT = 600_000;
+const LEXICON = "@intentius/chant-lexicon-sql";
+const VERSION = (JSON.parse(readFileSync(join(REPO, "lexicons", "sql", "package.json"), "utf-8")) as { version: string }).version;
+let server: TestPostgres | undefined;
+let databases = 0;
+
+beforeAll(async () => {
+  if (enabled) server = await startTestPostgres();
+}, TIMEOUT);
+
+afterAll(async () => {
+  await server?.stop();
+  cleanScratch();
+});
+
+function chant(cwd: string, ...argv: string[]) {
+  return spawnSync(process.execPath, ["--import", pathToFileURL(join(REPO, "node_modules/tsx/dist/loader.mjs")).href, join(REPO, "packages/core/src/cli/main.ts"), ...argv], {
+    cwd,
+    encoding: "utf-8",
+    timeout: TIMEOUT,
+    env: { ...process.env, NO_COLOR: "1", TSX_DISABLE_CACHE: "1" },
+  });
+}
+
+/** An empty database on the emulator, as a {@link ReplayDb}. */
+async function emptyDb(): Promise<ReplayDb> {
+  const admin = await server!.connect();
+  const name = `replay_${String((databases += 1))}`;
+  await admin.query(`CREATE DATABASE ${name}`);
+  await admin.end();
+  const client = await server!.connect(name);
+  return { dialect: "postgres", run: async (sql) => void (await client.query(sql)), rows: (sql) => client.query(sql), close: () => client.end() };
+}
+
+const SOURCES = "version: 2\n\nsources:\n  - name: shop\n    schema: public\n    tables:\n      - name: orders\n        columns:\n          - name: id\n          - name: customer_id\n";
+
+const FIXTURES: Record<"goose" | "golang-migrate", { first: Record<string, string>; drop: Record<string, string> }> = {
+  goose: {
+    first: {
+      "db/00001_orders.sql": "-- +goose Up\nCREATE TABLE orders (id bigint PRIMARY KEY, customer_id bigint NOT NULL);\n-- +goose Down\nDROP TABLE orders;\n",
+    },
+    drop: { "db/00002_drop_customer.sql": "-- +goose Up\nALTER TABLE orders DROP COLUMN customer_id;\n-- +goose Down\nALTER TABLE orders ADD COLUMN customer_id bigint;\n" },
+  },
+  "golang-migrate": {
+    first: {
+      "db/1_orders.up.sql": "CREATE TABLE orders (id bigint PRIMARY KEY, customer_id bigint NOT NULL);\n",
+      "db/1_orders.down.sql": "DROP TABLE orders;\n",
+    },
+    drop: { "db/2_drop_customer.up.sql": "ALTER TABLE orders DROP COLUMN customer_id;\n", "db/2_drop_customer.down.sql": "ALTER TABLE orders ADD COLUMN customer_id bigint;\n" },
+  },
+};
+
+async function workspace(kind: "goose" | "golang-migrate", extra: Record<string, string>): Promise<string> {
+  const root = repo({
+    "chant.workspace.json": declaration(
+      [
+        { name: "db", dir: "db", kind },
+        { name: "analytics", dir: "analytics", kind: "dbt", fields: { sqlMember: "db" } },
+      ],
+      { pins: [{ package: LEXICON, version: VERSION }] },
+    ),
+    "analytics/dbt_project.yml": "name: analytics\nversion: '1.0'\nprofile: analytics\n",
+    "analytics/models/staging/sources.yml": SOURCES,
+    ".gitignore": "node_modules\n",
+    ...FIXTURES[kind].first,
+    ...extra,
+  });
+  symlinkSync(join(REPO, "node_modules"), join(root, "node_modules"), "dir");
+  return root;
+}
+
+/** Replay `root/db` into a fresh database and write the snapshot beside the migrations. */
+async function record(root: string, kind: ReplayKind): Promise<void> {
+  const db = await emptyDb();
+  try {
+    const snapshot = await replayDirectory(db, kind, workingTree(join(root, "db")), "");
+    writeFileSync(join(root, "db", REPLAY_FILE), renderSnapshot(snapshot));
+  } finally {
+    await db.close();
+  }
+}
+
+describe.skipIf(!enabled)("migration directories replayed into the emulator (#4400)", () => {
+  for (const kind of ["goose", "golang-migrate"] as const) {
+    test(
+      `${kind}: the replayed columns are outputs a dbt member links to, and a migration that drops a linked column fails workspace check`,
+      async () => {
+        const ok = await workspace(kind, {});
+        await record(ok, kind);
+        expect(JSON.parse(readFileSync(join(ok, "db", REPLAY_FILE), "utf-8")).outputs).toEqual(["public.orders", "public.orders.id", "public.orders.customer_id"]);
+        const passed = chant(ok, "workspace", "check");
+        expect(passed.status, passed.stdout + passed.stderr).toBe(0);
+
+        const dropped = await workspace(kind, FIXTURES[kind].drop);
+        await record(dropped, kind);
+        const failed = chant(dropped, "workspace", "check");
+        expect(failed.status).toBe(1);
+        const text = failed.stdout + failed.stderr;
+        expect(text).toContain("analytics/models/staging/sources.yml:10");
+        expect(text).toContain("WSP093");
+        expect(text).toMatch(/link to db output public\.orders\.customer_id .*does not resolve/);
+      },
+      TIMEOUT,
+    );
+
+    test(
+      `${kind}: a migration added after the replay leaves the outputs unresolved until the replay runs again`,
+      async () => {
+        const root = await workspace(kind, {});
+        await record(root, kind);
+        for (const [path, text] of Object.entries(FIXTURES[kind].drop)) writeFileSync(join(root, path), text);
+        const stale = chant(root, "workspace", "check");
+        expect(stale.stdout + stale.stderr).toContain("changed since chant.replay.json was recorded");
+      },
+      TIMEOUT,
+    );
+  }
+
+  test(
+    "a migration that fails stops the replay and names its file",
+    async () => {
+      const root = await workspace("golang-migrate", { "db/2_bad.up.sql": "ALTER TABLE nowhere ADD COLUMN x int;\n" });
+      const db = await emptyDb();
+      try {
+        await expect(replayDirectory(db, "golang-migrate", workingTree(join(root, "db")), "")).rejects.toThrow(/2_bad\.up\.sql: statement 1 failed/);
+      } finally {
+        await db.close();
+      }
+    },
+    TIMEOUT,
+  );
+});

package/src/migration-replay.test.ts

@@ -0,0 +1,173 @@
+/**
+ * Replaying a migration directory (#4400), against a fake server: the
+ * statements it sends, the catalog names it records, and the round trip into
+ * the kind reader's outputs. The emulator runs in `migration-replay.e2e.test.ts`.
+ */
+
+import { describe, expect, test } from "vitest";
+import { LINK_READERS } from "@intentius/chant/workspace/kind-link-readers";
+import { REPLAY_FILE } from "@intentius/chant/workspace/migration-dirs";
+import type { WorkspaceTree } from "@intentius/chant/workspace/tree";
+import { applySteps, gooseUpStatements, readCatalogOutputs, renderSnapshot, replayDirectory, replaySteps, splitStatements, type ReplayDb } from "./migration-replay";
+
+function memoryTree(files: Record<string, string>): WorkspaceTree {
+  const dirs = new Set([""]);
+  for (const f of Object.keys(files)) {
+    const parts = f.split("/");
+    for (let i = 1; i < parts.length; i++) dirs.add(parts.slice(0, i).join("/"));
+  }
+  return {
+    label: "",
+    stat: (p) => (p in files ? "file" : dirs.has(p) ? "dir" : undefined),
+    list(p) {
+      if (!dirs.has(p)) return undefined;
+      const prefix = p === "" ? "" : `${p}/`;
+      const seen = new Map<string, "file" | "dir">();
+      for (const f of Object.keys(files)) {
+        if (!f.startsWith(prefix)) continue;
+        const rest = f.slice(prefix.length);
+        seen.set(rest.split("/")[0]!, rest.includes("/") ? "dir" : "file");
+      }
+      return [...seen].map(([name, type]) => ({ name, type }));
+    },
+    read(p) {
+      if (!(p in files)) throw new Error(`no such file ${p}`);
+      return files[p]!;
+    },
+  };
+}
+
+/** A server that records what it is sent and answers the catalog query with `catalog`. */
+function fakeDb(dialect: "postgres" | "clickhouse", catalog: () => Record<string, unknown>[], failOn?: RegExp): ReplayDb & { sent: string[] } {
+  const sent: string[] = [];
+  return {
+    dialect,
+    sent,
+    async run(sql) {
+      if (failOn?.test(sql)) throw new Error("relation does not exist");
+      sent.push(sql);
+    },
+    rows: async () => catalog(),
+    close: async () => {},
+  };
+}
+
+describe("splitting statements", () => {
+  test("semicolons inside strings, identifiers, comments and dollar quotes do not end a statement", () => {
+    const sql = `create table a (x text default 'a;b');\n-- c; d\ncreate function f() returns int as $$ begin return 1; end $$ language plpgsql;\n/* x; */ select "a;b" from a;\n;;`;
+    expect(splitStatements(sql)).toEqual([
+      "create table a (x text default 'a;b')",
+      "-- c; d\ncreate function f() returns int as $$ begin return 1; end $$ language plpgsql",
+      '/* x; */ select "a;b" from a',
+    ]);
+  });
+
+  test("a final statement without a semicolon is kept, and a comment alone is dropped", () => {
+    expect(splitStatements("select 1;\n-- done\n")).toEqual(["select 1"]);
+    expect(splitStatements("select 1;\nselect 2")).toEqual(["select 1", "select 2"]);
+  });
+});
+
+describe("goose Up statements", () => {
+  test("only the Up section runs, and StatementBegin to StatementEnd is one statement", () => {
+    const text = [
+      "-- +goose Up",
+      "create table a (id int);",
+      "-- +goose StatementBegin",
+      "create function f() returns int as $$ begin return 1; end; $$ language plpgsql;",
+      "select 1;",
+      "-- +goose StatementEnd",
+      "alter table a add c int;",
+      "-- +goose Down",
+      "drop table a;",
+    ].join("\n");
+    expect(gooseUpStatements(text)).toEqual(["create table a (id int)", "create function f() returns int as $$ begin return 1; end; $$ language plpgsql;\nselect 1;", "alter table a add c int"]);
+  });
+});
+
+describe("the steps per kind", () => {
+  test("golang-migrate runs the up files in version order, never the down files", () => {
+    const { steps } = replaySteps("golang-migrate", memoryTree({ "10_b.up.sql": "create table b (id int);", "2_a.up.sql": "create table a (id int);", "2_a.down.sql": "drop table a;" }), "");
+    expect(steps.map((s) => [s.migration.file, s.statements])).toEqual([["2_a.up.sql", ["create table a (id int)"]], ["10_b.up.sql", ["create table b (id int)"]]]);
+  });
+
+  test("Flyway substitutes placeholders, runs versioned then repeatable, and skips undo and callbacks", () => {
+    const { steps } = replaySteps(
+      "flyway",
+      memoryTree({
+        "flyway.conf": "flyway.placeholders.t=orders\n",
+        "V1__a.sql": "create table ${t} (id int);",
+        "R__v.sql": "create or replace view v as select 1;",
+        "U1__a.sql": "drop table orders;",
+        "beforeMigrate.sql": "select 1;",
+      }),
+      "",
+    );
+    expect(steps.map((s) => s.statements)).toEqual([["create table orders (id int)"], ["create or replace view v as select 1"]]);
+  });
+
+  test("a Flyway placeholder with no value fails naming the file and line", () => {
+    expect(() => replaySteps("flyway", memoryTree({ "V1__a.sql": "select 1;\ncreate table ${nope} (id int);" }), "")).toThrow(/V1__a\.sql:2: no value for the placeholder \$\{nope\}/);
+  });
+});
+
+describe("replaying", () => {
+  test("a failing statement names the migration file", async () => {
+    const { steps } = replaySteps("golang-migrate", memoryTree({ "1_a.up.sql": "create table a (id int);", "2_b.up.sql": "alter table missing add c int;" }), "");
+    await expect(applySteps(fakeDb("postgres", () => [], /missing/), steps)).rejects.toThrow(/2_b\.up\.sql: statement 1 failed: relation does not exist/);
+  });
+
+  test("the catalog rows become <schema>.<table> and <schema>.<table>.<column>", async () => {
+    const db = fakeDb("clickhouse", () => [
+      { schema: "app", relation: "events", col: "ts" },
+      { schema: "app", relation: "events", col: "kind" },
+      { schema: "app", relation: "empty", col: null },
+    ]);
+    expect(await readCatalogOutputs(db)).toEqual(["app.events", "app.events.ts", "app.events.kind", "app.empty"]);
+  });
+
+  test("a server that already has tables is refused", async () => {
+    await expect(replayDirectory(fakeDb("postgres", () => [{ schema: "public", relation: "old", col: "id" }]), "goose", memoryTree({}), "")).rejects.toThrow(/already has public\.old/);
+  });
+
+  test("a dialect mismatch is refused before anything runs", async () => {
+    const db = fakeDb("clickhouse", () => []);
+    await expect(replayDirectory(db, "goose", memoryTree({ "1_a.sql": "-- +goose Up\nselect 1;" }), "")).rejects.toThrow(/migrations are postgres and the server is clickhouse/);
+    expect(db.sent).toEqual([]);
+  });
+
+  test("the snapshot a replay writes is what the kind reader reads back as outputs, until a migration changes", async () => {
+    const files: Record<string, string> = {
+      "1_users.sql": "-- +goose Up\ncreate table users (id int, email text);\n-- +goose Down\ndrop table users;\n",
+      "2_orders.sql": "-- +goose Up\ncreate table orders (id int, user_id int);\n",
+    };
+    let created = false;
+    const db = fakeDb("postgres", () =>
+      created
+        ? [
+            { schema: "public", relation: "users", col: "id" },
+            { schema: "public", relation: "users", col: "email" },
+            { schema: "public", relation: "orders", col: "id" },
+            { schema: "public", relation: "orders", col: "user_id" },
+          ]
+        : [],
+    );
+    const original = db.run;
+    db.run = async (sql) => {
+      await original(sql);
+      created = true;
+    };
+    const snapshot = await replayDirectory(db, "goose", memoryTree(files), "");
+    expect(db.sent).toEqual(["create table users (id int, email text)", "create table orders (id int, user_id int)"]);
+    expect(snapshot.outputs).toEqual(["public.users", "public.users.id", "public.users.email", "public.orders", "public.orders.id", "public.orders.user_id"]);
+
+    const withSnapshot = { ...files, [REPLAY_FILE]: renderSnapshot(snapshot) };
+    const read = (f: Record<string, string>) => LINK_READERS["goose-dir"]({ member: { name: "db" } as never, dir: "", fields: null, tree: memoryTree(f) });
+    expect(read(withSnapshot).outputs!.map((o) => o.name)).toEqual(snapshot.outputs);
+    expect(read(withSnapshot).gaps).toEqual([]);
+
+    const dropped = read({ ...withSnapshot, "3_drop.sql": "-- +goose Up\nalter table orders drop column user_id;\n" });
+    expect(dropped.outputs).toBeUndefined();
+    expect(dropped.gaps[0]).toMatchObject({ outputs: true, message: expect.stringContaining("changed since") });
+  });
+});

package/src/migration-replay.ts

@@ -0,0 +1,289 @@
+/**
+ * Replay a migration directory into an empty server and read the catalog
+ * (#4400). The workspace kinds for goose, golang-migrate,
+ * clickhouse-migrations and Flyway list a directory's migrations without
+ * running anything (`packages/core/src/workspace/migration-dirs.ts`); the
+ * tables and columns they produce come from here.
+ *
+ *     chant sql replay <dir> --kind <kind> --env <env> [--dir-field migrationsDir=<sub>] [--check]
+ *
+ * The replay applies each migration's up statements in the order the kind's
+ * reader lists them, then reads the tables, views and columns the server has
+ * and writes them as `chant.replay.json` beside the migrations, named by the
+ * contract of #3931: `<schema>.<table>` and `<schema>.<table>.<column>` for
+ * Postgres, `<database>.<table>` and `<database>.<table>.<column>` for
+ * ClickHouse. The file also holds a digest of the files it ran, so the
+ * reader knows when a migration changed after it. `--check` replays and
+ * compares instead of writing, exiting 1 when the file differs, the way
+ * `chant sql types --check` does.
+ *
+ * What it runs per kind:
+ * - goose: the `-- +goose Up` section of each file; `StatementBegin` to
+ *   `StatementEnd` is one statement.
+ * - golang-migrate and clickhouse-migrations: the whole up file.
+ * - Flyway: the SQL files of the locations, versioned in version order and
+ *   then repeatable ones, with placeholders substituted. Baseline and undo
+ *   files are not run.
+ *
+ * The server must hold no table, view or column of its own: the replay
+ * refuses a server that does, since it cannot tell what the migrations made.
+ * It runs the migrations' SQL against the server `--env` binds, so point it
+ * at a throwaway server or emulator.
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { join, relative, resolve } from "node:path";
+import type { CommandGroupContext } from "@intentius/chant/cli/command-group";
+import { LINK_READERS, type KindReadMigration, type KindReaderInput } from "@intentius/chant/workspace/kind-link-readers";
+import { migrationDigest, REPLAY_FILE, replayFiles, type ReplaySnapshot } from "@intentius/chant/workspace/migration-dirs";
+import { findFlywayConfig, readFlywayConfig, substitute } from "@intentius/chant/workspace/flyway-layout";
+import { workingTree, type WorkspaceTree } from "@intentius/chant/workspace/tree";
+
+export const REPLAY_KINDS = ["goose", "golang-migrate", "clickhouse-migrations", "flyway"] as const;
+export type ReplayKind = (typeof REPLAY_KINDS)[number];
+
+const READER = { goose: "goose-dir", "golang-migrate": "golang-migrate-dir", "clickhouse-migrations": "clickhouse-migrations-dir", flyway: "flyway-dir" } as const;
+
+/** One server the replay runs on. */
+export interface ReplayDb {
+  dialect: "postgres" | "clickhouse";
+  run(sql: string): Promise<void>;
+  rows(sql: string): Promise<Record<string, unknown>[]>;
+  close(): Promise<void>;
+}
+
+/**
+ * Split SQL text into statements at `;`. Single-quoted strings, quoted
+ * identifiers, `--` and `/* *\/` comments and Postgres dollar-quoted bodies
+ * hold semicolons without ending the statement. A statement of only
+ * comments is dropped.
+ */
+export function splitStatements(text: string): string[] {
+  const out: string[] = [];
+  let start = 0;
+  let i = 0;
+  const flush = (end: number) => {
+    const s = text.slice(start, end).trim();
+    if (s !== "" && s.replace(/--[^\n]*/g, "").replace(/\/\*[\s\S]*?\*\//g, "").trim() !== "") out.push(s);
+    start = end + 1;
+  };
+  while (i < text.length) {
+    const c = text[i]!;
+    const two = text.slice(i, i + 2);
+    if (two === "--") {
+      const nl = text.indexOf("\n", i);
+      i = nl < 0 ? text.length : nl + 1;
+    } else if (two === "/*") {
+      const end = text.indexOf("*/", i + 2);
+      i = end < 0 ? text.length : end + 2;
+    } else if (c === "'" || c === '"' || c === "`") {
+      i++;
+      while (i < text.length) {
+        if (text[i] === "\\" && c !== '"') i += 2;
+        else if (text[i] === c) {
+          if (text[i + 1] === c) i += 2;
+          else break;
+        } else i++;
+      }
+      i++;
+    } else if (c === "$") {
+      const tag = /^\$[A-Za-z_]*\$/.exec(text.slice(i))?.[0];
+      if (tag) {
+        const end = text.indexOf(tag, i + tag.length);
+        i = end < 0 ? text.length : end + tag.length;
+      } else i++;
+    } else if (c === ";") {
+      flush(i);
+      i++;
+    } else i++;
+  }
+  flush(text.length);
+  return out;
+}
+
+/** goose's Up statements: the lines between `-- +goose Up` and `-- +goose Down`; a StatementBegin to StatementEnd block is one statement. */
+export function gooseUpStatements(text: string): string[] {
+  const out: string[] = [];
+  let section: "none" | "up" | "down" = "none";
+  let block: string[] | undefined;
+  let plain: string[] = [];
+  const annotation = (l: string) => {
+    const t = l.trim();
+    return t.startsWith("--") && t.includes("+goose") ? t.replace("+goose", "").replace(/^--\s*/, "").trim().toLowerCase() : undefined;
+  };
+  const flushPlain = () => {
+    out.push(...splitStatements(plain.join("\n")));
+    plain = [];
+  };
+  for (const line of text.split(/\r?\n/)) {
+    const a = annotation(line);
+    if (a === "up") section = "up";
+    else if (a === "down") {
+      flushPlain();
+      section = "down";
+    } else if (a === "statementbegin" && section === "up") {
+      flushPlain();
+      block = [];
+    } else if (a === "statementend" && section === "up" && block) {
+      const s = block.join("\n").trim();
+      if (s !== "") out.push(s);
+      block = undefined;
+    } else if (section === "up" && a === undefined) (block ?? plain).push(line);
+  }
+  flushPlain();
+  return out;
+}
+
+export interface ReplayStep {
+  migration: KindReadMigration;
+  statements: string[];
+}
+
+/** The statements to run, per migration, in order. Throws when a Flyway placeholder has no value. */
+export function replaySteps(kind: ReplayKind, tree: WorkspaceTree, dir: string, fields: KindReaderInput["fields"] = null): { steps: ReplayStep[]; files: string[]; dialect: string } {
+  const read = LINK_READERS[READER[kind]]({ member: { name: "replay" } as never, dir, fields, tree });
+  const migrations = (read.migrations ?? []).filter((m) => m.type === "versioned" || m.type === "repeatable");
+  const config = kind === "flyway" ? readFlywayConfig(tree, findFlywayConfig(tree, dir), dir) : undefined;
+  const steps = migrations.map((migration) => {
+    const text = tree.read(migration.file);
+    const statements = kind === "goose" ? gooseUpStatements(text) : splitStatements(config ? substitute(text, migration.file, config) : text);
+    return { migration, statements };
+  });
+  const extra = config?.file !== undefined ? [config.file] : [];
+  return { steps, files: replayFiles(tree, read.migrations ?? [], extra), dialect: read.migrations?.[0]?.dialect ?? "postgres" };
+}
+
+/** Apply each migration's statements; a failure names the migration file and the statement. */
+export async function applySteps(db: ReplayDb, steps: readonly ReplayStep[]): Promise<void> {
+  for (const { migration, statements } of steps) {
+    for (const [i, sql] of statements.entries()) {
+      try {
+        await db.run(sql);
+      } catch (err) {
+        throw new Error(`${migration.file}: statement ${String(i + 1)} failed: ${(err as Error).message}\n${sql.slice(0, 200)}`);
+      }
+    }
+  }
+}
+
+const POSTGRES_CATALOG = `
+select n.nspname as schema, c.relname as relation, a.attname as col
+from pg_class c
+join pg_namespace n on n.oid = c.relnamespace
+left join pg_attribute a on a.attrelid = c.oid and a.attnum > 0 and not a.attisdropped
+where c.relkind in ('r', 'p', 'v', 'm', 'f')
+  and n.nspname not in ('pg_catalog', 'information_schema')
+  and n.nspname not like 'pg\\_toast%' and n.nspname not like 'pg\\_temp%'
+order by 1, 2, a.attnum`;
+
+const CLICKHOUSE_CATALOG = `
+select t.database as schema, t.name as relation, c.name as col
+from system.tables t
+left join system.columns c on c.database = t.database and c.table = t.name
+where t.database not in ('system', 'INFORMATION_SCHEMA', 'information_schema')
+  and t.is_temporary = 0
+order by t.database, t.name, c.position`;
+
+/** The tables, views and columns the server has, as workspace output names (#3931). */
+export async function readCatalogOutputs(db: ReplayDb): Promise<string[]> {
+  const rows = await db.rows(db.dialect === "postgres" ? POSTGRES_CATALOG : CLICKHOUSE_CATALOG);
+  const out: string[] = [];
+  const seen = new Set<string>();
+  for (const r of rows) {
+    const rel = `${String(r.schema)}.${String(r.relation)}`;
+    if (!seen.has(rel)) {
+      seen.add(rel);
+      out.push(rel);
+    }
+    if (r.col !== null && r.col !== undefined) out.push(`${rel}.${String(r.col)}`);
+  }
+  return out;
+}
+
+/** Replay a directory into `db`, which must hold nothing, and return the snapshot to record. */
+export async function replayDirectory(db: ReplayDb, kind: ReplayKind, tree: WorkspaceTree, dir: string, fields: KindReaderInput["fields"] = null): Promise<ReplaySnapshot> {
+  const before = await readCatalogOutputs(db);
+  if (before.length > 0) throw new Error(`the server already has ${before[0]!.split(".").slice(0, 2).join(".")}${before.length > 1 ? ` and ${String(before.length - 1)} more names` : ""}; replay needs a server with no tables or views of its own, so the tables it finds are the migrations' (use a throwaway server or emulator)`);
+  const { steps, files, dialect } = replaySteps(kind, tree, dir, fields);
+  if (dialect !== db.dialect) throw new Error(`the migrations are ${dialect} and the server is ${db.dialect}; set the member's dialect field or bind the matching server`);
+  await applySteps(db, steps);
+  return { schema: 1, dialect, digest: migrationDigest((f) => tree.read(f), files), outputs: await readCatalogOutputs(db) };
+}
+
+export const renderSnapshot = (s: ReplaySnapshot): string => `${JSON.stringify(s, null, 2)}\n`;
+
+const USAGE = "usage: chant sql replay <dir> --kind goose|golang-migrate|clickhouse-migrations|flyway --env <env> [--migrations-dir <sub>] [--dialect postgres|clickhouse] [--check]";
+
+/** Open the server `--env` binds, of the migrations' dialect. */
+async function openBound(dialect: "postgres" | "clickhouse", environment: string): Promise<ReplayDb> {
+  if (dialect === "postgres") {
+    const { bindPostgres } = await import("./postgres/live/bind");
+    const { client } = await bindPostgres({ environment });
+    return { dialect, run: async (sql) => void (await client.query(sql)), rows: (sql) => client.query(sql), close: () => client.end() };
+  }
+  const { bindClickHouse } = await import("./clickhouse/live/bind");
+  const { clickhouseQuery } = await import("./clickhouse/http");
+  const target = await bindClickHouse({ environment });
+  return { dialect, run: async (sql) => void (await clickhouseQuery(target.endpoint, sql)), rows: (sql) => clickhouseQuery(target.endpoint, sql), async close() {} };
+}
+
+export async function runReplay(ctx: CommandGroupContext): Promise<number> {
+  const args = ctx.rawArgs;
+  let kind: string | undefined;
+  let env: string | undefined;
+  let sub: string | undefined;
+  let dialect: string | undefined;
+  const check = args.includes("--check");
+  const positional: string[] = [];
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i]!;
+    if (a === "--kind") kind = args[++i];
+    else if (a === "--env") env = args[++i];
+    else if (a === "--migrations-dir") sub = args[++i];
+    else if (a === "--dialect") dialect = args[++i];
+    else if (a === "--check") continue;
+    else if (a.startsWith("--")) {
+      console.error(`unknown argument ${a}\n${USAGE}`);
+      return 1;
+    } else positional.push(a);
+  }
+  if (positional.length !== 1 || !kind || !(REPLAY_KINDS as readonly string[]).includes(kind) || !env) {
+    console.error(USAGE);
+    return 1;
+  }
+  const abs = resolve(process.cwd(), positional[0]!);
+  const tree = workingTree(abs);
+  const fields = { ...(sub ? { migrationsDir: sub } : {}), ...(dialect ? { dialect } : {}) } as KindReaderInput["fields"];
+  const { steps, dialect: found } = replaySteps(kind as ReplayKind, tree, "", fields);
+  if (steps.length === 0) {
+    console.error(`chant sql replay: ${relative(process.cwd(), abs) || "."} lists no ${kind} migrations`);
+    return 1;
+  }
+  const db = await openBound(found as "postgres" | "clickhouse", env);
+  let snapshot: ReplaySnapshot;
+  try {
+    snapshot = await replayDirectory(db, kind as ReplayKind, tree, "", fields);
+  } catch (err) {
+    console.error(`chant sql replay: ${(err as Error).message}`);
+    return 1;
+  } finally {
+    await db.close();
+  }
+  const file = join(abs, REPLAY_FILE);
+  const text = renderSnapshot(snapshot);
+  const current = (() => {
+    try {
+      return readFileSync(file, "utf8");
+    } catch {
+      return undefined;
+    }
+  })();
+  if (check) {
+    if (current === text) return 0;
+    console.error(`chant sql replay --check: ${relative(process.cwd(), file)} ${current === undefined ? "is missing" : "differs from a replay of the migrations"}; run chant sql replay and commit it`);
+    return 1;
+  }
+  writeFileSync(file, text);
+  console.log(`recorded ${String(snapshot.outputs.length)} names from ${String(steps.length)} migrations in ${relative(process.cwd(), file)}`);
+  return 0;
+}

package/src/op/__snapshots__/watch-pipeline.test.ts.snap

@@ -20,7 +20,7 @@ jobs:
       CLICKHOUSE_READER_USER: chant_reader
       CLICKHOUSE_READER_PASSWORD: '\${{ secrets.CLICKHOUSE_READER_PASSWORD }}'
     steps:
-      - uses: https://code.forgejo.org/actions/checkout@v7
+      - uses: https://code.forgejo.org/actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - run: npm ci
       - run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
       - run: chant run schema-watch
@@ -52,7 +52,7 @@ jobs:
       CLICKHOUSE_READER_USER: chant_reader
       CLICKHOUSE_READER_PASSWORD: '\${{ secrets.CLICKHOUSE_READER_PASSWORD }}'
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - run: npm ci
       - run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
       - run: chant run schema-watch

package/src/op/activities/postgres-migration.ts

@@ -109,6 +109,8 @@ async function bound<T>(args: PostgresMigrationArgs, signal: AbortSignal | undef
       objects,
       column: args.column,
       ...(args.using !== undefined ? { using: args.using } : {}),
+      ...(args.reverse !== undefined ? { reverse: args.reverse } : {}),
+      ...(args.defaultAtContract ? { defaultAtContract: true } : {}),
       ...(marker ? { marker } : {}),
       ...(major !== undefined ? { major } : {}),
       batchSize: args.batchSize ?? 1000,
@@ -158,6 +160,8 @@ export async function postgresMigrationPlan(args: PostgresMigrationArgs, signal?
       column: run.column,
       ...(run.marker ? { marker: run.marker } : {}),
       ...(run.using !== undefined ? { using: run.using } : {}),
+      ...(run.reverse !== undefined ? { reverse: run.reverse } : {}),
+      ...(run.defaultAtContract ? { defaultAtContract: true } : {}),
       ...(run.major !== undefined ? { major: run.major } : {}),
     });
     const n = o.names;

package/src/op/activity-contracts.ts

@@ -124,6 +124,8 @@ const migrationArgs = z.strictObject({
   buildPath: z.string(),
   environment: z.string().optional(),
   using: z.string().optional(),
+  reverse: z.string().optional(),
+  defaultAtContract: z.boolean().optional(),
   retain: z.string().optional(),
   batchSize: z.number().int().positive().optional(),
   replicationLag: z.union([z.literal(false), z.strictObject({ max: z.string().optional(), wait: z.string().optional() })]).optional(),