@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/migrate/from-postgres/example.test.ts

@@ -0,0 +1,205 @@
+/**
+ * The first and third Done-when boxes of #4399, on the example pair.
+ *
+ * `examples/postgres-cdc-source` translated with `--use-composites` gives the
+ * `CdcMirror` calls of `examples/cdc-mirror`, apart from the choices the
+ * review diagnostics ask for. And the two projects' column names no longer
+ * need a test that compares them: the mirror links each column to the
+ * Postgres member's output, so dropping a Postgres column fails the link.
+ * The link check here is core's `resolveLinks`, the function behind WSP093;
+ * `workspace.e2e.test.ts` runs `chant workspace check` itself.
+ */
+
+import { describe, expect, test } from "vitest";
+import { parseDeclaration } from "@intentius/chant/workspace/declaration";
+import { resolveLinks, type MemberHandles } from "@intentius/chant/workspace/links";
+import type { Declarable } from "@intentius/chant/declarable";
+import { CdcMirror } from "../../composites/cdc-mirror";
+import { schema, table } from "../../postgres/entities";
+import { sqlWorkspaceOutputs } from "../../workspace-outputs";
+import * as source from "../../../examples/postgres-cdc-source/src/shop";
+import * as mirrors from "../../../examples/cdc-mirror/src/mirrors";
+import { mirrorColumns, orderBySql } from "./emit";
+import { translatePostgres, type Mirror } from "./translate";
+
+const sourceEntities = (): Map<string, Declarable> => new Map(Object.entries(source) as Array<[string, Declarable]>);
+const ddl = (e: unknown): string => (e as { props: { ddl: string } }).props.ddl;
+
+/**
+ * The call for a mirror once its review is done. `types` replaces a column's
+ * type, `call` adds the call's own props.
+ */
+function reviewed(m: Mirror, review: { types?: Record<string, string>; call?: Record<string, string> } = {}) {
+  const columns = m.columns.map((c) => ({ ...c, type: review.types?.[c.name] ?? c.type }));
+  return { name: m.name, columns: mirrorColumns({ ...m, columns }), primaryKey: orderBySql(m), ...review.call };
+}
+
+describe("postgres-cdc-source translated with --use-composites", () => {
+  const t = translatePostgres(sourceEntities(), { composite: true });
+  const byName = Object.fromEntries(t.mirrors.map((m) => [m.name, m]));
+
+  test("is the two tables, keyed on their primary keys", () => {
+    expect(t.mirrors.map((m) => m.name)).toEqual(["customers", "orders"]);
+    expect(t.mirrors.map((m) => m.primaryKey)).toEqual([["id"], ["id"]]);
+  });
+
+  test("the call's columns are the mechanical type map, with the column comment kept", () => {
+    expect(mirrorColumns(byName.customers!)).toBe("id Int64, email String COMMENT 'PII, hashed downstream', country String, created_at DateTime64(6, 'UTC')");
+    expect(mirrorColumns(byName.orders!)).toBe("id Int64, customer_id Int64, total Decimal(12, 2), status String, placed_at DateTime64(6, 'UTC')");
+  });
+
+  test("customers is examples/cdc-mirror's declaration once country is LowCardinality (SQLRV004)", () => {
+    const call = CdcMirror(reviewed(byName.customers!, { types: { country: "LowCardinality(String)" } }));
+    expect(ddl(call.table)).toBe(ddl(mirrors.customers.table));
+    expect(ddl(call.current)).toBe(ddl(mirrors.customers.current));
+  });
+
+  test("orders is examples/cdc-mirror's declaration once status is LowCardinality and the pipeline's names and partition are chosen (SQLRV004, SQLRV002)", () => {
+    const call = CdcMirror(
+      reviewed(byName.orders!, {
+        types: { status: "LowCardinality(String)" },
+        call: { version: "_peerdb_version", deleted: "_peerdb_is_deleted", partitionBy: "toYYYYMM(placed_at)" },
+      }),
+    );
+    expect(ddl(call.table)).toBe(ddl(mirrors.orders.table));
+    expect(ddl(call.current)).toBe(ddl(mirrors.orders.current));
+  });
+
+  test("each of those edits is asked for by an open diagnostic", () => {
+    const at = (rule: string, entity: string, text: string) => t.diagnostics.some((d) => d.ruleId === rule && d.entity === entity && d.message.includes(text));
+    expect(at("SQLRV004", "shop.customers", "shop.customers.country")).toBe(true);
+    expect(at("SQLRV004", "shop.orders", "shop.orders.status")).toBe(true);
+    expect(at("SQLRV002", "shop.orders", "ReplacingMergeTree(_version, _is_deleted)")).toBe(true);
+  });
+
+  test("the diagnostics are the decisions: no mechanical map writes one", () => {
+    const rules = new Set(t.diagnostics.map((d) => d.ruleId));
+    expect([...rules].sort()).toEqual(["SQLRV001", "SQLRV002", "SQLRV004", "SQLRV005", "SQLRV006", "SQLRV008", "SQLRV009", "SQLRV011"]);
+    // No type diagnostic (SQLRV007): bigint, text, numeric(12, 2) and timestamptz are mechanical.
+    expect(t.diagnostics.filter((d) => d.ruleId === "SQLRV007")).toEqual([]);
+    // No Nullable finding: every column is NOT NULL or a key.
+    expect(t.diagnostics.filter((d) => d.ruleId === "SQLRV003")).toEqual([]);
+  });
+
+  test("the foreign key from orders to customers and the index on orders are said not to be carried", () => {
+    const fk = t.diagnostics.find((d) => d.ruleId === "SQLRV005");
+    expect(fk).toMatchObject({ entity: "shop.orders", severity: "warning" });
+    expect(fk!.message).toContain("REFERENCES shop.customers (id)");
+    const ix = t.diagnostics.find((d) => d.ruleId === "SQLRV009");
+    expect(ix!.message).toContain("orders_by_customer_idx");
+  });
+
+  test("the links are the Postgres outputs of every mirrored column", () => {
+    expect(t.outputs).toEqual([
+      "shop.customers.id",
+      "shop.customers.email",
+      "shop.customers.country",
+      "shop.customers.created_at",
+      "shop.orders.id",
+      "shop.orders.customer_id",
+      "shop.orders.total",
+      "shop.orders.status",
+      "shop.orders.placed_at",
+    ]);
+  });
+});
+
+/** The two projects as a workspace: `db` is the Postgres member, `mirror` links to each column. */
+function workspace(entities: ReadonlyMap<string, Declarable>, links: readonly string[]) {
+  const declaration = parseDeclaration(
+    JSON.stringify({
+      name: "shop",
+      schema: 1,
+      members: [
+        { name: "db", dir: "postgres-cdc-source", kind: "chant" },
+        { name: "mirror", dir: "cdc-mirror", kind: "chant", links: links.map((output) => ({ member: "db", output })) },
+      ],
+    }),
+    "chant.workspace.json",
+  );
+  const report = sqlWorkspaceOutputs(entities as never)!;
+  const handles = new Map<string, MemberHandles>([
+    [
+      "db",
+      {
+        member: "db",
+        outputs: report.outputs.map((o) => ({ name: o.name, ...(o.previously?.length ? { previously: o.previously } : {}) })),
+        inputs: [],
+        complete: (report.gaps ?? []).length === 0,
+        why: null,
+        exposesNone: false,
+      },
+    ],
+  ]);
+  return resolveLinks(declaration, handles);
+}
+
+describe("the workspace link replaces the test that held the column names together", () => {
+  const links = translatePostgres(sourceEntities(), { composite: true }).outputs;
+
+  test("every link of the translated example resolves against the Postgres member", () => {
+    const rows = workspace(sourceEntities(), links);
+    expect(rows).toHaveLength(9);
+    expect(rows.map((r) => r.status)).toEqual(Array(9).fill("resolved"));
+  });
+
+  test("the mirror's hand-written column names are among the outputs, so a drift would show", () => {
+    // examples/cdc-mirror declares these columns; each is a Postgres output.
+    const declared = [...Object.values(mirrors)].flatMap((m) => ((m.table as unknown as { props: { columns: Array<{ name: string }> } }).props.columns.map((c) => c.name)));
+    const outputs = new Set(sqlWorkspaceOutputs(sourceEntities() as never)!.outputs.map((o) => o.name));
+    const named = declared.filter((n) => !n.startsWith("_"));
+    expect(named.length).toBe(9);
+    for (const n of named) expect([...outputs].some((o) => o.endsWith(`.${n}`)), n).toBe(true);
+  });
+
+  /** The customers declaration without email, as an author who drops the column would leave it. */
+  function withoutEmail(): Map<string, Declarable> {
+    const entities = sourceEntities();
+    const shop = schema`CREATE SCHEMA shop`;
+    entities.set("shop", shop as unknown as Declarable);
+    entities.set(
+      "customers",
+      table`
+        CREATE TABLE ${shop}.customers (
+          id         bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+          country    text NOT NULL,
+          created_at timestamptz NOT NULL DEFAULT now()
+        )` as unknown as Declarable,
+    );
+    return entities;
+  }
+
+  test("dropping a mirrored column from the Postgres declaration fails the mirror's link (WSP093)", () => {
+    const rows = workspace(withoutEmail(), links);
+    const missing = rows.filter((r) => r.status === "missing");
+    expect(missing).toHaveLength(1);
+    expect(missing[0]).toMatchObject({ consumer: "mirror", producer: "db", output: "shop.customers.email" });
+    expect(missing[0]!.reason).toContain("db has no output shop.customers.email");
+  });
+
+  test("the link passes again once the mirror follows: migrate again, and the dropped column is gone from the links", () => {
+    const entities = withoutEmail();
+    const again = translatePostgres(entities, { composite: true });
+    expect(again.outputs).not.toContain("shop.customers.email");
+    expect(workspace(entities, again.outputs).every((r) => r.status === "resolved")).toBe(true);
+  });
+
+  test("renaming a column fails the link, and a previously hint says where it went", () => {
+    const entities = sourceEntities();
+    const shop = schema`CREATE SCHEMA shop`;
+    entities.set("shop", shop as unknown as Declarable);
+    entities.set(
+      "customers",
+      table`
+        CREATE TABLE ${shop}.customers (
+          id         bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+          email_hash text NOT NULL, -- previously: email
+          country    text NOT NULL,
+          created_at timestamptz NOT NULL DEFAULT now()
+        )` as unknown as Declarable,
+    );
+    const missing = workspace(entities, links).filter((r) => r.status === "missing");
+    expect(missing).toHaveLength(1);
+    expect(missing[0]!.reason).toContain("renamed to shop.customers.email_hash");
+  });
+});

package/src/migrate/from-postgres/index.ts

@@ -0,0 +1,122 @@
+/**
+ * `chant migrate --from sql/postgres --to sql/clickhouse <file or member>`
+ * (#4399): the Postgres tables a CDC pipeline mirrors, as the ClickHouse
+ * tables that receive them.
+ *
+ * ## Naming
+ *
+ * A `chant migrate` side names a lexicon, and the sql lexicon holds two
+ * dialects whose builds never mix, so a side is `<lexicon>/<dialect>`:
+ * `sql/postgres` and `sql/clickhouse`. `--to sql/clickhouse` loads the sql
+ * lexicon and asks it for `migrationSource("sql/postgres", "sql/clickhouse")`.
+ * The same shape would name a later pair (`sql/clickhouse` to `sql/postgres`)
+ * without a new flag. A side with no slash is a lexicon alone, as for
+ * `--from github --to gitlab`.
+ *
+ * ## The source
+ *
+ * - a `.sql` file of Postgres DDL;
+ * - a TypeScript file or a member directory of `table` declarations, read by
+ *   a folded build of the project it belongs to (`chant sql types` reads the
+ *   same way), so `${shop}.customers` and other interpolations resolve.
+ *
+ * ## What it writes
+ *
+ * `--emit ts` (the default here) writes declarations: `table` templates, or
+ * `CdcMirror` calls with `--use-composites`. `--emit sql` writes the DDL.
+ * Every decision that is not the type map is a finding from the shared
+ * catalog (`../../clickhouse/review-catalog.ts`), reported as SARIF with
+ * `--report`, and a `warning` fails the run under `--strict`.
+ *
+ * `links` names the Postgres output each mirrored column reads
+ * (`shop.customers.email`). In a workspace, `chant migrate` writes them as
+ * `links` of the ClickHouse member (`../../../../packages/core/src/workspace/migrate-links.ts`),
+ * so `chant workspace check` fails when the Postgres declaration drops or
+ * renames one.
+ *
+ * `--validate` creates the emitted DDL on a throwaway server of the pinned
+ * ClickHouse image (`../../sqlserver/validate.ts` runs it).
+ */
+
+import { existsSync, statSync } from "node:fs";
+import { dirname } from "node:path";
+import type { Declarable } from "@intentius/chant/declarable";
+import type { MigrateOptions, MigrationResult, MigrationSource } from "@intentius/chant/lexicon";
+import { looksLikePostgres } from "./detect";
+import { emitSql, emitTs, mirrorObjects } from "./emit";
+import { translatePostgres, type Translation } from "./translate";
+
+export const FROM_POSTGRES = "sql/postgres";
+export const TO_CLICKHOUSE = "sql/clickhouse";
+
+/** Whether `from` and `to` name this pair. */
+export const isPostgresToClickHouse = (from: string, to?: string): boolean => from === FROM_POSTGRES && (to === undefined || to === TO_CLICKHOUSE);
+
+/** The Postgres declarations at `sourceFile`: a `.sql` file read as DDL, else a folded build of its project. */
+export async function loadPostgresEntities(content: string, sourceFile: string | undefined): Promise<Map<string, Declarable>> {
+  const isDir = sourceFile !== undefined && existsSync(sourceFile) && statSync(sourceFile).isDirectory();
+  if (!isDir && (sourceFile === undefined || sourceFile.endsWith(".sql"))) {
+    const { sqlFileEntities } = await import("../../sql-files");
+    return sqlFileEntities("postgres", [{ ddl: content, origin: sourceFile ?? "the DDL" }], { known: new Map() });
+  }
+  const { buildFolded, loadProject } = await import("../../sql-types-command");
+  const project = await loadProject(isDir ? sourceFile! : dirname(sourceFile!));
+  const result = await buildFolded(project, "chant migrate", "nothing was translated");
+  if (!result) throw new Error(`the build of ${sourceFile} failed`);
+  return new Map(result.entities as ReadonlyMap<string, Declarable>);
+}
+
+/** A finding as the lint diagnostic `chant migrate` writes as SARIF. */
+function toDiagnostic(file: string, d: Translation["diagnostics"][number], strict: boolean): Record<string, unknown> {
+  return {
+    file,
+    line: d.line ?? 1,
+    column: d.column ?? 1,
+    ruleId: d.ruleId,
+    // Under --strict a warning is an error, which is what fails the run (core reads the severity).
+    severity: strict && d.severity === "warning" ? "error" : d.severity,
+    message: d.message,
+    ...(d.entity ? { entity: d.entity } : {}),
+  };
+}
+
+export async function transform(content: string, opts: MigrateOptions): Promise<MigrationResult> {
+  const emit = opts.emit ?? "ts";
+  if (emit === "yaml") throw new Error("--emit yaml has no meaning here; use --emit ts or --emit sql");
+  const composite = opts.useComposites === true;
+  const file = opts.sourceFile ?? "schema.sql";
+  const entities = await loadPostgresEntities(content, opts.sourceFile);
+  const translation = translatePostgres(entities, { composite });
+  if (translation.mirrors.length === 0) {
+    const refused = translation.diagnostics.filter((d) => d.severity === "error").map((d) => d.message);
+    throw new Error(`no Postgres table to mirror in ${file}${refused.length > 0 ? `: ${refused.join(" ")}` : ""}`);
+  }
+  const output = emit === "sql" ? emitSql(translation.mirrors, composite) : emitTs(translation.mirrors, composite);
+  const diagnostics = translation.diagnostics.map((d) => toDiagnostic(file, d, opts.strict === true));
+  const provenance = translation.mirrors.map((m) => ({
+    sourceKey: `${m.source.schema}.${m.source.table}`,
+    rule: composite ? "SQLMIG-CDC-MIRROR" : "SQLMIG-TABLE",
+    category: "translated",
+    note: `${String(m.columns.length)} columns; ORDER BY ${m.primaryKey.length === 0 ? "tuple()" : `(${m.primaryKey.join(", ")})`}`,
+  }));
+  return {
+    output,
+    provenance,
+    diagnostics,
+    links: translation.outputs,
+    validate: async () => {
+      const [{ objectsToIR }, { validateOnEmulator }] = await Promise.all([import("../../clickhouse/import/ir"), import("../../sqlserver/validate")]);
+      const found = await validateOnEmulator(objectsToIR(mirrorObjects(translation.mirrors, composite)));
+      return found.map((d) => ({ file, line: 1, column: 1, ruleId: d.ruleId, severity: d.severity, message: d.message, ...(d.entity ? { entity: d.entity } : {}) }));
+    },
+  };
+}
+
+/** The `migrationSource` of the sql lexicon for this pair. */
+export function postgresToClickHouseSource(): MigrationSource {
+  return {
+    detect: looksLikePostgres,
+    directory: true,
+    transform,
+  };
+}

package/src/migrate/from-postgres/translate.test.ts

@@ -0,0 +1,242 @@
+import { describe, expect, test } from "vitest";
+import { sqlFileEntities } from "../../sql-files";
+import { table as clickhouseTable } from "../../clickhouse/entities";
+import { emitSql, emitTs, mirrorObjects, tableSql } from "./emit";
+import { looksLikePostgres } from "./detect";
+import { transform } from "./index";
+import { translatePostgres, type Translation } from "./translate";
+
+const entities = (ddl: string) => sqlFileEntities("postgres", [{ ddl, origin: "schema.sql" }], { known: new Map() });
+const run = (ddl: string, composite = false): Translation => translatePostgres(entities(ddl), { composite });
+const rules = (t: Translation, entity?: string) => t.diagnostics.filter((d) => entity === undefined || d.entity === entity).map((d) => d.ruleId);
+const find = (t: Translation, ruleId: string, text = "") => t.diagnostics.find((d) => d.ruleId === ruleId && d.message.includes(text));
+
+describe("sort key and engine", () => {
+  test("the primary key is the sort key; a composite key keeps its order", () => {
+    const t = run("CREATE TABLE docs (tenant_id bigint NOT NULL, id bigint NOT NULL, body text NOT NULL, PRIMARY KEY (tenant_id, id))");
+    expect(t.mirrors[0]!.primaryKey).toEqual(["tenant_id", "id"]);
+    expect(find(t, "SQLRV001")!.message).toContain("ORDER BY (tenant_id, id) suggested from the primary key");
+    expect(tableSql(t.mirrors[0]!)).toContain("ORDER BY (tenant_id, id)");
+  });
+
+  test("with no primary key the first unique key is the sort key, and the finding says it passed over nothing else", () => {
+    const t = run("CREATE TABLE docs (slug text NOT NULL UNIQUE, body text NOT NULL)");
+    expect(t.mirrors[0]!.primaryKey).toEqual(["slug"]);
+    expect(find(t, "SQLRV001")!.message).toContain("unique key");
+  });
+
+  test("with no key at all the sort key is tuple(), and the engine finding still warns", () => {
+    const t = run("CREATE TABLE log (line text NOT NULL)");
+    expect(t.mirrors[0]!.primaryKey).toEqual([]);
+    expect(find(t, "SQLRV001")!.message).toContain("tuple()");
+    expect(tableSql(t.mirrors[0]!)).toContain("ORDER BY tuple()");
+    expect(find(t, "SQLRV002")!.severity).toBe("warning");
+  });
+
+  test("every table gets the ReplacingMergeTree engine with the two columns the composite names", () => {
+    const sql = tableSql(run("CREATE TABLE t (id bigint PRIMARY KEY)").mirrors[0]!);
+    expect(sql).toContain("ENGINE = ReplacingMergeTree(_version, _is_deleted)");
+    expect(sql).toContain("_version UInt64");
+    expect(sql).toContain("_is_deleted UInt8");
+  });
+
+  test("a source column named like one the mirror adds refuses the table with an error", () => {
+    const t = run("CREATE TABLE t (id bigint PRIMARY KEY, _version int)");
+    expect(t.mirrors).toEqual([]);
+    expect(find(t, "SQLRV011", "_version")).toMatchObject({ severity: "error" });
+  });
+
+  test("a nullable unique key needs allow_nullable_key, which a CdcMirror call cannot carry", () => {
+    const ddl = "CREATE TABLE t (code text UNIQUE, n int NOT NULL)";
+    const bare = run(ddl);
+    expect(tableSql(bare.mirrors[0]!)).toContain("SETTINGS allow_nullable_key = 1");
+    const composite = run(ddl, true);
+    expect(find(composite, "SQLRV001", "no SETTINGS")).toMatchObject({ severity: "warning" });
+  });
+});
+
+describe("Nullable, defaults and column kinds", () => {
+  test("a nullable column is Nullable with a finding, and a default is named as the way out", () => {
+    const t = run("CREATE TABLE t (id bigint PRIMARY KEY, note text, qty integer DEFAULT 0, tag text NOT NULL)");
+    const types = Object.fromEntries(t.mirrors[0]!.columns.map((c) => [c.name, c.type]));
+    expect(types).toEqual({ id: "Int64", note: "Nullable(String)", qty: "Nullable(Int32)", tag: "String" });
+    expect(find(t, "SQLRV003", "t.qty")!.message).toContain("DEFAULT 0");
+    expect(find(t, "SQLRV003", "t.note")!.message).toContain("no default");
+    expect(t.diagnostics.filter((d) => d.ruleId === "SQLRV003")).toHaveLength(2);
+  });
+
+  test("a primary key, identity and serial column are not Nullable whatever the DDL says", () => {
+    const t = run("CREATE TABLE t (a bigint PRIMARY KEY, b bigint GENERATED BY DEFAULT AS IDENTITY, c serial, d bigint NOT NULL)");
+    expect(t.mirrors[0]!.columns.map((c) => c.type)).toEqual(["Int64", "Int64", "Int32", "Int64"]);
+    expect(rules(t).filter((r) => r === "SQLRV003")).toEqual([]);
+    expect(t.diagnostics.filter((d) => d.ruleId === "SQLRV006").map((d) => d.severity)).toEqual(["info", "info"]);
+  });
+
+  test("an array is not Nullable and says why", () => {
+    const t = run("CREATE TABLE t (id bigint PRIMARY KEY, tags text[])");
+    expect(t.mirrors[0]!.columns[1]!.type).toBe("Array(String)");
+    expect(find(t, "SQLRV007", "t.tags")!.message).toContain("Array(String)");
+    expect(rules(t)).not.toContain("SQLRV003");
+  });
+
+  test("a stored generated column is a warning; the type judgments are warnings; a default is an info", () => {
+    const t = run("CREATE TABLE t (id bigint PRIMARY KEY, a int NOT NULL, b int GENERATED ALWAYS AS (a * 2) STORED, m money NOT NULL, ts timestamptz NOT NULL DEFAULT now())");
+    expect(find(t, "SQLRV006", "t.b")).toMatchObject({ severity: "warning" });
+    expect(find(t, "SQLRV007", "money")).toMatchObject({ severity: "warning" });
+    expect(find(t, "SQLRV008", "t.ts")).toMatchObject({ severity: "info" });
+    expect(t.mirrors[0]!.columns.find((c) => c.name === "ts")!.type).toBe("DateTime64(6, 'UTC')");
+    expect(tableSql(t.mirrors[0]!)).not.toContain("DEFAULT");
+  });
+
+  test("the type of an enum or extension column is a judgment, never silent", () => {
+    const t = run("CREATE TABLE t (id bigint PRIMARY KEY, mood citext NOT NULL, geo geometry NOT NULL)");
+    expect(find(t, "SQLRV007", "t.geo")).toBeDefined();
+    expect(find(t, "SQLRV007", "t.mood")).toBeUndefined();
+  });
+
+  test("a column comment is kept as COMMENT; a table comment is kept bare and noted under a composite", () => {
+    const ddl = "CREATE TABLE t (id bigint PRIMARY KEY, email text NOT NULL); COMMENT ON COLUMN t.email IS 'PII'; COMMENT ON TABLE t IS 'The accounts'";
+    const bare = run(ddl);
+    expect(tableSql(bare.mirrors[0]!)).toContain("email String COMMENT 'PII'");
+    expect(tableSql(bare.mirrors[0]!)).toContain("COMMENT 'The accounts'");
+    const composite = run(ddl, true);
+    expect(find(composite, "SQLRV011", "description")).toMatchObject({ severity: "info" });
+    expect(emitTs(composite.mirrors, true)).toContain("email String COMMENT 'PII'");
+  });
+});
+
+describe("constraints, indexes and storage", () => {
+  const ddl = `
+    CREATE TABLE a (id bigint PRIMARY KEY);
+    CREATE TABLE b (
+      id bigint PRIMARY KEY,
+      a_id bigint NOT NULL REFERENCES a (id),
+      code text NOT NULL UNIQUE,
+      qty int NOT NULL CHECK (qty > 0),
+      during tstzrange,
+      EXCLUDE USING gist (during WITH &&)
+    ) WITH (fillfactor = 70);
+    CREATE UNIQUE INDEX b_code_lower ON b (lower(code));
+    CREATE INDEX b_qty ON b (qty);`;
+
+  test("foreign keys, uniques, checks and exclusions are findings and comment lines", () => {
+    const t = run(ddl);
+    const b = t.mirrors.find((m) => m.name === "b")!;
+    expect(b.constraintComments).toEqual([
+      "-- FOREIGN KEY (a_id) REFERENCES a (id)",
+      "-- UNIQUE (code)",
+      "-- CHECK qty > 0",
+      "-- EXCLUDE USING gist (during WITH &&)",
+    ]);
+    expect(t.diagnostics.filter((d) => d.ruleId === "SQLRV005" && d.entity === "b")).toHaveLength(4);
+    expect(tableSql(b).split("\n").slice(0, 4)).toEqual(b.constraintComments);
+  });
+
+  test("an index that is not a plain unique key, and the storage options, are warnings", () => {
+    const t = run(ddl);
+    expect(find(t, "SQLRV009", "b_qty")).toMatchObject({ severity: "warning", entity: "b" });
+    expect(find(t, "SQLRV009", "b_code_lower")).toBeDefined();
+    expect(find(t, "SQLRV009", "Storage options")!.message).toContain("WITH (fillfactor = 70)");
+  });
+
+  test("a view, a sequence and a partition are named as not carried", () => {
+    const t = run(`
+      CREATE TABLE events (id bigint, at timestamptz NOT NULL) PARTITION BY RANGE (at);
+      CREATE TABLE events_2026 PARTITION OF events FOR VALUES FROM ('2026-01-01') TO ('2027-01-01');
+      CREATE VIEW recent AS SELECT * FROM events;
+      CREATE SEQUENCE ticket_seq;`);
+    expect(t.mirrors.map((m) => m.name)).toEqual(["events"]);
+    expect(t.diagnostics.filter((d) => d.ruleId === "SQLRV011").map((d) => d.entity).sort()).toEqual(["events_2026", "sequence ticket_seq", "view recent"]);
+    expect(find(t, "SQLRV009", "PARTITION BY RANGE")).toBeDefined();
+  });
+
+  test("two schemas with one table name keep the schema as a database qualifier", () => {
+    const t = run("CREATE SCHEMA a; CREATE SCHEMA b; CREATE TABLE a.items (id bigint PRIMARY KEY); CREATE TABLE b.items (id bigint PRIMARY KEY); CREATE TABLE b.other (id bigint PRIMARY KEY)");
+    expect(t.mirrors.map((m) => m.name)).toEqual(["a.items", "b.items", "other"]);
+    expect(t.outputs).toContain("b.items.id");
+    expect(find(t, "SQLRV011", "keeps its schema")).toBeDefined();
+  });
+});
+
+describe("what is written", () => {
+  const t = run("CREATE TABLE accounts (tenant_id bigint NOT NULL, id bigint NOT NULL, name text NOT NULL, PRIMARY KEY (tenant_id, id)); CREATE TABLE \"Odd Name\" (id bigint PRIMARY KEY)");
+  const plain = run("CREATE TABLE accounts (tenant_id bigint NOT NULL, id bigint NOT NULL, name text NOT NULL, PRIMARY KEY (tenant_id, id))", true);
+
+  test("--emit ts with composites is CdcMirror calls with the sort key as primaryKey", () => {
+    const ts = emitTs(plain.mirrors, true);
+    expect(ts).toContain('import { CdcMirror } from "@intentius/chant-lexicon-sql/clickhouse";');
+    expect(ts).toContain('primaryKey: "(tenant_id, id)"');
+    expect(ts).toContain('columns: "tenant_id Int64, id Int64, name String"');
+  });
+
+  test("--emit ts without composites is table declarations whose DDL the ClickHouse tag reads", () => {
+    const ts = emitTs(t.mirrors, false);
+    expect(ts).toContain('import { table } from "@intentius/chant-lexicon-sql/clickhouse";');
+    for (const m of t.mirrors) {
+      const sql = tableSql(m);
+      const entity = clickhouseTable(Object.assign([sql], { raw: [sql] }) as unknown as TemplateStringsArray);
+      const props = (entity as unknown as { props: { engine: { name: string; args: string[] }; columns: Array<{ name: string }> } }).props;
+      expect(props.engine).toMatchObject({ name: "ReplacingMergeTree", args: ["_version", "_is_deleted"] });
+      expect(props.columns.map((c) => c.name).slice(-2)).toEqual(["_version", "_is_deleted"]);
+    }
+  });
+
+  test("--emit sql ends each statement; a composite adds the view of live rows", () => {
+    const sql = emitSql(plain.mirrors, true);
+    expect(sql.match(/^CREATE (TABLE|VIEW)/gm)).toHaveLength(2);
+    expect(sql).toContain("CREATE VIEW accounts_current AS");
+    expect(sql.trimEnd().endsWith(";")).toBe(true);
+    expect(mirrorObjects(t.mirrors, false).map((o) => o.name)).toEqual(["accounts", "Odd Name"]);
+  });
+
+  test("a name that needs quoting is quoted in DDL, and refused for a CdcMirror call, which writes it bare", () => {
+    expect(tableSql(t.mirrors[1]!)).toContain("CREATE TABLE `Odd Name`");
+    const composite = run('CREATE TABLE "Odd Name" (id bigint PRIMARY KEY)', true);
+    expect(composite.mirrors).toEqual([]);
+    expect(find(composite, "SQLRV011", "needs quoting")).toMatchObject({ severity: "error" });
+  });
+});
+
+describe("transform: the MigrationSource's result", () => {
+  const ddl = "CREATE TABLE t (id bigint PRIMARY KEY, status text NOT NULL, note text);";
+
+  test("findings are SARIF-shaped diagnostics against the source file; links name the Postgres outputs", async () => {
+    const r = await transform(ddl, { emit: "ts", useComposites: true, sourceFile: "db/schema.sql" });
+    expect(r.diagnostics.every((d) => d.file === "db/schema.sql" && d.line === 1 && d.column === 1)).toBe(true);
+    expect(new Set(r.diagnostics.map((d) => d.ruleId))).toEqual(new Set(["SQLRV001", "SQLRV002", "SQLRV003", "SQLRV004"]));
+    expect(r.links).toEqual(["public.t.id", "public.t.status", "public.t.note"]);
+    expect(r.output).toContain("CdcMirror({");
+    expect(r.provenance).toHaveLength(1);
+    expect(typeof r.validate).toBe("function");
+  });
+
+  test("--strict makes every open warning an error and leaves info alone", async () => {
+    const lax = await transform(ddl, { emit: "ts" });
+    const strict = await transform(ddl, { emit: "ts", strict: true });
+    expect(lax.diagnostics.some((d) => d.severity === "error")).toBe(false);
+    const warnings = lax.diagnostics.filter((d) => d.severity === "warning").length;
+    expect(warnings).toBeGreaterThan(0);
+    expect(strict.diagnostics.filter((d) => d.severity === "error")).toHaveLength(warnings);
+    expect(strict.diagnostics.filter((d) => d.severity === "warning")).toEqual([]);
+  });
+
+  test("a table with nothing left to decide still has the sort key and engine open, so strict has something to fail on", async () => {
+    const r = await transform("CREATE TABLE t (id bigint PRIMARY KEY);", { emit: "sql", strict: true });
+    expect(r.diagnostics.filter((d) => d.severity === "error").map((d) => d.ruleId)).toEqual(["SQLRV001", "SQLRV002"]);
+  });
+
+  test("--emit yaml is refused, and a file with no table to mirror fails by name", async () => {
+    await expect(transform(ddl, { emit: "yaml" })).rejects.toThrow("--emit yaml");
+    await expect(transform("CREATE VIEW v AS SELECT 1;", { emit: "ts", sourceFile: "x.sql" })).rejects.toThrow("no Postgres table to mirror in x.sql");
+  });
+});
+
+describe("looksLikePostgres", () => {
+  test("recognises Postgres DDL and declarations, and not ClickHouse or T-SQL", () => {
+    expect(looksLikePostgres("CREATE TABLE t (id int);")).toBe(true);
+    expect(looksLikePostgres('import { table } from "@intentius/chant-lexicon-sql/postgres";')).toBe(true);
+    expect(looksLikePostgres("CREATE TABLE t (id UInt64) ENGINE = MergeTree ORDER BY id")).toBe(false);
+    expect(looksLikePostgres("CREATE TABLE t (id int)\nGO\n")).toBe(false);
+    expect(looksLikePostgres("SELECT 1")).toBe(false);
+  });
+});