@intentius/chant-lexicon-sql 0.112.1 → 0.114.0

package/src/clickhouse/rebuild/steps.ts

@@ -14,6 +14,7 @@ import type { ClickHouseTarget } from "../live/bind";
 import { carriesMarker, readTrailerPairs, REBUILD_TRAILER_KEY, stampedComment } from "../ownership";
 import { createStatement, ident, qualifiedIdent, renamedDeclaration, sqlString, type DeclaredObject } from "../apply/statements";
 import { waitForMutations } from "../apply/mutations";
+import { renderFor } from "../topology";
 import { DDL_SETTINGS, syncReplicaWithin } from "./replicas";
 import {
   changesBetween,
@@ -60,12 +61,14 @@ export const utcLiteral = (ms: number): string => `toDateTime64(${sqlString(sqlU
 export const observe = (run: RebuildRun): Promise<RebuildObservation> => observeRebuild(run.target, run.declared, run.marker);
 
 /**
- * One DDL statement. In a Replicated database it goes on without the
- * replicas that are down, which run it from the database's log when they
- * come back (`./replicas.ts`).
+ * One DDL statement, rendered for the environment's topology when it names
+ * one (`../topology.ts`: `ON CLUSTER` and the engine). In a Replicated
+ * database it goes on without the replicas that are down, which run it from
+ * the database's log when they come back (`./replicas.ts`).
  */
-const q = (run: RebuildRun, sql: string) => {
+const q = (run: RebuildRun, statement: string) => {
   run.signal?.throwIfAborted();
+  const sql = renderFor(statement, run.target.topology);
   run.log(sql);
   return clickhouseQuery(run.target.endpoint, sql, { settings: DDL_SETTINGS, ...(run.signal ? { signal: run.signal } : {}) });
 };
@@ -373,7 +376,7 @@ export async function compensate(run: Pick<RebuildRun, "target" | "declared" | "
     const row = rows.find((r) => r.name === name);
     const pairs = readTrailerPairs(row?.comment);
     if (!row || pairs?.get(REBUILD_TRAILER_KEY) !== n.key || pairs.get("role") !== role || !carriesMarker(row.comment, run.marker)) continue;
-    const sql = `DROP ${what} ${qualifiedIdent(n.database, name)} SYNC`;
+    const sql = renderFor(`DROP ${what} ${qualifiedIdent(n.database, name)} SYNC`, run.target.topology);
     run.log(sql);
     await clickhouseQuery(run.target.endpoint, sql, { settings: DDL_SETTINGS });
     dropped.push(`${n.database}.${name}`);

package/src/clickhouse/topology-replicated.e2e.test.ts

@@ -0,0 +1,125 @@
+/**
+ * The replicated topology against a Replicated database (#3645): two
+ * replicas of the pinned server, Keeper embedded in the first (the scratch
+ * cluster the Replicated rebuild runs on).
+ *
+ * The source is written for a single node: a plain database, MergeTree
+ * tables. Applied with `CLICKHOUSE_TOPOLOGY=replicated`, the database is a
+ * `Replicated` one and its tables `Replicated*MergeTree` with no path, so
+ * the second replica gets them from the database's log and rows written on
+ * one reach the other. Applied again on the other replica, the server's
+ * printed engines (with the default Keeper path filled in) compare equal to
+ * the declarations, and nothing is sent; a change applies there and reaches
+ * the first.
+ *
+ * The scratch cluster names no cluster in `remote_servers`, so the second
+ * replica joins the database with the `CREATE DATABASE` the applier sent to
+ * the first, as an operator would (`replicated:<cluster>` sends it `ON
+ * CLUSTER` instead). Needs Docker; skips cleanly without it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { dockerAvailable, startScratchCluster, type ScratchCluster } from "./container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "./http";
+import { clickhouseImage } from "../spec/pin";
+import { clickhouseApply } from "../op/activities/clickhouse-apply";
+import { renderStatement } from "./topology";
+
+const DB = "yodel_3645";
+const enabled = await dockerAvailable();
+let cluster: ScratchCluster | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-topology-repl-"));
+
+beforeAll(async () => {
+  if (!enabled) return;
+  cluster = await startScratchCluster(clickhouseImage(), { replicas: 2, namePrefix: "chant-sql-topology-repl" });
+}, 600_000);
+
+afterAll(async () => {
+  await cluster?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+interface Obj {
+  export: string;
+  type: string;
+  ddl: string;
+  dependsOn?: string[];
+}
+
+const writeBuild = (file: string, objects: Obj[]): string => {
+  const path = join(dir, file);
+  writeFileSync(path, JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects: objects.map((o) => ({ dependsOn: [], ...o })) }));
+  return path;
+};
+
+const objects = (extraColumn = ""): Obj[] => [
+  { export: "db", type: "ClickHouse::Database", ddl: `CREATE DATABASE ${DB}` },
+  {
+    export: "events",
+    type: "ClickHouse::Table",
+    dependsOn: ["db"],
+    ddl: `CREATE TABLE ${DB}.events (id UInt64, ts DateTime, kind String${extraColumn}) ENGINE = ReplacingMergeTree(ts) ORDER BY id`,
+  },
+  {
+    export: "byKind",
+    type: "ClickHouse::MaterializedView",
+    dependsOn: ["events"],
+    ddl: `CREATE MATERIALIZED VIEW ${DB}.by_kind ENGINE = SummingMergeTree ORDER BY kind AS SELECT kind, count() AS n FROM ${DB}.events GROUP BY kind`,
+  },
+];
+
+const q = <T = Record<string, unknown>>(at: ClickHouseEndpoint, sql: string) => clickhouseQuery<T>(at, sql);
+
+async function apply(at: ClickHouseEndpoint, buildPath: string) {
+  const sent: string[] = [];
+  const outcome = await clickhouseApply({ buildPath, environment: "test" }, undefined, {
+    config: { ownership: { stack: "yodel", env: "e2e" } },
+    env: { CLICKHOUSE_URL: at.url, CLICKHOUSE_TOPOLOGY: "replicated" },
+    log: (l) => void (/^[A-Z]+ /.test(l) && sent.push(l)),
+  });
+  return { outcome, sent };
+}
+
+describe.skipIf(!enabled)("the replicated topology on a Replicated database (#3645)", () => {
+  test("a single node's source becomes a Replicated database that both replicas hold, and converges on either", async () => {
+    const [r1, r2] = cluster!.replicas as [ClickHouseEndpoint, ClickHouseEndpoint];
+    expect(renderStatement(`CREATE DATABASE ${DB}`, { kind: "replicated" })).toBe(`CREATE DATABASE ${DB} ENGINE = Replicated('/clickhouse/databases/${DB}', '{shard}', '{replica}')`);
+
+    const v1 = writeBuild("v1.json", objects());
+    const first = await apply(r1, v1);
+    expect(first.outcome.failed).toEqual([]);
+    // The second replica joins the database with the statement the applier sent to the first, and replays its tables from the log.
+    expect(first.sent[0]).toMatch(new RegExp(`^CREATE DATABASE ${DB} ENGINE = Replicated\\('/clickhouse/databases/${DB}', '\\{shard\\}', '\\{replica\\}'\\) COMMENT `));
+    await q(r2, first.sent[0]!);
+    await q(r2, `SYSTEM SYNC DATABASE REPLICA ${DB}`);
+    expect(first.sent.join("\n")).not.toMatch(/ON CLUSTER/);
+    expect(first.sent[1]).toMatch(/ENGINE = ReplicatedReplacingMergeTree\(ts\) ORDER BY id/);
+
+    await q(r1, `INSERT INTO ${DB}.events (id, ts, kind) VALUES (1, now(), 'click'), (2, now(), 'view')`);
+    await q(r2, `SYSTEM SYNC REPLICA ${DB}.events`);
+    expect(Number((await q<{ n: string | number }>(r2, `SELECT count() AS n FROM ${DB}.events`))[0]!.n)).toBe(2);
+    const engines = await q<{ name: string; engine: string }>(r2, `SELECT name, engine FROM system.tables WHERE database = '${DB}' ORDER BY name`);
+    expect(engines.map((t) => [t.name.startsWith(".inner") ? ".inner" : t.name, t.engine])).toEqual([
+      [".inner", "ReplicatedSummingMergeTree"],
+      ["by_kind", "MaterializedView"],
+      ["events", "ReplicatedReplacingMergeTree"],
+    ]);
+    expect((await q<{ engine: string }>(r2, `SELECT engine FROM system.databases WHERE name = '${DB}'`))[0]!.engine).toBe("Replicated");
+
+    // On the other replica: the printed engines (default Keeper path filled in) are the declarations; nothing to send.
+    const again = await apply(r2, v1);
+    expect(again.sent).toEqual([]);
+    expect(again.outcome.notAttempted).toEqual([]);
+
+    // A change there reaches the first replica through the database's log.
+    const change = await apply(r2, writeBuild("v2.json", objects(", source String DEFAULT 'web'")));
+    expect(change.sent).toEqual([`ALTER TABLE \`${DB}\`.\`events\` ADD COLUMN source String DEFAULT 'web' AFTER \`kind\``]);
+    await q(r1, `SYSTEM SYNC DATABASE REPLICA ${DB}`);
+    const cols = await q<{ name: string }>(r1, `SELECT name FROM system.columns WHERE database = '${DB}' AND table = 'events' ORDER BY position`);
+    expect(cols.map((c) => c.name)).toEqual(["id", "ts", "kind", "source"]);
+  }, 300_000);
+});

package/src/clickhouse/topology.e2e.test.ts

@@ -0,0 +1,183 @@
+/**
+ * The single-node rendering against a server (#3645): one migration source
+ * written for a cluster (`ON CLUSTER`, `Replicated*MergeTree` with Keeper
+ * paths, a `Replicated` database) is refused by a single node as written
+ * and applies once rendered for it, with the plain MergeTree family: as
+ * statements, and through the applier with `CLICKHOUSE_TOPOLOGY=single`,
+ * which then finds nothing left to change.
+ *
+ * Runs against `CLICKHOUSE_URL` when it is set (the emulator, `chant
+ * emulator up --lexicon sql`), else a scratch server of the pinned image.
+ * Everything it writes is in the database `yodel_3645`. Needs one or the
+ * other; skips cleanly without Docker.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { clickhouseApply } from "../op/activities/clickhouse-apply";
+import { dockerAvailable, startScratchServer, type ScratchServer } from "./container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "./http";
+import { clickhouseImage } from "../spec/pin";
+import { renderStatement, type Topology } from "./topology";
+
+const DB = "yodel_3645";
+const fromEnv = process.env.CLICKHOUSE_URL;
+const enabled = fromEnv !== undefined || (await dockerAvailable());
+let server: ScratchServer | undefined;
+let endpoint: ClickHouseEndpoint;
+
+const q = <T = Record<string, unknown>>(sql: string) => clickhouseQuery<T>(endpoint, sql);
+
+beforeAll(async () => {
+  if (!enabled) return;
+  if (fromEnv !== undefined) {
+    endpoint = {
+      url: fromEnv,
+      ...(process.env.CLICKHOUSE_USER ? { user: process.env.CLICKHOUSE_USER } : {}),
+      ...(process.env.CLICKHOUSE_PASSWORD ? { password: process.env.CLICKHOUSE_PASSWORD } : {}),
+    };
+  } else {
+    server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-topology" });
+    endpoint = server.endpoint;
+  }
+  await q(`DROP DATABASE IF EXISTS ${DB} SYNC`);
+}, 600_000);
+
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-topology-"));
+
+afterAll(async () => {
+  if (enabled) await q(`DROP DATABASE IF EXISTS ${DB} SYNC`).catch(() => undefined);
+  await server?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+const SINGLE: Topology = { kind: "single" };
+
+/** One migration, written for a cluster named `main`. */
+const SOURCE = [
+  `CREATE DATABASE ${DB} ON CLUSTER main ENGINE = Replicated('/clickhouse/databases/${DB}', '{shard}', '{replica}') COMMENT 'topology e2e'`,
+  `CREATE TABLE ${DB}.events ON CLUSTER main (
+    id UInt64,
+    ts DateTime,
+    kind LowCardinality(String)
+  ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/${DB}/events', '{replica}', ts)
+  ORDER BY id`,
+  `CREATE MATERIALIZED VIEW ${DB}.by_kind ON CLUSTER main
+  ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/${DB}/by_kind', '{replica}')
+  ORDER BY kind
+  AS SELECT kind, count() AS n FROM ${DB}.events GROUP BY kind`,
+  `CREATE TABLE ${DB}.staging ON CLUSTER main (id UInt64, ts DateTime, kind LowCardinality(String)) ENGINE = ReplicatedReplacingMergeTree(ts) ORDER BY id`,
+  `ALTER TABLE ${DB}.events ON CLUSTER main ADD COLUMN source String DEFAULT 'web' AFTER kind`,
+  `ALTER TABLE ${DB}.staging ON CLUSTER main ADD COLUMN source String DEFAULT 'web' AFTER kind`,
+  `ALTER TABLE ${DB}.events ON CLUSTER main MODIFY COMMENT 'events'`,
+  `EXCHANGE TABLES ${DB}.events AND ${DB}.staging ON CLUSTER main`,
+  `RENAME TABLE ${DB}.staging TO ${DB}.events_old ON CLUSTER main`,
+  `DROP TABLE ${DB}.events_old ON CLUSTER main SYNC`,
+];
+
+describe.skipIf(!enabled)("the single-node rendering against a server (#3645)", () => {
+  test("the source as written for a cluster is refused by a single node", async () => {
+    await expect(q(SOURCE[0]!)).rejects.toThrow();
+    await expect(q(renderStatement(SOURCE[0]!, SINGLE))).resolves.toEqual([]);
+    // The database is there now; the table as written still fails, on its ON CLUSTER or its Keeper path.
+    await expect(q(SOURCE[1]!)).rejects.toThrow();
+    await q(`DROP DATABASE ${DB} SYNC`);
+  });
+
+  test("every statement rendered for a single node applies, with the plain MergeTree family", async () => {
+    for (const sql of SOURCE) {
+      const rendered = renderStatement(sql, SINGLE);
+      expect(rendered).not.toMatch(/ON CLUSTER|Replicated/);
+      await q(rendered);
+    }
+    const db = await q<{ engine: string }>(`SELECT engine FROM system.databases WHERE name = '${DB}'`);
+    expect(db).toEqual([{ engine: "Atomic" }]);
+    const tables = await q<{ name: string; engine: string }>(
+      `SELECT name, engine FROM system.tables WHERE database = '${DB}' AND NOT startsWith(name, '.inner') ORDER BY name`,
+    );
+    expect(tables).toEqual([
+      { name: "by_kind", engine: "MaterializedView" },
+      { name: "events", engine: "ReplacingMergeTree" },
+    ]);
+    const inner = await q<{ engine: string }>(`SELECT engine FROM system.tables WHERE database = '${DB}' AND startsWith(name, '.inner')`);
+    expect(inner).toEqual([{ engine: "SummingMergeTree" }]);
+    const cols = await q<{ name: string }>(`SELECT name FROM system.columns WHERE database = '${DB}' AND table = 'events' ORDER BY position`);
+    expect(cols.map((c) => c.name)).toEqual(["id", "ts", "kind", "source"]);
+  });
+});
+
+describe.skipIf(!enabled)("the applier with a single-node topology (#3645)", () => {
+  interface Obj {
+    export: string;
+    type: string;
+    ddl: string;
+    dependsOn?: string[];
+  }
+  const writeBuild = (file: string, objects: Obj[]): string => {
+    const path = join(dir, file);
+    writeFileSync(path, JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects: objects.map((o) => ({ dependsOn: [], ...o })) }));
+    return path;
+  };
+  const objects = (extraColumn = ""): Obj[] => [
+    { export: "db", type: "ClickHouse::Database", ddl: `CREATE DATABASE ${DB} ON CLUSTER main ENGINE = Replicated('/clickhouse/databases/${DB}', '{shard}', '{replica}')` },
+    {
+      export: "events",
+      type: "ClickHouse::Table",
+      dependsOn: ["db"],
+      ddl: `CREATE TABLE ${DB}.events ON CLUSTER main (id UInt64, ts DateTime, kind String${extraColumn}) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/${DB}/events', '{replica}', ts) ORDER BY id`,
+    },
+    {
+      export: "byKind",
+      type: "ClickHouse::MaterializedView",
+      dependsOn: ["events"],
+      ddl: `CREATE MATERIALIZED VIEW ${DB}.by_kind ON CLUSTER main ENGINE = ReplicatedSummingMergeTree ORDER BY kind AS SELECT kind, count() AS n FROM ${DB}.events GROUP BY kind`,
+    },
+  ];
+  const apply = async (buildPath: string, topology?: string) => {
+    const sent: string[] = [];
+    const env: Record<string, string> = { CLICKHOUSE_URL: endpoint.url, ...(topology ? { CLICKHOUSE_TOPOLOGY: topology } : {}) };
+    if (endpoint.user) env.CLICKHOUSE_USER = endpoint.user;
+    if (endpoint.password) env.CLICKHOUSE_PASSWORD = endpoint.password;
+    const outcome = await clickhouseApply({ buildPath, environment: "test" }, undefined, {
+      config: { ownership: { stack: "yodel", env: "e2e" } },
+      env,
+      // The statements it sent; the log's other lines report each object.
+      log: (l) => void (/^[A-Z]+ /.test(l) && sent.push(l)),
+    }).catch((err: unknown) => ({ error: err as Error }));
+    return { outcome, sent };
+  };
+
+  test("as declared, the cluster's source fails on a single node; rendered for it, it applies and converges", async () => {
+    await q(`DROP DATABASE IF EXISTS ${DB} SYNC`);
+    const v1 = writeBuild("v1.json", objects());
+
+    const asDeclared = await apply(v1);
+    expect("error" in asDeclared.outcome || asDeclared.outcome.failed.length > 0).toBe(true);
+    await q(`DROP DATABASE IF EXISTS ${DB} SYNC`);
+
+    const first = await apply(v1, "single");
+    expect("error" in first.outcome).toBe(false);
+    expect(first.sent.join("\n")).not.toMatch(/ON CLUSTER|Replicated/);
+    expect(first.sent).toHaveLength(3);
+    const tables = await q<{ name: string; engine: string }>(`SELECT name, engine FROM system.tables WHERE database = '${DB}' ORDER BY name`);
+    expect(tables.map((t) => [t.name.startsWith(".inner") ? ".inner" : t.name, t.engine])).toEqual([
+      [".inner", "SummingMergeTree"],
+      ["by_kind", "MaterializedView"],
+      ["events", "ReplacingMergeTree"],
+    ]);
+
+    // Again: the server's plain engines are what the source renders to on a single node, so nothing is sent.
+    const again = await apply(v1, "single");
+    expect(again.sent).toEqual([]);
+    expect("error" in again.outcome ? again.outcome.error : again.outcome.notAttempted).toEqual([]);
+
+    // A change to the cluster's source applies as a single node's ALTER.
+    const v2 = writeBuild("v2.json", objects(", source String DEFAULT 'web'"));
+    const change = await apply(v2, "single");
+    expect(change.sent).toEqual([`ALTER TABLE \`${DB}\`.\`events\` ADD COLUMN source String DEFAULT 'web' AFTER \`kind\``]);
+    const cols = await q<{ name: string }>(`SELECT name FROM system.columns WHERE database = '${DB}' AND table = 'events' ORDER BY position`);
+    expect(cols.map((c) => c.name)).toEqual(["id", "ts", "kind", "source"]);
+  });
+});

package/src/clickhouse/topology.test.ts

@@ -0,0 +1,309 @@
+import { describe, expect, test } from "vitest";
+import { canonicalObject } from "./plan/normalize";
+import { diffSchemas } from "./plan/diff";
+import type { Change } from "./plan/diff";
+import { alterSteps, createStatement, dropStatement, type DeclaredObject } from "./apply/statements";
+import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "./entities";
+import { parseTopology, renderEngine, renderStatement, renderSteps, type Topology } from "./topology";
+
+const SINGLE: Topology = { kind: "single" };
+const CLUSTER: Topology = { kind: "cluster", cluster: "main" };
+const REPLICATED: Topology = { kind: "replicated" };
+const CLOUD: Topology = { kind: "cloud" };
+
+const EVENTS = `CREATE TABLE analytics.events (
+  id UInt64,
+  ts DateTime,
+  kind LowCardinality(String)
+)
+ENGINE = ReplacingMergeTree(ts)
+ORDER BY id
+COMMENT 'events'`;
+
+describe("CREATE TABLE per topology", () => {
+  test("a single node takes the declaration as written", () => {
+    expect(renderStatement(EVENTS, SINGLE)).toBe(EVENTS);
+  });
+
+  test("a cluster adds ON CLUSTER after the name and replicates the engine with its Keeper path and replica", () => {
+    expect(renderStatement(EVENTS, CLUSTER)).toBe(`CREATE TABLE analytics.events ON CLUSTER \`main\` (
+  id UInt64,
+  ts DateTime,
+  kind LowCardinality(String)
+)
+ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}', ts)
+ORDER BY id
+COMMENT 'events'`);
+  });
+
+  test("a cluster's Keeper path and replica name can be set, and a macro cluster name is a string literal", () => {
+    const t: Topology = { kind: "cluster", cluster: "{cluster}", replicaPath: "/ch/{shard}/{database}/{table}", replicaName: "{replica}" };
+    expect(renderStatement("CREATE TABLE db.t (id UInt64) ENGINE = MergeTree ORDER BY id", t)).toBe(
+      "CREATE TABLE db.t ON CLUSTER '{cluster}' (id UInt64) ENGINE = ReplicatedMergeTree('/ch/{shard}/{database}/{table}', '{replica}') ORDER BY id",
+    );
+  });
+
+  test("a Replicated database gets the Replicated family with no path arguments and no ON CLUSTER", () => {
+    expect(renderStatement(EVENTS, REPLICATED)).toBe(EVENTS.replace("ReplacingMergeTree(ts)", "ReplicatedReplacingMergeTree(ts)"));
+  });
+
+  test("Cloud takes the plain family, which it turns into SharedMergeTree itself", () => {
+    expect(renderStatement(EVENTS, CLOUD)).toBe(EVENTS);
+  });
+
+  describe("a declaration written for a cluster", () => {
+    const declared =
+      "CREATE TABLE IF NOT EXISTS db.t ON CLUSTER prod (id UInt64, v UInt32) ENGINE = ReplicatedSummingMergeTree('/p/{shard}/t', '{replica}', (v)) ORDER BY id";
+
+    test("keeps its own path on a cluster, under the topology's cluster name", () => {
+      expect(renderStatement(declared, CLUSTER)).toBe(
+        "CREATE TABLE IF NOT EXISTS db.t ON CLUSTER `main` (id UInt64, v UInt32) ENGINE = ReplicatedSummingMergeTree('/p/{shard}/t', '{replica}', (v)) ORDER BY id",
+      );
+    });
+
+    test("loses ON CLUSTER and replication on a single node", () => {
+      expect(renderStatement(declared, SINGLE)).toBe("CREATE TABLE IF NOT EXISTS db.t (id UInt64, v UInt32) ENGINE = SummingMergeTree((v)) ORDER BY id");
+    });
+
+    test("loses ON CLUSTER and the path arguments in a Replicated database", () => {
+      expect(renderStatement(declared, REPLICATED)).toBe(
+        "CREATE TABLE IF NOT EXISTS db.t (id UInt64, v UInt32) ENGINE = ReplicatedSummingMergeTree((v)) ORDER BY id",
+      );
+    });
+
+    test("loses ON CLUSTER and the path arguments on Cloud", () => {
+      expect(renderStatement(declared, CLOUD)).toBe("CREATE TABLE IF NOT EXISTS db.t (id UInt64, v UInt32) ENGINE = SummingMergeTree((v)) ORDER BY id");
+    });
+  });
+
+  test("ON CLUSTER goes after a UUID clause", () => {
+    expect(renderStatement("CREATE TABLE db.t UUID '0e9a7e46-0a7d-4c3a-8d47-3c1f2f9b0a11' (id UInt64) ENGINE = MergeTree ORDER BY id", CLUSTER)).toBe(
+      "CREATE TABLE db.t UUID '0e9a7e46-0a7d-4c3a-8d47-3c1f2f9b0a11' ON CLUSTER `main` (id UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') ORDER BY id",
+    );
+  });
+
+  test("an engine outside the MergeTree family is left alone; only ON CLUSTER changes", () => {
+    const log = "CREATE TABLE db.l (id UInt64) ENGINE = Log";
+    expect(renderStatement(log, REPLICATED)).toBe(log);
+    expect(renderStatement(log, CLUSTER)).toBe("CREATE TABLE db.l ON CLUSTER `main` (id UInt64) ENGINE = Log");
+  });
+
+  test("comments and spacing in the declaration survive", () => {
+    const sql = "CREATE TABLE db.t ( -- the key\n  id UInt64 /* id */\n)\nENGINE = MergeTree() -- engine\nORDER BY id";
+    expect(renderStatement(sql, REPLICATED)).toBe(sql.replace("MergeTree()", "ReplicatedMergeTree()"));
+  });
+});
+
+describe("views per topology", () => {
+  const mv = "CREATE MATERIALIZED VIEW db.counts ENGINE = SummingMergeTree ORDER BY kind AS SELECT kind, count() AS n FROM db.events GROUP BY kind";
+  const to = "CREATE MATERIALIZED VIEW db.feed TO db.events AS SELECT * FROM db.raw";
+
+  test("a materialized view's inner engine is replicated like a table's", () => {
+    expect(renderStatement(mv, CLUSTER)).toBe(
+      "CREATE MATERIALIZED VIEW db.counts ON CLUSTER `main` ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') ORDER BY kind AS SELECT kind, count() AS n FROM db.events GROUP BY kind",
+    );
+    expect(renderStatement(mv, REPLICATED)).toBe(mv.replace("SummingMergeTree", "ReplicatedSummingMergeTree"));
+    expect(renderStatement(mv, SINGLE)).toBe(mv);
+    expect(renderStatement(mv, CLOUD)).toBe(mv);
+  });
+
+  test("a view writing TO a table only gains ON CLUSTER", () => {
+    expect(renderStatement(to, CLUSTER)).toBe("CREATE MATERIALIZED VIEW db.feed ON CLUSTER `main` TO db.events AS SELECT * FROM db.raw");
+    for (const t of [SINGLE, REPLICATED, CLOUD]) expect(renderStatement(to, t)).toBe(to);
+  });
+
+  test("a SELECT inside a view is not mistaken for the view's own clauses", () => {
+    const v = "CREATE OR REPLACE VIEW db.v AS SELECT * FROM db.t ON CLUSTER_X JOIN db.u USING id";
+    expect(renderStatement(v, SINGLE)).toBe(v);
+  });
+});
+
+describe("CREATE DATABASE per topology", () => {
+  const db = "CREATE DATABASE IF NOT EXISTS analytics COMMENT 'the warehouse'";
+
+  test("single node and Cloud take it as written", () => {
+    expect(renderStatement(db, SINGLE)).toBe(db);
+    expect(renderStatement(db, CLOUD)).toBe(db);
+  });
+
+  test("a cluster adds ON CLUSTER", () => {
+    expect(renderStatement(db, CLUSTER)).toBe("CREATE DATABASE IF NOT EXISTS analytics ON CLUSTER `main` COMMENT 'the warehouse'");
+  });
+
+  test("the replicated topology makes it a Replicated database, on its cluster when it names one", () => {
+    expect(renderStatement(db, REPLICATED)).toBe(
+      "CREATE DATABASE IF NOT EXISTS analytics ENGINE = Replicated('/clickhouse/databases/analytics', '{shard}', '{replica}') COMMENT 'the warehouse'",
+    );
+    expect(renderStatement(db, { kind: "replicated", cluster: "all" })).toBe(
+      "CREATE DATABASE IF NOT EXISTS analytics ON CLUSTER `all` ENGINE = Replicated('/clickhouse/databases/analytics', '{shard}', '{replica}') COMMENT 'the warehouse'",
+    );
+  });
+
+  test("a declared Atomic engine becomes Replicated in the replicated topology, and a declared Replicated one keeps its arguments", () => {
+    expect(renderStatement("CREATE DATABASE `a` ENGINE = Atomic", REPLICATED)).toBe("CREATE DATABASE `a` ENGINE = Replicated('/clickhouse/databases/a', '{shard}', '{replica}')");
+    const r = "CREATE DATABASE a ENGINE = Replicated('/x/a', 's', 'r')";
+    expect(renderStatement(r, REPLICATED)).toBe(r);
+  });
+
+  test("a declared Replicated engine is dropped everywhere else", () => {
+    const r = "CREATE DATABASE a ENGINE = Replicated('/x/a', '{shard}', '{replica}') COMMENT 'c'";
+    expect(renderStatement(r, SINGLE)).toBe("CREATE DATABASE a COMMENT 'c'");
+    expect(renderStatement(r, CLOUD)).toBe("CREATE DATABASE a COMMENT 'c'");
+    expect(renderStatement(r, CLUSTER)).toBe("CREATE DATABASE a ON CLUSTER `main` COMMENT 'c'");
+  });
+});
+
+describe("the other DDL per topology", () => {
+  const cases: Array<[string, string]> = [
+    ["ALTER TABLE `db`.`t` ADD COLUMN `x` UInt8 AFTER `id`", "ALTER TABLE `db`.`t` ON CLUSTER `main` ADD COLUMN `x` UInt8 AFTER `id`"],
+    ["ALTER TABLE `db`.`t` MODIFY COMMENT 'c'", "ALTER TABLE `db`.`t` ON CLUSTER `main` MODIFY COMMENT 'c'"],
+    ["DROP TABLE `db`.`t` SYNC", "DROP TABLE `db`.`t` ON CLUSTER `main` SYNC"],
+    ["DROP VIEW IF EXISTS `db`.`v` SYNC", "DROP VIEW IF EXISTS `db`.`v` ON CLUSTER `main` SYNC"],
+    ["TRUNCATE TABLE db.t", "TRUNCATE TABLE db.t ON CLUSTER `main`"],
+    ["DETACH VIEW db.v", "DETACH VIEW db.v ON CLUSTER `main`"],
+    ["ATTACH TABLE db.v", "ATTACH TABLE db.v ON CLUSTER `main`"],
+    ["OPTIMIZE TABLE db.t FINAL", "OPTIMIZE TABLE db.t ON CLUSTER `main` FINAL"],
+    ["RENAME TABLE `db`.`a` TO `db`.`b`", "RENAME TABLE `db`.`a` TO `db`.`b` ON CLUSTER `main`"],
+    ["RENAME TABLE db.a TO db.b, db.c TO db.d;", "RENAME TABLE db.a TO db.b, db.c TO db.d ON CLUSTER `main`;"],
+    ["EXCHANGE TABLES db.a AND db.b", "EXCHANGE TABLES db.a AND db.b ON CLUSTER `main`"],
+    ["EXCHANGE TABLES db.a AND db.b SETTINGS x = 1", "EXCHANGE TABLES db.a AND db.b ON CLUSTER `main` SETTINGS x = 1"],
+  ];
+
+  test.each(cases)("%s gains ON CLUSTER on a cluster", (sql, onCluster) => {
+    expect(renderStatement(sql, CLUSTER)).toBe(onCluster);
+  });
+
+  test.each(cases)("%s is written without ON CLUSTER on every other topology", (sql, onCluster) => {
+    for (const t of [SINGLE, REPLICATED, CLOUD]) {
+      expect(renderStatement(sql, t)).toBe(sql);
+      expect(renderStatement(onCluster, t)).toBe(sql);
+    }
+  });
+
+  test("ALTER, RENAME and DROP DATABASE take ON CLUSTER on a cluster", () => {
+    expect(renderStatement("ALTER DATABASE `db` MODIFY COMMENT 'c'", CLUSTER)).toBe("ALTER DATABASE `db` ON CLUSTER `main` MODIFY COMMENT 'c'");
+    expect(renderStatement("RENAME DATABASE `a` TO `b`", CLUSTER)).toBe("RENAME DATABASE `a` TO `b` ON CLUSTER `main`");
+    expect(renderStatement("DROP DATABASE `db` SYNC", CLUSTER)).toBe("DROP DATABASE `db` ON CLUSTER `main` SYNC");
+  });
+
+  test("in the replicated topology only dropping a database carries its cluster", () => {
+    const t: Topology = { kind: "replicated", cluster: "all" };
+    expect(renderStatement("DROP DATABASE `db` SYNC", t)).toBe("DROP DATABASE `db` ON CLUSTER `all` SYNC");
+    expect(renderStatement("ALTER DATABASE `db` MODIFY COMMENT 'c'", t)).toBe("ALTER DATABASE `db` MODIFY COMMENT 'c'");
+    expect(renderStatement("ALTER TABLE db.t DROP COLUMN x", t)).toBe("ALTER TABLE db.t DROP COLUMN x");
+  });
+
+  test("a declared ON CLUSTER is replaced with the topology's cluster", () => {
+    expect(renderStatement("ALTER TABLE db.t ON CLUSTER old DROP COLUMN x", CLUSTER)).toBe("ALTER TABLE db.t ON CLUSTER `main` DROP COLUMN x");
+  });
+
+  test("KILL gains ON CLUSTER on a cluster, and is kept as written elsewhere", () => {
+    expect(renderStatement("KILL QUERY WHERE query_id = 'q' SYNC", CLUSTER)).toBe("KILL QUERY ON CLUSTER `main` WHERE query_id = 'q' SYNC");
+    const replicatedKill = "KILL QUERY ON CLUSTER 'db' WHERE query_id = 'q' SYNC";
+    expect(renderStatement(replicatedKill, REPLICATED)).toBe(replicatedKill);
+  });
+
+  test("statements that are not DDL come back unchanged", () => {
+    for (const sql of ["INSERT INTO db.t SELECT * FROM db.s", "SELECT 1", "SYSTEM SYNC REPLICA db.t LIGHTWEIGHT"]) {
+      for (const t of [SINGLE, CLUSTER, REPLICATED, CLOUD]) expect(renderStatement(sql, t)).toBe(sql);
+    }
+  });
+
+  test("a CREATE the parser does not take still gains ON CLUSTER", () => {
+    expect(renderStatement("CREATE TABLE db.t2 AS db.t", CLUSTER)).toBe("CREATE TABLE db.t2 ON CLUSTER `main` AS db.t");
+    expect(renderStatement("CREATE TABLE db.t2 ON CLUSTER x AS db.t", SINGLE)).toBe("CREATE TABLE db.t2 AS db.t");
+  });
+});
+
+describe("renderEngine", () => {
+  test("splits a Replicated engine's path and replica from the family's own arguments", () => {
+    expect(renderEngine({ name: "ReplicatedReplacingMergeTree", args: ["'/p'", "'{replica}'", "ver"] }, SINGLE)).toEqual({ name: "ReplacingMergeTree", args: ["ver"] });
+    expect(renderEngine({ name: "ReplicatedReplacingMergeTree", args: ["ver"] }, SINGLE)).toEqual({ name: "ReplacingMergeTree", args: ["ver"] });
+  });
+
+  test("leaves a Shared engine and the non-MergeTree engines alone", () => {
+    for (const name of ["SharedMergeTree", "Distributed", "Memory"]) {
+      const e = { name, args: ["a"] };
+      for (const t of [SINGLE, CLUSTER, REPLICATED, CLOUD]) expect(renderEngine(e, t)).toBe(e);
+    }
+  });
+
+  test("keeps an engine written without parentheses without them where it can", () => {
+    expect(renderEngine({ name: "MergeTree" }, REPLICATED)).toEqual({ name: "ReplicatedMergeTree" });
+    expect(renderEngine({ name: "ReplicatedMergeTree" }, CLOUD)).toEqual({ name: "MergeTree" });
+  });
+});
+
+describe("parseTopology", () => {
+  test("reads each topology's string form", () => {
+    expect(parseTopology("single")).toEqual(SINGLE);
+    expect(parseTopology("cluster:main")).toEqual(CLUSTER);
+    expect(parseTopology("cluster:{cluster}")).toEqual({ kind: "cluster", cluster: "{cluster}" });
+    expect(parseTopology("replicated")).toEqual(REPLICATED);
+    expect(parseTopology("replicated:all")).toEqual({ kind: "replicated", cluster: "all" });
+    expect(parseTopology("cloud")).toEqual(CLOUD);
+  });
+
+  test("refuses a cluster without a name, and anything unknown", () => {
+    expect(() => parseTopology("cluster")).toThrow(/needs the cluster's name/);
+    expect(() => parseTopology("single:x")).toThrow(/takes no argument/);
+    expect(() => parseTopology("sharded")).toThrow(/unknown topology/);
+  });
+});
+
+describe("the applier's own statements", () => {
+  const declared = (type: ClickHouseEntityType, exportName: string, ddl: string): DeclaredObject => {
+    const canonical = canonicalObject(ddl);
+    return { exportName, type, key: canonical.database ? `${canonical.database}.${canonical.name}` : canonical.name, ddl, canonical, dependsOn: [] };
+  };
+  const table = declared(CLICKHOUSE_ENTITY_TYPES.table, "events", "CREATE TABLE db.events (id UInt64, x UInt8) ENGINE = MergeTree ORDER BY id");
+  const marker = { stack: "s", env: "e" };
+  const added: Change = { object: "events", field: "columns.x", after: "UInt8", rule: "SQLCH201", class: "metadata" };
+
+  test("render for a cluster, with the marker comment and steps kept", () => {
+    const create = renderStatement(createStatement(table, marker), CLUSTER);
+    expect(create).toMatch(/^CREATE TABLE db\.events ON CLUSTER `main` \(id UInt64, x UInt8\) ENGINE = ReplicatedMergeTree\('\/clickhouse\/tables\/\{uuid\}\/\{shard\}', '\{replica\}'\) ORDER BY id COMMENT '/);
+    const steps = renderSteps(alterSteps(table, [added]), CLUSTER);
+    expect(steps).toEqual([{ sql: "ALTER TABLE `db`.`events` ON CLUSTER `main` ADD COLUMN x UInt8 AFTER `id`", rewrite: false, rule: "SQLCH201", class: "metadata" }]);
+    expect(renderStatement(dropStatement(CLICKHOUSE_ENTITY_TYPES.table, "db", "events"), CLUSTER)).toBe("DROP TABLE `db`.`events` ON CLUSTER `main` SYNC");
+  });
+
+  test("come back unchanged for a single node", () => {
+    const create = createStatement(table, marker);
+    expect(renderStatement(create, SINGLE)).toBe(create);
+    const steps = alterSteps(table, [added]);
+    expect(renderSteps(steps, SINGLE)).toEqual(steps);
+  });
+});
+
+describe("the classifier compares the engine a topology runs", () => {
+  const declared = "CREATE TABLE db.t (id UInt64, v UInt64) ENGINE = ReplacingMergeTree(v) ORDER BY id";
+  const changes = (live: string, topology: Topology) =>
+    diffSchemas([{ key: "t", canonical: canonicalObject(live) }], [{ key: "t", canonical: canonicalObject(renderStatement(declared, topology)) }]).changes;
+
+  test("a cluster's table, as the server prints it, is the rendered declaration", () => {
+    expect(changes("CREATE TABLE db.t (`id` UInt64, `v` UInt64) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}', v) ORDER BY id", CLUSTER)).toEqual([]);
+  });
+
+  test("a Replicated database's table, printed with the default path and replica, is the declaration with none", () => {
+    expect(changes("CREATE TABLE db.t (`id` UInt64, `v` UInt64) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}', v) ORDER BY id", REPLICATED)).toEqual([]);
+  });
+
+  test("Cloud's Shared engine is the plain family it was declared as", () => {
+    expect(changes("CREATE TABLE db.t (`id` UInt64, `v` UInt64) ENGINE = SharedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}', v) ORDER BY id", CLOUD)).toEqual([]);
+  });
+
+  test("a database declared with no engine takes whatever the server's default is, so a single node's Atomic is no change", () => {
+    const db = (live: string, declared: string) => diffSchemas([{ key: "d", canonical: canonicalObject(live) }], [{ key: "d", canonical: canonicalObject(declared) }]).changes;
+    const declaredForCluster = "CREATE DATABASE d ON CLUSTER main ENGINE = Replicated('/clickhouse/databases/d', '{shard}', '{replica}')";
+    expect(db("CREATE DATABASE d\nENGINE = Atomic", renderStatement(declaredForCluster, SINGLE))).toEqual([]);
+    expect(db("CREATE DATABASE d\nENGINE = Replicated('/clickhouse/databases/d', '{shard}', '{replica}')", renderStatement("CREATE DATABASE d", REPLICATED))).toEqual([]);
+    expect(db("CREATE DATABASE d\nENGINE = Atomic", "CREATE DATABASE d ENGINE = Lazy(60)")).toEqual([expect.objectContaining({ rule: "SQLCH232" })]);
+  });
+
+  test("the engine a topology does not run is still a change", () => {
+    expect(changes("CREATE TABLE db.t (`id` UInt64, `v` UInt64) ENGINE = ReplacingMergeTree(v) ORDER BY id", CLUSTER)).toEqual([
+      expect.objectContaining({ field: "engine", rule: "SQLCH223", class: "rebuild" }),
+    ]);
+  });
+});