@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/clickhouse/plan/stream.test.ts

@@ -0,0 +1,237 @@
+/**
+ * Stream engines (#4396): a change to a Kafka or S3Queue table is SQLCH225
+ * and not SQLCH223, the plan holds nothing back for it, the statements go
+ * detach, drop, create, attach, and a replacement that loses position says so.
+ */
+
+import { describe, expect, test } from "vitest";
+import { canonicalObject } from "./normalize";
+import { diffSchemas, type SchemaObject } from "./diff";
+import { renderDiff } from "./report";
+import { classifyDisruption } from "./disruption";
+import { CLASSIFIER_RULES } from "./rules";
+import { rebuildOpSuggestions } from "./rebuild-handoff";
+import { isStreamEngine, readsTable, STREAM_ENGINES, streamEngineArguments } from "./stream";
+import { planStatements, type DeclaredObject } from "../apply/statements";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
+import { observeRebuild, RebuildRefusal } from "../rebuild/observe";
+import type { ClickHouseTarget } from "../live/bind";
+
+const KAFKA = (topics: string, group = "ingest", extra = "") =>
+  `CREATE TABLE shop.events_stream (user_id UInt64, kind String) ENGINE = Kafka(kafka_creds) SETTINGS kafka_topic_list = '${topics}', kafka_group_name = '${group}', kafka_format = 'JSONEachRow'${extra}`;
+const S3QUEUE = (path: string, mode = "unordered") =>
+  `CREATE TABLE shop.files_stream (user_id UInt64) ENGINE = S3Queue(lake_creds) SETTINGS mode = '${mode}', keeper_path = '${path}'`;
+const MV = "CREATE MATERIALIZED VIEW shop.events_mv TO shop.events AS SELECT user_id, kind FROM shop.events_stream";
+const MV_OTHER = "CREATE MATERIALIZED VIEW shop.audit_mv TO shop.audit AS SELECT user_id FROM events_stream";
+const MV_UNRELATED = "CREATE MATERIALIZED VIEW shop.other_mv TO shop.other AS SELECT user_id FROM shop.users";
+
+const obj = (key: string, ddl: string): SchemaObject => ({ key, canonical: canonicalObject(ddl) });
+const one = (key: string, before: string, after: string) => diffSchemas([obj(key, before)], [obj(key, after)]);
+
+describe("stream engines", () => {
+  test("the six engines are the ones the pinned catalog has", () => {
+    expect([...STREAM_ENGINES].sort()).toEqual(["AzureQueue", "FileLog", "Kafka", "NATS", "RabbitMQ", "S3Queue"]);
+    expect(isStreamEngine("MergeTree")).toBe(false);
+    expect(isStreamEngine(undefined)).toBe(false);
+  });
+
+  test("the rule is its own class, with a rule id in the SQLCH2xx range", () => {
+    expect(CLASSIFIER_RULES.SQLCH225.class).toBe("replace");
+    expect(CLASSIFIER_RULES.SQLCH223.class).toBe("rebuild");
+  });
+});
+
+describe("classification", () => {
+  test("a different topic list is SQLCH225, not SQLCH223, and the plan refuses nothing", () => {
+    const diff = one("events_stream", KAFKA("events"), KAFKA("events,clicks"));
+    expect(diff.changes.map((c) => [c.rule, c.class, c.field])).toEqual([["SQLCH225", "replace", "settings.kafka_topic_list"]]);
+    expect(diff.rebuilds).toEqual([]);
+    expect(rebuildOpSuggestions(diff, new Map([["events_stream", canonicalObject(KAFKA("events,clicks"))]]), "prod")).toEqual([]);
+  });
+
+  test("a topic list given as an engine argument is the engine changing", () => {
+    const positional = (topic: string) => `CREATE TABLE shop.q (a String) ENGINE = Kafka('broker:9092', '${topic}', 'g', 'JSONEachRow')`;
+    const diff = one("q", positional("a"), positional("b"));
+    expect(diff.changes.map((c) => [c.rule, c.field])).toEqual([["SQLCH225", "engine"]]);
+  });
+
+  test("a format, a column and a setting added are all the stream rule", () => {
+    const diff = one(
+      "events_stream",
+      KAFKA("events"),
+      KAFKA("events", "ingest", ", kafka_num_consumers = 2").replace("kafka_format = 'JSONEachRow'", "kafka_format = 'CSV'").replace("kind String", "kind String, ts DateTime"),
+    );
+    expect(new Set(diff.changes.map((c) => c.rule))).toEqual(new Set(["SQLCH225"]));
+    expect(diff.changes.map((c) => c.field).sort()).toEqual(["columns.ts", "settings.kafka_format", "settings.kafka_num_consumers"]);
+  });
+
+  test("S3Queue, RabbitMQ and NATS tables are stream tables too", () => {
+    expect(one("f", S3QUEUE("/q/a"), S3QUEUE("/q/a", "ordered")).changes.map((c) => c.rule)).toEqual(["SQLCH225"]);
+    const nats = (subject: string) => `CREATE TABLE shop.n (a String) ENGINE = NATS(nats_creds) SETTINGS nats_subjects = '${subject}', nats_format = 'JSONEachRow'`;
+    expect(one("n", nats("a"), nats("b")).changes.map((c) => c.rule)).toEqual(["SQLCH225"]);
+    const rabbit = (queue: string) => `CREATE TABLE shop.r (a String) ENGINE = RabbitMQ(rabbit_creds) SETTINGS rabbitmq_queue_base = '${queue}', rabbitmq_format = 'JSONEachRow'`;
+    expect(one("r", rabbit("a"), rabbit("b")).changes.map((c) => c.rule)).toEqual(["SQLCH225"]);
+  });
+
+  test("a comment alone stays metadata", () => {
+    const diff = one("events_stream", KAFKA("events"), `${KAFKA("events")} COMMENT 'ingest'`);
+    expect(diff.changes.map((c) => c.rule)).toEqual(["SQLCH203"]);
+  });
+
+  test("a MergeTree engine change is still SQLCH223", () => {
+    const mt = (v: string) => `CREATE TABLE shop.t (a UInt64, v UInt64) ENGINE = ReplacingMergeTree(${v}) ORDER BY a`;
+    expect(one("t", mt("v"), mt("a")).changes.map((c) => c.rule)).toEqual(["SQLCH223"]);
+  });
+
+  test("a table that becomes a stream table is a rebuild with a note, and gets no Op suggestion", () => {
+    const mt = "CREATE TABLE shop.events_stream (user_id UInt64, kind String) ENGINE = MergeTree ORDER BY user_id";
+    const diff = one("events_stream", mt, KAFKA("events"));
+    expect(diff.changes[0]).toMatchObject({ rule: "SQLCH223", class: "rebuild" });
+    expect(diff.changes[0]!.note).toMatch(/stream engine/);
+    expect(rebuildOpSuggestions(diff, new Map([["events_stream", canonicalObject(KAFKA("events"))]]), "prod")).toEqual([]);
+  });
+
+  test("the report counts the class and names the views that are detached around the replacement", () => {
+    const diff = diffSchemas(
+      [obj("events_stream", KAFKA("events")), obj("events_mv", MV), obj("audit_mv", MV_OTHER), obj("other_mv", MV_UNRELATED)],
+      [obj("events_stream", KAFKA("clicks")), obj("events_mv", MV), obj("audit_mv", MV_OTHER), obj("other_mv", MV_UNRELATED)],
+    );
+    const text = renderDiff(diff);
+    expect(text).toMatch(/\[replace table\] settings\.kafka_topic_list/);
+    expect(text).toMatch(/SQLCH225 Change a stream table/);
+    expect(text).toMatch(/1 replace table/);
+    expect(text).toMatch(/hint: shop\.events_stream is replaced: shop\.events_mv, shop\.audit_mv read it and are detached first and attached after/);
+    expect(text).not.toMatch(/Refused/);
+  });
+});
+
+describe("position", () => {
+  test("a different kafka_group_name warns, naming both groups", () => {
+    const [c] = one("events_stream", KAFKA("events", "ingest"), KAFKA("events", "ingest-v2")).changes;
+    expect(c!.rule).toBe("SQLCH225");
+    expect(c!.note).toMatch(/loses its position: kafka_group_name changes from ingest to ingest-v2/);
+    expect(c!.note).toMatch(/committed offsets, or from the start of the topic/);
+  });
+
+  test("a group given as an engine argument is read too", () => {
+    const positional = (g: string) => `CREATE TABLE shop.q (a String) ENGINE = Kafka('broker:9092', 't', '${g}', 'JSONEachRow')`;
+    expect(one("q", positional("g1"), positional("g2")).changes[0]!.note).toMatch(/from g1 to g2/);
+  });
+
+  test("a topic list change keeps the group, so there is no warning", () => {
+    expect(one("events_stream", KAFKA("events"), KAFKA("clicks")).changes[0]!.note).toBeUndefined();
+  });
+
+  test("a different S3Queue keeper_path warns that every file is processed again", () => {
+    const [c] = one("files_stream", S3QUEUE("/q/a"), S3QUEUE("/q/b")).changes;
+    expect(c!.rule).toBe("SQLCH225");
+    expect(c!.note).toMatch(/keeper_path changes from \/q\/a to \/q\/b: .*every file already seen is processed again/);
+  });
+
+  test("the same keeper_path with another mode does not warn", () => {
+    expect(one("files_stream", S3QUEUE("/q/a"), S3QUEUE("/q/a", "ordered")).changes[0]!.note).toBeUndefined();
+  });
+});
+
+describe("reading a table", () => {
+  const table = canonicalObject(KAFKA("events"));
+  test("a view reads it by a bare name in its own database, or by a qualified one", () => {
+    expect(readsTable(canonicalObject(MV), table)).toBe(true);
+    expect(readsTable(canonicalObject(MV_OTHER), table)).toBe(true);
+    expect(readsTable(canonicalObject("CREATE MATERIALIZED VIEW audit.mv TO audit.t AS SELECT a FROM shop.events_stream"), table)).toBe(true);
+  });
+  test("a view of another table, another database's table of the same name, or a plain view is not a reader", () => {
+    expect(readsTable(canonicalObject(MV_UNRELATED), table)).toBe(false);
+    expect(readsTable(canonicalObject("CREATE MATERIALIZED VIEW audit.mv TO audit.t AS SELECT a FROM events_stream"), table)).toBe(false);
+    expect(readsTable(canonicalObject("CREATE VIEW shop.v AS SELECT a FROM shop.events_stream"), table)).toBe(false);
+    expect(readsTable(canonicalObject("CREATE MATERIALIZED VIEW shop.mv TO shop.t AS SELECT a FROM shop.events_stream_old"), table)).toBe(false);
+  });
+  test("engine arguments split outside quotes and parentheses", () => {
+    expect(streamEngineArguments("Kafka('a, b:9092', 't', 'g', 'JSONEachRow')")).toEqual(["a, b:9092", "t", "g", "JSONEachRow"]);
+    expect(streamEngineArguments("Kafka")).toEqual([]);
+  });
+});
+
+const declared = (key: string, type: DeclaredObject["type"], ddl: string): DeclaredObject => ({ exportName: key, type, key, ddl, canonical: canonicalObject(ddl), dependsOn: [] });
+
+describe("the statements", () => {
+  const before = [KAFKA("events"), MV, MV_OTHER, MV_UNRELATED].map((d) => canonicalObject(d));
+  const names = ["shop.events_stream", "shop.events_mv", "shop.audit_mv", "shop.other_mv"];
+  const current = new Map(names.map((n, i) => [n, before[i]!]));
+  const after = [
+    declared("shop.events_stream", CLICKHOUSE_ENTITY_TYPES.table, KAFKA("clicks")),
+    declared("shop.events_mv", CLICKHOUSE_ENTITY_TYPES.materializedView, MV),
+    declared("shop.audit_mv", CLICKHOUSE_ENTITY_TYPES.materializedView, MV_OTHER),
+    declared("shop.other_mv", CLICKHOUSE_ENTITY_TYPES.materializedView, MV_UNRELATED),
+  ];
+  const changes = diffSchemas(
+    names.map((n, i) => ({ key: n, canonical: before[i]! })),
+    after.map((o) => ({ key: o.key, canonical: o.canonical })),
+  ).changes;
+  const stepsOf = (plan: ReturnType<typeof planStatements>, i = 0) => (plan.objects[i] as { steps: { sql: string; rule: string; class: string }[] }).steps;
+
+  test("detach the views that read the table, replace it, attach them again", () => {
+    const plan = planStatements({ declared: after, changes, current });
+    expect(plan.objects[0]!.verdict).toBe("alter");
+    expect(stepsOf(plan).map((s) => s.sql.replace(/\s+/g, " "))).toEqual([
+      "DETACH TABLE `shop`.`events_mv`",
+      "DETACH TABLE `shop`.`audit_mv`",
+      "DROP TABLE `shop`.`events_stream` SYNC",
+      expect.stringMatching(/^CREATE TABLE shop\.events_stream \(user_id UInt64, kind String\) ENGINE = Kafka\(kafka_creds\) SETTINGS kafka_topic_list = 'clicks', .* COMMENT '.*managed-by=chant/),
+      "ATTACH TABLE `shop`.`events_mv`",
+      "ATTACH TABLE `shop`.`audit_mv`",
+    ]);
+    expect(stepsOf(plan).every((s) => s.rule === "SQLCH225" && s.class === "replace")).toBe(true);
+  });
+
+  test("the views themselves have nothing to do, and the unrelated one is not touched", () => {
+    const plan = planStatements({ declared: after, changes, current });
+    expect(plan.objects.slice(1).map((o) => [o.verdict, (o as { steps: unknown[] }).steps.length])).toEqual([
+      ["alter", 0],
+      ["alter", 0],
+      ["alter", 0],
+    ]);
+  });
+
+  test("a view the build drops in the same plan is not detached", () => {
+    const gone = [...changes, { object: "shop.audit_mv", field: "object", rule: "SQLCH250" as const, class: "drop" as const }];
+    const plan = planStatements({ declared: after.slice(0, 2), changes: gone, current });
+    expect(stepsOf(plan).filter((s) => s.sql.startsWith("DETACH")).map((s) => s.sql)).toEqual(["DETACH TABLE `shop`.`events_mv`"]);
+  });
+
+  test("on a cluster each statement runs ON CLUSTER", () => {
+    const plan = planStatements({ declared: after, changes, current, topology: { kind: "cluster", cluster: "main" } as never });
+    expect(stepsOf(plan).every((s) => /ON CLUSTER/.test(s.sql))).toBe(true);
+  });
+});
+
+describe("classifyDisruption", () => {
+  test("a stream engine's setting replaces; a MergeTree setting stays in place", () => {
+    const out = classifyDisruption({
+      environment: "prod",
+      changes: [
+        { name: "events_stream", type: "ClickHouse::Table", deltas: [{ path: "settings.kafka_topic_list" } as never] },
+        { name: "events", type: "ClickHouse::Table", deltas: [{ path: "settings.merge_with_ttl_timeout" } as never] },
+      ],
+    });
+    expect(out.events_stream).toMatchObject({ disruption: "replace" });
+    expect(out.events_stream!.detail).toMatch(/SQLCH225/);
+    expect(out.events).toMatchObject({ disruption: "in-place" });
+  });
+});
+
+describe("ClickHouseRebuildOp's observation", () => {
+  const target = { endpoint: { url: "http://unreachable.invalid" }, defaultDatabase: "shop" } as unknown as ClickHouseTarget;
+
+  test.each([
+    ["Kafka", KAFKA("events")],
+    ["S3Queue", S3QUEUE("/q/a")],
+  ])("refuses a %s table by name, before it asks the server anything", async (engine, ddl) => {
+    const o = declared("shop.t", CLICKHOUSE_ENTITY_TYPES.table, ddl);
+    const err = await observeRebuild(target, o, undefined).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(RebuildRefusal);
+    expect((err as Error).message).toContain(`the ${engine} engine`);
+    expect((err as Error).message).toMatch(/SQLCH225/);
+    expect((err as Error).message).toMatch(/stores no rows/);
+  });
+});

package/src/clickhouse/plan/stream.ts

@@ -0,0 +1,111 @@
+/**
+ * Stream engines (#4396): Kafka, S3Queue, AzureQueue, RabbitMQ, NATS and
+ * FileLog. A table on one of them stores no rows. It is a consumer: the
+ * materialized views that read it pull messages (or files) through it into a
+ * target, and the position it has reached lives outside the table, in the
+ * broker's committed offsets or in Keeper.
+ *
+ * What follows from that, and what this file holds:
+ *
+ * - There is no ALTER for the engine's arguments, and nothing to copy to a
+ *   new table, so a change is neither `metadata` nor a `rebuild`: the table is
+ *   replaced, with the views that read it detached around the replacement
+ *   (SQLCH225, `./rules.ts`).
+ * - A replacement can lose the table's position. The two changes that do
+ *   are found here and reported on the change as a note.
+ */
+
+import type { CanonicalObject } from "./normalize";
+
+/** The stream engines, as `system.tables.engine` names them. */
+export const STREAM_ENGINES: ReadonlySet<string> = new Set(["Kafka", "S3Queue", "AzureQueue", "RabbitMQ", "NATS", "FileLog"]);
+
+/** Whether an engine name is a stream engine. */
+export const isStreamEngine = (engine: string | undefined): boolean => engine !== undefined && STREAM_ENGINES.has(engine);
+
+/** Whether a canonical table is on a stream engine. */
+export const isStreamTable = (o: CanonicalObject): boolean => o.kind === "table" && isStreamEngine(o.engineName);
+
+/** The arguments inside an engine's parentheses, split at top-level commas, string quotes removed. Empty when it has none. */
+export function streamEngineArguments(engine: string | undefined): string[] {
+  const open = engine?.indexOf("(") ?? -1;
+  if (engine === undefined || open < 0 || !engine.endsWith(")")) return [];
+  const inner = engine.slice(open + 1, -1);
+  const out: string[] = [];
+  let depth = 0;
+  let quote: string | undefined;
+  let cur = "";
+  for (let i = 0; i < inner.length; i++) {
+    const ch = inner[i]!;
+    if (quote) {
+      cur += ch;
+      if (ch === "\\") cur += inner[++i] ?? "";
+      else if (ch === quote) quote = undefined;
+      continue;
+    }
+    if (ch === "'" || ch === '"' || ch === "`") quote = ch;
+    if (ch === "(" || ch === "[") depth++;
+    if (ch === ")" || ch === "]") depth--;
+    if (ch === "," && depth === 0) {
+      out.push(cur.trim());
+      cur = "";
+    } else cur += ch;
+  }
+  if (cur.trim()) out.push(cur.trim());
+  return out.map((a) => (/^'.*'$/.test(a) ? a.slice(1, -1) : a));
+}
+
+/** Kafka's positional arguments, in the order the engine takes them. */
+const KAFKA_POSITIONAL = ["kafka_broker_list", "kafka_topic_list", "kafka_group_name", "kafka_format"];
+
+/** A Kafka setting, from SETTINGS or from the engine's positional arguments. */
+function kafkaSetting(o: CanonicalObject, name: string): string | undefined {
+  const at = KAFKA_POSITIONAL.indexOf(name);
+  return o.settings[name] ?? (at >= 0 ? streamEngineArguments(o.engine)[at] : undefined);
+}
+
+/**
+ * What a replacement would lose, in a sentence each (#4396):
+ *
+ * - a different `kafka_group_name` is a different consumer group: it starts
+ *   from the group's committed offsets, or from the topic's start when the
+ *   group has none (`auto.offset.reset`);
+ * - a different S3Queue `keeper_path` is a different place for the table's
+ *   record of processed files, so every file already seen is processed again.
+ *
+ * Neither is a refusal: a new group or path is sometimes the point.
+ */
+export function positionWarnings(before: CanonicalObject, after: CanonicalObject): string[] {
+  const out: string[] = [];
+  if (before.engineName === "Kafka" && after.engineName === "Kafka") {
+    const b = kafkaSetting(before, "kafka_group_name");
+    const a = kafkaSetting(after, "kafka_group_name");
+    if (b !== a) {
+      out.push(
+        `kafka_group_name changes from ${b ?? "none"} to ${a ?? "none"}: the new group starts from its own committed offsets, or from the start of the topic when it has none, ` +
+          "so messages already consumed may be read again or skipped",
+      );
+    }
+  }
+  if (before.engineName === "S3Queue" && after.engineName === "S3Queue") {
+    const b = before.settings.keeper_path;
+    const a = after.settings.keeper_path;
+    if (b !== a) {
+      out.push(
+        `keeper_path changes from ${b ?? "none"} to ${a ?? "none"}: the new path holds no record of processed files, so every file already seen is processed again`,
+      );
+    }
+  }
+  return out;
+}
+
+/** Whether a materialized view reads from the table: its SELECT names the table, qualified or, in the table's own database, bare. */
+export function readsTable(view: CanonicalObject, table: CanonicalObject): boolean {
+  if (view.kind !== "materializedView" || view.select === undefined) return false;
+  const escaped = table.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const sameDatabase = view.database === table.database;
+  const database = table.database?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  // The canonical SELECT drops the view's own database from a name, so a bare name is a table in the view's database.
+  const names = [...(sameDatabase ? [escaped] : []), ...(database ? [`${database}\\s*\\.\\s*${escaped}`] : [])];
+  return names.some((n) => new RegExp(`(?:^|[^\\w.])${n}(?![\\w.])`).test(view.select!));
+}

package/src/clickhouse/rebuild/observe.ts

@@ -27,7 +27,9 @@
  *
  * Anything else is refused with a {@link RebuildRefusal}: a database that is
  * neither Atomic nor Replicated, a table that is not there, a change that is
- * not a rebuild, a working object chant did not make.
+ * not a rebuild, a working object chant did not make, a table on a stream
+ * engine (Kafka, S3Queue, AzureQueue, RabbitMQ, NATS, FileLog), which holds no
+ * rows to copy and is replaced by the applier instead (#4396).
  *
  * A Replicated database (#3249) is Atomic on every replica, with its DDL run
  * on each through Keeper, so the same EXCHANGE swaps the table everywhere.
@@ -47,6 +49,7 @@ import { dropFormattingOnly } from "../plan/server-format";
 import { CLASSIFIER_RULES } from "../plan/rules";
 import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
 import { carriesMarker, readTrailerPairs, REBUILD_TRAILER_KEY, stripMarkerFromStatement } from "../ownership";
+import { isStreamEngine } from "../plan/stream";
 import { ident, isRebuild, qualifiedIdent, sqlString, type DeclaredObject } from "../apply/statements";
 
 /** How the new table keeps up with writes to the old one while it is filled. */
@@ -250,10 +253,20 @@ function ownWorkingObject(o: ServerObject | undefined, names: RebuildNames, role
   return o;
 }
 
+/** The refusal for a stream-engine table, naming the engine and where its change goes instead (#4396). */
+function streamRefusal(key: string, engine: string, side: "declared" | "live"): RebuildRefusal {
+  return new RebuildRefusal(
+    `${key} is ${side === "declared" ? "declared on" : "on the server as"} the ${engine} engine, a stream engine: it stores no rows, so there is nothing to backfill, and a direct SELECT from it either fails ` +
+      `or consumes messages. A change to it is SQLCH225 (replace a stream table), which the applier makes in place: it detaches the materialized views that read the table, drops and creates it, and attaches the views again ` +
+      `(ApplyOp with target "clickhouse").`,
+  );
+}
+
 /** Observe the rebuild of the declared table on the server. */
 export async function observeRebuild(target: ClickHouseTarget, declared: DeclaredObject, marker: OwnershipMarker | undefined): Promise<RebuildObservation> {
   const database = declared.canonical.database ?? target.defaultDatabase;
   const names = rebuildNames(database, declared.canonical.name);
+  if (isStreamEngine(declared.canonical.engineName)) throw streamRefusal(names.key, declared.canonical.engineName!, "declared");
 
   const [db] = await clickhouseQuery<{ engine: string }>(target.endpoint, `SELECT engine FROM system.databases WHERE name = ${sqlString(database)}`);
   if (!db) throw new RebuildRefusal(`database ${database} is not on the server, so there is no ${names.key} to rebuild; the applier creates it`);
@@ -272,6 +285,7 @@ export async function observeRebuild(target: ClickHouseTarget, declared: Declare
   // Between EXCHANGE and the end of the swap, the table's own name holds the
   // new table, still carrying role=new, and the new table's name holds the
   // old table until it is renamed; the swap step finishes that.
+  if (isStreamEngine(liveRow.engine)) throw streamRefusal(names.key, liveRow.engine, "live");
   const exchanged = liveRow.pairs?.get(REBUILD_TRAILER_KEY) === names.key && liveRow.pairs.get("role") === "new";
   const newTable = exchanged ? undefined : ownWorkingObject(objects.get(names.newName), names, "new", marker);
   const exchangedOld = exchanged ? objects.get(names.newName) : undefined;

package/src/clickhouse/review-catalog.test.ts

@@ -0,0 +1,181 @@
+import { describe, expect, test } from "vitest";
+import { MERGE_TREE_SETTINGS, TABLE_ENGINES, TYPE_FAMILIES } from "../generated/clickhouse";
+import {
+  CDC_DELETED_COLUMN,
+  CDC_VERSION_COLUMN,
+  REVIEW_RULES,
+  constraintReview,
+  engineReview,
+  findingText,
+  lowCardinalityReview,
+  nullableReview,
+  sortKeyReview,
+} from "./review-catalog";
+
+const types = { id: "Int64", ts: "DateTime64(3)", tenant: "String", maybe: "Nullable(Int32)" };
+
+describe("sortKeyReview", () => {
+  test("a columnstore ORDER outranks a clustering key, a primary key and a unique key", () => {
+    const r = sortKeyReview({
+      object: "t",
+      columnTypes: types,
+      candidates: [
+        { kind: "unique-key", columns: ["tenant"], source: "UNIQUE u" },
+        { kind: "primary-key", columns: ["id"], source: "PRIMARY KEY pk" },
+        { kind: "clustering-key", columns: ["ts"], source: "CLUSTERED INDEX cx" },
+        { kind: "explicit-order", columns: ["tenant", "ts"], source: "COLUMNSTORE ORDER" },
+      ],
+    });
+    expect(r.orderBy).toEqual(["tenant", "ts"]);
+    expect(r.finding.ruleId).toBe("SQLRV001");
+    expect(r.finding.evidence[0]).toBe("COLUMNSTORE ORDER (tenant, ts)");
+    // The three it passed over are named in the order of their rank.
+    expect(r.finding.evidence.slice(1)).toEqual(["passed over: CLUSTERED INDEX cx (ts)", "passed over: PRIMARY KEY pk (id)", "passed over: UNIQUE u (tenant)"]);
+  });
+
+  test("with no candidate the key is tuple() and the finding says why", () => {
+    const r = sortKeyReview({ object: "t", columnTypes: types, candidates: [] });
+    expect(r.orderBy).toEqual([]);
+    expect(r.finding.message).toContain("tuple()");
+    expect(r.finding.evidence).toEqual(["no clustering key, primary key or unique key in the source"]);
+  });
+
+  test("a Nullable key column sets allow_nullable_key, a setting the pinned server has", () => {
+    const r = sortKeyReview({ object: "t", columnTypes: types, candidates: [{ kind: "primary-key", columns: ["maybe"], source: "PRIMARY KEY pk" }] });
+    expect(r.settings).toEqual({ allow_nullable_key: "1" });
+    expect(Object.keys(MERGE_TREE_SETTINGS)).toContain("allow_nullable_key");
+    expect(r.finding.evidence.join(" ")).toContain("allow_nullable_key = 1");
+  });
+
+  test("a descending column is said to be ascending here", () => {
+    const r = sortKeyReview({ object: "t", columnTypes: types, candidates: [{ kind: "clustering-key", columns: ["ts"], descending: ["ts"], source: "CLUSTERED INDEX cx" }] });
+    expect(r.finding.evidence).toContain("descending in the source, ascending here: ts");
+  });
+});
+
+describe("engineReview", () => {
+  test("an append table is MergeTree and adds no column", () => {
+    const r = engineReview({ object: "t", mode: "append", uniqueKey: ["id"] });
+    expect(r.engine).toBe("MergeTree");
+    expect(r.extraColumns).toEqual([]);
+    expect(r.finding.evidence).toContain("the source enforces uniqueness on (id); MergeTree does not");
+  });
+
+  test("a CDC mirror is ReplacingMergeTree(_version, _deleted) with the two columns", () => {
+    const r = engineReview({ object: "t", mode: "cdc", uniqueKey: ["id"] });
+    expect(r.engine).toBe("ReplacingMergeTree(_version, _deleted)");
+    expect(r.extraColumns).toEqual([CDC_VERSION_COLUMN, CDC_DELETED_COLUMN]);
+  });
+
+  test("a CDC mirror takes the caller's names for the two columns, in the engine, the columns and the advice (#4399)", () => {
+    const r = engineReview({ object: "t", mode: "cdc", cdcColumns: { version: "_peerdb_version", deleted: "_peerdb_is_deleted" } });
+    expect(r.engine).toBe("ReplacingMergeTree(_peerdb_version, _peerdb_is_deleted)");
+    expect(r.extraColumns).toEqual([
+      { name: "_peerdb_version", type: CDC_VERSION_COLUMN.type },
+      { name: "_peerdb_is_deleted", type: CDC_DELETED_COLUMN.type },
+    ]);
+    expect(r.finding.message).toContain("_peerdb_version and _peerdb_is_deleted");
+    expect(r.finding.suggestion).toContain("highest _peerdb_version");
+    expect(r.finding.suggestion).toContain("filter on _peerdb_is_deleted,");
+  });
+
+  test("the engines it writes exist in the pinned catalog and take the arguments it gives", () => {
+    expect(Object.keys(TABLE_ENGINES)).toContain("MergeTree");
+    const replacing = TABLE_ENGINES.ReplacingMergeTree;
+    expect(replacing.args?.map((a) => a.name)).toEqual(["ver", "is_deleted"]);
+    expect(replacing.mergeTree).toBe(true);
+    expect(CDC_VERSION_COLUMN.type in TYPE_FAMILIES).toBe(true);
+    expect(CDC_DELETED_COLUMN.type in TYPE_FAMILIES).toBe(true);
+  });
+});
+
+describe("nullableReview", () => {
+  test("a NOT NULL column keeps its type and raises nothing", () => {
+    expect(nullableReview({ object: "t", column: "id", type: "Int64", sourceNullable: false })).toEqual({ type: "Int64" });
+  });
+
+  test("a nullable column with a default says the default could replace Nullable", () => {
+    const r = nullableReview({ object: "t", column: "n", type: "Int32", sourceNullable: true, defaultExpr: "0" });
+    expect(r.type).toBe("Nullable(Int32)");
+    expect(r.finding?.evidence).toEqual(["n allows NULL in the source", "it has the default 0"]);
+    expect(r.finding?.suggestion).toContain("Int32 DEFAULT 0");
+  });
+
+  test("a nullable column with no default says so", () => {
+    const r = nullableReview({ object: "t", column: "n", type: "String", sourceNullable: true });
+    expect(r.type).toBe("Nullable(String)");
+    expect(r.finding?.evidence).toEqual(["n allows NULL in the source", "it has no default"]);
+  });
+
+  test("Nullable is a family of the pinned catalog", () => {
+    expect("Nullable" in TYPE_FAMILIES && "LowCardinality" in TYPE_FAMILIES).toBe(true);
+  });
+});
+
+describe("lowCardinalityReview", () => {
+  const at = (column: string, type = "String", maxLength?: number) => lowCardinalityReview({ object: "t", column, type, ...(maxLength !== undefined ? { maxLength } : {}) });
+
+  test("a String column named for a small set is a candidate and keeps its type", () => {
+    for (const name of ["status", "order_status", "OrderStatus", "country_code", "currency"]) {
+      const f = at(name);
+      expect(f?.ruleId, name).toBe("SQLRV004");
+      expect(f?.message).toContain("stays String");
+    }
+  });
+
+  test("the declared length is evidence when it is short", () => {
+    expect(at("status", "String", 20)?.evidence).toContain("the declared length is 20");
+    expect(at("status", "String", 4000)?.evidence.join(" ")).not.toContain("declared length");
+  });
+
+  test("other names, other types and words that merely end the same way are not candidates", () => {
+    expect(at("name")).toBeUndefined();
+    expect(at("birthday")).toBeUndefined();
+    expect(at("prostatus")).toBeUndefined();
+    expect(at("status", "Int32")).toBeUndefined();
+  });
+
+  test("a Nullable(String) column is a candidate and the suggestion names the nested form", () => {
+    expect(at("status", "Nullable(String)")?.suggestion).toContain("LowCardinality(Nullable(String))");
+  });
+});
+
+describe("constraintReview", () => {
+  test("a foreign key, a unique constraint and a check are comments and findings", () => {
+    const r = constraintReview({
+      object: "t",
+      constraints: [
+        { kind: "foreign-key", name: "fk", columns: ["a"], references: { table: "u", columns: ["id"] } },
+        { kind: "unique", name: "uq", columns: ["b", "c"] },
+        { kind: "check", name: "ck", expression: "([x] > 0\n AND [x] < 10)" },
+      ],
+    });
+    expect(r.comments).toEqual(["-- FOREIGN KEY fk (a) REFERENCES u (id)", "-- UNIQUE uq (b, c)", "-- CHECK ck ([x] > 0 AND [x] < 10)"]);
+    expect(r.findings.map((f) => f.ruleId)).toEqual(["SQLRV005", "SQLRV005", "SQLRV005"]);
+    expect(r.findings[0]?.column).toBe("a");
+    expect(r.findings[1]?.column).toBeUndefined();
+    expect(r.comments.every((c) => !c.includes("\n"))).toBe(true);
+  });
+});
+
+describe("constraintReview: an exclusion constraint (#4399)", () => {
+  test("is a comment and a finding with no column", () => {
+    const r = constraintReview({ object: "t", constraints: [{ kind: "exclusion", name: "no_overlap", expression: "USING gist (during WITH &&)" }] });
+    expect(r.comments).toEqual(["-- EXCLUDE no_overlap USING gist (during WITH &&)"]);
+    expect(r.findings[0]).toMatchObject({ ruleId: "SQLRV005", severity: "warning" });
+    expect(r.findings[0]?.column).toBeUndefined();
+    expect(r.findings[0]?.message).toContain("Exclusion constraint no_overlap is not enforced");
+  });
+});
+
+describe("the rules", () => {
+  test("every rule has a unique id and a name", () => {
+    const names = Object.values(REVIEW_RULES).map((r) => r.name);
+    expect(new Set(names).size).toBe(names.length);
+    for (const [id, r] of Object.entries(REVIEW_RULES)) expect(r.id).toBe(id);
+  });
+
+  test("findingText leads with the object and column, then evidence and suggestion", () => {
+    expect(findingText({ ruleId: "SQLRV003", severity: "warning", object: "t", column: "c", message: "m.", evidence: ["a", "b"], suggestion: "s." })).toBe("t.c: m. Evidence: a; b. Suggestion: s.");
+  });
+});