@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/clickhouse/plan/diff.ts

@@ -0,0 +1,203 @@
+/**
+ * The diff between two ClickHouse schemas, each change classified.
+ *
+ * Identity (#3047 question 3):
+ *
+ * - An object is identified by its export name between two revisions of the
+ *   declarations, so a changed name in the SQL under the same export is a
+ *   rename, not a drop and a create. Against a live server, which has no
+ *   export names, it is identified by `database.name`, and a declaration
+ *   carrying `-- previously: <old name>` before its CREATE is matched to the
+ *   object of that name.
+ * - A column is identified by its name: the SQL form has no column key apart
+ *   from the name. A column renamed in place says so with
+ *   `-- previously: <old name>` on its line; without the hint a renamed
+ *   column reads as a drop and an add, and the diff points that out when the
+ *   two look alike.
+ */
+
+import { CLASSIFIER_RULES, type ChangeClass, type ClassifierRuleId } from "./rules";
+import type { CanonicalObject } from "./normalize";
+import type { RebuildOpSuggestion } from "./rebuild-handoff";
+import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse";
+import { classifiedChange, type ChangeSet, type ClassifiedChange } from "../../core/classifier";
+import { changedEntries, matchByIdentity, type Keyed } from "../../core/diff";
+
+export type SchemaObject = Keyed<CanonicalObject>;
+
+export type Change = ClassifiedChange<ClassifierRuleId, ChangeClass>;
+
+export interface SchemaDiff extends ChangeSet<Change> {
+  /** The rebuilds a plan must refuse to make in place. */
+  rebuilds: Change[];
+  /** For each table refused as a rebuild, the rebuild migration Op to run instead (`./rebuild-handoff.ts`). */
+  rebuildOps?: RebuildOpSuggestion[];
+}
+
+const qualified = (o: CanonicalObject) => (o.database ? `${o.database}.${o.name}` : o.name);
+
+function change(object: string, field: string, rule: ClassifierRuleId, before?: unknown, after?: unknown, extra: Partial<Change> = {}): Change {
+  return classifiedChange(CLASSIFIER_RULES, object, field, rule, before, after, extra);
+}
+
+/** The column names a key expression mentions. */
+function keyColumns(o: CanonicalObject): Set<string> {
+  const out = new Set<string>();
+  for (const k of [o.orderBy, o.primaryKey ?? o.orderBy, o.partitionBy, o.sampleBy]) {
+    for (const word of (k ?? "").split(" ")) if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(word)) out.add(word);
+  }
+  return out;
+}
+
+const elements = (key: string | undefined) => (key ? key.replace(/^\( (.*) \)$/, "$1").split(" , ") : []);
+
+/** An ORDER BY change that only appends columns the same change adds (MODIFY ORDER BY). */
+function appendsAddedColumns(before: string | undefined, after: string | undefined, added: Set<string>): boolean {
+  const b = elements(before);
+  const a = elements(after);
+  if (a.length <= b.length || b.some((e, i) => a[i] !== e)) return false;
+  return a.slice(b.length).every((e) => added.has(e));
+}
+
+function diffColumns(key: string, before: CanonicalObject, after: CanonicalObject, out: Change[], hints: string[]): Set<string> {
+  const keys = keyColumns(before);
+  const pk = new Set(elements(before.primaryKey ?? before.orderBy).flatMap((e) => e.split(" ")));
+  const byName = new Map(before.columns.map((c) => [c.name, c]));
+  const afterNames = new Set(after.columns.map((c) => c.name));
+  const matched = new Set<string>();
+  const added = new Set<string>();
+
+  for (const a of after.columns) {
+    let b = byName.get(a.name);
+    if (!b && a.previously && byName.has(a.previously) && !afterNames.has(a.previously)) {
+      b = byName.get(a.previously)!;
+      out.push(change(key, `columns.${a.name}`, keys.has(b.name) ? "SQLCH213" : "SQLCH212", b.name, a.name));
+    }
+    if (!b) {
+      added.add(a.name);
+      out.push(change(key, `columns.${a.name}`, "SQLCH201", undefined, a.text));
+      continue;
+    }
+    matched.add(b.name);
+    const field = (f: string) => `columns.${a.name}.${f}`;
+    if (a.type !== b.type || (a.nullable ?? false) !== (b.nullable ?? false)) {
+      out.push(change(key, field("type"), pk.has(b.name) || keys.has(b.name) ? "SQLCH211" : "SQLCH210", b.type, a.type));
+    }
+    if (a.defaultKind !== b.defaultKind || a.defaultExpr !== b.defaultExpr) {
+      out.push(change(key, field("default"), "SQLCH207", [b.defaultKind, b.defaultExpr].filter(Boolean).join(" "), [a.defaultKind, a.defaultExpr].filter(Boolean).join(" ")));
+    }
+    if (a.codec !== b.codec) out.push(change(key, field("codec"), "SQLCH208", b.codec, a.codec));
+    if (a.ttl !== b.ttl) out.push(change(key, field("ttl"), "SQLCH205", b.ttl, a.ttl));
+    if (a.comment !== b.comment) out.push(change(key, field("comment"), "SQLCH203", b.comment, a.comment));
+  }
+
+  const dropped = before.columns.filter((c) => !matched.has(c.name));
+  for (const b of dropped) {
+    out.push(change(key, `columns.${b.name}`, keys.has(b.name) ? "SQLCH213" : "SQLCH202", b.text, undefined, { destructive: true }));
+    const twin = after.columns.find((a) => added.has(a.name) && a.type === b.type && a.position === b.position);
+    if (twin) {
+      hints.push(
+        `${key}: column ${b.name} is dropped and ${twin.name} added with the same type in the same place. ` +
+          `If it is a rename, write \`-- previously: ${b.name}\` on ${twin.name}'s line.`,
+      );
+    }
+  }
+
+  // Order, among the columns both sides have (a renamed column by its old name).
+  const beforeOrder = before.columns.map((c) => c.name).filter((n) => matched.has(n));
+  const afterOrder = after.columns
+    .map((c) => (byName.has(c.name) ? c.name : c.previously && matched.has(c.previously) ? c.previously : undefined))
+    .filter((n): n is string => n !== undefined && matched.has(n));
+  if (beforeOrder.join(",") !== afterOrder.join(",")) out.push(change(key, "columns.order", "SQLCH209", beforeOrder.join(", "), afterOrder.join(", ")));
+  return added;
+}
+
+function diffMap(key: string, field: string, before: Record<string, string>, after: Record<string, string>, rule: ClassifierRuleId, out: Change[]) {
+  for (const name of changedEntries(before, after)) out.push(change(key, `${field}.${name}`, rule, before[name], after[name]));
+}
+
+function diffObject(key: string, before: CanonicalObject, after: CanonicalObject, out: Change[], hints: string[]): void {
+  if (before.kind !== after.kind) {
+    out.push(change(key, "kind", "SQLCH224", before.kind, after.kind, { destructive: before.kind === "table" }));
+    return;
+  }
+  if (qualified(before) !== qualified(after)) {
+    out.push(change(key, "name", before.kind === "database" ? "SQLCH231" : "SQLCH230", qualified(before), qualified(after)));
+  }
+  if (before.comment !== after.comment) out.push(change(key, "comment", "SQLCH203", before.comment, after.comment));
+
+  if (before.kind === "database") {
+    if (before.engine !== after.engine) out.push(change(key, "engine", "SQLCH232", before.engine, after.engine));
+    return;
+  }
+
+  if (before.kind === "view") {
+    if (before.select !== after.select) out.push(change(key, "select", "SQLCH240", before.select, after.select));
+    return;
+  }
+
+  if (before.kind === "materializedView") {
+    if (before.to !== after.to) out.push(change(key, "to", "SQLCH242", before.to, after.to));
+    if (before.select !== after.select) out.push(change(key, "select", "SQLCH241", before.select, after.select, { note: "rows already in the target are not recomputed" }));
+    if (before.refresh !== after.refresh) out.push(change(key, "refresh", "SQLCH244", before.refresh, after.refresh));
+    for (const f of ["engine", "orderBy", "primaryKey", "partitionBy", "sampleBy", "ttl"] as const) {
+      if (before[f] !== after[f]) out.push(change(key, f, "SQLCH243", before[f], after[f]));
+    }
+    return;
+  }
+
+  // A table.
+  if (before.engine !== after.engine) out.push(change(key, "engine", "SQLCH223", before.engine, after.engine));
+  const added = diffColumns(key, before, after, out, hints);
+
+  const beforePk = before.primaryKey ?? before.orderBy;
+  const afterPk = after.primaryKey ?? after.orderBy;
+  if (before.orderBy !== after.orderBy) {
+    const append = appendsAddedColumns(before.orderBy, after.orderBy, added) && beforePk === afterPk;
+    out.push(
+      change(key, "orderBy", append ? "SQLCH216" : "SQLCH220", before.orderBy, after.orderBy, {
+        ...(appendsAddedColumns(before.orderBy, after.orderBy, added) && beforePk !== afterPk
+          ? { note: `appending new columns keeps the primary key; declare PRIMARY KEY ${beforePk} to make this an ALTER` }
+          : {}),
+      }),
+    );
+  }
+  if (beforePk !== afterPk && !(before.orderBy !== after.orderBy && before.primaryKey === undefined && after.primaryKey === undefined)) {
+    out.push(change(key, "primaryKey", "SQLCH221", beforePk, afterPk));
+  }
+  if (before.partitionBy !== after.partitionBy) out.push(change(key, "partitionBy", "SQLCH222", before.partitionBy, after.partitionBy));
+  if (before.sampleBy !== after.sampleBy) out.push(change(key, "sampleBy", "SQLCH217", before.sampleBy, after.sampleBy));
+  if (before.ttl !== after.ttl) out.push(change(key, "ttl", "SQLCH205", before.ttl, after.ttl));
+
+  for (const name of changedEntries(before.settings, after.settings)) {
+    const spec = (MERGE_TREE_SETTINGS as Record<string, { readonly: boolean } | undefined>)[name];
+    out.push(change(key, `settings.${name}`, spec?.readonly ? "SQLCH218" : "SQLCH206", before.settings[name], after.settings[name]));
+  }
+  diffMap(key, "indexes", before.indexes, after.indexes, "SQLCH204", out);
+  diffMap(key, "projections", before.projections, after.projections, "SQLCH214", out);
+  diffMap(key, "constraints", before.constraints, after.constraints, "SQLCH215", out);
+}
+
+/**
+ * Diff two schemas. `before` and `after` are matched by key; an `after`
+ * object whose key is new but whose `previously` names a `before` object
+ * with no `after` counterpart is that object, renamed.
+ */
+export function diffSchemas(before: readonly SchemaObject[], after: readonly SchemaObject[]): SchemaDiff {
+  const changes: Change[] = [];
+  const hints: string[] = [];
+  const matches = matchByIdentity(before, after, {
+    qualified,
+    previously: (o) => o.previously,
+    previousNames: (o, prev) => [prev, `${o.database}.${prev}`],
+  });
+  for (const m of matches) {
+    if (m.kind === "created") changes.push(change(m.after.key, "object", "SQLCH200", undefined, qualified(m.after.canonical)));
+    else if (m.kind === "matched") diffObject(m.after.key, m.before.canonical, m.after.canonical, changes, hints);
+    else {
+      const b = m.before;
+      changes.push(change(b.key, "object", "SQLCH250", qualified(b.canonical), undefined, { destructive: b.canonical.kind === "table" || b.canonical.kind === "materializedView" }));
+    }
+  }
+  return { changes, rebuilds: changes.filter((c) => c.class === "rebuild"), hints };
+}

package/src/clickhouse/plan/disruption.ts

@@ -0,0 +1,61 @@
+/**
+ * `classifyDisruption()`: what applying a pending update costs, from the
+ * paths `chant lifecycle plan` reports changed, mapped onto the classifier's
+ * rules (`./rules.ts`). A metadata-only change is `in-place`, a background
+ * rewrite `rolling`, a rebuild `replace`.
+ *
+ * A path alone cannot always say which rule applies: a column type change is a
+ * rewrite, or a rebuild when the column is in a key. Those answer `unknown`,
+ * naming both rules; `chant sql plan` resolves them with the whole definition
+ * in hand.
+ */
+
+import type { DisruptionQuery, DisruptionVerdict } from "@intentius/chant/lifecycle/disruption";
+import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse";
+import { classifyDisruptionWith } from "../../core/classifier";
+import { CHANGE_CLASSES, CLASSIFIER_RULES, type ClassifierRuleId } from "./rules";
+
+function ruleFor(path: string): ClassifierRuleId | "ambiguous" | undefined {
+  const root = path.split(/[.[]/)[0]!;
+  if (root === "name" || root === "database") return "SQLCH230";
+  if (root === "comment") return "SQLCH203";
+  if (root === "engine") return "SQLCH223";
+  if (root === "orderBy") return "ambiguous";
+  if (root === "primaryKey") return "SQLCH221";
+  if (root === "partitionBy") return "SQLCH222";
+  if (root === "sampleBy") return "SQLCH217";
+  if (root === "ttl") return "SQLCH205";
+  if (root === "indexes") return "SQLCH204";
+  if (root === "projections") return "SQLCH214";
+  if (root === "constraints") return "SQLCH215";
+  if (root === "select") return "SQLCH240";
+  if (root === "refresh") return "SQLCH244";
+  if (root === "settings") {
+    const key = path.split(".")[1];
+    const spec = key ? (MERGE_TREE_SETTINGS as Record<string, { readonly: boolean } | undefined>)[key] : undefined;
+    return spec?.readonly ? "SQLCH218" : "SQLCH206";
+  }
+  if (root === "columns") {
+    const field = /^columns\[\d+\]\.(\w+)/.exec(path)?.[1];
+    if (field === undefined) return "ambiguous"; // a column added, dropped or moved
+    if (field === "type" || field === "nullable" || field === "name") return "ambiguous";
+    if (field === "default") return "SQLCH207";
+    if (field === "codec") return "SQLCH208";
+    if (field === "ttl") return "SQLCH205";
+    if (field === "comment") return "SQLCH203";
+  }
+  return undefined;
+}
+
+export function classifyDisruption(options: { environment: string; changes: DisruptionQuery[] }): Record<string, DisruptionVerdict> {
+  return classifyDisruptionWith(
+    {
+      typePrefix: "ClickHouse::",
+      rules: CLASSIFIER_RULES,
+      classes: CHANGE_CLASSES,
+      ruleFor,
+      ambiguousDetail: "a sorting-key or column change is metadata, a rewrite or a rebuild depending on the whole definition; `chant sql plan` classifies it",
+    },
+    options,
+  );
+}

package/src/clickhouse/plan/normalize.ts

@@ -0,0 +1,352 @@
+/**
+ * Normalization: a ClickHouse object's definition in a form two definitions
+ * can be compared in, so a declaration and what the server reports for it are
+ * equal when they mean the same thing (#3047 question 4).
+ *
+ * The server rewrites what it is given. `SHOW CREATE` quotes every column,
+ * qualifies names with their database, writes `INTERVAL 180 DAY` as
+ * `toIntervalDay(180)`, fills in codec levels (`ZSTD(1)`, `Delta(4)`), drops a
+ * TTL's trailing `DELETE`, prints `index_granularity = 8192`, writes `int` as
+ * `Int32` and infers a view's column list. Each of those is undone here, on
+ * both sides, by rule:
+ *
+ * - expressions compare as their token sequences, whitespace, comments and
+ *   identifier quoting left out, keywords in upper case, `INTERVAL n UNIT`
+ *   rewritten to `toIntervalUnit(n)`, the object's own database dropped from a
+ *   qualified name;
+ * - a type's family is its canonical name (`int` is `Int32`), from the pinned
+ *   catalog's alias table;
+ * - a codec's default level goes (`ZSTD(1)` is `ZSTD`, `Delta(4)` is `Delta`);
+ * - a TTL rule's trailing `DELETE` goes;
+ * - a setting at the pinned server's default goes;
+ * - a view's column list is not compared: the server infers it.
+ *
+ * What the rules cannot see is the server's own expression formatting
+ * (`a+b*2` prints as `a + (b * 2)`). When a plan reads a live server it asks
+ * that server's formatter (`formatQuerySingleLine`, a read with no side
+ * effects) about any expression the rules leave different
+ * (`./server-format.ts`). Nothing is created anywhere to normalize: a scratch
+ * database on the target would start Kafka consumers and register replicas in
+ * Keeper, and a scratch container would need Docker at plan time and could
+ * differ from the server in version.
+ */
+
+import { isTrivia, tokenizeText, type Token } from "../tokens";
+import { parseCreate, unquote, type ColumnNode, type CreateNode, type Span } from "../parser";
+import { MERGE_TREE_SETTINGS, TYPE_FAMILIES } from "../../generated/clickhouse";
+import { previouslyIn, splitTopLevel as splitTop } from "../../core/normalize";
+
+export { previouslyIn } from "../../core/normalize";
+
+export type ObjectKind = "database" | "table" | "view" | "materializedView";
+
+export interface CanonicalColumn {
+  name: string;
+  position: number;
+  type: string;
+  nullable?: boolean;
+  defaultKind?: string;
+  defaultExpr?: string;
+  codec?: string;
+  ttl?: string;
+  comment?: string;
+  /** `-- previously: <name>` on the column's line: the name it had before. */
+  previously?: string;
+  /** The column's name and type as written, for a message. Not compared. */
+  text: string;
+}
+
+export interface CanonicalObject {
+  kind: ObjectKind;
+  database?: string;
+  name: string;
+  engine?: string;
+  /** The engine's name alone. */
+  engineName?: string;
+  columns: CanonicalColumn[];
+  orderBy?: string;
+  primaryKey?: string;
+  partitionBy?: string;
+  sampleBy?: string;
+  ttl?: string;
+  settings: Record<string, string>;
+  indexes: Record<string, string>;
+  projections: Record<string, string>;
+  constraints: Record<string, string>;
+  comment?: string;
+  select?: string;
+  to?: string;
+  refresh?: string;
+  /** `-- previously: <name>` before the statement: the object's previous name. */
+  previously?: string;
+}
+
+const KEYWORDS = new Set(
+  (
+    "AS AND OR NOT IN IS NULL LIKE ILIKE BETWEEN CASE WHEN THEN ELSE END INTERVAL ASC DESC NULLS FIRST LAST " +
+    "SELECT FROM WHERE GROUP BY ORDER HAVING LIMIT OFFSET JOIN LEFT RIGHT INNER OUTER FULL CROSS ON USING ARRAY " +
+    "DISTINCT ALL ANY UNION WITH TOTALS ROLLUP CUBE PREWHERE SETTINGS FORMAT GLOBAL SEMI ANTI ASOF FINAL SAMPLE " +
+    "DELETE TO DISK VOLUME RECOMPRESS SET TRUE FALSE EVERY AFTER RANDOMIZE FOR DEPENDS APPEND"
+  ).split(" "),
+);
+
+const INTERVAL_UNITS: Record<string, string> = {
+  NANOSECOND: "Nanosecond",
+  MICROSECOND: "Microsecond",
+  MILLISECOND: "Millisecond",
+  SECOND: "Second",
+  MINUTE: "Minute",
+  HOUR: "Hour",
+  DAY: "Day",
+  WEEK: "Week",
+  MONTH: "Month",
+  QUARTER: "Quarter",
+  YEAR: "Year",
+};
+
+const isName = (t: Token | undefined) => t !== undefined && (t.kind === "ident" || t.kind === "qident");
+
+/**
+ * The Keeper path and replica name a `Replicated*MergeTree` gets when its
+ * declaration leaves them out: the stock `default_replica_path` and
+ * `default_replica_name`, which `SHOW CREATE` then prints.
+ */
+export const DEFAULT_REPLICA_PATH = "'/clickhouse/tables/{uuid}/{shard}'";
+export const DEFAULT_REPLICA_NAME = "'{replica}'";
+
+/** One token's canonical text. */
+function tokenText(t: Token): string {
+  if (t.kind === "qident") return unquote(t.text);
+  if (t.kind === "ident" && KEYWORDS.has(t.text.toUpperCase())) return t.text.toUpperCase();
+  return t.text;
+}
+
+/**
+ * An expression's canonical text: its significant tokens joined by single
+ * spaces, with the rewrites the server makes undone.
+ */
+export function canonicalExpression(text: string, ownDatabase?: string): string {
+  let sig: Token[];
+  try {
+    sig = tokenizeText(text, 0).filter((t) => !isTrivia(t));
+  } catch {
+    return text.trim();
+  }
+  const out: string[] = [];
+  for (let i = 0; i < sig.length; i++) {
+    const t = sig[i]!;
+    // INTERVAL 180 DAY  ->  toIntervalDay(180)
+    if (t.kind === "ident" && t.text.toUpperCase() === "INTERVAL" && sig[i + 1] && sig[i + 2]) {
+      const unit = INTERVAL_UNITS[sig[i + 2]!.text.toUpperCase().replace(/S$/, "")];
+      if (unit && (sig[i + 1]!.kind === "number" || sig[i + 1]!.kind === "ref" || isName(sig[i + 1]))) {
+        out.push(`toInterval${unit}`, "(", tokenText(sig[i + 1]!), ")");
+        i += 2;
+        continue;
+      }
+    }
+    // own_db.name  ->  name
+    if (ownDatabase !== undefined && isName(t) && unquote(t.text) === ownDatabase && sig[i + 1]?.text === "." && isName(sig[i + 2])) {
+      const prev = sig[i - 1];
+      if (!(prev?.kind === "punct" && prev.text === ".")) {
+        i += 1;
+        continue;
+      }
+    }
+    out.push(tokenText(t));
+  }
+  return unwrapSingle(out).join(" ");
+}
+
+/** `( a )` with no top-level comma is `a`; `tuple ( )` is empty. */
+function unwrapSingle(tokens: string[]): string[] {
+  if (tokens.length === 3 && tokens[0]!.toLowerCase() === "tuple" && tokens[1] === "(" && tokens[2] === ")") return [];
+  if (tokens[0] !== "(" || tokens[tokens.length - 1] !== ")") return tokens;
+  let depth = 0;
+  for (let i = 0; i < tokens.length; i++) {
+    if (tokens[i] === "(") depth++;
+    else if (tokens[i] === ")") depth--;
+    if (depth === 0 && i < tokens.length - 1) return tokens;
+    if (depth === 1 && tokens[i] === ",") return tokens;
+  }
+  return unwrapSingle(tokens.slice(1, -1));
+}
+
+/** A type with its family names canonical: `int` is `Int32`, `BIGINT` is `Int64`. */
+export function canonicalType(text: string): string {
+  const families = TYPE_FAMILIES as Record<string, { canonical: string; caseInsensitive: boolean } | undefined>;
+  const lower = new Map<string, string>();
+  for (const [name, spec] of Object.entries(families)) if (spec?.caseInsensitive) lower.set(name.toLowerCase(), spec.canonical);
+  return canonicalExpression(text)
+    .split(" ")
+    .map((w) => families[w]?.canonical ?? lower.get(w.toLowerCase()) ?? w)
+    .join(" ");
+}
+
+const CODEC_DEFAULT_ARGS: Record<string, (args: string) => boolean> = {
+  ZSTD: (a) => a === "1",
+  LZ4HC: (a) => a === "0" || a === "9",
+  Delta: () => true,
+  DoubleDelta: () => true,
+  Gorilla: () => true,
+  T64: (a) => a === "'byte'",
+};
+
+/** A codec chain with each codec's default level left out. */
+export function canonicalCodec(text: string): string {
+  return splitTop(canonicalExpression(text))
+    .map((c) => {
+      const m = /^(\w+) \( (.*) \)$/.exec(c);
+      if (!m) return c;
+      const [, name, args] = m;
+      return CODEC_DEFAULT_ARGS[name!]?.(args!) ? name! : c;
+    })
+    .join(" , ");
+}
+
+/** A TTL with each rule's trailing `DELETE` (the default action) left out. */
+export function canonicalTtl(text: string, ownDatabase?: string): string {
+  return splitTop(canonicalExpression(text, ownDatabase))
+    .map((rule) => rule.replace(/ DELETE$/, ""))
+    .join(" , ");
+}
+
+function spanText(tokens: Token[], span: Span | undefined): string | undefined {
+  if (!span || span.to <= span.from) return undefined;
+  return tokens
+    .slice(span.from, span.to)
+    .map((t) => t.text)
+    .join("")
+    .trim();
+}
+
+/** The comments on the line a column starts on, after its name. */
+function lineComments(tokens: Token[], from: number): string[] {
+  const out: string[] = [];
+  for (let i = from + 1; i < tokens.length; i++) {
+    const t = tokens[i]!;
+    if (t.kind === "ws" && t.text.includes("\n")) break;
+    if (t.kind === "comment") out.push(t.text);
+  }
+  return out;
+}
+
+const kindOf = (node: CreateNode): ObjectKind =>
+  node.statement === "database" ? "database" : node.statement === "table" ? "table" : node.materialized ? "materializedView" : "view";
+
+const stringValue = (s: string | undefined) =>
+  s === undefined ? undefined : /^'.*'$/s.test(s) ? s.slice(1, -1).replace(/''/g, "'").replace(/\\(.)/g, "$1") : s;
+
+/**
+ * Parse a CREATE statement into its canonical form. `defaultDatabase` is the
+ * database an unqualified name is created in.
+ */
+export function canonicalObject(ddl: string, defaultDatabase = "default"): CanonicalObject {
+  const tokens = tokenizeText(ddl, 0);
+  const node = parseCreate(tokens);
+  const nameParts = tokens
+    .slice(node.name.from, node.name.to)
+    .filter((t) => !isTrivia(t) && !(t.kind === "punct" && t.text === "."))
+    .map((t) => unquote(t.text));
+  const kind = kindOf(node);
+  const name = nameParts[nameParts.length - 1]!;
+  const database = kind === "database" ? undefined : nameParts.length >= 2 ? nameParts[nameParts.length - 2] : defaultDatabase;
+  const expr = (span: Span | undefined) => {
+    const t = spanText(tokens, span);
+    return t === undefined ? undefined : canonicalExpression(t, database);
+  };
+  const leading = tokens.slice(0, tokens.findIndex((t) => !isTrivia(t))).filter((t) => t.kind === "comment").map((t) => t.text);
+
+  const obj: CanonicalObject = {
+    kind,
+    ...(database !== undefined ? { database } : {}),
+    name,
+    columns: [],
+    settings: {},
+    indexes: {},
+    projections: {},
+    constraints: {},
+  };
+  const previously = previouslyIn(leading);
+  if (previously) obj.previously = previously;
+
+  const engine = node.engine;
+  if (engine) {
+    let args = engine.args?.map((a) => canonicalExpression(spanText(tokens, a) ?? "", database)) ?? [];
+    // `ENGINE = ReplicatedMergeTree` with no Keeper path and replica name is
+    // printed back with the server's defaults filled in; the two say the same.
+    if (/^Replicated.*MergeTree$/.test(engine.name) && args[0] === DEFAULT_REPLICA_PATH && args[1] === DEFAULT_REPLICA_NAME) args = args.slice(2);
+    obj.engineName = engine.name;
+    obj.engine = args.length ? `${engine.name}(${args.join(", ")})` : engine.name;
+  }
+  const comment = stringValue(spanText(tokens, node.comment));
+  if (comment !== undefined && comment !== "") obj.comment = comment;
+
+  if (node.statement === "database") {
+    for (const s of node.settings ?? []) obj.settings[s.key] = canonicalExpression(spanText(tokens, s.value) ?? "").replace(/^'(.*)'$/, "$1");
+    return obj;
+  }
+
+  const set = (k: "orderBy" | "primaryKey" | "partitionBy" | "sampleBy", span: Span | undefined) => {
+    const v = expr(span);
+    if (v !== undefined && v !== "") obj[k] = v;
+  };
+  set("orderBy", node.orderBy);
+  set("primaryKey", node.primaryKey);
+  set("partitionBy", node.partitionBy);
+  set("sampleBy", node.sampleBy);
+  if (node.ttl) obj.ttl = canonicalTtl(spanText(tokens, node.ttl)!, database);
+  for (const s of node.settings ?? []) {
+    const value = canonicalExpression(spanText(tokens, s.value) ?? "").replace(/^'(.*)'$/, "$1");
+    const def = (MERGE_TREE_SETTINGS as Record<string, { default: string } | undefined>)[s.key];
+    if (def && def.default === value) continue;
+    obj.settings[s.key] = value;
+  }
+
+  if (node.statement === "table") {
+    node.columns.forEach((c, i) => obj.columns.push(column(tokens, c, i, database)));
+    for (const ix of node.indexes) {
+      obj.indexes[ix.name] = [expr(ix.expr), "TYPE", expr(ix.type), ix.granularity ? `GRANULARITY ${expr(ix.granularity)}` : ""]
+        .filter(Boolean)
+        .join(" ");
+    }
+    for (const p of node.projections) obj.projections[p.name] = expr(p.body) ?? "";
+    for (const c of node.constraints) obj.constraints[c.name] = `${c.kind} ${expr(c.expr)}`;
+    return obj;
+  }
+
+  obj.select = expr(node.select);
+  if (node.to) obj.to = qualified(tokens, node.to, defaultDatabase);
+  if (node.refresh) obj.refresh = expr(node.refresh);
+  return obj;
+}
+
+function qualified(tokens: Token[], span: Span, defaultDatabase: string): string {
+  const parts = tokens
+    .slice(span.from, span.to)
+    .filter((t) => !isTrivia(t) && !(t.kind === "punct" && t.text === "."))
+    .map((t) => unquote(t.text));
+  return parts.length >= 2 ? `${parts[parts.length - 2]}.${parts[parts.length - 1]}` : `${defaultDatabase}.${parts[0]}`;
+}
+
+function column(tokens: Token[], c: ColumnNode, position: number, database?: string): CanonicalColumn {
+  const text = (span: Span | undefined) => spanText(tokens, span);
+  const out: CanonicalColumn = {
+    name: c.name,
+    position,
+    type: c.type ? canonicalType(text(c.type)!) : "",
+    text: [c.name, text(c.type)].filter(Boolean).join(" "),
+  };
+  if (c.nullable !== undefined) out.nullable = c.nullable;
+  if (c.default) {
+    out.defaultKind = c.default.kind;
+    const e = text(c.default.expr);
+    if (e !== undefined) out.defaultExpr = canonicalExpression(e, database);
+  }
+  if (c.codec) out.codec = canonicalCodec(text(c.codec)!);
+  if (c.ttl) out.ttl = canonicalExpression(text(c.ttl)!, database);
+  const comment = stringValue(text(c.comment));
+  if (comment !== undefined && comment !== "") out.comment = comment;
+  const prev = previouslyIn(lineComments(tokens, c.nameSpan.from));
+  if (prev) out.previously = prev;
+  return out;
+}