@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/clickhouse/review-catalog.ts

@@ -0,0 +1,489 @@
+/**
+ * The needs-review catalog: the judgment calls that turn a table from another
+ * database into a ClickHouse table, each as a finding with the evidence it
+ * used. A source reader (`chant import` of SQL Server, #4398; `chant migrate`
+ * of Postgres, #4399) does the mechanical work, a type map and a rename,
+ * and asks this module for everything that is a decision.
+ *
+ * Nothing here knows a source dialect. A reader describes its table in the
+ * neutral facts below (a candidate sort key, a column's nullability, a
+ * constraint) and gets back the ClickHouse choice, a finding to report, and
+ * for a constraint the comment lines to keep in the DDL. The rules are the
+ * same whichever database the table came from.
+ *
+ * ## API
+ *
+ * - {@link sortKeyReview}: `ORDER BY` from the candidates the source offers,
+ *   in the priority `explicit-order`, `clustering-key`, `primary-key`,
+ *   `unique-key`, then `tuple()`. Returns `orderBy`, the `settings` the key
+ *   needs (`allow_nullable_key` when a key column is `Nullable`), and a
+ *   finding that names the chosen candidate and the others it passed over.
+ * - {@link engineReview}: `MergeTree` for an append-only table;
+ *   `ReplacingMergeTree(_version, _deleted)` for a CDC mirror, with the two
+ *   columns it adds in `extraColumns`. `cdcColumns` names them otherwise
+ *   (the Postgres pair uses the `CdcMirror` composite's `_is_deleted`).
+ * - {@link nullableReview}: a column's final type. A `NOT NULL` column keeps
+ *   its type with no finding. A nullable column becomes `Nullable(T)` with a
+ *   finding; when it has a default the finding says the default could
+ *   replace `Nullable`.
+ * - {@link lowCardinalityReview}: a finding for a `String` column whose name
+ *   and declared length suggest few distinct values. It never changes the
+ *   type; offline, the name and the type are all there is to read.
+ * - {@link constraintReview}: foreign keys, checks, unique and exclusion
+ *   constraints, which ClickHouse does not enforce. Returns `comments` (SQL `--` lines to
+ *   put before the `CREATE`) and one finding per constraint.
+ * - {@link REVIEW_RULES}: every rule id with its summary and the ClickHouse
+ *   documentation it rests on, for report metadata.
+ * - {@link findingText}: a finding as one line (object, message, evidence,
+ *   suggestion), the form a SARIF `message` takes.
+ *
+ * A finding's `severity` is `warning` for a decision the author must confirm
+ * and `info` for a note that changes nothing. The caller maps findings to its
+ * own diagnostic type; `at` carries the source line and column when the
+ * reader knows them.
+ *
+ * Every ClickHouse type and engine this module writes is checked against the
+ * pinned catalog (`src/generated/clickhouse.ts`, ClickHouse 26.8) in
+ * `review-catalog.test.ts`, so a pin move that drops one fails there.
+ *
+ * Sources: the MergeTree sorting key and `allow_nullable_key`
+ * (https://clickhouse.com/docs/engines/table-engines/mergetree-family/mergetree),
+ * `ReplacingMergeTree(ver, is_deleted)`
+ * (https://clickhouse.com/docs/engines/table-engines/mergetree-family/replacingmergetree),
+ * `Nullable` (https://clickhouse.com/docs/sql-reference/data-types/nullable),
+ * `LowCardinality`
+ * (https://clickhouse.com/docs/sql-reference/data-types/lowcardinality).
+ */
+
+export type ReviewSeverity = "warning" | "info";
+
+export type ReviewRuleId =
+  | "SQLRV001"
+  | "SQLRV002"
+  | "SQLRV003"
+  | "SQLRV004"
+  | "SQLRV005"
+  | "SQLRV006"
+  | "SQLRV007"
+  | "SQLRV008"
+  | "SQLRV009"
+  | "SQLRV010"
+  | "SQLRV011"
+  | "SQLRV012";
+
+export interface ReviewRule {
+  id: ReviewRuleId;
+  name: string;
+  summary: string;
+  docs: readonly string[];
+}
+
+const CH_MERGETREE = "https://clickhouse.com/docs/engines/table-engines/mergetree-family/mergetree";
+
+/**
+ * The rules. SQLRV001 to SQLRV005 are the shared ones; the rest are raised by
+ * a source reader for what only its dialect has, and live here so a report
+ * lists one catalog.
+ */
+export const REVIEW_RULES: Readonly<Record<ReviewRuleId, ReviewRule>> = {
+  SQLRV001: {
+    id: "SQLRV001",
+    name: "sort-key",
+    summary: "The ORDER BY key is a suggestion from the source's indexes.",
+    docs: [CH_MERGETREE],
+  },
+  SQLRV002: {
+    id: "SQLRV002",
+    name: "engine",
+    summary: "The table engine is a choice the source does not make.",
+    docs: [CH_MERGETREE, "https://clickhouse.com/docs/engines/table-engines/mergetree-family/replacingmergetree"],
+  },
+  SQLRV003: {
+    id: "SQLRV003",
+    name: "nullable",
+    summary: "A nullable column becomes Nullable(T), which costs storage and speed; a default can replace it.",
+    docs: ["https://clickhouse.com/docs/sql-reference/data-types/nullable"],
+  },
+  SQLRV004: {
+    id: "SQLRV004",
+    name: "low-cardinality",
+    summary: "A String column that may hold few distinct values could be LowCardinality(String).",
+    docs: ["https://clickhouse.com/docs/sql-reference/data-types/lowcardinality"],
+  },
+  SQLRV005: {
+    id: "SQLRV005",
+    name: "constraint",
+    summary: "ClickHouse does not enforce this constraint; it is kept as a comment.",
+    docs: ["https://clickhouse.com/docs/sql-reference/statements/create/table"],
+  },
+  SQLRV006: {
+    id: "SQLRV006",
+    name: "generated-column",
+    summary: "An identity, sequence, computed or system-generated column has no ClickHouse equivalent.",
+    docs: ["https://clickhouse.com/docs/sql-reference/statements/create/table"],
+  },
+  SQLRV007: {
+    id: "SQLRV007",
+    name: "type-judgment",
+    summary: "The type has no exact ClickHouse match; the nearest one was chosen.",
+    docs: ["https://clickhouse.com/docs/sql-reference/data-types"],
+  },
+  SQLRV008: {
+    id: "SQLRV008",
+    name: "default-expression",
+    summary: "A default expression was translated or dropped.",
+    docs: ["https://clickhouse.com/docs/sql-reference/statements/create/table"],
+  },
+  SQLRV009: {
+    id: "SQLRV009",
+    name: "storage-option",
+    summary: "A storage option of the source (filegroup, partitioning, compression, versioning) has no counterpart here.",
+    docs: [CH_MERGETREE],
+  },
+  SQLRV010: {
+    id: "SQLRV010",
+    name: "query-text",
+    summary: "A view body is copied as written and is not translated.",
+    docs: ["https://clickhouse.com/docs/sql-reference/statements/create/view"],
+  },
+  SQLRV011: {
+    id: "SQLRV011",
+    name: "not-carried",
+    summary: "An object or statement of the source is not carried over, and is named here.",
+    docs: [],
+  },
+  SQLRV012: {
+    id: "SQLRV012",
+    name: "emulator-rejected",
+    summary: "The ClickHouse emulator refused a statement the import would write.",
+    docs: [],
+  },
+};
+
+export interface SourcePosition {
+  /** 1-based. */
+  line: number;
+  /** 1-based. */
+  column: number;
+}
+
+export interface ReviewFinding {
+  ruleId: ReviewRuleId;
+  severity: ReviewSeverity;
+  /** The table or view the finding is about, as the source names it. */
+  object: string;
+  /** The column, when the finding is about one. */
+  column?: string;
+  message: string;
+  /** What the decision was read from: each entry is a fact in the source. */
+  evidence: readonly string[];
+  suggestion?: string;
+  at?: SourcePosition;
+}
+
+/** A finding as one line. */
+export function findingText(f: ReviewFinding): string {
+  const where = f.column ? `${f.object}.${f.column}` : f.object;
+  const parts = [`${where}: ${f.message}`];
+  if (f.evidence.length > 0) parts.push(`Evidence: ${f.evidence.join("; ")}.`);
+  if (f.suggestion) parts.push(`Suggestion: ${f.suggestion}`);
+  return parts.join(" ");
+}
+
+const q = (name: string): string => (/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) ? name : `\`${name.replace(/`/g, "\\`")}\``);
+const list = (names: readonly string[]): string => names.join(", ");
+
+// --- sort key -------------------------------------------------------------
+
+/**
+ * Where a candidate sort key came from, strongest first. `explicit-order` is
+ * an ordering the author declared for storage (a columnstore `ORDER`),
+ * `clustering-key` the key the source stores rows in (a clustered index),
+ * then the primary key and the first unique key.
+ */
+export type SortKeyKind = "explicit-order" | "clustering-key" | "primary-key" | "unique-key";
+
+const SORT_KEY_PRIORITY: readonly SortKeyKind[] = ["explicit-order", "clustering-key", "primary-key", "unique-key"];
+
+export interface SortKeyCandidate {
+  kind: SortKeyKind;
+  /** The index or constraint name, for the evidence. */
+  name?: string;
+  columns: readonly string[];
+  /** Columns the source sorts descending; ClickHouse's `ORDER BY` drops the direction. */
+  descending?: readonly string[];
+  /** The fact this candidate was read from, e.g. `CREATE CLUSTERED INDEX CX_t`. */
+  source: string;
+}
+
+export interface SortKeyInput {
+  object: string;
+  candidates: readonly SortKeyCandidate[];
+  /** Each column's ClickHouse type, by name; `Nullable(...)` marks a nullable key column. */
+  columnTypes: Readonly<Record<string, string>>;
+  at?: SourcePosition;
+}
+
+export interface SortKeyResult {
+  orderBy: string[];
+  /** MergeTree settings the key needs. */
+  settings: Record<string, string>;
+  finding: ReviewFinding;
+}
+
+export function sortKeyReview(input: SortKeyInput): SortKeyResult {
+  const ranked = SORT_KEY_PRIORITY.flatMap((kind) => input.candidates.filter((c) => c.kind === kind && c.columns.length > 0));
+  const chosen = ranked[0];
+  const evidence: string[] = [];
+  const settings: Record<string, string> = {};
+  let orderBy: string[] = [];
+  let message: string;
+  let suggestion: string | undefined;
+  if (chosen) {
+    orderBy = [...chosen.columns];
+    evidence.push(`${chosen.source} (${list(chosen.columns)})`);
+    for (const other of ranked.slice(1)) evidence.push(`passed over: ${other.source} (${list(other.columns)})`);
+    if (chosen.descending && chosen.descending.length > 0) evidence.push(`descending in the source, ascending here: ${list(chosen.descending)}`);
+    const nullable = orderBy.filter((c) => (input.columnTypes[c] ?? "").startsWith("Nullable("));
+    if (nullable.length > 0) {
+      settings.allow_nullable_key = "1";
+      evidence.push(`Nullable key column${nullable.length > 1 ? "s" : ""}: ${list(nullable)}, so allow_nullable_key = 1 is set`);
+    }
+    message = `ORDER BY (${list(orderBy.map(q))}) suggested from the ${chosen.kind.replace("-", " ")}.`;
+    suggestion = "Put the columns your queries filter on first, and the one with the fewest distinct values before the one with the most.";
+  } else {
+    orderBy = [];
+    evidence.push("no clustering key, primary key or unique key in the source");
+    message = "ORDER BY tuple() used because the source gives no key to suggest.";
+    suggestion = "Choose the columns your queries filter on; a MergeTree table without a sort key cannot skip granules.";
+  }
+  return {
+    orderBy,
+    settings,
+    finding: { ruleId: "SQLRV001", severity: "warning", object: input.object, message, evidence, ...(suggestion ? { suggestion } : {}), ...(input.at ? { at: input.at } : {}) },
+  };
+}
+
+// --- engine ---------------------------------------------------------------
+
+export interface EngineInput {
+  object: string;
+  /**
+   * `append` for a table that is loaded and queried, `cdc` for a mirror of a
+   * table that changes in place (#4399): rows arrive as new versions and a
+   * delete arrives as a marker row.
+   */
+  mode: "append" | "cdc";
+  /** Facts about the source table that bear on the choice, as plain sentences. */
+  facts?: readonly string[];
+  /** The columns the source enforces unique, when it does. */
+  uniqueKey?: readonly string[];
+  /**
+   * For `cdc`, the names of the version and deleted columns, when the pipeline
+   * or the `CdcMirror` composite names them other than `_version` and
+   * `_deleted`. The names are the caller's; the catalog only writes them.
+   */
+  cdcColumns?: { version: string; deleted: string };
+  at?: SourcePosition;
+}
+
+export interface EngineResult {
+  /** The text after `ENGINE =`. */
+  engine: string;
+  /** Columns the engine needs and the source does not have. */
+  extraColumns: Array<{ name: string; type: string }>;
+  finding: ReviewFinding;
+}
+
+export const CDC_VERSION_COLUMN = { name: "_version", type: "UInt64" } as const;
+export const CDC_DELETED_COLUMN = { name: "_deleted", type: "UInt8" } as const;
+
+export function engineReview(input: EngineInput): EngineResult {
+  const evidence = [...(input.facts ?? [])];
+  const unique = input.uniqueKey && input.uniqueKey.length > 0 ? list(input.uniqueKey) : undefined;
+  if (input.mode === "cdc") {
+    evidence.push(`the table is mirrored from a source that updates rows in place${unique ? `, keyed on (${unique})` : ""}`);
+    const version = input.cdcColumns?.version ?? CDC_VERSION_COLUMN.name;
+    const deleted = input.cdcColumns?.deleted ?? CDC_DELETED_COLUMN.name;
+    return {
+      engine: `ReplacingMergeTree(${version}, ${deleted})`,
+      extraColumns: [
+        { name: version, type: CDC_VERSION_COLUMN.type },
+        { name: deleted, type: CDC_DELETED_COLUMN.type },
+      ],
+      finding: {
+        ruleId: "SQLRV002",
+        severity: "warning",
+        object: input.object,
+        message: `ReplacingMergeTree(${version}, ${deleted}) chosen for a mirror; it adds the columns ${version} and ${deleted}.`,
+        evidence,
+        suggestion: `The sort key must be the source key, because rows with an equal sort key collapse to the highest ${version} on merge. Read with FINAL, or filter on ${deleted}, to see the collapsed rows before a merge.`,
+        ...(input.at ? { at: input.at } : {}),
+      },
+    };
+  }
+  if (unique) evidence.push(`the source enforces uniqueness on (${unique}); MergeTree does not`);
+  return {
+    engine: "MergeTree",
+    extraColumns: [],
+    finding: {
+      ruleId: "SQLRV002",
+      severity: "warning",
+      object: input.object,
+      message: "MergeTree chosen; the source has no engine to carry over.",
+      evidence,
+      suggestion: unique
+        ? "If rows can arrive twice, ReplacingMergeTree collapses duplicates of the sort key on merge, not on insert."
+        : "Pick another MergeTree variant for pre-aggregation (SummingMergeTree, AggregatingMergeTree) when the table is a rollup.",
+      ...(input.at ? { at: input.at } : {}),
+    },
+  };
+}
+
+// --- Nullable -------------------------------------------------------------
+
+export interface NullableInput {
+  object: string;
+  column: string;
+  /** The column's ClickHouse type before `Nullable`. */
+  type: string;
+  sourceNullable: boolean;
+  /** The default as the ClickHouse expression it became, or the source text when it did not translate. */
+  defaultExpr?: string;
+  at?: SourcePosition;
+}
+
+export interface NullableResult {
+  type: string;
+  finding?: ReviewFinding;
+}
+
+export function nullableReview(input: NullableInput): NullableResult {
+  if (!input.sourceNullable) return { type: input.type };
+  const evidence = [`${input.column} allows NULL in the source`];
+  let suggestion: string;
+  if (input.defaultExpr !== undefined) {
+    evidence.push(`it has the default ${input.defaultExpr}`);
+    suggestion = `If NULL is never stored on purpose, declare ${input.type} DEFAULT ${input.defaultExpr} and load the default instead of NULL.`;
+  } else {
+    evidence.push("it has no default");
+    suggestion = `If a sentinel value can stand for missing, declare ${input.type} DEFAULT <sentinel>; a Nullable column keeps a separate null mask.`;
+  }
+  return {
+    type: `Nullable(${input.type})`,
+    finding: {
+      ruleId: "SQLRV003",
+      severity: "warning",
+      object: input.object,
+      column: input.column,
+      message: `Kept as Nullable(${input.type}) so no stored NULL is lost.`,
+      evidence,
+      suggestion,
+      ...(input.at ? { at: input.at } : {}),
+    },
+  };
+}
+
+// --- LowCardinality -------------------------------------------------------
+
+/** Last words of a column name that usually mean a small set of values. */
+const LOW_CARDINALITY_WORDS = new Set("status state type kind category country currency region level channel gender lang language locale unit code flag method mode grade tier priority".split(" "));
+
+/** A column name's last word: `order_status` and `OrderStatus` both give `status`. */
+function lastWord(name: string): string {
+  const words = name.replace(/([a-z0-9])([A-Z])/g, "$1_$2").split(/[^A-Za-z0-9]+/).filter(Boolean);
+  return (words[words.length - 1] ?? "").toLowerCase();
+}
+
+export interface LowCardinalityInput {
+  object: string;
+  column: string;
+  /** The ClickHouse type as emitted. Only `String` and `Nullable(String)` are candidates. */
+  type: string;
+  /** The source's declared maximum length, when it has one. */
+  maxLength?: number;
+  at?: SourcePosition;
+}
+
+/** The longest declared length that still counts as supporting evidence. */
+export const LOW_CARDINALITY_MAX_LENGTH = 50;
+
+export function lowCardinalityReview(input: LowCardinalityInput): ReviewFinding | undefined {
+  if (input.type !== "String" && input.type !== "Nullable(String)") return undefined;
+  if (!LOW_CARDINALITY_WORDS.has(lastWord(input.column))) return undefined;
+  const evidence = [`the name ${input.column} ends in a word that usually names a small set of values`, `the type is ${input.type}`];
+  if (input.maxLength !== undefined && input.maxLength <= LOW_CARDINALITY_MAX_LENGTH) evidence.push(`the declared length is ${String(input.maxLength)}`);
+  evidence.push("no data was read, so the distinct count is unknown");
+  const wrapped = input.type.startsWith("Nullable(") ? "LowCardinality(Nullable(String))" : "LowCardinality(String)";
+  return {
+    ruleId: "SQLRV004",
+    severity: "warning",
+    object: input.object,
+    column: input.column,
+    message: `${input.column} is a LowCardinality candidate and stays ${input.type}.`,
+    evidence,
+    suggestion: `ClickHouse documents ${wrapped} as helpful below roughly ten thousand distinct values; count them with uniqExact(${q(input.column)}) before changing the type.`,
+    ...(input.at ? { at: input.at } : {}),
+  };
+}
+
+// --- constraints ----------------------------------------------------------
+
+export interface ConstraintFact {
+  kind: "foreign-key" | "check" | "unique" | "exclusion";
+  name?: string;
+  columns?: readonly string[];
+  references?: { table: string; columns: readonly string[] };
+  /** A check's expression, or an exclusion's `USING method (element WITH operator, ...)`, as the source wrote it. */
+  expression?: string;
+  at?: SourcePosition;
+}
+
+export interface ConstraintResult {
+  /** `-- ` lines to put before the `CREATE`; never contain a line break. */
+  comments: string[];
+  findings: ReviewFinding[];
+}
+
+const oneLine = (s: string): string => s.replace(/\s+/g, " ").trim();
+
+export function constraintReview(input: { object: string; constraints: readonly ConstraintFact[] }): ConstraintResult {
+  const comments: string[] = [];
+  const findings: ReviewFinding[] = [];
+  for (const c of input.constraints) {
+    const label = c.name ? ` ${c.name}` : "";
+    let text: string;
+    let message: string;
+    let suggestion: string;
+    if (c.kind === "foreign-key") {
+      const target = c.references ? `${c.references.table} (${list(c.references.columns)})` : "an unknown table";
+      text = `FOREIGN KEY${label} (${list(c.columns ?? [])}) REFERENCES ${target}`;
+      message = `Foreign key${label} is not enforced by ClickHouse and is kept as a comment.`;
+      suggestion = "Check referential integrity in the load pipeline, or join on the column and accept orphan rows.";
+    } else if (c.kind === "unique") {
+      text = `UNIQUE${label} (${list(c.columns ?? [])})`;
+      message = `Unique constraint${label} is not enforced by ClickHouse and is kept as a comment.`;
+      suggestion = "A MergeTree sort key does not make rows unique. ReplacingMergeTree collapses duplicates on merge only.";
+    } else if (c.kind === "exclusion") {
+      text = `EXCLUDE${label} ${oneLine(c.expression ?? "")}`;
+      message = `Exclusion constraint${label} is not enforced by ClickHouse and is kept as a comment.`;
+      suggestion = "ClickHouse has no constraint that rejects overlapping rows. Check for overlaps in the load, or in a query.";
+    } else {
+      text = `CHECK${label} ${oneLine(c.expression ?? "")}`;
+      message = `Check constraint${label} is not translated and is kept as a comment.`;
+      suggestion = "ClickHouse has CONSTRAINT name CHECK expr, enforced on INSERT. Rewrite the expression in ClickHouse SQL to use it.";
+    }
+    comments.push(`-- ${oneLine(text)}`);
+    findings.push({
+      ruleId: "SQLRV005",
+      severity: "warning",
+      object: input.object,
+      ...(c.kind !== "check" && c.kind !== "exclusion" && c.columns && c.columns.length === 1 ? { column: c.columns[0]! } : {}),
+      message,
+      evidence: [oneLine(text)],
+      suggestion,
+      ...(c.at ? { at: c.at } : {}),
+    });
+  }
+  return { comments, findings };
+}

package/src/codegen/classifier-page.ts

@@ -10,6 +10,7 @@ const LABEL: Record<string, string> = {
   drop: "drop",
   metadata: "metadata only",
   rewrite: "background rewrite",
+  replace: "replace table",
   rebuild: "rebuild",
 };
 
@@ -22,7 +23,7 @@ export function renderClassifierPage(): string {
   return `---
 title: "Planning ClickHouse Changes"
 label: "Planning and the Change Classifier"
-description: "How a ClickHouse schema change is classified as metadata only, a background rewrite or a rebuild, each with the ALTER restriction behind it"
+description: "How a ClickHouse schema change is classified as metadata only, a background rewrite, a replacement or a rebuild, each with the ALTER restriction behind it"
 diataxis: reference
 group: "ClickHouse"
 order: 1
@@ -36,9 +37,10 @@ Every change between two ClickHouse schemas is classified before anything runs:
 |---|---|
 | metadata only | the server records the change and touches no existing data |
 | background rewrite | a mutation rewrites existing parts in the background, with no rollback |
+| replace table | ClickHouse cannot alter a stream-engine table, and it stores no rows, so the applier drops and creates it around the views that read it |
 | rebuild | ClickHouse cannot make the change to the existing table, and the data has to be copied into a new one; a plan refuses to make it in place |
 
-Two commands report the classification, and \`chant lifecycle plan\` reports it as each update's disruption (metadata only is \`in-place\`, a rewrite \`rolling\`, a rebuild \`replace\`):
+Two commands report the classification, and \`chant lifecycle plan\` reports it as each update's disruption (metadata only is \`in-place\`, a rewrite \`rolling\`, a replacement or a rebuild \`replace\`):
 
 \`\`\`bash
 chant sql diff base.json head.json     # two chant build outputs, offline
@@ -47,13 +49,28 @@ chant sql plan prod schema.json        # a build output against the prod server
 
 Both exit 2 when a change needs a rebuild. \`chant sql diff\` compares a pull request's base and head builds with no server. \`chant sql plan\` reads the server \`sql.profiles.<env>\` binds, and asks that server's formatter about any expression the normalization rules leave different, so the server's own rewriting (\`INTERVAL 1 DAY\` as \`toIntervalDay(1)\`, \`a+b*2\` as \`a + (b * 2)\`) is not reported as a change.
 
-\`chant sql plan\` also exits 2 when a declared database holds an object whose definition it cannot read. It names each one under \`Refused\` (\`unreadable\` in \`--json\`) instead of leaving it out. Dictionaries are read and planned like the other objects: any change but a comment replaces one (SQLCH245).
+\`chant sql plan\` also exits 2 when a declared database holds an object whose definition it cannot read. It names each one under \`Refused\` (\`unreadable\` in \`--json\`) instead of leaving it out. It exits 2 as well when the build names a named collection the server doesn't have, naming each under \`Refused\` (\`namedCollections.missing\` in \`--json\`); see [Named collections](../clickhouse-ddl/#named-collections). Dictionaries are read and planned like the other objects: any change but a comment replaces one (SQLCH245).
 
 A clause chant's parser does not know ([Plain .sql Files](../sql-files/#clauses-chant-does-not-read)) is compared as text, and the rest of its object is planned as usual. A change to one is SQLCH290, refused for review with no rebuild Op suggested, since chant cannot tell what the change takes. A clause only the server prints is a hint, not a change: a newer server printing a clause with its default looks the same as a clause removed from the declaration.
 
+## Stream tables
+
+A table on Kafka, S3Queue, AzureQueue, RabbitMQ, NATS or FileLog stores no rows. It is the consumer that a materialized view reads through, and the position it has reached lives in the broker's committed offsets or in Keeper. There is no \`ALTER\` for its engine arguments, settings or columns, and there is nothing to copy, so a change to one is neither metadata nor a rebuild. It is SQLCH225, in the replace class (\`diffStreamTable\` in \`src/clickhouse/plan/diff.ts\`). The plan refuses nothing for it.
+
+The applier makes the change in place, in this order (\`replaceStreamSteps\` in \`src/clickhouse/apply/statements.ts\`): \`DETACH TABLE\` for each materialized view on the server that reads the table, \`DROP TABLE ... SYNC\`, the declared \`CREATE TABLE\`, then \`ATTACH TABLE\` for the same views. The plan names the views in a hint. Messages wait in the broker, and files wait in the bucket, while the views are detached. A change that moves a table to or from a stream engine is still SQLCH223, a rebuild, and \`ClickHouseRebuildOp\` refuses a stream-engine table by name (\`streamRefusal\` in \`src/clickhouse/rebuild/observe.ts\`).
+
+Two replacements lose the table's position, and the change carries a note saying so (\`positionWarnings\` in \`src/clickhouse/plan/stream.ts\`):
+
+- A different \`kafka_group_name\`, as a setting or as the third engine argument, is a different consumer group. It starts from its own committed offsets, or from the start of the topic when it has none.
+- A different S3Queue \`keeper_path\` is a different record of processed files. Every file already seen is processed again.
+
+An apply killed between the detach and the attach leaves the views detached, and a detached view is not in \`system.tables\`. Nothing re-attaches it: run \`ATTACH TABLE\` for each view by hand, then plan again. This has not been run against a server.
+
+The \`StreamIngest\` composite declares the stream table, the view and the MergeTree target together: see [Composites](../composites/#streamingest).
+
 ## After the plan
 
-The plan only reports. [\`clickhouseApply\`](../applying/) makes the metadata-only and background-rewrite changes, and refuses a rebuild the same way the plan does, sending nothing for that object. A rebuild runs as \`ClickHouseRebuildOp\`, a gated migration Op that creates the new table, backfills it, verifies it and swaps it in: see [Rebuilding a Table](../rebuild/). Both commands print the Op declaration to start from for each refused table.
+The plan only reports. [\`clickhouseApply\`](../applying/) makes the metadata-only, background-rewrite and stream-table changes, and refuses a rebuild the same way the plan does, sending nothing for that object. A rebuild runs as \`ClickHouseRebuildOp\`, a gated migration Op that creates the new table, backfills it, verifies it and swaps it in: see [Rebuilding a Table](../rebuild/). Both commands print the Op declaration to start from for each refused table.
 
 \`chant migrate\` plays no part in any of this. It translates a file from one lexicon's format into another's, such as a GitHub Actions workflow into GitLab CI, and does not run schema migrations.
 
@@ -76,6 +93,29 @@ A rebuild is never DDL: it is a step naming \`ClickHouseRebuildOp\`, or for a vi
 
 \`--topology\` renders the statements for where they will run: \`single\` (the default), \`cluster:<name>\`, \`replicated\` or \`cloud\`. On a cluster every statement carries \`ON CLUSTER\` and a MergeTree-family table becomes \`Replicated*MergeTree\` with its Keeper path; in a \`Replicated\` database the tables are \`Replicated*MergeTree\` with no path and no \`ON CLUSTER\`. \`diffStatements(before, after, { topology })\` takes the same choice; see [Applying to a Server](../applying/#topology).
 
+## Migration files on a pull request
+
+A workspace member that is a goose, golang-migrate, clickhouse-migrations or Flyway directory (see [Workspace kinds](/chant/reference/workspace-kinds/)) has no declaration to diff. On a pull request, \`chant components pr-plan\` lists each migration file the pull request adds to such a member, and the plan note gives every statement in it the class the rules below give the same change. This is the "check" level of adopting chant beside an existing migration tool: it reads files, applies nothing and contacts no server, so it needs no \`--env\`.
+
+The up statements are the ones \`chant sql replay\` would run: goose's \`-- +goose Up\` section, the whole \`.up.sql\` file for golang-migrate, the whole file for clickhouse-migrations, and a Flyway file after placeholder substitution with the values in \`flyway.toml\` or \`flyway.conf\`. A file the base commit already has is not new, whatever the pull request does to it. Undo files are left out.
+
+\`\`\`text
+\`db/00002_orders.sql\` (member \`ledger\`, postgres)
+- 1. \`CREATE INDEX orders_status_idx ON orders (status)\`: needs CONCURRENTLY (SQLPG241, SHARE)
+- 2. \`ALTER TABLE orders ALTER COLUMN a TYPE bigint\`: undecided between SQLPG206 metadata only and SQLPG207 ACCESS EXCLUSIVE rewrite or scan
+\`\`\`
+
+Each statement is read with the dialect's parser (\`CREATE\`, \`GRANT\`, \`COMMENT ON\`) or word by word (\`ALTER\`, \`DROP\`, \`RENAME\`), and the lock on a Postgres statement is the one its rule's text names. The code is \`lexicons/sql/src/postgres/plan/statements.ts\` and \`lexicons/sql/src/clickhouse/plan/statements.ts\`.
+
+A statement alone cannot always decide a rule, and then the note names the candidates and does not guess:
+
+- \`ALTER COLUMN ... TYPE\` is SQLPG206 or SQLPG207 by the column's old type, and SQLPG208 as well with a \`USING\` expression.
+- \`MODIFY COLUMN <type>\` is SQLCH210 or SQLCH211, \`DROP COLUMN\` SQLCH202 or SQLCH213, and \`RENAME COLUMN\` SQLCH212 or SQLCH213, by whether the column is in a key. When the same file creates the table, its keys are known and the statement is classified.
+- \`MODIFY ORDER BY\` is SQLCH220, a rebuild. It is SQLCH216 or SQLCH220 when the same \`ALTER\` adds the columns the new key ends with, unless the file created the table and shows the old key is kept.
+- \`CREATE OR REPLACE VIEW\` is SQLPG250 or SQLPG251 by whether the view keeps its columns.
+
+A statement that reads or writes rows (\`INSERT\`, \`UPDATE\`, \`TRUNCATE\`) is listed as data. A schema change no rule covers, such as \`ALTER TABLE ... OWNER TO\` or a ClickHouse \`DROP PARTITION\`, is listed as having no class.
+
 ## Identity and renames
 
 An object is identified by its export name between two builds, so a changed name in the SQL under the same export is a rename. Against a server, which has no export names, an object is identified by \`database.name\`; a declaration renamed since the server last saw it says so with \`-- previously: <old name>\` before its CREATE.

package/src/codegen/clickhouse-module.ts

@@ -269,6 +269,9 @@ export function renderClickHouseModule(catalog: ClickHouseCatalog): RenderedModu
     `/** The table functions (\`system.table_functions\`): \`numbers\`, \`remote\`, \`s3\`. */`,
     `export const TABLE_FUNCTIONS: readonly string[] = ${JSON.stringify(catalog.tableFunctions)};`,
     "",
+    `/** The input and output formats (\`system.formats\`): \`Parquet\`, \`JSONEachRow\`. */`,
+    `export const FORMATS: readonly string[] = ${JSON.stringify(catalog.formats.map((f) => f.name))};`,
+    "",
     `export const SKIP_INDEX_TYPES: Readonly<Record<SkipIndexType, SkipIndexSpec>> = {`,
     ...indexLines,
     "};",

package/src/codegen/coverage-page.ts

@@ -9,7 +9,7 @@ import type { SqlDialect } from "../dialects";
 
 const cell = (s: string) => s.replace(/\|/g, "\\|").replace(/[{}<>]/g, (c) => `\\${c}`);
 
-const STATUS: Record<KindRow["status"], string> = { declared: "declared", refused: "refused", "out-of-scope": "out of scope" };
+const STATUS: Record<KindRow["status"], string> = { declared: "declared", referenced: "referenced", refused: "refused", "out-of-scope": "out of scope" };
 
 function table(dialect: SqlDialect): string {
   const rows = [...KIND_COVERAGE[dialect]].sort((a, b) => (a.command < b.command ? -1 : a.command > b.command ? 1 : 0));
@@ -27,7 +27,8 @@ function table(dialect: SqlDialect): string {
 function counts(dialect: SqlDialect): string {
   const rows = KIND_COVERAGE[dialect];
   const n = (s: KindRow["status"]) => rows.filter((r) => r.status === s).length;
-  return `${n("declared")} declared, ${n("refused")} refused, ${n("out-of-scope")} out of scope`;
+  const referenced = n("referenced") > 0 ? `, ${n("referenced")} referenced` : "";
+  return `${n("declared")} declared${referenced}, ${n("refused")} refused, ${n("out-of-scope")} out of scope`;
 }
 
 export function renderCoveragePage(): string {
@@ -36,7 +37,7 @@ export function renderCoveragePage(): string {
   return `---
 title: "Object Coverage"
 label: "Object Coverage"
-description: "Every kind of object each pinned Postgres and ClickHouse server can create, marked declared, refused with a reason, or out of scope, and what import does with each"
+description: "Every kind of object each pinned Postgres and ClickHouse server can create, marked declared, referenced, refused with a reason, or out of scope, and what import does with each"
 diataxis: reference
 order: 3
 ---
@@ -46,6 +47,7 @@ order: 3
 Each row is a \`CREATE\` command a pinned server has, or one form of a command where chant treats the forms apart (\`CREATE TYPE ... AS ENUM\` is declared, \`CREATE TYPE ... AS RANGE\` is not). The commands come from the servers' own sources at each pinned version (\`scripts/sql-create-commands.ts\`), and a test fails when a pinned server has a command with no row here, so a new server release cannot add a kind nobody looked at.
 
 - **declared**: a tag declares it, and import writes it as that declaration.
+- **referenced**: the environment's, and never declared: a declaration names one, and chant never reads its values. A ClickHouse named collection is one ([#4395](https://github.com/INTENTIUS/chant/issues/4395)): \`ENGINE = S3(lake_creds, format = 'Parquet')\` names \`lake_creds\`, the build records the name, and \`chant sql plan\` fails when the server has no collection of that name. \`chant import --from <env>\` lists the server's SQL-defined collections as prerequisites and imports the rest. See [Named collections](../clickhouse-ddl/#named-collections).
 - **refused**: no declaration holds it yet. \`chant import --from <env>\` fails when the server has one, naming every such object (kind, schema, name) at once, and \`chant import schema.sql\` fails on the statement. Nothing is left out without a word.
 - **out of scope**: the environment's, not a schema's: import never reads it.
 

package/src/codegen/docs.ts

@@ -122,7 +122,7 @@ Postgres:
 - [Getting Started with Postgres](./postgres-getting-started/): declare a
   schema, apply it to a local server, add a column and an index, and see a
   rename refused.
-- [Declaring Postgres Objects](./postgres-ddl/): the eight tags,
+- [Declaring Postgres Objects](./postgres-ddl/): the tags for every schema-level kind,
   constraints, \`nextval(\${seq})\`, \`COMMENT ON\` and \`literal()\`.
 - [Importing a Postgres Server](./postgres-importing/): canonical text from
   the catalog's printers, and the objects that belong to someone else.

package/src/codegen/postgres-classifier-page.ts

@@ -65,14 +65,14 @@ A column rename or a type change across kinds is never DDL: it is a step naming
 
 ## Which major
 
-Two rules change class across the supported majors: a STORED generated column's expression (SQLPG212) is a rewrite from 17 and expand and contract before it, and a table's access method (SQLPG226) is a rewrite from 15 and expand and contract before it.
+Three rules change class across the supported majors: a STORED generated column's expression (SQLPG212) is a rewrite from 17 and expand and contract before it, a table's access method (SQLPG226) is a rewrite from 15 and expand and contract before it, and an operator's commutator, negator, HASHES and MERGES (SQLPG304) are altered in place from 17 and dropped and created before it.
 
 - \`chant sql diff\` classifies for the major the builds recorded (\`postgresMajor\` in the build output): the newer build's, else the older build's, else \`sql.postgresMajor\`, else 18. A diff between two revisions does not depend on today's config.
 - \`chant sql plan\` classifies for the major the server runs (\`server_version_num\`), since the server is what takes the locks. When the build targets another major, the plan says so in a hint.
 
 ## Identity and renames
 
-An object is identified by its export name between two builds, and by its schema-qualified name against a server, which has no export names. Tables, views, materialized views, sequences and indexes share one namespace per schema, types and domains another, and schemas and extensions are matched by name. Functions, procedures and aggregates share a namespace of their own and are told apart by their input parameter types, so an overload is another object; composite and range types share the types' namespace, collations have one of their own, a cast is named by its two types and an event trigger by its name in the database; a trigger is matched by its name on its table.
+An object is identified by its export name between two builds, and by its schema-qualified name against a server, which has no export names. Tables, foreign tables, views, materialized views, sequences and indexes share one namespace per schema, types and domains another, and schemas and extensions are matched by name. Functions, procedures and aggregates share a namespace of their own and are told apart by their input parameter types, so an overload is another object; composite and range types share the types' namespace, collations have one of their own, extended statistics another, an operator by its symbol and operand types, an operator class and an operator family each by its name and index method, text search configurations one namespace and dictionaries another, a cast by its two types and an event trigger, a foreign data wrapper, a foreign server and a publication by their names in the database; a trigger is matched by its name on its table.
 
 A rename is declared where it happens. Before a CREATE, \`-- previously: <old name>\` says the object had another name; on a column's line, it says the column did: