@intentius/chant-lexicon-sql 0.118.1 → 0.120.0

package/src/clickhouse/rebuild/rebuild.test.ts

@@ -24,6 +24,8 @@ import { partitionEffect } from "./backfill";
 import { insertTarget, waitForCutover, type CutoverProbe } from "./steps";
 import { intoShard, onShard, onShardServers, shardTable } from "./shards";
 import { renameColumns, sourcePartitionExpression } from "./partitions";
+import { collapseSemantics, collapsesRows, engineArguments, splitTopLevel, type EngineColumn } from "./engines";
+import { RebuildVerificationError } from "./verify";
 import { canonicalObject } from "../plan/normalize";
 import { diffSchemas } from "../plan/diff";
 import { renderDiff } from "../plan/report";
@@ -226,6 +228,42 @@ describe("the plan hands a rebuild to the Op", () => {
   });
 });
 
+describe("a table whose engine collapses rows by sorting key (#3674)", () => {
+  test("the collapsing MergeTree engines are read under FINAL, replicated or shared too; the others are not", () => {
+    for (const e of ["SummingMergeTree", "ReplacingMergeTree", "AggregatingMergeTree", "CollapsingMergeTree", "VersionedCollapsingMergeTree", "ReplicatedSummingMergeTree", "SharedReplacingMergeTree"]) {
+      expect(collapsesRows(e), e).toBe(true);
+    }
+    for (const e of ["MergeTree", "ReplicatedMergeTree", "Log", "Distributed", undefined]) expect(collapsesRows(e), String(e)).toBe(false);
+  });
+
+  test("a difference on such a table names the engine and the OPTIMIZE that merges it, not only a late write", () => {
+    const mismatch = [{ partition: "all", old: { partition: "all", rows: 3, checksum: "1" }, new: { partition: "all", rows: 1, checksum: "2" } }];
+    const collapsing = new RebuildVerificationError("db.daily", mismatch, "SummingMergeTree").message;
+    expect(collapsing).toMatch(/Nothing was swapped\. db\.daily is a SummingMergeTree, .*compared under FINAL/);
+    expect(collapsing).toContain("run OPTIMIZE TABLE db.daily FINAL");
+    const plain = new RebuildVerificationError("db.events", mismatch).message;
+    expect(plain).not.toMatch(/FINAL/);
+    expect(plain).toMatch(/The usual cause is in materialized-view mode a row written to the old table later than the cut-over delay/);
+  });
+
+  test("from a plain MergeTree, a difference names what was compared per key", () => {
+    const mismatch = [{ partition: "202610", old: { partition: "202610", rows: 2, checksum: "1" }, new: { partition: "202610", rows: 1, checksum: "2" } }];
+    const message = new RebuildVerificationError("db.daily", mismatch, { engine: "SummingMergeTree", how: "keys", describe: "the sums of total" }).message;
+    expect(message).toContain("202610: old 2 keys, checksum 1, new 1 keys, checksum 2");
+    expect(message).toMatch(/The new table is a SummingMergeTree, .*compared per key on the sums of total/);
+    expect(message).not.toMatch(/OPTIMIZE/);
+  });
+
+  test("the plan's hand-off says so for such a table", () => {
+    const ddl = (o: string) => `CREATE TABLE db.daily (day Date, total Decimal(18, 2), n UInt64) ENGINE = SummingMergeTree ORDER BY ${o}`;
+    const after = [{ key: "daily", canonical: canonicalObject(ddl("(day, n)")) }];
+    const diff = diffSchemas([{ key: "daily", canonical: canonicalObject(ddl("day")) }], after);
+    const ops = rebuildOpSuggestions(diff, new Map(after.map((o) => [o.key, o.canonical])), "prod");
+    expect(ops[0]!.note).toMatch(/^db\.daily is a SummingMergeTree: when the old table's engine collapses rows too, the Verify phase reads both tables under FINAL/);
+    expect(renderDiff({ ...diff, rebuildOps: ops })).toContain(`  Note: ${ops[0]!.note}`);
+  });
+});
+
 describe("a rebuild across the shards of a cluster (#3663)", () => {
   const sharding = { cluster: "main", shards: [{ num: 1, macro: "s1", slot: 0 }, { num: 2, macro: "s2", slot: 1 }] };
   const shard2 = sharding.shards[1]!;
@@ -294,3 +332,43 @@ describe("the wait for the cut-over (#3700)", () => {
     expect(insertTarget("SELECT * FROM shop.events", "shop")).toBeUndefined();
   });
 });
+
+describe("what a collapsing engine keeps per sorting key (#3727)", () => {
+  const col = (name: string, type: string, inSortingKey = false): EngineColumn => ({ name, type, inSortingKey });
+  const columns = [col("day", "Date", true), col("shop", "UInt32", true), col("total", "Decimal(18, 2)"), col("n", "UInt64"), col("ratio", "Float64"), col("note", "String"), col("sign", "Int8"), col("ver", "UInt32")];
+
+  test("splitTopLevel and engineArguments read engine_full, without a Replicated engine's Keeper path", () => {
+    expect(splitTopLevel("toStartOfInterval(ts, toIntervalDay(1)), id")).toEqual(["toStartOfInterval(ts, toIntervalDay(1))", "id"]);
+    expect(splitTopLevel("")).toEqual([]);
+    expect(engineArguments("SummingMergeTree((total, n)) PARTITION BY toYYYYMM(day) ORDER BY (day, shop) SETTINGS index_granularity = 8192")).toEqual(["(total, n)"]);
+    expect(engineArguments("ReplicatedCollapsingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}', sign) ORDER BY id")).toEqual(["sign"]);
+    expect(engineArguments("SummingMergeTree ORDER BY id")).toEqual([]);
+  });
+
+  test("SummingMergeTree: the sums of every numeric column outside the key, or of those listed, in their own types; a key whose sums are zero is removed", () => {
+    const all = collapseSemantics("SummingMergeTree", "SummingMergeTree ORDER BY (day, shop)", columns)!;
+    expect(all.aggregates).toEqual(["CAST(sum(`total`), 'Decimal(18, 2)')", "CAST(sum(`n`), 'UInt64')", "toFloat32(sum(`ratio`))", "CAST(sum(`sign`), 'Int8')", "CAST(sum(`ver`), 'UInt32')"]);
+    expect(all.having).toBe(`NOT (${all.aggregates.map((a) => `${a} = 0`).join(" AND ")})`);
+    const listed = collapseSemantics("ReplicatedSummingMergeTree", "ReplicatedSummingMergeTree('/p', '{replica}', (total, n)) ORDER BY (day, shop)", columns)!;
+    expect(listed).toMatchObject({ engine: "SummingMergeTree", aggregates: ["CAST(sum(`total`), 'Decimal(18, 2)')", "CAST(sum(`n`), 'UInt64')"], describe: "the sums of total, n" });
+    const partitioned = collapseSemantics("SummingMergeTree", "SummingMergeTree PARTITION BY g ORDER BY k", [col("k", "UInt32", true), { ...col("g", "UInt8"), inPartitionKey: true }, col("a", "UInt32")])!;
+    expect(partitioned.aggregates).toEqual(["CAST(sum(`a`), 'UInt32')"]);
+  });
+
+  test("Replacing keeps the highest version, Collapsing the sum of its signs, Aggregating its order-free simple functions, Coalescing which columns hold a value", () => {
+    expect(collapseSemantics("ReplacingMergeTree", "ReplacingMergeTree(ver) ORDER BY id", columns)).toMatchObject({ aggregates: ["max(`ver`)"], describe: "the highest ver" });
+    expect(collapseSemantics("ReplacingMergeTree", "ReplacingMergeTree ORDER BY id", columns)).toMatchObject({ aggregates: [], describe: "the keys" });
+    expect(collapseSemantics("CollapsingMergeTree", "CollapsingMergeTree(sign) ORDER BY id", columns)).toMatchObject({ aggregates: ["sum(`sign`)"], having: "sum(`sign`) != 0" });
+    expect(collapseSemantics("VersionedCollapsingMergeTree", "VersionedCollapsingMergeTree(sign, ver) ORDER BY id", columns)).toMatchObject({ aggregates: ["sum(`sign`)"], having: "sum(`sign`) != 0" });
+    const agg = [col("k", "UInt32", true), col("hi", "SimpleAggregateFunction(max, UInt64)"), col("any", "SimpleAggregateFunction(anyLast, String)"), col("u", "AggregateFunction(uniq, UInt64)")];
+    expect(collapseSemantics("AggregatingMergeTree", "AggregatingMergeTree ORDER BY k", agg)).toMatchObject({ aggregates: ["max(`hi`)"], describe: "max(hi)" });
+    const coalescing = [col("k", "UInt32", true), col("v", "Nullable(String)"), col("w", "UInt8")];
+    expect(collapseSemantics("CoalescingMergeTree", "CoalescingMergeTree ORDER BY k", coalescing)).toMatchObject({ aggregates: ["count(`v`) > 0"] });
+  });
+
+  test("an engine that keeps every row, or whose rows depend on the order of merges, has none", () => {
+    expect(collapseSemantics("MergeTree", "MergeTree ORDER BY id", columns)).toBeUndefined();
+    expect(collapseSemantics("GraphiteMergeTree", "GraphiteMergeTree('graphite_rollup') ORDER BY id", columns)).toBeUndefined();
+    expect(collapseSemantics("CollapsingMergeTree", "CollapsingMergeTree(missing) ORDER BY id", columns)).toBeUndefined();
+  });
+});

package/src/clickhouse/rebuild/verify.ts

@@ -24,6 +24,14 @@
  * later than the cut-over delay, with a time before the cut-over) stop the
  * swap too, rather than stay behind in the old table.
  *
+ * A table whose engine collapses rows that share a sorting key when parts
+ * merge (Summing, Replacing, Aggregating, Collapsing, VersionedCollapsing,
+ * Coalescing and Graphite, replicated or not), rebuilt into such an engine,
+ * is read under `FINAL` on both sides (#3674). The old table can hold unmerged parts with several rows for
+ * one key, which the copy inserts in one block and the new table collapses
+ * as it writes them, so the raw counts differ although the data is the same.
+ * Under `FINAL` both sides are compared as merged.
+ *
  * On a cluster of more than one shard (#3663) every shard is read, through
  * the profile's server, and each partition is compared per shard
  * (`shard<n>/<partition>`), so a shard the copy missed is a difference.
@@ -44,9 +52,11 @@ import { rebuildPlanSubject, RebuildRefusal, type RebuildObservation } from "./o
 import { sourcePartitionExpression } from "./partitions";
 import { cutoverOf, observe, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
 import { shardTable } from "./shards";
+import { collapseSemantics, collapsesRows, splitTopLevel, type CollapseSemantics, type EngineColumn } from "./engines";
 
 export interface PartitionCheck {
   partition: string;
+  /** Rows; keys of the new sorting key when the tables were compared by key (#3727). */
   rows: number;
   checksum: string;
 }
@@ -67,23 +77,42 @@ export interface VerifyResult {
 
 /** A verification that found the tables different. */
 export class RebuildVerificationError extends Error {
+  /** The engine, when it collapses rows and the tables were compared under FINAL (#3674) or by sorting key (#3727). */
+  readonly collapsingEngine?: string;
   constructor(
     readonly table: string,
     readonly mismatches: Mismatch[],
+    /** How the tables were compared when an engine collapses rows; an engine's name alone means under FINAL. */
+    collapsed?: string | Collapsed,
   ) {
+    const how: Collapsed | undefined = typeof collapsed === "string" ? { engine: collapsed, how: "final" } : collapsed;
     const shown = mismatches
       .slice(0, 10)
-      .map((m) => `${m.partition}: old ${m.old ? `${m.old.rows} rows, checksum ${m.old.checksum}` : "none"}, new ${m.new ? `${m.new.rows} rows, checksum ${m.new.checksum}` : "none"}`)
+      .map((m) => `${m.partition}: old ${m.old ? `${m.old.rows} ${unit(how)}, checksum ${m.old.checksum}` : "none"}, new ${m.new ? `${m.new.rows} ${unit(how)}, checksum ${m.new.checksum}` : "none"}`)
       .join("; ");
+    const lateWrite =
+      `in materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, ` +
+      `and in app mode a write the application did not stop`;
     super(
-      `${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. ` +
-        `Nothing was swapped. In materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, is the usual cause; ` +
-        `in app mode, a write the application did not stop.`,
+      `${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. Nothing was swapped. ` +
+        (how?.how === "final"
+          ? `${table} is a ${how.engine}, which collapses rows that share a sorting key when parts merge, and both tables were compared under FINAL. ` +
+            `They still differ when the new table collapses the copied rows by its sorting key differently from how the old one merges them: ` +
+            `run OPTIMIZE TABLE ${table} FINAL, drop the new table if the Op kept it, and run the rebuild again, so the copy reads merged rows. ` +
+            `The other causes are ${lateWrite}.`
+          : how?.how === "keys"
+            ? `The new table is a ${how.engine}, which collapses rows that share a sorting key, so both tables were grouped by its sorting key and partition ` +
+              `and compared per key on ${how.describe ?? "the keys"}, which its merges keep. A key missing from the new table, or one whose values differ, is rows the copy lost or changed. ` +
+              `The usual cause is ${lateWrite}.`
+            : `The usual cause is ${lateWrite}.`),
     );
     this.name = "RebuildVerificationError";
+    if (how) this.collapsingEngine = how.engine;
   }
 }
 
+const unit = (how: Collapsed | undefined) => (how?.how === "keys" ? "keys" : "rows");
+
 /** How often a difference is read again before it counts, and how long apart. */
 const COMPARE_ATTEMPTS = 5;
 const COMPARE_INTERVAL_MS = 1000;
@@ -92,6 +121,81 @@ const COMPARE_INTERVAL_MS = 1000;
 interface SideCounts {
   all: Map<string, PartitionCheck>;
   before: Map<string, PartitionCheck>;
+  /** Grouped by sorting key (#3727): the rows read, of which `all` counts the keys. */
+  rows?: number;
+}
+
+/**
+ * How the tables were compared when an engine collapses rows: both under
+ * FINAL (#3674), or grouped by the new table's sorting key on what its
+ * engine's merges keep (#3727).
+ */
+export interface Collapsed {
+  engine: string;
+  how: "final" | "keys";
+  /** With `keys`: what is compared per key. */
+  describe?: string;
+}
+
+/**
+ * Both tables grouped by the new table's sorting key and its partition, each
+ * key compared on what the new engine's merges keep (#3727): its sums, its
+ * highest version, the sum of its signs. Each side is read raw, so the
+ * comparison is the same whichever parts have merged. The old side's columns
+ * are cast to the new table's types and named as there, so the sorting key,
+ * the partition key and the aggregates read the same on both sides. A
+ * partition is the new table's (`shard<n>/` before it on a cluster), and its
+ * rows are its keys.
+ */
+async function keyComparison(run: RebuildRun, o: RebuildObservation, columns: readonly EngineColumn[]): Promise<{ semantics: CollapseSemantics; read: (side: "old" | "new") => Promise<SideCounts> } | undefined> {
+  const n = o.names;
+  const [t] = await clickhouseQuery<{ engine_full: string; sorting_key: string; partition_key: string }>(
+    run.target.endpoint,
+    `SELECT engine_full, sorting_key, partition_key FROM system.tables WHERE database = ${sqlString(n.database)} AND name = ${sqlString(n.newName)}`,
+  );
+  if (!t || !o.newTable) return undefined;
+  const semantics = collapseSemantics(o.newTable.engine, t.engine_full, columns);
+  if (!semantics) return undefined;
+  // Every column of the sorting key must come from the old table, or the old rows have no key to group by.
+  const copied = new Set(o.copied.map((c) => c.name));
+  if (columns.some((c) => c.inSortingKey && !copied.has(c.name))) return undefined;
+
+  const types = new Map(columns.map((c) => [c.name, c.type]));
+  const sharding = run.sharding;
+  const cutover = cutoverOf(run, o);
+  const cut = run.dualWrite.mode === "materialized-view" ? (run.dualWrite as { cutoverColumn: string }).cutoverColumn : undefined;
+  const before = cut && copied.has(cut) && cutover !== undefined ? `${ident(cut)} < ${utcLiteral(cutover)}` : "1";
+  const key = splitTopLevel(t.sorting_key).filter(Boolean);
+  const parts = splitTopLevel(t.partition_key).filter(Boolean);
+  const partition = parts.length === 0 ? "'all'" : `toString(${parts.length === 1 ? parts[0] : `tuple(${parts.join(", ")})`})`;
+  const label = sharding ? `concat('shard', toString(__shard), '/', ${partition})` : partition;
+  const hash = `cityHash64(toString(tuple(${[...key, ...semantics.aggregates].join(", ")})))`;
+
+  const read = async (side: "old" | "new"): Promise<SideCounts> => {
+    const name = side === "old" ? n.name : n.newName;
+    const from = sharding ? shardTable(sharding, n.database, name) : side === "old" ? n.table : n.newTable;
+    const projected = o.copied.map((c) => (side === "old" ? `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")}) AS ${ident(c.name)}` : ident(c.name)));
+    const rowsOf = `SELECT ${sharding ? "_shard_num AS __shard, " : ""}${projected.join(", ")} FROM ${from}`;
+    const perKey =
+      `SELECT ${label} AS __p, ${hash} AS __h, max(${before}) AS __b, count() AS __n, ${semantics.having ?? "1"} AS __k FROM (${rowsOf}) ` +
+      `GROUP BY __p${sharding ? ", __shard" : ""}${key.length > 0 ? `, ${key.join(", ")}` : ""}`;
+    // A key the engine removes (its sums all zero, its signs cancelled) is not compared, but its rows are counted as read.
+    const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string; raw: string | number }>(
+      run.target.endpoint,
+      `SELECT __p AS p, countIf(__k) AS rows, toString(sumIf(__h, __k)) AS checksum, countIf(__b AND __k) AS brows, toString(sumIf(__h, __b AND __k)) AS bchecksum, sum(__n) AS raw ` +
+        `FROM (${perKey}) GROUP BY p`,
+    );
+    const out: SideCounts = { all: new Map(), before: new Map(), rows: 0 };
+    for (const r of rows) {
+      out.rows! += Number(r.raw);
+      // A partition whose keys the engine all removes holds nothing in the new table, as if it had no rows.
+      if (Number(r.rows) === 0) continue;
+      out.all.set(r.p, { partition: r.p, rows: Number(r.rows), checksum: r.checksum });
+      if (Number(r.brows) > 0) out.before.set(r.p, { partition: r.p, rows: Number(r.brows), checksum: r.bchecksum });
+    }
+    return out;
+  };
+  return { semantics, read };
 }
 
 /**
@@ -103,21 +207,21 @@ async function compareTables(
   run: RebuildRun,
   o: RebuildObservation,
   opts: { attempts?: number; intervalMs?: number } = {},
-): Promise<{ old: SideCounts; new: SideCounts; mismatches: Mismatch[] }> {
+): Promise<{ old: SideCounts; new: SideCounts; mismatches: Mismatch[]; collapsed?: Collapsed }> {
   if (!o.newTable) throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
   const n = o.names;
   const cutover = cutoverOf(run, o);
   await waitOn(run, n.database, n.name);
   await waitOn(run, n.database, n.newName);
 
-  const types = new Map(
-    (
-      await clickhouseQuery<{ name: string; type: string }>(
-        run.target.endpoint,
-        `SELECT name, type FROM system.columns WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)}`,
-      )
-    ).map((c) => [c.name, c.type]),
-  );
+  const newColumns = (
+    await clickhouseQuery<{ name: string; type: string; key: number | string; partition: number | string }>(
+      run.target.endpoint,
+      `SELECT name, type, is_in_sorting_key AS key, is_in_partition_key AS partition FROM system.columns ` +
+        `WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)} ORDER BY position`,
+    )
+  ).map((c): EngineColumn => ({ name: c.name, type: c.type, inSortingKey: Number(c.key) === 1, inPartitionKey: Number(c.partition) === 1 }));
+  const types = new Map(newColumns.map((c) => [c.name, c.type]));
   const oldHash = rowHash(o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`));
   const newHash = rowHash(o.copied.map((c) => ident(c.name)));
   const cut = run.dualWrite.mode === "materialized-view" ? o.copied.find((c) => c.name === (run.dualWrite as { cutoverColumn: string }).cutoverColumn) : undefined;
@@ -127,8 +231,15 @@ async function compareTables(
 
   // Across the shards of a cluster (#3663), every shard's rows, each partition counted per shard as `shard<n>/<partition>`.
   const sharding = run.sharding;
-  const read = async (table: string, name: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
-    const from = sharding ? shardTable(sharding, n.database, name) : table;
+  // Tables whose engines collapse rows by sorting key are read as merged (#3674). Only when both do: a collapsing
+  // table rebuilt into a plain MergeTree keeps every row it copies, and is compared row for row.
+  const final = collapsesRows(o.live.engine) && collapsesRows(o.newTable.engine);
+  // A table that does not collapse rows rebuilt into one that does (#3727): the new table keeps one row per sorting
+  // key where the old one keeps them all, by design. Both are grouped by the new sorting key and compared on what
+  // the new engine's merges keep per key.
+  const keys = !final && collapsesRows(o.newTable.engine) ? await keyComparison(run, o, newColumns) : undefined;
+  const read = async (table: string, name: string, partition: string, hash: string, before: string, final: boolean): Promise<SideCounts> => {
+    const from = `${sharding ? shardTable(sharding, n.database, name) : table}${final ? " FINAL" : ""}`;
     const key = sharding ? `concat('shard', toString(_shard_num), '/', ${partition})` : partition;
     const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string }>(
       run.target.endpoint,
@@ -150,10 +261,18 @@ async function compareTables(
     await syncReplica(run, o, n.database, n.name);
     await syncReplica(run, o, n.database, n.newName);
     // The old table first: a row the view has yet to commit in the new one is read as missing there, never the other way round.
-    const before = await read(n.table, n.name, "_partition_id", oldHash, oldBefore);
-    const after = await read(n.newTable, n.newName, fromOld, newHash, newBefore);
+    const before = keys ? await keys.read("old") : await read(n.table, n.name, "_partition_id", oldHash, oldBefore, final);
+    const after = keys ? await keys.read("new") : await read(n.newTable, n.newName, fromOld, newHash, newBefore, final);
     const mismatches = differences(before.all, after.all);
-    if (mismatches.length === 0 || attempt >= attempts) return { old: before, new: after, mismatches };
+    if (mismatches.length === 0 || attempt >= attempts) {
+      return {
+        old: before,
+        new: after,
+        mismatches,
+        ...(final ? { collapsed: { engine: o.live.engine, how: "final" as const } } : {}),
+        ...(keys ? { collapsed: { engine: o.newTable.engine, how: "keys" as const, describe: keys.semantics.describe } } : {}),
+      };
+    }
     run.log(`-- ${n.key}: ${mismatches.length} partition(s) differ (${mismatches.map((m) => m.partition).slice(0, 5).join(", ")}); reading again (${attempt}/${attempts})`);
     await sleep(opts.intervalMs ?? COMPARE_INTERVAL_MS, run.signal);
   }
@@ -179,8 +298,8 @@ export const rowHash = (columns: readonly string[]): string => `cityHash64(toStr
  * {@link RebuildVerificationError} when it does not, so nothing is swapped.
  */
 export async function assertSameRows(run: RebuildRun, o: RebuildObservation): Promise<void> {
-  const { mismatches } = await compareTables(run, o);
-  if (mismatches.length > 0) throw new RebuildVerificationError(o.names.key, mismatches);
+  const { mismatches, collapsed } = await compareTables(run, o);
+  if (mismatches.length > 0) throw new RebuildVerificationError(o.names.key, mismatches, collapsed);
 }
 
 export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
@@ -188,15 +307,25 @@ export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
   if (o.state !== "rebuild") return { state: o.state, partitions: 0, rows: 0, verification: [], summary: `nothing to verify: ${o.state}` };
   const cutover = cutoverOf(run, o);
   const compared = await compareTables(run, o);
-  if (compared.mismatches.length > 0) throw new RebuildVerificationError(o.names.key, compared.mismatches);
+  if (compared.mismatches.length > 0) throw new RebuildVerificationError(o.names.key, compared.mismatches, compared.collapsed);
 
   const all = [...compared.old.all.values()];
-  const rows = all.reduce((s, v) => s + v.rows, 0);
+  const counted = all.reduce((s, v) => s + v.rows, 0);
+  const c = compared.collapsed;
+  // Grouped by sorting key (#3727), each partition counts keys; the old table's rows are still every row read.
+  const rows = c?.how === "keys" ? (compared.old.rows ?? counted) : counted;
   // What the approval binds: the rows before the cut-over (every row in app mode), which later writes do not change.
   const verification = [...compared.old.before.values()].sort((x, y) => x.partition.localeCompare(y.partition));
   const bound = verification.reduce((s, v) => s + v.rows, 0);
-  const range = cutover !== undefined ? ` (${bound} of them before the cut-over at ${new Date(cutover).toISOString()})` : "";
-  const summary = `${o.names.key}: ${all.length} partition(s), ${rows} row(s)${range}, counts and checksums equal in the old and new tables`;
+  const range = cutover !== undefined ? ` (${bound} ${c?.how === "keys" ? "key(s)" : "of them"} before the cut-over at ${new Date(cutover).toISOString()})` : "";
+  const counts = c?.how === "keys" ? `${rows} row(s) of the old table as ${counted} key(s)` : `${rows} row(s)`;
+  const merged =
+    c?.how === "final"
+      ? `, read under FINAL (${c.engine})`
+      : c?.how === "keys"
+        ? `, grouped by the sorting key of the new ${c.engine} and compared on ${c.describe}`
+        : "";
+  const summary = `${o.names.key}: ${all.length} partition(s), ${counts}${range}, counts and checksums equal in the old and new tables${merged}`;
   run.log(`-- ${summary}`);
   return {
     state: o.state,

package/src/clickhouse/testing/fake-server.ts

@@ -24,6 +24,8 @@ export interface FakeOptions {
   /** Answer every query with this status and body (an auth failure, say). */
   fail?: { status: number; body: string };
   version?: string;
+  /** The SQL user-defined functions `system.functions` lists. Default: none. */
+  functions?: Array<{ name: string; statement: string }>;
 }
 
 const row = (o: Record<string, unknown>) => `${JSON.stringify(o)}\n`;
@@ -52,8 +54,7 @@ export async function fakeClickHouse(objects: FakeObject[], options: FakeOptions
           .join(""),
       );
     }
-    // No SQL user-defined functions.
-    if (/FROM system\.functions/.test(body)) return ok("");
+    if (/FROM system\.functions/.test(body)) return ok((options.functions ?? []).map((f) => row(f)).join(""));
     const show = /^SHOW CREATE (DATABASE|TABLE) (`(?:[^`]|``)*`)(?:\.(`(?:[^`]|``)*`))?$/.exec(body.trim());
     if (show) {
       const [db, name] = show[3] ? [unquote(show[2]!), unquote(show[3])] : [undefined, unquote(show[2]!)];

package/src/codegen/classifier-page.ts

@@ -70,7 +70,7 @@ ALTER TABLE \`analytics\`.\`events\` MODIFY COLUMN \`kind\` LowCardinality(Strin
 --   export const { op } = ClickHouseRebuildOp({ name: "rebuild-analytics-sessions", env: "<env>", table: "analytics.sessions", dualWrite: { mode: "materialized-view", cutoverColumn: "ts" } });
 \`\`\`
 
-A rebuild is never DDL: it is a step naming \`ClickHouseRebuildOp\`, or for a view or a database, which the Op does not rebuild, a manual step. A column or object drop is a statement marked destructive. Each \`CREATE\` carries the project's ownership marker, as the applier stamps it. The command exits 2 when a step is not a statement. Two things the applier asks the server are left out: whether a difference is formatting only, and whether a database is empty before it is dropped.
+A rebuild is never DDL: it is a step naming \`ClickHouseRebuildOp\`, or for a view or a database, which the Op does not rebuild, a manual step. A column or object drop is a statement marked destructive. Each \`CREATE\` carries the project's ownership marker, as the applier stamps it. The command exits 2 when a step is not a statement. A migration runner's history table (\`schema_migrations\`, \`goose_db_version\`; see [Importing a Live Server](../importing/)) is never dropped: the report names it in a hint. Two things the applier asks the server are left out: whether a difference is formatting only, and whether a database is empty before it is dropped.
 
 \`--topology\` renders the statements for where they will run: \`single\` (the default), \`cluster:<name>\`, \`replicated\` or \`cloud\`. On a cluster every statement carries \`ON CLUSTER\` and a MergeTree-family table becomes \`Replicated*MergeTree\` with its Keeper path; in a \`Replicated\` database the tables are \`Replicated*MergeTree\` with no path and no \`ON CLUSTER\`. \`diffStatements(before, after, { topology })\` takes the same choice; see [Applying to a Server](../applying/#topology).
 

package/src/config.ts

@@ -104,6 +104,14 @@ export const sqlProfileSchema = z.strictObject({
   databases: z.array(z.string()).optional(),
   /** The database an unqualified declaration is created in. `default` when omitted. */
   defaultDatabase: z.string().optional(),
+  /**
+   * ClickHouse import (#3718): the SQL user-defined functions `chant import
+   * --from <env>` adopts besides those the imported objects call, by name, or
+   * by prefix with a trailing `*` (`shop_*`). A function belongs to no
+   * database, so without this only the functions the imported tables, views
+   * and dictionaries call (and the functions those call) are imported.
+   */
+  importFunctions: z.array(z.string()).optional(),
   /**
    * ClickHouse: the topology this environment runs, which the applier, the
    * plan and the rebuild migration render every statement for: `single`,
@@ -140,8 +148,11 @@ export const sqlProfileSchema = z.strictObject({
   /**
    * Postgres apply: how long one statement waits for its lock before it
    * fails (`lock_timeout`), in milliseconds, so a change blocked behind a
-   * long query fails fast instead of queueing every query behind it. 5000
-   * when omitted.
+   * long query fails fast instead of queueing every query behind it. The
+   * catalog reads (`chant lifecycle diff --live`, `chant sql plan`, the
+   * import, and an apply's read before its first statement) wait under it
+   * too, and name the session holding the lock when it passes. 5000 when
+   * omitted.
    */
   lockTimeoutMs: z.number().int().nonnegative().optional(),
   /**
@@ -156,11 +167,12 @@ export const sqlProfileSchema = z.strictObject({
    */
   scanTimeoutMs: z.number().int().nonnegative().optional(),
   /**
-   * Postgres: whether chant manages access in this environment (#3681): the
+   * Whether chant manages access in this environment. Postgres (#3681): the
    * declared row-level security policies and each table's row-level security,
    * the declared roles (created when missing, never dropped), and the
    * privileges on the declared objects and the default privileges, which a
-   * plan compares and an apply grants and revokes. Off when omitted: the
+   * plan compares and an apply grants and revokes. ClickHouse (#3716): the
+   * declared users, roles, row policies and grants. Off when omitted: the
    * access declarations are not planned or applied, and the server's access
    * is not read, so a project that manages access elsewhere is not affected.
    */

package/src/core/foreign-tables.ts

@@ -0,0 +1,28 @@
+/**
+ * Tables another tool keeps its migration history in, by name, with the tool.
+ *
+ * Both dialects' catalog readers mark a table by one of these names as
+ * foreign: import leaves it out with a warning, and a plan names it in a hint
+ * instead of proposing a drop, since chant changing it would change what that
+ * tool owns (#3047 question 10, #3676). golang-migrate, dbmate, goose and
+ * Flyway keep the same tables on ClickHouse as on Postgres.
+ */
+export const FOREIGN_TABLES: Readonly<Record<string, string>> = {
+  _prisma_migrations: "Prisma Migrate",
+  schema_migrations: "a migration runner (Rails, golang-migrate, dbmate)",
+  ar_internal_metadata: "Rails",
+  django_migrations: "Django",
+  alembic_version: "Alembic",
+  __drizzle_migrations: "drizzle-kit",
+  flyway_schema_history: "Flyway",
+  goose_db_version: "goose",
+  knex_migrations: "Knex",
+  knex_migrations_lock: "Knex",
+  SequelizeMeta: "Sequelize",
+  pgmigrations: "node-pg-migrate",
+};
+
+/** The tool that keeps a table by this name, or undefined when none does. */
+export function foreignTool(name: string): string | undefined {
+  return Object.prototype.hasOwnProperty.call(FOREIGN_TABLES, name) ? FOREIGN_TABLES[name] : undefined;
+}

package/src/files/clickhouse.ts

@@ -1,23 +1,79 @@
 /**
  * A file of ClickHouse DDL as the objects it declares: one per `CREATE
  * DATABASE`, `TABLE`, `VIEW`, `MATERIALIZED VIEW`, `DICTIONARY` or `FUNCTION`, as `chant import
- * <file>.sql` reads them. Any other statement, and any statement the parser
- * refuses, is a problem naming it.
+ * <file>.sql` reads them, and one per `CREATE USER`, `CREATE ROLE`, `CREATE
+ * ROW POLICY` and `GRANT` (#3711), each read by its tag, so what the tag
+ * refuses (`IDENTIFIED BY`, a `REVOKE`) is refused with the tag's message.
+ * Any other statement, and any statement the parser refuses, is a problem
+ * naming it.
  *
  * With `schema` (a database), an object's unqualified name is qualified with
- * it; a function belongs to no database and keeps its name.
+ * it, and so is the table a row policy or a grant is on; a function, a user
+ * and a role belong to no database and keep their names.
  */
 
 import type { Declarable } from "@intentius/chant/declarable";
 import { isTrivia, tokenizeText } from "../clickhouse/tokens";
 import { parseCreate } from "../clickhouse/parser";
-import { database, dictionary, func, table, view } from "../clickhouse/entities";
+import { CLICKHOUSE_ENTITY_TYPES, database, dictionary, func, grant, policy, role, table, user, view } from "../clickhouse/entities";
+import { isAccessStatement, parseAccess, type AccessNode } from "../clickhouse/access";
+import type { Token } from "../clickhouse/tokens";
 import { CLICKHOUSE_TAG_OF } from "../clickhouse/import/generator";
 import { describeStatement, type ImportedObject } from "../clickhouse/import/ir";
 import { firstLine, quoteBare, splitTokens, tokensText, type ReadOptions } from "./common";
+import { templateSafe, templateStrings } from "./postgres";
 
 type Tag = (strings: TemplateStringsArray, ...values: unknown[]) => Declarable;
-const TAGS: Record<string, Tag> = { database, table, view, dictionary, func };
+const TAGS: Record<string, Tag> = { database, table, view, dictionary, func, user, role, policy, grant };
+
+/** Tokens with `[db.]name`'s bare name qualified with `schema`. A `*` (all tables) is left alone. */
+function qualifyBare(tokens: Token[], span: { from: number; to: number } | undefined, schema: string): string | undefined {
+  if (!span) return undefined;
+  const name = tokens.slice(span.from, span.to).filter((t) => !isTrivia(t));
+  const [only] = name;
+  if (!only || name.length !== 1 || (only.kind !== "ident" && only.kind !== "qident")) return undefined;
+  const at = tokens.indexOf(only);
+  return tokensText(tokens.slice(0, at)) + `${quoteBare(schema)}.` + tokensText(tokens.slice(at));
+}
+
+/** An access statement as the object its tag declares (#3711), or the tag's refusal. */
+function readAccess(sql: string, options: ReadOptions): ImportedObject | string {
+  let tokens = tokenizeText(sql, 0);
+  let node: AccessNode;
+  try {
+    node = parseAccess(tokens);
+  } catch (e) {
+    // What the tag refuses (a password) is refused in the tag's words.
+    const words = sql.replace(/--[^\n]*\n/g, " ").trim().split(/\s+/, 2).map((w) => w.toUpperCase());
+    const tag = words[0] === "CREATE" ? (words[1] === "USER" ? user : words[1] === "ROLE" ? role : policy) : grant;
+    try {
+      tag(templateStrings([templateSafe(sql)]));
+    } catch (refused) {
+      return refused instanceof Error ? refused.message : String(refused);
+    }
+    throw e;
+  }
+  if (options.schema) {
+    const qualified = qualifyBare(tokens, node.statement === "rowPolicy" ? node.table : node.statement === "grant" ? node.target : undefined, options.schema);
+    if (qualified !== undefined) {
+      sql = qualified;
+      tokens = tokenizeText(sql, 0);
+      node = parseAccess(tokens);
+    }
+  }
+  const tag = node.statement === "user" ? user : node.statement === "role" ? role : node.statement === "rowPolicy" ? policy : grant;
+  let props: { name: string; database?: string; table?: string };
+  try {
+    props = (tag(templateStrings([templateSafe(sql)])) as unknown as { props: typeof props }).props;
+  } catch (e) {
+    return e instanceof Error ? e.message : String(e);
+  }
+  if (node.statement === "user") return { type: CLICKHOUSE_ENTITY_TYPES.user, name: props.name, ddl: sql };
+  if (node.statement === "role") return { type: CLICKHOUSE_ENTITY_TYPES.role, name: props.name, ddl: sql };
+  if (node.statement === "rowPolicy") return { type: CLICKHOUSE_ENTITY_TYPES.rowPolicy, ...(props.database ? { database: props.database } : {}), name: props.name, ddl: sql };
+  // A grant has no name of its own: what it grants, on what, to whom.
+  return { type: CLICKHOUSE_ENTITY_TYPES.grant, name: `grant ${tokensText(tokens.slice(tokens.findIndex((t) => !isTrivia(t)) + 1)).replace(/\s+/g, " ").trim().toLowerCase()}`, ddl: sql };
+}
 
 /** The tag that declares an entity type. */
 export function clickhouseTag(type: string): Tag | undefined {
@@ -30,6 +86,16 @@ export function readClickHouse(ddl: string, options: ReadOptions, problems: stri
   const objects: ImportedObject[] = [];
   for (const stmt of splitTokens(tokenizeText(ddl, 0), isTrivia)) {
     let sql = tokensText(stmt);
+    if (isAccessStatement(sql)) {
+      try {
+        const read = readAccess(sql, options);
+        if (typeof read === "string") problems.push(`${read}: ${firstLine(sql)}`);
+        else objects.push(read);
+      } catch (e) {
+        problems.push(`does not parse (${e instanceof Error ? e.message : String(e)}): ${firstLine(sql)}`);
+      }
+      continue;
+    }
     if (!/^CREATE\b/i.test(sql)) {
       problems.push(`not a CREATE statement: ${firstLine(sql)}`);
       continue;

package/src/files/postgres.ts

@@ -45,7 +45,8 @@ const isName = (t: Token | undefined): boolean => t !== undefined && (t.kind ===
 const word = (t: Token | undefined, ...words: string[]): boolean => t !== undefined && t.kind === "ident" && words.includes(t.text.toUpperCase());
 
 /** Text that sits inside a template literal: a backquote and `${` escaped. */
-const templateSafe = (s: string) => s.replace(/\\(?=`|\$\{)/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
+/** Text that sits inside a template literal: a backquote and `${` escaped. */
+export const templateSafe = (s: string) => s.replace(/\\(?=`|\$\{)/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
 
 interface PgFileObject {
   node: StatementNode;

package/src/import-parser.ts

@@ -25,6 +25,8 @@ const CLICKHOUSE_ONLY = [
   /^\s*CREATE\s+(?:DATABASE|(?:OR\s+REPLACE\s+)?DICTIONARY)\b/im,
   // A SQL function: CREATE FUNCTION f AS (x) -> ...
   /^\s*CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+[^\s(]+(?:\s+ON\s+CLUSTER\s+\S+)?\s+AS\s*\(/im,
+  // A row policy (#3711): Postgres says CREATE POLICY.
+  /^\s*CREATE\s+(?:OR\s+REPLACE\s+)?ROW\s+POLICY\b/im,
 ];
 const POSTGRES_ONLY = [
   /^\s*CREATE\s+(?:OR\s+REPLACE\s+)?(?:SCHEMA|EXTENSION|TYPE|DOMAIN|SEQUENCE|(?:UNIQUE\s+)?INDEX|(?:CONSTRAINT\s+)?TRIGGER|POLICY|ROLE|FUNCTION|PROCEDURE|(?:UNLOGGED|TEMP|TEMPORARY)\s+TABLE)\b/im,

package/src/migration-statements.test.ts

@@ -111,6 +111,39 @@ describe("ClickHouse", () => {
     expect((d.steps[0] as { detail: string }).detail).toMatch(/drop and create this object instead/);
   });
 
+  test("a user the environment creates gets its declared clauses with ALTER USER, its password left alone (#3717)", () => {
+    const reader: Obj = { export: "reader", type: "ClickHouse::Role", ddl: "CREATE ROLE reader" };
+    const app: Obj = {
+      export: "app",
+      type: "ClickHouse::User",
+      dependsOn: ["reader", "analytics"],
+      ddl: "CREATE USER app HOST LOCAL DEFAULT ROLE reader DEFAULT DATABASE analytics SETTINGS max_memory_usage = 1000000000",
+    };
+    const readerToApp: Obj = { export: "readerToApp", type: "ClickHouse::Grant", dependsOn: ["reader", "app"], ddl: "GRANT reader TO app" };
+    const d = diffStatements(build("clickhouse", [DB]), build("clickhouse", [DB, reader, app, readerToApp]));
+    const sql = sqlOf(d);
+    expect(sql).toEqual([
+      "CREATE ROLE reader",
+      "ALTER USER `app` HOST LOCAL",
+      "ALTER USER `app` DEFAULT DATABASE analytics",
+      "ALTER USER `app` SETTINGS max_memory_usage = 1000000000",
+      "GRANT `reader` TO `app`",
+      // A default role must be held first.
+      "ALTER USER `app` DEFAULT ROLE reader",
+    ]);
+    expect(sql.join("\n")).not.toMatch(/CREATE USER|IDENTIFIED/);
+    expect(d.steps.filter((s) => s.object === "app").map((s) => (s as StatementStep).rule)).toEqual(["SQLCH271", "SQLCH271", "SQLCH271"]);
+
+    // Without grants to it, the default role is set with the rest.
+    const alone = sqlOf(diffStatements(build("clickhouse", [DB, reader]), build("clickhouse", [DB, reader, app])));
+    expect(alone).toEqual([
+      "ALTER USER `app` HOST LOCAL",
+      "ALTER USER `app` DEFAULT DATABASE analytics",
+      "ALTER USER `app` SETTINGS max_memory_usage = 1000000000",
+      "ALTER USER `app` DEFAULT ROLE reader",
+    ]);
+  });
+
   test("two identical builds take no steps", () => {
     expect(diffStatements(CH_BEFORE, CH_BEFORE)).toEqual({ dialect: "clickhouse", defaultDatabase: "default", topology: "single", steps: [], refused: false, hints: [] });
   });