@intentius/chant-lexicon-sql 0.118.0 → 0.120.0

package/src/clickhouse/rebuild/rebuild-into-collapsing.e2e.test.ts

@@ -0,0 +1,198 @@
+/**
+ * A plain MergeTree rebuilt into a SummingMergeTree (#3727).
+ *
+ * The old table keeps every row it was given; the new one keeps one row per
+ * sorting key, its sums, and drops a key whose sums are all zero. So the row
+ * counts differ by design. The Verify phase groups both tables by the new
+ * sorting key and compares each key's sums, finds them equal, and the run
+ * swaps once approved. A key the new table lost, or a sum it changed, still
+ * fails the verification.
+ *
+ * Runs against `CHANT_SQL_E2E_CLICKHOUSE_URL` when it is set (a shared
+ * server: everything is in the databases `chant_e2e_3727` and
+ * `chant_e2e_3727_loss`, dropped afterwards, with this test's receipts),
+ * otherwise a scratch server, which needs Docker and skips cleanly without it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { memoryGateLedgerPort, runOpLocally, loadProfiles, type ActivityFn, type GateLedgerPort, type OpConfig } from "@intentius/chant/op";
+import { dockerAvailable, startScratchServer, type ScratchServer } from "../container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { clickhouseApply } from "../../op/activities/clickhouse-apply";
+import * as rebuildActivities from "../../op/activities/clickhouse-rebuild";
+import { ClickHouseRebuildOp } from "./op";
+import { RECEIPTS_DATABASE, RECEIPTS_TABLE } from "./receipts";
+
+const given = process.env.CHANT_SQL_E2E_CLICKHOUSE_URL;
+const enabled = given ? true : await dockerAvailable();
+const DATABASES = ["chant_e2e_3727", "chant_e2e_3727_loss"];
+const MARKER = { stack: "chant_e2e_3727", env: "test" };
+let server: ScratchServer | undefined;
+let endpoint: ClickHouseEndpoint;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-rebuild-into-collapsing-"));
+
+const q = <T = Record<string, unknown>>(sql: string) => clickhouseQuery<T>(endpoint, sql);
+const count = async (from: string) => Number((await q<{ n: string }>(`SELECT count() AS n FROM ${from}`))[0]!.n);
+const mutate = (sql: string) => clickhouseQuery(endpoint, sql, { settings: { mutations_sync: "2" } });
+
+async function cleanup(): Promise<void> {
+  for (const db of DATABASES) await q(`DROP DATABASE IF EXISTS ${db} SYNC`);
+  const receipts = await q<{ n: string }>(`SELECT count() AS n FROM system.tables WHERE database = '${RECEIPTS_DATABASE}' AND name = '${RECEIPTS_TABLE}'`);
+  if (Number(receipts[0]!.n) > 0) await mutate(`ALTER TABLE ${RECEIPTS_DATABASE}.${RECEIPTS_TABLE} DELETE WHERE position(address, '${MARKER.stack}') > 0`);
+}
+
+beforeAll(async () => {
+  if (!enabled) return;
+  if (given) endpoint = { url: given, user: "default", password: process.env.CHANT_SQL_E2E_CLICKHOUSE_PASSWORD ?? "" };
+  else {
+    server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-rebuild-into-collapsing" });
+    endpoint = server.endpoint;
+  }
+  await cleanup();
+}, 600_000);
+
+afterAll(async () => {
+  if (enabled) await cleanup().catch(() => undefined);
+  await server?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+const deps = () => ({ config: { ownership: MARKER }, env: { CLICKHOUSE_URL: endpoint.url, CLICKHOUSE_USER: endpoint.user, CLICKHOUSE_PASSWORD: endpoint.password }, log: () => undefined });
+
+function writeBuild(db: string, file: string, engine: string): string {
+  const objects = [
+    { export: "db", type: "ClickHouse::Database", ddl: `CREATE DATABASE ${db} ENGINE = Atomic`, dependsOn: [] },
+    {
+      export: "daily",
+      type: "ClickHouse::Table",
+      dependsOn: ["db"],
+      ddl: `CREATE TABLE ${db}.daily (day Date, shop UInt32, total Decimal(18, 2), n UInt64) ENGINE = ${engine} PARTITION BY toYYYYMM(day) ORDER BY (day, shop)`,
+    },
+  ];
+  writeFileSync(join(dir, file), JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects }));
+  return join(dir, file);
+}
+
+/** Eleven rows: shop 1 three times and shop 2 twice on one day, shop 3 whose rows sum to zero, and six single rows across two months. */
+async function seed(db: string): Promise<void> {
+  await q(
+    `INSERT INTO ${db}.daily VALUES ` +
+      `('2026-09-01', 1, 10, 1), ('2026-09-01', 1, 20, 1), ('2026-09-01', 1, 30, 1), ('2026-09-01', 2, 5, 1), ('2026-09-01', 2, 7, 2), ` +
+      `('2026-09-01', 3, 4, 0), ('2026-09-01', 3, -4, 0), ` +
+      `('2026-09-02', 1, 1, 1), ('2026-09-03', 1, 2, 1), ('2026-10-01', 1, 3, 1), ('2026-10-02', 2, 4, 1)`,
+  );
+}
+
+const activities = (): Map<string, ActivityFn> => {
+  const out = new Map<string, ActivityFn>();
+  for (const [name, fn] of Object.entries(rebuildActivities)) {
+    if (typeof fn !== "function" || !name.startsWith("clickhouseRebuild")) continue;
+    out.set(name, ((args: Record<string, unknown>, signal?: AbortSignal) => (fn as (a: unknown, s?: AbortSignal, d?: unknown) => Promise<unknown>)(args, signal, deps())) as ActivityFn);
+  }
+  return out;
+};
+
+/** An in-memory gate ledger: each approval answers the gate the last run stopped at, for the plan it recorded. */
+class Ledger {
+  private pending: unknown[] = [];
+  private resolutions: unknown[] = [];
+  port: GateLedgerPort = memoryGateLedgerPort();
+
+  approveLast(): void {
+    const last = (this.port as ReturnType<typeof memoryGateLedgerPort>).appended.at(-1)!;
+    this.pending.push(...(this.port as ReturnType<typeof memoryGateLedgerPort>).appended);
+    this.resolutions.push({
+      version: 1,
+      kind: "resolution",
+      op: last.op,
+      gate: last.gate,
+      resolvedBy: "e2e",
+      timestamp: new Date(Date.parse(last.timestamp) + 1000).toISOString(),
+      ...(last.planDigest ? { planDigest: last.planDigest } : {}),
+    });
+    this.port = memoryGateLedgerPort({ pending: this.pending as never, resolutions: this.resolutions as never });
+  }
+}
+
+describe.skipIf(!enabled)("a MergeTree rebuilt into a SummingMergeTree", () => {
+  const DB = "chant_e2e_3727";
+  test("the Verify phase compares each sorting key's sums, and the run swaps once approved", async () => {
+    await clickhouseApply({ buildPath: writeBuild(DB, "daily-v1.json", "MergeTree"), environment: "e2e" }, undefined, deps());
+    await seed(DB);
+    writeBuild(DB, "daily-v2.json", "SummingMergeTree");
+
+    const { op } = ClickHouseRebuildOp({
+      name: "rebuild-daily-3727",
+      env: "e2e",
+      table: `${DB}.daily`,
+      dualWrite: { mode: "materialized-view", cutoverColumn: "day", cutoverDelay: "1s" },
+      build: false,
+      path: dir,
+      output: "daily-v2.json",
+      stack: MARKER.stack,
+      ownershipEnv: MARKER.env,
+    });
+    const props = (op as unknown as { props: OpConfig }).props;
+    const ledger = new Ledger();
+    const run = async () => runOpLocally(props, activities(), await loadProfiles(), undefined, { gates: ledger.port, now: new Date().toISOString() });
+
+    const first = await run();
+    expect(first.records.find((x) => x.fn === "clickhouseRebuildVerify")?.error).toBeUndefined();
+    expect(first.status).toBe("gated");
+    expect(first.gate).toMatchObject({ gate: "approve-rebuild-daily-3727" });
+    const verification = first.records.flatMap((x) => x.outcomes ?? []).find((o) => o.name === "Verification")?.value;
+    expect(String(verification)).toMatch(/2 partition\(s\), 11 row\(s\) of the old table as 6 key\(s\).*grouped by the sorting key of the new SummingMergeTree and compared on the sums of total, n/);
+
+    ledger.approveLast();
+    const swapped = await run();
+    expect(swapped.records.find((x) => x.fn === "clickhouseRebuildSwap")?.status).toBe("ok");
+    expect(swapped.gate).toMatchObject({ gate: "approve-rebuild-daily-3727-drop" });
+    expect(await q<{ engine: string }>(`SELECT engine FROM system.tables WHERE database = '${DB}' AND name = 'daily'`)).toEqual([{ engine: "SummingMergeTree" }]);
+    expect(await q(`SELECT toString(day) AS day, shop, toString(total) AS total, toString(n) AS n FROM ${DB}.daily FINAL WHERE day = '2026-09-01' ORDER BY shop`)).toEqual([
+      { day: "2026-09-01", shop: 1, total: "60", n: "3" },
+      { day: "2026-09-01", shop: 2, total: "12", n: "3" },
+    ]);
+  }, 300_000);
+});
+
+describe.skipIf(!enabled)("a MergeTree rebuilt into a SummingMergeTree that lost rows", () => {
+  const DB = "chant_e2e_3727_loss";
+  const args = {
+    table: `${DB}.daily`,
+    buildPath: "loss-v2.json",
+    environment: "e2e",
+    dualWrite: { mode: "materialized-view" as const, cutoverColumn: "day", cutoverDelay: "1s" },
+    stack: MARKER.stack,
+    ownershipEnv: MARKER.env,
+    cwd: dir,
+  };
+
+  test("a changed sum, or a key missing from the new table, fails the verification and nothing is swapped", async () => {
+    await clickhouseApply({ buildPath: writeBuild(DB, "loss-v1.json", "MergeTree"), environment: "e2e" }, undefined, deps());
+    await seed(DB);
+    writeBuild(DB, "loss-v2.json", "SummingMergeTree");
+    await rebuildActivities.clickhouseRebuildCreate(args, undefined, deps());
+    await rebuildActivities.clickhouseRebuildDualWrite(args, undefined, deps());
+    await rebuildActivities.clickhouseRebuildBackfill(args, undefined, deps());
+    expect(await rebuildActivities.clickhouseRebuildVerify(args, undefined, deps())).toMatchObject({ rows: 11, partitions: 2 });
+
+    // A sum the copy got wrong.
+    await mutate(`ALTER TABLE ${DB}.daily__chant_new UPDATE total = total + 1 WHERE shop = 2 AND day = '2026-09-01'`);
+    await expect(rebuildActivities.clickhouseRebuildVerify(args, undefined, deps())).rejects.toThrow(
+      /does not match the old one in 1 partition\(s\): \(?202609\)?: old 4 keys.*new 4 keys.*compared per key on the sums of total, n/,
+    );
+    await mutate(`ALTER TABLE ${DB}.daily__chant_new UPDATE total = total - 1 WHERE shop = 2 AND day = '2026-09-01'`);
+    expect(await rebuildActivities.clickhouseRebuildVerify(args, undefined, deps())).toMatchObject({ rows: 11 });
+
+    // A key the copy lost: the swap compares again before the EXCHANGE and stops.
+    await mutate(`ALTER TABLE ${DB}.daily__chant_new DELETE WHERE shop = 1 AND day = '2026-10-01'`);
+    await expect(rebuildActivities.clickhouseRebuildSwap(args, undefined, deps())).rejects.toThrow(/does not match the old one in 1 partition\(s\): \(?202610\)?: old 2 keys.*new 1 keys/);
+    expect(await q<{ engine: string }>(`SELECT engine FROM system.tables WHERE database = '${DB}' AND name = 'daily'`)).toEqual([{ engine: "MergeTree" }]);
+    expect(await count(`${DB}.daily`)).toBe(11);
+    await rebuildActivities.clickhouseRebuildCompensate(args, undefined, deps());
+  }, 300_000);
+});

package/src/clickhouse/rebuild/rebuild.test.ts

@@ -24,6 +24,8 @@ import { partitionEffect } from "./backfill";
 import { insertTarget, waitForCutover, type CutoverProbe } from "./steps";
 import { intoShard, onShard, onShardServers, shardTable } from "./shards";
 import { renameColumns, sourcePartitionExpression } from "./partitions";
+import { collapseSemantics, collapsesRows, engineArguments, splitTopLevel, type EngineColumn } from "./engines";
+import { RebuildVerificationError } from "./verify";
 import { canonicalObject } from "../plan/normalize";
 import { diffSchemas } from "../plan/diff";
 import { renderDiff } from "../plan/report";
@@ -226,6 +228,42 @@ describe("the plan hands a rebuild to the Op", () => {
   });
 });
 
+describe("a table whose engine collapses rows by sorting key (#3674)", () => {
+  test("the collapsing MergeTree engines are read under FINAL, replicated or shared too; the others are not", () => {
+    for (const e of ["SummingMergeTree", "ReplacingMergeTree", "AggregatingMergeTree", "CollapsingMergeTree", "VersionedCollapsingMergeTree", "ReplicatedSummingMergeTree", "SharedReplacingMergeTree"]) {
+      expect(collapsesRows(e), e).toBe(true);
+    }
+    for (const e of ["MergeTree", "ReplicatedMergeTree", "Log", "Distributed", undefined]) expect(collapsesRows(e), String(e)).toBe(false);
+  });
+
+  test("a difference on such a table names the engine and the OPTIMIZE that merges it, not only a late write", () => {
+    const mismatch = [{ partition: "all", old: { partition: "all", rows: 3, checksum: "1" }, new: { partition: "all", rows: 1, checksum: "2" } }];
+    const collapsing = new RebuildVerificationError("db.daily", mismatch, "SummingMergeTree").message;
+    expect(collapsing).toMatch(/Nothing was swapped\. db\.daily is a SummingMergeTree, .*compared under FINAL/);
+    expect(collapsing).toContain("run OPTIMIZE TABLE db.daily FINAL");
+    const plain = new RebuildVerificationError("db.events", mismatch).message;
+    expect(plain).not.toMatch(/FINAL/);
+    expect(plain).toMatch(/The usual cause is in materialized-view mode a row written to the old table later than the cut-over delay/);
+  });
+
+  test("from a plain MergeTree, a difference names what was compared per key", () => {
+    const mismatch = [{ partition: "202610", old: { partition: "202610", rows: 2, checksum: "1" }, new: { partition: "202610", rows: 1, checksum: "2" } }];
+    const message = new RebuildVerificationError("db.daily", mismatch, { engine: "SummingMergeTree", how: "keys", describe: "the sums of total" }).message;
+    expect(message).toContain("202610: old 2 keys, checksum 1, new 1 keys, checksum 2");
+    expect(message).toMatch(/The new table is a SummingMergeTree, .*compared per key on the sums of total/);
+    expect(message).not.toMatch(/OPTIMIZE/);
+  });
+
+  test("the plan's hand-off says so for such a table", () => {
+    const ddl = (o: string) => `CREATE TABLE db.daily (day Date, total Decimal(18, 2), n UInt64) ENGINE = SummingMergeTree ORDER BY ${o}`;
+    const after = [{ key: "daily", canonical: canonicalObject(ddl("(day, n)")) }];
+    const diff = diffSchemas([{ key: "daily", canonical: canonicalObject(ddl("day")) }], after);
+    const ops = rebuildOpSuggestions(diff, new Map(after.map((o) => [o.key, o.canonical])), "prod");
+    expect(ops[0]!.note).toMatch(/^db\.daily is a SummingMergeTree: when the old table's engine collapses rows too, the Verify phase reads both tables under FINAL/);
+    expect(renderDiff({ ...diff, rebuildOps: ops })).toContain(`  Note: ${ops[0]!.note}`);
+  });
+});
+
 describe("a rebuild across the shards of a cluster (#3663)", () => {
   const sharding = { cluster: "main", shards: [{ num: 1, macro: "s1", slot: 0 }, { num: 2, macro: "s2", slot: 1 }] };
   const shard2 = sharding.shards[1]!;
@@ -294,3 +332,43 @@ describe("the wait for the cut-over (#3700)", () => {
     expect(insertTarget("SELECT * FROM shop.events", "shop")).toBeUndefined();
   });
 });
+
+describe("what a collapsing engine keeps per sorting key (#3727)", () => {
+  const col = (name: string, type: string, inSortingKey = false): EngineColumn => ({ name, type, inSortingKey });
+  const columns = [col("day", "Date", true), col("shop", "UInt32", true), col("total", "Decimal(18, 2)"), col("n", "UInt64"), col("ratio", "Float64"), col("note", "String"), col("sign", "Int8"), col("ver", "UInt32")];
+
+  test("splitTopLevel and engineArguments read engine_full, without a Replicated engine's Keeper path", () => {
+    expect(splitTopLevel("toStartOfInterval(ts, toIntervalDay(1)), id")).toEqual(["toStartOfInterval(ts, toIntervalDay(1))", "id"]);
+    expect(splitTopLevel("")).toEqual([]);
+    expect(engineArguments("SummingMergeTree((total, n)) PARTITION BY toYYYYMM(day) ORDER BY (day, shop) SETTINGS index_granularity = 8192")).toEqual(["(total, n)"]);
+    expect(engineArguments("ReplicatedCollapsingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}', sign) ORDER BY id")).toEqual(["sign"]);
+    expect(engineArguments("SummingMergeTree ORDER BY id")).toEqual([]);
+  });
+
+  test("SummingMergeTree: the sums of every numeric column outside the key, or of those listed, in their own types; a key whose sums are zero is removed", () => {
+    const all = collapseSemantics("SummingMergeTree", "SummingMergeTree ORDER BY (day, shop)", columns)!;
+    expect(all.aggregates).toEqual(["CAST(sum(`total`), 'Decimal(18, 2)')", "CAST(sum(`n`), 'UInt64')", "toFloat32(sum(`ratio`))", "CAST(sum(`sign`), 'Int8')", "CAST(sum(`ver`), 'UInt32')"]);
+    expect(all.having).toBe(`NOT (${all.aggregates.map((a) => `${a} = 0`).join(" AND ")})`);
+    const listed = collapseSemantics("ReplicatedSummingMergeTree", "ReplicatedSummingMergeTree('/p', '{replica}', (total, n)) ORDER BY (day, shop)", columns)!;
+    expect(listed).toMatchObject({ engine: "SummingMergeTree", aggregates: ["CAST(sum(`total`), 'Decimal(18, 2)')", "CAST(sum(`n`), 'UInt64')"], describe: "the sums of total, n" });
+    const partitioned = collapseSemantics("SummingMergeTree", "SummingMergeTree PARTITION BY g ORDER BY k", [col("k", "UInt32", true), { ...col("g", "UInt8"), inPartitionKey: true }, col("a", "UInt32")])!;
+    expect(partitioned.aggregates).toEqual(["CAST(sum(`a`), 'UInt32')"]);
+  });
+
+  test("Replacing keeps the highest version, Collapsing the sum of its signs, Aggregating its order-free simple functions, Coalescing which columns hold a value", () => {
+    expect(collapseSemantics("ReplacingMergeTree", "ReplacingMergeTree(ver) ORDER BY id", columns)).toMatchObject({ aggregates: ["max(`ver`)"], describe: "the highest ver" });
+    expect(collapseSemantics("ReplacingMergeTree", "ReplacingMergeTree ORDER BY id", columns)).toMatchObject({ aggregates: [], describe: "the keys" });
+    expect(collapseSemantics("CollapsingMergeTree", "CollapsingMergeTree(sign) ORDER BY id", columns)).toMatchObject({ aggregates: ["sum(`sign`)"], having: "sum(`sign`) != 0" });
+    expect(collapseSemantics("VersionedCollapsingMergeTree", "VersionedCollapsingMergeTree(sign, ver) ORDER BY id", columns)).toMatchObject({ aggregates: ["sum(`sign`)"], having: "sum(`sign`) != 0" });
+    const agg = [col("k", "UInt32", true), col("hi", "SimpleAggregateFunction(max, UInt64)"), col("any", "SimpleAggregateFunction(anyLast, String)"), col("u", "AggregateFunction(uniq, UInt64)")];
+    expect(collapseSemantics("AggregatingMergeTree", "AggregatingMergeTree ORDER BY k", agg)).toMatchObject({ aggregates: ["max(`hi`)"], describe: "max(hi)" });
+    const coalescing = [col("k", "UInt32", true), col("v", "Nullable(String)"), col("w", "UInt8")];
+    expect(collapseSemantics("CoalescingMergeTree", "CoalescingMergeTree ORDER BY k", coalescing)).toMatchObject({ aggregates: ["count(`v`) > 0"] });
+  });
+
+  test("an engine that keeps every row, or whose rows depend on the order of merges, has none", () => {
+    expect(collapseSemantics("MergeTree", "MergeTree ORDER BY id", columns)).toBeUndefined();
+    expect(collapseSemantics("GraphiteMergeTree", "GraphiteMergeTree('graphite_rollup') ORDER BY id", columns)).toBeUndefined();
+    expect(collapseSemantics("CollapsingMergeTree", "CollapsingMergeTree(missing) ORDER BY id", columns)).toBeUndefined();
+  });
+});

package/src/clickhouse/rebuild/verify.ts

@@ -24,6 +24,14 @@
  * later than the cut-over delay, with a time before the cut-over) stop the
  * swap too, rather than stay behind in the old table.
  *
+ * A table whose engine collapses rows that share a sorting key when parts
+ * merge (Summing, Replacing, Aggregating, Collapsing, VersionedCollapsing,
+ * Coalescing and Graphite, replicated or not), rebuilt into such an engine,
+ * is read under `FINAL` on both sides (#3674). The old table can hold unmerged parts with several rows for
+ * one key, which the copy inserts in one block and the new table collapses
+ * as it writes them, so the raw counts differ although the data is the same.
+ * Under `FINAL` both sides are compared as merged.
+ *
  * On a cluster of more than one shard (#3663) every shard is read, through
  * the profile's server, and each partition is compared per shard
  * (`shard<n>/<partition>`), so a shard the copy missed is a difference.
@@ -44,9 +52,11 @@ import { rebuildPlanSubject, RebuildRefusal, type RebuildObservation } from "./o
 import { sourcePartitionExpression } from "./partitions";
 import { cutoverOf, observe, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
 import { shardTable } from "./shards";
+import { collapseSemantics, collapsesRows, splitTopLevel, type CollapseSemantics, type EngineColumn } from "./engines";
 
 export interface PartitionCheck {
   partition: string;
+  /** Rows; keys of the new sorting key when the tables were compared by key (#3727). */
   rows: number;
   checksum: string;
 }
@@ -67,23 +77,42 @@ export interface VerifyResult {
 
 /** A verification that found the tables different. */
 export class RebuildVerificationError extends Error {
+  /** The engine, when it collapses rows and the tables were compared under FINAL (#3674) or by sorting key (#3727). */
+  readonly collapsingEngine?: string;
   constructor(
     readonly table: string,
     readonly mismatches: Mismatch[],
+    /** How the tables were compared when an engine collapses rows; an engine's name alone means under FINAL. */
+    collapsed?: string | Collapsed,
   ) {
+    const how: Collapsed | undefined = typeof collapsed === "string" ? { engine: collapsed, how: "final" } : collapsed;
     const shown = mismatches
       .slice(0, 10)
-      .map((m) => `${m.partition}: old ${m.old ? `${m.old.rows} rows, checksum ${m.old.checksum}` : "none"}, new ${m.new ? `${m.new.rows} rows, checksum ${m.new.checksum}` : "none"}`)
+      .map((m) => `${m.partition}: old ${m.old ? `${m.old.rows} ${unit(how)}, checksum ${m.old.checksum}` : "none"}, new ${m.new ? `${m.new.rows} ${unit(how)}, checksum ${m.new.checksum}` : "none"}`)
       .join("; ");
+    const lateWrite =
+      `in materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, ` +
+      `and in app mode a write the application did not stop`;
     super(
-      `${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. ` +
-        `Nothing was swapped. In materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, is the usual cause; ` +
-        `in app mode, a write the application did not stop.`,
+      `${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. Nothing was swapped. ` +
+        (how?.how === "final"
+          ? `${table} is a ${how.engine}, which collapses rows that share a sorting key when parts merge, and both tables were compared under FINAL. ` +
+            `They still differ when the new table collapses the copied rows by its sorting key differently from how the old one merges them: ` +
+            `run OPTIMIZE TABLE ${table} FINAL, drop the new table if the Op kept it, and run the rebuild again, so the copy reads merged rows. ` +
+            `The other causes are ${lateWrite}.`
+          : how?.how === "keys"
+            ? `The new table is a ${how.engine}, which collapses rows that share a sorting key, so both tables were grouped by its sorting key and partition ` +
+              `and compared per key on ${how.describe ?? "the keys"}, which its merges keep. A key missing from the new table, or one whose values differ, is rows the copy lost or changed. ` +
+              `The usual cause is ${lateWrite}.`
+            : `The usual cause is ${lateWrite}.`),
     );
     this.name = "RebuildVerificationError";
+    if (how) this.collapsingEngine = how.engine;
   }
 }
 
+const unit = (how: Collapsed | undefined) => (how?.how === "keys" ? "keys" : "rows");
+
 /** How often a difference is read again before it counts, and how long apart. */
 const COMPARE_ATTEMPTS = 5;
 const COMPARE_INTERVAL_MS = 1000;
@@ -92,6 +121,81 @@ const COMPARE_INTERVAL_MS = 1000;
 interface SideCounts {
   all: Map<string, PartitionCheck>;
   before: Map<string, PartitionCheck>;
+  /** Grouped by sorting key (#3727): the rows read, of which `all` counts the keys. */
+  rows?: number;
+}
+
+/**
+ * How the tables were compared when an engine collapses rows: both under
+ * FINAL (#3674), or grouped by the new table's sorting key on what its
+ * engine's merges keep (#3727).
+ */
+export interface Collapsed {
+  engine: string;
+  how: "final" | "keys";
+  /** With `keys`: what is compared per key. */
+  describe?: string;
+}
+
+/**
+ * Both tables grouped by the new table's sorting key and its partition, each
+ * key compared on what the new engine's merges keep (#3727): its sums, its
+ * highest version, the sum of its signs. Each side is read raw, so the
+ * comparison is the same whichever parts have merged. The old side's columns
+ * are cast to the new table's types and named as there, so the sorting key,
+ * the partition key and the aggregates read the same on both sides. A
+ * partition is the new table's (`shard<n>/` before it on a cluster), and its
+ * rows are its keys.
+ */
+async function keyComparison(run: RebuildRun, o: RebuildObservation, columns: readonly EngineColumn[]): Promise<{ semantics: CollapseSemantics; read: (side: "old" | "new") => Promise<SideCounts> } | undefined> {
+  const n = o.names;
+  const [t] = await clickhouseQuery<{ engine_full: string; sorting_key: string; partition_key: string }>(
+    run.target.endpoint,
+    `SELECT engine_full, sorting_key, partition_key FROM system.tables WHERE database = ${sqlString(n.database)} AND name = ${sqlString(n.newName)}`,
+  );
+  if (!t || !o.newTable) return undefined;
+  const semantics = collapseSemantics(o.newTable.engine, t.engine_full, columns);
+  if (!semantics) return undefined;
+  // Every column of the sorting key must come from the old table, or the old rows have no key to group by.
+  const copied = new Set(o.copied.map((c) => c.name));
+  if (columns.some((c) => c.inSortingKey && !copied.has(c.name))) return undefined;
+
+  const types = new Map(columns.map((c) => [c.name, c.type]));
+  const sharding = run.sharding;
+  const cutover = cutoverOf(run, o);
+  const cut = run.dualWrite.mode === "materialized-view" ? (run.dualWrite as { cutoverColumn: string }).cutoverColumn : undefined;
+  const before = cut && copied.has(cut) && cutover !== undefined ? `${ident(cut)} < ${utcLiteral(cutover)}` : "1";
+  const key = splitTopLevel(t.sorting_key).filter(Boolean);
+  const parts = splitTopLevel(t.partition_key).filter(Boolean);
+  const partition = parts.length === 0 ? "'all'" : `toString(${parts.length === 1 ? parts[0] : `tuple(${parts.join(", ")})`})`;
+  const label = sharding ? `concat('shard', toString(__shard), '/', ${partition})` : partition;
+  const hash = `cityHash64(toString(tuple(${[...key, ...semantics.aggregates].join(", ")})))`;
+
+  const read = async (side: "old" | "new"): Promise<SideCounts> => {
+    const name = side === "old" ? n.name : n.newName;
+    const from = sharding ? shardTable(sharding, n.database, name) : side === "old" ? n.table : n.newTable;
+    const projected = o.copied.map((c) => (side === "old" ? `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")}) AS ${ident(c.name)}` : ident(c.name)));
+    const rowsOf = `SELECT ${sharding ? "_shard_num AS __shard, " : ""}${projected.join(", ")} FROM ${from}`;
+    const perKey =
+      `SELECT ${label} AS __p, ${hash} AS __h, max(${before}) AS __b, count() AS __n, ${semantics.having ?? "1"} AS __k FROM (${rowsOf}) ` +
+      `GROUP BY __p${sharding ? ", __shard" : ""}${key.length > 0 ? `, ${key.join(", ")}` : ""}`;
+    // A key the engine removes (its sums all zero, its signs cancelled) is not compared, but its rows are counted as read.
+    const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string; raw: string | number }>(
+      run.target.endpoint,
+      `SELECT __p AS p, countIf(__k) AS rows, toString(sumIf(__h, __k)) AS checksum, countIf(__b AND __k) AS brows, toString(sumIf(__h, __b AND __k)) AS bchecksum, sum(__n) AS raw ` +
+        `FROM (${perKey}) GROUP BY p`,
+    );
+    const out: SideCounts = { all: new Map(), before: new Map(), rows: 0 };
+    for (const r of rows) {
+      out.rows! += Number(r.raw);
+      // A partition whose keys the engine all removes holds nothing in the new table, as if it had no rows.
+      if (Number(r.rows) === 0) continue;
+      out.all.set(r.p, { partition: r.p, rows: Number(r.rows), checksum: r.checksum });
+      if (Number(r.brows) > 0) out.before.set(r.p, { partition: r.p, rows: Number(r.brows), checksum: r.bchecksum });
+    }
+    return out;
+  };
+  return { semantics, read };
 }
 
 /**
@@ -103,21 +207,21 @@ async function compareTables(
   run: RebuildRun,
   o: RebuildObservation,
   opts: { attempts?: number; intervalMs?: number } = {},
-): Promise<{ old: SideCounts; new: SideCounts; mismatches: Mismatch[] }> {
+): Promise<{ old: SideCounts; new: SideCounts; mismatches: Mismatch[]; collapsed?: Collapsed }> {
   if (!o.newTable) throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
   const n = o.names;
   const cutover = cutoverOf(run, o);
   await waitOn(run, n.database, n.name);
   await waitOn(run, n.database, n.newName);
 
-  const types = new Map(
-    (
-      await clickhouseQuery<{ name: string; type: string }>(
-        run.target.endpoint,
-        `SELECT name, type FROM system.columns WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)}`,
-      )
-    ).map((c) => [c.name, c.type]),
-  );
+  const newColumns = (
+    await clickhouseQuery<{ name: string; type: string; key: number | string; partition: number | string }>(
+      run.target.endpoint,
+      `SELECT name, type, is_in_sorting_key AS key, is_in_partition_key AS partition FROM system.columns ` +
+        `WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)} ORDER BY position`,
+    )
+  ).map((c): EngineColumn => ({ name: c.name, type: c.type, inSortingKey: Number(c.key) === 1, inPartitionKey: Number(c.partition) === 1 }));
+  const types = new Map(newColumns.map((c) => [c.name, c.type]));
   const oldHash = rowHash(o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`));
   const newHash = rowHash(o.copied.map((c) => ident(c.name)));
   const cut = run.dualWrite.mode === "materialized-view" ? o.copied.find((c) => c.name === (run.dualWrite as { cutoverColumn: string }).cutoverColumn) : undefined;
@@ -127,8 +231,15 @@ async function compareTables(
 
   // Across the shards of a cluster (#3663), every shard's rows, each partition counted per shard as `shard<n>/<partition>`.
   const sharding = run.sharding;
-  const read = async (table: string, name: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
-    const from = sharding ? shardTable(sharding, n.database, name) : table;
+  // Tables whose engines collapse rows by sorting key are read as merged (#3674). Only when both do: a collapsing
+  // table rebuilt into a plain MergeTree keeps every row it copies, and is compared row for row.
+  const final = collapsesRows(o.live.engine) && collapsesRows(o.newTable.engine);
+  // A table that does not collapse rows rebuilt into one that does (#3727): the new table keeps one row per sorting
+  // key where the old one keeps them all, by design. Both are grouped by the new sorting key and compared on what
+  // the new engine's merges keep per key.
+  const keys = !final && collapsesRows(o.newTable.engine) ? await keyComparison(run, o, newColumns) : undefined;
+  const read = async (table: string, name: string, partition: string, hash: string, before: string, final: boolean): Promise<SideCounts> => {
+    const from = `${sharding ? shardTable(sharding, n.database, name) : table}${final ? " FINAL" : ""}`;
     const key = sharding ? `concat('shard', toString(_shard_num), '/', ${partition})` : partition;
     const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string }>(
       run.target.endpoint,
@@ -150,10 +261,18 @@ async function compareTables(
     await syncReplica(run, o, n.database, n.name);
     await syncReplica(run, o, n.database, n.newName);
     // The old table first: a row the view has yet to commit in the new one is read as missing there, never the other way round.
-    const before = await read(n.table, n.name, "_partition_id", oldHash, oldBefore);
-    const after = await read(n.newTable, n.newName, fromOld, newHash, newBefore);
+    const before = keys ? await keys.read("old") : await read(n.table, n.name, "_partition_id", oldHash, oldBefore, final);
+    const after = keys ? await keys.read("new") : await read(n.newTable, n.newName, fromOld, newHash, newBefore, final);
     const mismatches = differences(before.all, after.all);
-    if (mismatches.length === 0 || attempt >= attempts) return { old: before, new: after, mismatches };
+    if (mismatches.length === 0 || attempt >= attempts) {
+      return {
+        old: before,
+        new: after,
+        mismatches,
+        ...(final ? { collapsed: { engine: o.live.engine, how: "final" as const } } : {}),
+        ...(keys ? { collapsed: { engine: o.newTable.engine, how: "keys" as const, describe: keys.semantics.describe } } : {}),
+      };
+    }
     run.log(`-- ${n.key}: ${mismatches.length} partition(s) differ (${mismatches.map((m) => m.partition).slice(0, 5).join(", ")}); reading again (${attempt}/${attempts})`);
     await sleep(opts.intervalMs ?? COMPARE_INTERVAL_MS, run.signal);
   }
@@ -179,8 +298,8 @@ export const rowHash = (columns: readonly string[]): string => `cityHash64(toStr
  * {@link RebuildVerificationError} when it does not, so nothing is swapped.
  */
 export async function assertSameRows(run: RebuildRun, o: RebuildObservation): Promise<void> {
-  const { mismatches } = await compareTables(run, o);
-  if (mismatches.length > 0) throw new RebuildVerificationError(o.names.key, mismatches);
+  const { mismatches, collapsed } = await compareTables(run, o);
+  if (mismatches.length > 0) throw new RebuildVerificationError(o.names.key, mismatches, collapsed);
 }
 
 export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
@@ -188,15 +307,25 @@ export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
   if (o.state !== "rebuild") return { state: o.state, partitions: 0, rows: 0, verification: [], summary: `nothing to verify: ${o.state}` };
   const cutover = cutoverOf(run, o);
   const compared = await compareTables(run, o);
-  if (compared.mismatches.length > 0) throw new RebuildVerificationError(o.names.key, compared.mismatches);
+  if (compared.mismatches.length > 0) throw new RebuildVerificationError(o.names.key, compared.mismatches, compared.collapsed);
 
   const all = [...compared.old.all.values()];
-  const rows = all.reduce((s, v) => s + v.rows, 0);
+  const counted = all.reduce((s, v) => s + v.rows, 0);
+  const c = compared.collapsed;
+  // Grouped by sorting key (#3727), each partition counts keys; the old table's rows are still every row read.
+  const rows = c?.how === "keys" ? (compared.old.rows ?? counted) : counted;
   // What the approval binds: the rows before the cut-over (every row in app mode), which later writes do not change.
   const verification = [...compared.old.before.values()].sort((x, y) => x.partition.localeCompare(y.partition));
   const bound = verification.reduce((s, v) => s + v.rows, 0);
-  const range = cutover !== undefined ? ` (${bound} of them before the cut-over at ${new Date(cutover).toISOString()})` : "";
-  const summary = `${o.names.key}: ${all.length} partition(s), ${rows} row(s)${range}, counts and checksums equal in the old and new tables`;
+  const range = cutover !== undefined ? ` (${bound} ${c?.how === "keys" ? "key(s)" : "of them"} before the cut-over at ${new Date(cutover).toISOString()})` : "";
+  const counts = c?.how === "keys" ? `${rows} row(s) of the old table as ${counted} key(s)` : `${rows} row(s)`;
+  const merged =
+    c?.how === "final"
+      ? `, read under FINAL (${c.engine})`
+      : c?.how === "keys"
+        ? `, grouped by the sorting key of the new ${c.engine} and compared on ${c.describe}`
+        : "";
+  const summary = `${o.names.key}: ${all.length} partition(s), ${counts}${range}, counts and checksums equal in the old and new tables${merged}`;
   run.log(`-- ${summary}`);
   return {
     state: o.state,

package/src/clickhouse/testing/fake-server.ts

@@ -24,6 +24,8 @@ export interface FakeOptions {
   /** Answer every query with this status and body (an auth failure, say). */
   fail?: { status: number; body: string };
   version?: string;
+  /** The SQL user-defined functions `system.functions` lists. Default: none. */
+  functions?: Array<{ name: string; statement: string }>;
 }
 
 const row = (o: Record<string, unknown>) => `${JSON.stringify(o)}\n`;
@@ -52,8 +54,7 @@ export async function fakeClickHouse(objects: FakeObject[], options: FakeOptions
           .join(""),
       );
     }
-    // No SQL user-defined functions.
-    if (/FROM system\.functions/.test(body)) return ok("");
+    if (/FROM system\.functions/.test(body)) return ok((options.functions ?? []).map((f) => row(f)).join(""));
     const show = /^SHOW CREATE (DATABASE|TABLE) (`(?:[^`]|``)*`)(?:\.(`(?:[^`]|``)*`))?$/.exec(body.trim());
     if (show) {
       const [db, name] = show[3] ? [unquote(show[2]!), unquote(show[3])] : [undefined, unquote(show[2]!)];

package/src/codegen/classifier-page.ts

@@ -70,7 +70,7 @@ ALTER TABLE \`analytics\`.\`events\` MODIFY COLUMN \`kind\` LowCardinality(Strin
 --   export const { op } = ClickHouseRebuildOp({ name: "rebuild-analytics-sessions", env: "<env>", table: "analytics.sessions", dualWrite: { mode: "materialized-view", cutoverColumn: "ts" } });
 \`\`\`
 
-A rebuild is never DDL: it is a step naming \`ClickHouseRebuildOp\`, or for a view or a database, which the Op does not rebuild, a manual step. A column or object drop is a statement marked destructive. Each \`CREATE\` carries the project's ownership marker, as the applier stamps it. The command exits 2 when a step is not a statement. Two things the applier asks the server are left out: whether a difference is formatting only, and whether a database is empty before it is dropped.
+A rebuild is never DDL: it is a step naming \`ClickHouseRebuildOp\`, or for a view or a database, which the Op does not rebuild, a manual step. A column or object drop is a statement marked destructive. Each \`CREATE\` carries the project's ownership marker, as the applier stamps it. The command exits 2 when a step is not a statement. A migration runner's history table (\`schema_migrations\`, \`goose_db_version\`; see [Importing a Live Server](../importing/)) is never dropped: the report names it in a hint. Two things the applier asks the server are left out: whether a difference is formatting only, and whether a database is empty before it is dropped.
 
 \`--topology\` renders the statements for where they will run: \`single\` (the default), \`cluster:<name>\`, \`replicated\` or \`cloud\`. On a cluster every statement carries \`ON CLUSTER\` and a MergeTree-family table becomes \`Replicated*MergeTree\` with its Keeper path; in a \`Replicated\` database the tables are \`Replicated*MergeTree\` with no path and no \`ON CLUSTER\`. \`diffStatements(before, after, { topology })\` takes the same choice; see [Applying to a Server](../applying/#topology).
 

package/src/config.ts

@@ -104,6 +104,14 @@ export const sqlProfileSchema = z.strictObject({
   databases: z.array(z.string()).optional(),
   /** The database an unqualified declaration is created in. `default` when omitted. */
   defaultDatabase: z.string().optional(),
+  /**
+   * ClickHouse import (#3718): the SQL user-defined functions `chant import
+   * --from <env>` adopts besides those the imported objects call, by name, or
+   * by prefix with a trailing `*` (`shop_*`). A function belongs to no
+   * database, so without this only the functions the imported tables, views
+   * and dictionaries call (and the functions those call) are imported.
+   */
+  importFunctions: z.array(z.string()).optional(),
   /**
    * ClickHouse: the topology this environment runs, which the applier, the
    * plan and the rebuild migration render every statement for: `single`,
@@ -140,8 +148,11 @@ export const sqlProfileSchema = z.strictObject({
   /**
    * Postgres apply: how long one statement waits for its lock before it
    * fails (`lock_timeout`), in milliseconds, so a change blocked behind a
-   * long query fails fast instead of queueing every query behind it. 5000
-   * when omitted.
+   * long query fails fast instead of queueing every query behind it. The
+   * catalog reads (`chant lifecycle diff --live`, `chant sql plan`, the
+   * import, and an apply's read before its first statement) wait under it
+   * too, and name the session holding the lock when it passes. 5000 when
+   * omitted.
    */
   lockTimeoutMs: z.number().int().nonnegative().optional(),
   /**
@@ -156,11 +167,12 @@ export const sqlProfileSchema = z.strictObject({
    */
   scanTimeoutMs: z.number().int().nonnegative().optional(),
   /**
-   * Postgres: whether chant manages access in this environment (#3681): the
+   * Whether chant manages access in this environment. Postgres (#3681): the
    * declared row-level security policies and each table's row-level security,
    * the declared roles (created when missing, never dropped), and the
    * privileges on the declared objects and the default privileges, which a
-   * plan compares and an apply grants and revokes. Off when omitted: the
+   * plan compares and an apply grants and revokes. ClickHouse (#3716): the
+   * declared users, roles, row policies and grants. Off when omitted: the
    * access declarations are not planned or applied, and the server's access
    * is not read, so a project that manages access elsewhere is not affected.
    */