@intentius/chant-lexicon-sql 0.118.0 → 0.120.0

package/src/clickhouse/plan/deep.ts

@@ -17,17 +17,29 @@
  * (#3664). When a server's copy of a table differs from the profile
  * server's, that copy is what is reported, with the server in `observedOn`,
  * which the drift report prints as `seen on`.
+ *
+ * Where the profile manages access, the grants an apply would give or take
+ * from each declared grantee are reported as `pending` (#3733), the same
+ * statements `chant sql plan` plans (SQLCH273, SQLCH274), so a privilege
+ * granted or revoked by hand after an approval moves the plan digest.
  */
 
-import { deepObservation, type DeepObservationResult, type DeepResourceObservation, type DeepNormalizationHooks } from "@intentius/chant/deep-observation";
+import {
+  deepObservation,
+  type DeepObservationResult,
+  type DeepPendingChange,
+  type DeepResourceObservation,
+  type DeepNormalizationHooks,
+} from "@intentius/chant/deep-observation";
 import type { UnobservedEntity } from "@intentius/chant/lexicon";
 import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "../live/bind";
-import { accessOf, readLiveAccess, readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
+import { ACCESS_ENTITY_TYPES, ACCESS_UNMANAGED_DETAIL, accessOf, readLiveAccess, readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
 import { clickhouseQuery } from "../http";
 import { sqlString } from "../apply/statements";
 import { stripMarkerFromStatement } from "../ownership";
 import { database, dictionary, func, policy, role, table, user, view, CLICKHOUSE_ENTITY_TYPES, type ColumnDef } from "../entities";
-import { canonicalObject, objectKey, type CanonicalColumn, type CanonicalObject } from "./normalize";
+import { canonicalObject, grantsObject, objectKey, type CanonicalColumn, type CanonicalObject } from "./normalize";
+import { grantStatement, grantsByGrantee, revokeStatement, type GrantAtom } from "../access";
 import { renderFor } from "../topology";
 
 type Props = Record<string, unknown>;
@@ -207,7 +219,7 @@ export async function observeResourcesDeep(
   let live: LiveObject[];
   try {
     target = await bindClickHouse(options);
-    live = [...(await readLiveSchema(target)), ...(await readLiveAccess(target, accessOf(options.entities, target.defaultDatabase)))];
+    live = [...(await readLiveSchema(target)), ...(target.access === true ? await readLiveAccess(target, accessOf(options.entities, target.defaultDatabase)) : [])];
   } catch (err) {
     const why = classifyClickHouseFailure(err);
     for (const name of options.entityNames) {
@@ -227,17 +239,34 @@ export async function observeResourcesDeep(
     }
     return deepObservation({}, unobserved);
   }
+  const pending: DeepPendingChange[] = [];
+  if (target.access === true) {
+    try {
+      pending.push(...(await pendingGrants(target, options.entities)));
+    } catch (err) {
+      const why = classifyClickHouseFailure(err);
+      for (const name of options.entityNames) {
+        if (options.entities.get(name)?.entityType !== CLICKHOUSE_ENTITY_TYPES.grant) continue;
+        unobserved[name] = { type: CLICKHOUSE_ENTITY_TYPES.grant, reason: "read-failed", detail: `the server's grants could not be compared: ${why.detail}` };
+      }
+    }
+  }
   for (const name of options.entityNames) {
     const entity = options.entities.get(name);
     if (!entity || !entity.entityType.startsWith("ClickHouse::")) {
       unobserved[name] = { type: entity?.entityType ?? "", reason: "unsupported-kind" };
       continue;
     }
+    if (target.access !== true && ACCESS_ENTITY_TYPES.has(entity.entityType)) {
+      unobserved[name] = { type: entity.entityType, reason: "filtered", detail: ACCESS_UNMANAGED_DETAIL };
+      continue;
+    }
     if (entity.entityType === CLICKHOUSE_ENTITY_TYPES.grant) {
+      if (unobserved[name]) continue;
       unobserved[name] = {
         type: entity.entityType,
         reason: "unsupported-kind",
-        detail: "a grant is compared as part of its grantees' grants, which chant sql plan reports (SQLCH273, SQLCH274)",
+        detail: "a grant is compared as part of its grantees' grants: what an apply would grant or revoke is listed as pending (SQLCH273, SQLCH274)",
       };
       continue;
     }
@@ -280,5 +309,40 @@ export async function observeResourcesDeep(
       unobserved[name] = { type: entity.entityType, reason: "read-failed", detail: `the server's definition does not parse: ${(err as Error).message}` };
     }
   }
-  return deepObservation(resources, unobserved);
+  return deepObservation(resources, unobserved, pending);
+}
+
+/**
+ * The grants an apply would give or take from each declared grantee (#3733),
+ * as `chant sql plan` compares them: the declared grants against what
+ * `SHOW GRANTS FOR` prints, atom by atom. One change per grantee, its
+ * REVOKEs before its GRANTs as the applier sends them, with the grant
+ * declarations behind it. A grantee the server does not hold yet is given
+ * every declared grant.
+ */
+export async function pendingGrants(
+  target: ClickHouseTarget,
+  entities: ReadonlyMap<string, { entityType: string; props: Record<string, unknown> }>,
+): Promise<DeepPendingChange[]> {
+  const declared = grantsByGrantee([...entities].map(([name, e]) => ({ export: name, type: e.entityType, ddl: String(e.props.ddl ?? "") })));
+  if (declared.length === 0) return [];
+  const live = await readLiveAccess(target, declared.map((g) => ({ kind: "grants", name: g.grantee })));
+  const held = new Map(live.filter((o) => o.type === CLICKHOUSE_ENTITY_TYPES.grant).map((o) => [o.name, grantsObject(o.statement, o.name, target.defaultDatabase).access ?? {}]));
+  const atom = (json: string) => JSON.parse(json) as GrantAtom;
+  const out: DeepPendingChange[] = [];
+  for (const g of declared) {
+    const want = grantsObject(g.ddl, g.grantee, target.defaultDatabase).access ?? {};
+    const have = held.get(g.grantee) ?? {};
+    const statements = [
+      ...Object.keys(have)
+        .filter((k) => !(k in want))
+        .map((k) => revokeStatement(atom(have[k]!), g.grantee)),
+      ...Object.keys(want)
+        .filter((k) => !(k in have))
+        .map((k) => grantStatement(atom(want[k]!), g.grantee)),
+    ];
+    if (statements.length === 0) continue;
+    out.push({ subject: `grants ${g.grantee}`, change: statements.join("; "), entities: [...g.exports].sort() });
+  }
+  return out;
 }

package/src/clickhouse/plan/diff.ts

@@ -259,7 +259,10 @@ export function diffSchemas(before: readonly SchemaObject[], after: readonly Sch
   for (const m of matches) {
     if (m.kind === "created") changes.push(change(m.after.key, "object", "SQLCH200", undefined, qualified(m.after.canonical)));
     else if (m.kind === "matched") diffObject(m.after.key, m.before.canonical, m.after.canonical, changes, hints);
-    else {
+    else if (m.before.canonical.foreign) {
+      // A migration runner's history table (#3676): not chant's to drop.
+      hints.push(`${qualified(m.before.canonical)} is kept by ${m.before.canonical.foreign}; it is not chant's to drop and is left alone`);
+    } else {
       const b = m.before;
       changes.push(change(b.key, "object", "SQLCH250", qualified(b.canonical), undefined, { destructive: b.canonical.kind === "table" || b.canonical.kind === "materializedView" }));
     }

package/src/clickhouse/plan/normalize.ts

@@ -98,6 +98,8 @@ export interface CanonicalObject {
   access?: Record<string, string>;
   /** `-- previously: <name>` before the statement: the object's previous name. */
   previously?: string;
+  /** The tool that keeps a live table (a migration runner's history table): a plan never drops it. */
+  foreign?: string;
 }
 
 const KEYWORDS = new Set(
@@ -560,6 +562,7 @@ export function scopeOf(o: { kind?: ObjectKind; type?: string; database?: string
 }
 
 /** A live object's canonical form: a grantee's grants from its `SHOW GRANTS` lines, anything else from its `SHOW CREATE`. */
-export function liveCanonical(o: { type: string; name: string; statement: string }, defaultDatabase = "default"): CanonicalObject {
-  return o.type === "ClickHouse::Grant" ? grantsObject(o.statement, o.name, defaultDatabase) : canonicalObject(o.statement, defaultDatabase);
+export function liveCanonical(o: { type: string; name: string; statement: string; foreign?: string }, defaultDatabase = "default"): CanonicalObject {
+  const canonical = o.type === "ClickHouse::Grant" ? grantsObject(o.statement, o.name, defaultDatabase) : canonicalObject(o.statement, defaultDatabase);
+  return o.foreign ? { ...canonical, foreign: o.foreign } : canonical;
 }

package/src/clickhouse/plan/plan.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "vitest";
-import { canonicalCodec, canonicalExpression, canonicalObject, canonicalTtl, canonicalType } from "./normalize";
+import { canonicalCodec, canonicalExpression, canonicalObject, canonicalTtl, canonicalType, liveCanonical } from "./normalize";
 import { diffSchemas, type SchemaObject } from "./diff";
 import { renderDiff } from "./report";
 import { CLASSIFIER_RULES } from "./rules";
@@ -184,6 +184,14 @@ describe("classifying a change", () => {
     expect(classify("CREATE VIEW events AS SELECT 1")).toEqual([["kind", "SQLCH224", "rebuild"]]);
   });
 
+  test("a migration runner's history table is named in a hint, never dropped (#3676)", () => {
+    const statement = "CREATE TABLE legacy.schema_migrations (version Int64, dirty UInt8, sequence UInt64) ENGINE = MergeTree ORDER BY sequence";
+    const live = [{ key: "legacy.schema_migrations", canonical: liveCanonical({ type: "ClickHouse::Table", name: "schema_migrations", statement, foreign: "a migration runner (Rails, golang-migrate, dbmate)" }) }];
+    const d = diffSchemas(live, []);
+    expect(d.changes).toEqual([]);
+    expect(d.hints).toEqual(["legacy.schema_migrations is kept by a migration runner (Rails, golang-migrate, dbmate); it is not chant's to drop and is left alone"]);
+  });
+
   test("against a server, previously before the statement matches the old name", () => {
     const live = [{ key: "default.events", canonical: canonicalObject(BASE) }];
     const declared = [{ key: "default.raw_events", canonical: canonicalObject(`-- previously: events\n${BASE.replace("TABLE events", "TABLE raw_events")}`) }];

package/src/clickhouse/plan/rebuild-handoff.ts

@@ -19,11 +19,14 @@
 import type { CanonicalObject } from "./normalize";
 import type { SchemaDiff } from "./diff";
 import { migrationOpName, renderMigrationOps, type MigrationOpSuggestion } from "../../core/handoff";
+import { collapsesRows } from "../rebuild/engines";
 
 export interface RebuildOpSuggestion extends MigrationOpSuggestion {
   /** `database.name`, the Op's `table`. */
   table: string;
   dualWrite: { mode: "materialized-view"; cutoverColumn: string } | { mode: "app" };
+  /** What to know before running it: set for an engine that collapses rows by sorting key (#3674, #3727). */
+  note?: string;
 }
 
 const TIME_TYPE = /^(?:Nullable\s*\(\s*)?(?:Date|Date32|DateTime|DateTime64)\b/;
@@ -42,11 +45,17 @@ export function rebuildOpSuggestions(diff: SchemaDiff, declared: ReadonlyMap<str
     const dualWrite: RebuildOpSuggestion["dualWrite"] = time ? { mode: "materialized-view", cutoverColumn: time.name } : { mode: "app" };
     const name = migrationOpName("rebuild", table);
     const dw = dualWrite.mode === "app" ? `{ mode: "app" }` : `{ mode: "materialized-view", cutoverColumn: ${JSON.stringify(dualWrite.cutoverColumn)} }`;
+    const note = collapsesRows(c.engineName)
+      ? `${table} is a ${c.engineName}: when the old table's engine collapses rows too, the Verify phase reads both tables under FINAL, so rows the old table has not merged yet compare as merged. ` +
+        `A new sorting key that collapses rows differently from the old one still fails it; OPTIMIZE TABLE ${table} FINAL before the run avoids that. ` +
+        `From an engine that keeps every row (a plain MergeTree), it groups both tables by the new sorting key instead and compares what a ${c.engineName} keeps per key (its sums, its highest version, the sum of its signs).`
+      : undefined;
     out.push({
       table,
       name,
       env,
       dualWrite,
+      ...(note ? { note } : {}),
       declaration: `export const { op } = ClickHouseRebuildOp({ name: ${JSON.stringify(name)}, env: ${JSON.stringify(env)}, table: ${JSON.stringify(table)}, dualWrite: ${dw} });`,
     });
   }
@@ -55,5 +64,6 @@ export function rebuildOpSuggestions(diff: SchemaDiff, declared: ReadonlyMap<str
 
 /** The suggestions as text, for the report. */
 export function renderRebuildOps(ops: readonly RebuildOpSuggestion[]): string[] {
-  return renderMigrationOps(ops, { what: "the rebuild migration Op", exportName: "ClickHouseRebuildOp", importPath: "@intentius/chant-lexicon-sql/clickhouse" });
+  const notes = ops.flatMap((o) => (o.note ? [`  Note: ${o.note}`] : []));
+  return [...renderMigrationOps(ops, { what: "the rebuild migration Op", exportName: "ClickHouseRebuildOp", importPath: "@intentius/chant-lexicon-sql/clickhouse" }), ...notes];
 }

package/src/clickhouse/rebuild/engines.ts

@@ -0,0 +1,163 @@
+/**
+ * Which table engines the rebuild reads as merged (#3674): the verification
+ * (`./verify.ts`) and the plan's hand-off note (`../plan/rebuild-handoff.ts`).
+ */
+
+/**
+ * The MergeTree engines that collapse rows sharing a sorting key when parts
+ * merge, as `system.tables.engine` names them, replicated or shared too. A
+ * count of such a table depends on which parts have merged; under `FINAL` it
+ * does not.
+ */
+const COLLAPSING_ENGINE = /^(?:Replicated|Shared)?(?:Summing|Replacing|Aggregating|Collapsing|VersionedCollapsing|Coalescing|Graphite)MergeTree$/;
+
+/** Whether an engine collapses rows that share a sorting key (#3674). */
+export const collapsesRows = (engine: string | undefined): boolean => COLLAPSING_ENGINE.test(engine ?? "");
+
+/** A column of the new table, as `system.columns` reports it. */
+export interface EngineColumn {
+  name: string;
+  type: string;
+  /** Part of the sorting key: the engine never merges its values. */
+  inSortingKey: boolean;
+  /** Part of the partition key: rows that merge share its value, so a SummingMergeTree does not sum it. */
+  inPartitionKey?: boolean;
+}
+
+/**
+ * How a collapsing engine merges the rows that share a sorting key, as what a
+ * merge leaves the same (#3727): for each key, the aggregates it preserves and
+ * whether it keeps the key at all. Both tables are grouped by the new table's
+ * sorting key and compared on these, so it does not matter which parts have
+ * merged, and a table that never collapses (a plain MergeTree) compares with
+ * one that does.
+ */
+export interface CollapseSemantics {
+  /** The engine's name without `Replicated` or `Shared`. */
+  engine: string;
+  /** Per key: what merging leaves unchanged. SQL over the new table's column names. */
+  aggregates: string[];
+  /** Per key: whether the engine keeps a row for it. Absent: every key keeps one. */
+  having?: string;
+  /** One line for the summary and the error: what is compared per key. */
+  describe: string;
+}
+
+/** Split on the commas outside parentheses and quotes. */
+export function splitTopLevel(text: string): string[] {
+  const out: string[] = [];
+  let depth = 0;
+  let quote: string | undefined;
+  let start = 0;
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i]!;
+    if (quote) {
+      if (c === "\\") i++;
+      else if (c === quote) quote = undefined;
+    } else if (c === "'" || c === '"' || c === "`") quote = c;
+    else if (c === "(" || c === "[") depth++;
+    else if (c === ")" || c === "]") depth--;
+    else if (c === "," && depth === 0) {
+      out.push(text.slice(start, i).trim());
+      start = i + 1;
+    }
+  }
+  const last = text.slice(start).trim();
+  if (last || out.length > 0) out.push(last);
+  return out;
+}
+
+/** The engine's own arguments in `system.tables.engine_full`, without a `Replicated` engine's Keeper path and replica name. */
+export function engineArguments(engineFull: string): string[] {
+  const m = /^\s*(\w+)\s*\(/.exec(engineFull);
+  if (!m) return [];
+  let depth = 0;
+  let quote: string | undefined;
+  const from = m[0].length;
+  let end = -1;
+  for (let i = from - 1; i < engineFull.length && end < 0; i++) {
+    const c = engineFull[i]!;
+    if (quote) {
+      if (c === "\\") i++;
+      else if (c === quote) quote = undefined;
+    } else if (c === "'") quote = c;
+    else if (c === "(") depth++;
+    else if (c === ")" && --depth === 0) end = i;
+  }
+  if (end < 0) return [];
+  const args = splitTopLevel(engineFull.slice(from, end)).filter((a) => a !== "");
+  if (/^(?:Replicated|Shared)/.test(m[1]!)) while (args.length > 0 && args[0]!.startsWith("'")) args.shift();
+  return args;
+}
+
+const NUMERIC = /^(?:U?Int(?:8|16|32|64|128|256)|Float(?:32|64)|BFloat16|Decimal(?:32|64|128|256)?\s*\(.*\))$/;
+const FLOAT = /^(?:Float(?:32|64)|BFloat16)$/;
+/** The SimpleAggregateFunction functions whose result a merge keeps exactly, whatever the order of the rows. */
+const SIMPLE_ORDER_FREE = new Set(["sum", "sumWithOverflow", "min", "max", "groupBitAnd", "groupBitOr", "groupBitXor"]);
+
+const quoted = (name: string) => `\`${name.replace(/\\/g, "\\\\").replace(/`/g, "\\`")}\``;
+const typeLiteral = (type: string) => `'${type.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+/** A sum in the column's own type, as the engine stores it; a float to 32 bits, since the order rows are added in moves its last bits. */
+const summed = (c: EngineColumn) => (FLOAT.test(c.type) ? `toFloat32(sum(${quoted(c.name)}))` : `CAST(sum(${quoted(c.name)}), ${typeLiteral(c.type)})`);
+
+/**
+ * What a merge of `engine` keeps per sorting key, or undefined for an engine
+ * that does not collapse rows, or whose result depends on the order rows
+ * merge in (`GraphiteMergeTree` thins its rows by age).
+ */
+export function collapseSemantics(engine: string, engineFull: string, columns: readonly EngineColumn[]): CollapseSemantics | undefined {
+  if (!collapsesRows(engine)) return undefined;
+  const base = engine.replace(/^(?:Replicated|Shared)/, "").replace(/MergeTree$/, "");
+  const args = engineArguments(engineFull);
+  const values = columns.filter((c) => !c.inSortingKey);
+  const named = (n: string | undefined) => (n ? columns.find((c) => c.name === n.replace(/^`|`$/g, "")) : undefined);
+  switch (base) {
+    case "Summing": {
+      // The columns listed, else every numeric column outside the sorting and partition keys; a key whose sums are all zero is removed.
+      const listed = args[0] ? splitTopLevel(args[0].replace(/^\((.*)\)$/s, "$1")).map(named) : undefined;
+      const sums = (listed ? listed.filter((c): c is EngineColumn => c !== undefined) : values).filter((c) => !c.inSortingKey && !c.inPartitionKey && NUMERIC.test(c.type));
+      const aggregates = sums.map(summed);
+      return {
+        engine: `${base}MergeTree`,
+        aggregates,
+        ...(aggregates.length > 0 ? { having: `NOT (${aggregates.map((a) => `${a} = 0`).join(" AND ")})` } : {}),
+        describe: sums.length > 0 ? `the sums of ${sums.map((c) => c.name).join(", ")}` : "the keys",
+      };
+    }
+    case "Replacing": {
+      // The row with the highest version is kept; without a version, the last one written, so only the key is compared.
+      const ver = named(args[0]);
+      return { engine: `${base}MergeTree`, aggregates: ver ? [`max(${quoted(ver.name)})`] : [], describe: ver ? `the highest ${ver.name}` : "the keys" };
+    }
+    case "Collapsing":
+    case "VersionedCollapsing": {
+      // A pair of rows with opposite signs cancels: the sum of the signs is what a merge keeps, and a key whose signs cancel out is removed.
+      const sign = named(args[0]);
+      if (!sign) return undefined;
+      const s = `sum(${quoted(sign.name)})`;
+      return { engine: `${base}MergeTree`, aggregates: [s], having: `${s} != 0`, describe: `the sum of ${sign.name}` };
+    }
+    case "Aggregating": {
+      const simple = values.flatMap((c) => {
+        const m = /^SimpleAggregateFunction\(\s*(\w+)\s*,/.exec(c.type);
+        return m && SIMPLE_ORDER_FREE.has(m[1]!) ? [{ c, fn: m[1]! }] : [];
+      });
+      return {
+        engine: `${base}MergeTree`,
+        aggregates: simple.map(({ c, fn }) => `${fn}(${quoted(c.name)})`),
+        describe: simple.length > 0 ? `${simple.map(({ c, fn }) => `${fn}(${c.name})`).join(", ")}` : "the keys",
+      };
+    }
+    case "Coalescing": {
+      // The last value that is not NULL is kept: whether a column has one is what a merge keeps.
+      const nullable = values.filter((c) => /^Nullable\(/.test(c.type));
+      return {
+        engine: `${base}MergeTree`,
+        aggregates: nullable.map((c) => `count(${quoted(c.name)}) > 0`),
+        describe: nullable.length > 0 ? `which of ${nullable.map((c) => c.name).join(", ")} hold a value` : "the keys",
+      };
+    }
+    default:
+      return undefined;
+  }
+}

package/src/clickhouse/rebuild/rebuild-collapsing.e2e.test.ts

@@ -0,0 +1,120 @@
+/**
+ * A rebuild of a SummingMergeTree with unmerged parts (#3674).
+ *
+ * Three rows for one key, in three parts the server is kept from merging.
+ * The copy inserts them in one block, which the new table collapses into one
+ * row as it writes it, so the raw counts differ (3 and 1) although the data
+ * is the same. The Verify phase reads both tables under FINAL, finds them
+ * equal, and the run reaches the swap gate.
+ *
+ * Runs against `CHANT_SQL_E2E_CLICKHOUSE_URL` when it is set (a shared
+ * server: everything is in the database `chant_e2e_3674`, dropped
+ * afterwards, with this test's receipts), otherwise a scratch server, which
+ * needs Docker and skips cleanly without it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { memoryGateLedgerPort, runOpLocally, loadProfiles, type ActivityFn, type OpConfig } from "@intentius/chant/op";
+import { dockerAvailable, startScratchServer, type ScratchServer } from "../container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { clickhouseApply } from "../../op/activities/clickhouse-apply";
+import * as rebuildActivities from "../../op/activities/clickhouse-rebuild";
+import { ClickHouseRebuildOp } from "./op";
+import { RECEIPTS_DATABASE, RECEIPTS_TABLE } from "./receipts";
+
+const given = process.env.CHANT_SQL_E2E_CLICKHOUSE_URL;
+const enabled = given ? true : await dockerAvailable();
+const DB = "chant_e2e_3674";
+const MARKER = { stack: "chant_e2e_3674", env: "test" };
+let server: ScratchServer | undefined;
+let endpoint: ClickHouseEndpoint;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-rebuild-collapsing-"));
+
+const q = <T = Record<string, unknown>>(sql: string) => clickhouseQuery<T>(endpoint, sql);
+const count = async (from: string) => Number((await q<{ n: string }>(`SELECT count() AS n FROM ${from}`))[0]!.n);
+
+async function cleanup(): Promise<void> {
+  await q(`DROP DATABASE IF EXISTS ${DB} SYNC`);
+  const receipts = await q<{ n: string }>(`SELECT count() AS n FROM system.tables WHERE database = '${RECEIPTS_DATABASE}' AND name = '${RECEIPTS_TABLE}'`);
+  if (Number(receipts[0]!.n) > 0) {
+    await clickhouseQuery(endpoint, `ALTER TABLE ${RECEIPTS_DATABASE}.${RECEIPTS_TABLE} DELETE WHERE position(address, '${MARKER.stack}') > 0`, { settings: { mutations_sync: "2" } });
+  }
+}
+
+beforeAll(async () => {
+  if (!enabled) return;
+  if (given) endpoint = { url: given, user: "default", password: process.env.CHANT_SQL_E2E_CLICKHOUSE_PASSWORD ?? "" };
+  else {
+    server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-rebuild-collapsing" });
+    endpoint = server.endpoint;
+  }
+  await cleanup();
+}, 600_000);
+
+afterAll(async () => {
+  if (enabled) await cleanup().catch(() => undefined);
+  await server?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+const deps = () => ({ config: { ownership: MARKER }, env: { CLICKHOUSE_URL: endpoint.url, CLICKHOUSE_USER: endpoint.user, CLICKHOUSE_PASSWORD: endpoint.password }, log: () => undefined });
+
+function writeBuild(file: string, orderBy: string): string {
+  const objects = [
+    { export: "db", type: "ClickHouse::Database", ddl: `CREATE DATABASE ${DB} ENGINE = Atomic`, dependsOn: [] },
+    {
+      export: "daily",
+      type: "ClickHouse::Table",
+      dependsOn: ["db"],
+      ddl: `CREATE TABLE ${DB}.daily (day Date, total Decimal(18, 2), n UInt64 DEFAULT 0) ENGINE = SummingMergeTree ORDER BY ${orderBy}`,
+    },
+  ];
+  writeFileSync(join(dir, file), JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects }));
+  return join(dir, file);
+}
+
+describe.skipIf(!enabled)("a SummingMergeTree with unmerged parts, rebuilt with a new sorting key", () => {
+  test("the Verify phase compares the tables under FINAL and the run reaches the swap gate", async () => {
+    await clickhouseApply({ buildPath: writeBuild("daily-v1.json", "day"), environment: "e2e" }, undefined, deps());
+    await q(`SYSTEM STOP MERGES ${DB}.daily`);
+    for (const total of [30, 30, 40]) await q(`INSERT INTO ${DB}.daily VALUES ('2026-10-10', ${total}, 0)`);
+    expect(await count(`${DB}.daily`)).toBe(3);
+    writeBuild("daily-v2.json", "(day, n)");
+
+    const { op } = ClickHouseRebuildOp({
+      name: "rebuild-daily",
+      env: "e2e",
+      table: `${DB}.daily`,
+      dualWrite: { mode: "materialized-view", cutoverColumn: "day", cutoverDelay: "1s" },
+      build: false,
+      path: dir,
+      output: "daily-v2.json",
+      stack: MARKER.stack,
+      ownershipEnv: MARKER.env,
+    });
+    const activities = new Map<string, ActivityFn>();
+    for (const [name, fn] of Object.entries(rebuildActivities)) {
+      if (typeof fn !== "function" || !name.startsWith("clickhouseRebuild")) continue;
+      activities.set(name, ((args: Record<string, unknown>, signal?: AbortSignal) =>
+        (fn as (a: unknown, s?: AbortSignal, d?: unknown) => Promise<unknown>)(args, signal, deps())) as ActivityFn);
+    }
+    const r = await runOpLocally((op as unknown as { props: OpConfig }).props, activities, await loadProfiles(), undefined, {
+      gates: memoryGateLedgerPort(),
+      now: new Date().toISOString(),
+    });
+
+    const verify = r.records.find((x) => x.fn === "clickhouseRebuildVerify");
+    expect(verify?.error).toBeUndefined();
+    expect(r.status).toBe("gated");
+    expect(r.gate).toMatchObject({ gate: "approve-rebuild-daily" });
+    const verification = r.records.flatMap((x) => x.outcomes ?? []).find((o) => o.name === "Verification")?.value;
+    expect(String(verification)).toMatch(/1 row\(s\).*read under FINAL \(SummingMergeTree\)/);
+    // The old table still holds its three unmerged rows; the new one holds them collapsed.
+    expect(await count(`${DB}.daily`)).toBe(3);
+    expect(await q(`SELECT toString(day) AS day, toString(total) AS total FROM ${DB}.daily__chant_new`)).toEqual([{ day: "2026-10-10", total: "100" }]);
+  }, 300_000);
+});