@intentius/chant-lexicon-sql 0.113.0 → 0.114.1

package/src/clickhouse/rebuild/observe.ts

@@ -55,9 +55,10 @@ export type DualWrite =
       /**
        * A materialized view on the old table writes every row whose
        * `cutoverColumn` is at or after the cut-over into the new table; the
-       * backfill copies the rows before it. The cut-over is the server's time
-       * when the view is made plus `cutoverDelay` (default `1m`), and the
-       * backfill starts once it has passed. The column must be a time
+       * backfill copies the rows before it, and the rows at or after it that
+       * the table held before the view was made. The cut-over is the server's
+       * time when the view is made plus `cutoverDelay` (default `1m`), rounded
+       * up to a whole second, and the backfill starts once it has passed. The column must be a time
        * (`Date`, `DateTime`, `DateTime64`) that rows arrive in order of, give
        * or take the delay; a row that arrives later than that is caught by
        * the verification, which then fails the run.

package/src/clickhouse/rebuild/rebuild.e2e.test.ts

@@ -7,6 +7,9 @@
  *   table, keeps the old table, and drops it at the second gate.
  * - A failure: rows that land in the old table after the backfill fail the
  *   verification, and onFailure drops the new table.
+ * - Rows dated after the cut-over (sql-yodeler#45): the ones the table held
+ *   before the view are copied by the backfill, the ones written during the
+ *   rebuild arrive through the view, and the swap leaves none behind.
  * - An interrupted backfill: stopped after three partitions, then killed
  *   between a partition's INSERT and its receipt, then resumed: the receipts
  *   skip what was copied, the half-copied partition is cleared first, and the
@@ -174,7 +177,7 @@ describe.skipIf(!enabled)("a sorting-key change, rebuilt through its gates", ()
     expect(outcome(r, "RebuildState")).toBe("rebuild");
     expect(outcome(r, "VerifiedRows")).toBe(1600);
     expect(outcome(r, "VerifiedPartitions")).toBe(4);
-    expect(String(outcome(r, "Verification"))).toMatch(/4 partition\(s\), 1600 row\(s\) before the cut-over/);
+    expect(String(outcome(r, "Verification"))).toMatch(/4 partition\(s\), 1600 row\(s\) \(1600 of them before the cut-over/);
     expect(await count("shop.events__chant_new")).toBe(1600);
     expect(await exists("shop", "events__chant_dual")).toBe(true);
     // Nothing visible to a plan or an apply: the working objects are left out.
@@ -235,6 +238,108 @@ describe.skipIf(!enabled)("a sorting-key change, rebuilt through its gates", ()
   }, 300_000);
 });
 
+describe.skipIf(!enabled)("rows dated after the cut-over that the table already held (sql-yodeler#45)", () => {
+  const bookingsDdl = (orderBy: string) => `CREATE TABLE shop.bookings
+(
+  id UInt64,
+  kind LowCardinality(String),
+  at DateTime
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(at)
+ORDER BY ${orderBy}`;
+  const V1: Obj = { export: "bookings", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: bookingsDdl("(kind, at)") };
+  const V2: Obj = { ...V1, ddl: bookingsDdl("(kind, id)") };
+  const config = (): ClickHouseRebuildOpConfig => ({
+    name: "rebuild-bookings",
+    env: "e2e",
+    table: "shop.bookings",
+    dualWrite: { mode: "materialized-view", cutoverColumn: "at", cutoverDelay: "2s" },
+    build: false,
+    path: dir,
+    output: "bookings-v2.json",
+    retain: "0s",
+    stack: MARKER.stack,
+    ownershipEnv: MARKER.env,
+  });
+
+  test("every row is copied and compared, those written during the rebuild too, and the swap leaves none behind", async () => {
+    await clickhouseApply({ buildPath: join(dir, writeBuild("bookings-v1.json", [DB, V1])), environment: "e2e" }, undefined, deps());
+    // 600 rows seven hours apart, from four months ago to about two months
+    // ahead, with an identical pair among the future ones.
+    await q(
+      "INSERT INTO shop.bookings SELECT number, ['view','cart','buy'][number % 3 + 1], toStartOfHour(now()) - toIntervalDay(120) + toIntervalHour(number * 7) FROM numbers(600)",
+    );
+    await q("INSERT INTO shop.bookings VALUES (599, 'buy', toStartOfHour(now()) - toIntervalDay(120) + toIntervalHour(599 * 7))");
+    const future = Number((await q<{ n: string }>("SELECT count() AS n FROM shop.bookings WHERE at > now() + 60"))[0]!.n);
+    expect(future).toBeGreaterThan(150);
+    writeBuild("bookings-v2.json", [DB, V2]);
+
+    const ledger = new Ledger();
+    const first = await runOp(config(), ledger.port);
+    expect(first.status).toBe("gated");
+    expect(first.gate).toMatchObject({ gate: "approve-rebuild-bookings" });
+    expect(outcome(first, "VerifiedRows")).toBe(601);
+    expect(await count("shop.bookings__chant_new")).toBe(601);
+
+    // Written during the rebuild, after the cut-over: through the view.
+    await q("INSERT INTO shop.bookings VALUES (1000, 'buy', now() + toIntervalDay(40)), (1001, 'cart', now())");
+    const again = await runOp(config(), ledger.port);
+    expect(again.status).toBe("gated");
+    expect(outcome(again, "VerifiedRows")).toBe(603);
+    // The rows before the cut-over are what the approval binds; later rows do not change it.
+    expect(again.gate!.planDigest).toBe(first.gate!.planDigest);
+
+    ledger.approveLast();
+    await q("INSERT INTO shop.bookings VALUES (1002, 'view', now() + toIntervalDay(70))");
+    const swapped = await runOp(config(), ledger.port);
+    expect(swapped.status).toBe("gated");
+    expect(swapped.gate).toMatchObject({ gate: "approve-rebuild-bookings-drop" });
+    expect(await showCreate("shop.bookings")).toMatch(/ORDER BY \(kind, id\)/);
+    expect(await count("shop.bookings")).toBe(604);
+    expect(await count("shop.bookings__chant_old")).toBe(604);
+    const diff = await q("SELECT * FROM shop.bookings__chant_old EXCEPT ALL SELECT * FROM shop.bookings");
+    expect(diff).toEqual([]);
+  }, 300_000);
+
+  test("a row that reaches the old table alone after the verification stops the swap", async () => {
+    const clicksDdl = (orderBy: string) => `CREATE TABLE shop.visits
+(
+  id UInt64,
+  at DateTime
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(at)
+ORDER BY ${orderBy}`;
+    const W1: Obj = { export: "visits", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: clicksDdl("at") };
+    const W2: Obj = { ...W1, ddl: clicksDdl("(id, at)") };
+    await clickhouseApply({ buildPath: join(dir, writeBuild("visits-v1.json", [DB, W1])), environment: "e2e" }, undefined, deps());
+    await q("INSERT INTO shop.visits SELECT number, now() - toIntervalDay(number % 90) FROM numbers(300)");
+    writeBuild("visits-v2.json", [DB, W2]);
+    const args = {
+      table: "shop.visits",
+      buildPath: "visits-v2.json",
+      environment: "e2e",
+      dualWrite: { mode: "materialized-view" as const, cutoverColumn: "at", cutoverDelay: "2s" },
+      stack: MARKER.stack,
+      ownershipEnv: MARKER.env,
+      cwd: dir,
+    };
+    await rebuildActivities.clickhouseRebuildCreate(args, undefined, deps());
+    await rebuildActivities.clickhouseRebuildDualWrite(args, undefined, deps());
+    await rebuildActivities.clickhouseRebuildBackfill(args, undefined, deps());
+    expect(await rebuildActivities.clickhouseRebuildVerify(args, undefined, deps())).toMatchObject({ rows: 300 });
+
+    // Late: a time well before the cut-over, written after the verification.
+    await q("INSERT INTO shop.visits VALUES (9999, now() - toIntervalDay(30))");
+    await expect(rebuildActivities.clickhouseRebuildSwap(args, undefined, deps())).rejects.toThrow(/does not match the old one in 1 partition/);
+    expect(await showCreate("shop.visits")).toMatch(/ORDER BY at/);
+    expect(await count("shop.visits")).toBe(301);
+    expect(await count("shop.visits__chant_new")).toBe(300);
+    await rebuildActivities.clickhouseRebuildCompensate(args, undefined, deps());
+  }, 300_000);
+});
+
 describe.skipIf(!enabled)("a failure runs onFailure, which drops the new table", () => {
   const clicksDdl = (orderBy: string) => `CREATE TABLE shop.clicks (id UInt64, url String) ENGINE = MergeTree ORDER BY ${orderBy}`;
   const V1: Obj = { export: "clicks", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: clicksDdl("id") };

package/src/clickhouse/rebuild/receipts.ts

@@ -50,6 +50,7 @@ import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
 import { ident, sqlString } from "../apply/statements";
 import { CLICKHOUSE_COMMENT_OWNERSHIP_KEYS, RECEIPTS_TRAILER_KEY } from "../ownership";
 import { DDL_SETTINGS, syncReplicaWithin } from "./replicas";
+import { renderFor, type Topology } from "../topology";
 
 /** The database the receipts live in. Its comment marks it as chant's, so schema reads leave it out. */
 export const RECEIPTS_DATABASE = "chant_receipts";
@@ -79,22 +80,31 @@ export type ClickHouseReceiptStore = SqlReceiptStore;
  * project's ownership stack and env, which every address starts with.
  * `replicatedIn` names a Replicated database to keep them in instead, and
  * `replicaTimeoutMs` how long a read waits for this replica to catch up there.
+ * `topology` is the environment's (`../topology.ts`): the database and table
+ * are created for it, `ON CLUSTER` and a `Replicated*` engine on a cluster.
+ * In a Replicated database they are created for that database whatever it says.
  */
 export function clickhouseReceiptStore(
   endpoint: ClickHouseEndpoint,
   identity: { stack?: string; env?: string },
-  opts: { runId?: string; replicatedIn?: string; replicaTimeoutMs?: number } = {},
+  opts: { runId?: string; replicatedIn?: string; replicaTimeoutMs?: number; topology?: Topology } = {},
 ): ClickHouseReceiptStore {
   const where = receiptsTable(opts.replicatedIn);
+  const topology: Topology | undefined = opts.replicatedIn ? { kind: "replicated" } : opts.topology;
   const TABLE = `${ident(where.database)}.${ident(where.name)}`;
   let ensured = false;
   const ensure = async () => {
     if (ensured) return;
-    if (!opts.replicatedIn) await clickhouseQuery(endpoint, `CREATE DATABASE IF NOT EXISTS ${ident(RECEIPTS_DATABASE)} COMMENT ${sqlString(COMMENT)}`);
+    if (!opts.replicatedIn) {
+      await clickhouseQuery(endpoint, renderFor(`CREATE DATABASE IF NOT EXISTS ${ident(RECEIPTS_DATABASE)} COMMENT ${sqlString(COMMENT)}`, topology), { settings: DDL_SETTINGS });
+    }
     await clickhouseQuery(
       endpoint,
-      `CREATE TABLE IF NOT EXISTS ${TABLE} (address String, effect String, expectation String, run_id String, written_at DateTime64(3) DEFAULT now64(3)) ` +
-        `ENGINE = ${opts.replicatedIn ? "ReplicatedReplacingMergeTree" : "ReplacingMergeTree"}(written_at) ORDER BY address COMMENT ${sqlString(COMMENT)}`,
+      renderFor(
+        `CREATE TABLE IF NOT EXISTS ${TABLE} (address String, effect String, expectation String, run_id String, written_at DateTime64(3) DEFAULT now64(3)) ` +
+          `ENGINE = ReplacingMergeTree(written_at) ORDER BY address COMMENT ${sqlString(COMMENT)}`,
+        topology,
+      ),
       { settings: DDL_SETTINGS },
     );
     ensured = true;

package/src/clickhouse/rebuild/steps.ts

@@ -14,7 +14,9 @@ import type { ClickHouseTarget } from "../live/bind";
 import { carriesMarker, readTrailerPairs, REBUILD_TRAILER_KEY, stampedComment } from "../ownership";
 import { createStatement, ident, qualifiedIdent, renamedDeclaration, sqlString, type DeclaredObject } from "../apply/statements";
 import { waitForMutations } from "../apply/mutations";
+import { renderFor } from "../topology";
 import { DDL_SETTINGS, syncReplicaWithin } from "./replicas";
+import { assertSameRows } from "./verify";
 import {
   changesBetween,
   observeRebuild,
@@ -60,12 +62,14 @@ export const utcLiteral = (ms: number): string => `toDateTime64(${sqlString(sqlU
 export const observe = (run: RebuildRun): Promise<RebuildObservation> => observeRebuild(run.target, run.declared, run.marker);
 
 /**
- * One DDL statement. In a Replicated database it goes on without the
- * replicas that are down, which run it from the database's log when they
- * come back (`./replicas.ts`).
+ * One DDL statement, rendered for the environment's topology when it names
+ * one (`../topology.ts`: `ON CLUSTER` and the engine). In a Replicated
+ * database it goes on without the replicas that are down, which run it from
+ * the database's log when they come back (`./replicas.ts`).
  */
-const q = (run: RebuildRun, sql: string) => {
+const q = (run: RebuildRun, statement: string) => {
   run.signal?.throwIfAborted();
+  const sql = renderFor(statement, run.target.topology);
   run.log(sql);
   return clickhouseQuery(run.target.endpoint, sql, { settings: DDL_SETTINGS, ...(run.signal ? { signal: run.signal } : {}) });
 };
@@ -152,7 +156,8 @@ export interface DualWriteResult {
  * old table that writes every row at or after the cut-over into the new
  * table. The cut-over is the server's time now plus the delay, kept in the
  * view's own comment (`cutover=`), so a later run and the backfill read it
- * back from the server. App mode creates nothing; its gate is the Op's.
+ * back from the server. The rows at or after the cut-over that the old table
+ * already held are the backfill's (`./backfill.ts`). App mode creates nothing; its gate is the Op's.
  */
 export async function startDualWrite(run: RebuildRun): Promise<DualWriteResult> {
   const o = await observe(run);
@@ -169,7 +174,10 @@ export async function startDualWrite(run: RebuildRun): Promise<DualWriteResult>
         `(copied: ${o.copied.map((c) => c.name).join(", ")})`,
     );
   }
-  const cutover = (await serverNow(run.target)) + parseDuration(run.dualWrite.cutoverDelay ?? "1m");
+  // On a whole second: a DateTime column holds whole seconds, and a row written
+  // during the cut-over's second would otherwise carry a time before it and
+  // reach the old table after the backfill has started.
+  const cutover = Math.ceil(((await serverNow(run.target)) + parseDuration(run.dualWrite.cutoverDelay ?? "1m")) / 1000) * 1000;
   const iso = new Date(cutover).toISOString();
   const select = o.copied.map((c) => (c.source === c.name ? ident(c.name) : `${ident(c.source)} AS ${ident(c.name)}`)).join(", ");
   const comment = stampedComment(`chant rebuild of ${o.names.key}: rows at or after the cut-over`, run.marker, working(o.names.key, "dual", { cutover: iso }));
@@ -202,8 +210,9 @@ export interface SwapResult {
 }
 
 /**
- * Swap the new table in: `EXCHANGE TABLES`, then drop the dual-write view,
- * rename the old table (now under the new table's name) to `t__chant_old`
+ * Swap the new table in: compare every row of the two tables once more, as
+ * the verification does, and stop on any difference; then `EXCHANGE TABLES`,
+ * drop the dual-write view, rename the old table (now under the new table's name) to `t__chant_old`
  * with its retention date, recreate the materialized views that read the
  * table, and stamp the table's own comment as declared.
  *
@@ -235,6 +244,10 @@ export async function swapTables(run: RebuildRun): Promise<SwapResult> {
   }
   if (!o.exchanged) {
     if (!o.newTable) throw new RebuildRefusal(`${n.key}: there is no new table to swap in`);
+    // The new table holds every row the old one does, or nothing is swapped
+    // (sql-yodeler#45): rows that reached the old table alone since the
+    // verification would stay behind in it.
+    await assertSameRows(run, o);
     await q(run, `EXCHANGE TABLES ${n.table} AND ${n.newTable}`);
   }
   // The dual-write view now reads the new table by name; drop it before
@@ -373,7 +386,7 @@ export async function compensate(run: Pick<RebuildRun, "target" | "declared" | "
     const row = rows.find((r) => r.name === name);
     const pairs = readTrailerPairs(row?.comment);
     if (!row || pairs?.get(REBUILD_TRAILER_KEY) !== n.key || pairs.get("role") !== role || !carriesMarker(row.comment, run.marker)) continue;
-    const sql = `DROP ${what} ${qualifiedIdent(n.database, name)} SYNC`;
+    const sql = renderFor(`DROP ${what} ${qualifiedIdent(n.database, name)} SYNC`, run.target.topology);
     run.log(sql);
     await clickhouseQuery(run.target.endpoint, sql, { settings: DDL_SETTINGS });
     dropped.push(`${n.database}.${name}`);

package/src/clickhouse/rebuild/verify.ts

@@ -1,20 +1,36 @@
 /**
- * The verification (#3198): row counts and checksums per partition, the old
- * table against the new one, before anyone is asked to approve the swap.
+ * The verification (#3198, INTENTIUS/sql-yodeler#45): row counts and
+ * checksums per partition, the old table against the new one, before anyone
+ * is asked to approve the swap.
  *
  * Per partition of the old table: the number of rows and the sum of a
  * 64-bit hash of each row's copied columns, read in one grouped query on
  * each side. The old side casts each column to its type in the new table
  * first, so a column whose type the rebuild changes hashes the same when
- * its values survived the change. In materialized-view mode both sides are
- * limited to the rows before the cut-over, which are the backfill's; the
- * rows after it were written by the dual-write view and keep arriving.
+ * its values survived the change.
+ *
+ * Every row is compared, in both modes. In materialized-view mode the rows
+ * at or after the cut-over are in the new table twice over: those the old
+ * table held before the view existed, which the backfill copies, and those
+ * written since, which the view writes. Comparing only the rows before the
+ * cut-over let a swap go ahead without the first kind (sql-yodeler#45).
+ * Writes go on while the tables are read, and a row the old table has
+ * committed may be a moment from committing in the new one through the view,
+ * so the old table is read first and a difference is read again a few times
+ * before it counts.
+ *
+ * The same comparison runs again in the Swap step just before the
+ * `EXCHANGE`, so rows that reach the old table alone after this step (a row
+ * later than the cut-over delay, with a time before the cut-over) stop the
+ * swap too, rather than stay behind in the old table.
  *
  * Any difference fails the run, naming the partitions; the run's onFailure
  * then drops the new table. On a match the result carries the digest the
- * swap gate binds: the rebuild's plan and the verification together, so an
- * approval is for this plan with these counts, and a changed count needs a
- * new approval.
+ * swap gate binds: the rebuild's plan and the counts and checksums of the
+ * rows before the cut-over, which no later write changes, so an approval
+ * is for this plan with these counts, and a changed count needs a new
+ * approval. The rows at or after the cut-over keep arriving through the view
+ * and are compared, not bound.
  */
 
 import { computePlanDigest } from "@intentius/chant/op";
@@ -34,10 +50,13 @@ export interface VerifyResult {
   state: RebuildObservation["state"];
   /** The digest the swap gate binds: plan plus verification. Absent once the swap has run: there is nothing left for that gate to approve. */
   planDigest?: string;
+  /** Partitions of the old table compared. */
   partitions: number;
+  /** Rows compared: every row of the old table. */
   rows: number;
   /** One line for the run record and the gate. */
   summary: string;
+  /** Per partition, the rows before the cut-over (every row in app mode): what the swap gate binds. */
   verification: PartitionCheck[];
 }
 
@@ -45,7 +64,7 @@ export interface VerifyResult {
 export class RebuildVerificationError extends Error {
   constructor(
     readonly table: string,
-    readonly mismatches: Array<{ partition: string; old?: PartitionCheck; new?: PartitionCheck }>,
+    readonly mismatches: Mismatch[],
   ) {
     const shown = mismatches
       .slice(0, 10)
@@ -53,23 +72,38 @@ export class RebuildVerificationError extends Error {
       .join("; ");
     super(
       `${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. ` +
-        `Nothing was swapped. In materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, is the usual cause.`,
+        `Nothing was swapped. In materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, is the usual cause; ` +
+        `in app mode, a write the application did not stop.`,
     );
     this.name = "RebuildVerificationError";
   }
 }
 
-export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
-  const o = await observe(run);
-  if (o.state !== "rebuild") return { state: o.state, partitions: 0, rows: 0, verification: [], summary: `nothing to verify: ${o.state}` };
+/** How often a difference is read again before it counts, and how long apart. */
+const COMPARE_ATTEMPTS = 5;
+const COMPARE_INTERVAL_MS = 1000;
+
+/** One side's counts per old partition: every row, and the rows before the cut-over. */
+interface SideCounts {
+  all: Map<string, PartitionCheck>;
+  before: Map<string, PartitionCheck>;
+}
+
+/**
+ * Read both tables' counts and checksums per old partition, the old table
+ * first, until they agree or `attempts` reads have disagreed. Returns the
+ * last read and the partitions that differ in it.
+ */
+async function compareTables(
+  run: RebuildRun,
+  o: RebuildObservation,
+  opts: { attempts?: number; intervalMs?: number } = {},
+): Promise<{ old: SideCounts; new: SideCounts; mismatches: Mismatch[] }> {
   if (!o.newTable) throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
   const n = o.names;
   const cutover = cutoverOf(run, o);
   await waitOn(run, n.database, n.name);
   await waitOn(run, n.database, n.newName);
-  // In a Replicated database: count what every replica wrote, not what this one has fetched so far.
-  await syncReplica(run, o, n.database, n.name);
-  await syncReplica(run, o, n.database, n.newName);
 
   const types = new Map(
     (
@@ -79,35 +113,81 @@ export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
       )
     ).map((c) => [c.name, c.type]),
   );
-  const oldHash = `sum(cityHash64(toString(tuple(${o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`).join(", ")}))))`;
-  const newHash = `sum(cityHash64(toString(tuple(${o.copied.map((c) => ident(c.name)).join(", ")}))))`;
+  const oldHash = rowHash(o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`));
+  const newHash = rowHash(o.copied.map((c) => ident(c.name)));
   const cut = run.dualWrite.mode === "materialized-view" ? o.copied.find((c) => c.name === (run.dualWrite as { cutoverColumn: string }).cutoverColumn) : undefined;
-  const oldWhere = cut && cutover !== undefined ? ` WHERE ${ident(cut.source)} < ${utcLiteral(cutover)}` : "";
-  const newWhere = cut && cutover !== undefined ? ` WHERE ${ident(cut.name)} < ${utcLiteral(cutover)}` : "";
+  const oldBefore = cut && cutover !== undefined ? `${ident(cut.source)} < ${utcLiteral(cutover)}` : "1";
+  const newBefore = cut && cutover !== undefined ? `${ident(cut.name)} < ${utcLiteral(cutover)}` : "1";
   const fromOld = sourcePartitionExpression(o.live.partitionKey, o.newTable.partitionKey, o.copied);
 
-  const read = async (sql: string) =>
-    new Map(
-      (await clickhouseQuery<{ p: string; rows: string | number; checksum: string }>(run.target.endpoint, sql)).map((r) => [
-        r.p,
-        { partition: r.p, rows: Number(r.rows), checksum: r.checksum },
-      ]),
+  const read = async (table: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
+    const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string }>(
+      run.target.endpoint,
+      `SELECT ${partition} AS p, count() AS rows, toString(sum(${hash})) AS checksum, countIf(${before}) AS brows, toString(sumIf(${hash}, ${before})) AS bchecksum ` +
+        `FROM ${table} GROUP BY p`,
     );
-  const before = await read(`SELECT _partition_id AS p, count() AS rows, toString(${oldHash}) AS checksum FROM ${n.table}${oldWhere} GROUP BY p`);
-  const after = await read(`SELECT ${fromOld} AS p, count() AS rows, toString(${newHash}) AS checksum FROM ${n.newTable}${newWhere} GROUP BY p`);
+    const side: SideCounts = { all: new Map(), before: new Map() };
+    for (const r of rows) {
+      side.all.set(r.p, { partition: r.p, rows: Number(r.rows), checksum: r.checksum });
+      // A partition with no row before the cut-over is left out, as a WHERE would leave it out.
+      if (Number(r.brows) > 0) side.before.set(r.p, { partition: r.p, rows: Number(r.brows), checksum: r.bchecksum });
+    }
+    return side;
+  };
+
+  const attempts = Math.max(1, opts.attempts ?? COMPARE_ATTEMPTS);
+  for (let attempt = 1; ; attempt++) {
+    // In a Replicated database: count what every replica wrote, not what this one has fetched so far.
+    await syncReplica(run, o, n.database, n.name);
+    await syncReplica(run, o, n.database, n.newName);
+    // The old table first: a row the view has yet to commit in the new one is read as missing there, never the other way round.
+    const before = await read(n.table, "_partition_id", oldHash, oldBefore);
+    const after = await read(n.newTable, fromOld, newHash, newBefore);
+    const mismatches = differences(before.all, after.all);
+    if (mismatches.length === 0 || attempt >= attempts) return { old: before, new: after, mismatches };
+    run.log(`-- ${n.key}: ${mismatches.length} partition(s) differ (${mismatches.map((m) => m.partition).slice(0, 5).join(", ")}); reading again (${attempt}/${attempts})`);
+    await sleep(opts.intervalMs ?? COMPARE_INTERVAL_MS, run.signal);
+  }
+}
+
+export type Mismatch = { partition: string; old?: PartitionCheck; new?: PartitionCheck };
 
-  const mismatches: Array<{ partition: string; old?: PartitionCheck; new?: PartitionCheck }> = [];
+function differences(before: Map<string, PartitionCheck>, after: Map<string, PartitionCheck>): Mismatch[] {
+  const out: Mismatch[] = [];
   for (const p of [...new Set([...before.keys(), ...after.keys()])].sort()) {
     const a = before.get(p);
     const b = after.get(p);
-    if (!a || !b || a.rows !== b.rows || a.checksum !== b.checksum) mismatches.push({ partition: p, ...(a ? { old: a } : {}), ...(b ? { new: b } : {}) });
+    if (!a || !b || a.rows !== b.rows || a.checksum !== b.checksum) out.push({ partition: p, ...(a ? { old: a } : {}), ...(b ? { new: b } : {}) });
   }
-  if (mismatches.length > 0) throw new RebuildVerificationError(n.key, mismatches);
+  return out;
+}
+
+/** The 64-bit hash of a row's copied columns, as the backfill and the verification compute it. */
+export const rowHash = (columns: readonly string[]): string => `cityHash64(toString(tuple(${columns.join(", ")})))`;
+
+/**
+ * Before the swap: the new table holds every row the old one does. Throws a
+ * {@link RebuildVerificationError} when it does not, so nothing is swapped.
+ */
+export async function assertSameRows(run: RebuildRun, o: RebuildObservation): Promise<void> {
+  const { mismatches } = await compareTables(run, o);
+  if (mismatches.length > 0) throw new RebuildVerificationError(o.names.key, mismatches);
+}
+
+export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
+  const o = await observe(run);
+  if (o.state !== "rebuild") return { state: o.state, partitions: 0, rows: 0, verification: [], summary: `nothing to verify: ${o.state}` };
+  const cutover = cutoverOf(run, o);
+  const compared = await compareTables(run, o);
+  if (compared.mismatches.length > 0) throw new RebuildVerificationError(o.names.key, compared.mismatches);
 
-  const verification = [...before.values()].sort((x, y) => x.partition.localeCompare(y.partition));
-  const rows = verification.reduce((s, v) => s + v.rows, 0);
-  const range = cutover !== undefined ? ` before the cut-over at ${new Date(cutover).toISOString()}` : "";
-  const summary = `${n.key}: ${verification.length} partition(s), ${rows} row(s)${range}, counts and checksums equal in the old and new tables`;
+  const all = [...compared.old.all.values()];
+  const rows = all.reduce((s, v) => s + v.rows, 0);
+  // What the approval binds: the rows before the cut-over (every row in app mode), which later writes do not change.
+  const verification = [...compared.old.before.values()].sort((x, y) => x.partition.localeCompare(y.partition));
+  const bound = verification.reduce((s, v) => s + v.rows, 0);
+  const range = cutover !== undefined ? ` (${bound} of them before the cut-over at ${new Date(cutover).toISOString()})` : "";
+  const summary = `${o.names.key}: ${all.length} partition(s), ${rows} row(s)${range}, counts and checksums equal in the old and new tables`;
   run.log(`-- ${summary}`);
   return {
     state: o.state,
@@ -116,9 +196,24 @@ export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
       cutover: cutover === undefined ? null : new Date(cutover).toISOString(),
       verification,
     }),
-    partitions: verification.length,
+    partitions: all.length,
     rows,
     summary,
     verification,
   };
 }
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    signal?.throwIfAborted();
+    const t = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(t);
+        reject(signal.reason);
+      },
+      { once: true },
+    );
+  });
+}

package/src/clickhouse/topology-replicated.e2e.test.ts

@@ -0,0 +1,125 @@
+/**
+ * The replicated topology against a Replicated database (#3645): two
+ * replicas of the pinned server, Keeper embedded in the first (the scratch
+ * cluster the Replicated rebuild runs on).
+ *
+ * The source is written for a single node: a plain database, MergeTree
+ * tables. Applied with `CLICKHOUSE_TOPOLOGY=replicated`, the database is a
+ * `Replicated` one and its tables `Replicated*MergeTree` with no path, so
+ * the second replica gets them from the database's log and rows written on
+ * one reach the other. Applied again on the other replica, the server's
+ * printed engines (with the default Keeper path filled in) compare equal to
+ * the declarations, and nothing is sent; a change applies there and reaches
+ * the first.
+ *
+ * The scratch cluster names no cluster in `remote_servers`, so the second
+ * replica joins the database with the `CREATE DATABASE` the applier sent to
+ * the first, as an operator would (`replicated:<cluster>` sends it `ON
+ * CLUSTER` instead). Needs Docker; skips cleanly without it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { dockerAvailable, startScratchCluster, type ScratchCluster } from "./container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "./http";
+import { clickhouseImage } from "../spec/pin";
+import { clickhouseApply } from "../op/activities/clickhouse-apply";
+import { renderStatement } from "./topology";
+
+const DB = "yodel_3645";
+const enabled = await dockerAvailable();
+let cluster: ScratchCluster | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-topology-repl-"));
+
+beforeAll(async () => {
+  if (!enabled) return;
+  cluster = await startScratchCluster(clickhouseImage(), { replicas: 2, namePrefix: "chant-sql-topology-repl" });
+}, 600_000);
+
+afterAll(async () => {
+  await cluster?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+interface Obj {
+  export: string;
+  type: string;
+  ddl: string;
+  dependsOn?: string[];
+}
+
+const writeBuild = (file: string, objects: Obj[]): string => {
+  const path = join(dir, file);
+  writeFileSync(path, JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects: objects.map((o) => ({ dependsOn: [], ...o })) }));
+  return path;
+};
+
+const objects = (extraColumn = ""): Obj[] => [
+  { export: "db", type: "ClickHouse::Database", ddl: `CREATE DATABASE ${DB}` },
+  {
+    export: "events",
+    type: "ClickHouse::Table",
+    dependsOn: ["db"],
+    ddl: `CREATE TABLE ${DB}.events (id UInt64, ts DateTime, kind String${extraColumn}) ENGINE = ReplacingMergeTree(ts) ORDER BY id`,
+  },
+  {
+    export: "byKind",
+    type: "ClickHouse::MaterializedView",
+    dependsOn: ["events"],
+    ddl: `CREATE MATERIALIZED VIEW ${DB}.by_kind ENGINE = SummingMergeTree ORDER BY kind AS SELECT kind, count() AS n FROM ${DB}.events GROUP BY kind`,
+  },
+];
+
+const q = <T = Record<string, unknown>>(at: ClickHouseEndpoint, sql: string) => clickhouseQuery<T>(at, sql);
+
+async function apply(at: ClickHouseEndpoint, buildPath: string) {
+  const sent: string[] = [];
+  const outcome = await clickhouseApply({ buildPath, environment: "test" }, undefined, {
+    config: { ownership: { stack: "yodel", env: "e2e" } },
+    env: { CLICKHOUSE_URL: at.url, CLICKHOUSE_TOPOLOGY: "replicated" },
+    log: (l) => void (/^[A-Z]+ /.test(l) && sent.push(l)),
+  });
+  return { outcome, sent };
+}
+
+describe.skipIf(!enabled)("the replicated topology on a Replicated database (#3645)", () => {
+  test("a single node's source becomes a Replicated database that both replicas hold, and converges on either", async () => {
+    const [r1, r2] = cluster!.replicas as [ClickHouseEndpoint, ClickHouseEndpoint];
+    expect(renderStatement(`CREATE DATABASE ${DB}`, { kind: "replicated" })).toBe(`CREATE DATABASE ${DB} ENGINE = Replicated('/clickhouse/databases/${DB}', '{shard}', '{replica}')`);
+
+    const v1 = writeBuild("v1.json", objects());
+    const first = await apply(r1, v1);
+    expect(first.outcome.failed).toEqual([]);
+    // The second replica joins the database with the statement the applier sent to the first, and replays its tables from the log.
+    expect(first.sent[0]).toMatch(new RegExp(`^CREATE DATABASE ${DB} ENGINE = Replicated\\('/clickhouse/databases/${DB}', '\\{shard\\}', '\\{replica\\}'\\) COMMENT `));
+    await q(r2, first.sent[0]!);
+    await q(r2, `SYSTEM SYNC DATABASE REPLICA ${DB}`);
+    expect(first.sent.join("\n")).not.toMatch(/ON CLUSTER/);
+    expect(first.sent[1]).toMatch(/ENGINE = ReplicatedReplacingMergeTree\(ts\) ORDER BY id/);
+
+    await q(r1, `INSERT INTO ${DB}.events (id, ts, kind) VALUES (1, now(), 'click'), (2, now(), 'view')`);
+    await q(r2, `SYSTEM SYNC REPLICA ${DB}.events`);
+    expect(Number((await q<{ n: string | number }>(r2, `SELECT count() AS n FROM ${DB}.events`))[0]!.n)).toBe(2);
+    const engines = await q<{ name: string; engine: string }>(r2, `SELECT name, engine FROM system.tables WHERE database = '${DB}' ORDER BY name`);
+    expect(engines.map((t) => [t.name.startsWith(".inner") ? ".inner" : t.name, t.engine])).toEqual([
+      [".inner", "ReplicatedSummingMergeTree"],
+      ["by_kind", "MaterializedView"],
+      ["events", "ReplicatedReplacingMergeTree"],
+    ]);
+    expect((await q<{ engine: string }>(r2, `SELECT engine FROM system.databases WHERE name = '${DB}'`))[0]!.engine).toBe("Replicated");
+
+    // On the other replica: the printed engines (default Keeper path filled in) are the declarations; nothing to send.
+    const again = await apply(r2, v1);
+    expect(again.sent).toEqual([]);
+    expect(again.outcome.notAttempted).toEqual([]);
+
+    // A change there reaches the first replica through the database's log.
+    const change = await apply(r2, writeBuild("v2.json", objects(", source String DEFAULT 'web'")));
+    expect(change.sent).toEqual([`ALTER TABLE \`${DB}\`.\`events\` ADD COLUMN source String DEFAULT 'web' AFTER \`kind\``]);
+    await q(r1, `SYSTEM SYNC DATABASE REPLICA ${DB}`);
+    const cols = await q<{ name: string }>(r1, `SELECT name FROM system.columns WHERE database = '${DB}' AND table = 'events' ORDER BY position`);
+    expect(cols.map((c) => c.name)).toEqual(["id", "ts", "kind", "source"]);
+  }, 300_000);
+});