@intentius/chant-lexicon-sql 0.114.0 → 0.115.0

package/src/clickhouse/rebuild/rebuild.test.ts

@@ -20,6 +20,8 @@ import * as contractsModule from "../../op/activity-contracts";
 import * as activitiesModule from "../../op/activities";
 import { ClickHouseRebuildOp, type ClickHouseRebuildOpConfig } from "./op";
 import { copiedColumns } from "./observe";
+import { partitionEffect } from "./backfill";
+import { intoShard, onShard, onShardServers, shardTable } from "./shards";
 import { renameColumns, sourcePartitionExpression } from "./partitions";
 import { canonicalObject } from "../plan/normalize";
 import { diffSchemas } from "../plan/diff";
@@ -79,7 +81,37 @@ describe("ClickHouseRebuildOp", () => {
     expect(Object.keys(gate.approval.context)).toEqual(["verifiedPartitions", "verifiedRows"]);
   });
 
+  test('gates: "outer" leaves out both approval gates and the Drop phase, and keeps the verification (#3658)', () => {
+    const props = propsOf({ ...BASE, gates: "outer" });
+    expect(props.phases.map((p) => p.name)).toEqual(["Build", "Plan", "Create", "Dual write", "Backfill", "Verify", "Swap", "Retain"]);
+    expect(props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "gate")).toEqual([]);
+    const fns = props.phases.flatMap((p) => p.steps).map((s) => (s as { fn?: string }).fn);
+    expect(fns.indexOf("clickhouseRebuildVerify")).toBeLessThan(fns.indexOf("clickhouseRebuildSwap"));
+    expect(fns).not.toContain("clickhouseRebuildDrop");
+    // Its own onFailure is unchanged.
+    expect(props.onFailure?.map((p) => p.steps.map((s) => (s as { fn?: string }).fn))).toEqual([["clickhouseRebuildCompensate"]]);
+    expect(props.overview).toContain("the caller's approval");
+  });
+
+  test('gates: "outer" in app mode keeps the writes-stopped gate, which is not an approval (#3658)', () => {
+    const props = propsOf({ ...BASE, dualWrite: { mode: "app" }, gates: "outer" });
+    const gates = props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "gate") as Array<{ gate: string }>;
+    expect(gates.map((g) => g.gate)).toEqual(["rebuild-events-writes-stopped"]);
+  });
+
+  test('onFailure: "keep" has no onFailure and tells every step so (#3658)', () => {
+    const props = propsOf({ ...BASE, onFailure: "keep" });
+    expect(props.onFailure).toBeUndefined();
+    const steps = props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "activity" && (s as { fn: string }).fn.startsWith("clickhouseRebuild")) as Array<{ args: Record<string, unknown> }>;
+    expect(steps.length).toBeGreaterThan(0);
+    for (const s of steps) expect(s.args.keepOnFailure).toBe(true);
+    expect(props.phases.map((p) => p.name)).toContain("Approve");
+    expect((propsOf(BASE).phases[1]!.steps[0] as { args: Record<string, unknown> }).args).not.toHaveProperty("keepOnFailure");
+  });
+
   test("bad configuration fails when the Op is built", () => {
+    expect(() => ClickHouseRebuildOp({ ...BASE, gates: "none" as never })).toThrow(/gates must be/);
+    expect(() => ClickHouseRebuildOp({ ...BASE, onFailure: "retry" as never })).toThrow(/onFailure must be/);
     expect(() => ClickHouseRebuildOp({ ...BASE, table: "shop.events; DROP" })).toThrow(/table must be/);
     expect(() => ClickHouseRebuildOp({ ...BASE, dualWrite: { mode: "materialized-view", cutoverColumn: "" } })).toThrow(/cutoverColumn/);
     expect(() => ClickHouseRebuildOp({ ...BASE, dualWrite: { mode: "both" } as never })).toThrow(/dualWrite\.mode/);
@@ -89,19 +121,23 @@ describe("ClickHouseRebuildOp", () => {
     const contracts = new Map<string, ActivityContract>();
     collectActivityContracts(contractsModule as unknown as Record<string, unknown>, contracts);
     const merged = mergeActivityContracts(await loadActivityContracts([]), contracts);
-    for (const config of [BASE, { ...BASE, dualWrite: { mode: "app" as const }, retain: "3d", replicaTimeout: "30s", stack: "shop", ownershipEnv: "prod" }]) {
+    for (const config of [
+      BASE,
+      { ...BASE, dualWrite: { mode: "app" as const }, retain: "3d", replicaTimeout: "30s", stack: "shop", ownershipEnv: "prod" },
+      { ...BASE, gates: "outer" as const, onFailure: "keep" as const },
+    ]) {
       const props = propsOf(config);
       expect(validateActivitySteps(props, merged)).toEqual([]);
       expect(validateStepOutputRefs(props, merged)).toEqual([]);
     }
   });
 
-  test("every rebuild step is an activity the sql lexicon exports, and none is a receipt activity", () => {
+  test("every rebuild step is an activity the sql lexicon exports, and the receipt activities are only fallbacks", () => {
     const exported = new Set(Object.entries(activitiesModule).filter(([, v]) => typeof v === "function").map(([k]) => k));
     const steps = propsOf(BASE).phases.flatMap((p) => p.steps).filter((s) => s.kind === "activity" && (s as { fn: string }).fn.startsWith("clickhouseRebuild"));
     for (const s of steps) expect(exported.has((s as { fn: string }).fn)).toBe(true);
-    expect(exported.has("receiptRead")).toBe(false);
-    expect(exported.has("receiptWrite")).toBe(false);
+    // The receipt activities are exported for effect() steps only as fallbacks (#3657), so they never take over another lexicon's receipt row.
+    expect((activitiesModule as { ACTIVITY_FALLBACKS?: readonly string[] }).ACTIVITY_FALLBACKS).toEqual(["receiptRead", "receiptWrite", "receiptStaleness"]);
   });
 });
 
@@ -188,3 +224,19 @@ describe("the plan hands a rebuild to the Op", () => {
     expect(rebuildOpSuggestions(diff, new Map(after.map((o) => [o.key, o.canonical])), "<env>")[0]!.dualWrite).toEqual({ mode: "app" });
   });
 });
+
+describe("a rebuild across the shards of a cluster (#3663)", () => {
+  const sharding = { cluster: "main", shards: [{ num: 1, macro: "s1", slot: 0 }, { num: 2, macro: "s2", slot: 1 }] };
+  const shard2 = sharding.shards[1]!;
+
+  test("each (shard, partition) is its own receipt; one shard keeps the address it always had", () => {
+    expect(partitionEffect("shop.events", "202602")).toBe("rebuild/shop.events/202602");
+    expect(partitionEffect("shop.events", "202602", shard2)).toBe("rebuild/shop.events/shard2/202602");
+  });
+
+  test("a shard is read through the cluster, written back with its own sharding key, and cleared by its own macro", () => {
+    expect(`SELECT * FROM ${shardTable(sharding, "shop", "events")} WHERE ${onShard(shard2)}`).toBe("SELECT * FROM cluster('main', `shop`, `events`) WHERE _shard_num = 2");
+    expect(intoShard(sharding, shard2, "shop", "events__chant_new")).toBe("FUNCTION cluster('main', `shop`, `events__chant_new`, 1)");
+    expect(onShardServers(shard2)).toBe("getMacro('shard') = 's2'");
+  });
+});

package/src/clickhouse/rebuild/receipts.ts

@@ -21,12 +21,10 @@
  *
  * The store implements core's `ReceiptStore` seam
  * (`@intentius/chant/op/receipt-store`), as the sql lexicon's shared
- * `SqlReceiptStore` (`../../core/receipts.ts`). It is not exported as the
- * `receiptRead` / `receiptWrite` activities: those names are global to a
- * run, and a project that also configures aws or k8s would have its
- * `effect()` steps read ClickHouse instead of SSM or a ConfigMap. The backfill
- * activity reaches the store directly, the way the fly lexicon's release
- * component reaches its own.
+ * `SqlReceiptStore` (`../../core/receipts.ts`). The backfill activity
+ * reaches the store directly, the way the fly lexicon's release component
+ * reaches its own. The same table holds the receipts of a sql project's
+ * `effect()` steps (`../../receipts.ts`), under their own addresses.
  *
  * Write discipline: the backfill writes a partition's receipt after the
  * partition's copy succeeded, last, and nothing else writes one.

package/src/clickhouse/rebuild/shards.ts

@@ -0,0 +1,104 @@
+/**
+ * A rebuild across the shards of a cluster (#3663).
+ *
+ * On a `cluster:<name>` topology the rebuild's DDL runs `ON CLUSTER`, so
+ * every server gets the new table, the dual-write view and the swap. The
+ * rows do not move that way: each shard holds its own, and a copy run on
+ * the profile's server alone copies that server's shard and no other. So
+ * with more than one shard, the backfill and the verification work shard by
+ * shard, all through the profile's server:
+ *
+ * - a shard's rows are read with `cluster('<name>', db, t)` and
+ *   `_shard_num = <n>`, one replica per shard, as the cluster's own
+ *   configuration reaches it;
+ * - they are written back to the same shard with
+ *   `INSERT INTO FUNCTION cluster('<name>', db, t__chant_new, <slot>)`, a
+ *   constant sharding key that lands on that shard, inserted in the
+ *   foreground so the statement returns once the shard has the rows;
+ * - a shard's half-copied rows are deleted with an `ALTER ... ON CLUSTER`
+ *   whose condition holds on that shard's servers only
+ *   (`getMacro('shard')`, the macro the cluster's Keeper paths already use).
+ *
+ * Each (shard, partition) is one copy with its own receipt, so a rerun after
+ * a failure on one shard skips what every shard already copied.
+ *
+ * With one shard nothing changes: the replicas of a shard replicate what is
+ * written on any of them, which is what the rebuild has always relied on.
+ */
+
+import { clickhouseQuery } from "../http";
+import type { ClickHouseTarget } from "../live/bind";
+import { ident, sqlString } from "../apply/statements";
+import { RebuildRefusal } from "./observe";
+
+export interface Shard {
+  /** `_shard_num` / `system.clusters.shard_num`, from 1. */
+  num: number;
+  /** The shard's `{shard}` macro, the same on each of its replicas. */
+  macro: string;
+  /** The sharding key that lands on this shard: the sum of the weights of the shards before it. */
+  slot: number;
+}
+
+export interface Sharding {
+  cluster: string;
+  shards: Shard[];
+}
+
+/** Settings for a copy into one shard: in the foreground, and read from the cluster rather than pushed down to each shard. */
+export const SHARD_INSERT_SETTINGS: Record<string, string> = { distributed_foreground_insert: "1", parallel_distributed_insert_select: "0" };
+
+/** Settings for a delete on one shard: its condition is the server's own macro, the same on every replica of the shard. */
+export const SHARD_DELETE_SETTINGS: Record<string, string> = { allow_nondeterministic_mutations: "1", mutations_sync: "2" };
+
+/**
+ * The shards of the target's cluster, when its topology is `cluster:<name>`
+ * and the cluster has more than one. Undefined otherwise: the rebuild works
+ * on the profile's server as it always has.
+ */
+export async function clusterSharding(target: ClickHouseTarget): Promise<Sharding | undefined> {
+  if (target.topology?.kind !== "cluster") return undefined;
+  let cluster = target.topology.cluster;
+  const macro = /^\{(\w+)\}$/.exec(cluster);
+  if (macro) {
+    const [row] = await clickhouseQuery<{ c: string }>(target.endpoint, `SELECT getMacro(${sqlString(macro[1]!)}) AS c`);
+    cluster = row?.c ?? cluster;
+  }
+  const weights = await clickhouseQuery<{ shard: number | string; weight: number | string }>(
+    target.endpoint,
+    `SELECT shard_num AS shard, any(shard_weight) AS weight FROM system.clusters WHERE cluster = ${sqlString(cluster)} GROUP BY shard_num ORDER BY shard_num`,
+  );
+  if (weights.length <= 1) return undefined;
+  const macros = new Map(
+    (await clickhouseQuery<{ shard: number | string; macro: string }>(target.endpoint, `SELECT _shard_num AS shard, getMacro('shard') AS macro FROM cluster(${sqlString(cluster)}, system.one)`)).map(
+      (r) => [Number(r.shard), r.macro],
+    ),
+  );
+  let slot = 0;
+  const shards: Shard[] = [];
+  for (const w of weights) {
+    const num = Number(w.shard);
+    const m = macros.get(num);
+    if (m === undefined) throw new RebuildRefusal(`cluster ${cluster}: shard ${num} has no {shard} macro, which a rebuild across shards needs to delete a half-copied partition on that shard alone`);
+    shards.push({ num, macro: m, slot });
+    slot += Number(w.weight) || 1;
+  }
+  const distinct = new Set(shards.map((s) => s.macro));
+  if (distinct.size !== shards.length) {
+    throw new RebuildRefusal(`cluster ${cluster}: two shards share a {shard} macro (${shards.map((s) => `${s.num}=${s.macro}`).join(", ")}); each shard needs its own`);
+  }
+  return { cluster, shards };
+}
+
+/** One shard's rows of `database.table`, as a table expression; filter with {@link onShard}. */
+export const shardTable = (sharding: Sharding, database: string, table: string): string => `cluster(${sqlString(sharding.cluster)}, ${ident(database)}, ${ident(table)})`;
+
+/** The condition that keeps one shard's rows of a {@link shardTable}. */
+export const onShard = (shard: Shard): string => `_shard_num = ${shard.num}`;
+
+/** Where a copy into one shard of `database.table` writes. */
+export const intoShard = (sharding: Sharding, shard: Shard, database: string, table: string): string =>
+  `FUNCTION cluster(${sqlString(sharding.cluster)}, ${ident(database)}, ${ident(table)}, ${shard.slot})`;
+
+/** The condition, in an `ALTER ... ON CLUSTER` mutation, that holds on that shard's servers only. */
+export const onShardServers = (shard: Shard): string => `getMacro('shard') = ${sqlString(shard.macro)}`;

package/src/clickhouse/rebuild/steps.ts

@@ -16,6 +16,8 @@ import { createStatement, ident, qualifiedIdent, renamedDeclaration, sqlString,
 import { waitForMutations } from "../apply/mutations";
 import { renderFor } from "../topology";
 import { DDL_SETTINGS, syncReplicaWithin } from "./replicas";
+import { assertSameRows } from "./verify";
+import type { Sharding } from "./shards";
 import {
   changesBetween,
   observeRebuild,
@@ -44,6 +46,8 @@ export interface RebuildRun {
   signal?: AbortSignal;
   /** The run's id, recorded on the receipts it writes. */
   runId?: string;
+  /** On a cluster of more than one shard, its shards: the backfill and the verification work shard by shard (`./shards.ts`). */
+  sharding?: Sharding;
 }
 
 /** The server's clock, in milliseconds: the cut-over and the retention are the server's times, not this machine's. */
@@ -75,6 +79,7 @@ const q = (run: RebuildRun, statement: string) => {
 
 export async function waitOn(run: RebuildRun, database: string, table: string): Promise<void> {
   const ids = await waitForMutations(run.target.endpoint, database, table, {
+    ...(run.sharding ? { source: `clusterAllReplicas(${sqlString(run.sharding.cluster)}, system.mutations)` } : {}),
     ...(run.mutationTimeoutMs !== undefined ? { timeoutMs: run.mutationTimeoutMs } : {}),
     ...(run.signal ? { signal: run.signal } : {}),
   });
@@ -155,7 +160,8 @@ export interface DualWriteResult {
  * old table that writes every row at or after the cut-over into the new
  * table. The cut-over is the server's time now plus the delay, kept in the
  * view's own comment (`cutover=`), so a later run and the backfill read it
- * back from the server. App mode creates nothing; its gate is the Op's.
+ * back from the server. The rows at or after the cut-over that the old table
+ * already held are the backfill's (`./backfill.ts`). App mode creates nothing; its gate is the Op's.
  */
 export async function startDualWrite(run: RebuildRun): Promise<DualWriteResult> {
   const o = await observe(run);
@@ -172,7 +178,10 @@ export async function startDualWrite(run: RebuildRun): Promise<DualWriteResult>
         `(copied: ${o.copied.map((c) => c.name).join(", ")})`,
     );
   }
-  const cutover = (await serverNow(run.target)) + parseDuration(run.dualWrite.cutoverDelay ?? "1m");
+  // On a whole second: a DateTime column holds whole seconds, and a row written
+  // during the cut-over's second would otherwise carry a time before it and
+  // reach the old table after the backfill has started.
+  const cutover = Math.ceil(((await serverNow(run.target)) + parseDuration(run.dualWrite.cutoverDelay ?? "1m")) / 1000) * 1000;
   const iso = new Date(cutover).toISOString();
   const select = o.copied.map((c) => (c.source === c.name ? ident(c.name) : `${ident(c.source)} AS ${ident(c.name)}`)).join(", ");
   const comment = stampedComment(`chant rebuild of ${o.names.key}: rows at or after the cut-over`, run.marker, working(o.names.key, "dual", { cutover: iso }));
@@ -205,8 +214,9 @@ export interface SwapResult {
 }
 
 /**
- * Swap the new table in: `EXCHANGE TABLES`, then drop the dual-write view,
- * rename the old table (now under the new table's name) to `t__chant_old`
+ * Swap the new table in: compare every row of the two tables once more, as
+ * the verification does, and stop on any difference; then `EXCHANGE TABLES`,
+ * drop the dual-write view, rename the old table (now under the new table's name) to `t__chant_old`
  * with its retention date, recreate the materialized views that read the
  * table, and stamp the table's own comment as declared.
  *
@@ -238,6 +248,10 @@ export async function swapTables(run: RebuildRun): Promise<SwapResult> {
   }
   if (!o.exchanged) {
     if (!o.newTable) throw new RebuildRefusal(`${n.key}: there is no new table to swap in`);
+    // The new table holds every row the old one does, or nothing is swapped
+    // (sql-yodeler#45): rows that reached the old table alone since the
+    // verification would stay behind in it.
+    await assertSameRows(run, o);
     await q(run, `EXCHANGE TABLES ${n.table} AND ${n.newTable}`);
   }
   // The dual-write view now reads the new table by name; drop it before

package/src/clickhouse/rebuild/verify.ts

@@ -1,20 +1,40 @@
 /**
- * The verification (#3198): row counts and checksums per partition, the old
- * table against the new one, before anyone is asked to approve the swap.
+ * The verification (#3198, INTENTIUS/sql-yodeler#45): row counts and
+ * checksums per partition, the old table against the new one, before anyone
+ * is asked to approve the swap.
  *
  * Per partition of the old table: the number of rows and the sum of a
  * 64-bit hash of each row's copied columns, read in one grouped query on
  * each side. The old side casts each column to its type in the new table
  * first, so a column whose type the rebuild changes hashes the same when
- * its values survived the change. In materialized-view mode both sides are
- * limited to the rows before the cut-over, which are the backfill's; the
- * rows after it were written by the dual-write view and keep arriving.
+ * its values survived the change.
+ *
+ * Every row is compared, in both modes. In materialized-view mode the rows
+ * at or after the cut-over are in the new table twice over: those the old
+ * table held before the view existed, which the backfill copies, and those
+ * written since, which the view writes. Comparing only the rows before the
+ * cut-over let a swap go ahead without the first kind (sql-yodeler#45).
+ * Writes go on while the tables are read, and a row the old table has
+ * committed may be a moment from committing in the new one through the view,
+ * so the old table is read first and a difference is read again a few times
+ * before it counts.
+ *
+ * The same comparison runs again in the Swap step just before the
+ * `EXCHANGE`, so rows that reach the old table alone after this step (a row
+ * later than the cut-over delay, with a time before the cut-over) stop the
+ * swap too, rather than stay behind in the old table.
+ *
+ * On a cluster of more than one shard (#3663) every shard is read, through
+ * the profile's server, and each partition is compared per shard
+ * (`shard<n>/<partition>`), so a shard the copy missed is a difference.
  *
  * Any difference fails the run, naming the partitions; the run's onFailure
  * then drops the new table. On a match the result carries the digest the
- * swap gate binds: the rebuild's plan and the verification together, so an
- * approval is for this plan with these counts, and a changed count needs a
- * new approval.
+ * swap gate binds: the rebuild's plan and the counts and checksums of the
+ * rows before the cut-over, which no later write changes, so an approval
+ * is for this plan with these counts, and a changed count needs a new
+ * approval. The rows at or after the cut-over keep arriving through the view
+ * and are compared, not bound.
  */
 
 import { computePlanDigest } from "@intentius/chant/op";
@@ -23,6 +43,7 @@ import { ident, sqlString } from "../apply/statements";
 import { rebuildPlanSubject, RebuildRefusal, type RebuildObservation } from "./observe";
 import { sourcePartitionExpression } from "./partitions";
 import { cutoverOf, observe, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
+import { shardTable } from "./shards";
 
 export interface PartitionCheck {
   partition: string;
@@ -34,10 +55,13 @@ export interface VerifyResult {
   state: RebuildObservation["state"];
   /** The digest the swap gate binds: plan plus verification. Absent once the swap has run: there is nothing left for that gate to approve. */
   planDigest?: string;
+  /** Partitions of the old table compared. */
   partitions: number;
+  /** Rows compared: every row of the old table. */
   rows: number;
   /** One line for the run record and the gate. */
   summary: string;
+  /** Per partition, the rows before the cut-over (every row in app mode): what the swap gate binds. */
   verification: PartitionCheck[];
 }
 
@@ -45,7 +69,7 @@ export interface VerifyResult {
 export class RebuildVerificationError extends Error {
   constructor(
     readonly table: string,
-    readonly mismatches: Array<{ partition: string; old?: PartitionCheck; new?: PartitionCheck }>,
+    readonly mismatches: Mismatch[],
   ) {
     const shown = mismatches
       .slice(0, 10)
@@ -53,23 +77,38 @@ export class RebuildVerificationError extends Error {
       .join("; ");
     super(
       `${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. ` +
-        `Nothing was swapped. In materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, is the usual cause.`,
+        `Nothing was swapped. In materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, is the usual cause; ` +
+        `in app mode, a write the application did not stop.`,
     );
     this.name = "RebuildVerificationError";
   }
 }
 
-export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
-  const o = await observe(run);
-  if (o.state !== "rebuild") return { state: o.state, partitions: 0, rows: 0, verification: [], summary: `nothing to verify: ${o.state}` };
+/** How often a difference is read again before it counts, and how long apart. */
+const COMPARE_ATTEMPTS = 5;
+const COMPARE_INTERVAL_MS = 1000;
+
+/** One side's counts per old partition: every row, and the rows before the cut-over. */
+interface SideCounts {
+  all: Map<string, PartitionCheck>;
+  before: Map<string, PartitionCheck>;
+}
+
+/**
+ * Read both tables' counts and checksums per old partition, the old table
+ * first, until they agree or `attempts` reads have disagreed. Returns the
+ * last read and the partitions that differ in it.
+ */
+async function compareTables(
+  run: RebuildRun,
+  o: RebuildObservation,
+  opts: { attempts?: number; intervalMs?: number } = {},
+): Promise<{ old: SideCounts; new: SideCounts; mismatches: Mismatch[] }> {
   if (!o.newTable) throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
   const n = o.names;
   const cutover = cutoverOf(run, o);
   await waitOn(run, n.database, n.name);
   await waitOn(run, n.database, n.newName);
-  // In a Replicated database: count what every replica wrote, not what this one has fetched so far.
-  await syncReplica(run, o, n.database, n.name);
-  await syncReplica(run, o, n.database, n.newName);
 
   const types = new Map(
     (
@@ -79,35 +118,85 @@ export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
       )
     ).map((c) => [c.name, c.type]),
   );
-  const oldHash = `sum(cityHash64(toString(tuple(${o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`).join(", ")}))))`;
-  const newHash = `sum(cityHash64(toString(tuple(${o.copied.map((c) => ident(c.name)).join(", ")}))))`;
+  const oldHash = rowHash(o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`));
+  const newHash = rowHash(o.copied.map((c) => ident(c.name)));
   const cut = run.dualWrite.mode === "materialized-view" ? o.copied.find((c) => c.name === (run.dualWrite as { cutoverColumn: string }).cutoverColumn) : undefined;
-  const oldWhere = cut && cutover !== undefined ? ` WHERE ${ident(cut.source)} < ${utcLiteral(cutover)}` : "";
-  const newWhere = cut && cutover !== undefined ? ` WHERE ${ident(cut.name)} < ${utcLiteral(cutover)}` : "";
+  const oldBefore = cut && cutover !== undefined ? `${ident(cut.source)} < ${utcLiteral(cutover)}` : "1";
+  const newBefore = cut && cutover !== undefined ? `${ident(cut.name)} < ${utcLiteral(cutover)}` : "1";
   const fromOld = sourcePartitionExpression(o.live.partitionKey, o.newTable.partitionKey, o.copied);
 
-  const read = async (sql: string) =>
-    new Map(
-      (await clickhouseQuery<{ p: string; rows: string | number; checksum: string }>(run.target.endpoint, sql)).map((r) => [
-        r.p,
-        { partition: r.p, rows: Number(r.rows), checksum: r.checksum },
-      ]),
+  // Across the shards of a cluster (#3663), every shard's rows, each partition counted per shard as `shard<n>/<partition>`.
+  const sharding = run.sharding;
+  const read = async (table: string, name: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
+    const from = sharding ? shardTable(sharding, n.database, name) : table;
+    const key = sharding ? `concat('shard', toString(_shard_num), '/', ${partition})` : partition;
+    const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string }>(
+      run.target.endpoint,
+      `SELECT ${key} AS p, count() AS rows, toString(sum(${hash})) AS checksum, countIf(${before}) AS brows, toString(sumIf(${hash}, ${before})) AS bchecksum ` +
+        `FROM ${from} GROUP BY p`,
     );
-  const before = await read(`SELECT _partition_id AS p, count() AS rows, toString(${oldHash}) AS checksum FROM ${n.table}${oldWhere} GROUP BY p`);
-  const after = await read(`SELECT ${fromOld} AS p, count() AS rows, toString(${newHash}) AS checksum FROM ${n.newTable}${newWhere} GROUP BY p`);
+    const side: SideCounts = { all: new Map(), before: new Map() };
+    for (const r of rows) {
+      side.all.set(r.p, { partition: r.p, rows: Number(r.rows), checksum: r.checksum });
+      // A partition with no row before the cut-over is left out, as a WHERE would leave it out.
+      if (Number(r.brows) > 0) side.before.set(r.p, { partition: r.p, rows: Number(r.brows), checksum: r.bchecksum });
+    }
+    return side;
+  };
+
+  const attempts = Math.max(1, opts.attempts ?? COMPARE_ATTEMPTS);
+  for (let attempt = 1; ; attempt++) {
+    // In a Replicated database: count what every replica wrote, not what this one has fetched so far.
+    await syncReplica(run, o, n.database, n.name);
+    await syncReplica(run, o, n.database, n.newName);
+    // The old table first: a row the view has yet to commit in the new one is read as missing there, never the other way round.
+    const before = await read(n.table, n.name, "_partition_id", oldHash, oldBefore);
+    const after = await read(n.newTable, n.newName, fromOld, newHash, newBefore);
+    const mismatches = differences(before.all, after.all);
+    if (mismatches.length === 0 || attempt >= attempts) return { old: before, new: after, mismatches };
+    run.log(`-- ${n.key}: ${mismatches.length} partition(s) differ (${mismatches.map((m) => m.partition).slice(0, 5).join(", ")}); reading again (${attempt}/${attempts})`);
+    await sleep(opts.intervalMs ?? COMPARE_INTERVAL_MS, run.signal);
+  }
+}
 
-  const mismatches: Array<{ partition: string; old?: PartitionCheck; new?: PartitionCheck }> = [];
+export type Mismatch = { partition: string; old?: PartitionCheck; new?: PartitionCheck };
+
+function differences(before: Map<string, PartitionCheck>, after: Map<string, PartitionCheck>): Mismatch[] {
+  const out: Mismatch[] = [];
   for (const p of [...new Set([...before.keys(), ...after.keys()])].sort()) {
     const a = before.get(p);
     const b = after.get(p);
-    if (!a || !b || a.rows !== b.rows || a.checksum !== b.checksum) mismatches.push({ partition: p, ...(a ? { old: a } : {}), ...(b ? { new: b } : {}) });
+    if (!a || !b || a.rows !== b.rows || a.checksum !== b.checksum) out.push({ partition: p, ...(a ? { old: a } : {}), ...(b ? { new: b } : {}) });
   }
-  if (mismatches.length > 0) throw new RebuildVerificationError(n.key, mismatches);
+  return out;
+}
+
+/** The 64-bit hash of a row's copied columns, as the backfill and the verification compute it. */
+export const rowHash = (columns: readonly string[]): string => `cityHash64(toString(tuple(${columns.join(", ")})))`;
+
+/**
+ * Before the swap: the new table holds every row the old one does. Throws a
+ * {@link RebuildVerificationError} when it does not, so nothing is swapped.
+ */
+export async function assertSameRows(run: RebuildRun, o: RebuildObservation): Promise<void> {
+  const { mismatches } = await compareTables(run, o);
+  if (mismatches.length > 0) throw new RebuildVerificationError(o.names.key, mismatches);
+}
 
-  const verification = [...before.values()].sort((x, y) => x.partition.localeCompare(y.partition));
-  const rows = verification.reduce((s, v) => s + v.rows, 0);
-  const range = cutover !== undefined ? ` before the cut-over at ${new Date(cutover).toISOString()}` : "";
-  const summary = `${n.key}: ${verification.length} partition(s), ${rows} row(s)${range}, counts and checksums equal in the old and new tables`;
+export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
+  const o = await observe(run);
+  if (o.state !== "rebuild") return { state: o.state, partitions: 0, rows: 0, verification: [], summary: `nothing to verify: ${o.state}` };
+  const cutover = cutoverOf(run, o);
+  const compared = await compareTables(run, o);
+  if (compared.mismatches.length > 0) throw new RebuildVerificationError(o.names.key, compared.mismatches);
+
+  const all = [...compared.old.all.values()];
+  const rows = all.reduce((s, v) => s + v.rows, 0);
+  // What the approval binds: the rows before the cut-over (every row in app mode), which later writes do not change.
+  const verification = [...compared.old.before.values()].sort((x, y) => x.partition.localeCompare(y.partition));
+  const bound = verification.reduce((s, v) => s + v.rows, 0);
+  const range = cutover !== undefined ? ` (${bound} of them before the cut-over at ${new Date(cutover).toISOString()})` : "";
+  const summary = `${o.names.key}: ${all.length} partition(s), ${rows} row(s)${range}, counts and checksums equal in the old and new tables`;
   run.log(`-- ${summary}`);
   return {
     state: o.state,
@@ -116,9 +205,24 @@ export async function verifyRebuild(run: RebuildRun): Promise<VerifyResult> {
       cutover: cutover === undefined ? null : new Date(cutover).toISOString(),
       verification,
     }),
-    partitions: verification.length,
+    partitions: all.length,
     rows,
     summary,
     verification,
   };
 }
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    signal?.throwIfAborted();
+    const t = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(t);
+        reject(signal.reason);
+      },
+      { once: true },
+    );
+  });
+}

package/src/config.ts

@@ -115,6 +115,13 @@ export const sqlProfileSchema = z.strictObject({
   schemas: z.array(z.string()).optional(),
   /** Postgres: the schema an unqualified declaration is created in. `public` when omitted. */
   defaultSchema: z.string().optional(),
+  /**
+   * Postgres: the schema `effect()` receipts are kept in, as
+   * `<schema>.__chant_receipts`. When omitted they are in
+   * `chant_receipts.receipts`, a schema chant creates; name an existing schema
+   * on a server where the role cannot `CREATE SCHEMA`.
+   */
+  receiptsSchema: z.string().optional(),
   /**
    * The managed service this environment's Postgres runs on, when it differs
    * from `sql.provider`. Import reads the provider's own roles, schemas and

package/src/core/receipts.ts

@@ -10,9 +10,11 @@
  *
  * A dialect's store implements this interface over its own client and keeps
  * the receipts in a table whose comment carries the ownership trailer with
- * the `receipts` key (`./ownership.ts`), so schema reads leave it out. It is
- * not registered as the global `receiptRead` / `receiptWrite` activities,
- * which would take over every other lexicon's `effect()` steps in the run.
+ * the `receipts` key (`./ownership.ts`), so schema reads leave it out. A
+ * migration's store is reached by its backfill directly. The store `effect()`
+ * steps use (`../receipts.ts`) is registered as the `receiptRead` /
+ * `receiptWrite` activities only as a fallback, so it never takes over
+ * another configured lexicon's receipt row.
  */
 
 import type { ReceiptStore } from "@intentius/chant/op/receipt-store";