@intentius/chant-lexicon-sql 0.114.1 → 0.115.0

package/src/clickhouse/rebuild/rebuild-sharded.e2e.test.ts

@@ -0,0 +1,161 @@
+/**
+ * The rebuild migration on a cluster of two shards (#3663), against a
+ * scratch cluster of the pinned image: shard 1 of two replicas, shard 2 of
+ * one server, Keeper, and a cluster `chant` over all three. The profile
+ * names shard 1's first replica with topology `cluster:chant`.
+ *
+ * Each shard holds its own rows of `shop.events`, with a `Distributed`
+ * table over them. An `ORDER BY` change runs as a `ClickHouseRebuildOp`
+ * under an outer approval, keeping its new table on failure:
+ *
+ * - the first run copies shard 1's partitions and one of shard 2's, then
+ *   fails: each (shard, partition) copied has its own receipt;
+ * - the rerun skips those, copies the rest of shard 2, verifies every row of
+ *   both shards, and swaps: every shard's rows are in its live table, on
+ *   every replica, with the new sorting key.
+ *
+ * Runs the Op through core's local executor. Needs Docker; skips cleanly
+ * without it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { normalizeApply } from "@intentius/chant/apply";
+import { memoryGateLedgerPort, runOpLocally, loadProfiles, OpRunFailure, type ActivityFn, type OpConfig, type OpRunResult } from "@intentius/chant/op";
+import { dockerAvailable, startScratchCluster, type ScratchCluster } from "../container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { clickhouseApply, toApplyResult } from "../../op/activities/clickhouse-apply";
+import * as rebuildActivities from "../../op/activities/clickhouse-rebuild";
+import type { ClickHouseRebuildDeps } from "../../op/activities/clickhouse-rebuild";
+import { ClickHouseRebuildOp, type ClickHouseRebuildOpConfig } from "./op";
+
+const enabled = await dockerAvailable();
+let cluster: ScratchCluster | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-rebuild-sharded-"));
+
+beforeAll(async () => {
+  if (!enabled) return;
+  cluster = await startScratchCluster(clickhouseImage(), { shards: [2, 1], cluster: "chant", namePrefix: "chant-sql-rebuild-shards" });
+}, 600_000);
+
+afterAll(async () => {
+  await cluster?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+const MARKER = { stack: "e2e", env: "test" };
+const config = () => ({ ownership: MARKER, sql: { profiles: { e2e: { url: cluster!.replicas[0]!.url, topology: "cluster:chant" } } } });
+const deps = (extra: Partial<ClickHouseRebuildDeps> = {}): ClickHouseRebuildDeps => ({ config: config(), env: {}, log: () => undefined, ...extra });
+const q = <T = Record<string, unknown>>(endpoint: ClickHouseEndpoint, sql: string) => clickhouseQuery<T>(endpoint, sql);
+const count = async (endpoint: ClickHouseEndpoint, table: string) => Number((await q<{ n: string }>(endpoint, `SELECT count() AS n FROM ${table}`))[0]!.n);
+/** The servers of a shard, by its macro. */
+const servers = (shard: string) => cluster!.layout.flatMap((l, i) => (l.shard === shard ? [cluster!.replicas[i]!] : []));
+
+const eventsDdl = (orderBy: string) => `CREATE TABLE shop.events
+(
+  ts DateTime,
+  user_id UInt64,
+  kind LowCardinality(String)
+)
+ENGINE = MergeTree
+PARTITION BY toDate(ts)
+ORDER BY ${orderBy}`;
+const DISTRIBUTED = "CREATE TABLE shop.events_all (ts DateTime, user_id UInt64, kind LowCardinality(String)) ENGINE = Distributed(chant, shop, events, user_id)";
+
+function writeBuild(file: string, orderBy: string): string {
+  const objects = [
+    { export: "shop", type: "ClickHouse::Database", ddl: "CREATE DATABASE shop", dependsOn: [] },
+    { export: "events", type: "ClickHouse::Table", ddl: eventsDdl(orderBy), dependsOn: ["shop"] },
+    { export: "eventsAll", type: "ClickHouse::Table", ddl: DISTRIBUTED, dependsOn: ["events"] },
+  ];
+  writeFileSync(join(dir, file), JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects }));
+  return file;
+}
+
+const opConfig = (): ClickHouseRebuildOpConfig => ({
+  name: "rebuild-events",
+  env: "e2e",
+  table: "shop.events",
+  dualWrite: { mode: "materialized-view", cutoverColumn: "ts", cutoverDelay: "2s" },
+  build: false,
+  path: dir,
+  output: "events-v2.json",
+  stack: MARKER.stack,
+  ownershipEnv: MARKER.env,
+  gates: "outer",
+  onFailure: "keep",
+});
+
+async function runOp(extra: Partial<ClickHouseRebuildDeps> = {}): Promise<OpRunResult> {
+  const props = (ClickHouseRebuildOp(opConfig()).op as unknown as { props: OpConfig }).props;
+  const activities = new Map<string, ActivityFn>();
+  for (const [name, fn] of Object.entries(rebuildActivities)) {
+    if (typeof fn !== "function" || !name.startsWith("clickhouseRebuild")) continue;
+    activities.set(name, ((args: Record<string, unknown>, signal?: AbortSignal) => (fn as (a: unknown, s?: AbortSignal, d?: unknown) => Promise<unknown>)(args, signal, deps(extra))) as ActivityFn);
+  }
+  // One attempt per step, so the failure below fails the run where it happens.
+  const profiles = Object.fromEntries(Object.entries(await loadProfiles()).map(([k, v]) => [k, { ...v, retry: { maximumAttempts: 1 } }]));
+  return runOpLocally(props, activities, profiles, undefined, { gates: memoryGateLedgerPort(), now: new Date().toISOString() });
+}
+
+const outcome = (r: OpRunResult, name: string) => r.records.flatMap((x) => x.outcomes ?? []).find((o) => o.name === name)?.value;
+
+describe.skipIf(!enabled)("a rebuild on a cluster of two shards (#3663)", () => {
+  test("every shard's rows survive the rebuild, and a rerun after a failure on shard 2 skips what was copied", async () => {
+    const applied = normalizeApply(toApplyResult(await clickhouseApply({ buildPath: join(dir, writeBuild("events-v1.json", "(ts, user_id)")), environment: "e2e" }, undefined, deps())));
+    expect(applied.notAttempted).toEqual([]);
+    // Each shard's own rows, three days of them, written on that shard.
+    const [s1] = servers("s1");
+    const [s2] = servers("s2");
+    await q(s1!, "INSERT INTO shop.events SELECT toDateTime('2026-02-01 00:00:00') + (number % 3) * 86400 + number, number, 'view' FROM numbers(30)");
+    await q(s2!, "INSERT INTO shop.events SELECT toDateTime('2026-02-01 00:00:00') + (number % 3) * 86400 + number, 1000 + number, 'click' FROM numbers(24)");
+    expect(await count(s1!, "shop.events_all")).toBe(54);
+    writeBuild("events-v2.json", "(user_id, ts)");
+
+    // Shard 2's first partition is copied and its receipt written, then the backfill fails.
+    const failure = await runOp({
+      backfill: {
+        afterPartition: (_p, shard) => {
+          if (shard === 2) throw new Error("the backfill failed on shard 2");
+        },
+      },
+    }).then(
+      () => undefined,
+      (e: unknown) => e,
+    );
+    expect(failure).toBeInstanceOf(OpRunFailure);
+    expect((failure as OpRunFailure).result.records.find((x) => x.fn === "clickhouseRebuildBackfill")?.error).toMatch(/failed on shard 2/);
+    const receipts = async () =>
+      (await q<{ address: string }>(s1!, "SELECT DISTINCT address FROM chant_receipts.receipts WHERE address LIKE 'e2e/test/rebuild/shop.events/%' ORDER BY address")).map((r) => r.address);
+    expect(await receipts()).toEqual([
+      "e2e/test/rebuild/shop.events/shard1/20260201",
+      "e2e/test/rebuild/shop.events/shard1/20260202",
+      "e2e/test/rebuild/shop.events/shard1/20260203",
+      "e2e/test/rebuild/shop.events/shard2/20260201",
+    ]);
+    // Each shard's new table holds that shard's copied rows, and nothing of the other's.
+    expect(await count(s1!, "shop.events__chant_new")).toBe(30);
+    expect(await count(s2!, "shop.events__chant_new")).toBe(8);
+
+    // The rerun resumes: four (shard, partition) copies skipped, two copied, every row verified, swapped.
+    const r = await runOp();
+    expect(r.status).toBe("ok");
+    expect(outcome(r, "Skipped")).toBe(4);
+    expect(outcome(r, "Copied")).toBe(2);
+    expect(outcome(r, "VerifiedRows")).toBe(54);
+    expect(outcome(r, "VerifiedPartitions")).toBe(6);
+
+    for (const [shard, rows] of [["s1", 30], ["s2", 24]] as const) {
+      for (const server of servers(shard)) {
+        await q(server, "SYSTEM SYNC REPLICA shop.events");
+        expect(await count(server, "shop.events")).toBe(rows);
+        expect((await q<{ statement: string }>(server, "SHOW CREATE TABLE shop.events"))[0]!.statement).toMatch(/ORDER BY \(user_id, ts\)/);
+        expect(await count(server, "shop.events__chant_old")).toBe(rows);
+      }
+    }
+    expect(await count(s1!, "shop.events_all")).toBe(54);
+  }, 600_000);
+});

package/src/clickhouse/rebuild/rebuild.e2e.test.ts

@@ -10,6 +10,10 @@
  * - Rows dated after the cut-over (sql-yodeler#45): the ones the table held
  *   before the view are copied by the backfill, the ones written during the
  *   rebuild arrive through the view, and the swap leaves none behind.
+ * - Under an outer approval with the new table kept on failure (#3658): a
+ *   backfill that fails part way leaves the new table and its receipts, and
+ *   the next run skips the partitions already copied, verifies and swaps
+ *   with no gate, and keeps the old table.
  * - An interrupted backfill: stopped after three partitions, then killed
  *   between a partition's INSERT and its receipt, then resumed: the receipts
  *   skip what was copied, the half-copied partition is cleared first, and the
@@ -31,6 +35,7 @@ import { clickhouseImage } from "../../spec/pin";
 import { planAgainstServer } from "../plan/commands";
 import { clickhouseApply, toApplyResult } from "../../op/activities/clickhouse-apply";
 import * as rebuildActivities from "../../op/activities/clickhouse-rebuild";
+import type { ClickHouseRebuildDeps } from "../../op/activities/clickhouse-rebuild";
 import { ClickHouseRebuildOp, type ClickHouseRebuildOpConfig } from "./op";
 import { clickhouseReceiptStore, RECEIPTS_DATABASE, RECEIPTS_TABLE, type ClickHouseReceiptStore } from "./receipts";
 
@@ -68,20 +73,20 @@ const exists = async (database: string, name: string) => Number((await q<{ n: st
 const showCreate = async (name: string) => (await q<{ statement: string }>(`SHOW CREATE TABLE ${name}`))[0]!.statement;
 
 /** The rebuild activities, with the test's config, server and a quiet log, as the local executor resolves them. */
-function activities(): Map<string, ActivityFn> {
+function activities(extra: Partial<ClickHouseRebuildDeps> = {}): Map<string, ActivityFn> {
   const map = new Map<string, ActivityFn>();
   for (const [name, fn] of Object.entries(rebuildActivities)) {
     if (typeof fn !== "function" || !name.startsWith("clickhouseRebuild")) continue;
     map.set(name, ((args: Record<string, unknown>, signal?: AbortSignal) =>
-      (fn as (a: unknown, s?: AbortSignal, d?: unknown) => Promise<unknown>)(args, signal, deps())) as ActivityFn);
+      (fn as (a: unknown, s?: AbortSignal, d?: unknown) => Promise<unknown>)(args, signal, { ...deps(), ...extra })) as ActivityFn);
   }
   return map;
 }
 
-async function runOp(config: ClickHouseRebuildOpConfig, gates: GateLedgerPort): Promise<OpRunResult> {
+async function runOp(config: ClickHouseRebuildOpConfig, gates: GateLedgerPort, extra: Partial<ClickHouseRebuildDeps> = {}): Promise<OpRunResult> {
   const { op } = ClickHouseRebuildOp(config);
   const props = (op as unknown as { props: OpConfig }).props;
-  return runOpLocally(props, activities(), await loadProfiles(), undefined, { gates, now: new Date().toISOString() });
+  return runOpLocally(props, activities(extra), await loadProfiles(), undefined, { gates, now: new Date().toISOString() });
 }
 
 /** An in-memory gate ledger a person approves on: each approval answers the gate the last run stopped at, for the plan it recorded. */
@@ -454,3 +459,104 @@ describe.skipIf(!enabled)("an interrupted backfill resumes from its receipts", (
     expect(dropped.dropped).toEqual(["shop.page_views__chant_new"]);
   }, 300_000);
 });
+
+describe.skipIf(!enabled)("under an outer approval, a failed rebuild keeps its new table and the rerun resumes (#3658)", () => {
+  const sessionsDdl = (orderBy: string) => `CREATE TABLE shop.sessions
+(
+  day Date,
+  user_id UInt64,
+  ts DateTime
+)
+ENGINE = MergeTree
+PARTITION BY day
+ORDER BY ${orderBy}`;
+  const V1: Obj = { export: "sessions", type: "ClickHouse::Table", dependsOn: ["shop"], ddl: sessionsDdl("(day, user_id)") };
+  const V2: Obj = { ...V1, ddl: sessionsDdl("(user_id, day)") };
+  const config = (): ClickHouseRebuildOpConfig => ({
+    name: "rebuild-sessions",
+    env: "e2e",
+    table: "shop.sessions",
+    dualWrite: { mode: "materialized-view", cutoverColumn: "ts", cutoverDelay: "2s" },
+    build: false,
+    path: dir,
+    output: "sessions-v2.json",
+    stack: MARKER.stack,
+    ownershipEnv: MARKER.env,
+    gates: "outer",
+    onFailure: "keep",
+  });
+  const receipts = async () =>
+    (await q<{ address: string }>(`SELECT DISTINCT address FROM ${RECEIPTS_DATABASE}.${RECEIPTS_TABLE} WHERE address LIKE 'e2e/test/rebuild/shop.sessions/%'`)).length;
+
+  test("the failed run keeps the new table and its receipts; the rerun skips what was copied, swaps without a gate and keeps the old table", async () => {
+    await clickhouseApply({ buildPath: join(dir, writeBuild("sessions-v1.json", [DB, V1])), environment: "e2e" }, undefined, deps());
+    // Six days of rows, a partition each, all well before any cut-over.
+    await q("INSERT INTO shop.sessions SELECT toDate('2026-02-01') + number % 6, number, toDateTime('2026-02-01 00:00:00') + (number % 6) * 86400 FROM numbers(1200)");
+    writeBuild("sessions-v2.json", [DB, V2]);
+
+    // Every attempt of the backfill step fails after its partition: three attempts, three partitions copied, then the run fails.
+    let copied = 0;
+    const ledger = new Ledger();
+    const failure = await runOp(config(), ledger.port, {
+      backfill: {
+        afterPartition: () => {
+          copied++;
+          throw new Error(`the backfill failed after ${copied} partition(s)`);
+        },
+      },
+    }).then(
+      () => undefined,
+      (e: unknown) => e,
+    );
+    expect(failure).toBeInstanceOf(OpRunFailure);
+    const records = (failure as OpRunFailure).result.records;
+    expect(records.find((x) => x.fn === "clickhouseRebuildBackfill")?.error).toMatch(/the backfill failed after 3 partition/);
+    // No onFailure ran: the new table, the dual-write view and the receipts are there.
+    expect(records.find((x) => x.fn === "clickhouseRebuildCompensate")).toBeUndefined();
+    expect(await exists("shop", "sessions__chant_new")).toBe(true);
+    expect(await exists("shop", "sessions__chant_dual")).toBe(true);
+    expect(await count("shop.sessions__chant_new")).toBe(600);
+    expect(await receipts()).toBe(3);
+    expect(await showCreate("shop.sessions")).toMatch(/ORDER BY \(day, user_id\)/);
+
+    // The rerun resumes: three partitions skipped by receipt, three copied, verified, swapped, with no gate.
+    const r = await runOp(config(), ledger.port);
+    expect(r.status).toBe("ok");
+    expect(ledger.port.appended).toEqual([]);
+    expect(outcome(r, "Skipped")).toBe(3);
+    expect(outcome(r, "Copied")).toBe(3);
+    expect(outcome(r, "VerifiedRows")).toBe(1200);
+    expect(r.records.map((x) => x.fn)).not.toContain("clickhouseRebuildDrop");
+    expect(await showCreate("shop.sessions")).toMatch(/ORDER BY \(user_id, day\)/);
+    expect(await count("shop.sessions")).toBe(1200);
+    // The old table is kept: dropping it is a run with the Op's own gates.
+    expect(await count("shop.sessions__chant_old")).toBe(1200);
+    expect(await exists("shop", "sessions__chant_new")).toBe(false);
+    expect(await receipts()).toBe(6);
+
+    // Run with its own gates, the Op has nothing left to swap, and its drop gate binds that old table.
+    const ownLedger = new Ledger();
+    const own = await runOp({ ...config(), gates: "own" }, ownLedger.port);
+    expect(own.status).toBe("gated");
+    expect(own.gate).toMatchObject({ gate: "approve-rebuild-sessions" });
+    expect(outcome(own, "RebuildState")).toBe("swapped");
+    ownLedger.approveLast();
+    const drop = await runOp({ ...config(), gates: "own" }, ownLedger.port);
+    expect(drop.status).toBe("gated");
+    expect(drop.gate).toMatchObject({ gate: "approve-rebuild-sessions-drop" });
+    expect(await count("shop.sessions__chant_old")).toBe(1200);
+  }, 300_000);
+
+  test('a refusal says to drop the kept table by hand', async () => {
+    // A new table made from another declaration: refused, and nothing will drop it.
+    writeBuild("sessions-v3.json", [DB, { ...V1, ddl: sessionsDdl("(ts, user_id)") }]);
+    await q("CREATE TABLE shop.sessions__chant_new AS shop.sessions");
+    await q(`ALTER TABLE shop.sessions__chant_new MODIFY COMMENT 'x [chant managed-by=chant stack=e2e env=test rebuild=shop.sessions role=new]'`);
+    const args = { table: "shop.sessions", buildPath: "sessions-v3.json", environment: "e2e", dualWrite: { mode: "app" as const }, stack: MARKER.stack, ownershipEnv: MARKER.env, cwd: dir };
+    await expect(rebuildActivities.clickhouseRebuildCreate({ ...args, keepOnFailure: true }, undefined, deps())).rejects.toThrow(
+      /keeps its new table on failure \(onFailure: "keep"\), so nothing drops it: drop shop\.sessions__chant_new/,
+    );
+    await expect(rebuildActivities.clickhouseRebuildCreate(args, undefined, deps())).rejects.not.toThrow(/onFailure: "keep"/);
+    await q("DROP TABLE shop.sessions__chant_new SYNC");
+  }, 300_000);
+});

package/src/clickhouse/rebuild/rebuild.test.ts

@@ -20,6 +20,8 @@ import * as contractsModule from "../../op/activity-contracts";
 import * as activitiesModule from "../../op/activities";
 import { ClickHouseRebuildOp, type ClickHouseRebuildOpConfig } from "./op";
 import { copiedColumns } from "./observe";
+import { partitionEffect } from "./backfill";
+import { intoShard, onShard, onShardServers, shardTable } from "./shards";
 import { renameColumns, sourcePartitionExpression } from "./partitions";
 import { canonicalObject } from "../plan/normalize";
 import { diffSchemas } from "../plan/diff";
@@ -79,7 +81,37 @@ describe("ClickHouseRebuildOp", () => {
     expect(Object.keys(gate.approval.context)).toEqual(["verifiedPartitions", "verifiedRows"]);
   });
 
+  test('gates: "outer" leaves out both approval gates and the Drop phase, and keeps the verification (#3658)', () => {
+    const props = propsOf({ ...BASE, gates: "outer" });
+    expect(props.phases.map((p) => p.name)).toEqual(["Build", "Plan", "Create", "Dual write", "Backfill", "Verify", "Swap", "Retain"]);
+    expect(props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "gate")).toEqual([]);
+    const fns = props.phases.flatMap((p) => p.steps).map((s) => (s as { fn?: string }).fn);
+    expect(fns.indexOf("clickhouseRebuildVerify")).toBeLessThan(fns.indexOf("clickhouseRebuildSwap"));
+    expect(fns).not.toContain("clickhouseRebuildDrop");
+    // Its own onFailure is unchanged.
+    expect(props.onFailure?.map((p) => p.steps.map((s) => (s as { fn?: string }).fn))).toEqual([["clickhouseRebuildCompensate"]]);
+    expect(props.overview).toContain("the caller's approval");
+  });
+
+  test('gates: "outer" in app mode keeps the writes-stopped gate, which is not an approval (#3658)', () => {
+    const props = propsOf({ ...BASE, dualWrite: { mode: "app" }, gates: "outer" });
+    const gates = props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "gate") as Array<{ gate: string }>;
+    expect(gates.map((g) => g.gate)).toEqual(["rebuild-events-writes-stopped"]);
+  });
+
+  test('onFailure: "keep" has no onFailure and tells every step so (#3658)', () => {
+    const props = propsOf({ ...BASE, onFailure: "keep" });
+    expect(props.onFailure).toBeUndefined();
+    const steps = props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "activity" && (s as { fn: string }).fn.startsWith("clickhouseRebuild")) as Array<{ args: Record<string, unknown> }>;
+    expect(steps.length).toBeGreaterThan(0);
+    for (const s of steps) expect(s.args.keepOnFailure).toBe(true);
+    expect(props.phases.map((p) => p.name)).toContain("Approve");
+    expect((propsOf(BASE).phases[1]!.steps[0] as { args: Record<string, unknown> }).args).not.toHaveProperty("keepOnFailure");
+  });
+
   test("bad configuration fails when the Op is built", () => {
+    expect(() => ClickHouseRebuildOp({ ...BASE, gates: "none" as never })).toThrow(/gates must be/);
+    expect(() => ClickHouseRebuildOp({ ...BASE, onFailure: "retry" as never })).toThrow(/onFailure must be/);
     expect(() => ClickHouseRebuildOp({ ...BASE, table: "shop.events; DROP" })).toThrow(/table must be/);
     expect(() => ClickHouseRebuildOp({ ...BASE, dualWrite: { mode: "materialized-view", cutoverColumn: "" } })).toThrow(/cutoverColumn/);
     expect(() => ClickHouseRebuildOp({ ...BASE, dualWrite: { mode: "both" } as never })).toThrow(/dualWrite\.mode/);
@@ -89,19 +121,23 @@ describe("ClickHouseRebuildOp", () => {
     const contracts = new Map<string, ActivityContract>();
     collectActivityContracts(contractsModule as unknown as Record<string, unknown>, contracts);
     const merged = mergeActivityContracts(await loadActivityContracts([]), contracts);
-    for (const config of [BASE, { ...BASE, dualWrite: { mode: "app" as const }, retain: "3d", replicaTimeout: "30s", stack: "shop", ownershipEnv: "prod" }]) {
+    for (const config of [
+      BASE,
+      { ...BASE, dualWrite: { mode: "app" as const }, retain: "3d", replicaTimeout: "30s", stack: "shop", ownershipEnv: "prod" },
+      { ...BASE, gates: "outer" as const, onFailure: "keep" as const },
+    ]) {
       const props = propsOf(config);
       expect(validateActivitySteps(props, merged)).toEqual([]);
       expect(validateStepOutputRefs(props, merged)).toEqual([]);
     }
   });
 
-  test("every rebuild step is an activity the sql lexicon exports, and none is a receipt activity", () => {
+  test("every rebuild step is an activity the sql lexicon exports, and the receipt activities are only fallbacks", () => {
     const exported = new Set(Object.entries(activitiesModule).filter(([, v]) => typeof v === "function").map(([k]) => k));
     const steps = propsOf(BASE).phases.flatMap((p) => p.steps).filter((s) => s.kind === "activity" && (s as { fn: string }).fn.startsWith("clickhouseRebuild"));
     for (const s of steps) expect(exported.has((s as { fn: string }).fn)).toBe(true);
-    expect(exported.has("receiptRead")).toBe(false);
-    expect(exported.has("receiptWrite")).toBe(false);
+    // The receipt activities are exported for effect() steps only as fallbacks (#3657), so they never take over another lexicon's receipt row.
+    expect((activitiesModule as { ACTIVITY_FALLBACKS?: readonly string[] }).ACTIVITY_FALLBACKS).toEqual(["receiptRead", "receiptWrite", "receiptStaleness"]);
   });
 });
 
@@ -188,3 +224,19 @@ describe("the plan hands a rebuild to the Op", () => {
     expect(rebuildOpSuggestions(diff, new Map(after.map((o) => [o.key, o.canonical])), "<env>")[0]!.dualWrite).toEqual({ mode: "app" });
   });
 });
+
+describe("a rebuild across the shards of a cluster (#3663)", () => {
+  const sharding = { cluster: "main", shards: [{ num: 1, macro: "s1", slot: 0 }, { num: 2, macro: "s2", slot: 1 }] };
+  const shard2 = sharding.shards[1]!;
+
+  test("each (shard, partition) is its own receipt; one shard keeps the address it always had", () => {
+    expect(partitionEffect("shop.events", "202602")).toBe("rebuild/shop.events/202602");
+    expect(partitionEffect("shop.events", "202602", shard2)).toBe("rebuild/shop.events/shard2/202602");
+  });
+
+  test("a shard is read through the cluster, written back with its own sharding key, and cleared by its own macro", () => {
+    expect(`SELECT * FROM ${shardTable(sharding, "shop", "events")} WHERE ${onShard(shard2)}`).toBe("SELECT * FROM cluster('main', `shop`, `events`) WHERE _shard_num = 2");
+    expect(intoShard(sharding, shard2, "shop", "events__chant_new")).toBe("FUNCTION cluster('main', `shop`, `events__chant_new`, 1)");
+    expect(onShardServers(shard2)).toBe("getMacro('shard') = 's2'");
+  });
+});

package/src/clickhouse/rebuild/receipts.ts

@@ -21,12 +21,10 @@
  *
  * The store implements core's `ReceiptStore` seam
  * (`@intentius/chant/op/receipt-store`), as the sql lexicon's shared
- * `SqlReceiptStore` (`../../core/receipts.ts`). It is not exported as the
- * `receiptRead` / `receiptWrite` activities: those names are global to a
- * run, and a project that also configures aws or k8s would have its
- * `effect()` steps read ClickHouse instead of SSM or a ConfigMap. The backfill
- * activity reaches the store directly, the way the fly lexicon's release
- * component reaches its own.
+ * `SqlReceiptStore` (`../../core/receipts.ts`). The backfill activity
+ * reaches the store directly, the way the fly lexicon's release component
+ * reaches its own. The same table holds the receipts of a sql project's
+ * `effect()` steps (`../../receipts.ts`), under their own addresses.
  *
  * Write discipline: the backfill writes a partition's receipt after the
  * partition's copy succeeded, last, and nothing else writes one.

package/src/clickhouse/rebuild/shards.ts

@@ -0,0 +1,104 @@
+/**
+ * A rebuild across the shards of a cluster (#3663).
+ *
+ * On a `cluster:<name>` topology the rebuild's DDL runs `ON CLUSTER`, so
+ * every server gets the new table, the dual-write view and the swap. The
+ * rows do not move that way: each shard holds its own, and a copy run on
+ * the profile's server alone copies that server's shard and no other. So
+ * with more than one shard, the backfill and the verification work shard by
+ * shard, all through the profile's server:
+ *
+ * - a shard's rows are read with `cluster('<name>', db, t)` and
+ *   `_shard_num = <n>`, one replica per shard, as the cluster's own
+ *   configuration reaches it;
+ * - they are written back to the same shard with
+ *   `INSERT INTO FUNCTION cluster('<name>', db, t__chant_new, <slot>)`, a
+ *   constant sharding key that lands on that shard, inserted in the
+ *   foreground so the statement returns once the shard has the rows;
+ * - a shard's half-copied rows are deleted with an `ALTER ... ON CLUSTER`
+ *   whose condition holds on that shard's servers only
+ *   (`getMacro('shard')`, the macro the cluster's Keeper paths already use).
+ *
+ * Each (shard, partition) is one copy with its own receipt, so a rerun after
+ * a failure on one shard skips what every shard already copied.
+ *
+ * With one shard nothing changes: the replicas of a shard replicate what is
+ * written on any of them, which is what the rebuild has always relied on.
+ */
+
+import { clickhouseQuery } from "../http";
+import type { ClickHouseTarget } from "../live/bind";
+import { ident, sqlString } from "../apply/statements";
+import { RebuildRefusal } from "./observe";
+
+export interface Shard {
+  /** `_shard_num` / `system.clusters.shard_num`, from 1. */
+  num: number;
+  /** The shard's `{shard}` macro, the same on each of its replicas. */
+  macro: string;
+  /** The sharding key that lands on this shard: the sum of the weights of the shards before it. */
+  slot: number;
+}
+
+export interface Sharding {
+  cluster: string;
+  shards: Shard[];
+}
+
+/** Settings for a copy into one shard: in the foreground, and read from the cluster rather than pushed down to each shard. */
+export const SHARD_INSERT_SETTINGS: Record<string, string> = { distributed_foreground_insert: "1", parallel_distributed_insert_select: "0" };
+
+/** Settings for a delete on one shard: its condition is the server's own macro, the same on every replica of the shard. */
+export const SHARD_DELETE_SETTINGS: Record<string, string> = { allow_nondeterministic_mutations: "1", mutations_sync: "2" };
+
+/**
+ * The shards of the target's cluster, when its topology is `cluster:<name>`
+ * and the cluster has more than one. Undefined otherwise: the rebuild works
+ * on the profile's server as it always has.
+ */
+export async function clusterSharding(target: ClickHouseTarget): Promise<Sharding | undefined> {
+  if (target.topology?.kind !== "cluster") return undefined;
+  let cluster = target.topology.cluster;
+  const macro = /^\{(\w+)\}$/.exec(cluster);
+  if (macro) {
+    const [row] = await clickhouseQuery<{ c: string }>(target.endpoint, `SELECT getMacro(${sqlString(macro[1]!)}) AS c`);
+    cluster = row?.c ?? cluster;
+  }
+  const weights = await clickhouseQuery<{ shard: number | string; weight: number | string }>(
+    target.endpoint,
+    `SELECT shard_num AS shard, any(shard_weight) AS weight FROM system.clusters WHERE cluster = ${sqlString(cluster)} GROUP BY shard_num ORDER BY shard_num`,
+  );
+  if (weights.length <= 1) return undefined;
+  const macros = new Map(
+    (await clickhouseQuery<{ shard: number | string; macro: string }>(target.endpoint, `SELECT _shard_num AS shard, getMacro('shard') AS macro FROM cluster(${sqlString(cluster)}, system.one)`)).map(
+      (r) => [Number(r.shard), r.macro],
+    ),
+  );
+  let slot = 0;
+  const shards: Shard[] = [];
+  for (const w of weights) {
+    const num = Number(w.shard);
+    const m = macros.get(num);
+    if (m === undefined) throw new RebuildRefusal(`cluster ${cluster}: shard ${num} has no {shard} macro, which a rebuild across shards needs to delete a half-copied partition on that shard alone`);
+    shards.push({ num, macro: m, slot });
+    slot += Number(w.weight) || 1;
+  }
+  const distinct = new Set(shards.map((s) => s.macro));
+  if (distinct.size !== shards.length) {
+    throw new RebuildRefusal(`cluster ${cluster}: two shards share a {shard} macro (${shards.map((s) => `${s.num}=${s.macro}`).join(", ")}); each shard needs its own`);
+  }
+  return { cluster, shards };
+}
+
+/** One shard's rows of `database.table`, as a table expression; filter with {@link onShard}. */
+export const shardTable = (sharding: Sharding, database: string, table: string): string => `cluster(${sqlString(sharding.cluster)}, ${ident(database)}, ${ident(table)})`;
+
+/** The condition that keeps one shard's rows of a {@link shardTable}. */
+export const onShard = (shard: Shard): string => `_shard_num = ${shard.num}`;
+
+/** Where a copy into one shard of `database.table` writes. */
+export const intoShard = (sharding: Sharding, shard: Shard, database: string, table: string): string =>
+  `FUNCTION cluster(${sqlString(sharding.cluster)}, ${ident(database)}, ${ident(table)}, ${shard.slot})`;
+
+/** The condition, in an `ALTER ... ON CLUSTER` mutation, that holds on that shard's servers only. */
+export const onShardServers = (shard: Shard): string => `getMacro('shard') = ${sqlString(shard.macro)}`;

package/src/clickhouse/rebuild/steps.ts

@@ -17,6 +17,7 @@ import { waitForMutations } from "../apply/mutations";
 import { renderFor } from "../topology";
 import { DDL_SETTINGS, syncReplicaWithin } from "./replicas";
 import { assertSameRows } from "./verify";
+import type { Sharding } from "./shards";
 import {
   changesBetween,
   observeRebuild,
@@ -45,6 +46,8 @@ export interface RebuildRun {
   signal?: AbortSignal;
   /** The run's id, recorded on the receipts it writes. */
   runId?: string;
+  /** On a cluster of more than one shard, its shards: the backfill and the verification work shard by shard (`./shards.ts`). */
+  sharding?: Sharding;
 }
 
 /** The server's clock, in milliseconds: the cut-over and the retention are the server's times, not this machine's. */
@@ -76,6 +79,7 @@ const q = (run: RebuildRun, statement: string) => {
 
 export async function waitOn(run: RebuildRun, database: string, table: string): Promise<void> {
   const ids = await waitForMutations(run.target.endpoint, database, table, {
+    ...(run.sharding ? { source: `clusterAllReplicas(${sqlString(run.sharding.cluster)}, system.mutations)` } : {}),
     ...(run.mutationTimeoutMs !== undefined ? { timeoutMs: run.mutationTimeoutMs } : {}),
     ...(run.signal ? { signal: run.signal } : {}),
   });

package/src/clickhouse/rebuild/verify.ts

@@ -24,6 +24,10 @@
  * later than the cut-over delay, with a time before the cut-over) stop the
  * swap too, rather than stay behind in the old table.
  *
+ * On a cluster of more than one shard (#3663) every shard is read, through
+ * the profile's server, and each partition is compared per shard
+ * (`shard<n>/<partition>`), so a shard the copy missed is a difference.
+ *
  * Any difference fails the run, naming the partitions; the run's onFailure
  * then drops the new table. On a match the result carries the digest the
  * swap gate binds: the rebuild's plan and the counts and checksums of the
@@ -39,6 +43,7 @@ import { ident, sqlString } from "../apply/statements";
 import { rebuildPlanSubject, RebuildRefusal, type RebuildObservation } from "./observe";
 import { sourcePartitionExpression } from "./partitions";
 import { cutoverOf, observe, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
+import { shardTable } from "./shards";
 
 export interface PartitionCheck {
   partition: string;
@@ -120,11 +125,15 @@ async function compareTables(
   const newBefore = cut && cutover !== undefined ? `${ident(cut.name)} < ${utcLiteral(cutover)}` : "1";
   const fromOld = sourcePartitionExpression(o.live.partitionKey, o.newTable.partitionKey, o.copied);
 
-  const read = async (table: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
+  // Across the shards of a cluster (#3663), every shard's rows, each partition counted per shard as `shard<n>/<partition>`.
+  const sharding = run.sharding;
+  const read = async (table: string, name: string, partition: string, hash: string, before: string): Promise<SideCounts> => {
+    const from = sharding ? shardTable(sharding, n.database, name) : table;
+    const key = sharding ? `concat('shard', toString(_shard_num), '/', ${partition})` : partition;
     const rows = await clickhouseQuery<{ p: string; rows: string | number; checksum: string; brows: string | number; bchecksum: string }>(
       run.target.endpoint,
-      `SELECT ${partition} AS p, count() AS rows, toString(sum(${hash})) AS checksum, countIf(${before}) AS brows, toString(sumIf(${hash}, ${before})) AS bchecksum ` +
-        `FROM ${table} GROUP BY p`,
+      `SELECT ${key} AS p, count() AS rows, toString(sum(${hash})) AS checksum, countIf(${before}) AS brows, toString(sumIf(${hash}, ${before})) AS bchecksum ` +
+        `FROM ${from} GROUP BY p`,
     );
     const side: SideCounts = { all: new Map(), before: new Map() };
     for (const r of rows) {
@@ -141,8 +150,8 @@ async function compareTables(
     await syncReplica(run, o, n.database, n.name);
     await syncReplica(run, o, n.database, n.newName);
     // The old table first: a row the view has yet to commit in the new one is read as missing there, never the other way round.
-    const before = await read(n.table, "_partition_id", oldHash, oldBefore);
-    const after = await read(n.newTable, fromOld, newHash, newBefore);
+    const before = await read(n.table, n.name, "_partition_id", oldHash, oldBefore);
+    const after = await read(n.newTable, n.newName, fromOld, newHash, newBefore);
     const mismatches = differences(before.all, after.all);
     if (mismatches.length === 0 || attempt >= attempts) return { old: before, new: after, mismatches };
     run.log(`-- ${n.key}: ${mismatches.length} partition(s) differ (${mismatches.map((m) => m.partition).slice(0, 5).join(", ")}); reading again (${attempt}/${attempts})`);

package/src/config.ts

@@ -115,6 +115,13 @@ export const sqlProfileSchema = z.strictObject({
   schemas: z.array(z.string()).optional(),
   /** Postgres: the schema an unqualified declaration is created in. `public` when omitted. */
   defaultSchema: z.string().optional(),
+  /**
+   * Postgres: the schema `effect()` receipts are kept in, as
+   * `<schema>.__chant_receipts`. When omitted they are in
+   * `chant_receipts.receipts`, a schema chant creates; name an existing schema
+   * on a server where the role cannot `CREATE SCHEMA`.
+   */
+  receiptsSchema: z.string().optional(),
   /**
    * The managed service this environment's Postgres runs on, when it differs
    * from `sql.provider`. Import reads the provider's own roles, schemas and

package/src/core/receipts.ts

@@ -10,9 +10,11 @@
  *
  * A dialect's store implements this interface over its own client and keeps
  * the receipts in a table whose comment carries the ownership trailer with
- * the `receipts` key (`./ownership.ts`), so schema reads leave it out. It is
- * not registered as the global `receiptRead` / `receiptWrite` activities,
- * which would take over every other lexicon's `effect()` steps in the run.
+ * the `receipts` key (`./ownership.ts`), so schema reads leave it out. A
+ * migration's store is reached by its backfill directly. The store `effect()`
+ * steps use (`../receipts.ts`) is registered as the `receiptRead` /
+ * `receiptWrite` activities only as a fallback, so it never takes over
+ * another configured lexicon's receipt row.
  */
 
 import type { ReceiptStore } from "@intentius/chant/op/receipt-store";