@intentius/chant-lexicon-sql 0.114.1 → 0.116.0

package/src/clickhouse/container.ts

@@ -51,8 +51,12 @@ export async function startScratchServer(
 }
 
 export interface ScratchCluster {
-  /** One endpoint per replica, in replica order (`r1`, `r2`, ...). */
+  /** One endpoint per server, in order: shard 1's replicas first (`r1`, `r2`, ...), then shard 2's. */
   replicas: ClickHouseEndpoint[];
+  /** Each server's shard and replica macros, in the order of `replicas`. */
+  layout: Array<{ shard: string; replica: string; host: string }>;
+  /** The cluster `remote_servers` defines over every server: `ON CLUSTER` and `cluster:<name>` name it. */
+  cluster: string;
   /** The containers' common name prefix, for a log line. */
   name: string;
   /** Remove the containers and their network. Safe to call twice. */
@@ -72,23 +76,37 @@ export interface ScratchCluster {
 }
 
 /**
- * Start `replicas` servers of `image` as one shard of replicas, on their own
- * Docker network, and wait until each answers and reaches Keeper.
+ * Start servers of `image` on their own Docker network, and wait until each
+ * answers and reaches Keeper: `replicas` servers as one shard of replicas,
+ * or, with `shards`, that many replicas in each shard (`[2, 1]` is a shard of
+ * two replicas and a shard of one server).
  *
  * Keeper is the server's embedded one (`keeper_server`), run by the first
- * replica, so the pinned server image and its digest pin Keeper too. Each
- * replica gets the macros `{shard}` = `s1` and `{replica}` = `r<n>`, which
- * a `Replicated` database's arguments name. Configuration is copied into each
- * container before it starts, with no bind mount.
+ * server, so the pinned server image and its digest pin Keeper too. Each
+ * server gets the macros `{shard}` = `s<n>` and `{replica}` = `r<n>` (the
+ * replica numbered within its shard), which a `Replicated` database's and a
+ * `Replicated*MergeTree`'s arguments name, and `remote_servers` defines the
+ * cluster `cluster` (default `chant`) over all of them, with
+ * `internal_replication`, so `ON CLUSTER` reaches every server. Configuration
+ * is copied into each container before it starts, with no bind mount.
  */
 export async function startScratchCluster(
   image: string,
-  options: { replicas?: number; namePrefix?: string; readyTimeoutMs?: number } = {},
+  options: { replicas?: number; shards?: number[]; cluster?: string; namePrefix?: string; readyTimeoutMs?: number } = {},
 ): Promise<ScratchCluster> {
-  const count = options.replicas ?? 2;
+  const shards = options.shards ?? [options.replicas ?? 2];
+  const clusterName = options.cluster ?? "chant";
   const name = uniqueContainerName(options.namePrefix ?? "chant-sql-cluster");
   const network = `${name}-net`;
-  const nodes = Array.from({ length: count }, (_, i) => `${name}-r${i + 1}`);
+  // One shard keeps the names the single-shard cluster always had (`<name>-r<n>`).
+  const layout = shards.flatMap((replicas, s) =>
+    Array.from({ length: replicas }, (_, r) => ({
+      shard: `s${s + 1}`,
+      replica: `r${r + 1}`,
+      host: shards.length === 1 ? `${name}-r${r + 1}` : `${name}-s${s + 1}r${r + 1}`,
+    })),
+  );
+  const nodes = layout.map((l) => l.host);
   const dir = mkdtempSync(join(tmpdir(), "chant-sql-cluster-"));
   let removed = false;
   const stop = async (): Promise<void> => {
@@ -100,6 +118,12 @@ export async function startScratchCluster(
   };
 
   const keeper = nodes[0]!;
+  const remoteServers = shards
+    .map((_, s) => {
+      const hosts = layout.filter((l) => l.shard === `s${s + 1}`).map((l) => `<replica><host>${l.host}</host><port>9000</port></replica>`);
+      return `<shard><internal_replication>true</internal_replication>${hosts.join("")}</shard>`;
+    })
+    .join("");
   const config = (node: string, i: number) => `<clickhouse>
 ${
   i === 0
@@ -113,7 +137,9 @@ ${
 `
     : ""
 }  <zookeeper><node><host>${keeper}</host><port>9181</port></node></zookeeper>
-  <macros><shard>s1</shard><replica>r${i + 1}</replica></macros>
+  <distributed_ddl><path>/clickhouse/task_queue/ddl</path></distributed_ddl>
+  <remote_servers><${clusterName}>${remoteServers}</${clusterName}></remote_servers>
+  <macros><shard>${layout[i]!.shard}</shard><replica>${layout[i]!.replica}</replica></macros>
   <interserver_http_host>${node}</interserver_http_host>
 </clickhouse>
 `;
@@ -156,6 +182,8 @@ ${
     };
     return {
       replicas: endpoints,
+      layout,
+      cluster: clusterName,
       name,
       stop,
       async stopReplica(index) {

package/src/clickhouse/plan/cluster-drift.e2e.test.ts

@@ -0,0 +1,117 @@
+/**
+ * Drift on a multi-shard cluster (#3664), against a scratch cluster of the
+ * pinned image: two shards (two replicas and one server) with Keeper and a
+ * cluster `chant` over all three, the profile naming the first server with
+ * topology `cluster:chant`.
+ *
+ * - The schema applied `ON CLUSTER`, with a `Distributed` table declared with
+ *   unquoted identifiers, reads as no drift: the server's quoted arguments
+ *   are the same arguments.
+ * - A TTL changed on shard 2's server alone, without `ON CLUSTER`, is drift,
+ *   seen on that server, though the profile's server still holds the
+ *   declaration.
+ *
+ * Runs the sql lexicon's deep reader and core's deep diff, as
+ * `chant lifecycle diff --live` does. Needs Docker; skips cleanly without it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { diffDeepObservation } from "@intentius/chant/lifecycle/deep-observe";
+import { normalizeDeepObservation } from "@intentius/chant/deep-observation";
+import { dockerAvailable, startScratchCluster, type ScratchCluster } from "../container";
+import { clickhouseQuery } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { clickhouseApply, toApplyResult } from "../../op/activities/clickhouse-apply";
+import { normalizeApply } from "@intentius/chant/apply";
+import { database, table } from "../entities";
+import { observeResourcesDeep, sqlDeepNormalizationHooks } from "./deep";
+
+const enabled = await dockerAvailable();
+const DB = "chant_e2e_3664";
+let cluster: ScratchCluster | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-cluster-drift-"));
+
+const DB_DDL = `CREATE DATABASE ${DB}`;
+const EVENTS_DDL = `CREATE TABLE ${DB}.events
+(
+  id UInt64,
+  ts DateTime
+)
+ENGINE = MergeTree
+ORDER BY id
+TTL ts + INTERVAL 30 DAY`;
+const EVENTS_ALL_DDL = `CREATE TABLE ${DB}.events_all
+(
+  id UInt64,
+  ts DateTime
+)
+ENGINE = Distributed(chant, ${DB}, events, id)`;
+
+const config = () => ({ ownership: { stack: "e2e", env: "test" }, sql: { profiles: { e2e: { url: cluster!.replicas[0]!.url, topology: "cluster:chant" } } } });
+const tag = (fn: (strings: TemplateStringsArray) => unknown, ddl: string) => fn(Object.assign([ddl], { raw: [ddl] }) as unknown as TemplateStringsArray) as unknown as { entityType: string; props: Record<string, unknown> };
+
+const ENTITIES = new Map([
+  ["db", tag(database, DB_DDL)],
+  ["events", tag(table, EVENTS_DDL)],
+  ["eventsAll", tag(table, EVENTS_ALL_DDL)],
+]);
+
+async function drift() {
+  const entities = new Map([...ENTITIES].map(([k, v]) => [k, { entityType: v.entityType, props: v.props }]));
+  const live = await observeResourcesDeep({ environment: "e2e", entityNames: [...entities.keys()], entities, config: config(), env: {} });
+  return diffDeepObservation(entities, normalizeDeepObservation(live), sqlDeepNormalizationHooks).drifted;
+}
+
+beforeAll(async () => {
+  if (!enabled) return;
+  cluster = await startScratchCluster(clickhouseImage(), { shards: [2, 1], cluster: "chant", namePrefix: "chant-sql-cluster-drift" });
+}, 600_000);
+
+afterAll(async () => {
+  await cluster?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+describe.skipIf(!enabled)("drift on a two-shard cluster (#3664)", () => {
+  test("applied ON CLUSTER, with unquoted Distributed arguments, it reads as no drift", async () => {
+    const build = join(dir, "schema.json");
+    writeFileSync(
+      build,
+      JSON.stringify({
+        dialect: "clickhouse",
+        applyOrder: ["db", "events", "eventsAll"],
+        objects: [
+          { export: "db", type: "ClickHouse::Database", ddl: DB_DDL, dependsOn: [] },
+          { export: "events", type: "ClickHouse::Table", ddl: EVENTS_DDL, dependsOn: ["db"] },
+          { export: "eventsAll", type: "ClickHouse::Table", ddl: EVENTS_ALL_DDL, dependsOn: ["events"] },
+        ],
+      }),
+    );
+    const applied = normalizeApply(toApplyResult(await clickhouseApply({ buildPath: build, environment: "e2e" }, undefined, { config: config(), env: {}, log: () => undefined })));
+    expect(applied.notAttempted).toEqual([]);
+    // Every server has both tables.
+    for (const endpoint of cluster!.replicas) {
+      const rows = await clickhouseQuery<{ name: string }>(endpoint, `SELECT name FROM system.tables WHERE database = '${DB}' ORDER BY name`);
+      expect(rows.map((r) => r.name)).toEqual(["events", "events_all"]);
+    }
+    expect(await drift()).toEqual([]);
+  }, 300_000);
+
+  test("a TTL changed on shard 2 alone is drift, seen on that server", async () => {
+    const shard2 = cluster!.layout.findIndex((l) => l.shard === "s2");
+    await clickhouseQuery(cluster!.replicas[shard2]!, `ALTER TABLE ${DB}.events MODIFY TTL ts + INTERVAL 7 DAY`);
+    // The profile's server still holds the declaration.
+    const [mine] = await clickhouseQuery<{ q: string }>(cluster!.replicas[0]!, `SELECT create_table_query AS q FROM system.tables WHERE database = '${DB}' AND name = 'events'`);
+    expect(mine!.q).toContain("toIntervalDay(30)");
+
+    const drifted = await drift();
+    expect(drifted.map((d) => d.name)).toEqual(["events"]);
+    const ttl = drifted[0]!.changes.find((c) => c.path.startsWith("ttl"));
+    expect(ttl, JSON.stringify(drifted[0]!.changes)).toBeDefined();
+    expect(String(ttl!.live)).toMatch(/7/);
+    expect(ttl!.seenOn).toBe(cluster!.layout[shard2]!.host);
+  }, 300_000);
+});

package/src/clickhouse/plan/deep.ts

@@ -12,12 +12,20 @@
  * `deepNormalizationHooks` prunes what is not a property of the object (the
  * DDL text, the template source, lineage, a view's reads and `TO` target).
  * `chant sql plan` covers the target, and classifies every change.
+ *
+ * On a `cluster:<name>` topology every replica of every shard is read
+ * (#3664). When a server's copy of a table differs from the profile
+ * server's, that copy is what is reported, with the server in `observedOn`,
+ * which the drift report prints as `seen on`.
  */
 
 import { deepObservation, type DeepObservationResult, type DeepResourceObservation, type DeepNormalizationHooks } from "@intentius/chant/deep-observation";
 import type { UnobservedEntity } from "@intentius/chant/lexicon";
-import { bindClickHouse, classifyClickHouseFailure, type BindOptions } from "../live/bind";
-import { readLiveSchema, type LiveObject } from "../live/catalog";
+import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "../live/bind";
+import { readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
+import { clickhouseQuery } from "../http";
+import { sqlString } from "../apply/statements";
+import { stripMarkerFromStatement } from "../ownership";
 import { database, table, view, CLICKHOUSE_ENTITY_TYPES, type ColumnDef } from "../entities";
 import { canonicalObject, type CanonicalColumn, type CanonicalObject } from "./normalize";
 import { renderFor } from "../topology";
@@ -76,6 +84,73 @@ export function inDeclaredVocabulary(declared: Props, live: Props, d: CanonicalO
   return out;
 }
 
+/** Every server's copy of each table and view on a cluster (#3664), as `<database>.<name>` → host → statement. */
+interface ClusterCopies {
+  /** The host the profile names, whose copy the rest are compared with. */
+  self: string;
+  byKey: Map<string, Map<string, string>>;
+}
+
+/**
+ * On a `cluster:<name>` topology, read `system.tables` on every replica of
+ * every shard through the profile's server (`clusterAllReplicas`). A change
+ * made on one server without `ON CLUSTER` (a TTL altered on shard 2 alone)
+ * is then seen, which the profile's server alone would not show. Undefined
+ * for any other topology: there is one copy.
+ */
+async function clusterCopies(target: ClickHouseTarget): Promise<ClusterCopies | undefined> {
+  if (target.topology?.kind !== "cluster") return undefined;
+  const scope = target.databases
+    ? `IN (${target.databases.map(sqlString).join(", ")})`
+    : `NOT IN (${[...SYSTEM_DATABASES].map(sqlString).join(", ")})`;
+  const [self] = await clickhouseQuery<{ host: string }>(target.endpoint, "SELECT hostName() AS host");
+  const rows = await clickhouseQuery<{ host: string; database: string; name: string; statement: string }>(
+    target.endpoint,
+    `SELECT hostName() AS host, database, name, create_table_query AS statement ` +
+      `FROM clusterAllReplicas(${sqlString(target.topology.cluster)}, system.tables) ` +
+      `WHERE database ${scope} AND NOT is_temporary AND name NOT LIKE '.inner%' AND engine != 'Dictionary' ORDER BY host, database, name`,
+  );
+  const byKey = new Map<string, Map<string, string>>();
+  for (const r of rows) {
+    const key = `${r.database}.${r.name}`;
+    if (!byKey.has(key)) byKey.set(key, new Map());
+    byKey.get(key)!.set(r.host, stripMarkerFromStatement(r.statement));
+  }
+  return { self: self?.host ?? "", byKey };
+}
+
+/**
+ * When the servers' copies of `key` do not all read the same, the copy to
+ * report and its server: whichever departs furthest from the declaration
+ * `d`, the profile server's own winning a tie. A copy that differs from the
+ * declaration on any server is then the one the diff sees. Undefined when
+ * every copy reads the same.
+ */
+function copyToReport(copies: ClusterCopies, key: string, d: CanonicalObject, defaultDatabase: string): { host: string; statement: string } | undefined {
+  const byHost = copies.byKey.get(key);
+  const mine = byHost?.get(copies.self);
+  if (!byHost || mine === undefined) return undefined;
+  const shape = (c: CanonicalObject) => {
+    const { columns, ...rest } = c;
+    return JSON.stringify({ ...rest, columns: columns.map(({ text: _t, position: _p, ...col }) => col) });
+  };
+  const reference = shape(canonicalObject(mine, defaultDatabase));
+  const declared = JSON.parse(shape(d)) as Record<string, unknown>;
+  /** How many of the declaration's fields this copy holds differently. */
+  const departures = (statement: string) => {
+    const live = JSON.parse(shape(canonicalObject(statement, defaultDatabase))) as Record<string, unknown>;
+    return Object.keys({ ...declared, ...live }).filter((k) => k !== "name" && k !== "database" && JSON.stringify(declared[k]) !== JSON.stringify(live[k])).length;
+  };
+  let pick: { host: string; statement: string; score: number } | undefined;
+  for (const [host, statement] of byHost) {
+    if (host === copies.self || shape(canonicalObject(statement, defaultDatabase)) === reference) continue;
+    const score = departures(statement);
+    if (!pick || score > pick.score) pick = { host, statement, score };
+  }
+  if (!pick) return undefined;
+  return pick.score > departures(mine) ? { host: pick.host, statement: pick.statement } : { host: copies.self, statement: mine };
+}
+
 export async function observeResourcesDeep(
   options: {
     environment: string;
@@ -98,6 +173,17 @@ export async function observeResourcesDeep(
     return deepObservation({}, unobserved);
   }
   const byKey = new Map(live.map((o) => [`${o.database ?? ""}.${o.name}`, o]));
+  let copies: ClusterCopies | undefined;
+  try {
+    copies = await clusterCopies(target);
+  } catch (err) {
+    // A cluster whose servers cannot all be read is not reported as matching.
+    const why = classifyClickHouseFailure(err);
+    for (const name of options.entityNames) {
+      unobserved[name] = { type: options.entities.get(name)?.entityType ?? "", reason: why.reason, detail: `reading every server of the cluster: ${why.detail}` };
+    }
+    return deepObservation({}, unobserved);
+  }
   for (const name of options.entityNames) {
     const entity = options.entities.get(name);
     if (!entity || !entity.entityType.startsWith("ClickHouse::")) {
@@ -111,10 +197,23 @@ export async function observeResourcesDeep(
     if (!o) continue;
     try {
       const d = canonicalObject(renderFor(String(props.ddl), target.topology), target.defaultDatabase);
+      const physicalId = o.uuid ?? `${db ? `${db}.` : ""}${o.name}`;
+      const other = copies && !isDb ? copyToReport(copies, `${db ?? ""}.${o.name}`, d, target.defaultDatabase) : undefined;
+      if (other) {
+        // The servers disagree: report the copy that departs from the declaration, and where it was seen.
+        const copy: LiveObject = { ...o, statement: other.host === copies!.self ? o.statement : other.statement };
+        resources[name] = {
+          type: o.type,
+          physicalId,
+          properties: inDeclaredVocabulary(props, liveProps(copy), d, canonicalObject(copy.statement, target.defaultDatabase)),
+          observedOn: other.host,
+        };
+        continue;
+      }
       const l = canonicalObject(o.statement, target.defaultDatabase);
       resources[name] = {
         type: o.type,
-        physicalId: o.uuid ?? `${db ? `${db}.` : ""}${o.name}`,
+        physicalId,
         properties: inDeclaredVocabulary(props, liveProps(o), d, l),
       };
     } catch (err) {

package/src/clickhouse/plan/normalize.ts

@@ -284,6 +284,10 @@ export function canonicalObject(ddl: string, defaultDatabase = "default"): Canon
       name = shared[1]!;
       if (/^'.*'$/s.test(args[0] ?? "") && /^'.*'$/s.test(args[1] ?? "")) args = args.slice(2);
     }
+    // `Distributed(cluster, db, table, ...)` takes its first three arguments
+    // as identifiers or as strings, and the server prints them back quoted
+    // (#3664): an unquoted declaration says the same.
+    if (name === "Distributed") args = args.map((a, i) => (i < 3 ? quotedIdentifier(a) : a));
     obj.engineName = name;
     obj.engine = args.length ? `${name}(${args.join(", ")})` : name;
   }
@@ -359,3 +363,9 @@ function column(tokens: Token[], c: ColumnNode, position: number, database?: str
   if (prev) out.previously = prev;
   return out;
 }
+
+/** A bare or backquoted identifier as the string literal the server prints it as; anything else as it is. */
+function quotedIdentifier(arg: string): string {
+  const bare = /^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) ? arg : /^`((?:[^`]|``)+)`$/.exec(arg)?.[1]?.replace(/``/g, "`");
+  return bare === undefined ? arg : `'${bare.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+}

package/src/clickhouse/plan/plan.test.ts

@@ -87,6 +87,16 @@ describe("normalization", () => {
     expect(shown("ReplicatedMergeTree('/clickhouse/tables/{shard}/t', '{replica}')")).not.toBe(shown("ReplicatedMergeTree"));
   });
 
+  test("a Distributed engine's cluster, database and table read the same quoted or not (#3664)", () => {
+    const shown = (engine: string) => canonicalObject(`CREATE TABLE d.events_all (id UInt64) ENGINE = ${engine}`).engine;
+    expect(shown("Distributed(my_cluster, d, events, id)")).toBe(shown("Distributed('my_cluster', 'd', 'events', id)"));
+    expect(shown("Distributed(`my_cluster`, `d`, `events`)")).toBe(shown("Distributed('my_cluster', 'd', 'events')"));
+    expect(shown("Distributed('{cluster}', currentDatabase(), events, rand())")).toBe(shown("Distributed('{cluster}', currentDatabase(), 'events', rand())"));
+    // The sharding key is an expression, not a name: it stays as written.
+    expect(shown("Distributed(c, d, events, id)")).not.toBe(shown("Distributed(c, d, events, 'id')"));
+    expect(shown("Distributed(c, d, events)")).not.toBe(shown("Distributed(c, d, other)"));
+  });
+
   test("a setting at its default is not part of the definition", () => {
     expect(canonicalObject("CREATE TABLE t (a UInt8) ENGINE = MergeTree ORDER BY a SETTINGS index_granularity = 8192").settings).toEqual({});
     expect(canonicalObject("CREATE TABLE t (a UInt8) ENGINE = MergeTree ORDER BY a SETTINGS min_bytes_for_wide_part = '0'").settings).toEqual({ min_bytes_for_wide_part: "0" });

package/src/clickhouse/rebuild/backfill.ts

@@ -42,6 +42,11 @@
  * either finds nothing that replica alone has and returns, or stops the step
  * naming it (`./replicas.ts`).
  *
+ * On a cluster of more than one shard (#3663) the unit is a partition on one
+ * shard: read from that shard, written back to it, cleared on it alone, with
+ * its own receipt (`./shards.ts`), so a rerun after a failure on one shard
+ * skips what every shard already copied.
+ *
  * In materialized-view mode a partition is copied in two parts
  * (INTENTIUS/sql-yodeler#45). The rows before the cut-over are copied as
  * above. The rows at or after it are the view's only when they were written
@@ -64,9 +69,11 @@ import { ident, sqlString } from "../apply/statements";
 import { clickhouseReceiptStore, receiptAddress, type ClickHouseReceiptStore } from "./receipts";
 import { RebuildRefusal, type RebuildObservation } from "./observe";
 import { DDL_SETTINGS } from "./replicas";
-import { sourcePartitionExpression, sourcePartitions } from "./partitions";
+import { shardPartitions, sourcePartitionExpression, sourcePartitions } from "./partitions";
 import { rowHash } from "./verify";
 import { cutoverOf, observe, serverNow, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
+import { intoShard, onShard, onShardServers, shardTable, SHARD_DELETE_SETTINGS, SHARD_INSERT_SETTINGS, type Shard } from "./shards";
+import { renderFor } from "../topology";
 
 export interface BackfillResult {
   state: RebuildObservation["state"];
@@ -83,11 +90,11 @@ export interface BackfillResult {
 }
 
 /** The effect a partition's copy is, and its receipt's address suffix. */
-export const partitionEffect = (key: string, partition: string): string => `rebuild/${key}/${partition}`;
+export const partitionEffect = (key: string, partition: string, shard?: Shard): string => (shard ? `rebuild/${key}/shard${shard.num}/${partition}` : `rebuild/${key}/${partition}`);
 
-/** The receipt's expected value for one partition's copy. */
-export function partitionExpectation(o: RebuildObservation, partition: string, cutover: number | undefined): string {
-  const effect = partitionEffect(o.names.key, partition);
+/** The receipt's expected value for one partition's copy, on one shard of a cluster when `shard` is given. */
+export function partitionExpectation(o: RebuildObservation, partition: string, cutover: number | undefined, shard?: Shard): string {
+  const effect = partitionEffect(o.names.key, partition, shard);
   return receiptExpectation(
     EffectReceipt(effect, {
       effect,
@@ -96,6 +103,7 @@ export function partitionExpectation(o: RebuildObservation, partition: string, c
         table: o.names.key,
         newTable: o.newTable!.uuid,
         partition,
+        ...(shard ? { shard: shard.num } : {}),
         copied: o.copied,
         cutover: cutover === undefined ? null : new Date(cutover).toISOString(),
         // The rows at or after the cut-over the old table held are copied too (sql-yodeler#45); a receipt from before that copied fewer.
@@ -108,8 +116,8 @@ export function partitionExpectation(o: RebuildObservation, partition: string, c
 export interface BackfillDeps {
   /** The receipt store. Default: the receipts table on the target server. */
   receipts?: ClickHouseReceiptStore;
-  /** Called after each partition is copied and its receipt written; a test interrupts the backfill here. */
-  afterPartition?: (partition: string) => void | Promise<void>;
+  /** Called after each partition is copied and its receipt written, with its shard on a cluster of shards; a test interrupts the backfill here. */
+  afterPartition?: (partition: string, shard?: number) => void | Promise<void>;
 }
 
 export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promise<BackfillResult> {
@@ -140,6 +148,12 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
   const columns = o.copied.map((c) => ident(c.name)).join(", ");
   const select = o.copied.map((c) => ident(c.source)).join(", ");
   const later = cutColumn && cutover !== undefined ? await laterRowsCopy(run, o, cutColumn, cutover, fromOld) : undefined;
+  const sharding = run.sharding;
+  // The old and new tables as one shard's rows (#3663, `./shards.ts`), or as this server's.
+  const oldFrom = (shard?: Shard) => (shard && sharding ? `${shardTable(sharding, n.database, n.name)} WHERE ${onShard(shard)} AND` : `${n.table} WHERE`);
+  const newFrom = (shard?: Shard) => (shard && sharding ? `${shardTable(sharding, n.database, n.newName)} WHERE ${onShard(shard)} AND` : `${n.newTable} WHERE`);
+  const into = (shard?: Shard) => (shard && sharding ? intoShard(sharding, shard, n.database, n.newName) : n.newTable);
+  const insertSettings = sharding ? SHARD_INSERT_SETTINGS : {};
 
   const identity = { ...(run.marker?.stack ? { stack: run.marker.stack } : {}), ...(run.marker?.env ? { env: run.marker.env } : {}) };
   const receipts =
@@ -152,21 +166,31 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
     });
   const recorded = await receipts.readAll(receiptAddress(identity, `rebuild/${n.key}/`));
 
-  const partitions = await sourcePartitions(run.target, n);
-  result.partitions = partitions.length;
-  for (const p of partitions) {
+  // The unit of work: a partition of the old table, on each shard of a cluster that has more than one.
+  const units: Array<{ shard?: Shard; partition: string }> = sharding
+    ? (await shardPartitions(run, sharding, n.database, n.name)).map((u) => ({ shard: sharding.shards.find((x) => x.num === u.shard)!, partition: u.partition }))
+    : (await sourcePartitions(run.target, n)).map((partition) => ({ partition }));
+  result.partitions = units.length;
+  for (const { shard, partition: p } of units) {
     run.signal?.throwIfAborted();
-    const effect = partitionEffect(n.key, p);
-    const expectation = partitionExpectation(o, p, cutover);
+    const effect = partitionEffect(n.key, p, shard);
+    const expectation = partitionExpectation(o, p, cutover, shard);
     if (recorded.get(receiptAddress(identity, effect)) === expectation) {
       result.skipped++;
       continue;
     }
 
-    const queryId = `chant-rebuild-${o.newTable.uuid}-${p}`;
+    const queryId = `chant-rebuild-${o.newTable.uuid}-${shard ? `shard${shard.num}-` : ""}${p}`;
     const laterQueryId = `${queryId}-after`;
     const killWhere = `query_id IN (${sqlString(queryId)}, ${sqlString(laterQueryId)})`;
-    if (o.replicated) {
+    if (sharding) {
+      // The copy reads and writes every shard from this server; what it started on the others goes with it.
+      await clickhouseQuery(
+        run.target.endpoint,
+        `KILL QUERY ON CLUSTER ${sqlString(sharding.cluster)} WHERE ${killWhere} OR initial_query_id IN (${sqlString(queryId)}, ${sqlString(laterQueryId)}) SYNC`,
+        { settings: DDL_SETTINGS },
+      );
+    } else if (o.replicated) {
       // On every replica that answers: the copy may be running on another one
       // than this run's. A replica that is down is skipped once Keeper sees it
       // inactive; nothing of its can be running a copy that still writes,
@@ -179,24 +203,28 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
       await clickhouseQuery(run.target.endpoint, `KILL QUERY WHERE ${killWhere} SYNC`);
     }
     const mine = `${fromOld} = ${sqlString(p)}${targetRange}`;
-    const [left] = await clickhouseQuery<{ n: string | number }>(run.target.endpoint, `SELECT count() AS n FROM ${n.newTable} WHERE ${mine}`);
+    const where = shard ? `partition ${p} on shard ${shard.num}` : `partition ${p}`;
+    const [left] = await clickhouseQuery<{ n: string | number }>(run.target.endpoint, `SELECT count() AS n FROM ${newFrom(shard)} ${mine}`);
     if (Number(left?.n ?? 0) > 0) {
-      run.log(`-- partition ${p}: ${left!.n} row(s) from a copy that did not finish; deleting them before copying again`);
-      const sql = `ALTER TABLE ${n.newTable} DELETE WHERE ${mine}`;
+      run.log(`-- ${where}: ${left!.n} row(s) from a copy that did not finish; deleting them before copying again`);
+      const sql =
+        shard && sharding
+          ? renderFor(`ALTER TABLE ${n.newTable} DELETE WHERE ${onShardServers(shard)} AND ${mine}`, run.target.topology)
+          : `ALTER TABLE ${n.newTable} DELETE WHERE ${mine}`;
       run.log(sql);
-      await clickhouseQuery(run.target.endpoint, sql);
+      await clickhouseQuery(run.target.endpoint, sql, shard && sharding ? { settings: { ...DDL_SETTINGS, ...SHARD_DELETE_SETTINGS } } : {});
       await waitOn(run, n.database, n.newName);
       result.cleared++;
     }
 
-    const sql = `INSERT INTO ${n.newTable} (${columns}) SELECT ${select} FROM ${n.table} WHERE _partition_id = ${sqlString(p)}${sourceRange}`;
+    const sql = `INSERT INTO ${into(shard)} (${columns}) SELECT ${select} FROM ${oldFrom(shard)} _partition_id = ${sqlString(p)}${sourceRange}`;
     run.log(sql);
-    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0" }, ...(run.signal ? { signal: run.signal } : {}) });
-    if (later) result.repaired += await copyAfterCutover(run, o, later, p, laterQueryId);
+    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0", ...insertSettings }, ...(run.signal ? { signal: run.signal } : {}) });
+    if (later) result.repaired += await copyAfterCutover(run, o, later, p, laterQueryId, shard);
     // The receipt, last, on success only.
     await receipts.write({ name: effect, effect, flavor: "hash", inputs: {} }, expectation);
     result.copied++;
-    await deps.afterPartition?.(p);
+    await deps.afterPartition?.(p, shard?.num);
   }
   run.log(`-- backfill of ${n.key}: ${result.partitions} partition(s), ${result.copied} copied, ${result.skipped} already copied, ${result.cleared} cleared first${result.repaired > 0 ? `, ${result.repaired} row(s) after the cut-over copied twice and repaired` : ""}`);
   return result;
@@ -206,13 +234,14 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
 interface LaterRows {
   /**
    * Per distinct row (by its hash), the old table's copies past the number
-   * the new table holds, into the new table.
+   * the new table holds, into the new table. On a cluster of shards, one
+   * shard's rows into the same shard.
    */
-  copy: (partition: string) => string;
+  copy: (partition: string, shard?: Shard) => string;
   /** The hashes of the rows the new table holds more copies of than the old one. */
-  extras: (partition: string) => string;
+  extras: (partition: string, shard?: Shard) => string;
   /** Delete every copy of those rows from the new table. */
-  drop: (partition: string, hashes: readonly string[]) => string;
+  drop: (partition: string, hashes: readonly string[], shard?: Shard) => string;
 }
 
 /** The most extra rows one partition's repair deletes and copies again; past that the verification decides. */
@@ -245,19 +274,28 @@ async function laterRowsCopy(
   const columns = o.copied.map((c) => ident(c.name)).join(", ");
   const aliases = o.copied.map((_, i) => `__chant_c${i}`);
   const fromOldSide = o.copied.map((c, i) => `${ident(c.source)} AS ${aliases[i]}`).join(", ");
-  const oldRows = (p: string) => `FROM ${n.table} WHERE _partition_id = ${sqlString(p)} AND ${ident(cut.source)} >= ${utcLiteral(cutover)}`;
-  const newRows = (p: string) => `FROM ${n.newTable} WHERE ${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)}`;
+  const sharding = run.sharding;
+  const on = (shard?: Shard) => (shard && sharding ? ` AND ${onShard(shard)}` : "");
+  const oldTable = (shard?: Shard) => (shard && sharding ? shardTable(sharding, n.database, n.name) : n.table);
+  const newTable = (shard?: Shard) => (shard && sharding ? shardTable(sharding, n.database, n.newName) : n.newTable);
+  const oldRows = (p: string, shard?: Shard) => `FROM ${oldTable(shard)} WHERE _partition_id = ${sqlString(p)} AND ${ident(cut.source)} >= ${utcLiteral(cutover)}${on(shard)}`;
+  const newRows = (p: string, shard?: Shard) => `FROM ${newTable(shard)} WHERE ${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)}${on(shard)}`;
   return {
-    copy: (p) =>
-      `INSERT INTO ${n.newTable} (${columns}) SELECT ${aliases.join(", ")} FROM (` +
-      `SELECT ${fromOldSide}, ${oldHash} AS __chant_h, row_number() OVER (PARTITION BY __chant_h) AS __chant_k ${oldRows(p)}` +
-      `) AS old_rows LEFT JOIN (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p)} GROUP BY __chant_h) AS new_rows ` +
+    copy: (p, shard) =>
+      `INSERT INTO ${shard && sharding ? intoShard(sharding, shard, n.database, n.newName) : n.newTable} (${columns}) SELECT ${aliases.join(", ")} FROM (` +
+      `SELECT ${fromOldSide}, ${oldHash} AS __chant_h, row_number() OVER (PARTITION BY __chant_h) AS __chant_k ${oldRows(p, shard)}` +
+      `) AS old_rows LEFT JOIN (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p, shard)} GROUP BY __chant_h) AS new_rows ` +
       `USING (__chant_h) WHERE __chant_k > ifNull(__chant_n, 0)`,
-    extras: (p) =>
-      `SELECT toString(__chant_h) AS h FROM (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p)} GROUP BY __chant_h) AS new_rows ` +
-      `LEFT JOIN (SELECT ${oldHash} AS __chant_h, count() AS __chant_o ${oldRows(p)} GROUP BY __chant_h) AS old_rows ` +
+    extras: (p, shard) =>
+      `SELECT toString(__chant_h) AS h FROM (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p, shard)} GROUP BY __chant_h) AS new_rows ` +
+      `LEFT JOIN (SELECT ${oldHash} AS __chant_h, count() AS __chant_o ${oldRows(p, shard)} GROUP BY __chant_h) AS old_rows ` +
       `USING (__chant_h) WHERE __chant_n > ifNull(__chant_o, 0) LIMIT ${REPAIR_LIMIT + 1}`,
-    drop: (p, hashes) => `ALTER TABLE ${n.newTable} DELETE WHERE ${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)} AND ${newHash} IN (${hashes.join(", ")})`,
+    drop: (p, hashes, shard) => {
+      const where = `${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)} AND ${newHash} IN (${hashes.join(", ")})`;
+      return shard && sharding
+        ? renderFor(`ALTER TABLE ${n.newTable} DELETE WHERE ${onShardServers(shard)} AND ${where}`, run.target.topology)
+        : `ALTER TABLE ${n.newTable} DELETE WHERE ${where}`;
+    },
   };
 }
 
@@ -269,23 +307,28 @@ async function laterRowsCopy(
  * table and the difference copied again, a few rounds at most. What is left
  * after that, the verification finds. Returns the rows repaired.
  */
-async function copyAfterCutover(run: RebuildRun, o: RebuildObservation, later: LaterRows, p: string, queryId: string): Promise<number> {
+async function copyAfterCutover(run: RebuildRun, o: RebuildObservation, later: LaterRows, p: string, queryId: string, shard?: Shard): Promise<number> {
   const n = o.names;
+  const sharded = shard !== undefined && run.sharding !== undefined;
   const insert = async () => {
-    const sql = later.copy(p);
+    const sql = later.copy(p, shard);
     run.log(sql);
-    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0" }, ...(run.signal ? { signal: run.signal } : {}) });
+    await clickhouseQuery(run.target.endpoint, sql, {
+      queryId,
+      settings: { async_insert: "0", insert_deduplicate: "0", ...(sharded ? SHARD_INSERT_SETTINGS : {}) },
+      ...(run.signal ? { signal: run.signal } : {}),
+    });
   };
   await insert();
   let repaired = 0;
   for (let round = 0; round < REPAIR_ROUNDS; round++) {
     await syncReplica(run, o, n.database, n.newName);
-    const extras = (await clickhouseQuery<{ h: string }>(run.target.endpoint, later.extras(p))).map((r) => r.h);
+    const extras = (await clickhouseQuery<{ h: string }>(run.target.endpoint, later.extras(p, shard))).map((r) => r.h);
     if (extras.length === 0 || extras.length > REPAIR_LIMIT) break;
     run.log(`-- partition ${p}: ${extras.length} row(s) after the cut-over in the new table more often than in the old one; deleting them and copying again`);
-    const sql = later.drop(p, extras);
+    const sql = later.drop(p, extras, shard);
     run.log(sql);
-    await clickhouseQuery(run.target.endpoint, sql);
+    await clickhouseQuery(run.target.endpoint, sql, sharded ? { settings: { ...DDL_SETTINGS, ...SHARD_DELETE_SETTINGS } } : {});
     await waitOn(run, n.database, n.newName);
     repaired += extras.length;
     await insert();