@intentius/chant-lexicon-sql 0.114.1 → 0.115.0

package/src/clickhouse/plan/deep.ts

@@ -12,12 +12,20 @@
  * `deepNormalizationHooks` prunes what is not a property of the object (the
  * DDL text, the template source, lineage, a view's reads and `TO` target).
  * `chant sql plan` covers the target, and classifies every change.
+ *
+ * On a `cluster:<name>` topology every replica of every shard is read
+ * (#3664). When a server's copy of a table differs from the profile
+ * server's, that copy is what is reported, with the server in `observedOn`,
+ * which the drift report prints as `seen on`.
  */
 
 import { deepObservation, type DeepObservationResult, type DeepResourceObservation, type DeepNormalizationHooks } from "@intentius/chant/deep-observation";
 import type { UnobservedEntity } from "@intentius/chant/lexicon";
-import { bindClickHouse, classifyClickHouseFailure, type BindOptions } from "../live/bind";
-import { readLiveSchema, type LiveObject } from "../live/catalog";
+import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "../live/bind";
+import { readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
+import { clickhouseQuery } from "../http";
+import { sqlString } from "../apply/statements";
+import { stripMarkerFromStatement } from "../ownership";
 import { database, table, view, CLICKHOUSE_ENTITY_TYPES, type ColumnDef } from "../entities";
 import { canonicalObject, type CanonicalColumn, type CanonicalObject } from "./normalize";
 import { renderFor } from "../topology";
@@ -76,6 +84,73 @@ export function inDeclaredVocabulary(declared: Props, live: Props, d: CanonicalO
   return out;
 }
 
+/** Every server's copy of each table and view on a cluster (#3664), as `<database>.<name>` → host → statement. */
+interface ClusterCopies {
+  /** The host the profile names, whose copy the rest are compared with. */
+  self: string;
+  byKey: Map<string, Map<string, string>>;
+}
+
+/**
+ * On a `cluster:<name>` topology, read `system.tables` on every replica of
+ * every shard through the profile's server (`clusterAllReplicas`). A change
+ * made on one server without `ON CLUSTER` (a TTL altered on shard 2 alone)
+ * is then seen, which the profile's server alone would not show. Undefined
+ * for any other topology: there is one copy.
+ */
+async function clusterCopies(target: ClickHouseTarget): Promise<ClusterCopies | undefined> {
+  if (target.topology?.kind !== "cluster") return undefined;
+  const scope = target.databases
+    ? `IN (${target.databases.map(sqlString).join(", ")})`
+    : `NOT IN (${[...SYSTEM_DATABASES].map(sqlString).join(", ")})`;
+  const [self] = await clickhouseQuery<{ host: string }>(target.endpoint, "SELECT hostName() AS host");
+  const rows = await clickhouseQuery<{ host: string; database: string; name: string; statement: string }>(
+    target.endpoint,
+    `SELECT hostName() AS host, database, name, create_table_query AS statement ` +
+      `FROM clusterAllReplicas(${sqlString(target.topology.cluster)}, system.tables) ` +
+      `WHERE database ${scope} AND NOT is_temporary AND name NOT LIKE '.inner%' AND engine != 'Dictionary' ORDER BY host, database, name`,
+  );
+  const byKey = new Map<string, Map<string, string>>();
+  for (const r of rows) {
+    const key = `${r.database}.${r.name}`;
+    if (!byKey.has(key)) byKey.set(key, new Map());
+    byKey.get(key)!.set(r.host, stripMarkerFromStatement(r.statement));
+  }
+  return { self: self?.host ?? "", byKey };
+}
+
+/**
+ * When the servers' copies of `key` do not all read the same, the copy to
+ * report and its server: whichever departs furthest from the declaration
+ * `d`, the profile server's own winning a tie. A copy that differs from the
+ * declaration on any server is then the one the diff sees. Undefined when
+ * every copy reads the same.
+ */
+function copyToReport(copies: ClusterCopies, key: string, d: CanonicalObject, defaultDatabase: string): { host: string; statement: string } | undefined {
+  const byHost = copies.byKey.get(key);
+  const mine = byHost?.get(copies.self);
+  if (!byHost || mine === undefined) return undefined;
+  const shape = (c: CanonicalObject) => {
+    const { columns, ...rest } = c;
+    return JSON.stringify({ ...rest, columns: columns.map(({ text: _t, position: _p, ...col }) => col) });
+  };
+  const reference = shape(canonicalObject(mine, defaultDatabase));
+  const declared = JSON.parse(shape(d)) as Record<string, unknown>;
+  /** How many of the declaration's fields this copy holds differently. */
+  const departures = (statement: string) => {
+    const live = JSON.parse(shape(canonicalObject(statement, defaultDatabase))) as Record<string, unknown>;
+    return Object.keys({ ...declared, ...live }).filter((k) => k !== "name" && k !== "database" && JSON.stringify(declared[k]) !== JSON.stringify(live[k])).length;
+  };
+  let pick: { host: string; statement: string; score: number } | undefined;
+  for (const [host, statement] of byHost) {
+    if (host === copies.self || shape(canonicalObject(statement, defaultDatabase)) === reference) continue;
+    const score = departures(statement);
+    if (!pick || score > pick.score) pick = { host, statement, score };
+  }
+  if (!pick) return undefined;
+  return pick.score > departures(mine) ? { host: pick.host, statement: pick.statement } : { host: copies.self, statement: mine };
+}
+
 export async function observeResourcesDeep(
   options: {
     environment: string;
@@ -98,6 +173,17 @@ export async function observeResourcesDeep(
     return deepObservation({}, unobserved);
   }
   const byKey = new Map(live.map((o) => [`${o.database ?? ""}.${o.name}`, o]));
+  let copies: ClusterCopies | undefined;
+  try {
+    copies = await clusterCopies(target);
+  } catch (err) {
+    // A cluster whose servers cannot all be read is not reported as matching.
+    const why = classifyClickHouseFailure(err);
+    for (const name of options.entityNames) {
+      unobserved[name] = { type: options.entities.get(name)?.entityType ?? "", reason: why.reason, detail: `reading every server of the cluster: ${why.detail}` };
+    }
+    return deepObservation({}, unobserved);
+  }
   for (const name of options.entityNames) {
     const entity = options.entities.get(name);
     if (!entity || !entity.entityType.startsWith("ClickHouse::")) {
@@ -111,10 +197,23 @@ export async function observeResourcesDeep(
     if (!o) continue;
     try {
       const d = canonicalObject(renderFor(String(props.ddl), target.topology), target.defaultDatabase);
+      const physicalId = o.uuid ?? `${db ? `${db}.` : ""}${o.name}`;
+      const other = copies && !isDb ? copyToReport(copies, `${db ?? ""}.${o.name}`, d, target.defaultDatabase) : undefined;
+      if (other) {
+        // The servers disagree: report the copy that departs from the declaration, and where it was seen.
+        const copy: LiveObject = { ...o, statement: other.host === copies!.self ? o.statement : other.statement };
+        resources[name] = {
+          type: o.type,
+          physicalId,
+          properties: inDeclaredVocabulary(props, liveProps(copy), d, canonicalObject(copy.statement, target.defaultDatabase)),
+          observedOn: other.host,
+        };
+        continue;
+      }
       const l = canonicalObject(o.statement, target.defaultDatabase);
       resources[name] = {
         type: o.type,
-        physicalId: o.uuid ?? `${db ? `${db}.` : ""}${o.name}`,
+        physicalId,
         properties: inDeclaredVocabulary(props, liveProps(o), d, l),
       };
     } catch (err) {

package/src/clickhouse/plan/normalize.ts

@@ -284,6 +284,10 @@ export function canonicalObject(ddl: string, defaultDatabase = "default"): Canon
       name = shared[1]!;
       if (/^'.*'$/s.test(args[0] ?? "") && /^'.*'$/s.test(args[1] ?? "")) args = args.slice(2);
     }
+    // `Distributed(cluster, db, table, ...)` takes its first three arguments
+    // as identifiers or as strings, and the server prints them back quoted
+    // (#3664): an unquoted declaration says the same.
+    if (name === "Distributed") args = args.map((a, i) => (i < 3 ? quotedIdentifier(a) : a));
     obj.engineName = name;
     obj.engine = args.length ? `${name}(${args.join(", ")})` : name;
   }
@@ -359,3 +363,9 @@ function column(tokens: Token[], c: ColumnNode, position: number, database?: str
   if (prev) out.previously = prev;
   return out;
 }
+
+/** A bare or backquoted identifier as the string literal the server prints it as; anything else as it is. */
+function quotedIdentifier(arg: string): string {
+  const bare = /^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) ? arg : /^`((?:[^`]|``)+)`$/.exec(arg)?.[1]?.replace(/``/g, "`");
+  return bare === undefined ? arg : `'${bare.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+}

package/src/clickhouse/plan/plan.test.ts

@@ -87,6 +87,16 @@ describe("normalization", () => {
     expect(shown("ReplicatedMergeTree('/clickhouse/tables/{shard}/t', '{replica}')")).not.toBe(shown("ReplicatedMergeTree"));
   });
 
+  test("a Distributed engine's cluster, database and table read the same quoted or not (#3664)", () => {
+    const shown = (engine: string) => canonicalObject(`CREATE TABLE d.events_all (id UInt64) ENGINE = ${engine}`).engine;
+    expect(shown("Distributed(my_cluster, d, events, id)")).toBe(shown("Distributed('my_cluster', 'd', 'events', id)"));
+    expect(shown("Distributed(`my_cluster`, `d`, `events`)")).toBe(shown("Distributed('my_cluster', 'd', 'events')"));
+    expect(shown("Distributed('{cluster}', currentDatabase(), events, rand())")).toBe(shown("Distributed('{cluster}', currentDatabase(), 'events', rand())"));
+    // The sharding key is an expression, not a name: it stays as written.
+    expect(shown("Distributed(c, d, events, id)")).not.toBe(shown("Distributed(c, d, events, 'id')"));
+    expect(shown("Distributed(c, d, events)")).not.toBe(shown("Distributed(c, d, other)"));
+  });
+
   test("a setting at its default is not part of the definition", () => {
     expect(canonicalObject("CREATE TABLE t (a UInt8) ENGINE = MergeTree ORDER BY a SETTINGS index_granularity = 8192").settings).toEqual({});
     expect(canonicalObject("CREATE TABLE t (a UInt8) ENGINE = MergeTree ORDER BY a SETTINGS min_bytes_for_wide_part = '0'").settings).toEqual({ min_bytes_for_wide_part: "0" });

package/src/clickhouse/rebuild/backfill.ts

@@ -42,6 +42,11 @@
  * either finds nothing that replica alone has and returns, or stops the step
  * naming it (`./replicas.ts`).
  *
+ * On a cluster of more than one shard (#3663) the unit is a partition on one
+ * shard: read from that shard, written back to it, cleared on it alone, with
+ * its own receipt (`./shards.ts`), so a rerun after a failure on one shard
+ * skips what every shard already copied.
+ *
  * In materialized-view mode a partition is copied in two parts
  * (INTENTIUS/sql-yodeler#45). The rows before the cut-over are copied as
  * above. The rows at or after it are the view's only when they were written
@@ -64,9 +69,11 @@ import { ident, sqlString } from "../apply/statements";
 import { clickhouseReceiptStore, receiptAddress, type ClickHouseReceiptStore } from "./receipts";
 import { RebuildRefusal, type RebuildObservation } from "./observe";
 import { DDL_SETTINGS } from "./replicas";
-import { sourcePartitionExpression, sourcePartitions } from "./partitions";
+import { shardPartitions, sourcePartitionExpression, sourcePartitions } from "./partitions";
 import { rowHash } from "./verify";
 import { cutoverOf, observe, serverNow, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
+import { intoShard, onShard, onShardServers, shardTable, SHARD_DELETE_SETTINGS, SHARD_INSERT_SETTINGS, type Shard } from "./shards";
+import { renderFor } from "../topology";
 
 export interface BackfillResult {
   state: RebuildObservation["state"];
@@ -83,11 +90,11 @@ export interface BackfillResult {
 }
 
 /** The effect a partition's copy is, and its receipt's address suffix. */
-export const partitionEffect = (key: string, partition: string): string => `rebuild/${key}/${partition}`;
+export const partitionEffect = (key: string, partition: string, shard?: Shard): string => (shard ? `rebuild/${key}/shard${shard.num}/${partition}` : `rebuild/${key}/${partition}`);
 
-/** The receipt's expected value for one partition's copy. */
-export function partitionExpectation(o: RebuildObservation, partition: string, cutover: number | undefined): string {
-  const effect = partitionEffect(o.names.key, partition);
+/** The receipt's expected value for one partition's copy, on one shard of a cluster when `shard` is given. */
+export function partitionExpectation(o: RebuildObservation, partition: string, cutover: number | undefined, shard?: Shard): string {
+  const effect = partitionEffect(o.names.key, partition, shard);
   return receiptExpectation(
     EffectReceipt(effect, {
       effect,
@@ -96,6 +103,7 @@ export function partitionExpectation(o: RebuildObservation, partition: string, c
         table: o.names.key,
         newTable: o.newTable!.uuid,
         partition,
+        ...(shard ? { shard: shard.num } : {}),
         copied: o.copied,
         cutover: cutover === undefined ? null : new Date(cutover).toISOString(),
         // The rows at or after the cut-over the old table held are copied too (sql-yodeler#45); a receipt from before that copied fewer.
@@ -108,8 +116,8 @@ export function partitionExpectation(o: RebuildObservation, partition: string, c
 export interface BackfillDeps {
   /** The receipt store. Default: the receipts table on the target server. */
   receipts?: ClickHouseReceiptStore;
-  /** Called after each partition is copied and its receipt written; a test interrupts the backfill here. */
-  afterPartition?: (partition: string) => void | Promise<void>;
+  /** Called after each partition is copied and its receipt written, with its shard on a cluster of shards; a test interrupts the backfill here. */
+  afterPartition?: (partition: string, shard?: number) => void | Promise<void>;
 }
 
 export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promise<BackfillResult> {
@@ -140,6 +148,12 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
   const columns = o.copied.map((c) => ident(c.name)).join(", ");
   const select = o.copied.map((c) => ident(c.source)).join(", ");
   const later = cutColumn && cutover !== undefined ? await laterRowsCopy(run, o, cutColumn, cutover, fromOld) : undefined;
+  const sharding = run.sharding;
+  // The old and new tables as one shard's rows (#3663, `./shards.ts`), or as this server's.
+  const oldFrom = (shard?: Shard) => (shard && sharding ? `${shardTable(sharding, n.database, n.name)} WHERE ${onShard(shard)} AND` : `${n.table} WHERE`);
+  const newFrom = (shard?: Shard) => (shard && sharding ? `${shardTable(sharding, n.database, n.newName)} WHERE ${onShard(shard)} AND` : `${n.newTable} WHERE`);
+  const into = (shard?: Shard) => (shard && sharding ? intoShard(sharding, shard, n.database, n.newName) : n.newTable);
+  const insertSettings = sharding ? SHARD_INSERT_SETTINGS : {};
 
   const identity = { ...(run.marker?.stack ? { stack: run.marker.stack } : {}), ...(run.marker?.env ? { env: run.marker.env } : {}) };
   const receipts =
@@ -152,21 +166,31 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
     });
   const recorded = await receipts.readAll(receiptAddress(identity, `rebuild/${n.key}/`));
 
-  const partitions = await sourcePartitions(run.target, n);
-  result.partitions = partitions.length;
-  for (const p of partitions) {
+  // The unit of work: a partition of the old table, on each shard of a cluster that has more than one.
+  const units: Array<{ shard?: Shard; partition: string }> = sharding
+    ? (await shardPartitions(run, sharding, n.database, n.name)).map((u) => ({ shard: sharding.shards.find((x) => x.num === u.shard)!, partition: u.partition }))
+    : (await sourcePartitions(run.target, n)).map((partition) => ({ partition }));
+  result.partitions = units.length;
+  for (const { shard, partition: p } of units) {
     run.signal?.throwIfAborted();
-    const effect = partitionEffect(n.key, p);
-    const expectation = partitionExpectation(o, p, cutover);
+    const effect = partitionEffect(n.key, p, shard);
+    const expectation = partitionExpectation(o, p, cutover, shard);
     if (recorded.get(receiptAddress(identity, effect)) === expectation) {
       result.skipped++;
       continue;
     }
 
-    const queryId = `chant-rebuild-${o.newTable.uuid}-${p}`;
+    const queryId = `chant-rebuild-${o.newTable.uuid}-${shard ? `shard${shard.num}-` : ""}${p}`;
     const laterQueryId = `${queryId}-after`;
     const killWhere = `query_id IN (${sqlString(queryId)}, ${sqlString(laterQueryId)})`;
-    if (o.replicated) {
+    if (sharding) {
+      // The copy reads and writes every shard from this server; what it started on the others goes with it.
+      await clickhouseQuery(
+        run.target.endpoint,
+        `KILL QUERY ON CLUSTER ${sqlString(sharding.cluster)} WHERE ${killWhere} OR initial_query_id IN (${sqlString(queryId)}, ${sqlString(laterQueryId)}) SYNC`,
+        { settings: DDL_SETTINGS },
+      );
+    } else if (o.replicated) {
       // On every replica that answers: the copy may be running on another one
       // than this run's. A replica that is down is skipped once Keeper sees it
       // inactive; nothing of its can be running a copy that still writes,
@@ -179,24 +203,28 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
       await clickhouseQuery(run.target.endpoint, `KILL QUERY WHERE ${killWhere} SYNC`);
     }
     const mine = `${fromOld} = ${sqlString(p)}${targetRange}`;
-    const [left] = await clickhouseQuery<{ n: string | number }>(run.target.endpoint, `SELECT count() AS n FROM ${n.newTable} WHERE ${mine}`);
+    const where = shard ? `partition ${p} on shard ${shard.num}` : `partition ${p}`;
+    const [left] = await clickhouseQuery<{ n: string | number }>(run.target.endpoint, `SELECT count() AS n FROM ${newFrom(shard)} ${mine}`);
     if (Number(left?.n ?? 0) > 0) {
-      run.log(`-- partition ${p}: ${left!.n} row(s) from a copy that did not finish; deleting them before copying again`);
-      const sql = `ALTER TABLE ${n.newTable} DELETE WHERE ${mine}`;
+      run.log(`-- ${where}: ${left!.n} row(s) from a copy that did not finish; deleting them before copying again`);
+      const sql =
+        shard && sharding
+          ? renderFor(`ALTER TABLE ${n.newTable} DELETE WHERE ${onShardServers(shard)} AND ${mine}`, run.target.topology)
+          : `ALTER TABLE ${n.newTable} DELETE WHERE ${mine}`;
       run.log(sql);
-      await clickhouseQuery(run.target.endpoint, sql);
+      await clickhouseQuery(run.target.endpoint, sql, shard && sharding ? { settings: { ...DDL_SETTINGS, ...SHARD_DELETE_SETTINGS } } : {});
       await waitOn(run, n.database, n.newName);
       result.cleared++;
     }
 
-    const sql = `INSERT INTO ${n.newTable} (${columns}) SELECT ${select} FROM ${n.table} WHERE _partition_id = ${sqlString(p)}${sourceRange}`;
+    const sql = `INSERT INTO ${into(shard)} (${columns}) SELECT ${select} FROM ${oldFrom(shard)} _partition_id = ${sqlString(p)}${sourceRange}`;
     run.log(sql);
-    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0" }, ...(run.signal ? { signal: run.signal } : {}) });
-    if (later) result.repaired += await copyAfterCutover(run, o, later, p, laterQueryId);
+    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0", ...insertSettings }, ...(run.signal ? { signal: run.signal } : {}) });
+    if (later) result.repaired += await copyAfterCutover(run, o, later, p, laterQueryId, shard);
     // The receipt, last, on success only.
     await receipts.write({ name: effect, effect, flavor: "hash", inputs: {} }, expectation);
     result.copied++;
-    await deps.afterPartition?.(p);
+    await deps.afterPartition?.(p, shard?.num);
   }
   run.log(`-- backfill of ${n.key}: ${result.partitions} partition(s), ${result.copied} copied, ${result.skipped} already copied, ${result.cleared} cleared first${result.repaired > 0 ? `, ${result.repaired} row(s) after the cut-over copied twice and repaired` : ""}`);
   return result;
@@ -206,13 +234,14 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
 interface LaterRows {
   /**
    * Per distinct row (by its hash), the old table's copies past the number
-   * the new table holds, into the new table.
+   * the new table holds, into the new table. On a cluster of shards, one
+   * shard's rows into the same shard.
    */
-  copy: (partition: string) => string;
+  copy: (partition: string, shard?: Shard) => string;
   /** The hashes of the rows the new table holds more copies of than the old one. */
-  extras: (partition: string) => string;
+  extras: (partition: string, shard?: Shard) => string;
   /** Delete every copy of those rows from the new table. */
-  drop: (partition: string, hashes: readonly string[]) => string;
+  drop: (partition: string, hashes: readonly string[], shard?: Shard) => string;
 }
 
 /** The most extra rows one partition's repair deletes and copies again; past that the verification decides. */
@@ -245,19 +274,28 @@ async function laterRowsCopy(
   const columns = o.copied.map((c) => ident(c.name)).join(", ");
   const aliases = o.copied.map((_, i) => `__chant_c${i}`);
   const fromOldSide = o.copied.map((c, i) => `${ident(c.source)} AS ${aliases[i]}`).join(", ");
-  const oldRows = (p: string) => `FROM ${n.table} WHERE _partition_id = ${sqlString(p)} AND ${ident(cut.source)} >= ${utcLiteral(cutover)}`;
-  const newRows = (p: string) => `FROM ${n.newTable} WHERE ${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)}`;
+  const sharding = run.sharding;
+  const on = (shard?: Shard) => (shard && sharding ? ` AND ${onShard(shard)}` : "");
+  const oldTable = (shard?: Shard) => (shard && sharding ? shardTable(sharding, n.database, n.name) : n.table);
+  const newTable = (shard?: Shard) => (shard && sharding ? shardTable(sharding, n.database, n.newName) : n.newTable);
+  const oldRows = (p: string, shard?: Shard) => `FROM ${oldTable(shard)} WHERE _partition_id = ${sqlString(p)} AND ${ident(cut.source)} >= ${utcLiteral(cutover)}${on(shard)}`;
+  const newRows = (p: string, shard?: Shard) => `FROM ${newTable(shard)} WHERE ${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)}${on(shard)}`;
   return {
-    copy: (p) =>
-      `INSERT INTO ${n.newTable} (${columns}) SELECT ${aliases.join(", ")} FROM (` +
-      `SELECT ${fromOldSide}, ${oldHash} AS __chant_h, row_number() OVER (PARTITION BY __chant_h) AS __chant_k ${oldRows(p)}` +
-      `) AS old_rows LEFT JOIN (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p)} GROUP BY __chant_h) AS new_rows ` +
+    copy: (p, shard) =>
+      `INSERT INTO ${shard && sharding ? intoShard(sharding, shard, n.database, n.newName) : n.newTable} (${columns}) SELECT ${aliases.join(", ")} FROM (` +
+      `SELECT ${fromOldSide}, ${oldHash} AS __chant_h, row_number() OVER (PARTITION BY __chant_h) AS __chant_k ${oldRows(p, shard)}` +
+      `) AS old_rows LEFT JOIN (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p, shard)} GROUP BY __chant_h) AS new_rows ` +
       `USING (__chant_h) WHERE __chant_k > ifNull(__chant_n, 0)`,
-    extras: (p) =>
-      `SELECT toString(__chant_h) AS h FROM (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p)} GROUP BY __chant_h) AS new_rows ` +
-      `LEFT JOIN (SELECT ${oldHash} AS __chant_h, count() AS __chant_o ${oldRows(p)} GROUP BY __chant_h) AS old_rows ` +
+    extras: (p, shard) =>
+      `SELECT toString(__chant_h) AS h FROM (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p, shard)} GROUP BY __chant_h) AS new_rows ` +
+      `LEFT JOIN (SELECT ${oldHash} AS __chant_h, count() AS __chant_o ${oldRows(p, shard)} GROUP BY __chant_h) AS old_rows ` +
       `USING (__chant_h) WHERE __chant_n > ifNull(__chant_o, 0) LIMIT ${REPAIR_LIMIT + 1}`,
-    drop: (p, hashes) => `ALTER TABLE ${n.newTable} DELETE WHERE ${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)} AND ${newHash} IN (${hashes.join(", ")})`,
+    drop: (p, hashes, shard) => {
+      const where = `${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)} AND ${newHash} IN (${hashes.join(", ")})`;
+      return shard && sharding
+        ? renderFor(`ALTER TABLE ${n.newTable} DELETE WHERE ${onShardServers(shard)} AND ${where}`, run.target.topology)
+        : `ALTER TABLE ${n.newTable} DELETE WHERE ${where}`;
+    },
   };
 }
 
@@ -269,23 +307,28 @@ async function laterRowsCopy(
  * table and the difference copied again, a few rounds at most. What is left
  * after that, the verification finds. Returns the rows repaired.
  */
-async function copyAfterCutover(run: RebuildRun, o: RebuildObservation, later: LaterRows, p: string, queryId: string): Promise<number> {
+async function copyAfterCutover(run: RebuildRun, o: RebuildObservation, later: LaterRows, p: string, queryId: string, shard?: Shard): Promise<number> {
   const n = o.names;
+  const sharded = shard !== undefined && run.sharding !== undefined;
   const insert = async () => {
-    const sql = later.copy(p);
+    const sql = later.copy(p, shard);
     run.log(sql);
-    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0" }, ...(run.signal ? { signal: run.signal } : {}) });
+    await clickhouseQuery(run.target.endpoint, sql, {
+      queryId,
+      settings: { async_insert: "0", insert_deduplicate: "0", ...(sharded ? SHARD_INSERT_SETTINGS : {}) },
+      ...(run.signal ? { signal: run.signal } : {}),
+    });
   };
   await insert();
   let repaired = 0;
   for (let round = 0; round < REPAIR_ROUNDS; round++) {
     await syncReplica(run, o, n.database, n.newName);
-    const extras = (await clickhouseQuery<{ h: string }>(run.target.endpoint, later.extras(p))).map((r) => r.h);
+    const extras = (await clickhouseQuery<{ h: string }>(run.target.endpoint, later.extras(p, shard))).map((r) => r.h);
     if (extras.length === 0 || extras.length > REPAIR_LIMIT) break;
     run.log(`-- partition ${p}: ${extras.length} row(s) after the cut-over in the new table more often than in the old one; deleting them and copying again`);
-    const sql = later.drop(p, extras);
+    const sql = later.drop(p, extras, shard);
     run.log(sql);
-    await clickhouseQuery(run.target.endpoint, sql);
+    await clickhouseQuery(run.target.endpoint, sql, sharded ? { settings: { ...DDL_SETTINGS, ...SHARD_DELETE_SETTINGS } } : {});
     await waitOn(run, n.database, n.newName);
     repaired += extras.length;
     await insert();

package/src/clickhouse/rebuild/op.ts

@@ -25,6 +25,12 @@
  * | Drop | the old table, once its date has passed |
  * | onFailure | drop the new table and the dual-write view |
  *
+ * `gates: "outer"` (#3658) leaves out both approval gates and the Drop phase,
+ * for a caller that runs the Op under an approval of its own: the run goes
+ * from Plan to Retain, verification and the swap's own comparison included.
+ * `onFailure: "keep"` leaves out the onFailure, so a failed run's next run
+ * resumes the backfill from its receipts instead of starting again.
+ *
  * Run it until it is done: each run goes as far as the next gate. Every step
  * re-reads the server, so a run after a gate, a crash or an approval picks up
  * where the last stopped. A run killed during the backfill (Ctrl-C, a lost
@@ -55,6 +61,27 @@ export interface ClickHouseRebuildOpConfig {
   build?: boolean;
   /** How long the old table is kept after the swap, as a duration (`7d`, `36h`). Default: `7d`. */
   retain?: string;
+  /**
+   * Whose approval the swap and the drop wait for. `"own"` (the default): the
+   * Op's two gates. `"outer"`: an approval the caller already holds for a
+   * larger change that runs this Op as one of its steps. The swap gate, the
+   * drop gate and the Drop phase are left out, so the Op ends at Retain with
+   * the old table kept as `<table>__chant_old` with its retention date. Drop
+   * it after that date, or run the Op with `"own"`: its swap gate has
+   * nothing left to swap, and its drop gate binds that old table.
+   * The verification still runs, and the swap compares the tables again
+   * before the `EXCHANGE`: any difference fails the run with nothing swapped.
+   * App mode's writes-stopped gate is not an approval and stays.
+   */
+  gates?: "own" | "outer";
+  /**
+   * What a failed run does with the new table and the dual-write view.
+   * `"drop"` (the default): onFailure drops them, and the next run starts
+   * again from a new table. `"keep"`: there is no onFailure, and the next run
+   * resumes the backfill from its per-partition receipts. To start again
+   * instead, drop `<table>__chant_new` and `<table>__chant_dual`.
+   */
+  onFailure?: "drop" | "keep";
   /** The gate before the swap. Default name `approve-<name>`. */
   gate?: {
     gate?: string;
@@ -100,6 +127,8 @@ export interface ClickHouseRebuildArgs {
   stack?: string;
   ownershipEnv?: string;
   cwd?: string;
+  /** A failed run keeps the new table (`onFailure: "keep"`): a refusal says to drop it by hand. */
+  keepOnFailure?: boolean;
 }
 
 export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHouseRebuildOpResources {
@@ -112,6 +141,14 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
   if (config.dualWrite.mode === "materialized-view" && !config.dualWrite.cutoverColumn) {
     throw new Error(`ClickHouseRebuildOp "${config.name}": materialized-view mode needs cutoverColumn, the time column rows are cut over on`);
   }
+  if (config.gates !== undefined && config.gates !== "own" && config.gates !== "outer") {
+    throw new Error(`ClickHouseRebuildOp "${config.name}": gates must be "own" or "outer"`);
+  }
+  if (config.onFailure !== undefined && config.onFailure !== "drop" && config.onFailure !== "keep") {
+    throw new Error(`ClickHouseRebuildOp "${config.name}": onFailure must be "drop" or "keep"`);
+  }
+  const outer = config.gates === "outer";
+  const keep = config.onFailure === "keep";
 
   const args: ClickHouseRebuildArgs = {
     table: config.table,
@@ -124,6 +161,7 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
     ...(config.stack ? { stack: config.stack } : {}),
     ...(config.ownershipEnv ? { ownershipEnv: config.ownershipEnv } : {}),
     ...(config.path && config.path !== "." ? { cwd: config.path } : {}),
+    ...(keep ? { keepOnFailure: true } : {}),
   };
   const a = args as unknown as Record<string, unknown>;
   const swapGate = config.gate?.gate ?? `approve-${config.name}`;
@@ -133,6 +171,33 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
   const step = (fn: string, extra: Partial<StepDefinition & { id: string; outcomeAttribute: unknown; profile: string; timeout: string }> = {}): StepDefinition =>
     ({ ...activity(fn, a), ...extra }) as StepDefinition;
 
+  // The swap waits for this gate, unless the caller holds the approval (`gates: "outer"`).
+  const approve = phase("Approve", [
+    gate(swapGate, {
+      ...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
+      plan: stepOutput("verify", "planDigest"),
+      description:
+        config.gate?.description ??
+        `Approve swapping the rebuilt ${config.table} in on ${config.env}: the run record's Verification line has the counts this approval binds`,
+      ...(config.gate?.approval
+        ? {
+            approval: {
+              ...config.gate.approval,
+              ...(policy
+                ? {
+                    context: {
+                      verifiedPartitions: stepOutput("verify", "partitions"),
+                      verifiedRows: stepOutput("verify", "rows"),
+                      ...config.gate.approval.context,
+                    },
+                  }
+                : {}),
+            },
+          }
+        : {}),
+    }),
+  ]);
+
   const phases = [
     ...(config.build === false ? [] : [phase("Build", [activity("chantBuild", { path: config.path ?? "." })])]),
     phase("Plan", [step("clickhouseRebuildPlan", { id: "plan", outcomeAttribute: { name: "RebuildState", from: "state" } })]),
@@ -158,49 +223,29 @@ export function ClickHouseRebuildOp(config: ClickHouseRebuildOpConfig): ClickHou
         ],
       }),
     ]),
-    phase("Approve", [
-      gate(swapGate, {
-        ...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
-        plan: stepOutput("verify", "planDigest"),
-        description:
-          config.gate?.description ??
-          `Approve swapping the rebuilt ${config.table} in on ${config.env}: the run record's Verification line has the counts this approval binds`,
-        ...(config.gate?.approval
-          ? {
-              approval: {
-                ...config.gate.approval,
-                ...(policy
-                  ? {
-                      context: {
-                        verifiedPartitions: stepOutput("verify", "partitions"),
-                        verifiedRows: stepOutput("verify", "rows"),
-                        ...config.gate.approval.context,
-                      },
-                    }
-                  : {}),
-              },
-            }
-          : {}),
-      }),
-    ]),
+    ...(outer ? [] : [approve]),
     phase("Swap", [step("clickhouseRebuildSwap", { profile: "longInfra", outcomeAttribute: { name: "Dependents", from: "dependents" } })]),
     phase("Retain", [step("clickhouseRebuildRetain", { id: "retain", outcomeAttribute: { name: "RetainUntil", from: "retainUntil" } })]),
-    phase("Approve drop", [
-      gate(dropGate, {
-        ...(config.dropGate?.timeout ? { timeout: config.dropGate.timeout } : {}),
-        plan: stepOutput("retain", "dropDigest"),
-        description: config.dropGate?.description ?? `Approve dropping the old ${config.table} once its retention has passed`,
-      }),
-    ]),
-    phase("Drop", [step("clickhouseRebuildDrop", { outcomeAttribute: { name: "Dropped", from: "dropped" } })]),
+    ...(outer
+      ? []
+      : [
+          phase("Approve drop", [
+            gate(dropGate, {
+              ...(config.dropGate?.timeout ? { timeout: config.dropGate.timeout } : {}),
+              plan: stepOutput("retain", "dropDigest"),
+              description: config.dropGate?.description ?? `Approve dropping the old ${config.table} once its retention has passed`,
+            }),
+          ]),
+          phase("Drop", [step("clickhouseRebuildDrop", { outcomeAttribute: { name: "Dropped", from: "dropped" } })]),
+        ]),
   ];
 
   const op = Op({
     name: config.name,
-    overview: `Rebuild ${config.table} on ${config.env}: new table, ${config.dualWrite.mode === "app" ? "writes stopped" : "dual write"}, backfill, verify, gated swap`,
+    overview: `Rebuild ${config.table} on ${config.env}: new table, ${config.dualWrite.mode === "app" ? "writes stopped" : "dual write"}, backfill, verify, ${outer ? "swap under the caller's approval" : "gated swap"}`,
     labels: { Rebuild: "true", Env: config.env, Table: config.table },
     phases,
-    onFailure: [phase("Compensate", [step("clickhouseRebuildCompensate")])],
+    ...(keep ? {} : { onFailure: [phase("Compensate", [step("clickhouseRebuildCompensate")])] }),
   });
   return { op };
 }

package/src/clickhouse/rebuild/partitions.ts

@@ -15,6 +15,7 @@ import { clickhouseQuery } from "../http";
 import type { ClickHouseTarget } from "../live/bind";
 import { ident, sqlString } from "../apply/statements";
 import type { CopiedColumn, RebuildNames } from "./observe";
+import type { Sharding } from "./shards";
 
 /** The old table's active partitions, by id. */
 export async function sourcePartitions(target: ClickHouseTarget, names: RebuildNames): Promise<string[]> {
@@ -26,6 +27,16 @@ export async function sourcePartitions(target: ClickHouseTarget, names: RebuildN
   return rows.map((r) => r.partition_id);
 }
 
+/** The old table's active partitions on each shard of a cluster (#3663), through this server. */
+export async function shardPartitions(run: { target: ClickHouseTarget }, sharding: Sharding, database: string, table: string): Promise<Array<{ shard: number; partition: string }>> {
+  const rows = await clickhouseQuery<{ shard: number | string; partition_id: string }>(
+    run.target.endpoint,
+    `SELECT _shard_num AS shard, partition_id FROM cluster(${sqlString(sharding.cluster)}, system.parts) ` +
+      `WHERE database = ${sqlString(database)} AND table = ${sqlString(table)} AND active GROUP BY shard, partition_id ORDER BY shard, partition_id`,
+  );
+  return rows.map((r) => ({ shard: Number(r.shard), partition: r.partition_id }));
+}
+
 /** Rename the columns an expression names, old name to new, leaving function names and qualified names alone. */
 export function renameColumns(expr: string, copied: readonly CopiedColumn[]): string {
   let out = expr;