@intentius/chant-lexicon-sql 0.114.0 → 0.115.0

package/src/clickhouse/plan/cluster-drift.e2e.test.ts

@@ -0,0 +1,117 @@
+/**
+ * Drift on a multi-shard cluster (#3664), against a scratch cluster of the
+ * pinned image: two shards (two replicas and one server) with Keeper and a
+ * cluster `chant` over all three, the profile naming the first server with
+ * topology `cluster:chant`.
+ *
+ * - The schema applied `ON CLUSTER`, with a `Distributed` table declared with
+ *   unquoted identifiers, reads as no drift: the server's quoted arguments
+ *   are the same arguments.
+ * - A TTL changed on shard 2's server alone, without `ON CLUSTER`, is drift,
+ *   seen on that server, though the profile's server still holds the
+ *   declaration.
+ *
+ * Runs the sql lexicon's deep reader and core's deep diff, as
+ * `chant lifecycle diff --live` does. Needs Docker; skips cleanly without it.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { diffDeepObservation } from "@intentius/chant/lifecycle/deep-observe";
+import { normalizeDeepObservation } from "@intentius/chant/deep-observation";
+import { dockerAvailable, startScratchCluster, type ScratchCluster } from "../container";
+import { clickhouseQuery } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { clickhouseApply, toApplyResult } from "../../op/activities/clickhouse-apply";
+import { normalizeApply } from "@intentius/chant/apply";
+import { database, table } from "../entities";
+import { observeResourcesDeep, sqlDeepNormalizationHooks } from "./deep";
+
+const enabled = await dockerAvailable();
+const DB = "chant_e2e_3664";
+let cluster: ScratchCluster | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-cluster-drift-"));
+
+const DB_DDL = `CREATE DATABASE ${DB}`;
+const EVENTS_DDL = `CREATE TABLE ${DB}.events
+(
+  id UInt64,
+  ts DateTime
+)
+ENGINE = MergeTree
+ORDER BY id
+TTL ts + INTERVAL 30 DAY`;
+const EVENTS_ALL_DDL = `CREATE TABLE ${DB}.events_all
+(
+  id UInt64,
+  ts DateTime
+)
+ENGINE = Distributed(chant, ${DB}, events, id)`;
+
+const config = () => ({ ownership: { stack: "e2e", env: "test" }, sql: { profiles: { e2e: { url: cluster!.replicas[0]!.url, topology: "cluster:chant" } } } });
+const tag = (fn: (strings: TemplateStringsArray) => unknown, ddl: string) => fn(Object.assign([ddl], { raw: [ddl] }) as unknown as TemplateStringsArray) as unknown as { entityType: string; props: Record<string, unknown> };
+
+const ENTITIES = new Map([
+  ["db", tag(database, DB_DDL)],
+  ["events", tag(table, EVENTS_DDL)],
+  ["eventsAll", tag(table, EVENTS_ALL_DDL)],
+]);
+
+async function drift() {
+  const entities = new Map([...ENTITIES].map(([k, v]) => [k, { entityType: v.entityType, props: v.props }]));
+  const live = await observeResourcesDeep({ environment: "e2e", entityNames: [...entities.keys()], entities, config: config(), env: {} });
+  return diffDeepObservation(entities, normalizeDeepObservation(live), sqlDeepNormalizationHooks).drifted;
+}
+
+beforeAll(async () => {
+  if (!enabled) return;
+  cluster = await startScratchCluster(clickhouseImage(), { shards: [2, 1], cluster: "chant", namePrefix: "chant-sql-cluster-drift" });
+}, 600_000);
+
+afterAll(async () => {
+  await cluster?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+describe.skipIf(!enabled)("drift on a two-shard cluster (#3664)", () => {
+  test("applied ON CLUSTER, with unquoted Distributed arguments, it reads as no drift", async () => {
+    const build = join(dir, "schema.json");
+    writeFileSync(
+      build,
+      JSON.stringify({
+        dialect: "clickhouse",
+        applyOrder: ["db", "events", "eventsAll"],
+        objects: [
+          { export: "db", type: "ClickHouse::Database", ddl: DB_DDL, dependsOn: [] },
+          { export: "events", type: "ClickHouse::Table", ddl: EVENTS_DDL, dependsOn: ["db"] },
+          { export: "eventsAll", type: "ClickHouse::Table", ddl: EVENTS_ALL_DDL, dependsOn: ["events"] },
+        ],
+      }),
+    );
+    const applied = normalizeApply(toApplyResult(await clickhouseApply({ buildPath: build, environment: "e2e" }, undefined, { config: config(), env: {}, log: () => undefined })));
+    expect(applied.notAttempted).toEqual([]);
+    // Every server has both tables.
+    for (const endpoint of cluster!.replicas) {
+      const rows = await clickhouseQuery<{ name: string }>(endpoint, `SELECT name FROM system.tables WHERE database = '${DB}' ORDER BY name`);
+      expect(rows.map((r) => r.name)).toEqual(["events", "events_all"]);
+    }
+    expect(await drift()).toEqual([]);
+  }, 300_000);
+
+  test("a TTL changed on shard 2 alone is drift, seen on that server", async () => {
+    const shard2 = cluster!.layout.findIndex((l) => l.shard === "s2");
+    await clickhouseQuery(cluster!.replicas[shard2]!, `ALTER TABLE ${DB}.events MODIFY TTL ts + INTERVAL 7 DAY`);
+    // The profile's server still holds the declaration.
+    const [mine] = await clickhouseQuery<{ q: string }>(cluster!.replicas[0]!, `SELECT create_table_query AS q FROM system.tables WHERE database = '${DB}' AND name = 'events'`);
+    expect(mine!.q).toContain("toIntervalDay(30)");
+
+    const drifted = await drift();
+    expect(drifted.map((d) => d.name)).toEqual(["events"]);
+    const ttl = drifted[0]!.changes.find((c) => c.path.startsWith("ttl"));
+    expect(ttl, JSON.stringify(drifted[0]!.changes)).toBeDefined();
+    expect(String(ttl!.live)).toMatch(/7/);
+    expect(ttl!.seenOn).toBe(cluster!.layout[shard2]!.host);
+  }, 300_000);
+});

package/src/clickhouse/plan/deep.ts

@@ -12,12 +12,20 @@
  * `deepNormalizationHooks` prunes what is not a property of the object (the
  * DDL text, the template source, lineage, a view's reads and `TO` target).
  * `chant sql plan` covers the target, and classifies every change.
+ *
+ * On a `cluster:<name>` topology every replica of every shard is read
+ * (#3664). When a server's copy of a table differs from the profile
+ * server's, that copy is what is reported, with the server in `observedOn`,
+ * which the drift report prints as `seen on`.
  */
 
 import { deepObservation, type DeepObservationResult, type DeepResourceObservation, type DeepNormalizationHooks } from "@intentius/chant/deep-observation";
 import type { UnobservedEntity } from "@intentius/chant/lexicon";
-import { bindClickHouse, classifyClickHouseFailure, type BindOptions } from "../live/bind";
-import { readLiveSchema, type LiveObject } from "../live/catalog";
+import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "../live/bind";
+import { readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
+import { clickhouseQuery } from "../http";
+import { sqlString } from "../apply/statements";
+import { stripMarkerFromStatement } from "../ownership";
 import { database, table, view, CLICKHOUSE_ENTITY_TYPES, type ColumnDef } from "../entities";
 import { canonicalObject, type CanonicalColumn, type CanonicalObject } from "./normalize";
 import { renderFor } from "../topology";
@@ -76,6 +84,73 @@ export function inDeclaredVocabulary(declared: Props, live: Props, d: CanonicalO
   return out;
 }
 
+/** Every server's copy of each table and view on a cluster (#3664), as `<database>.<name>` → host → statement. */
+interface ClusterCopies {
+  /** The host the profile names, whose copy the rest are compared with. */
+  self: string;
+  byKey: Map<string, Map<string, string>>;
+}
+
+/**
+ * On a `cluster:<name>` topology, read `system.tables` on every replica of
+ * every shard through the profile's server (`clusterAllReplicas`). A change
+ * made on one server without `ON CLUSTER` (a TTL altered on shard 2 alone)
+ * is then seen, which the profile's server alone would not show. Undefined
+ * for any other topology: there is one copy.
+ */
+async function clusterCopies(target: ClickHouseTarget): Promise<ClusterCopies | undefined> {
+  if (target.topology?.kind !== "cluster") return undefined;
+  const scope = target.databases
+    ? `IN (${target.databases.map(sqlString).join(", ")})`
+    : `NOT IN (${[...SYSTEM_DATABASES].map(sqlString).join(", ")})`;
+  const [self] = await clickhouseQuery<{ host: string }>(target.endpoint, "SELECT hostName() AS host");
+  const rows = await clickhouseQuery<{ host: string; database: string; name: string; statement: string }>(
+    target.endpoint,
+    `SELECT hostName() AS host, database, name, create_table_query AS statement ` +
+      `FROM clusterAllReplicas(${sqlString(target.topology.cluster)}, system.tables) ` +
+      `WHERE database ${scope} AND NOT is_temporary AND name NOT LIKE '.inner%' AND engine != 'Dictionary' ORDER BY host, database, name`,
+  );
+  const byKey = new Map<string, Map<string, string>>();
+  for (const r of rows) {
+    const key = `${r.database}.${r.name}`;
+    if (!byKey.has(key)) byKey.set(key, new Map());
+    byKey.get(key)!.set(r.host, stripMarkerFromStatement(r.statement));
+  }
+  return { self: self?.host ?? "", byKey };
+}
+
+/**
+ * When the servers' copies of `key` do not all read the same, the copy to
+ * report and its server: whichever departs furthest from the declaration
+ * `d`, the profile server's own winning a tie. A copy that differs from the
+ * declaration on any server is then the one the diff sees. Undefined when
+ * every copy reads the same.
+ */
+function copyToReport(copies: ClusterCopies, key: string, d: CanonicalObject, defaultDatabase: string): { host: string; statement: string } | undefined {
+  const byHost = copies.byKey.get(key);
+  const mine = byHost?.get(copies.self);
+  if (!byHost || mine === undefined) return undefined;
+  const shape = (c: CanonicalObject) => {
+    const { columns, ...rest } = c;
+    return JSON.stringify({ ...rest, columns: columns.map(({ text: _t, position: _p, ...col }) => col) });
+  };
+  const reference = shape(canonicalObject(mine, defaultDatabase));
+  const declared = JSON.parse(shape(d)) as Record<string, unknown>;
+  /** How many of the declaration's fields this copy holds differently. */
+  const departures = (statement: string) => {
+    const live = JSON.parse(shape(canonicalObject(statement, defaultDatabase))) as Record<string, unknown>;
+    return Object.keys({ ...declared, ...live }).filter((k) => k !== "name" && k !== "database" && JSON.stringify(declared[k]) !== JSON.stringify(live[k])).length;
+  };
+  let pick: { host: string; statement: string; score: number } | undefined;
+  for (const [host, statement] of byHost) {
+    if (host === copies.self || shape(canonicalObject(statement, defaultDatabase)) === reference) continue;
+    const score = departures(statement);
+    if (!pick || score > pick.score) pick = { host, statement, score };
+  }
+  if (!pick) return undefined;
+  return pick.score > departures(mine) ? { host: pick.host, statement: pick.statement } : { host: copies.self, statement: mine };
+}
+
 export async function observeResourcesDeep(
   options: {
     environment: string;
@@ -98,6 +173,17 @@ export async function observeResourcesDeep(
     return deepObservation({}, unobserved);
   }
   const byKey = new Map(live.map((o) => [`${o.database ?? ""}.${o.name}`, o]));
+  let copies: ClusterCopies | undefined;
+  try {
+    copies = await clusterCopies(target);
+  } catch (err) {
+    // A cluster whose servers cannot all be read is not reported as matching.
+    const why = classifyClickHouseFailure(err);
+    for (const name of options.entityNames) {
+      unobserved[name] = { type: options.entities.get(name)?.entityType ?? "", reason: why.reason, detail: `reading every server of the cluster: ${why.detail}` };
+    }
+    return deepObservation({}, unobserved);
+  }
   for (const name of options.entityNames) {
     const entity = options.entities.get(name);
     if (!entity || !entity.entityType.startsWith("ClickHouse::")) {
@@ -111,10 +197,23 @@ export async function observeResourcesDeep(
     if (!o) continue;
     try {
       const d = canonicalObject(renderFor(String(props.ddl), target.topology), target.defaultDatabase);
+      const physicalId = o.uuid ?? `${db ? `${db}.` : ""}${o.name}`;
+      const other = copies && !isDb ? copyToReport(copies, `${db ?? ""}.${o.name}`, d, target.defaultDatabase) : undefined;
+      if (other) {
+        // The servers disagree: report the copy that departs from the declaration, and where it was seen.
+        const copy: LiveObject = { ...o, statement: other.host === copies!.self ? o.statement : other.statement };
+        resources[name] = {
+          type: o.type,
+          physicalId,
+          properties: inDeclaredVocabulary(props, liveProps(copy), d, canonicalObject(copy.statement, target.defaultDatabase)),
+          observedOn: other.host,
+        };
+        continue;
+      }
       const l = canonicalObject(o.statement, target.defaultDatabase);
       resources[name] = {
         type: o.type,
-        physicalId: o.uuid ?? `${db ? `${db}.` : ""}${o.name}`,
+        physicalId,
         properties: inDeclaredVocabulary(props, liveProps(o), d, l),
       };
     } catch (err) {

package/src/clickhouse/plan/normalize.ts

@@ -284,6 +284,10 @@ export function canonicalObject(ddl: string, defaultDatabase = "default"): Canon
       name = shared[1]!;
       if (/^'.*'$/s.test(args[0] ?? "") && /^'.*'$/s.test(args[1] ?? "")) args = args.slice(2);
     }
+    // `Distributed(cluster, db, table, ...)` takes its first three arguments
+    // as identifiers or as strings, and the server prints them back quoted
+    // (#3664): an unquoted declaration says the same.
+    if (name === "Distributed") args = args.map((a, i) => (i < 3 ? quotedIdentifier(a) : a));
     obj.engineName = name;
     obj.engine = args.length ? `${name}(${args.join(", ")})` : name;
   }
@@ -359,3 +363,9 @@ function column(tokens: Token[], c: ColumnNode, position: number, database?: str
   if (prev) out.previously = prev;
   return out;
 }
+
+/** A bare or backquoted identifier as the string literal the server prints it as; anything else as it is. */
+function quotedIdentifier(arg: string): string {
+  const bare = /^[A-Za-z_][A-Za-z0-9_]*$/.test(arg) ? arg : /^`((?:[^`]|``)+)`$/.exec(arg)?.[1]?.replace(/``/g, "`");
+  return bare === undefined ? arg : `'${bare.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+}

package/src/clickhouse/plan/plan.test.ts

@@ -87,6 +87,16 @@ describe("normalization", () => {
     expect(shown("ReplicatedMergeTree('/clickhouse/tables/{shard}/t', '{replica}')")).not.toBe(shown("ReplicatedMergeTree"));
   });
 
+  test("a Distributed engine's cluster, database and table read the same quoted or not (#3664)", () => {
+    const shown = (engine: string) => canonicalObject(`CREATE TABLE d.events_all (id UInt64) ENGINE = ${engine}`).engine;
+    expect(shown("Distributed(my_cluster, d, events, id)")).toBe(shown("Distributed('my_cluster', 'd', 'events', id)"));
+    expect(shown("Distributed(`my_cluster`, `d`, `events`)")).toBe(shown("Distributed('my_cluster', 'd', 'events')"));
+    expect(shown("Distributed('{cluster}', currentDatabase(), events, rand())")).toBe(shown("Distributed('{cluster}', currentDatabase(), 'events', rand())"));
+    // The sharding key is an expression, not a name: it stays as written.
+    expect(shown("Distributed(c, d, events, id)")).not.toBe(shown("Distributed(c, d, events, 'id')"));
+    expect(shown("Distributed(c, d, events)")).not.toBe(shown("Distributed(c, d, other)"));
+  });
+
   test("a setting at its default is not part of the definition", () => {
     expect(canonicalObject("CREATE TABLE t (a UInt8) ENGINE = MergeTree ORDER BY a SETTINGS index_granularity = 8192").settings).toEqual({});
     expect(canonicalObject("CREATE TABLE t (a UInt8) ENGINE = MergeTree ORDER BY a SETTINGS min_bytes_for_wide_part = '0'").settings).toEqual({ min_bytes_for_wide_part: "0" });

package/src/clickhouse/rebuild/backfill.ts

@@ -41,6 +41,26 @@
  * With a replica down (#3270) the kill goes on without it, and each sync
  * either finds nothing that replica alone has and returns, or stops the step
  * naming it (`./replicas.ts`).
+ *
+ * On a cluster of more than one shard (#3663) the unit is a partition on one
+ * shard: read from that shard, written back to it, cleared on it alone, with
+ * its own receipt (`./shards.ts`), so a rerun after a failure on one shard
+ * skips what every shard already copied.
+ *
+ * In materialized-view mode a partition is copied in two parts
+ * (INTENTIUS/sql-yodeler#45). The rows before the cut-over are copied as
+ * above. The rows at or after it are the view's only when they were written
+ * after the view was made; the ones the old table already held (a booking
+ * next month, an expiry date) reach the new table only through the backfill.
+ * The two kinds cannot be told apart by their values, so the second part
+ * copies the difference: for each distinct row at or after the cut-over, as
+ * many copies as the old table has more of it than the new one. That copy
+ * converges on its own, so a run killed during it, or a partition copied
+ * again, copies only what is still missing. A row the view is a moment from
+ * committing can be copied by both: the copy is followed by a check for rows
+ * the new table holds more often than the old one, which deletes and copies
+ * those again. Whatever is left, the verification finds, and the swap does
+ * not go ahead.
  */
 
 import { EffectReceipt, receiptExpectation } from "@intentius/chant/effect-receipt";
@@ -49,8 +69,11 @@ import { ident, sqlString } from "../apply/statements";
 import { clickhouseReceiptStore, receiptAddress, type ClickHouseReceiptStore } from "./receipts";
 import { RebuildRefusal, type RebuildObservation } from "./observe";
 import { DDL_SETTINGS } from "./replicas";
-import { sourcePartitionExpression, sourcePartitions } from "./partitions";
+import { shardPartitions, sourcePartitionExpression, sourcePartitions } from "./partitions";
+import { rowHash } from "./verify";
 import { cutoverOf, observe, serverNow, syncReplica, utcLiteral, waitOn, type RebuildRun } from "./steps";
+import { intoShard, onShard, onShardServers, shardTable, SHARD_DELETE_SETTINGS, SHARD_INSERT_SETTINGS, type Shard } from "./shards";
+import { renderFor } from "../topology";
 
 export interface BackfillResult {
   state: RebuildObservation["state"];
@@ -62,19 +85,30 @@ export interface BackfillResult {
   skipped: number;
   /** Partitions this run found partly copied and cleared before copying again. */
   cleared: number;
+  /** Distinct rows at or after the cut-over found in the new table too often (copied while the view was writing them), deleted and copied again. */
+  repaired: number;
 }
 
 /** The effect a partition's copy is, and its receipt's address suffix. */
-export const partitionEffect = (key: string, partition: string): string => `rebuild/${key}/${partition}`;
+export const partitionEffect = (key: string, partition: string, shard?: Shard): string => (shard ? `rebuild/${key}/shard${shard.num}/${partition}` : `rebuild/${key}/${partition}`);
 
-/** The receipt's expected value for one partition's copy. */
-export function partitionExpectation(o: RebuildObservation, partition: string, cutover: number | undefined): string {
-  const effect = partitionEffect(o.names.key, partition);
+/** The receipt's expected value for one partition's copy, on one shard of a cluster when `shard` is given. */
+export function partitionExpectation(o: RebuildObservation, partition: string, cutover: number | undefined, shard?: Shard): string {
+  const effect = partitionEffect(o.names.key, partition, shard);
   return receiptExpectation(
     EffectReceipt(effect, {
       effect,
       flavor: "hash",
-      inputs: { table: o.names.key, newTable: o.newTable!.uuid, partition, copied: o.copied, cutover: cutover === undefined ? null : new Date(cutover).toISOString() },
+      inputs: {
+        table: o.names.key,
+        newTable: o.newTable!.uuid,
+        partition,
+        ...(shard ? { shard: shard.num } : {}),
+        copied: o.copied,
+        cutover: cutover === undefined ? null : new Date(cutover).toISOString(),
+        // The rows at or after the cut-over the old table held are copied too (sql-yodeler#45); a receipt from before that copied fewer.
+        ...(cutover === undefined ? {} : { after: "difference" }),
+      },
     }),
   );
 }
@@ -82,13 +116,13 @@ export function partitionExpectation(o: RebuildObservation, partition: string, c
 export interface BackfillDeps {
   /** The receipt store. Default: the receipts table on the target server. */
   receipts?: ClickHouseReceiptStore;
-  /** Called after each partition is copied and its receipt written; a test interrupts the backfill here. */
-  afterPartition?: (partition: string) => void | Promise<void>;
+  /** Called after each partition is copied and its receipt written, with its shard on a cluster of shards; a test interrupts the backfill here. */
+  afterPartition?: (partition: string, shard?: number) => void | Promise<void>;
 }
 
 export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promise<BackfillResult> {
   const o = await observe(run);
-  const result: BackfillResult = { state: o.state, partitions: 0, copied: 0, skipped: 0, cleared: 0 };
+  const result: BackfillResult = { state: o.state, partitions: 0, copied: 0, skipped: 0, cleared: 0, repaired: 0 };
   if (o.state !== "rebuild") return result;
   if (!o.newTable) throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
   const n = o.names;
@@ -113,6 +147,13 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
   const fromOld = sourcePartitionExpression(o.live.partitionKey, o.newTable.partitionKey, o.copied);
   const columns = o.copied.map((c) => ident(c.name)).join(", ");
   const select = o.copied.map((c) => ident(c.source)).join(", ");
+  const later = cutColumn && cutover !== undefined ? await laterRowsCopy(run, o, cutColumn, cutover, fromOld) : undefined;
+  const sharding = run.sharding;
+  // The old and new tables as one shard's rows (#3663, `./shards.ts`), or as this server's.
+  const oldFrom = (shard?: Shard) => (shard && sharding ? `${shardTable(sharding, n.database, n.name)} WHERE ${onShard(shard)} AND` : `${n.table} WHERE`);
+  const newFrom = (shard?: Shard) => (shard && sharding ? `${shardTable(sharding, n.database, n.newName)} WHERE ${onShard(shard)} AND` : `${n.newTable} WHERE`);
+  const into = (shard?: Shard) => (shard && sharding ? intoShard(sharding, shard, n.database, n.newName) : n.newTable);
+  const insertSettings = sharding ? SHARD_INSERT_SETTINGS : {};
 
   const identity = { ...(run.marker?.stack ? { stack: run.marker.stack } : {}), ...(run.marker?.env ? { env: run.marker.env } : {}) };
   const receipts =
@@ -125,53 +166,176 @@ export async function backfill(run: RebuildRun, deps: BackfillDeps = {}): Promis
     });
   const recorded = await receipts.readAll(receiptAddress(identity, `rebuild/${n.key}/`));
 
-  const partitions = await sourcePartitions(run.target, n);
-  result.partitions = partitions.length;
-  for (const p of partitions) {
+  // The unit of work: a partition of the old table, on each shard of a cluster that has more than one.
+  const units: Array<{ shard?: Shard; partition: string }> = sharding
+    ? (await shardPartitions(run, sharding, n.database, n.name)).map((u) => ({ shard: sharding.shards.find((x) => x.num === u.shard)!, partition: u.partition }))
+    : (await sourcePartitions(run.target, n)).map((partition) => ({ partition }));
+  result.partitions = units.length;
+  for (const { shard, partition: p } of units) {
     run.signal?.throwIfAborted();
-    const effect = partitionEffect(n.key, p);
-    const expectation = partitionExpectation(o, p, cutover);
+    const effect = partitionEffect(n.key, p, shard);
+    const expectation = partitionExpectation(o, p, cutover, shard);
     if (recorded.get(receiptAddress(identity, effect)) === expectation) {
       result.skipped++;
       continue;
     }
 
-    const queryId = `chant-rebuild-${o.newTable.uuid}-${p}`;
-    if (o.replicated) {
+    const queryId = `chant-rebuild-${o.newTable.uuid}-${shard ? `shard${shard.num}-` : ""}${p}`;
+    const laterQueryId = `${queryId}-after`;
+    const killWhere = `query_id IN (${sqlString(queryId)}, ${sqlString(laterQueryId)})`;
+    if (sharding) {
+      // The copy reads and writes every shard from this server; what it started on the others goes with it.
+      await clickhouseQuery(
+        run.target.endpoint,
+        `KILL QUERY ON CLUSTER ${sqlString(sharding.cluster)} WHERE ${killWhere} OR initial_query_id IN (${sqlString(queryId)}, ${sqlString(laterQueryId)}) SYNC`,
+        { settings: DDL_SETTINGS },
+      );
+    } else if (o.replicated) {
       // On every replica that answers: the copy may be running on another one
       // than this run's. A replica that is down is skipped once Keeper sees it
       // inactive; nothing of its can be running a copy that still writes,
       // and a part it wrote before going down is waited for by the sync below.
-      await clickhouseQuery(run.target.endpoint, `KILL QUERY ON CLUSTER ${sqlString(n.database)} WHERE query_id = ${sqlString(queryId)} SYNC`, {
+      await clickhouseQuery(run.target.endpoint, `KILL QUERY ON CLUSTER ${sqlString(n.database)} WHERE ${killWhere} SYNC`, {
         settings: DDL_SETTINGS,
       });
       await syncReplica(run, o, n.database, n.newName);
     } else {
-      await clickhouseQuery(run.target.endpoint, `KILL QUERY WHERE query_id = ${sqlString(queryId)} SYNC`);
+      await clickhouseQuery(run.target.endpoint, `KILL QUERY WHERE ${killWhere} SYNC`);
     }
     const mine = `${fromOld} = ${sqlString(p)}${targetRange}`;
-    const [left] = await clickhouseQuery<{ n: string | number }>(run.target.endpoint, `SELECT count() AS n FROM ${n.newTable} WHERE ${mine}`);
+    const where = shard ? `partition ${p} on shard ${shard.num}` : `partition ${p}`;
+    const [left] = await clickhouseQuery<{ n: string | number }>(run.target.endpoint, `SELECT count() AS n FROM ${newFrom(shard)} ${mine}`);
     if (Number(left?.n ?? 0) > 0) {
-      run.log(`-- partition ${p}: ${left!.n} row(s) from a copy that did not finish; deleting them before copying again`);
-      const sql = `ALTER TABLE ${n.newTable} DELETE WHERE ${mine}`;
+      run.log(`-- ${where}: ${left!.n} row(s) from a copy that did not finish; deleting them before copying again`);
+      const sql =
+        shard && sharding
+          ? renderFor(`ALTER TABLE ${n.newTable} DELETE WHERE ${onShardServers(shard)} AND ${mine}`, run.target.topology)
+          : `ALTER TABLE ${n.newTable} DELETE WHERE ${mine}`;
       run.log(sql);
-      await clickhouseQuery(run.target.endpoint, sql);
+      await clickhouseQuery(run.target.endpoint, sql, shard && sharding ? { settings: { ...DDL_SETTINGS, ...SHARD_DELETE_SETTINGS } } : {});
       await waitOn(run, n.database, n.newName);
       result.cleared++;
     }
 
-    const sql = `INSERT INTO ${n.newTable} (${columns}) SELECT ${select} FROM ${n.table} WHERE _partition_id = ${sqlString(p)}${sourceRange}`;
+    const sql = `INSERT INTO ${into(shard)} (${columns}) SELECT ${select} FROM ${oldFrom(shard)} _partition_id = ${sqlString(p)}${sourceRange}`;
     run.log(sql);
-    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0" }, ...(run.signal ? { signal: run.signal } : {}) });
+    await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0", ...insertSettings }, ...(run.signal ? { signal: run.signal } : {}) });
+    if (later) result.repaired += await copyAfterCutover(run, o, later, p, laterQueryId, shard);
     // The receipt, last, on success only.
     await receipts.write({ name: effect, effect, flavor: "hash", inputs: {} }, expectation);
     result.copied++;
-    await deps.afterPartition?.(p);
+    await deps.afterPartition?.(p, shard?.num);
   }
-  run.log(`-- backfill of ${n.key}: ${result.partitions} partition(s), ${result.copied} copied, ${result.skipped} already copied, ${result.cleared} cleared first`);
+  run.log(`-- backfill of ${n.key}: ${result.partitions} partition(s), ${result.copied} copied, ${result.skipped} already copied, ${result.cleared} cleared first${result.repaired > 0 ? `, ${result.repaired} row(s) after the cut-over copied twice and repaired` : ""}`);
   return result;
 }
 
+/** The statements that copy one old partition's rows at or after the cut-over. */
+interface LaterRows {
+  /**
+   * Per distinct row (by its hash), the old table's copies past the number
+   * the new table holds, into the new table. On a cluster of shards, one
+   * shard's rows into the same shard.
+   */
+  copy: (partition: string, shard?: Shard) => string;
+  /** The hashes of the rows the new table holds more copies of than the old one. */
+  extras: (partition: string, shard?: Shard) => string;
+  /** Delete every copy of those rows from the new table. */
+  drop: (partition: string, hashes: readonly string[], shard?: Shard) => string;
+}
+
+/** The most extra rows one partition's repair deletes and copies again; past that the verification decides. */
+const REPAIR_LIMIT = 1000;
+const REPAIR_ROUNDS = 3;
+
+/**
+ * The statements for the rows at or after the cut-over. The old side casts
+ * each column to its type in the new table before hashing, as the
+ * verification does, so a row and its copy hash the same.
+ */
+async function laterRowsCopy(
+  run: RebuildRun,
+  o: RebuildObservation,
+  cut: RebuildObservation["copied"][number],
+  cutover: number,
+  fromOld: string,
+): Promise<LaterRows> {
+  const n = o.names;
+  const types = new Map(
+    (
+      await clickhouseQuery<{ name: string; type: string }>(
+        run.target.endpoint,
+        `SELECT name, type FROM system.columns WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)}`,
+      )
+    ).map((c) => [c.name, c.type]),
+  );
+  const oldHash = rowHash(o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`));
+  const newHash = rowHash(o.copied.map((c) => ident(c.name)));
+  const columns = o.copied.map((c) => ident(c.name)).join(", ");
+  const aliases = o.copied.map((_, i) => `__chant_c${i}`);
+  const fromOldSide = o.copied.map((c, i) => `${ident(c.source)} AS ${aliases[i]}`).join(", ");
+  const sharding = run.sharding;
+  const on = (shard?: Shard) => (shard && sharding ? ` AND ${onShard(shard)}` : "");
+  const oldTable = (shard?: Shard) => (shard && sharding ? shardTable(sharding, n.database, n.name) : n.table);
+  const newTable = (shard?: Shard) => (shard && sharding ? shardTable(sharding, n.database, n.newName) : n.newTable);
+  const oldRows = (p: string, shard?: Shard) => `FROM ${oldTable(shard)} WHERE _partition_id = ${sqlString(p)} AND ${ident(cut.source)} >= ${utcLiteral(cutover)}${on(shard)}`;
+  const newRows = (p: string, shard?: Shard) => `FROM ${newTable(shard)} WHERE ${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)}${on(shard)}`;
+  return {
+    copy: (p, shard) =>
+      `INSERT INTO ${shard && sharding ? intoShard(sharding, shard, n.database, n.newName) : n.newTable} (${columns}) SELECT ${aliases.join(", ")} FROM (` +
+      `SELECT ${fromOldSide}, ${oldHash} AS __chant_h, row_number() OVER (PARTITION BY __chant_h) AS __chant_k ${oldRows(p, shard)}` +
+      `) AS old_rows LEFT JOIN (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p, shard)} GROUP BY __chant_h) AS new_rows ` +
+      `USING (__chant_h) WHERE __chant_k > ifNull(__chant_n, 0)`,
+    extras: (p, shard) =>
+      `SELECT toString(__chant_h) AS h FROM (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p, shard)} GROUP BY __chant_h) AS new_rows ` +
+      `LEFT JOIN (SELECT ${oldHash} AS __chant_h, count() AS __chant_o ${oldRows(p, shard)} GROUP BY __chant_h) AS old_rows ` +
+      `USING (__chant_h) WHERE __chant_n > ifNull(__chant_o, 0) LIMIT ${REPAIR_LIMIT + 1}`,
+    drop: (p, hashes, shard) => {
+      const where = `${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)} AND ${newHash} IN (${hashes.join(", ")})`;
+      return shard && sharding
+        ? renderFor(`ALTER TABLE ${n.newTable} DELETE WHERE ${onShardServers(shard)} AND ${where}`, run.target.topology)
+        : `ALTER TABLE ${n.newTable} DELETE WHERE ${where}`;
+    },
+  };
+}
+
+/**
+ * Copy one old partition's rows at or after the cut-over that the new table
+ * lacks. A row the view had written to the old table but not yet to the new
+ * one when the copy read them is copied by both, and is then in the new
+ * table once too often: every copy of such a row is deleted from the new
+ * table and the difference copied again, a few rounds at most. What is left
+ * after that, the verification finds. Returns the rows repaired.
+ */
+async function copyAfterCutover(run: RebuildRun, o: RebuildObservation, later: LaterRows, p: string, queryId: string, shard?: Shard): Promise<number> {
+  const n = o.names;
+  const sharded = shard !== undefined && run.sharding !== undefined;
+  const insert = async () => {
+    const sql = later.copy(p, shard);
+    run.log(sql);
+    await clickhouseQuery(run.target.endpoint, sql, {
+      queryId,
+      settings: { async_insert: "0", insert_deduplicate: "0", ...(sharded ? SHARD_INSERT_SETTINGS : {}) },
+      ...(run.signal ? { signal: run.signal } : {}),
+    });
+  };
+  await insert();
+  let repaired = 0;
+  for (let round = 0; round < REPAIR_ROUNDS; round++) {
+    await syncReplica(run, o, n.database, n.newName);
+    const extras = (await clickhouseQuery<{ h: string }>(run.target.endpoint, later.extras(p, shard))).map((r) => r.h);
+    if (extras.length === 0 || extras.length > REPAIR_LIMIT) break;
+    run.log(`-- partition ${p}: ${extras.length} row(s) after the cut-over in the new table more often than in the old one; deleting them and copying again`);
+    const sql = later.drop(p, extras, shard);
+    run.log(sql);
+    await clickhouseQuery(run.target.endpoint, sql, sharded ? { settings: { ...DDL_SETTINGS, ...SHARD_DELETE_SETTINGS } } : {});
+    await waitOn(run, n.database, n.newName);
+    repaired += extras.length;
+    await insert();
+  }
+  return repaired;
+}
+
 function sleep(ms: number, signal?: AbortSignal): Promise<void> {
   return new Promise((resolve, reject) => {
     const t = setTimeout(resolve, ms);

package/src/clickhouse/rebuild/observe.ts

@@ -55,9 +55,10 @@ export type DualWrite =
       /**
        * A materialized view on the old table writes every row whose
        * `cutoverColumn` is at or after the cut-over into the new table; the
-       * backfill copies the rows before it. The cut-over is the server's time
-       * when the view is made plus `cutoverDelay` (default `1m`), and the
-       * backfill starts once it has passed. The column must be a time
+       * backfill copies the rows before it, and the rows at or after it that
+       * the table held before the view was made. The cut-over is the server's
+       * time when the view is made plus `cutoverDelay` (default `1m`), rounded
+       * up to a whole second, and the backfill starts once it has passed. The column must be a time
        * (`Date`, `DateTime`, `DateTime64`) that rows arrive in order of, give
        * or take the delay; a row that arrives later than that is caught by
        * the verification, which then fails the run.