@intentius/chant-lexicon-sql 0.124.31 → 0.124.32

package/dist/clickhouse/plan/rules.js

@@ -0,0 +1,72 @@
+/**
+ * The change classifier's rules (#3046 question 3): every way a declared
+ * ClickHouse object can change, the class of change it is on the server, and
+ * the `ALTER` restriction behind the class, cited.
+ *
+ * Three classes:
+ *
+ * - `metadata`: the server records the change and touches no existing data
+ *   (a new column reads its default from old parts; a new index covers new
+ *   parts until `MATERIALIZE INDEX`).
+ * - `rewrite`: the change is a mutation that rewrites existing parts in the
+ *   background (`system.mutations`), with no rollback.
+ * - `rebuild`: ClickHouse cannot make the change to the table it has; the data
+ *   has to be copied into a new table. A plan refuses to make one in place.
+ *
+ * Creating and dropping whole objects are classed `create` and `drop`.
+ */
+import { classifierRule } from "../../core/classifier.js";
+/** The classes as data: report order and labels, and what each costs for `classifyDisruption()`. */
+export const CHANGE_CLASSES = {
+    order: ["create", "metadata", "rewrite", "rebuild", "drop"],
+    label: { create: "create", drop: "drop", metadata: "metadata only", rewrite: "background rewrite", rebuild: "REBUILD" },
+    disruption: { metadata: "in-place", rewrite: "rolling", rebuild: "replace", create: "in-place", drop: "destroy" },
+};
+const DOCS = "https://clickhouse.com/docs/sql-reference/statements";
+const rule = (id, cls, title, restriction, cite) => classifierRule(id, cls, title, restriction, cite);
+export const CLASSIFIER_RULES = {
+    SQLCH200: rule("SQLCH200", "create", "Create an object", "A new database, table, view, dictionary, function, user, role, row policy, quota or settings profile is created, or a grantee gets its first grants; nothing existing changes.", `${DOCS}/create`),
+    SQLCH201: rule("SQLCH201", "metadata", "Add a column", "ADD COLUMN only changes metadata: parts written before it read the column's default until they are merged or the column is materialized.", `${DOCS}/alter/column#add-column`),
+    SQLCH202: rule("SQLCH202", "metadata", "Drop a column", "DROP COLUMN removes the column's data from every part; it is not undone. A column in a key cannot be dropped (SQLCH213 covers that case).", `${DOCS}/alter/column#drop-column`),
+    SQLCH203: rule("SQLCH203", "metadata", "Change a comment", "COMMENT COLUMN and MODIFY COMMENT change metadata only.", `${DOCS}/alter/comment`),
+    SQLCH204: rule("SQLCH204", "metadata", "Add, drop or change a skip index", "ADD INDEX and DROP INDEX change metadata; a new index covers parts written after it until MATERIALIZE INDEX rebuilds it for older parts.", `${DOCS}/alter/skipping-index`),
+    SQLCH205: rule("SQLCH205", "rewrite", "Change a TTL", "MODIFY TTL is recorded in metadata, and with materialize_ttl_after_modify (on by default) the server recalculates the TTL over existing data as a mutation.", `${DOCS}/alter/ttl`),
+    SQLCH206: rule("SQLCH206", "metadata", "Change a table setting", "MODIFY SETTING and RESET SETTING change metadata, for settings the server lets change after creation.", `${DOCS}/alter/setting`),
+    SQLCH207: rule("SQLCH207", "metadata", "Change a column's default", "MODIFY COLUMN with a new DEFAULT, MATERIALIZED or ALIAS expression changes metadata; values already written keep the old default until the column is materialized.", `${DOCS}/alter/column#modify-column`),
+    SQLCH208: rule("SQLCH208", "metadata", "Change a column's codec", "MODIFY COLUMN ... CODEC applies to parts written afterwards; existing parts keep their codec until they are merged or rewritten.", `${DOCS}/alter/column#modify-column`),
+    SQLCH209: rule("SQLCH209", "metadata", "Move a column", "MODIFY COLUMN ... FIRST | AFTER changes the column order in metadata only.", `${DOCS}/alter/column#modify-column`),
+    SQLCH210: rule("SQLCH210", "rewrite", "Change a column's type", "Changing the type of a column outside the primary key is a mutation that rewrites the column in every part in the background, with no rollback.", `${DOCS}/alter/column#modify-column`),
+    SQLCH211: rule("SQLCH211", "rebuild", "Change the type of a key column", "The type of a primary key column can change only when the change does not modify the data (for example adding values to an Enum); anything else needs a new table.", `${DOCS}/alter/column#modify-column`),
+    SQLCH212: rule("SQLCH212", "metadata", "Rename a column", "RENAME COLUMN changes metadata, for a column that is not in a key.", `${DOCS}/alter/column#rename-column`),
+    SQLCH213: rule("SQLCH213", "rebuild", "Rename or drop a key column", "A column in the sorting key, the primary key or the partition key cannot be renamed or dropped.", `${DOCS}/alter/column#rename-column`),
+    SQLCH214: rule("SQLCH214", "metadata", "Add, drop or change a projection", "ADD PROJECTION and DROP PROJECTION change metadata; a new projection covers new parts until MATERIALIZE PROJECTION builds it for older ones.", `${DOCS}/alter/projection`),
+    SQLCH215: rule("SQLCH215", "metadata", "Add or drop a constraint", "ADD CONSTRAINT and DROP CONSTRAINT change metadata; existing rows are not checked.", `${DOCS}/alter/constraint`),
+    SQLCH216: rule("SQLCH216", "metadata", "Append new columns to the sorting key", "MODIFY ORDER BY can only append columns added by ADD COLUMN in the same ALTER, and leaves the primary key as it was; the declaration must keep the old key as PRIMARY KEY.", `${DOCS}/alter/order-by`),
+    SQLCH217: rule("SQLCH217", "metadata", "Change the sampling key", "MODIFY SAMPLE BY changes metadata; the new sampling expression must be part of the primary key, and the server refuses it otherwise.", `${DOCS}/alter/sample-by`),
+    SQLCH218: rule("SQLCH218", "rebuild", "Change a setting fixed at creation", "A table setting the server marks read-only (such as index_granularity) is fixed when the table is created and needs a new table to change.", `${DOCS}/alter/setting`),
+    SQLCH219: rule("SQLCH219", "metadata", "Replace a table over a table function", "A table declared AS a table function (numbers, remote, s3, url) stores no data of its own and has no ALTER for its function; CREATE OR REPLACE TABLE replaces its call and columns, and queries read through the new one.", `${DOCS}/create/table#from-a-table-function`),
+    SQLCH220: rule("SQLCH220", "rebuild", "Change the sorting key", "The sorting key is the on-disk order of every part (and in ReplacingMergeTree the deduplication identity); beyond appending new columns (SQLCH216) it cannot change in place.", `${DOCS}/alter/order-by`),
+    SQLCH221: rule("SQLCH221", "rebuild", "Change the primary key", "The primary key cannot be changed by ALTER; MODIFY ORDER BY leaves it as it was.", `${DOCS}/alter/order-by`),
+    SQLCH222: rule("SQLCH222", "rebuild", "Change the partition key", "There is no ALTER for the partition key; parts are laid out by it when they are written.", `${DOCS}/alter`),
+    SQLCH223: rule("SQLCH223", "rebuild", "Change the table engine or its arguments", "There is no ALTER for a table's engine or its arguments (a ReplacingMergeTree version column, a Distributed sharding key); a different engine is a different table.", `${DOCS}/alter`),
+    SQLCH224: rule("SQLCH224", "rebuild", "Change what kind of object it is", "A table cannot become a view or a view a table; the old object is dropped and the new one created.", `${DOCS}/create`),
+    SQLCH230: rule("SQLCH230", "metadata", "Rename or move a table or view", "RENAME TABLE changes the name, or moves the object to another database, in metadata on an Atomic database.", `${DOCS}/rename`),
+    SQLCH231: rule("SQLCH231", "metadata", "Rename a database", "RENAME DATABASE is metadata only, for an Atomic database.", `${DOCS}/rename`),
+    SQLCH232: rule("SQLCH232", "rebuild", "Change a database's engine", "A database's engine is fixed at creation; there is no ALTER for it.", `${DOCS}/create/database`),
+    SQLCH240: rule("SQLCH240", "metadata", "Change a view's query", "A plain view stores no data; CREATE OR REPLACE VIEW replaces its query.", `${DOCS}/create/view`),
+    SQLCH241: rule("SQLCH241", "metadata", "Change a materialized view's query", "ALTER TABLE ... MODIFY QUERY replaces the query without stopping inserts; rows already written are not recomputed.", `${DOCS}/alter/view`),
+    SQLCH242: rule("SQLCH242", "rebuild", "Change a materialized view's target", "A materialized view's TO target is fixed at creation; the view is dropped and created again, and inserts into its source in between are not seen.", `${DOCS}/create/view`),
+    SQLCH243: rule("SQLCH243", "rebuild", "Change a materialized view's own storage", "A materialized view without TO keeps its rows in an inner table whose engine and keys are fixed like any table's.", `${DOCS}/create/view`),
+    SQLCH244: rule("SQLCH244", "metadata", "Change a refreshable view's schedule", "ALTER TABLE ... MODIFY REFRESH changes the schedule of a refreshable materialized view.", `${DOCS}/alter/view`),
+    SQLCH245: rule("SQLCH245", "metadata", "Change a dictionary", "A dictionary stores no data of its own: CREATE OR REPLACE DICTIONARY replaces its attributes, key, source, layout, lifetime or range, and it loads again from its source.", `${DOCS}/create/dictionary`),
+    SQLCH260: rule("SQLCH260", "metadata", "Change a function", "A SQL user-defined function stores no data: CREATE OR REPLACE FUNCTION replaces its parameters and expression, and queries that call it use the new one.", `${DOCS}/create/function`),
+    SQLCH270: rule("SQLCH270", "metadata", "Change a role", "ALTER ROLE ... SETTINGS replaces a role's settings; its grants are untouched.", `${DOCS}/alter/role`),
+    SQLCH271: rule("SQLCH271", "metadata", "Change a user", "ALTER USER sets one clause (HOST, DEFAULT ROLE, DEFAULT DATABASE, GRANTEES, SETTINGS, VALID UNTIL, IDENTIFIED) and leaves the rest, the password and the grants untouched. Sessions already open keep the old settings.", `${DOCS}/alter/user`),
+    SQLCH272: rule("SQLCH272", "metadata", "Change a row policy", "CREATE ROW POLICY OR REPLACE replaces the policy's condition, kind and roles; queries started after it see the new one.", `${DOCS}/create/row-policy`),
+    SQLCH273: rule("SQLCH273", "metadata", "Grant a privilege or a role", "GRANT adds a privilege or a role to a user or role; it takes effect for new queries.", `${DOCS}/grant`),
+    SQLCH274: rule("SQLCH274", "metadata", "Revoke a privilege or a role", "REVOKE takes a privilege or a role away, including one granted by hand to a grantee the build declares grants for; queries that relied on it fail from then on.", `${DOCS}/revoke`),
+    SQLCH275: rule("SQLCH275", "metadata", "Change a quota", "ALTER QUOTA sets the quota's key, one interval's limits (NO LIMITS takes the interval away) or whom it applies to, on the quota that is there, so what names it still does.", `${DOCS}/alter/quota`),
+    SQLCH276: rule("SQLCH276", "metadata", "Change a settings profile", "ALTER SETTINGS PROFILE replaces the profile's settings, constraints and inherited profiles, or whom it applies to; the users, roles and profiles that name it keep it. Sessions already open keep the old settings.", `${DOCS}/alter/settings-profile`),
+    SQLCH290: rule("SQLCH290", "rebuild", "Change a clause chant does not read", "chant's parser does not know this clause, so it cannot tell whether ALTER can make the change in place; the plan refuses it rather than guess. Review it: make it by hand with the ALTER the server documents for the clause, or drop and create the object.", `${DOCS}/create/table`),
+    SQLCH250: rule("SQLCH250", "drop", "Drop an object", "DROP removes the object and, for a table, its data; it is not undone.", `${DOCS}/drop`),
+};

package/dist/clickhouse/plan/schema.js

@@ -0,0 +1,67 @@
+/**
+ * The two sides a diff compares: a build's output (the declarations), or what
+ * a live server reports.
+ */
+import { readFileSync } from "node:fs";
+import { canonicalObject, grantsObject, liveCanonical, objectKey, scopeOf } from "./normalize.js";
+import { grantsByGrantee } from "../access.js";
+import { readLiveAccess, readLiveSchema } from "../live/catalog.js";
+import { renderFor } from "../topology.js";
+/** The objects a `chant build` output holds, keyed by export name; with a `topology`, each declaration as rendered for it (`../topology.ts`). */
+export function schemaFromBuildOutput(json, defaultDatabase = "default", topology) {
+    const doc = JSON.parse(json);
+    if (doc.dialect !== "clickhouse" || !Array.isArray(doc.objects)) {
+        throw new Error("not a sql lexicon build output: expected { dialect: \"clickhouse\", objects: [...] }");
+    }
+    const out = doc.objects
+        .filter((o) => o.type !== "ClickHouse::Grant")
+        .map((o) => ({ key: o.export, canonical: canonicalObject(renderFor(o.ddl, topology), defaultDatabase) }));
+    // Grants are compared per grantee (`../access.ts`).
+    for (const g of grantsByGrantee(doc.objects.map((o) => ({ ...o, type: o.type ?? "" })))) {
+        const canonical = grantsObject(g.ddl, g.grantee, defaultDatabase);
+        out.push({ key: objectKey(canonical), canonical });
+    }
+    return out;
+}
+export function schemaFromBuildFile(path, defaultDatabase = "default", topology) {
+    return schemaFromBuildOutput(readFileSync(path, "utf-8"), defaultDatabase, topology);
+}
+/**
+ * The declared objects re-keyed by `database.name`, the only identity a live
+ * server has. The export name is kept for the report.
+ */
+export function keyedByQualifiedName(objects) {
+    return objects.map((o) => ({
+        key: objectKey(o.canonical),
+        canonical: o.canonical,
+        exportName: o.key,
+    }));
+}
+/**
+ * What the server holds, keyed by `database.name`, limited to the databases
+ * the declarations use (and the target's scope) and the functions they
+ * declare (`scopeOf`). The `default` database is
+ * left out, as import leaves it out. An object whose definition chant cannot
+ * read is not left out silently (#3653): it goes in `unreadable`, when
+ * given, and otherwise fails the read.
+ */
+export async function schemaFromServer(target, databases, unreadable, access = []) {
+    const live = [...(await readLiveSchema(target)), ...(await readLiveAccess(target, access))];
+    const out = [];
+    for (const o of live) {
+        if (o.type === "ClickHouse::Database" && o.name === "default")
+            continue;
+        if (databases && !databases.has(scopeOf(o)))
+            continue;
+        const key = objectKey(o);
+        try {
+            out.push({ key, canonical: liveCanonical(o, target.defaultDatabase) });
+        }
+        catch (err) {
+            if (!unreadable)
+                throw err;
+            unreadable.push({ object: key, type: o.type, reason: `its definition does not parse: ${err.message}` });
+        }
+    }
+    return out;
+}

package/dist/clickhouse/plan/server-format.js

@@ -0,0 +1,35 @@
+/**
+ * The backstop for normalization against a live server: for each changed
+ * expression the rules leave different, ask the server to format both sides
+ * (`formatQuerySingleLine`, which parses and prints and touches nothing) and
+ * drop the change when they print the same. `a+b*2` and `a + (b * 2)` are one
+ * expression to the server.
+ */
+import { clickhouseQuery } from "../http.js";
+const EXPRESSION_FIELDS = /^(orderBy|primaryKey|partitionBy|sampleBy|select|lambda|using|columns\.[^.]+\.(default|ttl)|indexes\..+|projections\..+)$/;
+const quote = (s) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+function asQuery(field, text) {
+    if (field === "select")
+        return text;
+    if (field.endsWith(".default"))
+        return `SELECT ${text.replace(/^(DEFAULT|MATERIALIZED|ALIAS|EPHEMERAL) /, "")}`;
+    return `SELECT ${text}`;
+}
+/** The changes that remain once the server's formatter has had its say. */
+export async function dropFormattingOnly(endpoint, changes) {
+    const candidates = changes.filter((c) => EXPRESSION_FIELDS.test(c.field) && c.before !== undefined && c.after !== undefined);
+    if (candidates.length === 0)
+        return changes;
+    const same = new Set();
+    for (const c of candidates) {
+        try {
+            const [row] = await clickhouseQuery(endpoint, `SELECT formatQuerySingleLine(${quote(asQuery(c.field, c.before))}) AS a, formatQuerySingleLine(${quote(asQuery(c.field, c.after))}) AS b`);
+            if (row && row.a === row.b)
+                same.add(c);
+        }
+        catch {
+            // The server could not parse one side as a query: the rules' answer stands.
+        }
+    }
+    return changes.filter((c) => !same.has(c));
+}

package/dist/clickhouse/rebuild/backfill.js

@@ -0,0 +1,265 @@
+/**
+ * The backfill (#3198): copy the old table into the new one with
+ * `INSERT ... SELECT`, one partition at a time, each partition an effect
+ * with its own receipt.
+ *
+ * Per partition, the cycle core's `effect()` step runs
+ * (concepts/effect-receipts.mdx): read the receipt, skip the copy when it
+ * matches, otherwise copy and write the receipt last, on success only.
+ * `effect()` itself wraps steps declared at build time around one receipt
+ * declared at build time; the partitions are only known when the run reads
+ * the old table, so the cycle runs here, once per partition, over the same
+ * `ReceiptStore` seam (`./receipts.ts`).
+ *
+ * A receipt's expectation is a hash of what the copy is: the table, the new
+ * table's UUID, the partition, the columns copied and the cut-over. A new
+ * table made again, or a new cut-over, makes every receipt stale and the
+ * partitions are copied again into the new table.
+ *
+ * At-least-once is the receipt model's guarantee, and a partition copied
+ * twice would be in the new table twice, so a copy that may have run before
+ * is undone first. A partition with no matching receipt is a partition whose
+ * copy never finished as far as anything recorded; if the new table holds
+ * any of its rows (a run killed between the INSERT and the receipt), they are
+ * deleted, the delete is waited on in `system.mutations`, and the partition
+ * is copied once more. Each copy runs under a query id derived from the new
+ * table and the partition, and any query still running under it (the server
+ * goes on with an INSERT whose client went away) is killed before the
+ * partition is cleared. A backfill interrupted halfway therefore resumes at
+ * the first partition without a receipt, and no row is copied twice.
+ *
+ * In a Replicated database (#3249) a resumed backfill may run on another
+ * replica than the one the copy ran on. The kill is `KILL QUERY ON CLUSTER`
+ * the database's own cluster, so it reaches the replica still running the
+ * copy; before the old table's partitions are listed, and before the new
+ * table's rows are counted, this replica fetches what the others wrote
+ * (`SYSTEM SYNC REPLICA`); and the receipts replicate with the tables
+ * (`./receipts.ts`). Each copy runs with `insert_deduplicate = 0`: a
+ * replicated table remembers the blocks it was given, and a partition copied
+ * again after its rows were cleared would otherwise be dropped as a repeat.
+ *
+ * With a replica down (#3270) the kill goes on without it, and each sync
+ * either finds nothing that replica alone has and returns, or stops the step
+ * naming it (`./replicas.ts`).
+ *
+ * On a cluster of more than one shard (#3663) the unit is a partition on one
+ * shard: read from that shard, written back to it, cleared on it alone, with
+ * its own receipt (`./shards.ts`), so a rerun after a failure on one shard
+ * skips what every shard already copied.
+ *
+ * In materialized-view mode a partition is copied in two parts
+ * (INTENTIUS/sql-yodeler#45). The rows before the cut-over are copied as
+ * above. The rows at or after it are the view's only when they were written
+ * after the view was made; the ones the old table already held (a booking
+ * next month, an expiry date) reach the new table only through the backfill.
+ * The two kinds cannot be told apart by their values, so the second part
+ * copies the difference: for each distinct row at or after the cut-over, as
+ * many copies as the old table has more of it than the new one. That copy
+ * converges on its own, so a run killed during it, or a partition copied
+ * again, copies only what is still missing. A row the view is a moment from
+ * committing can be copied by both: the copy is followed by a check for rows
+ * the new table holds more often than the old one, which deletes and copies
+ * those again. Whatever is left, the verification finds, and the swap does
+ * not go ahead.
+ */
+import { EffectReceipt, receiptExpectation } from "@intentius/chant/effect-receipt";
+import { clickhouseQuery } from "../http.js";
+import { ident, sqlString } from "../apply/statements.js";
+import { clickhouseReceiptStore, receiptAddress } from "./receipts.js";
+import { RebuildRefusal } from "./observe.js";
+import { DDL_SETTINGS } from "./replicas.js";
+import { shardPartitions, sourcePartitionExpression, sourcePartitions } from "./partitions.js";
+import { rowHash } from "./verify.js";
+import { cutoverOf, observe, serverCutoverProbe, syncReplica, utcLiteral, waitForCutover, waitOn } from "./steps.js";
+import { intoShard, onShard, onShardServers, shardTable, SHARD_DELETE_SETTINGS, SHARD_INSERT_SETTINGS } from "./shards.js";
+import { renderFor } from "../topology.js";
+/** The effect a partition's copy is, and its receipt's address suffix. */
+export const partitionEffect = (key, partition, shard) => (shard ? `rebuild/${key}/shard${shard.num}/${partition}` : `rebuild/${key}/${partition}`);
+/** The receipt's expected value for one partition's copy, on one shard of a cluster when `shard` is given. */
+export function partitionExpectation(o, partition, cutover, shard) {
+    const effect = partitionEffect(o.names.key, partition, shard);
+    return receiptExpectation(EffectReceipt(effect, {
+        effect,
+        flavor: "hash",
+        inputs: {
+            table: o.names.key,
+            newTable: o.newTable.uuid,
+            partition,
+            ...(shard ? { shard: shard.num } : {}),
+            copied: o.copied,
+            cutover: cutover === undefined ? null : new Date(cutover).toISOString(),
+            // The rows at or after the cut-over the old table held are copied too (sql-yodeler#45); a receipt from before that copied fewer.
+            ...(cutover === undefined ? {} : { after: "difference" }),
+        },
+    }));
+}
+export async function backfill(run, deps = {}) {
+    const o = await observe(run);
+    const result = { state: o.state, partitions: 0, copied: 0, skipped: 0, cleared: 0, repaired: 0 };
+    if (o.state !== "rebuild")
+        return result;
+    if (!o.newTable)
+        throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
+    const n = o.names;
+    const cutover = cutoverOf(run, o);
+    // Rows before the cut-over are the backfill's; wait until none of them is still to arrive.
+    if (cutover !== undefined)
+        await waitForCutover(run, cutover, deps.cutoverProbe ?? serverCutoverProbe(run, o), run.cutoverTimeoutMs !== undefined ? { timeoutMs: run.cutoverTimeoutMs } : {});
+    await waitOn(run, n.database, n.name);
+    await syncReplica(run, o, n.database, n.name);
+    const cutColumn = run.dualWrite.mode === "materialized-view" ? o.copied.find((c) => c.name === run.dualWrite.cutoverColumn) : undefined;
+    const sourceRange = cutColumn && cutover !== undefined ? ` AND ${ident(cutColumn.source)} < ${utcLiteral(cutover)}` : "";
+    const targetRange = cutColumn && cutover !== undefined ? ` AND ${ident(cutColumn.name)} < ${utcLiteral(cutover)}` : "";
+    const fromOld = sourcePartitionExpression(o.live.partitionKey, o.newTable.partitionKey, o.copied);
+    const columns = o.copied.map((c) => ident(c.name)).join(", ");
+    const select = o.copied.map((c) => ident(c.source)).join(", ");
+    const later = cutColumn && cutover !== undefined ? await laterRowsCopy(run, o, cutColumn, cutover, fromOld) : undefined;
+    const sharding = run.sharding;
+    // The old and new tables as one shard's rows (#3663, `./shards.ts`), or as this server's.
+    const oldFrom = (shard) => (shard && sharding ? `${shardTable(sharding, n.database, n.name)} WHERE ${onShard(shard)} AND` : `${n.table} WHERE`);
+    const newFrom = (shard) => (shard && sharding ? `${shardTable(sharding, n.database, n.newName)} WHERE ${onShard(shard)} AND` : `${n.newTable} WHERE`);
+    const into = (shard) => (shard && sharding ? intoShard(sharding, shard, n.database, n.newName) : n.newTable);
+    const insertSettings = sharding ? SHARD_INSERT_SETTINGS : {};
+    const identity = { ...(run.marker?.stack ? { stack: run.marker.stack } : {}), ...(run.marker?.env ? { env: run.marker.env } : {}) };
+    const receipts = deps.receipts ??
+        clickhouseReceiptStore(run.target.endpoint, identity, {
+            ...(run.runId ? { runId: run.runId } : {}),
+            ...(o.replicated ? { replicatedIn: n.database } : {}),
+            ...(run.target.topology ? { topology: run.target.topology } : {}),
+            ...(run.replicaTimeoutMs !== undefined ? { replicaTimeoutMs: run.replicaTimeoutMs } : {}),
+        });
+    const recorded = await receipts.readAll(receiptAddress(identity, `rebuild/${n.key}/`));
+    // The unit of work: a partition of the old table, on each shard of a cluster that has more than one.
+    const units = sharding
+        ? (await shardPartitions(run, sharding, n.database, n.name)).map((u) => ({ shard: sharding.shards.find((x) => x.num === u.shard), partition: u.partition }))
+        : (await sourcePartitions(run.target, n)).map((partition) => ({ partition }));
+    result.partitions = units.length;
+    for (const { shard, partition: p } of units) {
+        run.signal?.throwIfAborted();
+        const effect = partitionEffect(n.key, p, shard);
+        const expectation = partitionExpectation(o, p, cutover, shard);
+        if (recorded.get(receiptAddress(identity, effect)) === expectation) {
+            result.skipped++;
+            continue;
+        }
+        const queryId = `chant-rebuild-${o.newTable.uuid}-${shard ? `shard${shard.num}-` : ""}${p}`;
+        const laterQueryId = `${queryId}-after`;
+        const killWhere = `query_id IN (${sqlString(queryId)}, ${sqlString(laterQueryId)})`;
+        if (sharding) {
+            // The copy reads and writes every shard from this server; what it started on the others goes with it.
+            await clickhouseQuery(run.target.endpoint, `KILL QUERY ON CLUSTER ${sqlString(sharding.cluster)} WHERE ${killWhere} OR initial_query_id IN (${sqlString(queryId)}, ${sqlString(laterQueryId)}) SYNC`, { settings: DDL_SETTINGS });
+        }
+        else if (o.replicated) {
+            // On every replica that answers: the copy may be running on another one
+            // than this run's. A replica that is down is skipped once Keeper sees it
+            // inactive; nothing of its can be running a copy that still writes,
+            // and a part it wrote before going down is waited for by the sync below.
+            await clickhouseQuery(run.target.endpoint, `KILL QUERY ON CLUSTER ${sqlString(n.database)} WHERE ${killWhere} SYNC`, {
+                settings: DDL_SETTINGS,
+            });
+            await syncReplica(run, o, n.database, n.newName);
+        }
+        else {
+            await clickhouseQuery(run.target.endpoint, `KILL QUERY WHERE ${killWhere} SYNC`);
+        }
+        const mine = `${fromOld} = ${sqlString(p)}${targetRange}`;
+        const where = shard ? `partition ${p} on shard ${shard.num}` : `partition ${p}`;
+        const [left] = await clickhouseQuery(run.target.endpoint, `SELECT count() AS n FROM ${newFrom(shard)} ${mine}`);
+        if (Number(left?.n ?? 0) > 0) {
+            run.log(`-- ${where}: ${left.n} row(s) from a copy that did not finish; deleting them before copying again`);
+            const sql = shard && sharding
+                ? renderFor(`ALTER TABLE ${n.newTable} DELETE WHERE ${onShardServers(shard)} AND ${mine}`, run.target.topology)
+                : `ALTER TABLE ${n.newTable} DELETE WHERE ${mine}`;
+            run.log(sql);
+            await clickhouseQuery(run.target.endpoint, sql, shard && sharding ? { settings: { ...DDL_SETTINGS, ...SHARD_DELETE_SETTINGS } } : {});
+            await waitOn(run, n.database, n.newName);
+            result.cleared++;
+        }
+        const sql = `INSERT INTO ${into(shard)} (${columns}) SELECT ${select} FROM ${oldFrom(shard)} _partition_id = ${sqlString(p)}${sourceRange}`;
+        run.log(sql);
+        await clickhouseQuery(run.target.endpoint, sql, { queryId, settings: { async_insert: "0", insert_deduplicate: "0", ...insertSettings }, ...(run.signal ? { signal: run.signal } : {}) });
+        if (later)
+            result.repaired += await copyAfterCutover(run, o, later, p, laterQueryId, shard);
+        // The receipt, last, on success only.
+        await receipts.write({ name: effect, effect, flavor: "hash", inputs: {} }, expectation);
+        result.copied++;
+        await deps.afterPartition?.(p, shard?.num);
+    }
+    run.log(`-- backfill of ${n.key}: ${result.partitions} partition(s), ${result.copied} copied, ${result.skipped} already copied, ${result.cleared} cleared first${result.repaired > 0 ? `, ${result.repaired} row(s) after the cut-over copied twice and repaired` : ""}`);
+    return result;
+}
+/** The most extra rows one partition's repair deletes and copies again; past that the verification decides. */
+const REPAIR_LIMIT = 1000;
+const REPAIR_ROUNDS = 3;
+/**
+ * The statements for the rows at or after the cut-over. The old side casts
+ * each column to its type in the new table before hashing, as the
+ * verification does, so a row and its copy hash the same.
+ */
+async function laterRowsCopy(run, o, cut, cutover, fromOld) {
+    const n = o.names;
+    const types = new Map((await clickhouseQuery(run.target.endpoint, `SELECT name, type FROM system.columns WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)}`)).map((c) => [c.name, c.type]));
+    const oldHash = rowHash(o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`));
+    const newHash = rowHash(o.copied.map((c) => ident(c.name)));
+    const columns = o.copied.map((c) => ident(c.name)).join(", ");
+    const aliases = o.copied.map((_, i) => `__chant_c${i}`);
+    const fromOldSide = o.copied.map((c, i) => `${ident(c.source)} AS ${aliases[i]}`).join(", ");
+    const sharding = run.sharding;
+    const on = (shard) => (shard && sharding ? ` AND ${onShard(shard)}` : "");
+    const oldTable = (shard) => (shard && sharding ? shardTable(sharding, n.database, n.name) : n.table);
+    const newTable = (shard) => (shard && sharding ? shardTable(sharding, n.database, n.newName) : n.newTable);
+    const oldRows = (p, shard) => `FROM ${oldTable(shard)} WHERE _partition_id = ${sqlString(p)} AND ${ident(cut.source)} >= ${utcLiteral(cutover)}${on(shard)}`;
+    const newRows = (p, shard) => `FROM ${newTable(shard)} WHERE ${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)}${on(shard)}`;
+    return {
+        copy: (p, shard) => `INSERT INTO ${shard && sharding ? intoShard(sharding, shard, n.database, n.newName) : n.newTable} (${columns}) SELECT ${aliases.join(", ")} FROM (` +
+            `SELECT ${fromOldSide}, ${oldHash} AS __chant_h, row_number() OVER (PARTITION BY __chant_h) AS __chant_k ${oldRows(p, shard)}` +
+            `) AS old_rows LEFT JOIN (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p, shard)} GROUP BY __chant_h) AS new_rows ` +
+            `USING (__chant_h) WHERE __chant_k > ifNull(__chant_n, 0)`,
+        extras: (p, shard) => `SELECT toString(__chant_h) AS h FROM (SELECT ${newHash} AS __chant_h, count() AS __chant_n ${newRows(p, shard)} GROUP BY __chant_h) AS new_rows ` +
+            `LEFT JOIN (SELECT ${oldHash} AS __chant_h, count() AS __chant_o ${oldRows(p, shard)} GROUP BY __chant_h) AS old_rows ` +
+            `USING (__chant_h) WHERE __chant_n > ifNull(__chant_o, 0) LIMIT ${REPAIR_LIMIT + 1}`,
+        drop: (p, hashes, shard) => {
+            const where = `${fromOld} = ${sqlString(p)} AND ${ident(cut.name)} >= ${utcLiteral(cutover)} AND ${newHash} IN (${hashes.join(", ")})`;
+            return shard && sharding
+                ? renderFor(`ALTER TABLE ${n.newTable} DELETE WHERE ${onShardServers(shard)} AND ${where}`, run.target.topology)
+                : `ALTER TABLE ${n.newTable} DELETE WHERE ${where}`;
+        },
+    };
+}
+/**
+ * Copy one old partition's rows at or after the cut-over that the new table
+ * lacks. A row the view had written to the old table but not yet to the new
+ * one when the copy read them is copied by both, and is then in the new
+ * table once too often: every copy of such a row is deleted from the new
+ * table and the difference copied again, a few rounds at most. What is left
+ * after that, the verification finds. Returns the rows repaired.
+ */
+async function copyAfterCutover(run, o, later, p, queryId, shard) {
+    const n = o.names;
+    const sharded = shard !== undefined && run.sharding !== undefined;
+    const insert = async () => {
+        const sql = later.copy(p, shard);
+        run.log(sql);
+        await clickhouseQuery(run.target.endpoint, sql, {
+            queryId,
+            settings: { async_insert: "0", insert_deduplicate: "0", ...(sharded ? SHARD_INSERT_SETTINGS : {}) },
+            ...(run.signal ? { signal: run.signal } : {}),
+        });
+    };
+    await insert();
+    let repaired = 0;
+    for (let round = 0; round < REPAIR_ROUNDS; round++) {
+        await syncReplica(run, o, n.database, n.newName);
+        const extras = (await clickhouseQuery(run.target.endpoint, later.extras(p, shard))).map((r) => r.h);
+        if (extras.length === 0 || extras.length > REPAIR_LIMIT)
+            break;
+        run.log(`-- partition ${p}: ${extras.length} row(s) after the cut-over in the new table more often than in the old one; deleting them and copying again`);
+        const sql = later.drop(p, extras, shard);
+        run.log(sql);
+        await clickhouseQuery(run.target.endpoint, sql, sharded ? { settings: { ...DDL_SETTINGS, ...SHARD_DELETE_SETTINGS } } : {});
+        await waitOn(run, n.database, n.newName);
+        repaired += extras.length;
+        await insert();
+    }
+    return repaired;
+}

package/dist/clickhouse/rebuild/engines.js

@@ -0,0 +1,146 @@
+/**
+ * Which table engines the rebuild reads as merged (#3674): the verification
+ * (`./verify.ts`) and the plan's hand-off note (`../plan/rebuild-handoff.ts`).
+ */
+/**
+ * The MergeTree engines that collapse rows sharing a sorting key when parts
+ * merge, as `system.tables.engine` names them, replicated or shared too. A
+ * count of such a table depends on which parts have merged; under `FINAL` it
+ * does not.
+ */
+const COLLAPSING_ENGINE = /^(?:Replicated|Shared)?(?:Summing|Replacing|Aggregating|Collapsing|VersionedCollapsing|Coalescing|Graphite)MergeTree$/;
+/** Whether an engine collapses rows that share a sorting key (#3674). */
+export const collapsesRows = (engine) => COLLAPSING_ENGINE.test(engine ?? "");
+/** Split on the commas outside parentheses and quotes. */
+export function splitTopLevel(text) {
+    const out = [];
+    let depth = 0;
+    let quote;
+    let start = 0;
+    for (let i = 0; i < text.length; i++) {
+        const c = text[i];
+        if (quote) {
+            if (c === "\\")
+                i++;
+            else if (c === quote)
+                quote = undefined;
+        }
+        else if (c === "'" || c === '"' || c === "`")
+            quote = c;
+        else if (c === "(" || c === "[")
+            depth++;
+        else if (c === ")" || c === "]")
+            depth--;
+        else if (c === "," && depth === 0) {
+            out.push(text.slice(start, i).trim());
+            start = i + 1;
+        }
+    }
+    const last = text.slice(start).trim();
+    if (last || out.length > 0)
+        out.push(last);
+    return out;
+}
+/** The engine's own arguments in `system.tables.engine_full`, without a `Replicated` engine's Keeper path and replica name. */
+export function engineArguments(engineFull) {
+    const m = /^\s*(\w+)\s*\(/.exec(engineFull);
+    if (!m)
+        return [];
+    let depth = 0;
+    let quote;
+    const from = m[0].length;
+    let end = -1;
+    for (let i = from - 1; i < engineFull.length && end < 0; i++) {
+        const c = engineFull[i];
+        if (quote) {
+            if (c === "\\")
+                i++;
+            else if (c === quote)
+                quote = undefined;
+        }
+        else if (c === "'")
+            quote = c;
+        else if (c === "(")
+            depth++;
+        else if (c === ")" && --depth === 0)
+            end = i;
+    }
+    if (end < 0)
+        return [];
+    const args = splitTopLevel(engineFull.slice(from, end)).filter((a) => a !== "");
+    if (/^(?:Replicated|Shared)/.test(m[1]))
+        while (args.length > 0 && args[0].startsWith("'"))
+            args.shift();
+    return args;
+}
+const NUMERIC = /^(?:U?Int(?:8|16|32|64|128|256)|Float(?:32|64)|BFloat16|Decimal(?:32|64|128|256)?\s*\(.*\))$/;
+const FLOAT = /^(?:Float(?:32|64)|BFloat16)$/;
+/** The SimpleAggregateFunction functions whose result a merge keeps exactly, whatever the order of the rows. */
+const SIMPLE_ORDER_FREE = new Set(["sum", "sumWithOverflow", "min", "max", "groupBitAnd", "groupBitOr", "groupBitXor"]);
+const quoted = (name) => `\`${name.replace(/\\/g, "\\\\").replace(/`/g, "\\`")}\``;
+const typeLiteral = (type) => `'${type.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+/** A sum in the column's own type, as the engine stores it; a float to 32 bits, since the order rows are added in moves its last bits. */
+const summed = (c) => (FLOAT.test(c.type) ? `toFloat32(sum(${quoted(c.name)}))` : `CAST(sum(${quoted(c.name)}), ${typeLiteral(c.type)})`);
+/**
+ * What a merge of `engine` keeps per sorting key, or undefined for an engine
+ * that does not collapse rows, or whose result depends on the order rows
+ * merge in (`GraphiteMergeTree` thins its rows by age).
+ */
+export function collapseSemantics(engine, engineFull, columns) {
+    if (!collapsesRows(engine))
+        return undefined;
+    const base = engine.replace(/^(?:Replicated|Shared)/, "").replace(/MergeTree$/, "");
+    const args = engineArguments(engineFull);
+    const values = columns.filter((c) => !c.inSortingKey);
+    const named = (n) => (n ? columns.find((c) => c.name === n.replace(/^`|`$/g, "")) : undefined);
+    switch (base) {
+        case "Summing": {
+            // The columns listed, else every numeric column outside the sorting and partition keys; a key whose sums are all zero is removed.
+            const listed = args[0] ? splitTopLevel(args[0].replace(/^\((.*)\)$/s, "$1")).map(named) : undefined;
+            const sums = (listed ? listed.filter((c) => c !== undefined) : values).filter((c) => !c.inSortingKey && !c.inPartitionKey && NUMERIC.test(c.type));
+            const aggregates = sums.map(summed);
+            return {
+                engine: `${base}MergeTree`,
+                aggregates,
+                ...(aggregates.length > 0 ? { having: `NOT (${aggregates.map((a) => `${a} = 0`).join(" AND ")})` } : {}),
+                describe: sums.length > 0 ? `the sums of ${sums.map((c) => c.name).join(", ")}` : "the keys",
+            };
+        }
+        case "Replacing": {
+            // The row with the highest version is kept; without a version, the last one written, so only the key is compared.
+            const ver = named(args[0]);
+            return { engine: `${base}MergeTree`, aggregates: ver ? [`max(${quoted(ver.name)})`] : [], describe: ver ? `the highest ${ver.name}` : "the keys" };
+        }
+        case "Collapsing":
+        case "VersionedCollapsing": {
+            // A pair of rows with opposite signs cancels: the sum of the signs is what a merge keeps, and a key whose signs cancel out is removed.
+            const sign = named(args[0]);
+            if (!sign)
+                return undefined;
+            const s = `sum(${quoted(sign.name)})`;
+            return { engine: `${base}MergeTree`, aggregates: [s], having: `${s} != 0`, describe: `the sum of ${sign.name}` };
+        }
+        case "Aggregating": {
+            const simple = values.flatMap((c) => {
+                const m = /^SimpleAggregateFunction\(\s*(\w+)\s*,/.exec(c.type);
+                return m && SIMPLE_ORDER_FREE.has(m[1]) ? [{ c, fn: m[1] }] : [];
+            });
+            return {
+                engine: `${base}MergeTree`,
+                aggregates: simple.map(({ c, fn }) => `${fn}(${quoted(c.name)})`),
+                describe: simple.length > 0 ? `${simple.map(({ c, fn }) => `${fn}(${c.name})`).join(", ")}` : "the keys",
+            };
+        }
+        case "Coalescing": {
+            // The last value that is not NULL is kept: whether a column has one is what a merge keeps.
+            const nullable = values.filter((c) => /^Nullable\(/.test(c.type));
+            return {
+                engine: `${base}MergeTree`,
+                aggregates: nullable.map((c) => `count(${quoted(c.name)}) > 0`),
+                describe: nullable.length > 0 ? `which of ${nullable.map((c) => c.name).join(", ")} hold a value` : "the keys",
+            };
+        }
+        default:
+            return undefined;
+    }
+}