@intentius/chant-lexicon-sql 0.124.31 → 0.124.32

package/dist/clickhouse/rebuild/observe.js

@@ -0,0 +1,260 @@
+/**
+ * The rebuild migration's view of the server (#3198): which table is being
+ * rebuilt, how far the rebuild has got, and what it will do.
+ *
+ * Every step of the Op re-observes from here rather than trusting what an
+ * earlier run or step said, so the Op converges: run it again after a crash,
+ * a gate or an approval, and each step finds its work done or still to do.
+ * The progress is read off the server's own objects:
+ *
+ * - the table being rebuilt, `db.t`, as declared and as the server has it;
+ * - `db.t__chant_new`, the new table being filled (`role=new` in its comment);
+ * - `db.t__chant_dual`, the dual-write materialized view (`role=dual`);
+ * - `db.t__chant_old`, the old table kept after the swap (`role=old`, with
+ *   `retain-until`).
+ *
+ * Each working object carries chant's ownership trailer plus
+ * `rebuild=<db.t>` and its role (`../ownership.ts`), so it is chant's and
+ * this rebuild's, and schema reads leave it out.
+ *
+ * States:
+ *
+ * - `rebuild`: the declaration differs from the server by at least one
+ *   rebuild-class change, and the old table has not been swapped out yet.
+ * - `swapped`: the swap ran; the old table is retained until its date.
+ * - `done`: the server holds the declaration and nothing of the rebuild is
+ *   left.
+ *
+ * Anything else is refused with a {@link RebuildRefusal}: a database that is
+ * neither Atomic nor Replicated, a table that is not there, a change that is
+ * not a rebuild, a working object chant did not make.
+ *
+ * A Replicated database (#3249) is Atomic on every replica, with its DDL run
+ * on each through Keeper, so the same EXCHANGE swaps the table everywhere.
+ * Its tables have to be `Replicated*MergeTree` on both sides of the rebuild:
+ * a plain MergeTree there keeps a different set of rows on each replica, and
+ * a copy run on one replica would fill the new table with that replica's rows
+ * alone.
+ */
+import { computePlanDigest } from "@intentius/chant/op";
+import { clickhouseQuery } from "../http.js";
+import { canonicalObject } from "../plan/normalize.js";
+import { diffSchemas } from "../plan/diff.js";
+import { dropFormattingOnly } from "../plan/server-format.js";
+import { CLASSIFIER_RULES } from "../plan/rules.js";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities.js";
+import { carriesMarker, readTrailerPairs, REBUILD_TRAILER_KEY, stripMarkerFromStatement } from "../ownership.js";
+import { ident, isRebuild, qualifiedIdent, sqlString } from "../apply/statements.js";
+/** A refusal: the rebuild cannot start, or cannot go on, for a reason a person has to act on. */
+export class RebuildRefusal extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "RebuildRefusal";
+    }
+}
+export function rebuildNames(database, name) {
+    const newName = `${name}__chant_new`;
+    const dualName = `${name}__chant_dual`;
+    const oldName = `${name}__chant_old`;
+    return {
+        key: `${database}.${name}`,
+        database,
+        name,
+        newName,
+        dualName,
+        oldName,
+        table: qualifiedIdent(database, name),
+        newTable: qualifiedIdent(database, newName),
+        dualView: qualifiedIdent(database, dualName),
+        oldTable: qualifiedIdent(database, oldName),
+    };
+}
+const qualifiedKey = (o) => `${o.canonical.database ?? "default"}.${o.canonical.name}`;
+/** The declared table with the given key (`db.t`), or a refusal naming what is declared. */
+export function declaredTable(declared, key) {
+    const found = declared.find((o) => o.key === key || o.exportName === key);
+    if (!found) {
+        throw new RebuildRefusal(`${key} is not declared in the build. Declared tables: ${declared.filter((o) => o.type === CLICKHOUSE_ENTITY_TYPES.table).map(qualifiedKey).join(", ") || "none"}`);
+    }
+    if (found.type !== CLICKHOUSE_ENTITY_TYPES.table)
+        throw new RebuildRefusal(`${key} is a ${found.type}, and only a table is rebuilt`);
+    return found;
+}
+/**
+ * The columns the backfill copies: every declared column the old table has
+ * under its name or its `-- previously:` name, except a `MATERIALIZED`,
+ * `ALIAS` or `EPHEMERAL` one, which the new table computes itself.
+ */
+export function copiedColumns(declared, live) {
+    const liveNames = new Set(live.columns.map((c) => c.name));
+    const out = [];
+    for (const c of declared.columns) {
+        if (c.defaultKind && c.defaultKind !== "DEFAULT")
+            continue;
+        const source = liveNames.has(c.name) ? c.name : c.previously && liveNames.has(c.previously) ? c.previously : undefined;
+        if (source)
+            out.push({ name: c.name, source });
+    }
+    return out;
+}
+/** A canonical definition without what names it, for comparing and digesting by role. */
+export function anonymous(c) {
+    const out = {
+        ...c,
+        columns: c.columns.map((col) => Object.fromEntries(Object.entries(col).filter(([k]) => k !== "text"))),
+    };
+    for (const k of ["name", "database", "previously", "comment"])
+        delete out[k];
+    return out;
+}
+async function serverObjects(target, names) {
+    const rows = await clickhouseQuery(target.endpoint, `SELECT name, toString(uuid) AS uuid, engine, comment, partition_key FROM system.tables ` +
+        `WHERE database = ${sqlString(names.database)} AND name IN (${[names.name, names.newName, names.dualName, names.oldName].map(sqlString).join(", ")})`);
+    return new Map(rows.map((r) => {
+        const pairs = readTrailerPairs(r.comment);
+        return [r.name, { name: r.name, uuid: r.uuid, engine: r.engine, comment: r.comment, partitionKey: r.partition_key, ...(pairs?.has(REBUILD_TRAILER_KEY) ? { pairs } : {}) }];
+    }));
+}
+/** The definition `SHOW CREATE` prints, less chant's trailer, canonical. */
+export async function showCreate(target, qualified) {
+    const [row] = await clickhouseQuery(target.endpoint, `SHOW CREATE TABLE ${qualified}`);
+    return canonicalObject(stripMarkerFromStatement(row?.statement ?? ""), target.defaultDatabase);
+}
+/** The classified changes from one definition to another, compared by role, with server formatting differences dropped. */
+export async function changesBetween(target, key, from, to) {
+    const diff = diffSchemas([{ key, canonical: from }], [{ key, canonical: to }]);
+    return dropFormattingOnly(target.endpoint, diff.changes);
+}
+/**
+ * A working object is this rebuild's only when it carries this project's
+ * marker, names this table and has the role its name says. One that has
+ * the name and not the marker is somebody else's, and the rebuild stops
+ * rather than drop or fill it.
+ */
+function ownWorkingObject(o, names, role, marker) {
+    if (!o)
+        return undefined;
+    if (!carriesMarker(o.comment, marker) || o.pairs?.get(REBUILD_TRAILER_KEY) !== names.key || o.pairs.get("role") !== role) {
+        throw new RebuildRefusal(`${names.database}.${o.name} exists and is not this rebuild's ${role} table (its comment carries no rebuild=${names.key} role=${role} marker for this project). ` +
+            `The rebuild uses that name, so it stops rather than touch it; rename or drop it by hand.`);
+    }
+    return o;
+}
+/** Observe the rebuild of the declared table on the server. */
+export async function observeRebuild(target, declared, marker) {
+    const database = declared.canonical.database ?? target.defaultDatabase;
+    const names = rebuildNames(database, declared.canonical.name);
+    const [db] = await clickhouseQuery(target.endpoint, `SELECT engine FROM system.databases WHERE name = ${sqlString(database)}`);
+    if (!db)
+        throw new RebuildRefusal(`database ${database} is not on the server, so there is no ${names.key} to rebuild; the applier creates it`);
+    if (db.engine !== "Atomic" && db.engine !== "Replicated") {
+        throw new RebuildRefusal(`database ${database} uses the ${db.engine} engine. The swap is EXCHANGE TABLES, which ClickHouse supports only in an Atomic database ` +
+            `(https://clickhouse.com/docs/sql-reference/statements/exchange) and the Replicated one built on it, and two RENAMEs in its place would leave a moment with no ${names.key} for writes to land in. ` +
+            `Convert the database to Atomic first (https://clickhouse.com/docs/engines/database-engines/atomic), then run the rebuild.`);
+    }
+    const replicated = db.engine === "Replicated";
+    const objects = await serverObjects(target, names);
+    const liveRow = objects.get(names.name);
+    if (!liveRow)
+        throw new RebuildRefusal(`${names.key} is not on the server, so there is nothing to rebuild; the applier creates it`);
+    // Between EXCHANGE and the end of the swap, the table's own name holds the
+    // new table, still carrying role=new, and the new table's name holds the
+    // old table until it is renamed; the swap step finishes that.
+    const exchanged = liveRow.pairs?.get(REBUILD_TRAILER_KEY) === names.key && liveRow.pairs.get("role") === "new";
+    const newTable = exchanged ? undefined : ownWorkingObject(objects.get(names.newName), names, "new", marker);
+    const exchangedOld = exchanged ? objects.get(names.newName) : undefined;
+    const dual = ownWorkingObject(objects.get(names.dualName), names, "dual", marker);
+    const old = ownWorkingObject(objects.get(names.oldName), names, "old", marker);
+    const liveCanonical = await showCreate(target, names.table);
+    const changes = (await changesBetween(target, names.key, { ...liveCanonical, name: declared.canonical.name }, declared.canonical)).filter((c) => !(c.field === "comment" && c.rule === "SQLCH203"));
+    const copied = copiedColumns(declared.canonical, liveCanonical);
+    let state;
+    if (old || exchanged) {
+        if (changes.some(isRebuild) && !exchanged) {
+            throw new RebuildRefusal(`${names.key} differs from its declaration by a rebuild again, and the old table of the last rebuild is still kept as ${names.database}.${names.oldName}. ` +
+                `Drop it (the rebuild's drop phase, or DROP TABLE) before rebuilding again.`);
+        }
+        state = "swapped";
+    }
+    else if (changes.some(isRebuild)) {
+        state = "rebuild";
+    }
+    else if (changes.length === 0) {
+        state = "done";
+    }
+    else {
+        throw new RebuildRefusal(`${names.key} has no change that needs a rebuild: ${changes.map((c) => `${c.rule} ${CLASSIFIER_RULES[c.rule].title} on ${c.field}`).join("; ")}. ` +
+            `The applier makes these in place (ApplyOp with target "clickhouse").`);
+    }
+    if (state === "rebuild") {
+        if (replicated)
+            refuseUnreplicated(names, liveRow.engine, declared.canonical.engineName);
+        refuseSharedKeeperPath(names, liveCanonical.engine, declared.canonical.engine);
+    }
+    if (state === "done" && (newTable || dual)) {
+        throw new RebuildRefusal(`${names.key} already holds its declaration, and ${[newTable, dual].filter(Boolean).map((o) => `${names.database}.${o.name}`).join(" and ")} from an unfinished rebuild is still there. ` +
+            `The run fails so onFailure drops it.`);
+    }
+    return {
+        state,
+        names,
+        declared,
+        live: liveRow,
+        liveCanonical,
+        changes,
+        ...(newTable ? { newTable } : {}),
+        ...(dual ? { dual } : {}),
+        ...(old ? { old } : {}),
+        ...(exchangedOld ? { exchangedOld } : {}),
+        exchanged,
+        copied,
+        replicated,
+    };
+}
+const REPLICATED_MERGE_TREE = /^Replicated.*MergeTree$/;
+/** In a Replicated database, both the table and its declaration have to replicate their rows. */
+function refuseUnreplicated(names, live, declared) {
+    const side = !REPLICATED_MERGE_TREE.test(live) ? `the table is ${live}` : declared !== undefined && !REPLICATED_MERGE_TREE.test(declared) ? `the declaration makes it ${declared}` : undefined;
+    if (!side)
+        return;
+    throw new RebuildRefusal(`${names.key} is in a Replicated database and ${side}, which keeps a separate set of rows on each replica. ` +
+        `The backfill runs on one replica and would copy that replica's rows alone, while the EXCHANGE runs on every replica, so the others would swap in a table without their rows. ` +
+        `Rebuild it from and to a Replicated*MergeTree engine (ENGINE = ReplicatedMergeTree, with no arguments in a Replicated database), or move it to an Atomic database first.`);
+}
+/** The Keeper path a `Replicated*MergeTree` engine names, when it names one. */
+function keeperPath(engine) {
+    const m = /^Replicated\w*MergeTree\(\s*('(?:[^'\\]|\\.)*')/.exec(engine ?? "");
+    return m?.[1];
+}
+/**
+ * The new table is made from the declaration under another name. With an
+ * explicit Keeper path that is the old table's own and has no `{uuid}` in
+ * it, both tables would be the same replicated table to Keeper, and the
+ * create would fail on a replica that is already there or, worse, share parts.
+ */
+function refuseSharedKeeperPath(names, live, declared) {
+    const path = keeperPath(declared);
+    if (!path || path.includes("{uuid}") || path !== keeperPath(live))
+        return;
+    throw new RebuildRefusal(`${names.key} names its Keeper path explicitly, ${path}, and the new table made from the declaration would get the same path as the old table. ` +
+        `Put {uuid} in the path (or leave the engine's arguments out, which defaults to '/clickhouse/tables/{uuid}/{shard}'), so each table has its own.`);
+}
+/**
+ * What the rebuild will do, independent of how far it has got: the changes,
+ * the definitions on both sides, the columns copied and how writes are kept
+ * up. The Plan phase publishes its digest; the gate binds a digest of this
+ * plus the verification.
+ */
+export function rebuildPlanSubject(o, dualWrite) {
+    return {
+        table: o.names.key,
+        from: anonymous(o.liveCanonical),
+        to: anonymous(o.declared.canonical),
+        changes: o.changes.map((c) => ({ rule: c.rule, field: c.field, ...(c.before !== undefined ? { before: c.before } : {}), ...(c.after !== undefined ? { after: c.after } : {}) })),
+        copied: o.copied,
+        dualWrite,
+    };
+}
+export const rebuildPlanDigest = (o, dualWrite) => computePlanDigest("clickhouse-rebuild", rebuildPlanSubject(o, dualWrite));
+/** `ident` re-exported for the steps, which quote column names. */
+export { ident };

package/dist/clickhouse/rebuild/op.js

@@ -0,0 +1,152 @@
+/**
+ * `ClickHouseRebuildOp` (#3198): the change ClickHouse cannot make with
+ * `ALTER` (a sorting key, a primary key, a partition key, an engine, a key
+ * column's type), run as a gated composite Op.
+ *
+ *     export const { op } = ClickHouseRebuildOp({
+ *       name: "rebuild-shop-events",
+ *       env: "prod",
+ *       table: "shop.events",
+ *       dualWrite: { mode: "materialized-view", cutoverColumn: "ts" },
+ *     });
+ *
+ * | Phase | Step |
+ * |---|---|
+ * | Build | `chant build` (`build: false` skips it) |
+ * | Plan | classify the table's change against the server; refuse what is not a rebuild |
+ * | Create | the new table, `t__chant_new`, from the declaration |
+ * | Dual write | a materialized view from the old table to the new one, or a gate for the app to stop writing |
+ * | Backfill | `INSERT ... SELECT` per partition, each with a receipt |
+ * | Verify | row counts and checksums per partition |
+ * | Approve | a gate bound to the plan and the verification |
+ * | Swap | `EXCHANGE TABLES`, then recreate the materialized views that read the table |
+ * | Retain | the old table, `t__chant_old`, until `retain` has passed |
+ * | Approve drop | a gate bound to that old table |
+ * | Drop | the old table, once its date has passed |
+ * | onFailure | drop the new table and the dual-write view |
+ *
+ * `gates: "outer"` (#3658) leaves out both approval gates and the Drop phase,
+ * for a caller that runs the Op under an approval of its own: the run goes
+ * from Plan to Retain, verification and the swap's own comparison included.
+ * `onFailure: "keep"` leaves out the onFailure, so a failed run's next run
+ * resumes the backfill from its receipts instead of starting again.
+ *
+ * Run it until it is done: each run goes as far as the next gate. Every step
+ * re-reads the server, so a run after a gate, a crash or an approval picks up
+ * where the last stopped. A run killed during the backfill (Ctrl-C, a lost
+ * machine) is not a failure, runs no onFailure, and the next run resumes the
+ * backfill from its receipts; a step that fails runs onFailure, and the next
+ * run starts again from a new table.
+ */
+import { Op, activity, gate, phase, stepOutput } from "@intentius/chant/op";
+export function ClickHouseRebuildOp(config) {
+    if (!/^[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*$|^[A-Za-z_][A-Za-z0-9_]*$/.test(config.table)) {
+        throw new Error(`ClickHouseRebuildOp "${config.name}": table must be "database.name" or an export name, got ${JSON.stringify(config.table)}`);
+    }
+    if (config.dualWrite?.mode !== "materialized-view" && config.dualWrite?.mode !== "app") {
+        throw new Error(`ClickHouseRebuildOp "${config.name}": dualWrite.mode must be "materialized-view" or "app"`);
+    }
+    if (config.dualWrite.mode === "materialized-view" && !config.dualWrite.cutoverColumn) {
+        throw new Error(`ClickHouseRebuildOp "${config.name}": materialized-view mode needs cutoverColumn, the time column rows are cut over on`);
+    }
+    if (config.gates !== undefined && config.gates !== "own" && config.gates !== "outer") {
+        throw new Error(`ClickHouseRebuildOp "${config.name}": gates must be "own" or "outer"`);
+    }
+    if (config.onFailure !== undefined && config.onFailure !== "drop" && config.onFailure !== "keep") {
+        throw new Error(`ClickHouseRebuildOp "${config.name}": onFailure must be "drop" or "keep"`);
+    }
+    const outer = config.gates === "outer";
+    const keep = config.onFailure === "keep";
+    const args = {
+        table: config.table,
+        buildPath: config.output ?? "dist/schema.json",
+        environment: config.env,
+        dualWrite: config.dualWrite,
+        ...(config.retain ? { retain: config.retain } : {}),
+        ...(config.mutationTimeout ? { mutationTimeout: config.mutationTimeout } : {}),
+        ...(config.replicaTimeout ? { replicaTimeout: config.replicaTimeout } : {}),
+        ...(config.cutoverTimeout ? { cutoverTimeout: config.cutoverTimeout } : {}),
+        ...(config.stack ? { stack: config.stack } : {}),
+        ...(config.ownershipEnv ? { ownershipEnv: config.ownershipEnv } : {}),
+        ...(config.path && config.path !== "." ? { cwd: config.path } : {}),
+        ...(keep ? { keepOnFailure: true } : {}),
+    };
+    const a = args;
+    const swapGate = config.gate?.gate ?? `approve-${config.name}`;
+    const dropGate = config.dropGate?.gate ?? `approve-${config.name}-drop`;
+    const policy = config.gate?.approval?.policy;
+    const step = (fn, extra = {}) => ({ ...activity(fn, a), ...extra });
+    // The swap waits for this gate, unless the caller holds the approval (`gates: "outer"`).
+    const approve = phase("Approve", [
+        gate(swapGate, {
+            ...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
+            plan: stepOutput("verify", "planDigest"),
+            description: config.gate?.description ??
+                `Approve swapping the rebuilt ${config.table} in on ${config.env}: the run record's Verification line has the counts this approval binds`,
+            ...(config.gate?.approval
+                ? {
+                    approval: {
+                        ...config.gate.approval,
+                        ...(policy
+                            ? {
+                                context: {
+                                    verifiedPartitions: stepOutput("verify", "partitions"),
+                                    verifiedRows: stepOutput("verify", "rows"),
+                                    ...config.gate.approval.context,
+                                },
+                            }
+                            : {}),
+                    },
+                }
+                : {}),
+        }),
+    ]);
+    const phases = [
+        ...(config.build === false ? [] : [phase("Build", [activity("chantBuild", { path: config.path ?? "." })])]),
+        phase("Plan", [step("clickhouseRebuildPlan", { id: "plan", outcomeAttribute: { name: "RebuildState", from: "state" } })]),
+        phase("Create", [step("clickhouseRebuildCreate")]),
+        phase("Dual write", [
+            config.dualWrite.mode === "app"
+                ? gate(config.writesGate?.gate ?? `${config.name}-writes-stopped`, {
+                    ...(config.writesGate?.timeout ? { timeout: config.writesGate.timeout } : {}),
+                    description: config.writesGate?.description ??
+                        `Stop the application's writes to ${config.table} (pause or buffer them) and approve; they resume against ${config.table} after the swap`,
+                })
+                : step("clickhouseRebuildDualWrite", { outcomeAttribute: { name: "Cutover", from: "cutover" } }),
+        ]),
+        phase("Backfill", [step("clickhouseRebuildBackfill", { profile: "fastIdempotent", timeout: config.backfillTimeout ?? "6h", outcomeAttribute: [{ name: "Copied", from: "copied" }, { name: "Skipped", from: "skipped" }] })]),
+        phase("Verify", [
+            step("clickhouseRebuildVerify", {
+                id: "verify",
+                outcomeAttribute: [
+                    { name: "Verification", from: "summary" },
+                    { name: "VerifiedPartitions", from: "partitions" },
+                    { name: "VerifiedRows", from: "rows" },
+                ],
+            }),
+        ]),
+        ...(outer ? [] : [approve]),
+        phase("Swap", [step("clickhouseRebuildSwap", { profile: "longInfra", outcomeAttribute: { name: "Dependents", from: "dependents" } })]),
+        phase("Retain", [step("clickhouseRebuildRetain", { id: "retain", outcomeAttribute: { name: "RetainUntil", from: "retainUntil" } })]),
+        ...(outer
+            ? []
+            : [
+                phase("Approve drop", [
+                    gate(dropGate, {
+                        ...(config.dropGate?.timeout ? { timeout: config.dropGate.timeout } : {}),
+                        plan: stepOutput("retain", "dropDigest"),
+                        description: config.dropGate?.description ?? `Approve dropping the old ${config.table} once its retention has passed`,
+                    }),
+                ]),
+                phase("Drop", [step("clickhouseRebuildDrop", { outcomeAttribute: { name: "Dropped", from: "dropped" } })]),
+            ]),
+    ];
+    const op = Op({
+        name: config.name,
+        overview: `Rebuild ${config.table} on ${config.env}: new table, ${config.dualWrite.mode === "app" ? "writes stopped" : "dual write"}, backfill, verify, ${outer ? "swap under the caller's approval" : "gated swap"}`,
+        labels: { Rebuild: "true", Env: config.env, Table: config.table },
+        phases,
+        ...(keep ? {} : { onFailure: [phase("Compensate", [step("clickhouseRebuildCompensate")])] }),
+    });
+    return { op };
+}

package/dist/clickhouse/rebuild/partitions.js

@@ -0,0 +1,65 @@
+/**
+ * Partitions, as the backfill copies them and the verification counts them.
+ *
+ * The unit of work is a partition of the old table. Its rows can land in
+ * other partitions of the new table when the rebuild changes the partition
+ * key, so "the rows of old partition p" is found on the new table by
+ * computing the old partition key over the new table's columns:
+ * `partitionID(<old key>)`, which the server computes the same way it named
+ * the old partition (`_partition_id`). When the key is unchanged that is the
+ * new table's own `_partition_id`; when the old table has no partition key
+ * every row is in partition `all`.
+ */
+import { clickhouseQuery } from "../http.js";
+import { ident, sqlString } from "../apply/statements.js";
+/** The old table's active partitions, by id. */
+export async function sourcePartitions(target, names) {
+    const rows = await clickhouseQuery(target.endpoint, `SELECT partition_id FROM system.parts WHERE database = ${sqlString(names.database)} AND table = ${sqlString(names.name)} AND active ` +
+        `GROUP BY partition_id ORDER BY partition_id`);
+    return rows.map((r) => r.partition_id);
+}
+/** The old table's active partitions on each shard of a cluster (#3663), through this server. */
+export async function shardPartitions(run, sharding, database, table) {
+    const rows = await clickhouseQuery(run.target.endpoint, `SELECT _shard_num AS shard, partition_id FROM cluster(${sqlString(sharding.cluster)}, system.parts) ` +
+        `WHERE database = ${sqlString(database)} AND table = ${sqlString(table)} AND active GROUP BY shard, partition_id ORDER BY shard, partition_id`);
+    return rows.map((r) => ({ shard: Number(r.shard), partition: r.partition_id }));
+}
+/** Rename the columns an expression names, old name to new, leaving function names and qualified names alone. */
+export function renameColumns(expr, copied) {
+    let out = expr;
+    for (const c of copied) {
+        if (c.source === c.name)
+            continue;
+        const re = new RegExp(`(?<![\\w.\`])\`?${c.source.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\`?(?![\\w\`(])`, "g");
+        out = out.replace(re, ident(c.name));
+    }
+    return out;
+}
+/** A tuple key's elements as arguments (`(a, b)` is `a, b`); any other key as itself. */
+function asArguments(key) {
+    const k = key.trim();
+    if (!k.startsWith("(") || !k.endsWith(")"))
+        return k;
+    let depth = 0;
+    for (let i = 0; i < k.length; i++) {
+        if (k[i] === "(")
+            depth++;
+        else if (k[i] === ")")
+            depth--;
+        if (depth === 0 && i < k.length - 1)
+            return k; // `(a) + (b)`: not one outer pair
+    }
+    return k.slice(1, -1);
+}
+/**
+ * The expression that gives, for a row of the new table, the id of the old
+ * table's partition the row came from.
+ */
+export function sourcePartitionExpression(oldKey, newKey, copied) {
+    if (oldKey.trim() === "")
+        return "'all'";
+    const renamed = renameColumns(oldKey, copied);
+    if (renamed === oldKey && oldKey.trim() === newKey.trim())
+        return "_partition_id";
+    return `partitionID(${asArguments(renamed)})`;
+}

package/dist/clickhouse/rebuild/receipts.js

@@ -0,0 +1,120 @@
+/**
+ * Effect receipts kept in ClickHouse (#3198): where the rebuild migration's
+ * backfill records which partitions it has copied.
+ *
+ * A receipt is the proof that one effect ran, kept where the effect happened
+ * (concepts/effect-receipts.mdx). The backfill's effect is "this partition of
+ * the old table is in the new table", and both tables are on the server, so
+ * the receipt is a row on the same server:
+ *
+ *     SELECT * FROM chant_receipts.receipts
+ *
+ * Why not git (core's lifecycle receipt store, `chant/lifecycle`): the
+ * receipt has to die with the data it witnesses. The rebuild's `onFailure`
+ * drops the new table; a receipt kept in git would go on saying partitions
+ * were copied into a table that no longer exists, and a resumed backfill
+ * would skip them. Here every receipt's expectation is bound to the new
+ * table's UUID, so a new table made again reads every old receipt as stale,
+ * and the receipts are read and written over the connection the copy already
+ * uses, with no commit and push per partition. Readable with nothing but a
+ * ClickHouse client, which is the walk-away property the other rows have.
+ *
+ * The store implements core's `ReceiptStore` seam
+ * (`@intentius/chant/op/receipt-store`), as the sql lexicon's shared
+ * `SqlReceiptStore` (`../../core/receipts.ts`). The backfill activity
+ * reaches the store directly, the way the fly lexicon's release component
+ * reaches its own. The same table holds the receipts of a sql project's
+ * `effect()` steps (`../../receipts.ts`), under their own addresses.
+ *
+ * Write discipline: the backfill writes a partition's receipt after the
+ * partition's copy succeeded, last, and nothing else writes one.
+ *
+ * In a Replicated database (#3249) the receipts sit beside the tables they
+ * witness, in `<db>.__chant_receipts`, a `ReplicatedReplacingMergeTree`: the
+ * database runs its CREATE on every replica and the table replicates its
+ * rows, so a receipt written by a copy on one replica is read by a resumed
+ * backfill on another. Each read first waits for this replica to fetch what
+ * the others wrote (`SYSTEM SYNC REPLICA ... LIGHTWEIGHT`). A separate
+ * `chant_receipts` database would exist on the one replica the backfill
+ * first ran on. With a replica down (#3270) the CREATE goes on without it,
+ * and a read that cannot fetch a receipt only that replica has stops the
+ * backfill, naming it (`./replicas.ts`).
+ */
+import { OWNERSHIP_MANAGED_BY_VALUE } from "@intentius/chant/ownership";
+import { receiptAddress } from "../../core/receipts.js";
+import { clickhouseQuery } from "../http.js";
+import { ident, sqlString } from "../apply/statements.js";
+import { CLICKHOUSE_COMMENT_OWNERSHIP_KEYS, RECEIPTS_TRAILER_KEY } from "../ownership.js";
+import { DDL_SETTINGS, syncReplicaWithin } from "./replicas.js";
+import { renderFor } from "../topology.js";
+/** The database the receipts live in. Its comment marks it as chant's, so schema reads leave it out. */
+export const RECEIPTS_DATABASE = "chant_receipts";
+/** The table, one row per receipt write. */
+export const RECEIPTS_TABLE = "receipts";
+/** The table in a Replicated database that holds the receipts of the rebuilds in it. */
+export const REPLICATED_RECEIPTS_TABLE = "__chant_receipts";
+/** Where the receipts are kept: `chant_receipts.receipts`, or `<db>.__chant_receipts` for a Replicated database. */
+export function receiptsTable(replicatedIn) {
+    return replicatedIn ? { database: replicatedIn, name: REPLICATED_RECEIPTS_TABLE } : { database: RECEIPTS_DATABASE, name: RECEIPTS_TABLE };
+}
+const COMMENT = `chant effect receipts [chant ${CLICKHOUSE_COMMENT_OWNERSHIP_KEYS.managedBy}=${OWNERSHIP_MANAGED_BY_VALUE} ${RECEIPTS_TRAILER_KEY}=effects]`;
+/** Plain synchronous inserts: an async insert can be acknowledged before it is written. */
+const SYNC = { async_insert: "0" };
+export { receiptAddress } from "../../core/receipts.js";
+/**
+ * A receipt store on the server at `endpoint`. Creates the receipts database
+ * and table on first write if they are not there. `identity` is the
+ * project's ownership stack and env, which every address starts with.
+ * `replicatedIn` names a Replicated database to keep them in instead, and
+ * `replicaTimeoutMs` how long a read waits for this replica to catch up there.
+ * `topology` is the environment's (`../topology.ts`): the database and table
+ * are created for it, `ON CLUSTER` and a `Replicated*` engine on a cluster.
+ * In a Replicated database they are created for that database whatever it says.
+ */
+export function clickhouseReceiptStore(endpoint, identity, opts = {}) {
+    const where = receiptsTable(opts.replicatedIn);
+    const topology = opts.replicatedIn ? { kind: "replicated" } : opts.topology;
+    const TABLE = `${ident(where.database)}.${ident(where.name)}`;
+    let ensured = false;
+    const ensure = async () => {
+        if (ensured)
+            return;
+        if (!opts.replicatedIn) {
+            await clickhouseQuery(endpoint, renderFor(`CREATE DATABASE IF NOT EXISTS ${ident(RECEIPTS_DATABASE)} COMMENT ${sqlString(COMMENT)}`, topology), { settings: DDL_SETTINGS });
+        }
+        await clickhouseQuery(endpoint, renderFor(`CREATE TABLE IF NOT EXISTS ${TABLE} (address String, effect String, expectation String, run_id String, written_at DateTime64(3) DEFAULT now64(3)) ` +
+            `ENGINE = ReplacingMergeTree(written_at) ORDER BY address COMMENT ${sqlString(COMMENT)}`, topology), { settings: DDL_SETTINGS });
+        ensured = true;
+    };
+    const exists = async () => {
+        const rows = await clickhouseQuery(endpoint, `SELECT count() AS n FROM system.tables WHERE database = ${sqlString(where.database)} AND name = ${sqlString(where.name)}`);
+        return Number(rows[0]?.n ?? 0) > 0;
+    };
+    /** Whether there is a table to read; in a Replicated database, after this replica has caught up with it. */
+    const readable = async () => {
+        if (!ensured && !(await exists()))
+            return false;
+        if (opts.replicatedIn)
+            await syncReplicaWithin(endpoint, where.database, where.name, opts.replicaTimeoutMs !== undefined ? { timeoutMs: opts.replicaTimeoutMs } : {});
+        return true;
+    };
+    const address = (receipt) => receiptAddress(identity, receipt.effect);
+    return {
+        async read(receipt) {
+            if (!(await readable()))
+                return undefined;
+            const rows = await clickhouseQuery(endpoint, `SELECT expectation FROM ${TABLE} WHERE address = ${sqlString(address(receipt))} ORDER BY written_at DESC LIMIT 1`);
+            return rows[0]?.expectation;
+        },
+        async readAll(prefix) {
+            if (!(await readable()))
+                return new Map();
+            const rows = await clickhouseQuery(endpoint, `SELECT address, argMax(expectation, written_at) AS expectation FROM ${TABLE} WHERE startsWith(address, ${sqlString(prefix)}) GROUP BY address`);
+            return new Map(rows.map((r) => [r.address, r.expectation]));
+        },
+        async write(receipt, expectation) {
+            await ensure();
+            await clickhouseQuery(endpoint, `INSERT INTO ${TABLE} (address, effect, expectation, run_id) VALUES (${sqlString(address(receipt))}, ${sqlString(receipt.effect)}, ${sqlString(expectation)}, ${sqlString(opts.runId ?? "")})`, { settings: SYNC });
+        },
+    };
+}