@intentius/chant-lexicon-sql 0.124.31 → 0.124.33

package/dist/clickhouse/rebuild/replicas.js

@@ -0,0 +1,121 @@
+/**
+ * A replica of a Replicated database that is down (#3270): what each step of
+ * the rebuild does about it.
+ *
+ * DDL (create, dual-write view, EXCHANGE, the views' DETACH and ATTACH, the
+ * drops, the receipts table) goes through the database's log in Keeper. The
+ * replica the run talks to executes it, and the others take it from the log;
+ * one that is down runs it in order when it comes back. So DDL proceeds on
+ * the live replicas: every statement carries
+ * `distributed_ddl_output_mode = throw_only_active`, which "doesn't wait for
+ * inactive replicas of the Replicated database" and still throws when a live
+ * one fails the statement
+ * (https://clickhouse.com/docs/reference/settings/session-settings/distributed-ddl).
+ * With the default `throw` the statement would wait the whole
+ * `distributed_ddl_task_timeout` (180s) for the down replica and then fail,
+ * after the live replicas had already run it. A replica counts as inactive
+ * once its Keeper session has expired (the client's `session_timeout_ms`,
+ * 30s by default), so a statement right after a replica goes down waits that
+ * long and then goes on. `KILL QUERY ON CLUSTER` takes the same setting and
+ * behaves the same way.
+ *
+ * Reading rows is different. A part written on a replica that went down
+ * before the others fetched it is on that replica alone, and the rows in it
+ * cannot be read anywhere else. A backfill that went on without them would
+ * copy the table less those rows, the verification would compare two tables
+ * that both lack them and pass, and the swap would lose them. So before a
+ * step reads rows it waits, for a bounded time, until this replica has
+ * fetched everything the others wrote (`SYSTEM SYNC REPLICA ... LIGHTWEIGHT`,
+ * https://clickhouse.com/docs/sql-reference/statements/system#sync-replica).
+ * When the down replica holds nothing this one lacks, which is the usual case,
+ * the wait returns at once and the step goes on. Otherwise the step stops
+ * with a {@link ReplicaFetchError} naming the replica and its parts.
+ */
+import { clickhouseQuery, ClickHouseQueryError } from "../http.js";
+import { qualifiedIdent, sqlString } from "../apply/statements.js";
+/** Settings every DDL statement of the rebuild carries: go on without the replicas that are down. Ignored in an Atomic database. */
+export const DDL_SETTINGS = { distributed_ddl_output_mode: "throw_only_active" };
+/** How long a step waits for this replica to fetch what the others wrote, by default. */
+export const DEFAULT_REPLICA_TIMEOUT_MS = 120000;
+/**
+ * One `SYSTEM SYNC REPLICA` waits at most this long (its `receive_timeout`),
+ * and the wait is repeated until the deadline. It keeps each HTTP request
+ * well inside the client's own five-minute response timeout, and puts a log
+ * line between them.
+ */
+const SYNC_SLICE_S = 60;
+/** A step stopped because this replica could not fetch parts that only another replica has. */
+export class ReplicaFetchError extends Error {
+    constructor(
+    /** `db.t`. */
+    table, 
+    /** The replica the run talks to. */
+    replica, missing, waitedMs) {
+        const sources = [...new Set(missing.map((m) => m.source))];
+        const shown = missing
+            .slice(0, 10)
+            .map((m) => m.part + " from " + m.source + (m.sourceActive ? "" : " (inactive)") + (m.lastError ? " (last fetch error: " + m.lastError + ")" : ""))
+            .join(", ");
+        super(`${table}: waited ${Math.round(waitedMs / 1000)}s for replica ${replica} to fetch what the other replicas wrote, and ${missing.length} part(s) are still to fetch: ${shown}${missing.length > 10 ? ", ..." : ""}. ` +
+            `Those rows are on ${sources.join(", ")} alone, so the step stops rather than read the table without them; nothing has been swapped. ` +
+            `Bring ${sources.join(", ")} back (it rejoins through Keeper and this replica fetches the parts), then run again. ` +
+            `replicaTimeout sets how long a step waits.`);
+        this.table = table;
+        this.replica = replica;
+        this.missing = missing;
+        this.waitedMs = waitedMs;
+        this.name = "ReplicaFetchError";
+    }
+}
+const isTimeout = (err) => err instanceof ClickHouseQueryError && /TIMEOUT_EXCEEDED|command timed out/.test(err.serverMessage);
+/**
+ * Wait until this replica has fetched every part the others have written to
+ * `database.table`, for at most `timeoutMs`; then throw a
+ * {@link ReplicaFetchError} naming what it still lacks and where that is.
+ */
+export async function syncReplicaWithin(endpoint, database, table, opts = {}) {
+    const started = Date.now();
+    const deadline = started + (opts.timeoutMs ?? DEFAULT_REPLICA_TIMEOUT_MS);
+    const sql = `SYSTEM SYNC REPLICA ${qualifiedIdent(database, table)} LIGHTWEIGHT`;
+    opts.log?.(sql);
+    for (;;) {
+        opts.signal?.throwIfAborted();
+        const slice = Math.max(1, Math.min(SYNC_SLICE_S, Math.ceil((deadline - Date.now()) / 1000)));
+        try {
+            await clickhouseQuery(endpoint, sql, { settings: { receive_timeout: String(slice) }, ...(opts.signal ? { signal: opts.signal } : {}) });
+            return;
+        }
+        catch (err) {
+            if (!isTimeout(err))
+                throw err;
+        }
+        const { replica, missing } = await missingParts(endpoint, database, table);
+        // Fetched between the timeout and the look: nothing is left to wait for.
+        if (missing.length === 0)
+            return;
+        if (Date.now() >= deadline)
+            throw new ReplicaFetchError(`${database}.${table}`, replica, missing, Date.now() - started);
+        opts.log?.(`-- ${database}.${table}: replica ${replica} is still to fetch ${missing.length} part(s) from ${[...new Set(missing.map((m) => m.source))].join(", ")}; waiting`);
+    }
+}
+/** The parts this replica's queue is still to fetch for `database.table`, with where from. */
+export async function missingParts(endpoint, database, table) {
+    const where = `database = ${sqlString(database)} AND table = ${sqlString(table)}`;
+    const [replicas] = await clickhouseQuery(endpoint, `SELECT replica_name, replica_is_active FROM system.replicas WHERE ${where}`);
+    // The entries LIGHTWEIGHT waits for.
+    const queue = await clickhouseQuery(endpoint, `SELECT new_part_name, source_replica, last_exception FROM system.replication_queue WHERE ${where} ` +
+        `AND type IN ('GET_PART', 'ATTACH_PART', 'DROP_RANGE', 'REPLACE_RANGE', 'DROP_PART') ORDER BY new_part_name`);
+    const active = replicas?.replica_is_active ?? {};
+    return {
+        replica: replicas?.replica_name ?? "this replica",
+        missing: queue.map((e) => {
+            const lastError = e.last_exception.trim().split("\n")[0];
+            return {
+                part: e.new_part_name,
+                source: e.source_replica || "another replica",
+                sourceActive: Number(active[e.source_replica] ?? 0) === 1,
+                ...(lastError ? { lastError } : {}),
+            };
+        }),
+    };
+}

package/dist/clickhouse/rebuild/shards.js

@@ -0,0 +1,76 @@
+/**
+ * A rebuild across the shards of a cluster (#3663).
+ *
+ * On a `cluster:<name>` topology the rebuild's DDL runs `ON CLUSTER`, so
+ * every server gets the new table, the dual-write view and the swap. The
+ * rows do not move that way: each shard holds its own, and a copy run on
+ * the profile's server alone copies that server's shard and no other. So
+ * with more than one shard, the backfill and the verification work shard by
+ * shard, all through the profile's server:
+ *
+ * - a shard's rows are read with `cluster('<name>', db, t)` and
+ *   `_shard_num = <n>`, one replica per shard, as the cluster's own
+ *   configuration reaches it;
+ * - they are written back to the same shard with
+ *   `INSERT INTO FUNCTION cluster('<name>', db, t__chant_new, <slot>)`, a
+ *   constant sharding key that lands on that shard, inserted in the
+ *   foreground so the statement returns once the shard has the rows;
+ * - a shard's half-copied rows are deleted with an `ALTER ... ON CLUSTER`
+ *   whose condition holds on that shard's servers only
+ *   (`getMacro('shard')`, the macro the cluster's Keeper paths already use).
+ *
+ * Each (shard, partition) is one copy with its own receipt, so a rerun after
+ * a failure on one shard skips what every shard already copied.
+ *
+ * With one shard nothing changes: the replicas of a shard replicate what is
+ * written on any of them, which is what the rebuild has always relied on.
+ */
+import { clickhouseQuery } from "../http.js";
+import { ident, sqlString } from "../apply/statements.js";
+import { RebuildRefusal } from "./observe.js";
+/** Settings for a copy into one shard: in the foreground, and read from the cluster rather than pushed down to each shard. */
+export const SHARD_INSERT_SETTINGS = { distributed_foreground_insert: "1", parallel_distributed_insert_select: "0" };
+/** Settings for a delete on one shard: its condition is the server's own macro, the same on every replica of the shard. */
+export const SHARD_DELETE_SETTINGS = { allow_nondeterministic_mutations: "1", mutations_sync: "2" };
+/**
+ * The shards of the target's cluster, when its topology is `cluster:<name>`
+ * and the cluster has more than one. Undefined otherwise: the rebuild works
+ * on the profile's server as it always has.
+ */
+export async function clusterSharding(target) {
+    if (target.topology?.kind !== "cluster")
+        return undefined;
+    let cluster = target.topology.cluster;
+    const macro = /^\{(\w+)\}$/.exec(cluster);
+    if (macro) {
+        const [row] = await clickhouseQuery(target.endpoint, `SELECT getMacro(${sqlString(macro[1])}) AS c`);
+        cluster = row?.c ?? cluster;
+    }
+    const weights = await clickhouseQuery(target.endpoint, `SELECT shard_num AS shard, any(shard_weight) AS weight FROM system.clusters WHERE cluster = ${sqlString(cluster)} GROUP BY shard_num ORDER BY shard_num`);
+    if (weights.length <= 1)
+        return undefined;
+    const macros = new Map((await clickhouseQuery(target.endpoint, `SELECT _shard_num AS shard, getMacro('shard') AS macro FROM cluster(${sqlString(cluster)}, system.one)`)).map((r) => [Number(r.shard), r.macro]));
+    let slot = 0;
+    const shards = [];
+    for (const w of weights) {
+        const num = Number(w.shard);
+        const m = macros.get(num);
+        if (m === undefined)
+            throw new RebuildRefusal(`cluster ${cluster}: shard ${num} has no {shard} macro, which a rebuild across shards needs to delete a half-copied partition on that shard alone`);
+        shards.push({ num, macro: m, slot });
+        slot += Number(w.weight) || 1;
+    }
+    const distinct = new Set(shards.map((s) => s.macro));
+    if (distinct.size !== shards.length) {
+        throw new RebuildRefusal(`cluster ${cluster}: two shards share a {shard} macro (${shards.map((s) => `${s.num}=${s.macro}`).join(", ")}); each shard needs its own`);
+    }
+    return { cluster, shards };
+}
+/** One shard's rows of `database.table`, as a table expression; filter with {@link onShard}. */
+export const shardTable = (sharding, database, table) => `cluster(${sqlString(sharding.cluster)}, ${ident(database)}, ${ident(table)})`;
+/** The condition that keeps one shard's rows of a {@link shardTable}. */
+export const onShard = (shard) => `_shard_num = ${shard.num}`;
+/** Where a copy into one shard of `database.table` writes. */
+export const intoShard = (sharding, shard, database, table) => `FUNCTION cluster(${sqlString(sharding.cluster)}, ${ident(database)}, ${ident(table)}, ${shard.slot})`;
+/** The condition, in an `ALTER ... ON CLUSTER` mutation, that holds on that shard's servers only. */
+export const onShardServers = (shard) => `getMacro('shard') = ${sqlString(shard.macro)}`;

package/dist/clickhouse/rebuild/steps.js

@@ -0,0 +1,383 @@
+/**
+ * The rebuild migration's steps (#3198), each one convergent: it observes
+ * the server (`./observe.ts`), does what is left of its own part, and
+ * reports. Running a step twice does what running it once did.
+ *
+ * The backfill and the verification are in `./backfill.ts` and
+ * `./verify.ts`; the Op that orders them is `./op.ts`.
+ */
+import { computePlanDigest, parseDuration } from "@intentius/chant/op";
+import { clickhouseQuery } from "../http.js";
+import { carriesMarker, readTrailerPairs, REBUILD_TRAILER_KEY, stampedComment } from "../ownership.js";
+import { createStatement, ident, qualifiedIdent, renamedDeclaration, sqlString } from "../apply/statements.js";
+import { waitForMutations } from "../apply/mutations.js";
+import { renderFor } from "../topology.js";
+import { DDL_SETTINGS, syncReplicaWithin } from "./replicas.js";
+import { assertSameRows } from "./verify.js";
+import { changesBetween, observeRebuild, rebuildNames, RebuildRefusal, showCreate, } from "./observe.js";
+/** The server's clock, in milliseconds: the cut-over and the retention are the server's times, not this machine's. */
+export async function serverNow(target) {
+    const [row] = await clickhouseQuery(target.endpoint, "SELECT toUnixTimestamp64Milli(now64(3)) AS t");
+    return Number(row?.t ?? Date.now());
+}
+/** `2026-10-02 12:00:00.000`, UTC: how a cut-over is written into SQL. */
+export const sqlUtc = (ms) => new Date(ms).toISOString().replace("T", " ").replace("Z", "");
+/** A `DateTime64` literal in UTC. */
+export const utcLiteral = (ms) => `toDateTime64(${sqlString(sqlUtc(ms))}, 3, 'UTC')`;
+export const observe = (run) => observeRebuild(run.target, run.declared, run.marker);
+/**
+ * One DDL statement, rendered for the environment's topology when it names
+ * one (`../topology.ts`: `ON CLUSTER` and the engine). In a Replicated
+ * database it goes on without the replicas that are down, which run it from
+ * the database's log when they come back (`./replicas.ts`).
+ */
+const q = (run, statement) => {
+    run.signal?.throwIfAborted();
+    const sql = renderFor(statement, run.target.topology);
+    run.log(sql);
+    return clickhouseQuery(run.target.endpoint, sql, { settings: DDL_SETTINGS, ...(run.signal ? { signal: run.signal } : {}) });
+};
+export async function waitOn(run, database, table) {
+    const ids = await waitForMutations(run.target.endpoint, database, table, {
+        ...(run.sharding ? { source: `clusterAllReplicas(${sqlString(run.sharding.cluster)}, system.mutations)` } : {}),
+        ...(run.mutationTimeoutMs !== undefined ? { timeoutMs: run.mutationTimeoutMs } : {}),
+        ...(run.signal ? { signal: run.signal } : {}),
+    });
+    if (ids.length > 0)
+        run.log(`-- waited for ${database}.${table} mutation(s) ${ids.join(", ")}`);
+}
+/**
+ * In a Replicated database, wait until this replica has every part the
+ * others have written to `database.table` (`SYSTEM SYNC REPLICA ...
+ * LIGHTWEIGHT`, which waits for fetches and not for merges). A step that
+ * reads rows on one replica after a copy ran on another reads them all.
+ * A replica that is down and holds parts this one lacks stops the step once
+ * `replicaTimeoutMs` has passed, naming it (`./replicas.ts`). Nothing to
+ * wait for in an Atomic database.
+ */
+export async function syncReplica(run, o, database, table) {
+    if (!o.replicated)
+        return;
+    await syncReplicaWithin(run.target.endpoint, database, table, {
+        ...(run.replicaTimeoutMs !== undefined ? { timeoutMs: run.replicaTimeoutMs } : {}),
+        log: run.log,
+        ...(run.signal ? { signal: run.signal } : {}),
+    });
+}
+/** The trailer pairs a working object carries besides the marker. */
+const working = (key, role, extra = {}) => ({ [REBUILD_TRAILER_KEY]: key, role, ...extra });
+/**
+ * Create the new table from the declaration, under `t__chant_new`, marked as
+ * this project's and this rebuild's. The old table's unfinished mutations are
+ * waited on first, so what is copied is what they produce. A new table that
+ * is already there is kept when it matches the declaration; one made from an
+ * earlier declaration is refused, and the run's onFailure drops it.
+ */
+export async function createNewTable(run) {
+    const o = await observe(run);
+    const table = `${o.names.database}.${o.names.newName}`;
+    if (o.state !== "rebuild")
+        return { state: o.state, table, created: false };
+    await waitOn(run, o.names.database, o.names.name);
+    if (o.newTable) {
+        const live = await showCreate(run.target, o.names.newTable);
+        const stale = (await changesBetween(run.target, o.names.key, { ...live, name: run.declared.canonical.name }, run.declared.canonical)).filter((c) => c.field !== "comment");
+        if (stale.length > 0) {
+            throw new RebuildRefusal(`${table} was made from another declaration of ${o.names.key} (${stale.map((c) => `${c.rule} on ${c.field}`).join(", ")}). ` +
+                `The run fails so onFailure drops it, and the next run starts again from the current declaration.`);
+        }
+        run.log(`-- ${table} is there and matches the declaration`);
+        return { state: o.state, table, created: false };
+    }
+    const renamed = renamedDeclaration(run.declared, o.names.newName, run.target.defaultDatabase);
+    await q(run, createStatement(renamed, run.marker, { trailer: working(o.names.key, "new") }));
+    return { state: o.state, table, created: true };
+}
+/**
+ * Materialized-view mode: create `t__chant_dual`, a materialized view on the
+ * old table that writes every row at or after the cut-over into the new
+ * table. The cut-over is the server's time now plus the delay, kept in the
+ * view's own comment (`cutover=`), so a later run and the backfill read it
+ * back from the server. The rows at or after the cut-over that the old table
+ * already held are the backfill's (`./backfill.ts`). App mode creates nothing; its gate is the Op's.
+ */
+export async function startDualWrite(run) {
+    const o = await observe(run);
+    const mode = run.dualWrite.mode;
+    if (o.state !== "rebuild" || run.dualWrite.mode === "app")
+        return { state: o.state, mode, created: false };
+    if (o.dual)
+        return { state: o.state, mode, cutover: o.dual.pairs.get("cutover"), created: false };
+    if (!o.newTable)
+        throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
+    const column = run.dualWrite.cutoverColumn;
+    const cut = o.copied.find((c) => c.name === column);
+    if (!cut) {
+        throw new RebuildRefusal(`${o.names.key}: the cut-over column ${column} is not a column the new table copies from the old one ` +
+            `(copied: ${o.copied.map((c) => c.name).join(", ")})`);
+    }
+    // On a whole second: a DateTime column holds whole seconds, and a row written
+    // during the cut-over's second would otherwise carry a time before it and
+    // reach the old table after the backfill has started.
+    const cutover = Math.ceil(((await serverNow(run.target)) + parseDuration(run.dualWrite.cutoverDelay ?? CUTOVER_DELAY)) / 1000) * 1000;
+    const iso = new Date(cutover).toISOString();
+    const select = o.copied.map((c) => (c.source === c.name ? ident(c.name) : `${ident(c.source)} AS ${ident(c.name)}`)).join(", ");
+    const comment = stampedComment(`chant rebuild of ${o.names.key}: rows at or after the cut-over`, run.marker, working(o.names.key, "dual", { cutover: iso }));
+    await q(run, `CREATE MATERIALIZED VIEW ${o.names.dualView} TO ${o.names.newTable} AS SELECT ${select} FROM ${o.names.table} ` +
+        `WHERE ${ident(cut.source)} >= ${utcLiteral(cutover)} COMMENT ${sqlString(comment)}`);
+    return { state: o.state, mode, cutover: iso, created: true };
+}
+/** How far after the dual-write view is made the cut-over falls, unless `cutoverDelay` says otherwise. */
+export const CUTOVER_DELAY = "5s";
+/** The cut-over in milliseconds, from the dual-write view; undefined in app mode. */
+export function cutoverOf(run, o) {
+    if (run.dualWrite.mode === "app")
+        return undefined;
+    const iso = o.dual?.pairs?.get("cutover");
+    if (!iso)
+        throw new RebuildRefusal(`${o.names.key}: the dual-write view ${o.names.database}.${o.names.dualName} is not there; the Dual write phase makes it`);
+    return Date.parse(iso);
+}
+/** How long the backfill waits, once the cut-over has passed, for the writes begun before it. Default: ten minutes. */
+export const CUTOVER_TIMEOUT_MS = 10 * 60000;
+/**
+ * Wait until no row before the cut-over is still to arrive in the old table,
+ * then return. Two conditions, both polled: the server's clock has passed
+ * the cut-over (until then a row can still be written with a time before
+ * it), and every write into the old table that began before the cut-over has
+ * finished (a long INSERT, or an asynchronous insert still in its buffer,
+ * can carry such rows after the clock has passed). The second wait is
+ * bounded by `timeoutMs`; past it the step stops naming the writes, rather
+ * than copy without their rows. A row whose time lags further behind its
+ * write than `cutoverDelay` is neither: the verification finds it.
+ */
+export async function waitForCutover(run, cutover, probe, opts = {}) {
+    const at = new Date(cutover).toISOString();
+    for (;;) {
+        const now = await probe.now();
+        if (now > cutover)
+            break;
+        run.log(`-- waiting ${Math.ceil((cutover - now) / 1000)}s for the cut-over at ${at}`);
+        await sleep(Math.min(cutover - now + 50, 10000), run.signal);
+    }
+    const timeoutMs = opts.timeoutMs ?? CUTOVER_TIMEOUT_MS;
+    const deadline = Date.now() + timeoutMs;
+    let pollMs = opts.pollMs ?? 100;
+    let logged = "";
+    for (;;) {
+        const pending = await probe.pendingWrites(cutover);
+        if (pending.length === 0)
+            return;
+        if (Date.now() >= deadline) {
+            throw new RebuildRefusal(`writes into the old table begun before the cut-over at ${at} are still running after ${Math.round(timeoutMs / 1000)}s: ${pending.join(", ")}. ` +
+                `Let them finish, or stop them, and run again; cutoverTimeout sets how long the backfill waits.`);
+        }
+        const line = pending.join(", ");
+        if (line !== logged)
+            run.log(`-- waiting for writes begun before the cut-over at ${at}: ${line}`);
+        logged = line;
+        await sleep(pollMs, run.signal);
+        pollMs = Math.min(pollMs * 2, 2000);
+    }
+}
+/** The table an `INSERT INTO` statement writes to, or undefined for anything else (`INSERT INTO FUNCTION` included). */
+export function insertTarget(query, currentDatabase) {
+    const text = query.replace(/^(?:\s+|--[^\n]*\n|\/\*[\s\S]*?\*\/)*/, "");
+    const part = "(`(?:[^`\\\\]|\\\\.)+`|\"(?:[^\"\\\\]|\\\\.)+\"|[A-Za-z_][A-Za-z0-9_]*)";
+    const m = new RegExp(`^INSERT\\s+INTO\\s+(?:TABLE\\s+)?${part}(?:\\s*\\.\\s*${part})?`, "i").exec(text);
+    if (!m || /^FUNCTION$/i.test(m[1]))
+        return undefined;
+    const unquote = (s) => (/^[`"]/.test(s) ? s.slice(1, -1).replace(/\\(.)/g, "$1") : s);
+    return m[2] !== undefined ? { database: unquote(m[1]), name: unquote(m[2]) } : { database: currentDatabase, name: unquote(m[1]) };
+}
+/**
+ * The probe against the server. On a cluster of shards it reads every
+ * server's queries and buffers, and in a Replicated database every
+ * replica's, since a write can be running on any of them. A server that
+ * cannot be reached is left out rather than failing the wait.
+ */
+export function serverCutoverProbe(run, o) {
+    const { database, name } = o.names;
+    const cluster = run.sharding?.cluster ?? (o.replicated ? database : undefined);
+    const from = (table) => (cluster ? `clusterAllReplicas(${sqlString(cluster)}, system.${table})` : `system.${table}`);
+    // A server that cannot be reached runs no write the backfill could miss
+    // from here: its parts reach this one by replication, which the backfill
+    // waits for (`syncReplica`), or the verification finds them missing. So an
+    // unreachable replica or shard is skipped, as every other step goes on
+    // with the live ones; `clusterAllReplicas` counts each replica as a shard.
+    const opts = { ...(cluster ? { settings: { skip_unavailable_shards: "1" } } : {}), ...(run.signal ? { signal: run.signal } : {}) };
+    return {
+        now: () => serverNow(run.target),
+        async pendingWrites(cutover) {
+            const running = await clickhouseQuery(run.target.endpoint, `SELECT query_id, query, current_database, elapsed, toUnixTimestamp64Milli(now64(3)) AS now FROM ${from("processes")} WHERE query_kind = 'Insert'`, opts);
+            const queued = await clickhouseQuery(run.target.endpoint, `SELECT entries.query_id AS ids FROM ${from("asynchronous_inserts")} ` +
+                `WHERE database = ${sqlString(database)} AND table = ${sqlString(name)} AND toUnixTimestamp64Milli(first_update) < ${cutover}`, opts);
+            const ids = running
+                .filter((r) => {
+                const t = insertTarget(r.query, r.current_database);
+                return t?.database === database && t.name === name && Number(r.now) - Number(r.elapsed) * 1000 < cutover;
+            })
+                .map((r) => r.query_id);
+            for (const r of queued)
+                ids.push(...r.ids);
+            return [...new Set(ids)];
+        },
+    };
+}
+export function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        signal?.throwIfAborted();
+        const t = setTimeout(resolve, ms);
+        signal?.addEventListener("abort", () => {
+            clearTimeout(t);
+            reject(signal.reason);
+        }, { once: true });
+    });
+}
+/**
+ * Swap the new table in: compare every row of the two tables once more, as
+ * the verification does, and stop on any difference; then `EXCHANGE TABLES`,
+ * drop the dual-write view, rename the old table (now under the new table's name) to `t__chant_old`
+ * with its retention date, recreate the materialized views that read the
+ * table, and stamp the table's own comment as declared.
+ *
+ * Each part checks for itself whether it is done, so a swap interrupted
+ * anywhere finishes on the next run. The table's comment is stamped last:
+ * until it is, the table still carries `role=new` and the next run knows the
+ * swap has not finished.
+ *
+ * Recreating a dependent view: in this server line a materialized view
+ * follows its source by name, so after the EXCHANGE it already reads the new
+ * table. It is detached and attached again so the server analyzes its query
+ * against the new table's columns and types, and a view the new definition
+ * breaks fails here, with the old table still kept, rather than on the next
+ * insert. A view with an inner table keeps its data, which dropping and
+ * creating it would not. Inserts that land while a view is detached are not
+ * passed to it; in materialized-view mode writes go on through the swap, so
+ * that window is the length of one DETACH and one ATTACH. In a Replicated
+ * database the DETACH is `PERMANENTLY`, the form that database takes, and
+ * the EXCHANGE, the DETACH and the ATTACH each run on every replica: now on
+ * the live ones, and from the database's log on one that is down, when it
+ * comes back.
+ */
+export async function swapTables(run) {
+    const o = await observe(run);
+    const n = o.names;
+    if (o.state === "done")
+        return { state: o.state, swapped: false, dependents: [] };
+    if (o.state === "swapped" && !o.exchanged) {
+        return { state: o.state, swapped: false, dependents: [], oldTable: `${n.database}.${n.oldName}`, ...retention(o) };
+    }
+    if (!o.exchanged) {
+        if (!o.newTable)
+            throw new RebuildRefusal(`${n.key}: there is no new table to swap in`);
+        // The new table holds every row the old one does, or nothing is swapped
+        // (sql-yodeler#45): rows that reached the old table alone since the
+        // verification would stay behind in it.
+        await assertSameRows(run, o);
+        await q(run, `EXCHANGE TABLES ${n.table} AND ${n.newTable}`);
+    }
+    // The dual-write view now reads the new table by name; drop it before
+    // anything else so it writes nothing more into the old one.
+    const dual = await objectComment(run, n.database, n.dualName);
+    if (dual !== undefined && readTrailerPairs(dual)?.get(REBUILD_TRAILER_KEY) === n.key && carriesMarker(dual, run.marker)) {
+        await q(run, `DROP VIEW ${n.dualView} SYNC`);
+    }
+    if ((await objectComment(run, n.database, n.oldName)) === undefined) {
+        await q(run, `RENAME TABLE ${n.newTable} TO ${n.oldTable}`);
+    }
+    const oldComment = await objectComment(run, n.database, n.oldName);
+    let retainUntil = readTrailerPairs(oldComment)?.get("retain-until");
+    if (readTrailerPairs(oldComment)?.get("role") !== "old") {
+        retainUntil = new Date((await serverNow(run.target)) + run.retainMs).toISOString();
+        const comment = stampedComment(`chant rebuild of ${n.key}: the old table, kept until ${retainUntil}`, run.marker, working(n.key, "old", { "retain-until": retainUntil }));
+        await q(run, `ALTER TABLE ${n.oldTable} MODIFY COMMENT ${sqlString(comment)}`);
+    }
+    const dependents = await dependentViews(run, n.database, n.name);
+    for (const d of dependents) {
+        const [database, name] = d;
+        // A Replicated database takes DETACH only as PERMANENTLY, run on every replica.
+        const permanently = database === n.database ? o.replicated : await isReplicatedDatabase(run, database);
+        await q(run, `DETACH TABLE ${qualifiedIdent(database, name)}${permanently ? " PERMANENTLY" : ""}`);
+        await q(run, `ATTACH TABLE ${qualifiedIdent(database, name)}`);
+    }
+    await q(run, `ALTER TABLE ${n.table} MODIFY COMMENT ${sqlString(stampedComment(run.declared.canonical.comment, run.marker))}`);
+    return { state: "swapped", swapped: true, dependents: dependents.map(([d, v]) => `${d}.${v}`), oldTable: `${n.database}.${n.oldName}`, ...(retainUntil ? { retainUntil } : {}) };
+}
+async function isReplicatedDatabase(run, database) {
+    const [row] = await clickhouseQuery(run.target.endpoint, `SELECT engine FROM system.databases WHERE name = ${sqlString(database)}`);
+    return row?.engine === "Replicated";
+}
+async function objectComment(run, database, name) {
+    const [row] = await clickhouseQuery(run.target.endpoint, `SELECT comment FROM system.tables WHERE database = ${sqlString(database)} AND name = ${sqlString(name)}`);
+    return row?.comment;
+}
+/** The materialized views that read `database.name`, as `system.tables` lists them. */
+export async function dependentViews(run, database, name) {
+    const [row] = await clickhouseQuery(run.target.endpoint, `SELECT dependencies_database, dependencies_table FROM system.tables WHERE database = ${sqlString(database)} AND name = ${sqlString(name)}`);
+    if (!row)
+        return [];
+    return row.dependencies_table.map((t, i) => [row.dependencies_database[i], t]);
+}
+// ── retain and drop ────────────────────────────────────────────────────
+function retention(o) {
+    const until = o.old?.pairs?.get("retain-until");
+    return until ? { retainUntil: until } : {};
+}
+/** Where the old table is kept and until when, and the digest the drop gate binds. */
+export async function retainPlan(run) {
+    const o = await observe(run);
+    if (!o.old)
+        return { state: o.state, due: false };
+    const retainUntil = o.old.pairs?.get("retain-until");
+    const due = retainUntil !== undefined && (await serverNow(run.target)) >= Date.parse(retainUntil);
+    const oldTable = `${o.names.database}.${o.names.oldName}`;
+    return {
+        state: o.state,
+        oldTable,
+        ...(retainUntil ? { retainUntil } : {}),
+        due,
+        dropDigest: computePlanDigest("clickhouse-rebuild-drop", { table: o.names.key, old: oldTable, uuid: o.old.uuid, retainUntil: retainUntil ?? null }),
+    };
+}
+/** Drop the old table once its retention date has passed. Before then, say until when and drop nothing. */
+export async function dropOldTable(run) {
+    const plan = await retainPlan(run);
+    if (!plan.oldTable)
+        return { state: plan.state, dropped: false };
+    if (!plan.due) {
+        run.log(`-- ${plan.oldTable} is kept until ${plan.retainUntil}; a run after that drops it`);
+        return { state: plan.state, dropped: false, oldTable: plan.oldTable, ...(plan.retainUntil ? { retainUntil: plan.retainUntil } : {}) };
+    }
+    const n = rebuildNames(run.declared.canonical.database ?? run.target.defaultDatabase, run.declared.canonical.name);
+    await q(run, `DROP TABLE ${n.oldTable} SYNC`);
+    return { state: "done", dropped: true, oldTable: plan.oldTable };
+}
+/**
+ * onFailure: drop the dual-write view and the new table, and nothing else.
+ * Each is dropped only when its comment names this rebuild and its role and
+ * carries this project's marker. After the EXCHANGE the new table is under
+ * the table's own name and the old one under the new table's name without
+ * `role=new`, so neither is touched: the swap is finished by the next run,
+ * not undone. The old table kept after a swap (`role=old`) is never dropped
+ * here; the drop gate decides that.
+ */
+export async function compensate(run) {
+    const n = rebuildNames(run.declared.canonical.database ?? run.target.defaultDatabase, run.declared.canonical.name);
+    const rows = await clickhouseQuery(run.target.endpoint, `SELECT name, comment FROM system.tables WHERE database = ${sqlString(n.database)} AND name IN (${sqlString(n.dualName)}, ${sqlString(n.newName)})`);
+    const dropped = [];
+    for (const [name, role, what] of [
+        [n.dualName, "dual", "VIEW"],
+        [n.newName, "new", "TABLE"],
+    ]) {
+        const row = rows.find((r) => r.name === name);
+        const pairs = readTrailerPairs(row?.comment);
+        if (!row || pairs?.get(REBUILD_TRAILER_KEY) !== n.key || pairs.get("role") !== role || !carriesMarker(row.comment, run.marker))
+            continue;
+        const sql = renderFor(`DROP ${what} ${qualifiedIdent(n.database, name)} SYNC`, run.target.topology);
+        run.log(sql);
+        await clickhouseQuery(run.target.endpoint, sql, { settings: DDL_SETTINGS });
+        dropped.push(`${n.database}.${name}`);
+    }
+    return { dropped };
+}