@intentius/chant-lexicon-sql 0.124.31 → 0.124.33

package/dist/clickhouse/rebuild/verify.js

@@ -0,0 +1,279 @@
+/**
+ * The verification (#3198, INTENTIUS/sql-yodeler#45): row counts and
+ * checksums per partition, the old table against the new one, before anyone
+ * is asked to approve the swap.
+ *
+ * Per partition of the old table: the number of rows and the sum of a
+ * 64-bit hash of each row's copied columns, read in one grouped query on
+ * each side. The old side casts each column to its type in the new table
+ * first, so a column whose type the rebuild changes hashes the same when
+ * its values survived the change.
+ *
+ * Every row is compared, in both modes. In materialized-view mode the rows
+ * at or after the cut-over are in the new table twice over: those the old
+ * table held before the view existed, which the backfill copies, and those
+ * written since, which the view writes. Comparing only the rows before the
+ * cut-over let a swap go ahead without the first kind (sql-yodeler#45).
+ * Writes go on while the tables are read, and a row the old table has
+ * committed may be a moment from committing in the new one through the view,
+ * so the old table is read first and a difference is read again a few times
+ * before it counts.
+ *
+ * The same comparison runs again in the Swap step just before the
+ * `EXCHANGE`, so rows that reach the old table alone after this step (a row
+ * later than the cut-over delay, with a time before the cut-over) stop the
+ * swap too, rather than stay behind in the old table.
+ *
+ * A table whose engine collapses rows that share a sorting key when parts
+ * merge (Summing, Replacing, Aggregating, Collapsing, VersionedCollapsing,
+ * Coalescing and Graphite, replicated or not), rebuilt into such an engine,
+ * is read under `FINAL` on both sides (#3674). The old table can hold unmerged parts with several rows for
+ * one key, which the copy inserts in one block and the new table collapses
+ * as it writes them, so the raw counts differ although the data is the same.
+ * Under `FINAL` both sides are compared as merged.
+ *
+ * On a cluster of more than one shard (#3663) every shard is read, through
+ * the profile's server, and each partition is compared per shard
+ * (`shard<n>/<partition>`), so a shard the copy missed is a difference.
+ *
+ * Any difference fails the run, naming the partitions; the run's onFailure
+ * then drops the new table. On a match the result carries the digest the
+ * swap gate binds: the rebuild's plan and the counts and checksums of the
+ * rows before the cut-over, which no later write changes, so an approval
+ * is for this plan with these counts, and a changed count needs a new
+ * approval. The rows at or after the cut-over keep arriving through the view
+ * and are compared, not bound.
+ */
+import { computePlanDigest } from "@intentius/chant/op";
+import { clickhouseQuery } from "../http.js";
+import { ident, sqlString } from "../apply/statements.js";
+import { rebuildPlanSubject, RebuildRefusal } from "./observe.js";
+import { sourcePartitionExpression } from "./partitions.js";
+import { cutoverOf, observe, syncReplica, utcLiteral, waitOn } from "./steps.js";
+import { shardTable } from "./shards.js";
+import { collapseSemantics, collapsesRows, splitTopLevel } from "./engines.js";
+/** A verification that found the tables different. */
+export class RebuildVerificationError extends Error {
+    constructor(table, mismatches, 
+    /** How the tables were compared when an engine collapses rows; an engine's name alone means under FINAL. */
+    collapsed) {
+        const how = typeof collapsed === "string" ? { engine: collapsed, how: "final" } : collapsed;
+        const shown = mismatches
+            .slice(0, 10)
+            .map((m) => `${m.partition}: old ${m.old ? `${m.old.rows} ${unit(how)}, checksum ${m.old.checksum}` : "none"}, new ${m.new ? `${m.new.rows} ${unit(how)}, checksum ${m.new.checksum}` : "none"}`)
+            .join("; ");
+        const lateWrite = `in materialized-view mode a row written to the old table later than the cut-over delay, with a time before the cut-over, ` +
+            `and in app mode a write the application did not stop`;
+        super(`${table}: the new table does not match the old one in ${mismatches.length} partition(s): ${shown}${mismatches.length > 10 ? "; ..." : ""}. Nothing was swapped. ` +
+            (how?.how === "final"
+                ? `${table} is a ${how.engine}, which collapses rows that share a sorting key when parts merge, and both tables were compared under FINAL. ` +
+                    `They still differ when the new table collapses the copied rows by its sorting key differently from how the old one merges them: ` +
+                    `run OPTIMIZE TABLE ${table} FINAL, drop the new table if the Op kept it, and run the rebuild again, so the copy reads merged rows. ` +
+                    `The other causes are ${lateWrite}.`
+                : how?.how === "keys"
+                    ? `The new table is a ${how.engine}, which collapses rows that share a sorting key, so both tables were grouped by its sorting key and partition ` +
+                        `and compared per key on ${how.describe ?? "the keys"}, which its merges keep. A key missing from the new table, or one whose values differ, is rows the copy lost or changed. ` +
+                        `The usual cause is ${lateWrite}.`
+                    : `The usual cause is ${lateWrite}.`));
+        this.table = table;
+        this.mismatches = mismatches;
+        this.name = "RebuildVerificationError";
+        if (how)
+            this.collapsingEngine = how.engine;
+    }
+}
+const unit = (how) => (how?.how === "keys" ? "keys" : "rows");
+/** How often a difference is read again before it counts, and how long apart. */
+const COMPARE_ATTEMPTS = 5;
+const COMPARE_INTERVAL_MS = 1000;
+/**
+ * Both tables grouped by the new table's sorting key and its partition, each
+ * key compared on what the new engine's merges keep (#3727): its sums, its
+ * highest version, the sum of its signs. Each side is read raw, so the
+ * comparison is the same whichever parts have merged. The old side's columns
+ * are cast to the new table's types and named as there, so the sorting key,
+ * the partition key and the aggregates read the same on both sides. A
+ * partition is the new table's (`shard<n>/` before it on a cluster), and its
+ * rows are its keys.
+ */
+async function keyComparison(run, o, columns) {
+    const n = o.names;
+    const [t] = await clickhouseQuery(run.target.endpoint, `SELECT engine_full, sorting_key, partition_key FROM system.tables WHERE database = ${sqlString(n.database)} AND name = ${sqlString(n.newName)}`);
+    if (!t || !o.newTable)
+        return undefined;
+    const semantics = collapseSemantics(o.newTable.engine, t.engine_full, columns);
+    if (!semantics)
+        return undefined;
+    // Every column of the sorting key must come from the old table, or the old rows have no key to group by.
+    const copied = new Set(o.copied.map((c) => c.name));
+    if (columns.some((c) => c.inSortingKey && !copied.has(c.name)))
+        return undefined;
+    const types = new Map(columns.map((c) => [c.name, c.type]));
+    const sharding = run.sharding;
+    const cutover = cutoverOf(run, o);
+    const cut = run.dualWrite.mode === "materialized-view" ? run.dualWrite.cutoverColumn : undefined;
+    const before = cut && copied.has(cut) && cutover !== undefined ? `${ident(cut)} < ${utcLiteral(cutover)}` : "1";
+    const key = splitTopLevel(t.sorting_key).filter(Boolean);
+    const parts = splitTopLevel(t.partition_key).filter(Boolean);
+    const partition = parts.length === 0 ? "'all'" : `toString(${parts.length === 1 ? parts[0] : `tuple(${parts.join(", ")})`})`;
+    const label = sharding ? `concat('shard', toString(__shard), '/', ${partition})` : partition;
+    const hash = `cityHash64(toString(tuple(${[...key, ...semantics.aggregates].join(", ")})))`;
+    const read = async (side) => {
+        const name = side === "old" ? n.name : n.newName;
+        const from = sharding ? shardTable(sharding, n.database, name) : side === "old" ? n.table : n.newTable;
+        const projected = o.copied.map((c) => (side === "old" ? `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")}) AS ${ident(c.name)}` : ident(c.name)));
+        const rowsOf = `SELECT ${sharding ? "_shard_num AS __shard, " : ""}${projected.join(", ")} FROM ${from}`;
+        const perKey = `SELECT ${label} AS __p, ${hash} AS __h, max(${before}) AS __b, count() AS __n, ${semantics.having ?? "1"} AS __k FROM (${rowsOf}) ` +
+            `GROUP BY __p${sharding ? ", __shard" : ""}${key.length > 0 ? `, ${key.join(", ")}` : ""}`;
+        // A key the engine removes (its sums all zero, its signs cancelled) is not compared, but its rows are counted as read.
+        const rows = await clickhouseQuery(run.target.endpoint, `SELECT __p AS p, countIf(__k) AS rows, toString(sumIf(__h, __k)) AS checksum, countIf(__b AND __k) AS brows, toString(sumIf(__h, __b AND __k)) AS bchecksum, sum(__n) AS raw ` +
+            `FROM (${perKey}) GROUP BY p`);
+        const out = { all: new Map(), before: new Map(), rows: 0 };
+        for (const r of rows) {
+            out.rows += Number(r.raw);
+            // A partition whose keys the engine all removes holds nothing in the new table, as if it had no rows.
+            if (Number(r.rows) === 0)
+                continue;
+            out.all.set(r.p, { partition: r.p, rows: Number(r.rows), checksum: r.checksum });
+            if (Number(r.brows) > 0)
+                out.before.set(r.p, { partition: r.p, rows: Number(r.brows), checksum: r.bchecksum });
+        }
+        return out;
+    };
+    return { semantics, read };
+}
+/**
+ * Read both tables' counts and checksums per old partition, the old table
+ * first, until they agree or `attempts` reads have disagreed. Returns the
+ * last read and the partitions that differ in it.
+ */
+async function compareTables(run, o, opts = {}) {
+    if (!o.newTable)
+        throw new RebuildRefusal(`${o.names.key}: the new table is not there yet; the Create phase makes it`);
+    const n = o.names;
+    const cutover = cutoverOf(run, o);
+    await waitOn(run, n.database, n.name);
+    await waitOn(run, n.database, n.newName);
+    const newColumns = (await clickhouseQuery(run.target.endpoint, `SELECT name, type, is_in_sorting_key AS key, is_in_partition_key AS partition FROM system.columns ` +
+        `WHERE database = ${sqlString(n.database)} AND table = ${sqlString(n.newName)} ORDER BY position`)).map((c) => ({ name: c.name, type: c.type, inSortingKey: Number(c.key) === 1, inPartitionKey: Number(c.partition) === 1 }));
+    const types = new Map(newColumns.map((c) => [c.name, c.type]));
+    const oldHash = rowHash(o.copied.map((c) => `CAST(${ident(c.source)}, ${sqlString(types.get(c.name) ?? "String")})`));
+    const newHash = rowHash(o.copied.map((c) => ident(c.name)));
+    const cut = run.dualWrite.mode === "materialized-view" ? o.copied.find((c) => c.name === run.dualWrite.cutoverColumn) : undefined;
+    const oldBefore = cut && cutover !== undefined ? `${ident(cut.source)} < ${utcLiteral(cutover)}` : "1";
+    const newBefore = cut && cutover !== undefined ? `${ident(cut.name)} < ${utcLiteral(cutover)}` : "1";
+    const fromOld = sourcePartitionExpression(o.live.partitionKey, o.newTable.partitionKey, o.copied);
+    // Across the shards of a cluster (#3663), every shard's rows, each partition counted per shard as `shard<n>/<partition>`.
+    const sharding = run.sharding;
+    // Tables whose engines collapse rows by sorting key are read as merged (#3674). Only when both do: a collapsing
+    // table rebuilt into a plain MergeTree keeps every row it copies, and is compared row for row.
+    const final = collapsesRows(o.live.engine) && collapsesRows(o.newTable.engine);
+    // A table that does not collapse rows rebuilt into one that does (#3727): the new table keeps one row per sorting
+    // key where the old one keeps them all, by design. Both are grouped by the new sorting key and compared on what
+    // the new engine's merges keep per key.
+    const keys = !final && collapsesRows(o.newTable.engine) ? await keyComparison(run, o, newColumns) : undefined;
+    const read = async (table, name, partition, hash, before, final) => {
+        const from = `${sharding ? shardTable(sharding, n.database, name) : table}${final ? " FINAL" : ""}`;
+        const key = sharding ? `concat('shard', toString(_shard_num), '/', ${partition})` : partition;
+        const rows = await clickhouseQuery(run.target.endpoint, `SELECT ${key} AS p, count() AS rows, toString(sum(${hash})) AS checksum, countIf(${before}) AS brows, toString(sumIf(${hash}, ${before})) AS bchecksum ` +
+            `FROM ${from} GROUP BY p`);
+        const side = { all: new Map(), before: new Map() };
+        for (const r of rows) {
+            side.all.set(r.p, { partition: r.p, rows: Number(r.rows), checksum: r.checksum });
+            // A partition with no row before the cut-over is left out, as a WHERE would leave it out.
+            if (Number(r.brows) > 0)
+                side.before.set(r.p, { partition: r.p, rows: Number(r.brows), checksum: r.bchecksum });
+        }
+        return side;
+    };
+    const attempts = Math.max(1, opts.attempts ?? COMPARE_ATTEMPTS);
+    for (let attempt = 1;; attempt++) {
+        // In a Replicated database: count what every replica wrote, not what this one has fetched so far.
+        await syncReplica(run, o, n.database, n.name);
+        await syncReplica(run, o, n.database, n.newName);
+        // The old table first: a row the view has yet to commit in the new one is read as missing there, never the other way round.
+        const before = keys ? await keys.read("old") : await read(n.table, n.name, "_partition_id", oldHash, oldBefore, final);
+        const after = keys ? await keys.read("new") : await read(n.newTable, n.newName, fromOld, newHash, newBefore, final);
+        const mismatches = differences(before.all, after.all);
+        if (mismatches.length === 0 || attempt >= attempts) {
+            return {
+                old: before,
+                new: after,
+                mismatches,
+                ...(final ? { collapsed: { engine: o.live.engine, how: "final" } } : {}),
+                ...(keys ? { collapsed: { engine: o.newTable.engine, how: "keys", describe: keys.semantics.describe } } : {}),
+            };
+        }
+        run.log(`-- ${n.key}: ${mismatches.length} partition(s) differ (${mismatches.map((m) => m.partition).slice(0, 5).join(", ")}); reading again (${attempt}/${attempts})`);
+        await sleep(opts.intervalMs ?? COMPARE_INTERVAL_MS, run.signal);
+    }
+}
+function differences(before, after) {
+    const out = [];
+    for (const p of [...new Set([...before.keys(), ...after.keys()])].sort()) {
+        const a = before.get(p);
+        const b = after.get(p);
+        if (!a || !b || a.rows !== b.rows || a.checksum !== b.checksum)
+            out.push({ partition: p, ...(a ? { old: a } : {}), ...(b ? { new: b } : {}) });
+    }
+    return out;
+}
+/** The 64-bit hash of a row's copied columns, as the backfill and the verification compute it. */
+export const rowHash = (columns) => `cityHash64(toString(tuple(${columns.join(", ")})))`;
+/**
+ * Before the swap: the new table holds every row the old one does. Throws a
+ * {@link RebuildVerificationError} when it does not, so nothing is swapped.
+ */
+export async function assertSameRows(run, o) {
+    const { mismatches, collapsed } = await compareTables(run, o);
+    if (mismatches.length > 0)
+        throw new RebuildVerificationError(o.names.key, mismatches, collapsed);
+}
+export async function verifyRebuild(run) {
+    const o = await observe(run);
+    if (o.state !== "rebuild")
+        return { state: o.state, partitions: 0, rows: 0, verification: [], summary: `nothing to verify: ${o.state}` };
+    const cutover = cutoverOf(run, o);
+    const compared = await compareTables(run, o);
+    if (compared.mismatches.length > 0)
+        throw new RebuildVerificationError(o.names.key, compared.mismatches, compared.collapsed);
+    const all = [...compared.old.all.values()];
+    const counted = all.reduce((s, v) => s + v.rows, 0);
+    const c = compared.collapsed;
+    // Grouped by sorting key (#3727), each partition counts keys; the old table's rows are still every row read.
+    const rows = c?.how === "keys" ? (compared.old.rows ?? counted) : counted;
+    // What the approval binds: the rows before the cut-over (every row in app mode), which later writes do not change.
+    const verification = [...compared.old.before.values()].sort((x, y) => x.partition.localeCompare(y.partition));
+    const bound = verification.reduce((s, v) => s + v.rows, 0);
+    const range = cutover !== undefined ? ` (${bound} ${c?.how === "keys" ? "key(s)" : "of them"} before the cut-over at ${new Date(cutover).toISOString()})` : "";
+    const counts = c?.how === "keys" ? `${rows} row(s) of the old table as ${counted} key(s)` : `${rows} row(s)`;
+    const merged = c?.how === "final"
+        ? `, read under FINAL (${c.engine})`
+        : c?.how === "keys"
+            ? `, grouped by the sorting key of the new ${c.engine} and compared on ${c.describe}`
+            : "";
+    const summary = `${o.names.key}: ${all.length} partition(s), ${counts}${range}, counts and checksums equal in the old and new tables${merged}`;
+    run.log(`-- ${summary}`);
+    return {
+        state: o.state,
+        planDigest: computePlanDigest("clickhouse-rebuild", {
+            plan: rebuildPlanSubject(o, run.dualWrite),
+            cutover: cutover === undefined ? null : new Date(cutover).toISOString(),
+            verification,
+        }),
+        partitions: all.length,
+        rows,
+        summary,
+        verification,
+    };
+}
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        signal?.throwIfAborted();
+        const t = setTimeout(resolve, ms);
+        signal?.addEventListener("abort", () => {
+            clearTimeout(t);
+            reject(signal.reason);
+        }, { once: true });
+    });
+}

package/dist/clickhouse/row-types.js

@@ -0,0 +1,151 @@
+/**
+ * What a ClickHouse column's value is in TypeScript, for `RowOf` (#3751): the
+ * value a row has in ClickHouse's JSON formats (`JSONEachRow`, what the HTTP
+ * clients read) with the server's default settings. 64-bit and wider
+ * integers are quoted (`output_format_json_quote_64bit_integers`), so they
+ * are strings; narrower integers, floats and decimals are numbers; dates,
+ * times, UUIDs and addresses are their text; a named tuple is an object
+ * (`output_format_json_named_tuples_as_objects`) and an unnamed one an array.
+ * `Nullable(T)` adds `| null` and `LowCardinality(T)` is `T`.
+ *
+ * The table names every type family of the pinned server's catalog
+ * (`TYPE_FAMILIES`, aliases resolved), and a test fails on one it does not,
+ * so a family a new release adds is a decision rather than a silent
+ * `unknown`.
+ */
+import { familyOf } from "../lint/post-synth/clickhouse-type-check.js";
+import { readType, typeTokens } from "./type-syntax.js";
+import { isTrivia } from "./tokens.js";
+const NUMBER = "number";
+const STRING = "string";
+const UNKNOWN = "unknown";
+const POINT = "[number, number]";
+/** A union needs parentheses before `[]`. */
+const element = (ts) => (/[|&]/.test(ts) && !/^\{.*\}$/s.test(ts) ? `(${ts})` : ts);
+/** An enum's labels as a union of string literals; its text when none read. */
+const enumLabels = (params) => {
+    const labels = params.flatMap((p) => {
+        const first = p.tokens.find((t) => !isTrivia(t));
+        return first?.kind === "string" ? [first.text.slice(1, -1).replace(/\\(.)/g, "$1").replace(/''/g, "'")] : [];
+    });
+    return labels.length > 0 ? labels.map((l) => JSON.stringify(l)).join(" | ") : STRING;
+};
+/** The TypeScript type of each type family's value, or how its parameters make it. */
+export const CLICKHOUSE_ROW_TYPES = {
+    UInt8: NUMBER,
+    UInt16: NUMBER,
+    UInt32: NUMBER,
+    Int8: NUMBER,
+    Int16: NUMBER,
+    Int32: NUMBER,
+    Float32: NUMBER,
+    Float64: NUMBER,
+    BFloat16: NUMBER,
+    Decimal: NUMBER,
+    Decimal32: NUMBER,
+    Decimal64: NUMBER,
+    Decimal128: NUMBER,
+    Decimal256: NUMBER,
+    // Quoted in JSON, so text.
+    UInt64: STRING,
+    UInt128: STRING,
+    UInt256: STRING,
+    Int64: STRING,
+    Int128: STRING,
+    Int256: STRING,
+    IntervalNanosecond: STRING,
+    IntervalMicrosecond: STRING,
+    IntervalMillisecond: STRING,
+    IntervalSecond: STRING,
+    IntervalMinute: STRING,
+    IntervalHour: STRING,
+    IntervalDay: STRING,
+    IntervalWeek: STRING,
+    IntervalMonth: STRING,
+    IntervalQuarter: STRING,
+    IntervalYear: STRING,
+    Bool: "boolean",
+    String: STRING,
+    FixedString: STRING,
+    UUID: STRING,
+    IPv4: STRING,
+    IPv6: STRING,
+    Date: STRING,
+    Date32: STRING,
+    DateTime: STRING,
+    DateTime32: STRING,
+    DateTime64: STRING,
+    Time: STRING,
+    Time64: STRING,
+    Enum: enumLabels,
+    Enum8: enumLabels,
+    Enum16: enumLabels,
+    // Wrappers and containers.
+    Nullable: (p, inner) => `${inner(p[0]?.text ?? "")} | null`,
+    LowCardinality: (p, inner) => inner(p[0]?.text ?? ""),
+    Array: (p, inner) => `${element(inner(p[0]?.text ?? ""))}[]`,
+    Map: (p, inner) => `Record<string, ${inner(p[1]?.text ?? "")}>`,
+    Tuple: (p, inner) => {
+        const elements = p.map((x) => named(x));
+        if (elements.length > 0 && elements.every((e) => e.name !== undefined)) {
+            return `{ ${elements.map((e) => `${propertyName(e.name)}: ${inner(e.type)}`).join("; ")} }`;
+        }
+        return `[${elements.map((e) => inner(e.type)).join(", ")}]`;
+    },
+    SimpleAggregateFunction: (p, inner) => inner(p[1]?.text ?? ""),
+    AggregateFunction: UNKNOWN,
+    Nested: UNKNOWN,
+    // Self-describing: the application's own shape.
+    JSON: UNKNOWN,
+    Dynamic: UNKNOWN,
+    Variant: UNKNOWN,
+    QBit: UNKNOWN,
+    Nothing: "null",
+    // Geometry, as the JSON formats write it: points as pairs, nested in arrays.
+    Point: POINT,
+    Ring: `${POINT}[]`,
+    LineString: `${POINT}[]`,
+    Polygon: `${POINT}[][]`,
+    MultiLineString: `${POINT}[][]`,
+    MultiPoint: `${POINT}[]`,
+    MultiPolygon: `${POINT}[][][]`,
+    Geometry: UNKNOWN,
+};
+/** A tuple element: `a String` is named `a`, `String` is not. */
+function named(param) {
+    const syn = readType(param.tokens);
+    if (!syn || syn.words.length < 2)
+        return { type: param.text };
+    if (familyOf(syn.words.map((w) => w.text).join(" ")))
+        return { type: param.text };
+    const first = syn.words[0];
+    const rest = param.text.slice(param.text.indexOf(first.text) + first.text.length).trim();
+    return { name: first.kind === "qident" ? first.text.slice(1, -1) : first.text, type: rest };
+}
+const propertyName = (name) => (/^[A-Za-z_$][\w$]*$/.test(name) ? name : JSON.stringify(name));
+/** A type's row type, wrappers and all; `unknown` for a type this cannot read. */
+export function clickhouseTsType(type) {
+    const tokens = typeTokens(type);
+    const syn = tokens && readType(tokens);
+    if (!syn)
+        return UNKNOWN;
+    const spec = familyOf(syn.words.map((w) => w.text).join(" "));
+    const entry = spec ? CLICKHOUSE_ROW_TYPES[spec.canonical] : undefined;
+    if (entry === undefined)
+        return UNKNOWN;
+    return typeof entry === "string" ? entry : entry(syn.params ?? [], clickhouseTsType);
+}
+/** A column's row type and whether it can be NULL: `Nullable(...)` at the top, under any `LowCardinality(...)`. */
+export function clickhouseRowType(type) {
+    let t = type.trim();
+    let nullable = false;
+    for (;;) {
+        const m = /^(Nullable|LowCardinality)\s*\(([\s\S]*)\)$/i.exec(t);
+        if (!m)
+            break;
+        if (m[1].toLowerCase() === "nullable")
+            nullable = true;
+        t = m[2].trim();
+    }
+    return { ts: clickhouseTsType(t), nullable };
+}

package/dist/clickhouse/testing/fake-server.js

@@ -0,0 +1,47 @@
+/**
+ * A stand-in for a ClickHouse server's HTTP interface, for tests: answers the
+ * catalog queries chant's live reads make from a fixed set of objects.
+ * Not a SQL engine.
+ */
+import { startHttpStub } from "../../core/testing/http-stub.js";
+const row = (o) => `${JSON.stringify(o)}\n`;
+const unquote = (s) => s.replace(/^`|`$/g, "").replace(/``/g, "`");
+export async function fakeClickHouse(objects, options = {}) {
+    const queries = [];
+    const ok = (text) => ({ status: 200, text });
+    const stub = await startHttpStub((body) => {
+        queries.push(body);
+        if (options.fail)
+            return { status: options.fail.status, text: options.fail.body };
+        if (/version\(\)/.test(body))
+            return ok(row({ v: options.version ?? "26.8.15.10" }));
+        // A probe for one undeclarable kind (`../live/undeclarable.ts`), which its comment names.
+        const probe = /^\/\* chant undeclarable: (.+?) \*\//.exec(body);
+        if (probe)
+            return ok((options.undeclarable ?? []).filter((u) => u.kind === probe[1]).map((u) => row({ database: u.database ?? "", name: u.name })).join(""));
+        if (/FROM system\.databases/.test(body)) {
+            return ok(objects
+                .filter((o) => o.database === undefined)
+                .map((o) => row({ name: o.name, engine: o.engine, uuid: "00000000-0000-0000-0000-000000000000", comment: o.comment ?? "" }))
+                .join(""));
+        }
+        if (/FROM system\.tables/.test(body)) {
+            return ok(objects
+                .filter((o) => o.database !== undefined)
+                .map((o) => row({ database: o.database, name: o.name, engine: o.engine, uuid: "00000000-0000-0000-0000-000000000000", comment: o.comment ?? "" }))
+                .join(""));
+        }
+        if (/FROM system\.functions/.test(body))
+            return ok((options.functions ?? []).map((f) => row(f)).join(""));
+        const show = /^SHOW CREATE (DATABASE|TABLE) (`(?:[^`]|``)*`)(?:\.(`(?:[^`]|``)*`))?$/.exec(body.trim());
+        if (show) {
+            const [db, name] = show[3] ? [unquote(show[2]), unquote(show[3])] : [undefined, unquote(show[2])];
+            const o = objects.find((x) => x.database === db && x.name === name);
+            if (o)
+                return ok(row({ statement: o.statement }));
+            return { status: 404, text: `Code: 60. DB::Exception: Table ${name} does not exist. (UNKNOWN_TABLE)` };
+        }
+        return { status: 400, text: `fake ClickHouse cannot answer: ${body.slice(0, 80)}` };
+    });
+    return { url: stub.url, queries, close: stub.close };
+}

package/dist/clickhouse/testing/roundtrip-schema.js

@@ -0,0 +1,25 @@
+/** The schema the round trip tests import, build, and compare: roundtrip.e2e.test.ts against live servers, import/roundtrip.test.ts offline. */
+export const SCHEMA = [
+    "CREATE DATABASE analytics ENGINE = Atomic COMMENT 'product analytics'",
+    `CREATE TABLE analytics.events (
+     user_id UUID,
+     kind LowCardinality(String) DEFAULT 'click' COMMENT 'what happened',
+     ts DateTime CODEC(Delta, ZSTD(3)),
+     props Map(String, String),
+     n Nullable(UInt32),
+     total UInt64 MATERIALIZED ifNull(n, 0) * 2,
+     INDEX by_kind kind TYPE bloom_filter(0.01) GRANULARITY 4,
+     PROJECTION per_kind (SELECT kind, count() GROUP BY kind)
+   )
+   ENGINE = MergeTree
+   PARTITION BY toYYYYMM(ts)
+   ORDER BY (user_id, kind, ts)
+   TTL ts + INTERVAL 180 DAY
+   SETTINGS min_bytes_for_wide_part = 0
+   COMMENT 'raw events'`,
+    "CREATE TABLE analytics.users (id UUID, email String, updated_at DateTime) ENGINE = ReplacingMergeTree(updated_at) ORDER BY id",
+    "CREATE TABLE analytics.daily (day Date, kind LowCardinality(String), users AggregateFunction(uniq, UUID)) ENGINE = AggregatingMergeTree ORDER BY (day, kind)",
+    "CREATE MATERIALIZED VIEW analytics.daily_mv TO analytics.daily AS SELECT toDate(ts) AS day, kind, uniqState(user_id) AS users FROM analytics.events GROUP BY day, kind",
+    "CREATE VIEW analytics.by_kind AS SELECT kind, count() AS n FROM analytics.events GROUP BY kind",
+    "CREATE TABLE default.scratch (a UInt8) ENGINE = Log",
+];

package/dist/clickhouse/testing/writable-server.js

@@ -0,0 +1,123 @@
+/**
+ * A stand-in for a ClickHouse server that an apply can write to, for the
+ * applier's tests (#3208). It keeps a set of objects, answers the catalog
+ * reads observation makes, and takes the writes an apply sends: a `CREATE`
+ * adds an object with the statement as its definition, a `DROP` removes one,
+ * `MODIFY COMMENT` sets a comment. Every other `ALTER` is recorded and
+ * answered, and does not change the definition. Not a SQL engine.
+ */
+import { startHttpStub } from "../../core/testing/http-stub.js";
+import { canonicalObject } from "../plan/normalize.js";
+const row = (o) => `${JSON.stringify(o)}\n`;
+const unquote = (s) => s.replace(/^`|`$/g, "").replace(/``/g, "`");
+const literal = (s) => s.slice(1, -1).replace(/\\(.)/g, "$1");
+function engineOf(statement) {
+    const c = canonicalObject(statement);
+    if (c.kind === "view")
+        return "View";
+    if (c.kind === "materializedView")
+        return "MaterializedView";
+    return c.engineName ?? (c.kind === "database" ? "Atomic" : "MergeTree");
+}
+export async function writableClickHouse(initial = []) {
+    const state = {
+        url: "",
+        objects: initial.map((o) => ({ ...o })),
+        writes: [],
+        deletes: [],
+        mutations: [],
+        close: async () => undefined,
+    };
+    const find = (db, name) => state.objects.find((o) => o.database === db && o.name === name);
+    const answer = (body) => {
+        const sql = body.trim();
+        if (/^SELECT version\(\)/.test(sql))
+            return { status: 200, text: row({ v: "26.8.15.10" }) };
+        if (/FROM system\.databases/.test(sql)) {
+            return {
+                status: 200,
+                text: state.objects
+                    .filter((o) => o.database === undefined)
+                    .map((o) => row({ name: o.name, engine: o.engine, uuid: "00000000-0000-0000-0000-000000000000", comment: o.comment ?? "" }))
+                    .join(""),
+            };
+        }
+        if (/^SELECT count\(\) AS n FROM system\.tables WHERE database = '(.*)'$/.test(sql)) {
+            const db = literal(/'(.*)'$/.exec(sql)[0]);
+            return { status: 200, text: row({ n: state.objects.filter((o) => o.database === db).length }) };
+        }
+        if (/FROM system\.tables/.test(sql)) {
+            return {
+                status: 200,
+                text: state.objects
+                    .filter((o) => o.database !== undefined)
+                    .map((o) => row({ database: o.database, name: o.name, engine: o.engine, uuid: "00000000-0000-0000-0000-000000000000", comment: o.comment ?? "" }))
+                    .join(""),
+            };
+        }
+        // No SQL user-defined functions.
+        if (/FROM system\.functions/.test(sql))
+            return { status: 200, text: "" };
+        if (/FROM system\.mutations/.test(sql)) {
+            const m = /database = '([^']*)' AND table = '([^']*)'/.exec(sql);
+            return {
+                status: 200,
+                text: state.mutations
+                    .filter((x) => m && x.database === m[1] && x.table === m[2])
+                    .map((x) => row({ mutation_id: x.mutation_id, command: "(MODIFY COLUMN)", latest_fail_reason: x.latest_fail_reason ?? "" }))
+                    .join(""),
+            };
+        }
+        const show = /^SHOW CREATE (DATABASE|TABLE) (`(?:[^`]|``)*`)(?:\.(`(?:[^`]|``)*`))?$/.exec(sql);
+        if (show) {
+            const [db, name] = show[3] ? [unquote(show[2]), unquote(show[3])] : [undefined, unquote(show[2])];
+            const o = find(db, name);
+            return o ? { status: 200, text: row({ statement: o.statement }) } : { status: 404, text: `Code: 60. DB::Exception: Table ${name} does not exist. (UNKNOWN_TABLE)` };
+        }
+        if (/^SELECT formatQuerySingleLine/.test(sql))
+            return { status: 400, text: "fake ClickHouse does not format" };
+        state.writes.push(sql);
+        if (state.refuse?.test(sql))
+            return { status: 500, text: `Code: 36. DB::Exception: refused by the test. (BAD_ARGUMENTS)` };
+        if (/^(?:--[^\n]*\n\s*)*CREATE\b/i.test(sql)) {
+            const c = canonicalObject(sql);
+            const existing = find(c.kind === "database" ? undefined : c.database, c.name);
+            if (existing && !/^(?:--[^\n]*\n\s*)*CREATE OR REPLACE/i.test(sql))
+                return { status: 500, text: `Code: 57. DB::Exception: Table ${c.name} already exists. (TABLE_ALREADY_EXISTS)` };
+            if (existing)
+                state.objects.splice(state.objects.indexOf(existing), 1);
+            state.objects.push({
+                ...(c.kind === "database" ? {} : { database: c.database }),
+                name: c.name,
+                engine: engineOf(sql),
+                statement: sql.replace(/^CREATE OR REPLACE /i, "CREATE "),
+                ...(c.comment !== undefined ? { comment: c.comment } : {}),
+            });
+            return { status: 200, text: "" };
+        }
+        const drop = /^DROP (DATABASE|TABLE|VIEW) (`(?:[^`]|``)*`)(?:\.(`(?:[^`]|``)*`))?/.exec(sql);
+        if (drop) {
+            const [db, name] = drop[3] ? [unquote(drop[2]), unquote(drop[3])] : [undefined, unquote(drop[2])];
+            state.deletes.push(db ? `${db}.${name}` : name);
+            const o = find(db, name);
+            if (o)
+                state.objects.splice(state.objects.indexOf(o), 1);
+            return { status: 200, text: "" };
+        }
+        const comment = /^ALTER (DATABASE|TABLE) (`(?:[^`]|``)*`)(?:\.(`(?:[^`]|``)*`))? MODIFY COMMENT ('(?:[^'\\]|\\.)*')$/.exec(sql);
+        if (comment) {
+            const [db, name] = comment[3] ? [unquote(comment[2]), unquote(comment[3])] : [undefined, unquote(comment[2])];
+            const o = find(db, name);
+            if (o) {
+                o.comment = literal(comment[4]);
+                o.statement = `${o.statement.replace(/\s+COMMENT '(?:[^'\\]|\\.)*'\s*$/, "")}\nCOMMENT ${comment[4]}`;
+            }
+            return { status: 200, text: "" };
+        }
+        return { status: 200, text: "" };
+    };
+    const stub = await startHttpStub((body) => answer(body));
+    state.url = stub.url;
+    state.close = stub.close;
+    return state;
+}

package/dist/clickhouse/tokens.js

@@ -0,0 +1,23 @@
+/**
+ * ClickHouse's lexical rules over the shared lossless tokenizer
+ * (`../core/tokens.ts`, chant #3196): `` ` `` and `"` quote an identifier, a
+ * backslash escapes inside any quote, `( ) , ; . =` are punctuation, and a
+ * number may run into letters (`00662_events`, which the parser accepts as a
+ * name).
+ */
+import { tokenize as tokenizeWith, tokenizeText as tokenizeTextWith } from "../core/tokens.js";
+export { isTrivia, SqlSyntaxError, untokenize } from "../core/tokens.js";
+export const CLICKHOUSE_LEXICAL = {
+    identQuotes: '`"',
+    backslashEscapes: true,
+    punct: "(),;.=",
+    opChars: /[+\-*/%<>!|&^~?:[\]{}@]/,
+};
+/** Tokenize one string with ClickHouse's rules. `part` is the part index recorded on each token and on an error. */
+export function tokenizeText(src, part) {
+    return tokenizeTextWith(src, part, CLICKHOUSE_LEXICAL);
+}
+/** Tokenize a template's parts with ClickHouse's rules, with a `ref` token for each interpolation between them. */
+export function tokenize(parts) {
+    return tokenizeWith(parts, CLICKHOUSE_LEXICAL);
+}