@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/dist/clickhouse/plan/statements.js

@@ -0,0 +1,334 @@
+/**
+ * The ClickHouse statement classifier (#4407): a migration file's statements,
+ * each given the class and rule the schema-diff classifier (`./rules.ts`)
+ * gives the same change. It reads text and contacts no server.
+ *
+ * `CREATE` goes through the declaration parser (`../parser.ts`), which also
+ * tells the classifier a table's keys when the file creates the table. The
+ * parser has no `ALTER` grammar, so `ALTER`, `DROP` and `RENAME` are read word
+ * by word from the same tokens.
+ *
+ * A statement alone cannot decide some changes, and those answer `undecided`
+ * naming both rules, the way `./disruption.ts` answers `unknown`:
+ *
+ * - `MODIFY COLUMN c <type>` is SQLCH210 (a rewrite), or SQLCH211 (a rebuild)
+ *   when `c` is in a key. The file states the keys only when it creates the
+ *   table.
+ * - `DROP COLUMN` is SQLCH202 or SQLCH213, and `RENAME COLUMN` SQLCH212 or
+ *   SQLCH213, on the same condition.
+ * - `MODIFY ORDER BY` that the same `ALTER` pairs with `ADD COLUMN` is SQLCH216
+ *   when it only appends those columns and SQLCH220 (a rebuild) otherwise.
+ *   Without such a column it is SQLCH220.
+ */
+import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse.js";
+import { isTrivia, tokenizeText } from "../tokens.js";
+import { parseCreate } from "../parser.js";
+import { classifiedOf, splitTopLevel, sqlHead } from "../../core/statement-class.js";
+import { CHANGE_CLASSES, CLASSIFIER_RULES } from "./rules.js";
+const classified = (id, extra = {}) => classifiedOf(CLASSIFIER_RULES, CHANGE_CLASSES.label, id, extra);
+const rule = (id, extra) => ({ rules: [{ id, ...(extra ? { extra } : {}) }] });
+/** Settings only a stream engine takes (the same list `./disruption.ts` reads). */
+const STREAM_SETTING = /^(?:kafka_|rabbitmq_|nats_|s3queue_|azure_queue_|keeper_path$)/;
+const bare = (name) => (name.split(".").pop() ?? name).replace(/^[`"]|[`"]$/g, "").toLowerCase();
+const text = (tokens, s) => (s ? tokens.slice(s.from, s.to).filter((t) => !isTrivia(t)).map((t) => t.text).join(" ") : "");
+/** The identifier tokens of a key expression that are not function names. */
+function columnsOf(tokens, s) {
+    if (!s)
+        return [];
+    const toks = tokens.slice(s.from, s.to).filter((t) => !isTrivia(t));
+    return toks.flatMap((t, i) => ((t.kind === "ident" || t.kind === "qident") && toks[i + 1]?.text !== "(" ? [bare(t.text)] : []));
+}
+function nameAt(words, i) {
+    let name = words[i] ?? "";
+    let j = i + 1;
+    while (words[j] === "." && words[j + 1] !== undefined) {
+        name += `.${words[j + 1]}`;
+        j += 2;
+    }
+    return { name, next: j };
+}
+const KNOWN_VERBS = new Set(["ADD", "DROP", "RENAME", "MODIFY", "COMMENT", "CLEAR", "MATERIALIZE", "UPDATE", "DELETE", "ATTACH", "DETACH", "FREEZE", "FETCH", "MOVE", "REPLACE", "RESET", "REMOVE", "APPLY", "UNFREEZE", "EXECUTE"]);
+/** Normalise a key expression for comparing an old and a new one: no spaces, no outer parentheses. */
+const keyText = (words) => {
+    let t = words.join("").toLowerCase();
+    while (t.startsWith("(") && t.endsWith(")"))
+        t = t.slice(1, -1);
+    return t;
+};
+function columnVerdict(kind, col, ctx) {
+    const plain = kind === "drop" ? "SQLCH202" : "SQLCH212";
+    const extra = kind === "drop" ? { destructive: true } : undefined;
+    if (ctx.created) {
+        return ctx.created.keys.has(bare(col)) ? rule("SQLCH213", extra) : rule(plain, extra);
+    }
+    return { undecided: [plain, "SQLCH213"], note: `${col} is a column change or, in a key, a rebuild; the file does not create the table, so its keys are not known offline` };
+}
+function tableAction(a, ctx) {
+    const u = a.map((w) => w.toUpperCase());
+    const first = u[0];
+    const second = u[1];
+    if (first === "ADD") {
+        if (second === "COLUMN")
+            return rule("SQLCH201");
+        if (second === "INDEX")
+            return rule("SQLCH204");
+        if (second === "PROJECTION")
+            return rule("SQLCH214");
+        if (second === "CONSTRAINT")
+            return rule("SQLCH215");
+        return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+    }
+    if (first === "DROP") {
+        if (second === "COLUMN") {
+            const k = u[2] === "IF" ? 4 : 2;
+            return columnVerdict("drop", a[k] ?? "column", ctx);
+        }
+        if (second === "INDEX")
+            return rule("SQLCH204");
+        if (second === "PROJECTION")
+            return rule("SQLCH214");
+        if (second === "CONSTRAINT")
+            return rule("SQLCH215");
+        return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")} (removes data parts, no rule covers it)` };
+    }
+    if (first === "CLEAR") {
+        if (second === "INDEX")
+            return rule("SQLCH204");
+        if (second === "PROJECTION")
+            return rule("SQLCH214");
+        return { none: `ALTER TABLE CLEAR ${second ?? ""}`.trim() };
+    }
+    if (first === "RENAME" && second === "COLUMN") {
+        const k = u[2] === "IF" ? 4 : 2;
+        return columnVerdict("rename", a[k] ?? "column", ctx);
+    }
+    if (first === "COMMENT" && second === "COLUMN")
+        return rule("SQLCH203");
+    if (first === "MODIFY" || first === "RESET") {
+        if (second === "COLUMN") {
+            let k = 2;
+            if (u[k] === "IF" && u[k + 1] === "EXISTS")
+                k += 2;
+            const col = a[k] ?? "column";
+            const after = u[k + 1];
+            if (after === "REMOVE") {
+                const what = u[k + 2];
+                return what === "CODEC" ? rule("SQLCH208") : what === "COMMENT" ? rule("SQLCH203") : what === "TTL" ? rule("SQLCH205") : rule("SQLCH207");
+            }
+            if (after === "DEFAULT" || after === "MATERIALIZED" || after === "ALIAS" || after === "EPHEMERAL")
+                return rule("SQLCH207");
+            if (after === "CODEC")
+                return rule("SQLCH208");
+            if (after === "COMMENT")
+                return rule("SQLCH203");
+            if (after === "TTL")
+                return rule("SQLCH205");
+            if (after === "MODIFY" || after === "RESET")
+                return { none: "MODIFY COLUMN ... SETTING" };
+            if (after === undefined)
+                return { none: "MODIFY COLUMN with no change" };
+            if (ctx.created)
+                return ctx.created.keys.has(bare(col)) ? rule("SQLCH211") : rule("SQLCH210");
+            return { undecided: ["SQLCH210", "SQLCH211"], note: `a type change of ${col} is a background rewrite or, when ${col} is in a key, a rebuild; the file does not create the table, so its keys are not known offline` };
+        }
+        if (second === "ORDER" && u[2] === "BY") {
+            const next = a.slice(3);
+            if (ctx.created?.orderBy !== undefined) {
+                const was = ctx.created.orderBy;
+                const now = keyText(next);
+                const tail = now.startsWith(was) ? now.slice(was.length) : undefined;
+                const extra = tail === undefined ? [] : tail.split(/[^a-z0-9_]+/).filter(Boolean);
+                if (tail !== undefined && extra.length > 0 && extra.every((c) => ctx.added.has(c)))
+                    return rule("SQLCH216");
+                return rule("SQLCH220");
+            }
+            const mentions = next.some((w) => ctx.added.has(bare(w)));
+            if (mentions)
+                return { undecided: ["SQLCH216", "SQLCH220"], note: "the new key appends the columns this ALTER adds only if its other elements are the old key's, which the file does not state" };
+            return rule("SQLCH220");
+        }
+        if (second === "SAMPLE" && u[2] === "BY")
+            return rule("SQLCH217");
+        if (second === "TTL")
+            return rule("SQLCH205");
+        if (second === "COMMENT")
+            return rule("SQLCH203");
+        if (second === "QUERY")
+            return rule("SQLCH241");
+        if (second === "REFRESH")
+            return rule("SQLCH244");
+        if (second === "SETTING" || second === "SETTINGS") {
+            const key = (a[2] ?? "").toLowerCase();
+            return settingRule(key);
+        }
+        return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+    }
+    if (first === "REMOVE") {
+        if (second === "TTL")
+            return rule("SQLCH205");
+        if (second === "SAMPLE" && u[2] === "BY")
+            return rule("SQLCH217");
+        return { none: `ALTER TABLE REMOVE ${second ?? ""}`.trim() };
+    }
+    if (first === "UPDATE" || first === "DELETE")
+        return { data: "a mutation: it rewrites the parts it touches in the background (system.mutations) and has no rollback" };
+    return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+}
+function settingRule(key) {
+    if (STREAM_SETTING.test(key))
+        return rule("SQLCH225", { note: `${key} belongs to a stream engine, which has no MODIFY SETTING` });
+    const spec = MERGE_TREE_SETTINGS[key];
+    return spec?.readonly ? rule("SQLCH218", { note: `${key} is fixed at creation` }) : rule("SQLCH206");
+}
+/** The `ALTER TABLE` actions of one statement, in order, a `MODIFY SETTING` list kept together. */
+function alterTable(words, u, created) {
+    let i = 2;
+    if (u[i] === "IF" && u[i + 1] === "EXISTS")
+        i += 2;
+    const { name, next } = nameAt(words, i);
+    i = next;
+    if (u[i] === "ON" && u[i + 1] === "CLUSTER")
+        i += 3;
+    const actions = splitTopLevel(words.slice(i));
+    const added = new Set();
+    for (const a of actions) {
+        const au = a.map((w) => w.toUpperCase());
+        if (au[0] === "ADD" && au[1] === "COLUMN") {
+            const k = au[2] === "IF" ? 5 : 2;
+            if (a[k])
+                added.add(bare(a[k]));
+        }
+    }
+    const ctx = { added, ...(created.has(bare(name)) ? { created: created.get(bare(name)) } : {}) };
+    const out = [];
+    let inSettings = false;
+    for (const a of actions) {
+        const au = a.map((w) => w.toUpperCase());
+        if (inSettings && au[0] !== undefined && !KNOWN_VERBS.has(au[0])) {
+            out.push(settingRule((a[0] ?? "").toLowerCase()));
+            continue;
+        }
+        inSettings = (au[0] === "MODIFY" || au[0] === "RESET") && (au[1] === "SETTING" || au[1] === "SETTINGS");
+        out.push(tableAction(a, ctx));
+    }
+    return out;
+}
+const OBJECTS = { USER: "SQLCH271", ROLE: "SQLCH270", QUOTA: "SQLCH275" };
+function alterOther(u) {
+    const obj = u[1];
+    if (obj && OBJECTS[obj])
+        return rule(OBJECTS[obj]);
+    if (obj === "ROW" && u[2] === "POLICY")
+        return rule("SQLCH272");
+    if (obj === "POLICY")
+        return rule("SQLCH272");
+    if (obj === "SETTINGS" && u[2] === "PROFILE")
+        return rule("SQLCH276");
+    if (obj === "DATABASE" && u.includes("COMMENT"))
+        return rule("SQLCH203");
+    return { none: `ALTER ${obj ?? ""}`.trim() };
+}
+const DATA_WORDS = new Set(["INSERT", "SELECT", "WITH", "SYSTEM", "OPTIMIZE", "TRUNCATE", "SET", "USE", "EXPLAIN", "EXISTS", "DESCRIBE", "DESC", "SHOW", "CHECK", "KILL", "WATCH"]);
+/** A `CREATE` or `ATTACH`: the parser reads tables, views, dictionaries, databases and functions; the rest go by key words. */
+function create(tokens, u, created) {
+    let i = 1;
+    const orReplace = u[i] === "OR" && u[i + 1] === "REPLACE";
+    if (orReplace)
+        i += 2;
+    let node;
+    try {
+        node = parseCreate(tokens);
+    }
+    catch {
+        node = undefined;
+    }
+    if (node?.statement === "table") {
+        const name = text(tokens, node.name).replace(/\s*\.\s*/g, ".");
+        const keys = new Set([...columnsOf(tokens, node.orderBy), ...columnsOf(tokens, node.primaryKey), ...columnsOf(tokens, node.partitionBy), ...columnsOf(tokens, node.sampleBy)]);
+        for (const c of node.columns)
+            if (c.primaryKey)
+                keys.add(bare(c.name));
+        created.set(bare(name), { keys, ...(node.orderBy ? { orderBy: keyText(text(tokens, node.orderBy).split(" ")) } : {}) });
+    }
+    if (orReplace && u[i] === "FUNCTION")
+        return rule("SQLCH260");
+    if (orReplace && (u[i] === "TABLE" || node?.statement === "table"))
+        return rule("SQLCH200", { note: "OR REPLACE drops an existing table of that name, and its rows, before it creates this one" });
+    return rule("SQLCH200");
+}
+function read(sql, created) {
+    let tokens;
+    try {
+        tokens = tokenizeText(sql, 0);
+    }
+    catch {
+        return [{ none: "text the tokenizer could not read" }];
+    }
+    const words = tokens.filter((t) => !isTrivia(t) && t.text !== ";").map((t) => t.text);
+    const u = words.map((w) => w.toUpperCase());
+    const head = u[0];
+    if (head === undefined)
+        return [{ data: "an empty statement" }];
+    if (head === "CREATE")
+        return [create(tokens, u, created)];
+    if (head === "ALTER" && u[1] === "TABLE")
+        return alterTable(words, u, created);
+    if (head === "ALTER")
+        return [alterOther(u)];
+    if (head === "DROP")
+        return [rule("SQLCH250", { destructive: true })];
+    if (head === "RENAME") {
+        if (u[1] === "DATABASE")
+            return [rule("SQLCH231")];
+        if (u[1] === "TABLE" || u[1] === "DICTIONARY")
+            return [rule("SQLCH230")];
+        return [{ none: `RENAME ${u[1] ?? ""}`.trim() }];
+    }
+    if (head === "GRANT")
+        return [rule("SQLCH273")];
+    if (head === "REVOKE")
+        return [rule("SQLCH274")];
+    if (head === "EXCHANGE")
+        return [{ none: "EXCHANGE TABLES swaps two tables; no rule covers it" }];
+    if (head === "TRUNCATE")
+        return [{ data: "TRUNCATE removes every row of the table" }];
+    if (DATA_WORDS.has(head))
+        return [{ data: `${head} reads or writes rows or the session, not the schema` }];
+    return [{ none: `${head} is not a statement the classifier reads` }];
+}
+/**
+ * Classify a migration's up statements, in order. The statements are the
+ * file's own text, one per entry.
+ */
+export function classifyClickHouseStatements(statements) {
+    const created = new Map();
+    return statements.map((sql, k) => {
+        const readings = read(sql, created);
+        const out = { index: k + 1, sql: sqlHead(sql), outcome: "classified", changes: [] };
+        const none = [];
+        const notes = [];
+        for (const r of readings) {
+            if ("rules" in r)
+                out.changes.push(...r.rules.map((x) => classified(x.id, x.extra)));
+            else if ("undecided" in r) {
+                out.candidates = [...(out.candidates ?? []), ...r.undecided.map((id) => classified(id))];
+                notes.push(r.note);
+            }
+            else if ("none" in r)
+                none.push(r.none);
+            else
+                notes.push(r.data);
+        }
+        if (out.candidates)
+            out.outcome = "undecided";
+        else if (out.changes.length === 0 && none.length === 0)
+            out.outcome = "data";
+        else if (none.length > 0) {
+            out.outcome = out.changes.length > 0 ? "classified" : "unclassified";
+            notes.push(`no rule covers ${none.join("; ")}`);
+        }
+        if (notes.length > 0)
+            out.note = notes.join("; ");
+        return out;
+    });
+}

package/dist/clickhouse/plan/stream.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Stream engines (#4396): Kafka, S3Queue, AzureQueue, RabbitMQ, NATS and
+ * FileLog. A table on one of them stores no rows. It is a consumer: the
+ * materialized views that read it pull messages (or files) through it into a
+ * target, and the position it has reached lives outside the table, in the
+ * broker's committed offsets or in Keeper.
+ *
+ * What follows from that, and what this file holds:
+ *
+ * - There is no ALTER for the engine's arguments, and nothing to copy to a
+ *   new table, so a change is neither `metadata` nor a `rebuild`: the table is
+ *   replaced, with the views that read it detached around the replacement
+ *   (SQLCH225, `./rules.ts`).
+ * - A replacement can lose the table's position. The two changes that do
+ *   are found here and reported on the change as a note.
+ */
+import type { CanonicalObject } from "./normalize.js";
+/** The stream engines, as `system.tables.engine` names them. */
+export declare const STREAM_ENGINES: ReadonlySet<string>;
+/** Whether an engine name is a stream engine. */
+export declare const isStreamEngine: (engine: string | undefined) => boolean;
+/** Whether a canonical table is on a stream engine. */
+export declare const isStreamTable: (o: CanonicalObject) => boolean;
+/** The arguments inside an engine's parentheses, split at top-level commas, string quotes removed. Empty when it has none. */
+export declare function streamEngineArguments(engine: string | undefined): string[];
+/**
+ * What a replacement would lose, in a sentence each (#4396):
+ *
+ * - a different `kafka_group_name` is a different consumer group: it starts
+ *   from the group's committed offsets, or from the topic's start when the
+ *   group has none (`auto.offset.reset`);
+ * - a different S3Queue `keeper_path` is a different place for the table's
+ *   record of processed files, so every file already seen is processed again.
+ *
+ * Neither is a refusal: a new group or path is sometimes the point.
+ */
+export declare function positionWarnings(before: CanonicalObject, after: CanonicalObject): string[];
+/** Whether a materialized view reads from the table: its SELECT names the table, qualified or, in the table's own database, bare. */
+export declare function readsTable(view: CanonicalObject, table: CanonicalObject): boolean;
+//# sourceMappingURL=stream.d.ts.map

package/dist/clickhouse/plan/stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../../src/clickhouse/plan/stream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,gEAAgE;AAChE,eAAO,MAAM,cAAc,EAAE,WAAW,CAAC,MAAM,CAA8E,CAAC;AAE9H,iDAAiD;AACjD,eAAO,MAAM,cAAc,GAAI,QAAQ,MAAM,GAAG,SAAS,KAAG,OAA6D,CAAC;AAE1H,uDAAuD;AACvD,eAAO,MAAM,aAAa,GAAI,GAAG,eAAe,KAAG,OAA6D,CAAC;AAEjH,8HAA8H;AAC9H,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CA0B1E;AAWD;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe,GAAG,MAAM,EAAE,CAsB1F;AAED,qIAAqI;AACrI,wBAAgB,UAAU,CAAC,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe,GAAG,OAAO,CAQjF"}

package/dist/clickhouse/plan/stream.js

@@ -0,0 +1,107 @@
+/**
+ * Stream engines (#4396): Kafka, S3Queue, AzureQueue, RabbitMQ, NATS and
+ * FileLog. A table on one of them stores no rows. It is a consumer: the
+ * materialized views that read it pull messages (or files) through it into a
+ * target, and the position it has reached lives outside the table, in the
+ * broker's committed offsets or in Keeper.
+ *
+ * What follows from that, and what this file holds:
+ *
+ * - There is no ALTER for the engine's arguments, and nothing to copy to a
+ *   new table, so a change is neither `metadata` nor a `rebuild`: the table is
+ *   replaced, with the views that read it detached around the replacement
+ *   (SQLCH225, `./rules.ts`).
+ * - A replacement can lose the table's position. The two changes that do
+ *   are found here and reported on the change as a note.
+ */
+/** The stream engines, as `system.tables.engine` names them. */
+export const STREAM_ENGINES = new Set(["Kafka", "S3Queue", "AzureQueue", "RabbitMQ", "NATS", "FileLog"]);
+/** Whether an engine name is a stream engine. */
+export const isStreamEngine = (engine) => engine !== undefined && STREAM_ENGINES.has(engine);
+/** Whether a canonical table is on a stream engine. */
+export const isStreamTable = (o) => o.kind === "table" && isStreamEngine(o.engineName);
+/** The arguments inside an engine's parentheses, split at top-level commas, string quotes removed. Empty when it has none. */
+export function streamEngineArguments(engine) {
+    const open = engine?.indexOf("(") ?? -1;
+    if (engine === undefined || open < 0 || !engine.endsWith(")"))
+        return [];
+    const inner = engine.slice(open + 1, -1);
+    const out = [];
+    let depth = 0;
+    let quote;
+    let cur = "";
+    for (let i = 0; i < inner.length; i++) {
+        const ch = inner[i];
+        if (quote) {
+            cur += ch;
+            if (ch === "\\")
+                cur += inner[++i] ?? "";
+            else if (ch === quote)
+                quote = undefined;
+            continue;
+        }
+        if (ch === "'" || ch === '"' || ch === "`")
+            quote = ch;
+        if (ch === "(" || ch === "[")
+            depth++;
+        if (ch === ")" || ch === "]")
+            depth--;
+        if (ch === "," && depth === 0) {
+            out.push(cur.trim());
+            cur = "";
+        }
+        else
+            cur += ch;
+    }
+    if (cur.trim())
+        out.push(cur.trim());
+    return out.map((a) => (/^'.*'$/.test(a) ? a.slice(1, -1) : a));
+}
+/** Kafka's positional arguments, in the order the engine takes them. */
+const KAFKA_POSITIONAL = ["kafka_broker_list", "kafka_topic_list", "kafka_group_name", "kafka_format"];
+/** A Kafka setting, from SETTINGS or from the engine's positional arguments. */
+function kafkaSetting(o, name) {
+    const at = KAFKA_POSITIONAL.indexOf(name);
+    return o.settings[name] ?? (at >= 0 ? streamEngineArguments(o.engine)[at] : undefined);
+}
+/**
+ * What a replacement would lose, in a sentence each (#4396):
+ *
+ * - a different `kafka_group_name` is a different consumer group: it starts
+ *   from the group's committed offsets, or from the topic's start when the
+ *   group has none (`auto.offset.reset`);
+ * - a different S3Queue `keeper_path` is a different place for the table's
+ *   record of processed files, so every file already seen is processed again.
+ *
+ * Neither is a refusal: a new group or path is sometimes the point.
+ */
+export function positionWarnings(before, after) {
+    const out = [];
+    if (before.engineName === "Kafka" && after.engineName === "Kafka") {
+        const b = kafkaSetting(before, "kafka_group_name");
+        const a = kafkaSetting(after, "kafka_group_name");
+        if (b !== a) {
+            out.push(`kafka_group_name changes from ${b ?? "none"} to ${a ?? "none"}: the new group starts from its own committed offsets, or from the start of the topic when it has none, ` +
+                "so messages already consumed may be read again or skipped");
+        }
+    }
+    if (before.engineName === "S3Queue" && after.engineName === "S3Queue") {
+        const b = before.settings.keeper_path;
+        const a = after.settings.keeper_path;
+        if (b !== a) {
+            out.push(`keeper_path changes from ${b ?? "none"} to ${a ?? "none"}: the new path holds no record of processed files, so every file already seen is processed again`);
+        }
+    }
+    return out;
+}
+/** Whether a materialized view reads from the table: its SELECT names the table, qualified or, in the table's own database, bare. */
+export function readsTable(view, table) {
+    if (view.kind !== "materializedView" || view.select === undefined)
+        return false;
+    const escaped = table.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const sameDatabase = view.database === table.database;
+    const database = table.database?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    // The canonical SELECT drops the view's own database from a name, so a bare name is a table in the view's database.
+    const names = [...(sameDatabase ? [escaped] : []), ...(database ? [`${database}\\s*\\.\\s*${escaped}`] : [])];
+    return names.some((n) => new RegExp(`(?:^|[^\\w.])${n}(?![\\w.])`).test(view.select));
+}

package/dist/clickhouse/rebuild/observe.d.ts

@@ -27,7 +27,9 @@
  *
  * Anything else is refused with a {@link RebuildRefusal}: a database that is
  * neither Atomic nor Replicated, a table that is not there, a change that is
- * not a rebuild, a working object chant did not make.
+ * not a rebuild, a working object chant did not make, a table on a stream
+ * engine (Kafka, S3Queue, AzureQueue, RabbitMQ, NATS, FileLog), which holds no
+ * rows to copy and is replaced by the applier instead (#4396).
  *
  * A Replicated database (#3249) is Atomic on every replica, with its DDL run
  * on each through Keeper, so the same EXCHANGE swaps the table everywhere.

package/dist/clickhouse/rebuild/observe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../../src/clickhouse/rebuild/observe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAGlE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAmB,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,EAAe,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAKxD,OAAO,EAAE,KAAK,EAAwC,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAEvG,gFAAgF;AAChF,MAAM,MAAM,SAAS,GACjB;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,mBAAmB,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,KAAK,CAAC;CACb,CAAC;AAEN,iGAAiG;AACjG,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,YAAY,CAgBzE;AAED,+DAA+D;AAC/D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,iEAAiE;IACjE,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B;AAED,2DAA2D;AAC3D,MAAM,WAAW,YAAY;IAC3B,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;IACtC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,cAAc,CAAC;IACzB,oCAAoC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,mEAAmE;IACnE,aAAa,EAAE,eAAe,CAAC;IAC/B,yEAAyE;IACzE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,GAAG,CAAC,EAAE,YAAY,CAAC;IACnB,wGAAwG;IACxG,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,uGAAuG;IACvG,SAAS,EAAE,OAAO,CAAC;IACnB,sEAAsE;IACtE,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB;;;;OAIG;IACH,UAAU,EAAE,OAAO,CAAC;CACrB;AAID,4FAA4F;AAC5F,wBAAgB,aAAa,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,cAAc,CAO9F;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,eAAe,EAAE,IAAI,EAAE,eAAe,GAAG,YAAY,EAAE,CAS9F;AAED,yFAAyF;AACzF,wBAAgB,SAAS,CAAC,CAAC,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAOrE;AAgBD,4EAA4E;AAC5E,wBAAsB,UAAU,CAAC,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAGtG;AAED,2HAA2H;AAC3H,wBAAsB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGzI;AAmBD,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,eAAe,GAAG,SAAS,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA2EzJ;AAoCD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,kBAAkB,EAAE,SAAS,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CASvG;AAED,eAAO,MAAM,iBAAiB,GAAI,GAAG,kBAAkB,EAAE,WAAW,SAAS,KAAG,MAAmF,CAAC;AAEpK,mEAAmE;AACnE,OAAO,EAAE,KAAK,EAAE,CAAC"}
+{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../../src/clickhouse/rebuild/observe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAGlE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAmB,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,EAAe,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAMxD,OAAO,EAAE,KAAK,EAAwC,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAEvG,gFAAgF;AAChF,MAAM,MAAM,SAAS,GACjB;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,mBAAmB,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,KAAK,CAAC;CACb,CAAC;AAEN,iGAAiG;AACjG,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AAED,uDAAuD;AACvD,MAAM,WAAW,YAAY;IAC3B,sCAAsC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,YAAY,CAgBzE;AAED,+DAA+D;AAC/D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,iEAAiE;IACjE,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B;AAED,2DAA2D;AAC3D,MAAM,WAAW,YAAY;IAC3B,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;IACtC,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,cAAc,CAAC;IACzB,oCAAoC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,mEAAmE;IACnE,aAAa,EAAE,eAAe,CAAC;IAC/B,yEAAyE;IACzE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,GAAG,CAAC,EAAE,YAAY,CAAC;IACnB,wGAAwG;IACxG,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,uGAAuG;IACvG,SAAS,EAAE,OAAO,CAAC;IACnB,sEAAsE;IACtE,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB;;;;OAIG;IACH,UAAU,EAAE,OAAO,CAAC;CACrB;AAID,4FAA4F;AAC5F,wBAAgB,aAAa,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,cAAc,CAO9F;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,eAAe,EAAE,IAAI,EAAE,eAAe,GAAG,YAAY,EAAE,CAS9F;AAED,yFAAyF;AACzF,wBAAgB,SAAS,CAAC,CAAC,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAOrE;AAgBD,4EAA4E;AAC5E,wBAAsB,UAAU,CAAC,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAGtG;AAED,2HAA2H;AAC3H,wBAAsB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGzI;AA4BD,+DAA+D;AAC/D,wBAAsB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,eAAe,GAAG,SAAS,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA6EzJ;AAoCD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,kBAAkB,EAAE,SAAS,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CASvG;AAED,eAAO,MAAM,iBAAiB,GAAI,GAAG,kBAAkB,EAAE,WAAW,SAAS,KAAG,MAAmF,CAAC;AAEpK,mEAAmE;AACnE,OAAO,EAAE,KAAK,EAAE,CAAC"}

package/dist/clickhouse/rebuild/observe.js

@@ -27,7 +27,9 @@
  *
  * Anything else is refused with a {@link RebuildRefusal}: a database that is
  * neither Atomic nor Replicated, a table that is not there, a change that is
- * not a rebuild, a working object chant did not make.
+ * not a rebuild, a working object chant did not make, a table on a stream
+ * engine (Kafka, S3Queue, AzureQueue, RabbitMQ, NATS, FileLog), which holds no
+ * rows to copy and is replaced by the applier instead (#4396).
  *
  * A Replicated database (#3249) is Atomic on every replica, with its DDL run
  * on each through Keeper, so the same EXCHANGE swaps the table everywhere.
@@ -44,6 +46,7 @@ import { dropFormattingOnly } from "../plan/server-format.js";
 import { CLASSIFIER_RULES } from "../plan/rules.js";
 import { CLICKHOUSE_ENTITY_TYPES } from "../entities.js";
 import { carriesMarker, readTrailerPairs, REBUILD_TRAILER_KEY, stripMarkerFromStatement } from "../ownership.js";
+import { isStreamEngine } from "../plan/stream.js";
 import { ident, isRebuild, qualifiedIdent, sqlString } from "../apply/statements.js";
 /** A refusal: the rebuild cannot start, or cannot go on, for a reason a person has to act on. */
 export class RebuildRefusal extends Error {
@@ -140,10 +143,18 @@ function ownWorkingObject(o, names, role, marker) {
     }
     return o;
 }
+/** The refusal for a stream-engine table, naming the engine and where its change goes instead (#4396). */
+function streamRefusal(key, engine, side) {
+    return new RebuildRefusal(`${key} is ${side === "declared" ? "declared on" : "on the server as"} the ${engine} engine, a stream engine: it stores no rows, so there is nothing to backfill, and a direct SELECT from it either fails ` +
+        `or consumes messages. A change to it is SQLCH225 (replace a stream table), which the applier makes in place: it detaches the materialized views that read the table, drops and creates it, and attaches the views again ` +
+        `(ApplyOp with target "clickhouse").`);
+}
 /** Observe the rebuild of the declared table on the server. */
 export async function observeRebuild(target, declared, marker) {
     const database = declared.canonical.database ?? target.defaultDatabase;
     const names = rebuildNames(database, declared.canonical.name);
+    if (isStreamEngine(declared.canonical.engineName))
+        throw streamRefusal(names.key, declared.canonical.engineName, "declared");
     const [db] = await clickhouseQuery(target.endpoint, `SELECT engine FROM system.databases WHERE name = ${sqlString(database)}`);
     if (!db)
         throw new RebuildRefusal(`database ${database} is not on the server, so there is no ${names.key} to rebuild; the applier creates it`);
@@ -160,6 +171,8 @@ export async function observeRebuild(target, declared, marker) {
     // Between EXCHANGE and the end of the swap, the table's own name holds the
     // new table, still carrying role=new, and the new table's name holds the
     // old table until it is renamed; the swap step finishes that.
+    if (isStreamEngine(liveRow.engine))
+        throw streamRefusal(names.key, liveRow.engine, "live");
     const exchanged = liveRow.pairs?.get(REBUILD_TRAILER_KEY) === names.key && liveRow.pairs.get("role") === "new";
     const newTable = exchanged ? undefined : ownWorkingObject(objects.get(names.newName), names, "new", marker);
     const exchangedOld = exchanged ? objects.get(names.newName) : undefined;