@intentius/chant-lexicon-sql 0.124.46 → 0.124.48

package/src/clickhouse/plan/rules.ts

@@ -10,6 +10,9 @@
  *   parts until `MATERIALIZE INDEX`).
  * - `rewrite`: the change is a mutation that rewrites existing parts in the
  *   background (`system.mutations`), with no rollback.
+ * - `replace`: ClickHouse cannot alter the object, but it holds no rows to
+ *   keep, so the applier drops and creates it, around the views that read it
+ *   (a stream-engine table, #4396). Nothing is copied.
  * - `rebuild`: ClickHouse cannot make the change to the table it has; the data
  *   has to be copied into a new table. A plan refuses to make one in place.
  *
@@ -18,16 +21,16 @@
 
 import { classifierRule, type ChangeClasses, type ClassifierRule as SqlClassifierRule } from "../../core/classifier";
 
-export type ChangeClass = "metadata" | "rewrite" | "rebuild" | "create" | "drop";
+export type ChangeClass = "metadata" | "rewrite" | "replace" | "rebuild" | "create" | "drop";
 
 /** A ClickHouse rule: the shared core's rule shape over these classes, citing a clickhouse.com page. */
 export type ClassifierRule = SqlClassifierRule<ChangeClass>;
 
 /** The classes as data: report order and labels, and what each costs for `classifyDisruption()`. */
 export const CHANGE_CLASSES: ChangeClasses<ChangeClass> = {
-  order: ["create", "metadata", "rewrite", "rebuild", "drop"],
-  label: { create: "create", drop: "drop", metadata: "metadata only", rewrite: "background rewrite", rebuild: "REBUILD" },
-  disruption: { metadata: "in-place", rewrite: "rolling", rebuild: "replace", create: "in-place", drop: "destroy" },
+  order: ["create", "metadata", "rewrite", "replace", "rebuild", "drop"],
+  label: { create: "create", drop: "drop", metadata: "metadata only", rewrite: "background rewrite", replace: "replace table", rebuild: "REBUILD" },
+  disruption: { metadata: "in-place", rewrite: "rolling", replace: "replace", rebuild: "replace", create: "in-place", drop: "destroy" },
 };
 
 const DOCS = "https://clickhouse.com/docs/sql-reference/statements";
@@ -192,6 +195,13 @@ export const CLASSIFIER_RULES = {
     "A table cannot become a view or a view a table; the old object is dropped and the new one created.",
     `${DOCS}/create`,
   ),
+  SQLCH225: rule(
+    "SQLCH225",
+    "replace",
+    "Change a stream table",
+    "A Kafka, S3Queue, AzureQueue, RabbitMQ, NATS or FileLog table stores no rows and has no ALTER for its engine arguments, settings or columns; the views that read it are detached, the table is dropped and created, and the views are attached again. Its position (a Kafka group's offsets, an S3Queue's record of processed files) is kept by the broker or Keeper, not by the table.",
+    `${DOCS}/create/table`,
+  ),
   SQLCH230: rule(
     "SQLCH230",
     "metadata",

package/src/clickhouse/plan/stream-s3queue.e2e.test.ts

@@ -0,0 +1,148 @@
+/**
+ * Replacing an S3Queue table against the pinned server (#4396): a change to
+ * the table's settings is planned as SQLCH225, the apply detaches the view,
+ * drops and creates the table and attaches the view, and the view keeps
+ * feeding the target with no file processed twice. A different keeper_path
+ * plans with the reprocessing warning.
+ *
+ * Needs Docker; skips cleanly without it. A one-server cluster (its embedded
+ * Keeper is what S3Queue records processed files in) and a MinIO container on
+ * its network as the object store, both removed afterwards even on failure.
+ * The credentials are MinIO's published test values, held in a named
+ * collection on the server (#4395) and never in the declaration.
+ */
+
+import { execFile } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { expandComposite } from "@intentius/chant/composite";
+import type { Declarable } from "@intentius/chant/declarable";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { dockerAvailable, startScratchCluster, type ScratchCluster } from "../container";
+import { clickhouseQuery } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { sqlSerializer } from "../../serializer";
+import { StreamIngest } from "../../composites";
+import { bindClickHouse } from "../live/bind";
+import { applyClickHouse, declaredObjects } from "../apply/apply";
+import { planAgainstServer } from "./commands";
+import { renderDiff } from "./report";
+
+const exec = promisify(execFile);
+const enabled = await dockerAvailable();
+let cluster: ScratchCluster | undefined;
+let minio: string | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-s3queue-"));
+const buildFile = join(dir, "files.json");
+
+const MINIO_IMAGE = "minio/minio:RELEASE.2025-04-22T22-12-26Z";
+const q = <T = unknown>(sql: string) => clickhouseQuery<T>(cluster!.replicas[0]!, sql);
+const env = () => ({ config: {}, env: { CLICKHOUSE_URL: cluster!.replicas[0]!.url } });
+
+beforeAll(async () => {
+  if (!enabled) return;
+  cluster = await startScratchCluster(clickhouseImage(), { replicas: 1, namePrefix: "chant-sql-s3queue" });
+  minio = `${cluster.name}-minio`;
+  await exec("docker", [
+    "run", "-d", "--name", minio, "--network", `${cluster.name}-net`,
+    "-e", "MINIO_ROOT_USER=minioadmin", "-e", "MINIO_ROOT_PASSWORD=minioadmin",
+    MINIO_IMAGE, "server", "/data",
+  ], { timeout: 600_000 });
+  await exec("docker", ["exec", minio, "mkdir", "-p", "/data/lake"]);
+  await q(`CREATE NAMED COLLECTION lake_creds AS url = 'http://${minio}:9000/lake/in/*.csv', access_key_id = 'minioadmin', secret_access_key = 'minioadmin'`);
+  await q("CREATE DATABASE shop");
+}, 900_000);
+
+afterAll(async () => {
+  if (minio) await exec("docker", ["rm", "-f", minio]).catch(() => undefined);
+  await cluster?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+/** The ingest chain with the S3Queue table's settings as given. */
+function buildOf(settings: string): string {
+  const entities = new Map<string, Declarable>();
+  for (const [k, v] of expandComposite(
+    "files",
+    StreamIngest({
+      name: "shop.files",
+      columns: "id UInt64, v String",
+      engine: "S3Queue(lake_creds, format = 'CSV')",
+      settings,
+      orderBy: "id",
+    }),
+  )) {
+    entities.set(k, v);
+  }
+  return (sqlSerializer.serialize(entities) as SerializerResult).primary;
+}
+
+const SETTINGS = (maxMs: number, path = "/chant/e2e/files") =>
+  `mode = 'unordered', keeper_path = '${path}', s3queue_polling_min_timeout_ms = 100, s3queue_polling_max_timeout_ms = ${maxMs}`;
+
+async function putFile(name: string, rows: string): Promise<void> {
+  await q(`INSERT INTO FUNCTION s3('http://${minio}:9000/lake/in/${name}', 'minioadmin', 'minioadmin', 'CSV', 'id UInt64, v String') VALUES ${rows}`);
+}
+
+async function waitForRows(n: number): Promise<void> {
+  const deadline = Date.now() + 90_000;
+  for (;;) {
+    const [row] = await q<{ n: string }>("SELECT count() AS n FROM shop.files");
+    if (Number(row?.n) >= n) return;
+    if (Date.now() > deadline) throw new Error(`shop.files held ${row?.n} rows, wanted ${n}`);
+    await new Promise((r) => setTimeout(r, 500));
+  }
+}
+
+async function apply(json: string) {
+  const target = await bindClickHouse({ environment: "e2e", ...env() });
+  return applyClickHouse(target, declaredObjects(json, "default", target.topology));
+}
+
+describe.skipIf(!enabled)("replacing an S3Queue table", () => {
+  test("the view keeps feeding the target, and no file is processed twice", async () => {
+    await apply(buildOf(SETTINGS(500)));
+    await putFile("f1.csv", "(1, 'a'), (2, 'b')");
+    await putFile("f2.csv", "(3, 'c')");
+    await waitForRows(3);
+    const [before] = await q<{ uuid: string }>("SELECT toString(uuid) AS uuid FROM system.tables WHERE database = 'shop' AND name = 'files_stream'");
+
+    // The settings change: planned as a stream-table replacement, not a rebuild, with nothing refused.
+    const next = buildOf(SETTINGS(1000));
+    writeFileSync(buildFile, next);
+    const diff = await planAgainstServer("e2e", buildFile, env());
+    expect(diff.changes.map((c) => [c.rule, c.field])).toEqual([["SQLCH225", "settings.s3queue_polling_max_timeout_ms"]]);
+    expect(diff.rebuilds).toEqual([]);
+    expect(diff.changes[0]!.note).toBeUndefined();
+    expect(diff.hints.join("\n")).toMatch(/shop\.files_stream is replaced: shop\.files_mv reads it/);
+
+    const outcome = await apply(next);
+    expect(outcome.failed).toEqual([]);
+    const replaced = outcome.applied.find((a) => a.name === "shop.files_stream");
+    expect(replaced?.statements.map((s) => s.split(" ").slice(0, 2).join(" "))).toEqual(["DETACH TABLE", "DROP TABLE", "CREATE TABLE", "ATTACH TABLE"]);
+
+    const [after] = await q<{ uuid: string }>("SELECT toString(uuid) AS uuid FROM system.tables WHERE database = 'shop' AND name = 'files_stream'");
+    expect(after!.uuid).not.toBe(before!.uuid);
+    const [view] = await q<{ n: string }>("SELECT count() AS n FROM system.tables WHERE database = 'shop' AND name = 'files_mv'");
+    expect(Number(view!.n)).toBe(1);
+
+    // A file that lands after the replacement is read by the new table through the same view.
+    await putFile("f3.csv", "(4, 'd'), (5, 'e')");
+    await waitForRows(5);
+    // Give the new table the chance to read f1 and f2 again, which it must not.
+    await new Promise((r) => setTimeout(r, 5000));
+    const [rows] = await q<{ n: string; distinct: string }>("SELECT count() AS n, uniqExact(id) AS distinct FROM shop.files");
+    expect([Number(rows!.n), Number(rows!.distinct)]).toEqual([5, 5]);
+    expect((await planAgainstServer("e2e", buildFile, env())).changes).toEqual([]);
+  }, 300_000);
+
+  test("a different keeper_path plans with the reprocessing warning", async () => {
+    writeFileSync(buildFile, buildOf(SETTINGS(1000, "/chant/e2e/files-v2")));
+    const diff = await planAgainstServer("e2e", buildFile, env());
+    expect(diff.changes.map((c) => [c.rule, c.field])).toEqual([["SQLCH225", "settings.keeper_path"]]);
+    expect(renderDiff(diff)).toMatch(/every file already seen is processed again/);
+  }, 120_000);
+});

package/src/clickhouse/plan/stream.test.ts

@@ -0,0 +1,237 @@
+/**
+ * Stream engines (#4396): a change to a Kafka or S3Queue table is SQLCH225
+ * and not SQLCH223, the plan holds nothing back for it, the statements go
+ * detach, drop, create, attach, and a replacement that loses position says so.
+ */
+
+import { describe, expect, test } from "vitest";
+import { canonicalObject } from "./normalize";
+import { diffSchemas, type SchemaObject } from "./diff";
+import { renderDiff } from "./report";
+import { classifyDisruption } from "./disruption";
+import { CLASSIFIER_RULES } from "./rules";
+import { rebuildOpSuggestions } from "./rebuild-handoff";
+import { isStreamEngine, readsTable, STREAM_ENGINES, streamEngineArguments } from "./stream";
+import { planStatements, type DeclaredObject } from "../apply/statements";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
+import { observeRebuild, RebuildRefusal } from "../rebuild/observe";
+import type { ClickHouseTarget } from "../live/bind";
+
+const KAFKA = (topics: string, group = "ingest", extra = "") =>
+  `CREATE TABLE shop.events_stream (user_id UInt64, kind String) ENGINE = Kafka(kafka_creds) SETTINGS kafka_topic_list = '${topics}', kafka_group_name = '${group}', kafka_format = 'JSONEachRow'${extra}`;
+const S3QUEUE = (path: string, mode = "unordered") =>
+  `CREATE TABLE shop.files_stream (user_id UInt64) ENGINE = S3Queue(lake_creds) SETTINGS mode = '${mode}', keeper_path = '${path}'`;
+const MV = "CREATE MATERIALIZED VIEW shop.events_mv TO shop.events AS SELECT user_id, kind FROM shop.events_stream";
+const MV_OTHER = "CREATE MATERIALIZED VIEW shop.audit_mv TO shop.audit AS SELECT user_id FROM events_stream";
+const MV_UNRELATED = "CREATE MATERIALIZED VIEW shop.other_mv TO shop.other AS SELECT user_id FROM shop.users";
+
+const obj = (key: string, ddl: string): SchemaObject => ({ key, canonical: canonicalObject(ddl) });
+const one = (key: string, before: string, after: string) => diffSchemas([obj(key, before)], [obj(key, after)]);
+
+describe("stream engines", () => {
+  test("the six engines are the ones the pinned catalog has", () => {
+    expect([...STREAM_ENGINES].sort()).toEqual(["AzureQueue", "FileLog", "Kafka", "NATS", "RabbitMQ", "S3Queue"]);
+    expect(isStreamEngine("MergeTree")).toBe(false);
+    expect(isStreamEngine(undefined)).toBe(false);
+  });
+
+  test("the rule is its own class, with a rule id in the SQLCH2xx range", () => {
+    expect(CLASSIFIER_RULES.SQLCH225.class).toBe("replace");
+    expect(CLASSIFIER_RULES.SQLCH223.class).toBe("rebuild");
+  });
+});
+
+describe("classification", () => {
+  test("a different topic list is SQLCH225, not SQLCH223, and the plan refuses nothing", () => {
+    const diff = one("events_stream", KAFKA("events"), KAFKA("events,clicks"));
+    expect(diff.changes.map((c) => [c.rule, c.class, c.field])).toEqual([["SQLCH225", "replace", "settings.kafka_topic_list"]]);
+    expect(diff.rebuilds).toEqual([]);
+    expect(rebuildOpSuggestions(diff, new Map([["events_stream", canonicalObject(KAFKA("events,clicks"))]]), "prod")).toEqual([]);
+  });
+
+  test("a topic list given as an engine argument is the engine changing", () => {
+    const positional = (topic: string) => `CREATE TABLE shop.q (a String) ENGINE = Kafka('broker:9092', '${topic}', 'g', 'JSONEachRow')`;
+    const diff = one("q", positional("a"), positional("b"));
+    expect(diff.changes.map((c) => [c.rule, c.field])).toEqual([["SQLCH225", "engine"]]);
+  });
+
+  test("a format, a column and a setting added are all the stream rule", () => {
+    const diff = one(
+      "events_stream",
+      KAFKA("events"),
+      KAFKA("events", "ingest", ", kafka_num_consumers = 2").replace("kafka_format = 'JSONEachRow'", "kafka_format = 'CSV'").replace("kind String", "kind String, ts DateTime"),
+    );
+    expect(new Set(diff.changes.map((c) => c.rule))).toEqual(new Set(["SQLCH225"]));
+    expect(diff.changes.map((c) => c.field).sort()).toEqual(["columns.ts", "settings.kafka_format", "settings.kafka_num_consumers"]);
+  });
+
+  test("S3Queue, RabbitMQ and NATS tables are stream tables too", () => {
+    expect(one("f", S3QUEUE("/q/a"), S3QUEUE("/q/a", "ordered")).changes.map((c) => c.rule)).toEqual(["SQLCH225"]);
+    const nats = (subject: string) => `CREATE TABLE shop.n (a String) ENGINE = NATS(nats_creds) SETTINGS nats_subjects = '${subject}', nats_format = 'JSONEachRow'`;
+    expect(one("n", nats("a"), nats("b")).changes.map((c) => c.rule)).toEqual(["SQLCH225"]);
+    const rabbit = (queue: string) => `CREATE TABLE shop.r (a String) ENGINE = RabbitMQ(rabbit_creds) SETTINGS rabbitmq_queue_base = '${queue}', rabbitmq_format = 'JSONEachRow'`;
+    expect(one("r", rabbit("a"), rabbit("b")).changes.map((c) => c.rule)).toEqual(["SQLCH225"]);
+  });
+
+  test("a comment alone stays metadata", () => {
+    const diff = one("events_stream", KAFKA("events"), `${KAFKA("events")} COMMENT 'ingest'`);
+    expect(diff.changes.map((c) => c.rule)).toEqual(["SQLCH203"]);
+  });
+
+  test("a MergeTree engine change is still SQLCH223", () => {
+    const mt = (v: string) => `CREATE TABLE shop.t (a UInt64, v UInt64) ENGINE = ReplacingMergeTree(${v}) ORDER BY a`;
+    expect(one("t", mt("v"), mt("a")).changes.map((c) => c.rule)).toEqual(["SQLCH223"]);
+  });
+
+  test("a table that becomes a stream table is a rebuild with a note, and gets no Op suggestion", () => {
+    const mt = "CREATE TABLE shop.events_stream (user_id UInt64, kind String) ENGINE = MergeTree ORDER BY user_id";
+    const diff = one("events_stream", mt, KAFKA("events"));
+    expect(diff.changes[0]).toMatchObject({ rule: "SQLCH223", class: "rebuild" });
+    expect(diff.changes[0]!.note).toMatch(/stream engine/);
+    expect(rebuildOpSuggestions(diff, new Map([["events_stream", canonicalObject(KAFKA("events"))]]), "prod")).toEqual([]);
+  });
+
+  test("the report counts the class and names the views that are detached around the replacement", () => {
+    const diff = diffSchemas(
+      [obj("events_stream", KAFKA("events")), obj("events_mv", MV), obj("audit_mv", MV_OTHER), obj("other_mv", MV_UNRELATED)],
+      [obj("events_stream", KAFKA("clicks")), obj("events_mv", MV), obj("audit_mv", MV_OTHER), obj("other_mv", MV_UNRELATED)],
+    );
+    const text = renderDiff(diff);
+    expect(text).toMatch(/\[replace table\] settings\.kafka_topic_list/);
+    expect(text).toMatch(/SQLCH225 Change a stream table/);
+    expect(text).toMatch(/1 replace table/);
+    expect(text).toMatch(/hint: shop\.events_stream is replaced: shop\.events_mv, shop\.audit_mv read it and are detached first and attached after/);
+    expect(text).not.toMatch(/Refused/);
+  });
+});
+
+describe("position", () => {
+  test("a different kafka_group_name warns, naming both groups", () => {
+    const [c] = one("events_stream", KAFKA("events", "ingest"), KAFKA("events", "ingest-v2")).changes;
+    expect(c!.rule).toBe("SQLCH225");
+    expect(c!.note).toMatch(/loses its position: kafka_group_name changes from ingest to ingest-v2/);
+    expect(c!.note).toMatch(/committed offsets, or from the start of the topic/);
+  });
+
+  test("a group given as an engine argument is read too", () => {
+    const positional = (g: string) => `CREATE TABLE shop.q (a String) ENGINE = Kafka('broker:9092', 't', '${g}', 'JSONEachRow')`;
+    expect(one("q", positional("g1"), positional("g2")).changes[0]!.note).toMatch(/from g1 to g2/);
+  });
+
+  test("a topic list change keeps the group, so there is no warning", () => {
+    expect(one("events_stream", KAFKA("events"), KAFKA("clicks")).changes[0]!.note).toBeUndefined();
+  });
+
+  test("a different S3Queue keeper_path warns that every file is processed again", () => {
+    const [c] = one("files_stream", S3QUEUE("/q/a"), S3QUEUE("/q/b")).changes;
+    expect(c!.rule).toBe("SQLCH225");
+    expect(c!.note).toMatch(/keeper_path changes from \/q\/a to \/q\/b: .*every file already seen is processed again/);
+  });
+
+  test("the same keeper_path with another mode does not warn", () => {
+    expect(one("files_stream", S3QUEUE("/q/a"), S3QUEUE("/q/a", "ordered")).changes[0]!.note).toBeUndefined();
+  });
+});
+
+describe("reading a table", () => {
+  const table = canonicalObject(KAFKA("events"));
+  test("a view reads it by a bare name in its own database, or by a qualified one", () => {
+    expect(readsTable(canonicalObject(MV), table)).toBe(true);
+    expect(readsTable(canonicalObject(MV_OTHER), table)).toBe(true);
+    expect(readsTable(canonicalObject("CREATE MATERIALIZED VIEW audit.mv TO audit.t AS SELECT a FROM shop.events_stream"), table)).toBe(true);
+  });
+  test("a view of another table, another database's table of the same name, or a plain view is not a reader", () => {
+    expect(readsTable(canonicalObject(MV_UNRELATED), table)).toBe(false);
+    expect(readsTable(canonicalObject("CREATE MATERIALIZED VIEW audit.mv TO audit.t AS SELECT a FROM events_stream"), table)).toBe(false);
+    expect(readsTable(canonicalObject("CREATE VIEW shop.v AS SELECT a FROM shop.events_stream"), table)).toBe(false);
+    expect(readsTable(canonicalObject("CREATE MATERIALIZED VIEW shop.mv TO shop.t AS SELECT a FROM shop.events_stream_old"), table)).toBe(false);
+  });
+  test("engine arguments split outside quotes and parentheses", () => {
+    expect(streamEngineArguments("Kafka('a, b:9092', 't', 'g', 'JSONEachRow')")).toEqual(["a, b:9092", "t", "g", "JSONEachRow"]);
+    expect(streamEngineArguments("Kafka")).toEqual([]);
+  });
+});
+
+const declared = (key: string, type: DeclaredObject["type"], ddl: string): DeclaredObject => ({ exportName: key, type, key, ddl, canonical: canonicalObject(ddl), dependsOn: [] });
+
+describe("the statements", () => {
+  const before = [KAFKA("events"), MV, MV_OTHER, MV_UNRELATED].map((d) => canonicalObject(d));
+  const names = ["shop.events_stream", "shop.events_mv", "shop.audit_mv", "shop.other_mv"];
+  const current = new Map(names.map((n, i) => [n, before[i]!]));
+  const after = [
+    declared("shop.events_stream", CLICKHOUSE_ENTITY_TYPES.table, KAFKA("clicks")),
+    declared("shop.events_mv", CLICKHOUSE_ENTITY_TYPES.materializedView, MV),
+    declared("shop.audit_mv", CLICKHOUSE_ENTITY_TYPES.materializedView, MV_OTHER),
+    declared("shop.other_mv", CLICKHOUSE_ENTITY_TYPES.materializedView, MV_UNRELATED),
+  ];
+  const changes = diffSchemas(
+    names.map((n, i) => ({ key: n, canonical: before[i]! })),
+    after.map((o) => ({ key: o.key, canonical: o.canonical })),
+  ).changes;
+  const stepsOf = (plan: ReturnType<typeof planStatements>, i = 0) => (plan.objects[i] as { steps: { sql: string; rule: string; class: string }[] }).steps;
+
+  test("detach the views that read the table, replace it, attach them again", () => {
+    const plan = planStatements({ declared: after, changes, current });
+    expect(plan.objects[0]!.verdict).toBe("alter");
+    expect(stepsOf(plan).map((s) => s.sql.replace(/\s+/g, " "))).toEqual([
+      "DETACH TABLE `shop`.`events_mv`",
+      "DETACH TABLE `shop`.`audit_mv`",
+      "DROP TABLE `shop`.`events_stream` SYNC",
+      expect.stringMatching(/^CREATE TABLE shop\.events_stream \(user_id UInt64, kind String\) ENGINE = Kafka\(kafka_creds\) SETTINGS kafka_topic_list = 'clicks', .* COMMENT '.*managed-by=chant/),
+      "ATTACH TABLE `shop`.`events_mv`",
+      "ATTACH TABLE `shop`.`audit_mv`",
+    ]);
+    expect(stepsOf(plan).every((s) => s.rule === "SQLCH225" && s.class === "replace")).toBe(true);
+  });
+
+  test("the views themselves have nothing to do, and the unrelated one is not touched", () => {
+    const plan = planStatements({ declared: after, changes, current });
+    expect(plan.objects.slice(1).map((o) => [o.verdict, (o as { steps: unknown[] }).steps.length])).toEqual([
+      ["alter", 0],
+      ["alter", 0],
+      ["alter", 0],
+    ]);
+  });
+
+  test("a view the build drops in the same plan is not detached", () => {
+    const gone = [...changes, { object: "shop.audit_mv", field: "object", rule: "SQLCH250" as const, class: "drop" as const }];
+    const plan = planStatements({ declared: after.slice(0, 2), changes: gone, current });
+    expect(stepsOf(plan).filter((s) => s.sql.startsWith("DETACH")).map((s) => s.sql)).toEqual(["DETACH TABLE `shop`.`events_mv`"]);
+  });
+
+  test("on a cluster each statement runs ON CLUSTER", () => {
+    const plan = planStatements({ declared: after, changes, current, topology: { kind: "cluster", cluster: "main" } as never });
+    expect(stepsOf(plan).every((s) => /ON CLUSTER/.test(s.sql))).toBe(true);
+  });
+});
+
+describe("classifyDisruption", () => {
+  test("a stream engine's setting replaces; a MergeTree setting stays in place", () => {
+    const out = classifyDisruption({
+      environment: "prod",
+      changes: [
+        { name: "events_stream", type: "ClickHouse::Table", deltas: [{ path: "settings.kafka_topic_list" } as never] },
+        { name: "events", type: "ClickHouse::Table", deltas: [{ path: "settings.merge_with_ttl_timeout" } as never] },
+      ],
+    });
+    expect(out.events_stream).toMatchObject({ disruption: "replace" });
+    expect(out.events_stream!.detail).toMatch(/SQLCH225/);
+    expect(out.events).toMatchObject({ disruption: "in-place" });
+  });
+});
+
+describe("ClickHouseRebuildOp's observation", () => {
+  const target = { endpoint: { url: "http://unreachable.invalid" }, defaultDatabase: "shop" } as unknown as ClickHouseTarget;
+
+  test.each([
+    ["Kafka", KAFKA("events")],
+    ["S3Queue", S3QUEUE("/q/a")],
+  ])("refuses a %s table by name, before it asks the server anything", async (engine, ddl) => {
+    const o = declared("shop.t", CLICKHOUSE_ENTITY_TYPES.table, ddl);
+    const err = await observeRebuild(target, o, undefined).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(RebuildRefusal);
+    expect((err as Error).message).toContain(`the ${engine} engine`);
+    expect((err as Error).message).toMatch(/SQLCH225/);
+    expect((err as Error).message).toMatch(/stores no rows/);
+  });
+});

package/src/clickhouse/plan/stream.ts

@@ -0,0 +1,111 @@
+/**
+ * Stream engines (#4396): Kafka, S3Queue, AzureQueue, RabbitMQ, NATS and
+ * FileLog. A table on one of them stores no rows. It is a consumer: the
+ * materialized views that read it pull messages (or files) through it into a
+ * target, and the position it has reached lives outside the table, in the
+ * broker's committed offsets or in Keeper.
+ *
+ * What follows from that, and what this file holds:
+ *
+ * - There is no ALTER for the engine's arguments, and nothing to copy to a
+ *   new table, so a change is neither `metadata` nor a `rebuild`: the table is
+ *   replaced, with the views that read it detached around the replacement
+ *   (SQLCH225, `./rules.ts`).
+ * - A replacement can lose the table's position. The two changes that do
+ *   are found here and reported on the change as a note.
+ */
+
+import type { CanonicalObject } from "./normalize";
+
+/** The stream engines, as `system.tables.engine` names them. */
+export const STREAM_ENGINES: ReadonlySet<string> = new Set(["Kafka", "S3Queue", "AzureQueue", "RabbitMQ", "NATS", "FileLog"]);
+
+/** Whether an engine name is a stream engine. */
+export const isStreamEngine = (engine: string | undefined): boolean => engine !== undefined && STREAM_ENGINES.has(engine);
+
+/** Whether a canonical table is on a stream engine. */
+export const isStreamTable = (o: CanonicalObject): boolean => o.kind === "table" && isStreamEngine(o.engineName);
+
+/** The arguments inside an engine's parentheses, split at top-level commas, string quotes removed. Empty when it has none. */
+export function streamEngineArguments(engine: string | undefined): string[] {
+  const open = engine?.indexOf("(") ?? -1;
+  if (engine === undefined || open < 0 || !engine.endsWith(")")) return [];
+  const inner = engine.slice(open + 1, -1);
+  const out: string[] = [];
+  let depth = 0;
+  let quote: string | undefined;
+  let cur = "";
+  for (let i = 0; i < inner.length; i++) {
+    const ch = inner[i]!;
+    if (quote) {
+      cur += ch;
+      if (ch === "\\") cur += inner[++i] ?? "";
+      else if (ch === quote) quote = undefined;
+      continue;
+    }
+    if (ch === "'" || ch === '"' || ch === "`") quote = ch;
+    if (ch === "(" || ch === "[") depth++;
+    if (ch === ")" || ch === "]") depth--;
+    if (ch === "," && depth === 0) {
+      out.push(cur.trim());
+      cur = "";
+    } else cur += ch;
+  }
+  if (cur.trim()) out.push(cur.trim());
+  return out.map((a) => (/^'.*'$/.test(a) ? a.slice(1, -1) : a));
+}
+
+/** Kafka's positional arguments, in the order the engine takes them. */
+const KAFKA_POSITIONAL = ["kafka_broker_list", "kafka_topic_list", "kafka_group_name", "kafka_format"];
+
+/** A Kafka setting, from SETTINGS or from the engine's positional arguments. */
+function kafkaSetting(o: CanonicalObject, name: string): string | undefined {
+  const at = KAFKA_POSITIONAL.indexOf(name);
+  return o.settings[name] ?? (at >= 0 ? streamEngineArguments(o.engine)[at] : undefined);
+}
+
+/**
+ * What a replacement would lose, in a sentence each (#4396):
+ *
+ * - a different `kafka_group_name` is a different consumer group: it starts
+ *   from the group's committed offsets, or from the topic's start when the
+ *   group has none (`auto.offset.reset`);
+ * - a different S3Queue `keeper_path` is a different place for the table's
+ *   record of processed files, so every file already seen is processed again.
+ *
+ * Neither is a refusal: a new group or path is sometimes the point.
+ */
+export function positionWarnings(before: CanonicalObject, after: CanonicalObject): string[] {
+  const out: string[] = [];
+  if (before.engineName === "Kafka" && after.engineName === "Kafka") {
+    const b = kafkaSetting(before, "kafka_group_name");
+    const a = kafkaSetting(after, "kafka_group_name");
+    if (b !== a) {
+      out.push(
+        `kafka_group_name changes from ${b ?? "none"} to ${a ?? "none"}: the new group starts from its own committed offsets, or from the start of the topic when it has none, ` +
+          "so messages already consumed may be read again or skipped",
+      );
+    }
+  }
+  if (before.engineName === "S3Queue" && after.engineName === "S3Queue") {
+    const b = before.settings.keeper_path;
+    const a = after.settings.keeper_path;
+    if (b !== a) {
+      out.push(
+        `keeper_path changes from ${b ?? "none"} to ${a ?? "none"}: the new path holds no record of processed files, so every file already seen is processed again`,
+      );
+    }
+  }
+  return out;
+}
+
+/** Whether a materialized view reads from the table: its SELECT names the table, qualified or, in the table's own database, bare. */
+export function readsTable(view: CanonicalObject, table: CanonicalObject): boolean {
+  if (view.kind !== "materializedView" || view.select === undefined) return false;
+  const escaped = table.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const sameDatabase = view.database === table.database;
+  const database = table.database?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  // The canonical SELECT drops the view's own database from a name, so a bare name is a table in the view's database.
+  const names = [...(sameDatabase ? [escaped] : []), ...(database ? [`${database}\\s*\\.\\s*${escaped}`] : [])];
+  return names.some((n) => new RegExp(`(?:^|[^\\w.])${n}(?![\\w.])`).test(view.select!));
+}

package/src/clickhouse/rebuild/observe.ts

@@ -27,7 +27,9 @@
  *
  * Anything else is refused with a {@link RebuildRefusal}: a database that is
  * neither Atomic nor Replicated, a table that is not there, a change that is
- * not a rebuild, a working object chant did not make.
+ * not a rebuild, a working object chant did not make, a table on a stream
+ * engine (Kafka, S3Queue, AzureQueue, RabbitMQ, NATS, FileLog), which holds no
+ * rows to copy and is replaced by the applier instead (#4396).
  *
  * A Replicated database (#3249) is Atomic on every replica, with its DDL run
  * on each through Keeper, so the same EXCHANGE swaps the table everywhere.
@@ -47,6 +49,7 @@ import { dropFormattingOnly } from "../plan/server-format";
 import { CLASSIFIER_RULES } from "../plan/rules";
 import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
 import { carriesMarker, readTrailerPairs, REBUILD_TRAILER_KEY, stripMarkerFromStatement } from "../ownership";
+import { isStreamEngine } from "../plan/stream";
 import { ident, isRebuild, qualifiedIdent, sqlString, type DeclaredObject } from "../apply/statements";
 
 /** How the new table keeps up with writes to the old one while it is filled. */
@@ -250,10 +253,20 @@ function ownWorkingObject(o: ServerObject | undefined, names: RebuildNames, role
   return o;
 }
 
+/** The refusal for a stream-engine table, naming the engine and where its change goes instead (#4396). */
+function streamRefusal(key: string, engine: string, side: "declared" | "live"): RebuildRefusal {
+  return new RebuildRefusal(
+    `${key} is ${side === "declared" ? "declared on" : "on the server as"} the ${engine} engine, a stream engine: it stores no rows, so there is nothing to backfill, and a direct SELECT from it either fails ` +
+      `or consumes messages. A change to it is SQLCH225 (replace a stream table), which the applier makes in place: it detaches the materialized views that read the table, drops and creates it, and attaches the views again ` +
+      `(ApplyOp with target "clickhouse").`,
+  );
+}
+
 /** Observe the rebuild of the declared table on the server. */
 export async function observeRebuild(target: ClickHouseTarget, declared: DeclaredObject, marker: OwnershipMarker | undefined): Promise<RebuildObservation> {
   const database = declared.canonical.database ?? target.defaultDatabase;
   const names = rebuildNames(database, declared.canonical.name);
+  if (isStreamEngine(declared.canonical.engineName)) throw streamRefusal(names.key, declared.canonical.engineName!, "declared");
 
   const [db] = await clickhouseQuery<{ engine: string }>(target.endpoint, `SELECT engine FROM system.databases WHERE name = ${sqlString(database)}`);
   if (!db) throw new RebuildRefusal(`database ${database} is not on the server, so there is no ${names.key} to rebuild; the applier creates it`);
@@ -272,6 +285,7 @@ export async function observeRebuild(target: ClickHouseTarget, declared: Declare
   // Between EXCHANGE and the end of the swap, the table's own name holds the
   // new table, still carrying role=new, and the new table's name holds the
   // old table until it is renamed; the swap step finishes that.
+  if (isStreamEngine(liveRow.engine)) throw streamRefusal(names.key, liveRow.engine, "live");
   const exchanged = liveRow.pairs?.get(REBUILD_TRAILER_KEY) === names.key && liveRow.pairs.get("role") === "new";
   const newTable = exchanged ? undefined : ownWorkingObject(objects.get(names.newName), names, "new", marker);
   const exchangedOld = exchanged ? objects.get(names.newName) : undefined;