@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/clickhouse/plan/rules.ts

@@ -10,6 +10,9 @@
  *   parts until `MATERIALIZE INDEX`).
  * - `rewrite`: the change is a mutation that rewrites existing parts in the
  *   background (`system.mutations`), with no rollback.
+ * - `replace`: ClickHouse cannot alter the object, but it holds no rows to
+ *   keep, so the applier drops and creates it, around the views that read it
+ *   (a stream-engine table, #4396). Nothing is copied.
  * - `rebuild`: ClickHouse cannot make the change to the table it has; the data
  *   has to be copied into a new table. A plan refuses to make one in place.
  *
@@ -18,16 +21,16 @@
 
 import { classifierRule, type ChangeClasses, type ClassifierRule as SqlClassifierRule } from "../../core/classifier";
 
-export type ChangeClass = "metadata" | "rewrite" | "rebuild" | "create" | "drop";
+export type ChangeClass = "metadata" | "rewrite" | "replace" | "rebuild" | "create" | "drop";
 
 /** A ClickHouse rule: the shared core's rule shape over these classes, citing a clickhouse.com page. */
 export type ClassifierRule = SqlClassifierRule<ChangeClass>;
 
 /** The classes as data: report order and labels, and what each costs for `classifyDisruption()`. */
 export const CHANGE_CLASSES: ChangeClasses<ChangeClass> = {
-  order: ["create", "metadata", "rewrite", "rebuild", "drop"],
-  label: { create: "create", drop: "drop", metadata: "metadata only", rewrite: "background rewrite", rebuild: "REBUILD" },
-  disruption: { metadata: "in-place", rewrite: "rolling", rebuild: "replace", create: "in-place", drop: "destroy" },
+  order: ["create", "metadata", "rewrite", "replace", "rebuild", "drop"],
+  label: { create: "create", drop: "drop", metadata: "metadata only", rewrite: "background rewrite", replace: "replace table", rebuild: "REBUILD" },
+  disruption: { metadata: "in-place", rewrite: "rolling", replace: "replace", rebuild: "replace", create: "in-place", drop: "destroy" },
 };
 
 const DOCS = "https://clickhouse.com/docs/sql-reference/statements";
@@ -192,6 +195,13 @@ export const CLASSIFIER_RULES = {
     "A table cannot become a view or a view a table; the old object is dropped and the new one created.",
     `${DOCS}/create`,
   ),
+  SQLCH225: rule(
+    "SQLCH225",
+    "replace",
+    "Change a stream table",
+    "A Kafka, S3Queue, AzureQueue, RabbitMQ, NATS or FileLog table stores no rows and has no ALTER for its engine arguments, settings or columns; the views that read it are detached, the table is dropped and created, and the views are attached again. Its position (a Kafka group's offsets, an S3Queue's record of processed files) is kept by the broker or Keeper, not by the table.",
+    `${DOCS}/create/table`,
+  ),
   SQLCH230: rule(
     "SQLCH230",
     "metadata",

package/src/clickhouse/plan/statements.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, test } from "vitest";
+import { classifyClickHouseStatements } from "./statements";
+
+const one = (sql: string) => classifyClickHouseStatements([sql])[0]!;
+const rules = (sql: string) => one(sql).changes.map((c) => c.rule);
+
+describe("ClickHouse statements", () => {
+  test("a sort-key change is SQLCH220, a rebuild", () => {
+    const v = one("ALTER TABLE events MODIFY ORDER BY (user_id, ts)");
+    expect(v.outcome).toBe("classified");
+    expect(v.changes).toMatchObject([{ rule: "SQLCH220", class: "rebuild", label: "REBUILD" }]);
+  });
+
+  test("appending the column the same ALTER adds is undecided between SQLCH216 and SQLCH220", () => {
+    const v = one("ALTER TABLE events ADD COLUMN region String, MODIFY ORDER BY (user_id, region)");
+    expect(v.outcome).toBe("undecided");
+    expect(v.candidates!.map((c) => c.rule)).toEqual(["SQLCH216", "SQLCH220"]);
+    expect(v.changes.map((c) => c.rule)).toEqual(["SQLCH201"]);
+  });
+
+  test("a table created in the same file decides the key questions", () => {
+    const create = "CREATE TABLE events (user_id UInt64, ts DateTime, region String) ENGINE = MergeTree ORDER BY (user_id, ts)";
+    const [, appended] = classifyClickHouseStatements([create, "ALTER TABLE events ADD COLUMN extra String, MODIFY ORDER BY (user_id, ts, extra)"]);
+    expect(appended!.outcome).toBe("classified");
+    expect(appended!.changes.map((c) => c.rule)).toEqual(["SQLCH201", "SQLCH216"]);
+    expect(classifyClickHouseStatements([create, "ALTER TABLE events MODIFY COLUMN user_id UInt32"])[1]!.changes[0]!.rule).toBe("SQLCH211");
+    expect(classifyClickHouseStatements([create, "ALTER TABLE events MODIFY COLUMN region LowCardinality(String)"])[1]!.changes[0]!.rule).toBe("SQLCH210");
+  });
+
+  test("a column type change on a table the file does not create is undecided", () => {
+    const v = one("ALTER TABLE events MODIFY COLUMN user_id UInt32");
+    expect(v.outcome).toBe("undecided");
+    expect(v.candidates!.map((c) => c.rule)).toEqual(["SQLCH210", "SQLCH211"]);
+    expect(one("ALTER TABLE events DROP COLUMN a").candidates!.map((c) => c.rule)).toEqual(["SQLCH202", "SQLCH213"]);
+    expect(one("ALTER TABLE events RENAME COLUMN a TO b").candidates!.map((c) => c.rule)).toEqual(["SQLCH212", "SQLCH213"]);
+  });
+
+  test("metadata and rewrite changes", () => {
+    expect(rules("ALTER TABLE events ADD COLUMN IF NOT EXISTS a String DEFAULT ''")).toEqual(["SQLCH201"]);
+    expect(rules("ALTER TABLE events ON CLUSTER c MODIFY COLUMN a DEFAULT 'x'")).toEqual(["SQLCH207"]);
+    expect(rules("ALTER TABLE events MODIFY TTL ts + INTERVAL 30 DAY")).toEqual(["SQLCH205"]);
+    expect(rules("ALTER TABLE events ADD INDEX i a TYPE minmax GRANULARITY 1")).toEqual(["SQLCH204"]);
+    expect(rules("ALTER TABLE events MODIFY SETTING ttl_only_drop_parts = 1")).toEqual(["SQLCH206"]);
+    expect(rules("ALTER TABLE events MODIFY COMMENT 'x'")).toEqual(["SQLCH203"]);
+  });
+
+  test("create, drop, rename", () => {
+    expect(rules("CREATE TABLE t (a UInt8) ENGINE = MergeTree ORDER BY a")).toEqual(["SQLCH200"]);
+    expect(rules("CREATE MATERIALIZED VIEW mv TO t AS SELECT a FROM s")).toEqual(["SQLCH200"]);
+    expect(rules("CREATE USER u IDENTIFIED WITH no_password")).toEqual(["SQLCH200"]);
+    expect(one("DROP TABLE t").changes[0]).toMatchObject({ rule: "SQLCH250", destructive: true });
+    expect(rules("RENAME TABLE a TO b")).toEqual(["SQLCH230"]);
+    expect(rules("GRANT SELECT ON db.* TO r")).toEqual(["SQLCH273"]);
+  });
+
+  test("mutations are data; partition operations have no rule", () => {
+    expect(one("ALTER TABLE t UPDATE a = 1 WHERE b = 2").outcome).toBe("data");
+    expect(one("ALTER TABLE t DROP PARTITION 2024").outcome).toBe("unclassified");
+    expect(one("INSERT INTO t VALUES (1)").outcome).toBe("data");
+  });
+});

package/src/clickhouse/plan/statements.ts

@@ -0,0 +1,306 @@
+/**
+ * The ClickHouse statement classifier (#4407): a migration file's statements,
+ * each given the class and rule the schema-diff classifier (`./rules.ts`)
+ * gives the same change. It reads text and contacts no server.
+ *
+ * `CREATE` goes through the declaration parser (`../parser.ts`), which also
+ * tells the classifier a table's keys when the file creates the table. The
+ * parser has no `ALTER` grammar, so `ALTER`, `DROP` and `RENAME` are read word
+ * by word from the same tokens.
+ *
+ * A statement alone cannot decide some changes, and those answer `undecided`
+ * naming both rules, the way `./disruption.ts` answers `unknown`:
+ *
+ * - `MODIFY COLUMN c <type>` is SQLCH210 (a rewrite), or SQLCH211 (a rebuild)
+ *   when `c` is in a key. The file states the keys only when it creates the
+ *   table.
+ * - `DROP COLUMN` is SQLCH202 or SQLCH213, and `RENAME COLUMN` SQLCH212 or
+ *   SQLCH213, on the same condition.
+ * - `MODIFY ORDER BY` that the same `ALTER` pairs with `ADD COLUMN` is SQLCH216
+ *   when it only appends those columns and SQLCH220 (a rebuild) otherwise.
+ *   Without such a column it is SQLCH220.
+ */
+
+import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse";
+import { isTrivia, tokenizeText, type Token } from "../tokens";
+import { parseCreate, type CreateNode } from "../parser";
+import type { Span } from "../../core/cursor";
+import { classifiedOf, splitTopLevel, sqlHead, type StatementClassified, type StatementVerdict } from "../../core/statement-class";
+import { CHANGE_CLASSES, CLASSIFIER_RULES, type ClassifierRuleId } from "./rules";
+
+const classified = (id: ClassifierRuleId, extra: Partial<StatementClassified> = {}): StatementClassified => classifiedOf(CLASSIFIER_RULES, CHANGE_CLASSES.label, id, extra);
+
+type Reading = { rules: Array<{ id: ClassifierRuleId; extra?: Partial<StatementClassified> }> } | { undecided: ClassifierRuleId[]; note: string } | { none: string } | { data: string };
+
+const rule = (id: ClassifierRuleId, extra?: Partial<StatementClassified>): Reading => ({ rules: [{ id, ...(extra ? { extra } : {}) }] });
+
+/** Settings only a stream engine takes (the same list `./disruption.ts` reads). */
+const STREAM_SETTING = /^(?:kafka_|rabbitmq_|nats_|s3queue_|azure_queue_|keeper_path$)/;
+
+/** What a table the file creates tells later statements in it. */
+interface Created {
+  /** The columns in the sorting, primary, partition or sampling key. */
+  keys: Set<string>;
+  orderBy?: string;
+}
+
+const bare = (name: string): string => (name.split(".").pop() ?? name).replace(/^[`"]|[`"]$/g, "").toLowerCase();
+
+const text = (tokens: readonly Token[], s: Span | undefined): string => (s ? tokens.slice(s.from, s.to).filter((t) => !isTrivia(t)).map((t) => t.text).join(" ") : "");
+
+/** The identifier tokens of a key expression that are not function names. */
+function columnsOf(tokens: readonly Token[], s: Span | undefined): string[] {
+  if (!s) return [];
+  const toks = tokens.slice(s.from, s.to).filter((t) => !isTrivia(t));
+  return toks.flatMap((t, i) => ((t.kind === "ident" || t.kind === "qident") && toks[i + 1]?.text !== "(" ? [bare(t.text)] : []));
+}
+
+function nameAt(words: readonly string[], i: number): { name: string; next: number } {
+  let name = words[i] ?? "";
+  let j = i + 1;
+  while (words[j] === "." && words[j + 1] !== undefined) {
+    name += `.${words[j + 1]}`;
+    j += 2;
+  }
+  return { name, next: j };
+}
+
+const KNOWN_VERBS = new Set(["ADD", "DROP", "RENAME", "MODIFY", "COMMENT", "CLEAR", "MATERIALIZE", "UPDATE", "DELETE", "ATTACH", "DETACH", "FREEZE", "FETCH", "MOVE", "REPLACE", "RESET", "REMOVE", "APPLY", "UNFREEZE", "EXECUTE"]);
+
+/** Normalise a key expression for comparing an old and a new one: no spaces, no outer parentheses. */
+const keyText = (words: readonly string[]): string => {
+  let t = words.join("").toLowerCase();
+  while (t.startsWith("(") && t.endsWith(")")) t = t.slice(1, -1);
+  return t;
+};
+
+interface TableContext {
+  created?: Created;
+  /** Columns the same ALTER adds. */
+  added: Set<string>;
+}
+
+function columnVerdict(kind: "drop" | "rename", col: string, ctx: TableContext): Reading {
+  const plain: ClassifierRuleId = kind === "drop" ? "SQLCH202" : "SQLCH212";
+  const extra = kind === "drop" ? { destructive: true } : undefined;
+  if (ctx.created) {
+    return ctx.created.keys.has(bare(col)) ? rule("SQLCH213", extra) : rule(plain, extra);
+  }
+  return { undecided: [plain, "SQLCH213"], note: `${col} is a column change or, in a key, a rebuild; the file does not create the table, so its keys are not known offline` };
+}
+
+function tableAction(a: readonly string[], ctx: TableContext): Reading {
+  const u = a.map((w) => w.toUpperCase());
+  const first = u[0];
+  const second = u[1];
+  if (first === "ADD") {
+    if (second === "COLUMN") return rule("SQLCH201");
+    if (second === "INDEX") return rule("SQLCH204");
+    if (second === "PROJECTION") return rule("SQLCH214");
+    if (second === "CONSTRAINT") return rule("SQLCH215");
+    return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+  }
+  if (first === "DROP") {
+    if (second === "COLUMN") {
+      const k = u[2] === "IF" ? 4 : 2;
+      return columnVerdict("drop", a[k] ?? "column", ctx);
+    }
+    if (second === "INDEX") return rule("SQLCH204");
+    if (second === "PROJECTION") return rule("SQLCH214");
+    if (second === "CONSTRAINT") return rule("SQLCH215");
+    return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")} (removes data parts, no rule covers it)` };
+  }
+  if (first === "CLEAR") {
+    if (second === "INDEX") return rule("SQLCH204");
+    if (second === "PROJECTION") return rule("SQLCH214");
+    return { none: `ALTER TABLE CLEAR ${second ?? ""}`.trim() };
+  }
+  if (first === "RENAME" && second === "COLUMN") {
+    const k = u[2] === "IF" ? 4 : 2;
+    return columnVerdict("rename", a[k] ?? "column", ctx);
+  }
+  if (first === "COMMENT" && second === "COLUMN") return rule("SQLCH203");
+  if (first === "MODIFY" || first === "RESET") {
+    if (second === "COLUMN") {
+      let k = 2;
+      if (u[k] === "IF" && u[k + 1] === "EXISTS") k += 2;
+      const col = a[k] ?? "column";
+      const after = u[k + 1];
+      if (after === "REMOVE") {
+        const what = u[k + 2];
+        return what === "CODEC" ? rule("SQLCH208") : what === "COMMENT" ? rule("SQLCH203") : what === "TTL" ? rule("SQLCH205") : rule("SQLCH207");
+      }
+      if (after === "DEFAULT" || after === "MATERIALIZED" || after === "ALIAS" || after === "EPHEMERAL") return rule("SQLCH207");
+      if (after === "CODEC") return rule("SQLCH208");
+      if (after === "COMMENT") return rule("SQLCH203");
+      if (after === "TTL") return rule("SQLCH205");
+      if (after === "MODIFY" || after === "RESET") return { none: "MODIFY COLUMN ... SETTING" };
+      if (after === undefined) return { none: "MODIFY COLUMN with no change" };
+      if (ctx.created) return ctx.created.keys.has(bare(col)) ? rule("SQLCH211") : rule("SQLCH210");
+      return { undecided: ["SQLCH210", "SQLCH211"], note: `a type change of ${col} is a background rewrite or, when ${col} is in a key, a rebuild; the file does not create the table, so its keys are not known offline` };
+    }
+    if (second === "ORDER" && u[2] === "BY") {
+      const next = a.slice(3);
+      if (ctx.created?.orderBy !== undefined) {
+        const was = ctx.created.orderBy;
+        const now = keyText(next);
+        const tail = now.startsWith(was) ? now.slice(was.length) : undefined;
+        const extra = tail === undefined ? [] : tail.split(/[^a-z0-9_]+/).filter(Boolean);
+        if (tail !== undefined && extra.length > 0 && extra.every((c) => ctx.added.has(c))) return rule("SQLCH216");
+        return rule("SQLCH220");
+      }
+      const mentions = next.some((w) => ctx.added.has(bare(w)));
+      if (mentions) return { undecided: ["SQLCH216", "SQLCH220"], note: "the new key appends the columns this ALTER adds only if its other elements are the old key's, which the file does not state" };
+      return rule("SQLCH220");
+    }
+    if (second === "SAMPLE" && u[2] === "BY") return rule("SQLCH217");
+    if (second === "TTL") return rule("SQLCH205");
+    if (second === "COMMENT") return rule("SQLCH203");
+    if (second === "QUERY") return rule("SQLCH241");
+    if (second === "REFRESH") return rule("SQLCH244");
+    if (second === "SETTING" || second === "SETTINGS") {
+      const key = (a[2] ?? "").toLowerCase();
+      return settingRule(key);
+    }
+    return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+  }
+  if (first === "REMOVE") {
+    if (second === "TTL") return rule("SQLCH205");
+    if (second === "SAMPLE" && u[2] === "BY") return rule("SQLCH217");
+    return { none: `ALTER TABLE REMOVE ${second ?? ""}`.trim() };
+  }
+  if (first === "UPDATE" || first === "DELETE") return { data: "a mutation: it rewrites the parts it touches in the background (system.mutations) and has no rollback" };
+  return { none: `ALTER TABLE ${u.slice(0, 2).join(" ")}` };
+}
+
+function settingRule(key: string): Reading {
+  if (STREAM_SETTING.test(key)) return rule("SQLCH225", { note: `${key} belongs to a stream engine, which has no MODIFY SETTING` });
+  const spec = (MERGE_TREE_SETTINGS as Record<string, { readonly: boolean } | undefined>)[key];
+  return spec?.readonly ? rule("SQLCH218", { note: `${key} is fixed at creation` }) : rule("SQLCH206");
+}
+
+/** The `ALTER TABLE` actions of one statement, in order, a `MODIFY SETTING` list kept together. */
+function alterTable(words: readonly string[], u: readonly string[], created: Map<string, Created>): Reading[] {
+  let i = 2;
+  if (u[i] === "IF" && u[i + 1] === "EXISTS") i += 2;
+  const { name, next } = nameAt(words, i);
+  i = next;
+  if (u[i] === "ON" && u[i + 1] === "CLUSTER") i += 3;
+  const actions = splitTopLevel(words.slice(i));
+  const added = new Set<string>();
+  for (const a of actions) {
+    const au = a.map((w) => w.toUpperCase());
+    if (au[0] === "ADD" && au[1] === "COLUMN") {
+      const k = au[2] === "IF" ? 5 : 2;
+      if (a[k]) added.add(bare(a[k]!));
+    }
+  }
+  const ctx: TableContext = { added, ...(created.has(bare(name)) ? { created: created.get(bare(name))! } : {}) };
+  const out: Reading[] = [];
+  let inSettings = false;
+  for (const a of actions) {
+    const au = a.map((w) => w.toUpperCase());
+    if (inSettings && au[0] !== undefined && !KNOWN_VERBS.has(au[0])) {
+      out.push(settingRule((a[0] ?? "").toLowerCase()));
+      continue;
+    }
+    inSettings = (au[0] === "MODIFY" || au[0] === "RESET") && (au[1] === "SETTING" || au[1] === "SETTINGS");
+    out.push(tableAction(a, ctx));
+  }
+  return out;
+}
+
+const OBJECTS: Record<string, ClassifierRuleId> = { USER: "SQLCH271", ROLE: "SQLCH270", QUOTA: "SQLCH275" };
+
+function alterOther(u: readonly string[]): Reading {
+  const obj = u[1];
+  if (obj && OBJECTS[obj]) return rule(OBJECTS[obj]!);
+  if (obj === "ROW" && u[2] === "POLICY") return rule("SQLCH272");
+  if (obj === "POLICY") return rule("SQLCH272");
+  if (obj === "SETTINGS" && u[2] === "PROFILE") return rule("SQLCH276");
+  if (obj === "DATABASE" && u.includes("COMMENT")) return rule("SQLCH203");
+  return { none: `ALTER ${obj ?? ""}`.trim() };
+}
+
+const DATA_WORDS = new Set(["INSERT", "SELECT", "WITH", "SYSTEM", "OPTIMIZE", "TRUNCATE", "SET", "USE", "EXPLAIN", "EXISTS", "DESCRIBE", "DESC", "SHOW", "CHECK", "KILL", "WATCH"]);
+
+
+/** A `CREATE` or `ATTACH`: the parser reads tables, views, dictionaries, databases and functions; the rest go by key words. */
+function create(tokens: Token[], u: readonly string[], created: Map<string, Created>): Reading {
+  let i = 1;
+  const orReplace = u[i] === "OR" && u[i + 1] === "REPLACE";
+  if (orReplace) i += 2;
+  let node: CreateNode | undefined;
+  try {
+    node = parseCreate(tokens);
+  } catch {
+    node = undefined;
+  }
+  if (node?.statement === "table") {
+    const name = text(tokens, node.name).replace(/\s*\.\s*/g, ".");
+    const keys = new Set<string>([...columnsOf(tokens, node.orderBy), ...columnsOf(tokens, node.primaryKey), ...columnsOf(tokens, node.partitionBy), ...columnsOf(tokens, node.sampleBy)]);
+    for (const c of node.columns) if (c.primaryKey) keys.add(bare(c.name));
+    created.set(bare(name), { keys, ...(node.orderBy ? { orderBy: keyText(text(tokens, node.orderBy).split(" ")) } : {}) });
+  }
+  if (orReplace && u[i] === "FUNCTION") return rule("SQLCH260");
+  if (orReplace && (u[i] === "TABLE" || node?.statement === "table")) return rule("SQLCH200", { note: "OR REPLACE drops an existing table of that name, and its rows, before it creates this one" });
+  return rule("SQLCH200");
+}
+
+function read(sql: string, created: Map<string, Created>): Reading[] {
+  let tokens: Token[];
+  try {
+    tokens = tokenizeText(sql, 0);
+  } catch {
+    return [{ none: "text the tokenizer could not read" }];
+  }
+  const words = tokens.filter((t) => !isTrivia(t) && t.text !== ";").map((t) => t.text);
+  const u = words.map((w) => w.toUpperCase());
+  const head = u[0];
+  if (head === undefined) return [{ data: "an empty statement" }];
+  if (head === "CREATE") return [create(tokens, u, created)];
+  if (head === "ALTER" && u[1] === "TABLE") return alterTable(words, u, created);
+  if (head === "ALTER") return [alterOther(u)];
+  if (head === "DROP") return [rule("SQLCH250", { destructive: true })];
+  if (head === "RENAME") {
+    if (u[1] === "DATABASE") return [rule("SQLCH231")];
+    if (u[1] === "TABLE" || u[1] === "DICTIONARY") return [rule("SQLCH230")];
+    return [{ none: `RENAME ${u[1] ?? ""}`.trim() }];
+  }
+  if (head === "GRANT") return [rule("SQLCH273")];
+  if (head === "REVOKE") return [rule("SQLCH274")];
+  if (head === "EXCHANGE") return [{ none: "EXCHANGE TABLES swaps two tables; no rule covers it" }];
+  if (head === "TRUNCATE") return [{ data: "TRUNCATE removes every row of the table" }];
+  if (DATA_WORDS.has(head)) return [{ data: `${head} reads or writes rows or the session, not the schema` }];
+  return [{ none: `${head} is not a statement the classifier reads` }];
+}
+
+/**
+ * Classify a migration's up statements, in order. The statements are the
+ * file's own text, one per entry.
+ */
+export function classifyClickHouseStatements(statements: readonly string[]): StatementVerdict[] {
+  const created = new Map<string, Created>();
+  return statements.map((sql, k) => {
+    const readings = read(sql, created);
+    const out: StatementVerdict = { index: k + 1, sql: sqlHead(sql), outcome: "classified", changes: [] };
+    const none: string[] = [];
+    const notes: string[] = [];
+    for (const r of readings) {
+      if ("rules" in r) out.changes.push(...r.rules.map((x) => classified(x.id, x.extra)));
+      else if ("undecided" in r) {
+        out.candidates = [...(out.candidates ?? []), ...r.undecided.map((id) => classified(id))];
+        notes.push(r.note);
+      } else if ("none" in r) none.push(r.none);
+      else notes.push(r.data);
+    }
+    if (out.candidates) out.outcome = "undecided";
+    else if (out.changes.length === 0 && none.length === 0) out.outcome = "data";
+    else if (none.length > 0) {
+      out.outcome = out.changes.length > 0 ? "classified" : "unclassified";
+      notes.push(`no rule covers ${none.join("; ")}`);
+    }
+    if (notes.length > 0) out.note = notes.join("; ");
+    return out;
+  });
+}

package/src/clickhouse/plan/stream-s3queue.e2e.test.ts

@@ -0,0 +1,148 @@
+/**
+ * Replacing an S3Queue table against the pinned server (#4396): a change to
+ * the table's settings is planned as SQLCH225, the apply detaches the view,
+ * drops and creates the table and attaches the view, and the view keeps
+ * feeding the target with no file processed twice. A different keeper_path
+ * plans with the reprocessing warning.
+ *
+ * Needs Docker; skips cleanly without it. A one-server cluster (its embedded
+ * Keeper is what S3Queue records processed files in) and a MinIO container on
+ * its network as the object store, both removed afterwards even on failure.
+ * The credentials are MinIO's published test values, held in a named
+ * collection on the server (#4395) and never in the declaration.
+ */
+
+import { execFile } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { expandComposite } from "@intentius/chant/composite";
+import type { Declarable } from "@intentius/chant/declarable";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { dockerAvailable, startScratchCluster, type ScratchCluster } from "../container";
+import { clickhouseQuery } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { sqlSerializer } from "../../serializer";
+import { StreamIngest } from "../../composites";
+import { bindClickHouse } from "../live/bind";
+import { applyClickHouse, declaredObjects } from "../apply/apply";
+import { planAgainstServer } from "./commands";
+import { renderDiff } from "./report";
+
+const exec = promisify(execFile);
+const enabled = await dockerAvailable();
+let cluster: ScratchCluster | undefined;
+let minio: string | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-s3queue-"));
+const buildFile = join(dir, "files.json");
+
+const MINIO_IMAGE = "minio/minio:RELEASE.2025-04-22T22-12-26Z";
+const q = <T = unknown>(sql: string) => clickhouseQuery<T>(cluster!.replicas[0]!, sql);
+const env = () => ({ config: {}, env: { CLICKHOUSE_URL: cluster!.replicas[0]!.url } });
+
+beforeAll(async () => {
+  if (!enabled) return;
+  cluster = await startScratchCluster(clickhouseImage(), { replicas: 1, namePrefix: "chant-sql-s3queue" });
+  minio = `${cluster.name}-minio`;
+  await exec("docker", [
+    "run", "-d", "--name", minio, "--network", `${cluster.name}-net`,
+    "-e", "MINIO_ROOT_USER=minioadmin", "-e", "MINIO_ROOT_PASSWORD=minioadmin",
+    MINIO_IMAGE, "server", "/data",
+  ], { timeout: 600_000 });
+  await exec("docker", ["exec", minio, "mkdir", "-p", "/data/lake"]);
+  await q(`CREATE NAMED COLLECTION lake_creds AS url = 'http://${minio}:9000/lake/in/*.csv', access_key_id = 'minioadmin', secret_access_key = 'minioadmin'`);
+  await q("CREATE DATABASE shop");
+}, 900_000);
+
+afterAll(async () => {
+  if (minio) await exec("docker", ["rm", "-f", minio]).catch(() => undefined);
+  await cluster?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+/** The ingest chain with the S3Queue table's settings as given. */
+function buildOf(settings: string): string {
+  const entities = new Map<string, Declarable>();
+  for (const [k, v] of expandComposite(
+    "files",
+    StreamIngest({
+      name: "shop.files",
+      columns: "id UInt64, v String",
+      engine: "S3Queue(lake_creds, format = 'CSV')",
+      settings,
+      orderBy: "id",
+    }),
+  )) {
+    entities.set(k, v);
+  }
+  return (sqlSerializer.serialize(entities) as SerializerResult).primary;
+}
+
+const SETTINGS = (maxMs: number, path = "/chant/e2e/files") =>
+  `mode = 'unordered', keeper_path = '${path}', s3queue_polling_min_timeout_ms = 100, s3queue_polling_max_timeout_ms = ${maxMs}`;
+
+async function putFile(name: string, rows: string): Promise<void> {
+  await q(`INSERT INTO FUNCTION s3('http://${minio}:9000/lake/in/${name}', 'minioadmin', 'minioadmin', 'CSV', 'id UInt64, v String') VALUES ${rows}`);
+}
+
+async function waitForRows(n: number): Promise<void> {
+  const deadline = Date.now() + 90_000;
+  for (;;) {
+    const [row] = await q<{ n: string }>("SELECT count() AS n FROM shop.files");
+    if (Number(row?.n) >= n) return;
+    if (Date.now() > deadline) throw new Error(`shop.files held ${row?.n} rows, wanted ${n}`);
+    await new Promise((r) => setTimeout(r, 500));
+  }
+}
+
+async function apply(json: string) {
+  const target = await bindClickHouse({ environment: "e2e", ...env() });
+  return applyClickHouse(target, declaredObjects(json, "default", target.topology));
+}
+
+describe.skipIf(!enabled)("replacing an S3Queue table", () => {
+  test("the view keeps feeding the target, and no file is processed twice", async () => {
+    await apply(buildOf(SETTINGS(500)));
+    await putFile("f1.csv", "(1, 'a'), (2, 'b')");
+    await putFile("f2.csv", "(3, 'c')");
+    await waitForRows(3);
+    const [before] = await q<{ uuid: string }>("SELECT toString(uuid) AS uuid FROM system.tables WHERE database = 'shop' AND name = 'files_stream'");
+
+    // The settings change: planned as a stream-table replacement, not a rebuild, with nothing refused.
+    const next = buildOf(SETTINGS(1000));
+    writeFileSync(buildFile, next);
+    const diff = await planAgainstServer("e2e", buildFile, env());
+    expect(diff.changes.map((c) => [c.rule, c.field])).toEqual([["SQLCH225", "settings.s3queue_polling_max_timeout_ms"]]);
+    expect(diff.rebuilds).toEqual([]);
+    expect(diff.changes[0]!.note).toBeUndefined();
+    expect(diff.hints.join("\n")).toMatch(/shop\.files_stream is replaced: shop\.files_mv reads it/);
+
+    const outcome = await apply(next);
+    expect(outcome.failed).toEqual([]);
+    const replaced = outcome.applied.find((a) => a.name === "shop.files_stream");
+    expect(replaced?.statements.map((s) => s.split(" ").slice(0, 2).join(" "))).toEqual(["DETACH TABLE", "DROP TABLE", "CREATE TABLE", "ATTACH TABLE"]);
+
+    const [after] = await q<{ uuid: string }>("SELECT toString(uuid) AS uuid FROM system.tables WHERE database = 'shop' AND name = 'files_stream'");
+    expect(after!.uuid).not.toBe(before!.uuid);
+    const [view] = await q<{ n: string }>("SELECT count() AS n FROM system.tables WHERE database = 'shop' AND name = 'files_mv'");
+    expect(Number(view!.n)).toBe(1);
+
+    // A file that lands after the replacement is read by the new table through the same view.
+    await putFile("f3.csv", "(4, 'd'), (5, 'e')");
+    await waitForRows(5);
+    // Give the new table the chance to read f1 and f2 again, which it must not.
+    await new Promise((r) => setTimeout(r, 5000));
+    const [rows] = await q<{ n: string; distinct: string }>("SELECT count() AS n, uniqExact(id) AS distinct FROM shop.files");
+    expect([Number(rows!.n), Number(rows!.distinct)]).toEqual([5, 5]);
+    expect((await planAgainstServer("e2e", buildFile, env())).changes).toEqual([]);
+  }, 300_000);
+
+  test("a different keeper_path plans with the reprocessing warning", async () => {
+    writeFileSync(buildFile, buildOf(SETTINGS(1000, "/chant/e2e/files-v2")));
+    const diff = await planAgainstServer("e2e", buildFile, env());
+    expect(diff.changes.map((c) => [c.rule, c.field])).toEqual([["SQLCH225", "settings.keeper_path"]]);
+    expect(renderDiff(diff)).toMatch(/every file already seen is processed again/);
+  }, 120_000);
+});