@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/clickhouse/named-collections.test.ts

@@ -0,0 +1,187 @@
+/**
+ * Named collections referenced by name (#4395): the build records each one a
+ * declaration names, a plan checks each exists by name and never reads the
+ * values, and import lists the server's as prerequisites instead of refusing
+ * the server.
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { makePostSynthCtx } from "@intentius/chant-test-utils";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { sqlSerializer } from "../serializer";
+import { database, dictionary, table, view } from "./entities";
+import { postSynthChecks } from "../lint/post-synth/index";
+import { COLLECTION_ENGINES, COLLECTION_FUNCTIONS, createdCollectionName, functionCollections, namedCollectionReferences, sourceCollections, statementCollections } from "./named-collections";
+import { checkNamedCollections, referencedCollections, renderNamedCollections } from "./live/named-collections";
+import { renderDiff } from "./plan/report";
+import { fakeClickHouse, type FakeServer } from "./testing/fake-server";
+import { exportResources } from "./import/live-export";
+import { SqlFileParser } from "../import-parser";
+import { KIND_COVERAGE, kindCoverageProblems } from "../coverage";
+import { CLICKHOUSE_PROBES } from "./live/undeclarable";
+
+type Doc = { namedCollections?: string[]; objects: Array<{ export: string; namedCollections?: string[] }> };
+
+const serialize = (entities: Record<string, unknown>) => sqlSerializer.serialize(new Map(Object.entries(entities)) as never) as SerializerResult;
+
+const lake = table`CREATE TABLE lake (a Int32) ENGINE = S3(lake_creds, url = 'http://minio:9000/bucket/x.parquet', format = 'Parquet')`;
+
+describe("a declaration names a collection", () => {
+  test("ENGINE = S3(lake_creds, url = '...', format = 'Parquet') builds and records lake_creds", () => {
+    const out = serialize({ lake });
+    const doc = JSON.parse(out.primary) as Doc;
+    expect(doc.namedCollections).toEqual(["lake_creds"]);
+    expect(doc.objects[0]).toMatchObject({ export: "lake", namedCollections: ["lake_creds"] });
+    // No post-synth check objects to it.
+    const ctx = makePostSynthCtx("sql", out.primary);
+    expect(postSynthChecks.flatMap((c) => c.check(ctx)).filter((d) => d.severity === "error")).toEqual([]);
+  });
+
+  test("an engine, a database engine, a table function, a view's SELECT and a dictionary SOURCE each name one", () => {
+    const pg = database`CREATE DATABASE pg ENGINE = PostgreSQL(pg_creds)`;
+    const kafka = table`CREATE TABLE k (a String) ENGINE = Kafka(kafka_creds, kafka_format = 'JSONEachRow')`;
+    const over = table`CREATE TABLE over AS s3(\`lake creds\`, filename = 'x.parquet')`;
+    const v = view`CREATE VIEW v AS SELECT * FROM mysql(mysql_creds, table = 'orders')`;
+    const d = dictionary`CREATE DICTIONARY d (id UInt64, name String) PRIMARY KEY id SOURCE(MYSQL(NAME mysql_creds TABLE 'names')) LAYOUT(HASHED()) LIFETIME(300)`;
+    const doc = JSON.parse(serialize({ pg, kafka, over, v, d }).primary) as Doc;
+    expect(doc.namedCollections).toEqual(["kafka_creds", "lake creds", "mysql_creds", "pg_creds"]);
+    expect(Object.fromEntries(doc.objects.map((o) => [o.export, o.namedCollections]))).toEqual({
+      pg: ["pg_creds"],
+      kafka: ["kafka_creds"],
+      over: ["lake creds"],
+      v: ["mysql_creds"],
+      d: ["mysql_creds"],
+    });
+  });
+
+  test("a build that names none records nothing", () => {
+    const t = table`CREATE TABLE t (a Int32) ENGINE = S3('http://minio:9000/bucket/x.parquet', 'Parquet')`;
+    const plain = table`CREATE TABLE p (a Int32) ENGINE = MergeTree ORDER BY a`;
+    const doc = JSON.parse(serialize({ t, plain }).primary) as Doc;
+    expect(doc.namedCollections).toBeUndefined();
+    expect(doc.objects.map((o) => o.namedCollections)).toEqual([undefined, undefined]);
+  });
+
+  test("a first argument that is a string, NOSIGN, a cluster or a column is not a collection", () => {
+    expect(namedCollectionReferences({ engine: { name: "S3", args: ["NOSIGN"] } })).toEqual([]);
+    expect(namedCollectionReferences({ engine: { name: "Distributed", args: ["main", "db", "t"] } })).toEqual([]);
+    expect(namedCollectionReferences({ engine: { name: "ReplacingMergeTree", args: ["ver"] } })).toEqual([]);
+    expect(functionCollections("SELECT * FROM s3Cluster(main, 'http://x/y', 'CSV')")).toEqual([]);
+    expect(functionCollections("SELECT * FROM remote('h:9000', db, t)")).toEqual([]);
+    expect(functionCollections("SELECT x.s3(a) FROM t")).toEqual([]);
+  });
+
+  test("the engines and table functions listed are ones the pinned server has", () => {
+    for (const e of ["S3", "Kafka", "MySQL", "PostgreSQL", "MongoDB", "AzureBlobStorage", "RabbitMQ", "NATS"]) expect(COLLECTION_ENGINES, e).toContain(e);
+    for (const f of ["s3", "mysql", "postgresql", "url", "remote", "azureblobstorage"]) expect(COLLECTION_FUNCTIONS, f).toContain(f);
+    expect([...COLLECTION_FUNCTIONS].some((f) => f.endsWith("cluster"))).toBe(false);
+  });
+
+  test("a statement as the server prints it", () => {
+    expect(statementCollections("CREATE TABLE analytics.lake\n(\n    `a` Int32\n)\nENGINE = S3(lake_creds, url = 'http://x/y', format = 'Parquet')")).toEqual(["lake_creds"]);
+    expect(statementCollections("CREATE DICTIONARY d (`id` UInt64, `name` String) PRIMARY KEY id SOURCE(MYSQL(NAME 'mysql_creds')) LAYOUT(HASHED()) LIFETIME(300)")).toEqual(["mysql_creds"]);
+    // A column called name is not a source's NAME.
+    expect(statementCollections("CREATE TABLE t (`name` String) ENGINE = MergeTree ORDER BY name")).toEqual([]);
+    expect(sourceCollections("CLICKHOUSE(TABLE 'rates')")).toEqual([]);
+    expect(createdCollectionName("CREATE NAMED COLLECTION IF NOT EXISTS `lake creds` AS url = 'x'")).toBe("lake creds");
+  });
+});
+
+describe("a plan checks each exists, by name", () => {
+  let server: FakeServer;
+  beforeAll(async () => {
+    server = await fakeClickHouse([], {});
+  });
+  afterAll(() => server.close());
+
+  test("the build output's list is what is checked", () => {
+    expect(referencedCollections(serialize({ lake }).primary)).toEqual(["lake_creds"]);
+  });
+
+  test("it reads names only, never a collection's values", async () => {
+    await checkNamedCollections({ url: server.url }, ["lake_creds", "kafka_creds"]);
+    const sent = server.queries.filter((q) => q.includes("named_collections"));
+    expect(sent).toEqual(["/* chant plan: named collections */ SELECT name FROM system.named_collections WHERE name IN ('kafka_creds', 'lake_creds')"]);
+  });
+
+  test("one the server does not have fails the plan, naming it", () => {
+    const check = { referenced: ["kafka_creds", "lake_creds"], missing: ["lake_creds"] };
+    const text = renderDiff({ changes: [], rebuilds: [], hints: [], namedCollections: check });
+    expect(text).toContain("No changes.");
+    expect(text).toMatch(/Refused: the build references a named collection the server does not have: lake_creds\./);
+    expect(renderNamedCollections({ referenced: ["lake_creds"], missing: [] })).toEqual([]);
+  });
+
+  test("a server whose collections cannot be read fails the plan too", async () => {
+    const failing = await fakeClickHouse([], { fail: { status: 500, body: "Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED)" } });
+    try {
+      const check = await checkNamedCollections({ url: failing.url }, ["lake_creds"]);
+      expect(check).toMatchObject({ referenced: ["lake_creds"], missing: ["lake_creds"], unread: expect.stringMatching(/ACCESS_DENIED/) });
+      expect(renderNamedCollections(check).join("\n")).toMatch(/could not be read to check them/);
+    } finally {
+      await failing.close();
+    }
+  });
+});
+
+describe("import lists the server's as prerequisites", () => {
+  let server: FakeServer;
+  beforeAll(async () => {
+    server = await fakeClickHouse(
+      [
+        { name: "analytics", engine: "Atomic", statement: "CREATE DATABASE analytics\nENGINE = Atomic" },
+        {
+          database: "analytics",
+          name: "lake",
+          engine: "S3",
+          statement: "CREATE TABLE analytics.lake\n(\n    `a` Int32\n)\nENGINE = S3(lake_creds, url = 'http://minio:9000/bucket/x.parquet', format = 'Parquet')",
+        },
+      ],
+      { undeclarable: [{ kind: "named collection", name: "lake_creds" }, { kind: "named collection", name: "spare" }] },
+    );
+  });
+  afterAll(() => server.close());
+
+  test("a server with SQL-defined named collections imports, and lists each with what names it", async () => {
+    const ir = await exportResources({ environment: "prod", config: {}, env: { CLICKHOUSE_URL: server.url } });
+    expect(ir.resources.map((r) => r.properties.name)).toEqual(["analytics", "lake"]);
+    expect(ir.metadata?.prerequisites).toEqual([
+      { kind: "named collection", name: "lake_creds", referencedBy: ["analytics.lake"] },
+      { kind: "named collection", name: "spare", referencedBy: [] },
+    ]);
+    const warnings = (ir.warnings ?? []).filter((w) => w.startsWith("named collection"));
+    expect(warnings).toEqual([
+      expect.stringMatching(/^named collection lake_creds is a prerequisite of prod, named by analytics\.lake: the server holds it, and chant does not declare it or read its values/),
+      expect.stringMatching(/^named collection spare is a prerequisite of prod: the server holds it/),
+    ]);
+    // Names only: no query reads a collection's keys, values or create query.
+    const reads = server.queries.filter((q) => q.includes("named_collections"));
+    expect(reads).toHaveLength(1);
+    expect(reads[0]).toMatch(/SELECT database, name FROM \(SELECT '' AS database, name FROM system\.named_collections WHERE source = 'SQL'\)/);
+  });
+
+  test("a .sql file's CREATE NAMED COLLECTION is a prerequisite, not a refusal", () => {
+    const ir = new SqlFileParser("clickhouse").parse(
+      "CREATE DATABASE a;\nCREATE NAMED COLLECTION lake_creds AS url = 'x', secret_access_key = 'y';\nCREATE TABLE a.t (x Int32) ENGINE = S3(lake_creds, format = 'Parquet')",
+    );
+    expect(ir.resources.map((r) => r.type)).toEqual(["ClickHouse::Database", "ClickHouse::Table"]);
+    expect(ir.metadata?.prerequisites).toEqual([{ kind: "named collection", name: "lake_creds", referencedBy: ["a.t"] }]);
+    expect(ir.warnings).toEqual([expect.stringMatching(/^named collection lake_creds is a prerequisite, named by a\.t: the DDL creates it, and chant does not declare it/)]);
+    // Its values are not carried anywhere.
+    expect(JSON.stringify(ir)).not.toContain("secret_access_key");
+  });
+});
+
+describe("the coverage table's referenced status", () => {
+  test("a named collection is referenced, with a reason and a live query", () => {
+    const row = KIND_COVERAGE.clickhouse.find((r) => r.kind === "named collection");
+    expect(row).toMatchObject({ status: "referenced" });
+    expect(row?.reason).toMatch(/never declares one or reads its values/);
+    expect(Object.keys(CLICKHOUSE_PROBES)).toContain("named collection");
+  });
+
+  test("a referenced row needs a reason, as a refused one does", () => {
+    const rows = [{ command: "CREATE NAMED COLLECTION", kind: "named collection", status: "referenced" as const }];
+    expect(kindCoverageProblems("clickhouse", { pins: [{ version: "x", commands: ["CREATE NAMED COLLECTION"] }], rows })).toEqual(["named collection is referenced, and gives no reason"]);
+  });
+});

package/src/clickhouse/named-collections.ts

@@ -0,0 +1,224 @@
+/**
+ * Named collections, referenced by name (#4395).
+ *
+ * A named collection holds an integration's connection settings and its
+ * credentials. Those are the environment's, so chant never declares one and
+ * never reads its values (the coverage table's `referenced` status,
+ * `../coverage/clickhouse.json`). A declaration may name one where the server
+ * takes a collection in place of inline arguments:
+ *
+ * - an integration engine's first argument: `ENGINE = S3(lake_creds, format = 'Parquet')`,
+ *   `ENGINE = Kafka(kafka_creds)`, a database's `ENGINE = PostgreSQL(pg_creds)`;
+ * - a table function's first argument: `AS s3(lake_creds)`, `SELECT ... FROM mysql(mysql_creds, table = 't')`;
+ * - a dictionary source's `NAME`: `SOURCE(MYSQL(NAME mysql_creds))`.
+ *
+ * The build records each reference (`namedCollections` on the object and on
+ * the document, `../serializer.ts`), and `chant sql plan` checks that every
+ * one exists on the server by name (`./plan/commands.ts`).
+ */
+
+import { DATABASE_ENGINES, TABLE_ENGINES, TABLE_FUNCTIONS } from "../generated/clickhouse";
+import { unquoteWith } from "../core/cursor";
+import { isTrivia, tokenizeText, type Token } from "./tokens";
+
+/**
+ * The table and database engines whose first argument may be a named
+ * collection, as the 26.8 reference documents them for each engine.
+ * Filtered to the engines the pinned server has.
+ */
+const COLLECTION_ENGINE_NAMES = [
+  "S3",
+  "S3Queue",
+  "AzureBlobStorage",
+  "AzureQueue",
+  "Iceberg",
+  "IcebergS3",
+  "IcebergAzure",
+  "DeltaLake",
+  "DeltaLakeS3",
+  "DeltaLakeAzure",
+  "Hudi",
+  "Paimon",
+  "PaimonS3",
+  "HDFS",
+  "URL",
+  "Kafka",
+  "RabbitMQ",
+  "NATS",
+  "MySQL",
+  "PostgreSQL",
+  "MaterializedPostgreSQL",
+  "MongoDB",
+  "Redis",
+  "ArrowFlight",
+  "YTsaurus",
+  "DataLakeCatalog",
+];
+
+const ENGINES = { ...TABLE_ENGINES, ...DATABASE_ENGINES } as Readonly<Record<string, unknown>>;
+
+/** The engines a named collection may be the first argument of. */
+export const COLLECTION_ENGINES: ReadonlySet<string> = new Set(COLLECTION_ENGINE_NAMES.filter((n) => n in ENGINES));
+
+/**
+ * The table functions a named collection may be the first argument of, lower
+ * case. A `...Cluster` function's first argument is the cluster, so those are
+ * left out.
+ */
+export const COLLECTION_FUNCTIONS: ReadonlySet<string> = new Set(
+  TABLE_FUNCTIONS.filter(
+    (f) => !/Cluster$/.test(f) && /^(s3|gcs|oss|cosn|url|hdfs|mysql|postgresql|mongodb|redis|remote|azureblobstorage|iceberg|deltalake|hudi|paimon|arrowflight|ytsaurus)/i.test(f),
+  ).map((f) => f.toLowerCase()),
+);
+
+/** `lake_creds` or `` `lake creds` ``: an identifier, not a string, a call or `key = value`. */
+function identifierName(text: string): string | undefined {
+  const t = text.trim();
+  if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(t)) return t.toUpperCase() === "NOSIGN" ? undefined : t;
+  if (/^`(?:[^`\\]|\\.)+`$/.test(t)) return unquoteWith(t, "`");
+  return undefined;
+}
+
+/** The collection an engine names in its first argument, when it is one of the engines that take one. */
+export function engineCollection(engine: { name: string; args?: readonly string[] } | undefined): string | undefined {
+  if (!engine || !COLLECTION_ENGINES.has(engine.name) || !engine.args || engine.args.length === 0) return undefined;
+  return identifierName(engine.args[0]!);
+}
+
+const significant = (text: string): Token[] => {
+  try {
+    return tokenizeText(text, 0).filter((t) => !isTrivia(t));
+  } catch {
+    return [];
+  }
+};
+
+/** The collections a SQL text's table function calls name: `s3(lake_creds, ...)`, `mysql(mysql_creds)`. */
+export function functionCollections(text: string): string[] {
+  const tokens = significant(text);
+  const out: string[] = [];
+  for (let i = 0; i + 3 < tokens.length; i++) {
+    const [fn, open, arg, after] = [tokens[i]!, tokens[i + 1]!, tokens[i + 2]!, tokens[i + 3]!];
+    if (fn.kind !== "ident" || !COLLECTION_FUNCTIONS.has(fn.text.toLowerCase())) continue;
+    if (open.kind !== "punct" || open.text !== "(") continue;
+    if (tokens[i - 1]?.kind === "punct" && tokens[i - 1]!.text === ".") continue;
+    if (arg.kind !== "ident" && arg.kind !== "qident") continue;
+    if (after.kind !== "punct" || (after.text !== "," && after.text !== ")")) continue;
+    const name = identifierName(arg.text);
+    if (name) out.push(name);
+  }
+  return out;
+}
+
+/** The collection a dictionary source names: `MYSQL(NAME mysql_creds)`. */
+export function sourceCollections(dataSource: string): string[] {
+  const tokens = significant(dataSource);
+  const out: string[] = [];
+  for (let i = 0; i + 1 < tokens.length; i++) {
+    if (tokens[i]!.kind !== "ident" || tokens[i]!.text.toUpperCase() !== "NAME") continue;
+    const v = tokens[i + 1]!;
+    if (v.kind === "ident" || v.kind === "qident") out.push(v.kind === "qident" ? unquoteWith(v.text, '`"') : v.text);
+    else if (v.kind === "string") out.push(unquoteWith(v.text, "'"));
+  }
+  return out;
+}
+
+/** What a declaration's props hold that can name a collection. */
+export interface CollectionReferrer {
+  engine?: { name: string; args?: readonly string[] };
+  asFunction?: string;
+  select?: string;
+  dataSource?: string;
+}
+
+/** Every named collection an object names, once each, in the order written. */
+export function namedCollectionReferences(o: CollectionReferrer): string[] {
+  const names = [
+    ...(engineCollection(o.engine) !== undefined ? [engineCollection(o.engine)!] : []),
+    ...(o.asFunction ? functionCollections(o.asFunction) : []),
+    ...(o.select ? functionCollections(o.select) : []),
+    ...(o.dataSource ? sourceCollections(o.dataSource) : []),
+  ];
+  return [...new Set(names)];
+}
+
+/** `CREATE NAMED COLLECTION [IF NOT EXISTS] name ...`: the name, or undefined for another statement. */
+export function createdCollectionName(sql: string): string | undefined {
+  const m = /^\s*CREATE\s+NAMED\s+COLLECTION\s+(?:IF\s+NOT\s+EXISTS\s+)?(`(?:[^`\\]|\\.)+`|"(?:[^"\\]|\\.)+"|[A-Za-z_][A-Za-z0-9_]*)/i.exec(sql);
+  return m ? unquoteWith(m[1]!, '`"') : undefined;
+}
+
+/**
+ * The collections a whole statement names, read off its tokens: what
+ * {@link namedCollectionReferences} reads from a parsed declaration, for a
+ * statement import has only as text (`SHOW CREATE`).
+ */
+export function statementCollections(ddl: string): string[] {
+  const tokens = significant(ddl);
+  const out: string[] = [];
+  for (let i = 0; i < tokens.length; i++) {
+    const t = tokens[i]!;
+    if (t.kind !== "ident") continue;
+    const word = t.text.toUpperCase();
+    if (word === "ENGINE" && tokens[i + 1]?.text === "=" && tokens[i + 3]?.text === "(") {
+      const engine = tokens[i + 2]!;
+      const arg = tokens[i + 4];
+      const after = tokens[i + 5];
+      if (COLLECTION_ENGINES.has(engine.text) && arg && (arg.kind === "ident" || arg.kind === "qident") && (after?.text === "," || after?.text === ")")) {
+        const name = identifierName(arg.text);
+        if (name) out.push(name);
+      }
+    } else if (word === "SOURCE" && tokens[i + 1]?.text === "(") {
+      let depth = 0;
+      let j = i + 1;
+      for (; j < tokens.length; j++) {
+        if (tokens[j]!.text === "(") depth++;
+        else if (tokens[j]!.text === ")" && --depth === 0) break;
+      }
+      out.push(...sourceCollections(tokens.slice(i + 2, j).map((x) => x.text).join(" ")));
+      i = j;
+    }
+  }
+  out.push(...functionCollections(ddl));
+  return [...new Set(out)];
+}
+
+/** One environment prerequisite an import found: a named collection (#4395). */
+export interface Prerequisite {
+  kind: "named collection";
+  name: string;
+  /** The imported objects that name it, `database.name`. */
+  referencedBy: string[];
+}
+
+/**
+ * The named collections an import found as prerequisites (#4395), each with
+ * the imported objects that name it, and a warning for each. A collection an
+ * imported object names that was not found is listed too: a live reader
+ * without `SHOW NAMED COLLECTIONS` sees none.
+ */
+export function namedCollectionPrerequisites(
+  found: ReadonlyArray<{ name: string }>,
+  objects: ReadonlyArray<{ database?: string; name: string; ddl: string }>,
+  wording: { of?: string; found: string; unseen: string },
+): { prerequisites: Prerequisite[]; warnings: string[] } {
+  const by = new Map<string, string[]>(found.map((o) => [o.name, []]));
+  for (const o of objects) {
+    for (const c of statementCollections(o.ddl)) {
+      if (!by.has(c)) by.set(c, []);
+      by.get(c)!.push(o.database ? `${o.database}.${o.name}` : o.name);
+    }
+  }
+  const prerequisites: Prerequisite[] = [...by]
+    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+    .map(([name, referencedBy]) => ({ kind: "named collection", name, referencedBy }));
+  const seen = new Set(found.map((o) => o.name));
+  const warnings = prerequisites.map((p) => {
+    const used = p.referencedBy.length > 0 ? `, named by ${p.referencedBy.join(", ")}` : "";
+    return (
+      `named collection ${p.name} is a prerequisite${wording.of ? ` of ${wording.of}` : ""}${used}: ${seen.has(p.name) ? wording.found : wording.unseen}, ` +
+      "and chant does not declare it or read its values. Create it on every server the project applies to; chant sql plan checks that it exists"
+    );
+  });
+  return { prerequisites, warnings };
+}

package/src/clickhouse/parser.ts

@@ -227,7 +227,8 @@ export const REFUSED_CREATE_KINDS: ReadonlyArray<{ words: readonly string[]; kin
   {
     words: ["NAMED", "COLLECTION"],
     kind: "NAMED COLLECTION",
-    reason: "a named collection holds credentials (a password, a secret key), which are the environment's and never in a build; the server hides its values, so a plan could not compare them either. Create it from the server's configuration or by hand",
+    reason:
+      "a named collection holds credentials (a password, a secret key), which are the environment's and never in a build. Create it on the server, and name it where it is used: ENGINE = S3(lake_creds, format = 'Parquet'), s3(lake_creds), SOURCE(MYSQL(NAME mysql_creds)). A plan checks that it exists",
   },
   { words: ["MASKING", "POLICY"], kind: "MASKING POLICY", reason: "masking policies are available only in ClickHouse Cloud, not on the pinned server" },
   { words: ["WORKLOAD"], kind: "WORKLOAD", reason: "a workload is server-level scheduling, configured with the server, not a schema object" },

package/src/clickhouse/plan/commands.ts

@@ -13,7 +13,8 @@
  * expressions the rules leave different. Both exit 2 when a change needs a
  * rebuild, which a plan refuses to make in place, and `plan` also when the
  * declared databases hold an object whose definition it cannot read (#3653),
- * which it names; 0 otherwise. For each
+ * which it names, or when the build references a named collection the
+ * server does not have (#4395), which it names; 0 otherwise. For each
  * refused table they name the `ClickHouseRebuildOp` to run instead
  * (`rebuildOps` in `--json`, `./rebuild-handoff.ts`).
  *
@@ -40,6 +41,7 @@ import { bindClickHouse } from "../live/bind";
 import { rebuildOpSuggestions } from "./rebuild-handoff";
 import { readFileSync } from "node:fs";
 import { parseTopology, type Topology } from "../topology";
+import { checkNamedCollections, namedCollectionsFail, referencedCollectionsOf } from "../live/named-collections";
 
 /** The dialect a build output names. */
 function outputDialect(path: string): string | undefined {
@@ -90,7 +92,7 @@ async function runDiffStatements(before: string, after: string, json: boolean, t
 
 function emit(diff: SchemaDiff, json: boolean, title: string): number {
   console.log(json ? JSON.stringify(diff, null, 2) : renderDiff(diff, { title }));
-  return diff.rebuilds.length > 0 || (diff.unreadable?.length ?? 0) > 0 ? 2 : 0;
+  return diff.rebuilds.length > 0 || (diff.unreadable?.length ?? 0) > 0 || namedCollectionsFail(diff.namedCollections) ? 2 : 0;
 }
 
 export async function runDiff(ctx: CommandGroupContext): Promise<number> {
@@ -134,6 +136,8 @@ export async function planAgainstServer(environment: string, buildFile: string,
     new Map(declared.map((o) => [o.key, o.canonical])),
     environment,
   );
+  // Each named collection the build references must exist on the server, by name; its values are never read (#4395).
+  const namedCollections = await checkNamedCollections(target.endpoint, referencedCollectionsOf(buildFile));
   const label = new Map(declared.map((o) => [o.key, o.exportName === o.key ? o.key : `${o.exportName} (${o.key})`]));
   const relabel = changes.map((c) => ({ ...c, object: label.get(c.object) ?? c.object }));
   return {
@@ -142,6 +146,7 @@ export async function planAgainstServer(environment: string, buildFile: string,
     hints: unmanaged > 0 ? [...diff.hints, accessUnmanagedHint(unmanaged, environment)] : diff.hints,
     ...(rebuildOps.length > 0 ? { rebuildOps } : {}),
     ...(unreadable.length > 0 ? { unreadable } : {}),
+    ...(namedCollections ? { namedCollections } : {}),
   };
 }
 
@@ -170,5 +175,15 @@ export const sqlCommands: CommandGroup = {
       description: "Write sql-types.d.ts: typed columns for each table, view and dictionary the build declares, for tags given their name (table<\"app.users\">) and RowOf; --output <path> writes it elsewhere; --check compares instead of writing and exits 1 when the file differs",
       handler: async (ctx) => (await import("../../sql-types-command")).runTypes(ctx),
     },
+    {
+      name: "replay",
+      description: "Replay a goose, golang-migrate, clickhouse-migrations or Flyway directory into the empty server --env binds, and record the tables and columns it makes as chant.replay.json; --check compares instead of writing",
+      handler: async (ctx) => (await import("../../migration-replay")).runReplay(ctx),
+    },
+    {
+      name: "drizzle",
+      description: "Write a Drizzle schema module (drizzle-schema.ts) from a Postgres build: pgTable per table with columns, keys, indexes, enums, sequences and views, so an application queries with Drizzle's row types while chant owns the schema; --output <path> writes it elsewhere; --check compares instead of writing and exits 1 when the file differs; --bigint bigint reads bigint columns as bigint, not number",
+      handler: async (ctx) => (await import("../../drizzle-command")).runDrizzle(ctx),
+    },
   ],
 };

package/src/clickhouse/plan/diff.ts

@@ -20,6 +20,7 @@ import { CLASSIFIER_RULES, type ChangeClass, type ClassifierRuleId } from "./rul
 import type { CanonicalColumn, CanonicalObject } from "./normalize";
 import type { RebuildOpSuggestion } from "./rebuild-handoff";
 import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse";
+import { isStreamEngine, isStreamTable, positionWarnings, readsTable } from "./stream";
 import { classifiedChange, type ChangeSet, type ClassifiedChange } from "../../core/classifier";
 import { changedEntries, matchByIdentity, type Keyed } from "../../core/diff";
 
@@ -38,6 +39,22 @@ export interface SchemaDiff extends ChangeSet<Change> {
    * it out.
    */
   unreadable?: UnreadableEntry[];
+  /**
+   * The named collections the build references, checked against the server
+   * by name (#4395). A plan with one `missing`, or whose check could not run
+   * (`unread`), fails naming it. Their values are never read.
+   */
+  namedCollections?: NamedCollectionCheck;
+}
+
+/** What a plan found of the named collections a build references: each by name, never its values. */
+export interface NamedCollectionCheck {
+  /** Every collection the build names, sorted. */
+  referenced: string[];
+  /** Those the server has no collection of, or that the plan's user may not see. */
+  missing: string[];
+  /** Why the server's collections could not be read, when they could not. */
+  unread?: string;
 }
 
 /** An object a plan could not read: `database.name`, its type and why. */
@@ -228,8 +245,14 @@ function diffObject(key: string, before: CanonicalObject, after: CanonicalObject
     return;
   }
 
+  // A stream table (#4396): replaced, not altered, and not rebuilt, since it holds no rows.
+  if (isStreamTable(before) && isStreamTable(after)) {
+    diffStreamTable(key, before, after, out);
+    return;
+  }
+
   // A table.
-  if (before.engine !== after.engine) out.push(change(key, "engine", "SQLCH223", before.engine, after.engine));
+  if (before.engine !== after.engine) out.push(change(key, "engine", "SQLCH223", before.engine, after.engine, isStreamEngine(before.engineName) || isStreamEngine(after.engineName) ? { note: "one side is a stream engine and the other is not, which is a different kind of table: neither the replace nor the rebuild applies" } : {}));
   const added = diffColumns(key, before, after, out, hints);
 
   const beforePk = before.primaryKey ?? before.orderBy;
@@ -260,6 +283,28 @@ function diffObject(key: string, before: CanonicalObject, after: CanonicalObject
   diffMap(key, "constraints", before.constraints, after.constraints, "SQLCH215", out);
 }
 
+/**
+ * A table on a stream engine, on both sides (#4396). Its engine, its settings
+ * and its columns are all replaced the same way (SQLCH225); the comment is
+ * compared before this. A change that loses the table's position says so in
+ * its note.
+ */
+function diffStreamTable(key: string, before: CanonicalObject, after: CanonicalObject, out: Change[]): void {
+  const start = out.length;
+  if (before.engine !== after.engine) out.push(change(key, "engine", "SQLCH225", before.engine, after.engine));
+  for (const name of changedEntries(before.settings, after.settings)) out.push(change(key, `settings.${name}`, "SQLCH225", before.settings[name], after.settings[name]));
+  const column = (c: CanonicalColumn) => [c.type, c.nullable ? "NULL" : "", c.defaultKind ?? "", c.defaultExpr ?? "", c.codec ? `CODEC(${c.codec})` : ""].filter(Boolean).join(" ");
+  const b = new Map(before.columns.map((c) => [c.name, column(c)]));
+  const a = new Map(after.columns.map((c) => [c.name, column(c)]));
+  for (const name of new Set([...b.keys(), ...a.keys()])) {
+    if (b.get(name) !== a.get(name)) out.push(change(key, `columns.${name}`, "SQLCH225", b.get(name), a.get(name)));
+  }
+  const order = (o: CanonicalObject) => o.columns.map((c) => c.name).join(", ");
+  if (order(before) !== order(after) && a.size === b.size && [...a.keys()].every((n) => b.has(n))) out.push(change(key, "columns.order", "SQLCH225", order(before), order(after)));
+  const warnings = positionWarnings(before, after);
+  if (warnings.length > 0 && out.length > start) out[start] = { ...out[start]!, note: `loses its position: ${warnings.join("; ")}` };
+}
+
 /**
  * A table over a table function (#3911), on either side. It stores nothing,
  * so every change but its comment replaces it (SQLCH219); a table with data
@@ -326,5 +371,16 @@ export function diffSchemas(before: readonly SchemaObject[], after: readonly Sch
       changes.push(change(b.key, "object", "SQLCH250", qualified(b.canonical), undefined, { destructive: b.canonical.kind === "table" || b.canonical.kind === "materializedView" }));
     }
   }
+  // A replaced stream table is replaced around the views that read it (#4396): say which.
+  const replaced = new Set(changes.filter((c) => c.rule === "SQLCH225").map((c) => c.object));
+  for (const m of matches) {
+    if (m.kind !== "matched" || !replaced.has(m.after.key)) continue;
+    const readers = before.filter((o) => readsTable(o.canonical, m.before.canonical)).map((o) => qualified(o.canonical));
+    hints.push(
+      readers.length > 0
+        ? `${qualified(m.before.canonical)} is replaced: ${readers.join(", ")} ${readers.length === 1 ? "reads it and is" : "read it and are"} detached first and attached after`
+        : `${qualified(m.before.canonical)} is replaced; no materialized view reads it`,
+    );
+  }
   return { changes, rebuilds: changes.filter((c) => c.class === "rebuild"), hints };
 }

package/src/clickhouse/plan/disruption.ts

@@ -15,6 +15,9 @@ import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse";
 import { classifyDisruptionWith } from "../../core/classifier";
 import { CHANGE_CLASSES, CLASSIFIER_RULES, type ClassifierRuleId } from "./rules";
 
+/** Settings only a stream engine takes; the path alone cannot name the engine. */
+const STREAM_SETTING = /^(?:kafka_|rabbitmq_|nats_|s3queue_|azure_queue_|keeper_path$)/;
+
 function ruleFor(path: string): ClassifierRuleId | "ambiguous" | undefined {
   const root = path.split(/[.[]/)[0]!;
   if (root === "name" || root === "database") return "SQLCH230";
@@ -32,6 +35,8 @@ function ruleFor(path: string): ClassifierRuleId | "ambiguous" | undefined {
   if (root === "refresh") return "SQLCH244";
   if (root === "settings") {
     const key = path.split(".")[1];
+    // A stream engine's own settings (Kafka, RabbitMQ, NATS, S3Queue's keeper_path) have no MODIFY SETTING: the table is replaced (#4396).
+    if (key !== undefined && STREAM_SETTING.test(key)) return "SQLCH225";
     const spec = key ? (MERGE_TREE_SETTINGS as Record<string, { readonly: boolean } | undefined>)[key] : undefined;
     return spec?.readonly ? "SQLCH218" : "SQLCH206";
   }

package/src/clickhouse/plan/rebuild-handoff.ts

@@ -19,6 +19,7 @@
 import type { CanonicalObject } from "./normalize";
 import type { SchemaDiff } from "./diff";
 import { migrationOpName, renderMigrationOps, type MigrationOpSuggestion } from "../../core/handoff";
+import { isStreamEngine } from "./stream";
 import { collapsesRows } from "../rebuild/engines";
 
 export interface RebuildOpSuggestion extends MigrationOpSuggestion {
@@ -41,6 +42,8 @@ export function rebuildOpSuggestions(diff: SchemaDiff, declared: ReadonlyMap<str
   for (const key of [...new Set(diff.rebuilds.filter((c) => c.rule !== "SQLCH290").map((c) => c.object))]) {
     const c = declared.get(key);
     if (!c || c.kind !== "table") continue;
+    // The Op refuses a stream-engine table (#4396); a change to one that reaches here is a move to or from a stream engine, for review.
+    if (isStreamEngine(c.engineName)) continue;
     const table = `${c.database ?? "default"}.${c.name}`;
     const time = c.columns.find((col) => !col.nullable && TIME_TYPE.test(col.type) && (!col.defaultKind || col.defaultKind === "DEFAULT"));
     const dualWrite: RebuildOpSuggestion["dualWrite"] = time ? { mode: "materialized-view", cutoverColumn: time.name } : { mode: "app" };

package/src/clickhouse/plan/report.ts

@@ -4,6 +4,7 @@ import { renderChangeSet } from "../../core/classifier";
 import { CHANGE_CLASSES, CLASSIFIER_RULES } from "./rules";
 import type { SchemaDiff } from "./diff";
 import { renderRebuildOps } from "./rebuild-handoff";
+import { renderNamedCollections } from "../live/named-collections";
 
 export function renderDiff(diff: SchemaDiff, opts: { title?: string } = {}): string {
   const unreadable = diff.unreadable ?? [];
@@ -17,6 +18,7 @@ export function renderDiff(diff: SchemaDiff, opts: { title?: string } = {}): str
       : [];
   const trailer = [
     ...refusedRead,
+    ...renderNamedCollections(diff.namedCollections),
     ...(diff.rebuilds.length > 0
       ? [
           "",
@@ -26,5 +28,8 @@ export function renderDiff(diff: SchemaDiff, opts: { title?: string } = {}): str
         ]
       : []),
   ];
-  return renderChangeSet(diff, { ...(opts.title ? { title: opts.title } : {}), rules: CLASSIFIER_RULES, classes: CHANGE_CLASSES, trailer });
+  const text = renderChangeSet(diff, { ...(opts.title ? { title: opts.title } : {}), rules: CLASSIFIER_RULES, classes: CHANGE_CLASSES, trailer });
+  // The shared report prints no trailer under "No changes."; a missing named collection fails the plan all the same (#4395).
+  const missing = renderNamedCollections(diff.namedCollections);
+  return diff.changes.length === 0 && missing.length > 0 ? [text, ...missing].join("\n") : text;
 }