@intentius/chant-lexicon-sql 0.124.46 → 0.124.49

package/src/composites/stream-ingest.test.ts

@@ -0,0 +1,114 @@
+/**
+ * `StreamIngest` (#4396): the stream table, the view and the MergeTree
+ * target it declares, the references between them, and what the lexicon's own
+ * checks say about a chain that names a named collection and about one that
+ * writes a password.
+ */
+
+import { describe, expect, test } from "vitest";
+import { expandComposite } from "@intentius/chant/composite";
+import type { Declarable } from "@intentius/chant/declarable";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { makePostSynthCtx } from "@intentius/chant-test-utils";
+import { StreamIngest } from "./index";
+import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseObject } from "../clickhouse/entities";
+import { sqlSerializer } from "../serializer";
+import { postSynthChecks } from "../lint/post-synth";
+import { diffStatements } from "../migration-statements";
+
+const flat = (ddl: string) => ddl.replace(/\s+/g, " ").trim();
+const ddlOf = (entity: unknown) => flat((entity as ClickHouseObject & { props: { ddl: string } }).props.ddl);
+
+const ingest = (over: Partial<Parameters<typeof StreamIngest>[0]> = {}) =>
+  StreamIngest({
+    name: "events",
+    columns: "ts DateTime, user_id UInt64, kind LowCardinality(String)",
+    engine: "Kafka(kafka_creds)",
+    settings: "kafka_topic_list = 'events', kafka_group_name = 'ingest', kafka_format = 'JSONEachRow'",
+    orderBy: "(kind, user_id, ts)",
+    ...over,
+  });
+
+describe("StreamIngest", () => {
+  test("a target, a stream table named after it, and a view from the one into the other", () => {
+    const { target, stream, view } = ingest();
+    expect(target.entityType).toBe(CLICKHOUSE_ENTITY_TYPES.table);
+    expect(stream.entityType).toBe(CLICKHOUSE_ENTITY_TYPES.table);
+    expect(view.entityType).toBe(CLICKHOUSE_ENTITY_TYPES.materializedView);
+    expect([target.props.name, stream.props.name, view.props.name]).toEqual(["events", "events_stream", "events_mv"]);
+    expect(target.props.engine).toEqual({ name: "MergeTree" });
+    expect(stream.props.engine).toMatchObject({ name: "Kafka" });
+  });
+
+  test("the DDL: the named collection is the engine's argument, the settings follow, the view selects everything", () => {
+    const { stream, view, target } = ingest();
+    expect(ddlOf(stream)).toBe(
+      "CREATE TABLE events_stream (ts DateTime, user_id UInt64, kind LowCardinality(String)) ENGINE = Kafka(kafka_creds) SETTINGS kafka_topic_list = 'events', kafka_group_name = 'ingest', kafka_format = 'JSONEachRow'",
+    );
+    expect(ddlOf(view)).toBe("CREATE MATERIALIZED VIEW events_mv TO events AS SELECT * FROM events_stream");
+    expect(ddlOf(target)).toBe("CREATE TABLE events (ts DateTime, user_id UInt64, kind LowCardinality(String)) ENGINE = MergeTree ORDER BY (kind, user_id, ts)");
+  });
+
+  test("the view writes TO the target and reads the stream table, by reference, so the build orders all three", () => {
+    const { target, stream, view } = ingest();
+    expect(view.props.to).toBe(target);
+    expect(view.props.reads).toEqual([stream]);
+    expect(view.dependsOn).toEqual([target, stream]);
+  });
+
+  test("the message columns, the select list, the target engine and the partition key are parameters", () => {
+    const { stream, view, target } = ingest({
+      streamColumns: "payload String",
+      select: "parseDateTimeBestEffort(JSONExtractString(payload, 'ts')) AS ts, JSONExtractUInt(payload, 'user_id') AS user_id, JSONExtractString(payload, 'kind') AS kind",
+      targetEngine: "ReplacingMergeTree",
+      partitionBy: "toYYYYMM(ts)",
+      engine: "S3Queue(lake_creds)",
+      settings: "mode = 'unordered', keeper_path = '/chant/events'",
+    });
+    expect(ddlOf(stream)).toBe("CREATE TABLE events_stream (payload String) ENGINE = S3Queue(lake_creds) SETTINGS mode = 'unordered', keeper_path = '/chant/events'");
+    expect(ddlOf(view)).toMatch(/^CREATE MATERIALIZED VIEW events_mv TO events AS SELECT parseDateTimeBestEffort\(/);
+    expect(target.props.engine).toEqual({ name: "ReplacingMergeTree" });
+    expect(target.props.partitionBy).toBe("toYYYYMM(ts)");
+  });
+
+  test("an engine with no settings declares none", () => {
+    expect(ddlOf(ingest({ engine: "Kafka(kafka_creds)", settings: undefined }).stream)).toMatch(/ENGINE = Kafka\(kafka_creds\)$/);
+  });
+
+  function build(props: Parameters<typeof ingest>[0]) {
+    const entities = new Map<string, Declarable>();
+    for (const [k, v] of expandComposite("events", ingest(props))) entities.set(k, v);
+    const out = sqlSerializer.serialize(entities) as SerializerResult;
+    return { out, doc: JSON.parse(out.primary) as { applyOrder: string[]; namedCollections?: string[] }, entities };
+  }
+  const findings = (b: ReturnType<typeof build>) => {
+    const ctx = makePostSynthCtx("sql", b.out.primary, b.entities);
+    return postSynthChecks.filter((c) => c.id.startsWith("SQLCH") || c.id === "SQL101").flatMap((check) => check.check(ctx));
+  };
+
+  test("the build applies the target and the stream table before the view, records the collection, and the lexicon's checks find nothing", () => {
+    const b = build({ columns: "ts DateTime, user_id UInt64, kind LowCardinality(String)", select: "ts, user_id, kind", streamColumns: "ts DateTime, user_id UInt64, kind LowCardinality(String)" });
+    const order = b.doc.applyOrder;
+    expect(order.indexOf("eventsTarget")).toBeLessThan(order.indexOf("eventsView"));
+    expect(order.indexOf("eventsStream")).toBeLessThan(order.indexOf("eventsView"));
+    expect(b.doc.namedCollections).toEqual(["kafka_creds"]);
+    expect(findings(b)).toEqual([]);
+  });
+
+  test("a view that writes a column the target lacks is SQLCH110, and a password in the engine is SQLCH128", () => {
+    const wrong = findings(build({ select: "ts, user_id, kind, extra" }));
+    expect(wrong.map((f) => f.checkId)).toContain("SQLCH110");
+    const inline = findings(build({ engine: "Kafka('broker:9092', 'events', 'ingest', 'JSONEachRow')", settings: "kafka_sasl_password = 'hunter2'" }));
+    expect(inline.map((f) => f.checkId)).toContain("SQLCH128");
+  });
+
+  test("a change to the stream table between two builds is a replacement in the migration's statements, not an Op", () => {
+    const settings = (topic: string) => `kafka_topic_list = '${topic}', kafka_group_name = 'ingest', kafka_format = 'JSONEachRow'`;
+    const doc = diffStatements(build({ settings: settings("events") }).out.primary, build({ settings: settings("events,clicks") }).out.primary);
+    expect(doc.refused).toBeFalsy();
+    const steps = doc.steps.filter((s) => s.kind === "statement") as { sql: string; rule: string; class: string }[];
+    expect(steps.map((s) => s.sql.replace(/\s+/g, " ").split(" ").slice(0, 2).join(" "))).toEqual(["DETACH TABLE", "DROP TABLE", "CREATE TABLE", "ATTACH TABLE"]);
+    expect(steps.every((s) => s.rule === "SQLCH225" && s.class === "replace")).toBe(true);
+    expect(doc.steps.every((s) => s.kind === "statement")).toBe(true);
+  });
+});

package/src/composites/stream-ingest.ts

@@ -0,0 +1,82 @@
+/**
+ * `StreamIngest`: the usual ClickHouse ingestion chain, a stream table, a materialized view reading it and a MergeTree target.
+ *
+ * The stream table (Kafka, S3Queue, AzureQueue, RabbitMQ, NATS or FileLog)
+ * stores no rows: it is the consumer. The materialized view pulls each batch
+ * through it with `TO` and writes the target, which is an ordinary MergeTree
+ * table that can be queried, altered and backfilled on its own. Every column
+ * the view's select list writes has to be one the target declares (SQLCH110).
+ *
+ * The stream engine usually carries credentials. Name a named collection as
+ * its first argument (`Kafka(kafka_creds)`, `S3Queue(lake_creds)`) and keep
+ * the broker, bucket and keys in the collection on the server: chant never
+ * declares or reads one (#4395), and SQLCH128 fails a build that writes a
+ * password in the engine's arguments.
+ *
+ * A change to the stream table is planned as SQLCH225, not as a rebuild: the
+ * view is detached, the table replaced and the view attached again. A
+ * different `kafka_group_name` or S3Queue `keeper_path` loses the table's
+ * position, and the plan says so (#4396).
+ */
+
+import { Composite } from "@intentius/chant/composite";
+import { table, view, type ClickHouseTable, type ClickHouseView } from "../clickhouse/entities";
+import type { ClickHouseMemberName } from "../core/typed";
+import type { Instance, TypedComposite } from "./typed";
+
+export interface StreamIngestProps {
+  /** The target table's name. The stream table is `<name>_stream` and the view `<name>_mv`. */
+  name: string;
+  /** The target's columns, as SQL: `ts DateTime, user_id UInt64, kind LowCardinality(String)`. */
+  columns: string;
+  /**
+   * The stream table's engine, with a named collection as its first argument
+   * rather than inline credentials: `Kafka(kafka_creds)`, `S3Queue(lake_creds)`.
+   */
+  engine: string;
+  /**
+   * The engine's settings, the body of a `SETTINGS` clause:
+   * `kafka_topic_list = 'events', kafka_group_name = 'ingest', kafka_format = 'JSONEachRow'`.
+   * Settings the named collection already holds are left out.
+   */
+  settings?: string;
+  /** The stream table's columns when they differ from the target's: the message as it arrives. Default: `columns`. */
+  streamColumns?: string;
+  /** The view's select list over the stream table, each item aliased to a target column. Default: `*`. */
+  select?: string;
+  /** The target's sort key (a column list or an expression). */
+  orderBy: string;
+  /** The target's engine (default `MergeTree`). */
+  targetEngine?: string;
+  /** The target's partition key. Left out by default. */
+  partitionBy?: string;
+}
+
+/** `Target`, `Stream` and `View` are the members' names, when the call's props give them (chant#3787). */
+export type StreamIngestMembers<Target extends string = string, Stream extends string = string, View extends string = string> = {
+  target: ClickHouseTable<Target>;
+  stream: ClickHouseTable<Stream>;
+  view: ClickHouseView<View>;
+};
+
+type StreamIngestCall = <const N extends string>(
+  props: StreamIngestProps & { name: N },
+) => Instance<StreamIngestMembers<ClickHouseMemberName<N>, ClickHouseMemberName<N, "_stream">, ClickHouseMemberName<N, "_mv">>>;
+
+/** A stream table, the materialized view that reads it, and the MergeTree target the view writes. */
+export const StreamIngest = Composite<StreamIngestProps, StreamIngestMembers, TypedComposite<StreamIngestProps, StreamIngestMembers, StreamIngestCall>>((props) => {
+  const target = table`
+    CREATE TABLE ${props.name} (${props.columns})
+    ENGINE = ${props.targetEngine ?? "MergeTree"}
+    ${props.partitionBy ? `PARTITION BY ${props.partitionBy}` : ""}
+    ORDER BY ${props.orderBy}`;
+  const stream = table`
+    CREATE TABLE ${`${props.name}_stream`} (${props.streamColumns ?? props.columns})
+    ENGINE = ${props.engine}
+    ${props.settings ? `SETTINGS ${props.settings}` : ""}`;
+  const ingest = view`
+    CREATE MATERIALIZED VIEW ${`${props.name}_mv`} TO ${target} AS
+    SELECT ${props.select ?? "*"}
+    FROM ${stream}`;
+  return { target, stream, view: ingest };
+}, "StreamIngest");

package/src/core/statement-class.ts

@@ -0,0 +1,88 @@
+/**
+ * What the statement classifiers (`../clickhouse/plan/statements.ts`,
+ * `../postgres/plan/statements.ts`) answer, in one shape for both dialects
+ * (#4407). They read a migration file's statements offline and give each the
+ * class the schema-diff classifier gives the same change, citing its rule.
+ * Nothing runs against a server.
+ */
+
+import type { ClassifierRule } from "./classifier";
+
+/** One change a statement makes, under one rule. */
+export interface StatementClassified {
+  rule: string;
+  /** The rule's class, e.g. `rebuild`, `concurrently`. */
+  class: string;
+  /** The class as a report names it, e.g. `REBUILD`. */
+  label: string;
+  title: string;
+  /** Postgres: the lock the rule's restriction names, e.g. `SHARE`. */
+  lock?: string;
+  /** Data is removed and not recoverable. */
+  destructive?: boolean;
+  note?: string;
+}
+
+/**
+ * - `classified`: every change has a rule.
+ * - `undecided`: the statement alone cannot say which of `candidates` applies
+ *   (a column type change is a rewrite, or a rebuild when the column is in a
+ *   key); the answer `classifyDisruption()` calls `unknown`.
+ * - `unclassified`: a schema change no rule covers; `note` says what.
+ * - `data`: the statement reads or writes rows or the session, not the schema.
+ */
+export type StatementOutcome = "classified" | "undecided" | "unclassified" | "data";
+
+export interface StatementVerdict {
+  /** 1-based position among the file's up statements. */
+  index: number;
+  /** The statement's first line, cut to 120 characters. */
+  sql: string;
+  outcome: StatementOutcome;
+  /** `classified`: one entry per change in the statement (an `ALTER TABLE` can hold several). */
+  changes: StatementClassified[];
+  /** `undecided`: the rules that could apply, in the order a reader should weigh them. */
+  candidates?: StatementClassified[];
+  note?: string;
+}
+
+/** A statement's own text cut for a note: the first line, at most 120 characters. */
+export function sqlHead(sql: string): string {
+  const line = sql.split(/\r?\n/).find((l) => l.trim() !== "" && !l.trim().startsWith("--")) ?? sql;
+  const t = line.trim().replace(/\s+/g, " ");
+  return t.length > 120 ? `${t.slice(0, 117)}...` : t;
+}
+
+/** The pieces of a classified change from a dialect's rule table. */
+export function classifiedOf<C extends string>(
+  rules: Readonly<Record<string, ClassifierRule<C>>>,
+  labels: Readonly<Record<C, string>>,
+  id: string,
+  extra: Partial<StatementClassified> = {},
+): StatementClassified {
+  const r = rules[id];
+  if (!r) throw new Error(`no classifier rule ${id}`);
+  return { rule: id, class: r.class, label: labels[r.class], title: r.title, ...extra };
+}
+
+/** Lock names, longest first so `SHARE UPDATE EXCLUSIVE` is not read as `SHARE`. */
+const LOCK = /\b(ACCESS EXCLUSIVE|SHARE UPDATE EXCLUSIVE|SHARE ROW EXCLUSIVE|ROW EXCLUSIVE|ROW SHARE|EXCLUSIVE|SHARE)\b/;
+
+/** The first lock a rule's restriction names, or undefined when it names none. */
+export const lockOf = (restriction: string): string | undefined => LOCK.exec(restriction)?.[1];
+
+/**
+ * Split a statement's tokens' text into the pieces between top-level commas
+ * (an `ALTER TABLE`'s actions). Parentheses and brackets nest.
+ */
+export function splitTopLevel(words: readonly string[]): string[][] {
+  const out: string[][] = [[]];
+  let depth = 0;
+  for (const w of words) {
+    if (w === "(" || w === "[") depth++;
+    else if (w === ")" || w === "]") depth--;
+    if (w === "," && depth === 0) out.push([]);
+    else out[out.length - 1]!.push(w);
+  }
+  return out.filter((a) => a.length > 0);
+}

package/src/core/undeclarable.test.ts

@@ -100,14 +100,15 @@ describe("ClickHouse live import", () => {
   afterAll(() => server.close());
   const run = (extra: Record<string, unknown> = {}) => exportClickHouse({ environment: "x", config: {}, env: { CLICKHOUSE_URL: server.url }, ...extra });
 
-  test("fails naming them", async () => {
-    await expect(run()).rejects.toThrow(/  - named collection nc1\n  - window view analytics\.wv\n  - workload all\n/);
+  test("fails naming them; a named collection is a prerequisite, not among them (#4395)", async () => {
+    await expect(run()).rejects.toThrow(/:\n  - window view analytics\.wv\n  - workload all\nWhy/);
   });
 
   test("with skipUndeclarable, warns, and the window view is not imported as a table", async () => {
     const ir = await run({ skipUndeclarable: true });
     expect(ir.resources.map((r) => r.properties.name)).toEqual(["analytics", "events"]);
-    expect((ir.warnings ?? []).filter((w) => w.includes("--skip-undeclarable"))).toHaveLength(3);
+    expect((ir.warnings ?? []).filter((w) => w.includes("--skip-undeclarable"))).toHaveLength(2);
+    expect((ir.warnings ?? []).filter((w) => w.startsWith("named collection nc1 is a prerequisite"))).toHaveLength(1);
   });
 });
 
@@ -134,8 +135,8 @@ CREATE COLLATION app.ci (provider = icu, locale = 'und');`;
     ]);
   });
 
-  test("ClickHouse: a named collection in a file is refused too; a quota is declared (#3911)", () => {
-    expect(() => new SqlFileParser("clickhouse").parse("CREATE DATABASE a;\nCREATE NAMED COLLECTION c AS url = 'x'")).toThrow(/named collection: chant cannot declare it/);
+  test("ClickHouse: a named collection in a file is a prerequisite (#4395); a quota is declared (#3911)", () => {
+    expect(new SqlFileParser("clickhouse").parse("CREATE DATABASE a;\nCREATE NAMED COLLECTION c AS url = 'x'").warnings).toEqual([expect.stringMatching(/^named collection c is a prerequisite/)]);
     expect(new SqlFileParser("clickhouse").parse("CREATE DATABASE a;\nCREATE QUOTA q1").resources.map((r) => r.type)).toEqual(["ClickHouse::Database", "ClickHouse::Quota"]);
   });
 });

package/src/coverage/clickhouse.json

@@ -8,7 +8,7 @@
     { "command": "CREATE HANDLER", "kind": "HTTP handler", "status": "refused", "reason": "an HTTP handler is server-level configuration, not a schema object" },
     { "command": "CREATE INDEX", "kind": "index", "status": "declared", "entity": "ClickHouse::Table", "note": "an index is part of its table: SHOW CREATE TABLE prints it, and the table's declaration holds it" },
     { "command": "CREATE MASKING POLICY", "kind": "masking policy", "status": "refused", "reason": "masking policies are available only in ClickHouse Cloud, not on the pinned server" },
-    { "command": "CREATE NAMED COLLECTION", "kind": "named collection", "status": "refused", "reason": "a named collection holds credentials (a password, a secret key), which are the environment's and never in a build; the server hides its values, so a plan could not compare them either" },
+    { "command": "CREATE NAMED COLLECTION", "kind": "named collection", "status": "referenced", "reason": "a named collection holds credentials (a password, a secret key), which are the environment's: chant never declares one or reads its values. An engine, a table function or a dictionary SOURCE names it, a plan checks that it exists, and import lists the server's as prerequisites" },
     { "command": "CREATE QUOTA", "kind": "quota", "status": "declared", "entity": "ClickHouse::Quota", "note": "import reads only the access objects a build declares" },
     { "command": "CREATE RESOURCE", "kind": "workload resource", "status": "refused", "reason": "a resource is server-level scheduling (a disk's reads or writes), configured with the server, not a schema object" },
     { "command": "CREATE ROLE", "kind": "role", "status": "declared", "entity": "ClickHouse::Role", "note": "import reads only the access objects a build declares" },

package/src/coverage/postgres.json

@@ -10,32 +10,32 @@
     { "command": "CREATE DOMAIN", "kind": "domain", "status": "declared", "entity": "Postgres::Domain" },
     { "command": "CREATE EVENT TRIGGER", "kind": "event trigger", "status": "declared", "entity": "Postgres::EventTrigger", "note": "the database's, not a schema's; creating one takes a superuser" },
     { "command": "CREATE EXTENSION", "kind": "extension", "status": "declared", "entity": "Postgres::Extension" },
-    { "command": "CREATE FOREIGN DATA WRAPPER", "kind": "foreign data wrapper", "status": "refused", "reason": "no tag declares a foreign data wrapper yet; one an extension installs comes with the extension" },
-    { "command": "CREATE FOREIGN TABLE", "kind": "foreign table", "status": "refused", "reason": "no tag declares a foreign table yet" },
+    { "command": "CREATE FOREIGN DATA WRAPPER", "kind": "foreign data wrapper", "status": "declared", "entity": "Postgres::ForeignDataWrapper", "note": "the database's, not a schema's; creating one takes a superuser; one an extension installs comes with the extension; an option that holds a credential is refused" },
+    { "command": "CREATE FOREIGN TABLE", "kind": "foreign table", "status": "declared", "entity": "Postgres::ForeignTable", "note": "its own tag, not table: another CREATE and ALTER, no key, index or storage clauses; its server cannot be altered; one that is a partition, inherits or has a generated column is not declared; an option that holds a credential is refused" },
     { "command": "CREATE FUNCTION", "kind": "function", "status": "declared", "entity": "Postgres::Function" },
     { "command": "CREATE FUNCTION", "form": "RETURN or BEGIN ATOMIC body", "match": "\\b(?:RETURN|BEGIN\\s+ATOMIC)\\b", "kind": "routine with a SQL-standard body", "status": "refused", "reason": "the server stores a SQL-standard body parsed and prints it back rewritten, so a declaration cannot be compared with it; declare the routine with a string body (AS $$ ... $$)" },
     { "command": "CREATE GROUP", "kind": "group", "status": "declared", "entity": "Postgres::Role", "note": "an alias of CREATE ROLE" },
     { "command": "CREATE INDEX", "kind": "index", "status": "declared", "entity": "Postgres::Index" },
     { "command": "CREATE LANGUAGE", "kind": "procedural language", "status": "refused", "reason": "no tag declares a procedural language; an extension installs one, and the extension is declared instead" },
     { "command": "CREATE MATERIALIZED VIEW", "kind": "materialized view", "status": "declared", "entity": "Postgres::MaterializedView" },
-    { "command": "CREATE OPERATOR", "kind": "operator", "status": "refused", "reason": "no tag declares an operator yet" },
-    { "command": "CREATE OPERATOR CLASS", "kind": "operator class", "status": "refused", "reason": "no tag declares an operator class yet" },
-    { "command": "CREATE OPERATOR FAMILY", "kind": "operator family", "status": "refused", "reason": "no tag declares an operator family yet" },
+    { "command": "CREATE OPERATOR", "kind": "operator", "status": "declared", "entity": "Postgres::Operator", "note": "identified by its name and operand types; RESTRICT and JOIN change in place, and from 17 so does setting a commutator or negator or turning on HASHES or MERGES; anything else is a drop and a create" },
+    { "command": "CREATE OPERATOR CLASS", "kind": "operator class", "status": "declared", "entity": "Postgres::OperatorClass", "note": "identified by its name and index method; a change is a drop and a create, since ALTER OPERATOR CLASS only renames or moves it; its own family, named as the class, comes with it" },
+    { "command": "CREATE OPERATOR FAMILY", "kind": "operator family", "status": "declared", "entity": "Postgres::OperatorFamily", "note": "identified by its name and index method; its members come with the classes that name it, and loose members added with ALTER OPERATOR FAMILY are not declared" },
     { "command": "CREATE POLICY", "kind": "policy", "status": "declared", "entity": "Postgres::Policy", "note": "read where the profile manages access (sql.profiles.<env>.access)" },
     { "command": "CREATE PROCEDURE", "kind": "procedure", "status": "declared", "entity": "Postgres::Procedure" },
-    { "command": "CREATE PUBLICATION", "kind": "publication", "status": "refused", "reason": "no tag declares a publication yet" },
+    { "command": "CREATE PUBLICATION", "kind": "publication", "status": "declared", "entity": "Postgres::Publication", "note": "the database's, not a schema's; a table of it may carry a column list and row filter from 15" },
     { "command": "CREATE ROLE", "kind": "role", "status": "declared", "entity": "Postgres::Role", "note": "import reads only the roles a build declares, since a role is the cluster's" },
     { "command": "CREATE RULE", "kind": "rule", "status": "refused", "reason": "no tag declares a rewrite rule; a view's own rule is the view's" },
     { "command": "CREATE SCHEMA", "kind": "schema", "status": "declared", "entity": "Postgres::Schema" },
     { "command": "CREATE SEQUENCE", "kind": "sequence", "status": "declared", "entity": "Postgres::Sequence" },
-    { "command": "CREATE SERVER", "kind": "foreign server", "status": "refused", "reason": "no tag declares a foreign server yet" },
-    { "command": "CREATE STATISTICS", "kind": "extended statistics", "status": "refused", "reason": "no tag declares extended statistics yet" },
-    { "command": "CREATE SUBSCRIPTION", "kind": "subscription", "status": "refused", "reason": "no tag declares a subscription yet" },
+    { "command": "CREATE SERVER", "kind": "foreign server", "status": "declared", "entity": "Postgres::ForeignServer", "note": "the database's, not a schema's; its user mappings are refused, so credentials stay out of the declaration" },
+    { "command": "CREATE STATISTICS", "kind": "extended statistics", "status": "declared", "entity": "Postgres::Statistics" },
+    { "command": "CREATE SUBSCRIPTION", "kind": "subscription", "status": "refused", "reason": "a subscription's connection string carries a credential, and creating one opens a replication slot on the publisher, outside this database; the environment makes it" },
     { "command": "CREATE TABLE", "kind": "table", "status": "declared", "entity": "Postgres::Table" },
     { "command": "CREATE TABLE AS", "kind": "table from a query", "status": "declared", "entity": "Postgres::Table", "note": "the server keeps no query, so the table it made reads as a CREATE TABLE" },
     { "command": "CREATE TABLESPACE", "kind": "tablespace", "status": "out-of-scope", "reason": "a tablespace is the cluster's, a directory on the server; a table's TABLESPACE clause is kept" },
-    { "command": "CREATE TEXT SEARCH CONFIGURATION", "kind": "text search configuration", "status": "refused", "reason": "no tag declares a text search configuration yet" },
-    { "command": "CREATE TEXT SEARCH DICTIONARY", "kind": "text search dictionary", "status": "refused", "reason": "no tag declares a text search dictionary yet" },
+    { "command": "CREATE TEXT SEARCH CONFIGURATION", "kind": "text search configuration", "status": "declared", "entity": "Postgres::TextSearchConfiguration", "note": "PARSER and its ADD MAPPING statements; COPY is refused, since it copies mappings chant does not read; mappings change in place, the parser by drop and create" },
+    { "command": "CREATE TEXT SEARCH DICTIONARY", "kind": "text search dictionary", "status": "declared", "entity": "Postgres::TextSearchDictionary", "note": "a template and its options; options change in place, the template by drop and create" },
     { "command": "CREATE TEXT SEARCH PARSER", "kind": "text search parser", "status": "refused", "reason": "no tag declares a text search parser; one is C code an extension installs" },
     { "command": "CREATE TEXT SEARCH TEMPLATE", "kind": "text search template", "status": "refused", "reason": "no tag declares a text search template; one is C code an extension installs" },
     { "command": "CREATE TRANSFORM", "kind": "transform", "status": "refused", "reason": "no tag declares a transform; an extension installs one" },
@@ -45,7 +45,7 @@
     { "command": "CREATE TYPE", "form": "AS RANGE", "match": "\\bAS\\s+RANGE\\b", "kind": "range type", "status": "declared", "entity": "Postgres::RangeType", "note": "its multirange type comes with it (MULTIRANGE_TYPE_NAME)" },
     { "command": "CREATE TYPE", "form": "INPUT, OUTPUT, or a shell type", "kind": "base type", "status": "refused", "reason": "no tag declares a base type; its input and output functions are C code an extension installs" },
     { "command": "CREATE USER", "kind": "user", "status": "declared", "entity": "Postgres::Role", "note": "an alias of CREATE ROLE ... LOGIN" },
-    { "command": "CREATE USER MAPPING", "kind": "user mapping", "status": "refused", "reason": "no tag declares a user mapping yet" },
+    { "command": "CREATE USER MAPPING", "kind": "user mapping", "status": "refused", "reason": "a user mapping carries the remote user and password, which the environment supplies and a declaration never holds" },
     { "command": "CREATE VIEW", "kind": "view", "status": "declared", "entity": "Postgres::View" }
   ]
 }

package/src/coverage.test.ts

@@ -3,6 +3,8 @@ import { KIND_COVERAGE, PINNED_COMMANDS, kindCoverageProblems, pinnedCommandsCur
 import { POSTGRES_PROBES } from "./postgres/live/undeclarable";
 import { CLICKHOUSE_PROBES } from "./clickhouse/live/undeclarable";
 import { SQL_DIALECTS } from "./dialects";
+import { POSTGRES_ENTITY_TYPES } from "./postgres/entity-types";
+import { CLICKHOUSE_ENTITY_TYPES } from "./clickhouse/entities";
 
 describe("the object coverage table (#3911)", () => {
   test.each(SQL_DIALECTS)("%s: every CREATE command a pinned server has is marked, and nothing else", (dialect) => {
@@ -29,10 +31,32 @@ describe("the object coverage table (#3911)", () => {
     ]);
   });
 
-  test("every refused kind has the live query that finds it, and every query a refused-or-declared row", () => {
+  test("a declared row names an entity type the dialect has, and every entity type but a grant has a declared row", () => {
+    // A grant is a GRANT statement, not a CREATE command, so no row is about it.
+    const types = {
+      postgres: { all: Object.values(POSTGRES_ENTITY_TYPES) as string[], notCreate: ["Postgres::Grant", "Postgres::DefaultPrivileges"] },
+      clickhouse: { all: Object.values(CLICKHOUSE_ENTITY_TYPES) as string[], notCreate: ["ClickHouse::Grant"] },
+    };
+    for (const dialect of SQL_DIALECTS) {
+      const entities = KIND_COVERAGE[dialect].filter((r) => r.status === "declared").map((r) => r.entity!);
+      for (const e of entities) expect(types[dialect].all, `${dialect} row entity ${e}`).toContain(e);
+      for (const e of types[dialect].all.filter((t) => !types[dialect].notCreate.includes(t))) expect(entities, `${dialect} entity type ${e} has no declared row`).toContain(e);
+    }
+  });
+
+  test("a kind that is declared, referenced, refused or out of scope is told apart: only a declared row names an entity", () => {
+    for (const dialect of SQL_DIALECTS) {
+      for (const r of KIND_COVERAGE[dialect]) {
+        if (r.status === "declared") expect(r.reason, `${dialect} ${r.kind}`).toBeUndefined();
+        else expect(r.entity, `${dialect} ${r.kind}`).toBeUndefined();
+      }
+    }
+  });
+
+  test("every refused or referenced kind has the live query that finds it, and every query a row", () => {
     const probes = { postgres: POSTGRES_PROBES, clickhouse: CLICKHOUSE_PROBES };
     for (const dialect of SQL_DIALECTS) {
-      const refused = KIND_COVERAGE[dialect].filter((r) => r.status === "refused").map((r) => r.kind);
+      const refused = KIND_COVERAGE[dialect].filter((r) => r.status === "refused" || r.status === "referenced").map((r) => r.kind);
       for (const kind of refused) expect(Object.keys(probes[dialect]), `${dialect} ${kind}`).toContain(kind);
       const kinds = KIND_COVERAGE[dialect].map((r) => r.kind);
       for (const kind of Object.keys(probes[dialect])) expect(kinds, `${dialect} probe ${kind}`).toContain(kind);
@@ -68,6 +92,7 @@ describe("a statement's kind", () => {
     ["CREATE SETTINGS PROFILE p SETTINGS max_threads = 1", "settings profile"],
     ["CREATE OR REPLACE FUNCTION f AS (x) -> x + 1", "SQL function"],
     ["CREATE QUOTA q FOR INTERVAL 1 hour MAX queries = 10", "quota"],
+    ["CREATE NAMED COLLECTION IF NOT EXISTS c AS url = 'x'", "named collection"],
   ])("clickhouse: %s is a %s", (sql, kind) => {
     expect(statementKind("clickhouse", sql)?.kind).toBe(kind);
   });

package/src/coverage.ts

@@ -4,8 +4,10 @@
  * Which kinds of object each dialect declares (#3911): every `CREATE` command
  * a pinned server has (`./coverage/<dialect>-commands.json`, written by
  * `scripts/sql-create-commands.ts` from the server's own sources) has a row
- * in `./coverage/<dialect>.json` marking it declared, refused with a reason,
- * or out of scope. A command with several forms that differ in what chant can
+ * in `./coverage/<dialect>.json` marking it declared, referenced, refused
+ * with a reason, or out of scope. A referenced kind (a ClickHouse named
+ * collection, #4395) is never declared: a declaration names one, a plan
+ * checks it exists, and import lists the server's as prerequisites. A command with several forms that differ in what chant can
  * hold (`CREATE TYPE ... AS ENUM` and `CREATE TYPE ... AS RANGE`) has a row
  * per form. Import refuses an object of a refused kind by that row's `kind`
  * (`./core/undeclarable.ts`), and the docs page is rendered from the rows
@@ -31,8 +33,16 @@ import clickhouseCommands from "./coverage/clickhouse-commands.json" with { type
 
 // ── Object kinds ───────────────────────────────────────────────────────
 
-/** `declared`: a tag holds it. `refused`: import names it and stops. `out-of-scope`: the environment's, never imported. */
-export type KindStatus = "declared" | "refused" | "out-of-scope";
+/**
+ * `declared`: a tag holds it. `referenced`: the environment's, never declared
+ * and its values never read; a declaration names it, a plan checks it exists,
+ * and import lists the server's as prerequisites (#4395). `refused`: import
+ * names it and stops. `out-of-scope`: the environment's, never imported.
+ */
+export type KindStatus = "declared" | "referenced" | "refused" | "out-of-scope";
+
+/** Every status a coverage row may have. */
+export const KIND_STATUSES: readonly KindStatus[] = ["declared", "referenced", "refused", "out-of-scope"];
 
 /** One row of a dialect's coverage table: a `CREATE` command, or one form of it. */
 export interface KindRow {
@@ -51,7 +61,7 @@ export interface KindRow {
   status: KindStatus;
   /** The entity type that declares it, when declared. */
   entity?: string;
-  /** Why it is refused or out of scope. */
+  /** Why it is referenced, refused or out of scope. */
   reason?: string;
   /** Anything else a reader should know. */
   note?: string;
@@ -80,7 +90,8 @@ export function kindRow(dialect: SqlDialect, kind: string): KindRow {
  * What is wrong with a dialect's coverage table, against its pinned servers'
  * commands: a command a pinned server has and no row covers, a row for a
  * command no pinned server has, a kind named twice, a declared row with no
- * entity, a refused or out-of-scope row with no reason. Empty when complete.
+ * entity, a referenced, refused or out-of-scope row with no reason. Empty
+ * when complete.
  */
 export function kindCoverageProblems(dialect: SqlDialect, opts: { pins?: ReadonlyArray<{ version: string; commands: readonly string[] }>; rows?: readonly KindRow[] } = {}): string[] {
   const rows = opts.rows ?? KIND_COVERAGE[dialect];
@@ -96,7 +107,7 @@ export function kindCoverageProblems(dialect: SqlDialect, opts: { pins?: Readonl
   for (const r of rows) {
     if (seen.has(r.kind)) problems.push(`the kind "${r.kind}" has two rows`);
     seen.add(r.kind);
-    if (!["declared", "refused", "out-of-scope"].includes(r.status)) problems.push(`${r.kind}: status "${r.status}" is not declared, refused or out-of-scope`);
+    if (!KIND_STATUSES.includes(r.status)) problems.push(`${r.kind}: status "${r.status}" is not ${KIND_STATUSES.slice(0, -1).join(", ")} or ${KIND_STATUSES.at(-1)}`);
     if (r.status === "declared" && !r.entity) problems.push(`${r.kind} is declared, and names no entity type`);
     if (r.status !== "declared" && !r.reason) problems.push(`${r.kind} is ${r.status}, and gives no reason`);
   }

package/src/drizzle-command.ts

@@ -0,0 +1,94 @@
+/**
+ * `chant sql drizzle [--output <path>] [--check] [--bigint number|bigint]`
+ * (#4401): build the project the way `chant build` does, folded, and write the
+ * Drizzle schema module {@link renderDrizzle} renders from its entities.
+ *
+ * The file goes to `<project>/drizzle-schema.ts` by default, outside the
+ * source directory the build reads, and starts with chant's generated-file
+ * marker either way. `-o` and `--output` put it elsewhere, which is also the
+ * flag a workspace `generated` entry's generator passes. `--check` renders the
+ * same text and compares it with the file instead of writing it: exit 1,
+ * naming the file, when they differ.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import type { CommandGroupContext } from "@intentius/chant/cli/command-group";
+import { DrizzleDialectError, renderDrizzle } from "./drizzle";
+import { buildFolded, loadProject } from "./sql-types-command";
+
+/** The file `chant sql drizzle` writes by default, in the project directory. */
+export const DRIZZLE_SCHEMA_FILE = "drizzle-schema.ts";
+
+const USAGE = "usage: chant sql drizzle [--output <path>] [--check] [--bigint number|bigint]";
+
+interface Args {
+  output?: string;
+  check: boolean;
+  bigint?: "number" | "bigint";
+}
+
+function parseArgs(args: readonly string[]): Args | { error: string } {
+  const out: Args = { check: false };
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i]!;
+    if (a === "--output" || a === "-o") {
+      const v = args[++i];
+      if (!v) return { error: `${a} needs a path` };
+      out.output = v;
+    } else if (a.startsWith("--output=")) out.output = a.slice("--output=".length);
+    else if (a === "--check") out.check = true;
+    else if (a === "--bigint" || a.startsWith("--bigint=")) {
+      const v = a === "--bigint" ? args[++i] : a.slice("--bigint=".length);
+      if (v !== "number" && v !== "bigint") return { error: "--bigint takes number or bigint" };
+      out.bigint = v;
+    } else return { error: `unknown argument ${a}` };
+  }
+  return out;
+}
+
+export async function runDrizzle(ctx: CommandGroupContext): Promise<number> {
+  const args = parseArgs(ctx.rawArgs);
+  if ("error" in args) {
+    console.error(`${args.error}\n${USAGE}`);
+    return 1;
+  }
+  const cwd = process.cwd();
+  const project = await loadProject(cwd);
+  const output = args.output !== undefined ? resolve(cwd, args.output) : join(project.projectDir, DRIZZLE_SCHEMA_FILE);
+  const result = await buildFolded(project, "chant sql drizzle", "nothing was written");
+  if (!result) return 1;
+
+  let rendered;
+  try {
+    rendered = renderDrizzle(result.entities, { config: project.config as unknown as Record<string, unknown>, ...(args.bigint ? { bigintMode: args.bigint } : {}) });
+  } catch (err) {
+    if (!(err instanceof DrizzleDialectError)) throw err;
+    console.error(`chant sql drizzle: ${err.message}`);
+    return 1;
+  }
+  const shown = relative(cwd, output);
+  const named = shown && !shown.startsWith("..") ? shown : output;
+  if (args.check) {
+    // Compares only; the file is never written.
+    const onDisk = existsSync(output) ? readFileSync(output, "utf8") : undefined;
+    if (onDisk === rendered.text) {
+      console.log(`${named} is current`);
+      return 0;
+    }
+    console.error(
+      onDisk === undefined
+        ? `chant sql drizzle: ${named} does not exist; run \`chant sql drizzle\` to write it`
+        : `chant sql drizzle: ${named} differs from what the build declares; run \`chant sql drizzle\` to rewrite it`,
+    );
+    return 1;
+  }
+  mkdirSync(dirname(output), { recursive: true });
+  writeFileSync(output, rendered.text);
+  const { tables, views, enums, sequences } = rendered.counts;
+  const plural = (n: number, word: string) => `${n} ${word}${n === 1 ? "" : "s"}`;
+  console.log(`wrote ${named}: ${[plural(tables, "table"), plural(views, "view"), plural(enums, "enum"), plural(sequences, "sequence")].join(", ")}`);
+  for (const s of rendered.skipped) console.log(`not written: ${s.kind} ${s.names.join(", ")} (${s.reason})`);
+  for (const n of rendered.approximated) console.log(`partly written: ${n}`);
+  return 0;
+}