@intentius/chant-lexicon-sql 0.124.18 → 0.124.21

package/src/clickhouse/live/catalog.ts

@@ -10,6 +10,8 @@ import type { ClickHouseTarget } from "./bind";
 import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "../entities";
 import { isChantWorkingObject, stripMarkerFromStatement } from "../ownership";
 import { foreignTool } from "../../core/foreign-tables";
+import { parseCreate, unquote } from "../parser";
+import { tokenizeText } from "../tokens";
 
 /** The server's own databases, never part of a declared schema. */
 export const SYSTEM_DATABASES = new Set(["system", "information_schema", "INFORMATION_SCHEMA"]);
@@ -69,8 +71,10 @@ export async function readLiveSchema(target: ClickHouseTarget, opts: { withState
   const databases = await q<{ name: string; engine: string; uuid: string; comment: string }>(
     `SELECT name, engine, toString(uuid) AS uuid, comment FROM system.databases WHERE name ${scope} ORDER BY name`,
   );
-  const tables = await q<{ database: string; name: string; engine: string; uuid: string; comment: string; dictionary: number }>(
-    `SELECT database, name, engine, toString(uuid) AS uuid, comment, startsWith(create_table_query, 'CREATE DICTIONARY') AS dictionary FROM system.tables ` +
+  // `system.tables` shows no comment for a table over a table function, whatever it was created with: its statement has it (#3911).
+  const tables = await q<{ database: string; name: string; engine: string; uuid: string; comment: string; dictionary: number; proxy: string }>(
+    `SELECT database, name, engine, toString(uuid) AS uuid, comment, startsWith(create_table_query, 'CREATE DICTIONARY') AS dictionary, ` +
+      `if(comment = '' AND position(create_table_query, ' COMMENT ') > 0, create_table_query, '') AS proxy FROM system.tables ` +
       `WHERE database ${scope} AND NOT is_temporary AND name NOT LIKE '.inner%' ` +
       `ORDER BY database, name`,
   );
@@ -94,6 +98,7 @@ export async function readLiveSchema(target: ClickHouseTarget, opts: { withState
     });
   }
   for (const t of tables) {
+    if (t.proxy) t.comment = commentOf(t.proxy) ?? t.comment;
     if (working.has(t.database) || isChantWorkingObject(t.comment)) continue;
     const type = Number(t.dictionary) === 1 ? CLICKHOUSE_ENTITY_TYPES.dictionary : entityTypeOfEngine(t.engine);
     const foreign = type === CLICKHOUSE_ENTITY_TYPES.table ? foreignTool(t.name) : undefined;
@@ -119,6 +124,23 @@ export async function readLiveSchema(target: ClickHouseTarget, opts: { withState
 }
 
 
+/** A `CREATE` statement's own `COMMENT`, its value unquoted; undefined when it has none or does not parse. */
+function commentOf(statement: string): string | undefined {
+  try {
+    const tokens = tokenizeText(statement, 0);
+    const node = parseCreate(tokens);
+    if (!node.comment) return undefined;
+    const literal = tokens
+      .slice(node.comment.from, node.comment.to)
+      .map((t) => t.text)
+      .join("")
+      .trim();
+    return /^'.*'$/s.test(literal) ? literal.slice(1, -1).replace(/\\(.)/g, "$1").replace(/''/g, "'") : unquote(literal);
+  } catch {
+    return undefined;
+  }
+}
+
 /** The access objects a build declares, by kind and name: what `readLiveAccess` reads. */
 export interface DeclaredAccess {
   kind: string;
@@ -128,9 +150,10 @@ export interface DeclaredAccess {
 }
 
 /**
- * The users, roles, row policies and grantees' grants the build declares,
- * as the server holds them (#3682): `SHOW CREATE USER`, `SHOW CREATE ROLE`,
- * `SHOW CREATE ROW POLICY`, and for a grantee every line of
+ * The users, roles, row policies, quotas, settings profiles and grantees'
+ * grants the build declares, as the server holds them (#3682, #3911):
+ * `SHOW CREATE USER`, `SHOW CREATE ROLE`, `SHOW CREATE ROW POLICY`,
+ * `SHOW CREATE QUOTA`, `SHOW CREATE SETTINGS PROFILE`, and for a grantee every line of
  * `SHOW GRANTS FOR`, one per line. Nothing else is read, so a project that
  * declares none needs no access to `system.users` and the like, and an
  * object made by hand is never seen. One that does not exist is left out.
@@ -155,6 +178,17 @@ export async function readLiveAccess(target: ClickHouseTarget, declared: readonl
   for (const name of roles.filter((r) => existingRoles.has(r))) {
     out.push({ type: CLICKHOUSE_ENTITY_TYPES.role, name, engine: "", statement: await first(`SHOW CREATE ROLE ${ident(name)}`) });
   }
+  for (const [kind, type, table, show] of [
+    ["quota", CLICKHOUSE_ENTITY_TYPES.quota, "system.quotas", "QUOTA"],
+    ["settingsProfile", CLICKHOUSE_ENTITY_TYPES.settingsProfile, "system.settings_profiles", "SETTINGS PROFILE"],
+  ] as const) {
+    const wanted = names(kind);
+    if (wanted.length === 0) continue;
+    const existing = new Set((await q<{ name: string }>(`SELECT name FROM ${table} WHERE name IN (${list(wanted)})`)).map((r) => r.name));
+    for (const name of wanted.filter((n) => existing.has(n))) {
+      out.push({ type, name, engine: "", statement: await first(`SHOW CREATE ${show} ${ident(name)}`) });
+    }
+  }
   const policies = declared.filter((d) => d.kind === "rowPolicy");
   if (policies.length > 0) {
     const live = await q<{ name: string; database: string; table: string }>(
@@ -180,13 +214,20 @@ export async function readLiveAccess(target: ClickHouseTarget, declared: readonl
 }
 
 /** The canonical kinds a profile's `access` switch decides (#3716). */
-export const ACCESS_KINDS: ReadonlySet<string> = new Set(["user", "role", "rowPolicy", "grants"]);
+export const ACCESS_KINDS: ReadonlySet<string> = new Set(["user", "role", "rowPolicy", "quota", "settingsProfile", "grants"]);
 
 /** The entity types a profile's `access` switch decides (#3716). */
-export const ACCESS_ENTITY_TYPES: ReadonlySet<string> = new Set([CLICKHOUSE_ENTITY_TYPES.user, CLICKHOUSE_ENTITY_TYPES.role, CLICKHOUSE_ENTITY_TYPES.rowPolicy, CLICKHOUSE_ENTITY_TYPES.grant]);
+export const ACCESS_ENTITY_TYPES: ReadonlySet<string> = new Set([
+  CLICKHOUSE_ENTITY_TYPES.user,
+  CLICKHOUSE_ENTITY_TYPES.role,
+  CLICKHOUSE_ENTITY_TYPES.rowPolicy,
+  CLICKHOUSE_ENTITY_TYPES.quota,
+  CLICKHOUSE_ENTITY_TYPES.settingsProfile,
+  CLICKHOUSE_ENTITY_TYPES.grant,
+]);
 
 /** Why an access declaration is not read, planned or applied where the profile does not manage access. */
-export const ACCESS_UNMANAGED_DETAIL = "access is not managed in this environment: set sql.profiles.<env>.access to plan and apply users, roles, row policies and grants";
+export const ACCESS_UNMANAGED_DETAIL = "access is not managed in this environment: set sql.profiles.<env>.access to plan and apply users, roles, row policies, quotas, settings profiles and grants";
 
 /** The hint a plan gives for the access declarations it left out. */
 export function accessUnmanagedHint(n: number, environment: string): string {
@@ -200,9 +241,15 @@ export function declaredAccess(objects: ReadonlyArray<{ kind: string; name: stri
     .map((o) => ({ kind: o.kind, name: o.name, ...(o.database !== undefined ? { database: o.database } : {}), ...(o.table !== undefined ? { table: o.table } : {}) }));
 }
 
-/** The users, roles and row policies among declared entities, for `readLiveAccess`. */
+/** The users, roles, row policies, quotas and settings profiles among declared entities, for `readLiveAccess`. */
 export function accessOf(entities: ReadonlyMap<string, { entityType: string; props: Record<string, unknown> }>, defaultDatabase: string): DeclaredAccess[] {
-  const kinds: Record<string, string> = { [CLICKHOUSE_ENTITY_TYPES.user]: "user", [CLICKHOUSE_ENTITY_TYPES.role]: "role", [CLICKHOUSE_ENTITY_TYPES.rowPolicy]: "rowPolicy" };
+  const kinds: Record<string, string> = {
+    [CLICKHOUSE_ENTITY_TYPES.user]: "user",
+    [CLICKHOUSE_ENTITY_TYPES.role]: "role",
+    [CLICKHOUSE_ENTITY_TYPES.rowPolicy]: "rowPolicy",
+    [CLICKHOUSE_ENTITY_TYPES.quota]: "quota",
+    [CLICKHOUSE_ENTITY_TYPES.settingsProfile]: "settingsProfile",
+  };
   const out: DeclaredAccess[] = [];
   for (const e of entities.values()) {
     const kind = kinds[e.entityType];

package/src/clickhouse/live/describe-resources.ts

@@ -42,7 +42,9 @@ export function declaredAddress(entity: DeclaredEntity, defaultDatabase: string)
     entity.type === CLICKHOUSE_ENTITY_TYPES.database ||
     entity.type === CLICKHOUSE_ENTITY_TYPES.function ||
     entity.type === CLICKHOUSE_ENTITY_TYPES.user ||
-    entity.type === CLICKHOUSE_ENTITY_TYPES.role
+    entity.type === CLICKHOUSE_ENTITY_TYPES.role ||
+    entity.type === CLICKHOUSE_ENTITY_TYPES.quota ||
+    entity.type === CLICKHOUSE_ENTITY_TYPES.settingsProfile
   ) {
     return { name };
   }

package/src/clickhouse/live/kinds.e2e.test.ts

@@ -0,0 +1,179 @@
+/**
+ * The ClickHouse kinds #3911 adds, against the pinned server: quotas,
+ * settings profiles, a table over a table function, and a refreshable
+ * materialized view and a table's skip index and projection as the server
+ * prints them back.
+ *
+ * Each round-trips: declared in a source file, built, applied, and then
+ * planned with no change; imported back from the server (`exportResources`
+ * for the schema, `SHOW CREATE` through a `.sql` file for access control,
+ * which an import never reads on its own), built again, and planned with no
+ * change. A change to each takes the statement its rule says: `ALTER QUOTA`
+ * and `ALTER SETTINGS PROFILE`, never `OR REPLACE`, so a profile that
+ * inherits a changed one keeps it; `CREATE OR REPLACE TABLE` for the table
+ * over a table function.
+ *
+ * Needs Docker; skips cleanly without it. A throwaway server at the pin.
+ */
+
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { build } from "@intentius/chant/build";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { dockerAvailable, startScratchServer, type ScratchServer } from "../container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { exportResources } from "../import/live-export";
+import { ClickHouseGenerator } from "../import/generator";
+import { ClickHouseSqlParser } from "../import/parser";
+import { readLiveAccess } from "./catalog";
+import { planAgainstServer } from "../plan/commands";
+import { observeResourcesDeep, sqlDeepNormalizationHooks } from "../plan/deep";
+import { clickhouseApply } from "../../op/activities/clickhouse-apply";
+import { sqlPlugin } from "../../plugin";
+import { sqlSerializer } from "../../serializer";
+
+const DB = "chant_e2e_3911";
+const enabled = await dockerAvailable();
+let server: ScratchServer | undefined;
+let endpoint: ClickHouseEndpoint;
+const root = join(import.meta.dirname, "..", "..", "..", `.kinds-tmp-${process.pid}`);
+const q = <T = Record<string, unknown>>(sql: string) => clickhouseQuery<T>(endpoint, sql);
+
+beforeAll(async () => {
+  if (!enabled) return;
+  server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-kinds" });
+  endpoint = server.endpoint;
+}, 600_000);
+
+afterAll(async () => {
+  await server?.stop();
+  rmSync(root, { recursive: true, force: true });
+});
+
+const config = () =>
+  ({ ownership: { stack: "e2e3911", env: "test" }, sql: { profiles: { test: { url: endpoint.url, access: true } } } }) as never;
+
+/** A project of these files, built: the build document's path and the entities. */
+async function buildProject(name: string, files: Record<string, string>) {
+  const dir = join(root, name);
+  mkdirSync(join(dir, "src"), { recursive: true });
+  writeFileSync(join(dir, "chant.config.ts"), 'export default { lexicons: ["sql"] };\n');
+  for (const [file, content] of Object.entries(files)) writeFileSync(join(dir, "src", file), content);
+  const result = await build(join(dir, "src"), [sqlSerializer], undefined, { fold: true, intrinsics: sqlPlugin.intrinsics!(), lexicons: ["sql"] });
+  expect(result.errors).toEqual([]);
+  const buildPath = join(dir, "build.json");
+  writeFileSync(buildPath, (result.outputs.get("sql") as SerializerResult).primary);
+  const entities = new Map([...result.entities].map(([k, v]) => [k, { entityType: v.entityType, props: (v as unknown as { props: Record<string, unknown> }).props }]));
+  return { buildPath, entities };
+}
+
+async function apply(buildPath: string) {
+  const sent: string[] = [];
+  const outcome = await clickhouseApply({ buildPath, environment: "test" }, undefined, { config: config(), env: {}, log: (l) => void (/^[A-Z]+ /.test(l) && sent.push(l)) });
+  expect(outcome.failed).toEqual([]);
+  return { outcome, sent };
+}
+
+const plan = async (buildPath: string) => (await planAgainstServer("test", buildPath, { config: config(), env: {} })).changes.map((c) => [c.object, c.field, c.rule]);
+
+const compared = (props: object): Record<string, unknown> =>
+  Object.fromEntries(Object.entries(JSON.parse(JSON.stringify(props)) as Record<string, unknown>).filter(([k]) => !sqlDeepNormalizationHooks.prune!({ pattern: k } as never)));
+
+const source = (v: { limit: number; threads: number; nums: number }) => `import { database, table, view, role, quota, settingsProfile } from "@intentius/chant-lexicon-sql/clickhouse";
+
+export const shop = database\`CREATE DATABASE ${DB}\`;
+
+export const events = table\`
+  CREATE TABLE \${shop}.events (
+    id UInt64,
+    kind LowCardinality(String),
+    INDEX by_kind kind TYPE bloom_filter(0.01) GRANULARITY 4,
+    PROJECTION per_kind (SELECT kind, count() GROUP BY kind)
+  ) ENGINE = MergeTree ORDER BY id\`;
+
+export const hourly = view\`
+  CREATE MATERIALIZED VIEW \${shop}.hourly REFRESH EVERY 1 HOUR
+  ENGINE = MergeTree ORDER BY kind
+  AS SELECT kind, count() AS n FROM \${events} GROUP BY kind\`;
+
+export const nums = table\`CREATE TABLE \${shop}.nums AS numbers(${v.nums})\`;
+export const typedNums = table\`CREATE TABLE \${shop}.typed_nums (number UInt64) AS numbers(10) COMMENT 'counted'\`;
+
+export const reader = role\`CREATE ROLE ${DB}_reader\`;
+
+export const base = settingsProfile\`CREATE SETTINGS PROFILE ${DB}_base SETTINGS max_threads = ${v.threads}\`;
+
+export const analysts = settingsProfile\`
+  CREATE PROFILE ${DB}_analysts
+  SETTINGS PROFILE \${base}, max_memory_usage = 10000000000 MIN 1000000 MAX 20000000000, readonly = 1 READONLY
+  TO \${reader}\`;
+
+export const perUser = quota\`
+  CREATE QUOTA ${DB}_per_user KEYED BY 'user_name'
+  FOR INTERVAL 60 MINUTE MAX QUERIES ${v.limit}, errors = 50,
+  FOR RANDOMIZED INTERVAL 1 DAY MAX read_bytes = 1e12
+  TO \${reader}\`;
+`;
+
+describe.skipIf(!enabled)("the kinds #3911 adds, against the pinned server", () => {
+  test("declared, built, applied, planned clean; imported back and planned clean; changed with the statement each rule says", async () => {
+    const v1 = await buildProject("v1", { "schema.ts": source({ limit: 1000, threads: 4, nums: 10 }) });
+    const first = await apply(v1.buildPath);
+    expect(first.outcome.applied.filter((a) => a.action === "created").map((a) => a.name)).toEqual(
+      expect.arrayContaining([`${DB}.events`, `${DB}.hourly`, `${DB}.nums`, `${DB}.typed_nums`, `quota ${DB}_per_user`, `settings profile ${DB}_base`, `settings profile ${DB}_analysts`]),
+    );
+    expect(await plan(v1.buildPath)).toEqual([]);
+
+    // The deep read: every declared object read back as declared.
+    const deep = await observeResourcesDeep({ environment: "test", entityNames: [...v1.entities.keys()], entities: v1.entities, config: config(), env: {} });
+    expect(deep.unobserved ?? {}).toEqual({});
+    for (const k of ["perUser", "base", "analysts", "nums", "typedNums", "hourly", "events"]) {
+      expect(compared(deep.resources[k]!.properties), k).toEqual(compared(v1.entities.get(k)!.props));
+    }
+
+    // Imported back: the schema from the catalog, access control from SHOW CREATE in a .sql file.
+    const exported = await exportResources({ environment: "test", config: config(), env: {} });
+    const [schemaFile] = new ClickHouseGenerator().generate(exported);
+    const access = await readLiveAccess({ endpoint, source: "test", defaultDatabase: "default" }, [
+      { kind: "role", name: `${DB}_reader` },
+      { kind: "settingsProfile", name: `${DB}_base` },
+      { kind: "settingsProfile", name: `${DB}_analysts` },
+      { kind: "quota", name: `${DB}_per_user` },
+    ]);
+    expect(access.map((a) => a.statement)).toEqual([
+      `CREATE ROLE ${DB}_reader`,
+      `CREATE QUOTA ${DB}_per_user KEYED BY user_name FOR INTERVAL 1 hour MAX queries = 1000, errors = 50, FOR RANDOMIZED INTERVAL 1 day MAX read_bytes = 1000000000000 TO ${DB}_reader`,
+      `CREATE SETTINGS PROFILE \`${DB}_base\` SETTINGS max_threads = 4`,
+      `CREATE SETTINGS PROFILE \`${DB}_analysts\` SETTINGS INHERIT \`${DB}_base\`, max_memory_usage = 10000000000 MIN 1000000 MAX 20000000000, readonly = 1 CONST TO ${DB}_reader`,
+    ]);
+    const [accessFile] = new ClickHouseGenerator().generate(new ClickHouseSqlParser().parse(access.map((a) => `${a.statement};`).join("\n")));
+    expect(schemaFile!.content).toMatch(/CREATE TABLE \$\{\w+\}\.nums\s*\(\s*number UInt64\s*\) AS numbers\(10\)/);
+    expect(schemaFile!.content).toContain("REFRESH EVERY 1 HOUR");
+    const imported = await buildProject("imported", { "schema.ts": schemaFile!.content, "access.ts": accessFile!.content });
+    expect(await plan(imported.buildPath)).toEqual([]);
+    expect((await apply(imported.buildPath)).sent).toEqual([]);
+
+    // Changed: a quota's limit, the inherited profile's settings, the table function's call.
+    const v2 = await buildProject("v2", { "schema.ts": source({ limit: 2000, threads: 2, nums: 20 }) });
+    expect(await plan(v2.buildPath)).toEqual([
+      [`base (settings profile ${DB}_base)`, "settings", "SQLCH276"],
+      [`nums (${DB}.nums)`, "asFunction", "SQLCH219"],
+      [`perUser (quota ${DB}_per_user)`, "interval.3600", "SQLCH275"],
+    ]);
+    const second = await apply(v2.buildPath);
+    expect(second.sent).toEqual(
+      expect.arrayContaining([
+        expect.stringMatching(new RegExp(`^CREATE OR REPLACE TABLE ${DB}\\.nums AS numbers\\(20\\) COMMENT '`)),
+        `ALTER QUOTA \`${DB}_per_user\` FOR INTERVAL 3600 second MAX errors = 50, queries = 2000`,
+        `ALTER SETTINGS PROFILE \`${DB}_base\` SETTINGS max_threads = 2`,
+      ]),
+    );
+    expect(second.sent).toHaveLength(3);
+    expect(await plan(v2.buildPath)).toEqual([]);
+    // ALTER, not OR REPLACE: the profile that inherits the changed one still does.
+    expect(Object.values((await q<Record<string, string>>(`SHOW CREATE SETTINGS PROFILE ${DB}_analysts`))[0]!)[0]).toContain(`INHERIT \`${DB}_base\``);
+    expect(Number((await q<{ n: string }>(`SELECT count() AS n FROM ${DB}.nums`))[0]!.n)).toBe(20);
+  }, 600_000);
+});

package/src/clickhouse/live/undeclarable.e2e.test.ts

@@ -0,0 +1,65 @@
+/**
+ * Import against a real ClickHouse server never leaves an object out without
+ * a word (#3911): a server holding a named collection and a workload made
+ * with SQL, beside a table, fails the import, which names both; a quota and a
+ * settings profile are declared kinds, which an import reads only when a
+ * build declares them, so they are not refused; with `skipUndeclarable` the table is imported and
+ * they are warnings. What `users.xml` defines (the `default` quota and
+ * profile) is the server's, and is not refused.
+ *
+ * Needs Docker; skips cleanly without it. One throwaway
+ * `clickhouse/clickhouse-server` container at the pin, removed afterwards
+ * even on failure.
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { dockerAvailable, startScratchServer, type ScratchServer } from "../container";
+import { clickhouseQuery } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { exportResources } from "../import/live-export";
+import { UndeclarableObjectsError } from "../../core/undeclarable";
+
+const enabled = await dockerAvailable();
+let server: ScratchServer | undefined;
+
+const SEED = [
+  "CREATE DATABASE analytics",
+  "CREATE TABLE analytics.events (id UInt64) ENGINE = MergeTree ORDER BY id",
+  "CREATE QUOTA q_reports FOR INTERVAL 1 hour MAX queries = 100",
+  "CREATE SETTINGS PROFILE p_reports SETTINGS max_threads = 2",
+  "CREATE NAMED COLLECTION nc_s3 AS url = 'https://example.com/bucket'",
+  "CREATE WORKLOAD all",
+];
+
+beforeAll(async () => {
+  if (!enabled) return;
+  server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-undeclarable" });
+  for (const stmt of SEED) await clickhouseQuery(server.endpoint, stmt);
+}, 600_000);
+
+afterAll(async () => {
+  await server?.stop();
+});
+
+const run = (extra: Record<string, unknown> = {}) => exportResources({ environment: "x", config: {}, env: { CLICKHOUSE_URL: server!.endpoint.url }, ...extra });
+
+const NAMED = ["named collection nc_s3", "workload all"];
+
+describe.skipIf(!enabled)("importing a server with objects chant cannot declare", () => {
+  test("fails, naming every one of them", async () => {
+    const err = await run().then(
+      () => undefined,
+      (e: unknown) => e,
+    );
+    expect(err).toBeInstanceOf(UndeclarableObjectsError);
+    const objects = (err as UndeclarableObjectsError).objects.map((o) => `${o.kind} ${o.schema ? `${o.schema}.` : ""}${o.name}`);
+    expect([...objects].sort()).toEqual(NAMED);
+  }, 120_000);
+
+  test("with skipUndeclarable, imports the rest and lists them as warnings", async () => {
+    const ir = await run({ skipUndeclarable: true });
+    expect(ir.resources.map((r) => r.properties.name)).toEqual(["analytics", "events"]);
+    const left = (ir.warnings ?? []).filter((w) => w.includes("is left out (--skip-undeclarable)"));
+    expect(left.map((w) => w.split(" is left out")[0]).sort()).toEqual(NAMED);
+  }, 120_000);
+});

package/src/clickhouse/live/undeclarable.ts

@@ -0,0 +1,66 @@
+/**
+ * The objects on a live ClickHouse server of the kinds no declaration holds
+ * (#3911), found so import can refuse them by name instead of leaving them
+ * out (`../../core/undeclarable.ts`).
+ *
+ * One `system.*` query per kind the coverage table marks refused
+ * (`../../coverage/clickhouse.json`), keyed by the row's `kind`. A kind whose
+ * row is flipped to declared is no longer asked about.
+ *
+ * What the server's configuration files define (`storage = 'users_xml'`, a
+ * named collection whose source is the config, an executable function) is
+ * the server's, not a schema's, and is left out; so is what is in a database
+ * out of scope. A server-wide object (a quota, a settings profile, a
+ * workload) is always looked for.
+ */
+
+import { clickhouseQuery } from "../http";
+import type { ClickHouseTarget } from "./bind";
+import { SYSTEM_DATABASES } from "./catalog";
+import { KIND_COVERAGE } from "../../coverage";
+import type { UndeclarableObject } from "../../core/undeclarable";
+
+const quote = (s: string) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+
+/** A query returning `database` (empty for a server-wide object) and `name`; `scope` is the predicate on a database column. */
+type Probe = (scope: (column: string) => string) => string;
+
+const serverWide = (table: string, where = "1") => () => `SELECT '' AS database, name FROM system.${table} WHERE ${where}`;
+const SQL_DEFINED = "storage != 'users_xml'";
+
+export const CLICKHOUSE_PROBES: Readonly<Record<string, Probe>> = {
+  "non-SQL function": serverWide("functions", "origin NOT IN ('System', 'SQLUserDefined', 'ExecutableUserDefined')"),
+  "HTTP handler": serverWide("handlers"),
+  "masking policy": (scope) => `SELECT database, short_name || ' ON ' || table AS name FROM system.masking_policies WHERE ${SQL_DEFINED} AND ${scope("database")}`,
+  "named collection": serverWide("named_collections", "source = 'SQL'"),
+  quota: serverWide("quotas", SQL_DEFINED),
+  "workload resource": serverWide("resources"),
+  "settings profile": serverWide("settings_profiles", SQL_DEFINED),
+  "window view": (scope) => `SELECT database, name FROM system.tables WHERE engine = 'WindowView' AND NOT is_temporary AND ${scope("database")}`,
+  workload: serverWide("workloads"),
+};
+
+/**
+ * Every object in scope of a kind the coverage table refuses, kind by kind
+ * in the table's order, each by database then name. A kind the reader may
+ * not read (no grant on its `system` table) is a problem in `unread`, not a
+ * pass.
+ */
+export async function readUndeclarable(target: ClickHouseTarget): Promise<{ objects: UndeclarableObject[]; unread: string[] }> {
+  const scope = (column: string) =>
+    target.databases ? `${column} IN (${target.databases.map(quote).join(", ")})` : `${column} NOT IN (${[...SYSTEM_DATABASES].map(quote).join(", ")})`;
+  const objects: UndeclarableObject[] = [];
+  const unread: string[] = [];
+  for (const row of KIND_COVERAGE.clickhouse) {
+    if (row.status !== "refused") continue;
+    const probe = CLICKHOUSE_PROBES[row.kind];
+    if (!probe) throw new Error(`no system table query finds a ClickHouse ${row.kind} (src/clickhouse/live/undeclarable.ts)`);
+    try {
+      const rows = await clickhouseQuery<{ database: string; name: string }>(target.endpoint, `/* chant undeclarable: ${row.kind} */ SELECT database, name FROM (${probe(scope)}) ORDER BY database, name`);
+      for (const r of rows) objects.push({ kind: row.kind, ...(r.database ? { schema: r.database } : {}), name: r.name });
+    } catch (err) {
+      unread.push(`${row.kind}: ${err instanceof Error ? err.message.split("\n")[0] : String(err)}`);
+    }
+  }
+  return { objects, unread };
+}

package/src/clickhouse/parser.test.ts

@@ -84,8 +84,42 @@ describe("parsing CREATE TABLE", () => {
     expect(t.engine!.name).toBe("MergeTree");
   });
 
-  test("CREATE TABLE ... AS is refused, since it declares no columns", () => {
-    expect(() => parse("CREATE TABLE t AS other")).toThrow(/column list/);
+  test("CREATE TABLE ... AS a table, AS SELECT, CLONE AS and a table with no columns are refused, each saying why (#3911)", () => {
+    expect(() => parse("CREATE TABLE t AS other")).toThrow(/AS <table> is not declared: the server copies the other table's columns/);
+    expect(() => parse("CREATE TABLE t AS db.other ENGINE = Log")).toThrow(/AS <table> is not declared/);
+    expect(() => parse("CREATE TABLE t ENGINE = Log AS SELECT 1 AS a")).toThrow(/expected the column list \(a table declared without one/);
+    expect(() => parse("CREATE TABLE t AS SELECT 1 AS a")).toThrow(/AS SELECT is not declared/);
+    expect(() => parse("CREATE TABLE t (a UInt8) ENGINE = Memory AS SELECT 1 AS a")).toThrow(/AS SELECT is not declared/);
+    expect(() => parse("CREATE TABLE t CLONE AS other")).toThrow(/CLONE AS is not declared/);
+    expect(() => parse("CREATE TABLE t EMPTY AS SELECT 1")).toThrow(/EMPTY AS SELECT is not declared/);
+  });
+
+  test("a table over a table function, with or without its columns (#3911)", () => {
+    const bareSql = "CREATE TABLE t AS numbers(10)";
+    const bare = parse(bareSql) as TableNode;
+    expect(bare.columns).toEqual([]);
+    expect(bare.engine).toBeUndefined();
+    expect(text(bareSql, bare.asFunction)).toBe("numbers(10)");
+    // As SHOW CREATE prints it: the inferred columns, then the call, then the comment.
+    const shownSql = "CREATE TABLE default.tf\n(\n    `a` UInt64,\n    `b` String\n) AS remote('localhost', 'default.t')\nCOMMENT 'x'";
+    const shown = parse(shownSql) as TableNode;
+    expect(shown.columns.map((c) => c.name)).toEqual(["a", "b"]);
+    expect(text(shownSql, shown.asFunction)).toBe("remote('localhost', 'default.t')");
+    expect(shown.comment).toBeDefined();
+    expect(() => parse("CREATE TABLE t AS numbers(10) ENGINE = Log")).toThrow(/no engine or storage clauses/);
+  });
+
+  test("the CREATE kinds chant does not declare are refused by name, with why (#3911)", () => {
+    expect(() => parse("CREATE LIVE VIEW lv AS SELECT 1")).toThrow(/CREATE LIVE VIEW is not declared: the pinned server no longer has live views/);
+    expect(() => parse("CREATE WINDOW VIEW wv AS SELECT 1")).toThrow(/CREATE WINDOW VIEW is not declared: a window view is experimental/);
+    expect(() => parse("CREATE NAMED COLLECTION c AS user = 'u', password = 'p'")).toThrow(/CREATE NAMED COLLECTION is not declared: a named collection holds credentials/);
+    expect(() => parse("CREATE MASKING POLICY m ON t UPDATE a = 'x' TO r")).toThrow(/only in ClickHouse Cloud/);
+    expect(() => parse("CREATE WORKLOAD all")).toThrow(/CREATE WORKLOAD is not declared: a workload is server-level/);
+    expect(() => parse("CREATE RESOURCE r (WRITE DISK d)")).toThrow(/CREATE RESOURCE is not declared/);
+    expect(() => parse("CREATE TEMPORARY TABLE t (a UInt8)")).toThrow(/CREATE TEMPORARY TABLE is not declared/);
+    expect(() => parse("CREATE INDEX ix ON t (b) TYPE minmax")).toThrow(/CREATE INDEX is not declared: a skip index belongs to its table/);
+    expect(() => parse("CREATE QUOTA q")).toThrow(/CREATE QUOTA is access control: declare it with the quota tag/);
+    expect(() => parse("CREATE SETTINGS PROFILE p")).toThrow(/settingsProfile tag/);
   });
 });
 
@@ -103,6 +137,15 @@ describe("parsing views and databases", () => {
     expect(text(sql, v.select)).toBe("SELECT a, b FROM src");
   });
 
+  test("a refreshable materialized view as SHOW CREATE prints it: the column list right after the schedule (#3911)", () => {
+    const sql =
+      "CREATE MATERIALIZED VIEW default.rmv\nREFRESH EVERY 1 HOUR OFFSET 5 MINUTES\n(\n    `a` UInt64\n)\nENGINE = MergeTree\nORDER BY a\nSETTINGS index_granularity = 8192\nDEFINER = default SQL SECURITY DEFINER\nAS SELECT a\nFROM default.t";
+    const v = parse(sql) as ViewNode;
+    expect(text(sql, v.refresh)).toBe("EVERY 1 HOUR OFFSET 5 MINUTES");
+    expect(v.columns.map((c) => c.name)).toEqual(["a"]);
+    expect(text(sql, v.orderBy)).toBe("a");
+  });
+
   test("a view the server printed back, with its UUID", () => {
     const v = parse("CREATE VIEW default.v UUID '6a1b' (`x` UInt8) AS SELECT 1 AS x") as ViewNode;
     expect(v.columns[0]!.name).toBe("x");