@intentius/chant-lexicon-sql 0.117.0 → 0.118.0

package/src/clickhouse/live/catalog.ts

@@ -1,7 +1,8 @@
 /**
- * Reading a live server's schema: the databases and the tables, views and
- * materialized views in them, from `system.databases`, `system.tables` and
- * `SHOW CREATE`.
+ * Reading a live server's schema: the databases and the tables, views,
+ * materialized views and dictionaries in them, from `system.databases`,
+ * `system.tables` and `SHOW CREATE`, and the SQL user-defined functions,
+ * from `system.functions`.
  */
 
 import { clickhouseQuery } from "../http";
@@ -16,13 +17,15 @@ export interface LiveObject {
   type: ClickHouseEntityType;
   database?: string;
   name: string;
-  /** The engine `system.tables` / `system.databases` report: `MergeTree`, `View`, `MaterializedView`, `Atomic`. */
+  /** The engine `system.tables` / `system.databases` report: `MergeTree`, `View`, `MaterializedView`, `Atomic`, `Dictionary`. */
   engine: string;
   uuid?: string;
   /** The object's comment as the server holds it, chant's ownership trailer included (`../ownership.ts`). */
   comment?: string;
   /** `SHOW CREATE` as the server prints it, less chant's ownership trailer, so it reads as the declaration. */
   statement: string;
+  /** A row policy's table. */
+  table?: string;
 }
 
 const quote = (s: string) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
@@ -38,8 +41,10 @@ export function entityTypeOfEngine(engine: string): ClickHouseEntityType {
 /**
  * Every object in the target's databases, sorted by database then name.
  * A materialized view's inner table (`.inner_id.<uuid>`, `.inner.<name>`)
- * belongs to its view and is left out, as are temporary tables and
- * dictionaries (not declared in slice 1).
+ * belongs to its view and is left out, as are temporary tables. A
+ * dictionary is read (#3682) from `system.tables`, where a reader granted
+ * `SELECT` on the database sees it without a grant on `system.dictionaries`;
+ * a table with the `Dictionary` engine is a table.
  *
  * So are chant's own working objects (`../ownership.ts`
  * `isChantWorkingObject`): a rebuild migration's new, dual-write and retained
@@ -56,9 +61,9 @@ export async function readLiveSchema(target: ClickHouseTarget, opts: { withState
   const databases = await q<{ name: string; engine: string; uuid: string; comment: string }>(
     `SELECT name, engine, toString(uuid) AS uuid, comment FROM system.databases WHERE name ${scope} ORDER BY name`,
   );
-  const tables = await q<{ database: string; name: string; engine: string; uuid: string; comment: string }>(
-    `SELECT database, name, engine, toString(uuid) AS uuid, comment FROM system.tables ` +
-      `WHERE database ${scope} AND NOT is_temporary AND name NOT LIKE '.inner%' AND engine != 'Dictionary' ` +
+  const tables = await q<{ database: string; name: string; engine: string; uuid: string; comment: string; dictionary: number }>(
+    `SELECT database, name, engine, toString(uuid) AS uuid, comment, startsWith(create_table_query, 'CREATE DICTIONARY') AS dictionary FROM system.tables ` +
+      `WHERE database ${scope} AND NOT is_temporary AND name NOT LIKE '.inner%' ` +
       `ORDER BY database, name`,
   );
 
@@ -83,7 +88,7 @@ export async function readLiveSchema(target: ClickHouseTarget, opts: { withState
   for (const t of tables) {
     if (working.has(t.database) || isChantWorkingObject(t.comment)) continue;
     out.push({
-      type: entityTypeOfEngine(t.engine),
+      type: Number(t.dictionary) === 1 ? CLICKHOUSE_ENTITY_TYPES.dictionary : entityTypeOfEngine(t.engine),
       database: t.database,
       name: t.name,
       engine: t.engine,
@@ -92,5 +97,96 @@ export async function readLiveSchema(target: ClickHouseTarget, opts: { withState
       statement: await statement("TABLE", `${ident(t.database)}.${ident(t.name)}`),
     });
   }
+  // SQL user-defined functions belong to no database: every one is read, and a caller keeps those it declares.
+  const functions = await q<{ name: string; statement: string }>(
+    `SELECT name, create_query AS statement FROM system.functions WHERE origin = 'SQLUserDefined' ORDER BY name`,
+  );
+  for (const f of functions) {
+    out.push({ type: CLICKHOUSE_ENTITY_TYPES.function, name: f.name, engine: "", statement: opts.withStatements === false ? "" : f.statement });
+  }
+  return out;
+}
+
+
+/** The access objects a build declares, by kind and name: what `readLiveAccess` reads. */
+export interface DeclaredAccess {
+  kind: string;
+  name: string;
+  database?: string;
+  table?: string;
+}
+
+/**
+ * The users, roles, row policies and grantees' grants the build declares,
+ * as the server holds them (#3682): `SHOW CREATE USER`, `SHOW CREATE ROLE`,
+ * `SHOW CREATE ROW POLICY`, and for a grantee every line of
+ * `SHOW GRANTS FOR`, one per line. Nothing else is read, so a project that
+ * declares none needs no access to `system.users` and the like, and an
+ * object made by hand is never seen. One that does not exist is left out.
+ */
+export async function readLiveAccess(target: ClickHouseTarget, declared: readonly DeclaredAccess[], opts: { withStatements?: boolean } = {}): Promise<LiveObject[]> {
+  if (declared.length === 0) return [];
+  const q = <T>(sql: string) => clickhouseQuery<T>(target.endpoint, sql);
+  const first = async (sql: string) => (opts.withStatements === false ? "" : Object.values((await q<Record<string, string>>(sql))[0] ?? {})[0] ?? "");
+  const all = async (sql: string) => (await q<Record<string, string>>(sql)).map((r) => Object.values(r)[0] ?? "");
+  const names = (kind: string) => [...new Set(declared.filter((d) => d.kind === kind).map((d) => d.name))];
+  const users = names("user");
+  const roles = names("role");
+  const grantees = names("grants");
+  const principals = [...new Set([...users, ...roles, ...grantees])];
+  const list = (xs: string[]) => xs.map(quote).join(", ");
+  const existingUsers = new Set(principals.length ? (await q<{ name: string }>(`SELECT name FROM system.users WHERE name IN (${list(principals)})`)).map((r) => r.name) : []);
+  const existingRoles = new Set(principals.length ? (await q<{ name: string }>(`SELECT name FROM system.roles WHERE name IN (${list(principals)})`)).map((r) => r.name) : []);
+  const out: LiveObject[] = [];
+  for (const name of users.filter((u) => existingUsers.has(u))) {
+    out.push({ type: CLICKHOUSE_ENTITY_TYPES.user, name, engine: "", statement: await first(`SHOW CREATE USER ${ident(name)}`) });
+  }
+  for (const name of roles.filter((r) => existingRoles.has(r))) {
+    out.push({ type: CLICKHOUSE_ENTITY_TYPES.role, name, engine: "", statement: await first(`SHOW CREATE ROLE ${ident(name)}`) });
+  }
+  const policies = declared.filter((d) => d.kind === "rowPolicy");
+  if (policies.length > 0) {
+    const live = await q<{ name: string; database: string; table: string }>(
+      `SELECT short_name AS name, database, table FROM system.row_policies WHERE short_name IN (${list(policies.map((p) => p.name))})`,
+    );
+    for (const p of policies) {
+      if (!live.some((l) => l.name === p.name && l.database === p.database && l.table === p.table)) continue;
+      out.push({
+        type: CLICKHOUSE_ENTITY_TYPES.rowPolicy,
+        name: p.name,
+        database: p.database!,
+        table: p.table!,
+        engine: "",
+        statement: await first(`SHOW CREATE ROW POLICY ${ident(p.name)} ON ${ident(p.database!)}.${ident(p.table!)}`),
+      });
+    }
+  }
+  for (const name of grantees.filter((g) => existingUsers.has(g) || existingRoles.has(g))) {
+    const lines = opts.withStatements === false ? [] : await all(`SHOW GRANTS FOR ${ident(name)}`);
+    out.push({ type: CLICKHOUSE_ENTITY_TYPES.grant, name, engine: "", statement: lines.join("\n") });
+  }
+  return out;
+}
+
+const ACCESS_KINDS = new Set(["user", "role", "rowPolicy", "grants"]);
+
+/** The access objects among declared canonical objects, for `readLiveAccess`. */
+export function declaredAccess(objects: ReadonlyArray<{ kind: string; name: string; database?: string; table?: string }>): DeclaredAccess[] {
+  return objects
+    .filter((o) => ACCESS_KINDS.has(o.kind))
+    .map((o) => ({ kind: o.kind, name: o.name, ...(o.database !== undefined ? { database: o.database } : {}), ...(o.table !== undefined ? { table: o.table } : {}) }));
+}
+
+/** The users, roles and row policies among declared entities, for `readLiveAccess`. */
+export function accessOf(entities: ReadonlyMap<string, { entityType: string; props: Record<string, unknown> }>, defaultDatabase: string): DeclaredAccess[] {
+  const kinds: Record<string, string> = { [CLICKHOUSE_ENTITY_TYPES.user]: "user", [CLICKHOUSE_ENTITY_TYPES.role]: "role", [CLICKHOUSE_ENTITY_TYPES.rowPolicy]: "rowPolicy" };
+  const out: DeclaredAccess[] = [];
+  for (const e of entities.values()) {
+    const kind = kinds[e.entityType];
+    if (!kind) continue;
+    const name = String(e.props.name ?? "");
+    if (kind === "rowPolicy") out.push({ kind, name, database: typeof e.props.database === "string" ? e.props.database : defaultDatabase, table: String(e.props.table ?? "") });
+    else out.push({ kind, name });
+  }
   return out;
 }

package/src/clickhouse/live/describe-resources.ts

@@ -2,8 +2,9 @@
  * `describeResources()` for the ClickHouse dialect: which declared databases,
  * tables and views exist on the environment's server.
  *
- * One read of `system.databases` and `system.tables` per run, then a lookup
- * per declared entity, so N entities are two queries, not N. An object the
+ * One read of `system.databases`, `system.tables` and `system.functions` per
+ * run, then a lookup per declared entity, so N entities are three queries,
+ * not N. An object the
  * catalog does not list is absent, and the `queried` address says which
  * server and name were asked. A server that cannot be reached, or refuses the
  * credentials, leaves every entity unobserved with that reason: never absent.
@@ -23,8 +24,9 @@ import {
   type ObserverAdapter,
 } from "@intentius/chant/observation";
 import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "./bind";
-import { readLiveSchema, type LiveObject } from "./catalog";
+import { accessOf, readLiveAccess, readLiveSchema, type DeclaredAccess, type LiveObject } from "./catalog";
 import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
+import { objectKey } from "../plan/normalize";
 import { isChantManaged, readMarker, stripMarker } from "../ownership";
 
 interface Bound {
@@ -32,22 +34,29 @@ interface Bound {
   byKey: Map<string, LiveObject>;
 }
 
-const keyOf = (database: string | undefined, name: string) => `${database ?? ""}.${name}`;
 
 /** The database and name a declared entity is created as. */
 export function declaredAddress(entity: DeclaredEntity, defaultDatabase: string): { database?: string; name: string } {
   const name = String(entity.props.name ?? "");
-  if (entity.type === CLICKHOUSE_ENTITY_TYPES.database) return { name };
+  if (
+    entity.type === CLICKHOUSE_ENTITY_TYPES.database ||
+    entity.type === CLICKHOUSE_ENTITY_TYPES.function ||
+    entity.type === CLICKHOUSE_ENTITY_TYPES.user ||
+    entity.type === CLICKHOUSE_ENTITY_TYPES.role
+  ) {
+    return { name };
+  }
   return { database: typeof entity.props.database === "string" ? entity.props.database : defaultDatabase, name };
 }
 
-function adapter(options: BindOptions & { owned?: boolean }): ObserverAdapter<Bound> {
+function adapter(options: BindOptions & { owned?: boolean; access?: (defaultDatabase: string) => DeclaredAccess[] }): ObserverAdapter<Bound> {
   return {
     async bind() {
       const target = await bindClickHouse(options);
       const byKey = new Map<string, LiveObject>();
-      for (const o of await readLiveSchema(target, { withStatements: false })) {
-        byKey.set(o.type === CLICKHOUSE_ENTITY_TYPES.database ? keyOf(undefined, o.name) : keyOf(o.database, o.name), o);
+      const access = await readLiveAccess(target, options.access?.(target.defaultDatabase) ?? [], { withStatements: false });
+      for (const o of [...(await readLiveSchema(target, { withStatements: false })), ...access]) {
+        byKey.set(objectKey(o), o);
       }
       return { target, byKey };
     },
@@ -56,9 +65,13 @@ function adapter(options: BindOptions & { owned?: boolean }): ObserverAdapter<Bo
       if (!entity.type.startsWith("ClickHouse::")) {
         return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
       }
+      if (entity.type === CLICKHOUSE_ENTITY_TYPES.grant) {
+        return { unobserved: { reason: "unsupported-kind", detail: "a grant is compared as part of its grantees' grants, which chant sql plan reports" } };
+      }
       const { database, name } = declaredAddress(entity, target.defaultDatabase);
       const queried = `${target.endpoint.url} ${database ? `${database}.` : ""}${name}`;
-      const live = byKey.get(keyOf(database, name));
+      const table = typeof entity.props.table === "string" ? entity.props.table : undefined;
+      const live = byKey.get(objectKey({ type: entity.type, ...(database !== undefined ? { database } : {}), name, ...(table !== undefined ? { table } : {}) }));
       if (!live) return { absent: true, queried };
       const owned = isChantManaged(live.comment);
       if (options.owned && !owned) {
@@ -94,5 +107,5 @@ export async function describeResources(
     const entity = options.entities.get(name);
     return { name, type: entity?.entityType ?? "", props: entity?.props ?? {} };
   });
-  return observeEntities(declared, adapter(options));
+  return observeEntities(declared, adapter({ ...options, access: (db) => accessOf(options.entities, db) }));
 }

package/src/clickhouse/live/dictionaries.e2e.test.ts

@@ -0,0 +1,186 @@
+/**
+ * Dictionaries against a server (#3653, #3682).
+ *
+ * On a single node: a declared dictionary is created with chant's marker,
+ * plans with no changes and reads back with no drift; a layout and a
+ * lifetime changed in the declaration are SQLCH245 and the apply replaces
+ * the dictionary; a dictionary made by hand in a declared database is
+ * listed, as a drop, never left out; a prune drops the declared one once
+ * the build no longer holds it, and leaves the foreign one. A table with
+ * the `Dictionary` engine is read as a table.
+ *
+ * On a Replicated database (two replicas of the pinned server in Docker):
+ * the dictionary is created once, through the database, with no
+ * `ON CLUSTER`, and both replicas hold it and answer `dictGet`.
+ *
+ * The single node: `CLICKHOUSE_URL` (with `CLICKHOUSE_USER` and
+ * `CLICKHOUSE_PASSWORD`) names a running one, such as the one
+ * `chant emulator up --lexicon sql` starts. Without it, a throwaway server at
+ * the pin, skipped cleanly when Docker is not available. The test writes
+ * only to the database `chant_e2e_3682`, dropped afterwards.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { dockerAvailable, startScratchCluster, startScratchServer, type ScratchCluster, type ScratchServer } from "../container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { database, dictionary, table } from "../entities";
+import { planAgainstServer } from "../plan/commands";
+import { observeResourcesDeep, sqlDeepNormalizationHooks } from "../plan/deep";
+import { clickhouseApply } from "../../op/activities/clickhouse-apply";
+
+const DB = "chant_e2e_3682";
+const fromEnv = process.env.CLICKHOUSE_URL;
+const docker = await dockerAvailable();
+const enabled = fromEnv !== undefined || docker;
+let server: ScratchServer | undefined;
+let endpoint: ClickHouseEndpoint;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-dictionaries-"));
+const q = <T = Record<string, unknown>>(sql: string, at: ClickHouseEndpoint = endpoint) => clickhouseQuery<T>(at, sql);
+
+beforeAll(async () => {
+  if (!enabled) return;
+  if (fromEnv !== undefined) {
+    endpoint = {
+      url: fromEnv,
+      ...(process.env.CLICKHOUSE_USER ? { user: process.env.CLICKHOUSE_USER } : {}),
+      ...(process.env.CLICKHOUSE_PASSWORD ? { password: process.env.CLICKHOUSE_PASSWORD } : {}),
+    };
+  } else {
+    server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-dictionaries" });
+    endpoint = server.endpoint;
+  }
+  await q(`DROP DATABASE IF EXISTS ${DB} SYNC`);
+}, 600_000);
+
+afterAll(async () => {
+  if (enabled) await q(`DROP DATABASE IF EXISTS ${DB} SYNC`).catch(() => undefined);
+  await server?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+const envOf = (at: ClickHouseEndpoint, topology?: string) => ({
+  CLICKHOUSE_URL: at.url,
+  ...(at.user ? { CLICKHOUSE_USER: at.user } : {}),
+  ...(at.password ? { CLICKHOUSE_PASSWORD: at.password } : {}),
+  ...(topology ? { CLICKHOUSE_TOPOLOGY: topology } : {}),
+});
+const OWNERSHIP = { stack: "e2e3682", env: "test" };
+
+type Entity = { entityType: string; props: { ddl: string } };
+function writeBuild(file: string, declared: Record<string, Entity>): string {
+  const path = join(dir, file);
+  const objects = Object.entries(declared).map(([k, e]) => ({ export: k, type: e.entityType, ddl: e.props.ddl, dependsOn: [] }));
+  writeFileSync(path, JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects }));
+  return path;
+}
+
+async function apply(at: ClickHouseEndpoint, buildPath: string, opts: { prune?: boolean; topology?: string } = {}) {
+  const sent: string[] = [];
+  const outcome = await clickhouseApply({ buildPath, environment: "test", ...(opts.prune ? { prune: true } : {}) }, undefined, {
+    config: { ownership: OWNERSHIP },
+    env: envOf(at, opts.topology),
+    log: (l) => void (/^[A-Z]+ /.test(l) && sent.push(l)),
+  });
+  return { outcome, sent };
+}
+
+const ratesDict = (layout: string, lifetime: string) =>
+  dictionary([
+    `CREATE DICTIONARY ${DB}.rates_dict (code String, rate Float64 DEFAULT 1) PRIMARY KEY code SOURCE(clickhouse(table 'rates' db '${DB}')) LAYOUT(${layout}) LIFETIME(${lifetime}) COMMENT 'exchange rates'`,
+  ] as unknown as TemplateStringsArray);
+
+/** The props a deep diff compares: the declaration's, less what the hooks prune. */
+const compared = (props: object): Record<string, unknown> =>
+  Object.fromEntries(Object.entries(JSON.parse(JSON.stringify(props)) as Record<string, unknown>).filter(([k]) => !sqlDeepNormalizationHooks.prune!({ pattern: k } as never)));
+
+describe.skipIf(!enabled)("a dictionary on a single node (#3682)", () => {
+  const shop = database([`CREATE DATABASE ${DB}`] as unknown as TemplateStringsArray);
+  const rates = table([`CREATE TABLE ${DB}.rates (code String, rate Float64) ENGINE = MergeTree ORDER BY code`] as unknown as TemplateStringsArray);
+
+  test("created, planned, read back, changed, listed beside a foreign one, and pruned", async () => {
+    const v1 = { shop, rates, ratesDict: ratesDict("complex_key_hashed()", "300") };
+    const first = await apply(endpoint, writeBuild("v1.json", v1));
+    expect(first.outcome.failed).toEqual([]);
+    expect(first.outcome.applied.map((a) => [a.name, a.action])).toEqual([
+      [DB, "created"],
+      [`${DB}.rates`, "created"],
+      [`${DB}.rates_dict`, "created"],
+    ]);
+    await q(`INSERT INTO ${DB}.rates VALUES ('EUR', 1.1), ('GBP', 1.3)`);
+    expect((await q<{ r: number }>(`SELECT dictGet('${DB}.rates_dict', 'rate', 'GBP') AS r`))[0]!.r).toBe(1.3);
+    // The server prints the layout upper case, the lifetime as MIN and MAX: still the declaration.
+    expect((await q<{ s: string }>(`SELECT create_table_query AS s FROM system.tables WHERE database = '${DB}' AND name = 'rates_dict'`))[0]!.s).toMatch(/LIFETIME\(MIN 0 MAX 300\) LAYOUT\(COMPLEX_KEY_HASHED\(\)\)/);
+
+    const diff = await planAgainstServer("test", writeBuild("v1.json", v1), { config: {}, env: envOf(endpoint) });
+    expect(diff.changes).toEqual([]);
+    expect(diff.unreadable).toBeUndefined();
+    const entities = new Map(Object.entries(v1).map(([k, e]) => [k, { entityType: e.entityType, props: e.props as unknown as Record<string, unknown> }]));
+    const deep = await observeResourcesDeep({ environment: "test", entityNames: [...entities.keys()], entities, config: {}, env: envOf(endpoint) });
+    expect(deep.unobserved ?? {}).toEqual({});
+    expect(compared(deep.resources.ratesDict!.properties)).toEqual(compared(v1.ratesDict.props));
+    expect(deep.resources.ratesDict!.type).toBe("ClickHouse::Dictionary");
+
+    // A foreign dictionary and a Dictionary-engine table, made by hand: listed, not left out.
+    await q(`CREATE DICTIONARY ${DB}.foreign_dict (code String, rate Float64) PRIMARY KEY code SOURCE(CLICKHOUSE(TABLE 'rates' DB '${DB}')) LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(0)`);
+    await q(`CREATE TABLE ${DB}.rates_view (code String, rate Float64) ENGINE = Dictionary(${DB}.rates_dict)`);
+
+    const v2 = { ...v1, ratesDict: ratesDict("complex_key_sparse_hashed()", "MIN 10 MAX 600") };
+    const planned = await planAgainstServer("test", writeBuild("v2.json", v2), { config: {}, env: envOf(endpoint) });
+    expect(planned.changes.map((c) => [c.object, c.field, c.rule])).toEqual([
+      [`ratesDict (${DB}.rates_dict)`, "layout", "SQLCH245"],
+      [`ratesDict (${DB}.rates_dict)`, "lifetime", "SQLCH245"],
+      [`${DB}.foreign_dict`, "object", "SQLCH250"],
+      [`${DB}.rates_view`, "object", "SQLCH250"],
+    ]);
+
+    const changed = await apply(endpoint, writeBuild("v2.json", v2));
+    expect(changed.sent).toEqual([expect.stringMatching(new RegExp(`^CREATE OR REPLACE DICTIONARY ${DB}\\.rates_dict .*LAYOUT\\(complex_key_sparse_hashed\\(\\)\\) LIFETIME\\(MIN 10 MAX 600\\) COMMENT 'exchange rates \\[chant`))]);
+    expect((await q<{ r: number }>(`SELECT dictGet('${DB}.rates_dict', 'rate', 'EUR') AS r`))[0]!.r).toBe(1.1);
+    expect((await planAgainstServer("test", writeBuild("v2.json", v2), { config: {}, env: envOf(endpoint) })).changes.map((c) => c.object)).toEqual([
+      `${DB}.foreign_dict`,
+      `${DB}.rates_view`,
+    ]);
+
+    // Pruned once the build no longer declares it; the foreign dictionary carries no marker and stays.
+    await q(`DROP TABLE ${DB}.rates_view SYNC`);
+    const pruned = await apply(endpoint, writeBuild("v3.json", { shop, rates }), { prune: true });
+    expect(pruned.outcome.pruned.map((p) => p.name)).toEqual([`${DB}.rates_dict`]);
+    expect(pruned.sent).toEqual([`DROP DICTIONARY \`${DB}\`.\`rates_dict\` SYNC`]);
+    const left = await q<{ name: string }>(`SELECT name FROM system.tables WHERE database = '${DB}' ORDER BY name`);
+    expect(left.map((t) => t.name)).toEqual(["foreign_dict", "rates"]);
+  }, 300_000);
+});
+
+describe.skipIf(!docker)("a dictionary in a Replicated database (#3682)", () => {
+  let cluster: ScratchCluster | undefined;
+  beforeAll(async () => {
+    cluster = await startScratchCluster(clickhouseImage(), { replicas: 2, namePrefix: "chant-sql-dictionaries-repl" });
+  }, 600_000);
+  afterAll(async () => {
+    await cluster?.stop();
+  });
+
+  test("created once through the database, held by both replicas, with no ON CLUSTER", async () => {
+    const [r1, r2] = cluster!.replicas as [ClickHouseEndpoint, ClickHouseEndpoint];
+    const shop = database([`CREATE DATABASE ${DB}`] as unknown as TemplateStringsArray);
+    const rates = table([`CREATE TABLE ${DB}.rates (code String, rate Float64) ENGINE = MergeTree ORDER BY code`] as unknown as TemplateStringsArray);
+    const build = writeBuild("repl.json", { shop, rates, ratesDict: ratesDict("complex_key_hashed()", "300") });
+    const first = await apply(r1, build, { topology: "replicated" });
+    expect(first.outcome.failed).toEqual([]);
+    expect(first.sent.join("\n")).not.toMatch(/ON CLUSTER/);
+    // The second replica joins the database with the statement the first was sent, and replays the rest from its log.
+    await q(first.sent[0]!, r2);
+    await q(`SYSTEM SYNC DATABASE REPLICA ${DB}`, r2);
+    await q(`INSERT INTO ${DB}.rates VALUES ('EUR', 1.1)`, r1);
+    await q(`SYSTEM SYNC REPLICA ${DB}.rates`, r2);
+    expect((await q<{ r: number }>(`SELECT dictGet('${DB}.rates_dict', 'rate', 'EUR') AS r`, r2))[0]!.r).toBe(1.1);
+    // On the other replica the dictionary is as declared: nothing to send.
+    const again = await apply(r2, build, { topology: "replicated" });
+    expect(again.sent).toEqual([]);
+    expect(again.outcome.notAttempted).toEqual([]);
+  }, 300_000);
+});

package/src/clickhouse/live/functions.e2e.test.ts

@@ -0,0 +1,155 @@
+/**
+ * SQL user-defined functions against a server (#3682).
+ *
+ * On a single node: a declared function is created as declared, plans with
+ * no changes although the server prints its expression its own way, and
+ * reads back with no drift; a changed expression is SQLCH260 and the apply
+ * replaces it; a function the build does not declare is never read, so
+ * never listed or dropped.
+ *
+ * Beside a Replicated database (two replicas of the pinned server in
+ * Docker): a function belongs to no database, so the database's log does
+ * not carry it. With no cluster named, each replica's apply creates it
+ * there, and a second apply sends nothing.
+ *
+ * The single node: `CLICKHOUSE_URL` (with `CLICKHOUSE_USER` and
+ * `CLICKHOUSE_PASSWORD`) names a running one, such as the one
+ * `chant emulator up --lexicon sql` starts. Without it, a throwaway server
+ * at the pin, skipped cleanly when Docker is not available. A function is
+ * global to the server, so the test names its functions `chant_e2e_3682_*`
+ * and drops them afterwards.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { dockerAvailable, startScratchCluster, startScratchServer, type ScratchCluster, type ScratchServer } from "../container";
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { database, func, table } from "../entities";
+import { planAgainstServer } from "../plan/commands";
+import { observeResourcesDeep, sqlDeepNormalizationHooks } from "../plan/deep";
+import { clickhouseApply } from "../../op/activities/clickhouse-apply";
+
+const PREFIX = "chant_e2e_3682";
+const LINEAR = `${PREFIX}_linear`;
+const FOREIGN = `${PREFIX}_foreign`;
+const DB = `${PREFIX}_fn`;
+const fromEnv = process.env.CLICKHOUSE_URL;
+const docker = await dockerAvailable();
+const enabled = fromEnv !== undefined || docker;
+let server: ScratchServer | undefined;
+let endpoint: ClickHouseEndpoint;
+const dir = mkdtempSync(join(tmpdir(), "chant-sql-functions-"));
+const q = <T = Record<string, unknown>>(sql: string, at: ClickHouseEndpoint = endpoint) => clickhouseQuery<T>(at, sql);
+const cleanup = async (at: ClickHouseEndpoint) => {
+  for (const f of [LINEAR, FOREIGN]) await q(`DROP FUNCTION IF EXISTS ${f}`, at).catch(() => undefined);
+  await q(`DROP DATABASE IF EXISTS ${DB} SYNC`, at).catch(() => undefined);
+};
+
+beforeAll(async () => {
+  if (!enabled) return;
+  if (fromEnv !== undefined) {
+    endpoint = {
+      url: fromEnv,
+      ...(process.env.CLICKHOUSE_USER ? { user: process.env.CLICKHOUSE_USER } : {}),
+      ...(process.env.CLICKHOUSE_PASSWORD ? { password: process.env.CLICKHOUSE_PASSWORD } : {}),
+    };
+  } else {
+    server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-functions" });
+    endpoint = server.endpoint;
+  }
+  await cleanup(endpoint);
+}, 600_000);
+
+afterAll(async () => {
+  if (enabled) await cleanup(endpoint);
+  await server?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+const envOf = (at: ClickHouseEndpoint, topology?: string) => ({
+  CLICKHOUSE_URL: at.url,
+  ...(at.user ? { CLICKHOUSE_USER: at.user } : {}),
+  ...(at.password ? { CLICKHOUSE_PASSWORD: at.password } : {}),
+  ...(topology ? { CLICKHOUSE_TOPOLOGY: topology } : {}),
+});
+
+type Entity = { entityType: string; props: { ddl: string } };
+function writeBuild(file: string, declared: Record<string, Entity>): string {
+  const path = join(dir, file);
+  const objects = Object.entries(declared).map(([k, e]) => ({ export: k, type: e.entityType, ddl: e.props.ddl, dependsOn: [] }));
+  writeFileSync(path, JSON.stringify({ dialect: "clickhouse", applyOrder: objects.map((o) => o.export), objects }));
+  return path;
+}
+
+async function apply(at: ClickHouseEndpoint, buildPath: string, opts: { prune?: boolean; topology?: string } = {}) {
+  const sent: string[] = [];
+  const outcome = await clickhouseApply({ buildPath, environment: "test", ...(opts.prune ? { prune: true } : {}) }, undefined, {
+    config: { ownership: { stack: "e2e3682", env: "test" } },
+    env: envOf(at, opts.topology),
+    log: (l) => void (/^[A-Z]+ /.test(l) && sent.push(l)),
+  });
+  return { outcome, sent };
+}
+
+const linear = (expr: string) => func([`CREATE FUNCTION ${LINEAR} AS (x, k, b) -> ${expr}`] as unknown as TemplateStringsArray);
+const compared = (props: object): Record<string, unknown> =>
+  Object.fromEntries(Object.entries(JSON.parse(JSON.stringify(props)) as Record<string, unknown>).filter(([k]) => !sqlDeepNormalizationHooks.prune!({ pattern: k } as never)));
+
+describe.skipIf(!enabled)("a function on a single node (#3682)", () => {
+  test("created, planned and read back as declared, changed, and a foreign one left alone", async () => {
+    const v1 = { linear: linear("k*x + b") };
+    const first = await apply(endpoint, writeBuild("v1.json", v1));
+    expect(first.outcome.failed).toEqual([]);
+    expect(first.sent).toEqual([`CREATE FUNCTION ${LINEAR} AS (x, k, b) -> k*x + b`]);
+    expect((await q<{ y: number }>(`SELECT ${LINEAR}(2, 3, 1) AS y`))[0]!.y).toBe(7);
+
+    // The server prints `((k * x) + b)`: its formatter says that is the declaration.
+    await q(`CREATE FUNCTION ${FOREIGN} AS (x) -> x`);
+    const diff = await planAgainstServer("test", writeBuild("v1.json", v1), { config: {}, env: envOf(endpoint) });
+    expect(diff.changes).toEqual([]);
+    const entities = new Map(Object.entries(v1).map(([k, e]) => [k, { entityType: e.entityType, props: e.props as unknown as Record<string, unknown> }]));
+    const deep = await observeResourcesDeep({ environment: "test", entityNames: [...entities.keys()], entities, config: {}, env: envOf(endpoint) });
+    expect(deep.unobserved ?? {}).toEqual({});
+    expect(compared(deep.resources.linear!.properties)).toEqual(compared(v1.linear.props));
+
+    const v2 = { linear: linear("k*x + b + 1") };
+    const planned = await planAgainstServer("test", writeBuild("v2.json", v2), { config: {}, env: envOf(endpoint) });
+    expect(planned.changes.map((c) => [c.object, c.field, c.rule])).toEqual([[`linear (function ${LINEAR})`, "lambda", "SQLCH260"]]);
+    const changed = await apply(endpoint, writeBuild("v2.json", v2), { prune: true });
+    expect(changed.sent).toEqual([`CREATE OR REPLACE FUNCTION ${LINEAR} AS (x, k, b) -> k*x + b + 1`]);
+    expect(changed.outcome.pruned).toEqual([]);
+    expect((await q<{ y: number }>(`SELECT ${LINEAR}(2, 3, 1) AS y`))[0]!.y).toBe(8);
+    // The foreign function is never read, so never dropped.
+    expect((await q<{ n: number }>(`SELECT count() AS n FROM system.functions WHERE name = '${FOREIGN}'`))[0]!.n).toBe(1);
+  }, 300_000);
+});
+
+describe.skipIf(!docker)("a function beside a Replicated database (#3682)", () => {
+  let cluster: ScratchCluster | undefined;
+  beforeAll(async () => {
+    cluster = await startScratchCluster(clickhouseImage(), { replicas: 2, namePrefix: "chant-sql-functions-repl" });
+  }, 600_000);
+  afterAll(async () => {
+    await cluster?.stop();
+  });
+
+  test("each replica's apply creates it there, with no ON CLUSTER, and the next apply sends nothing", async () => {
+    const [r1, r2] = cluster!.replicas as [ClickHouseEndpoint, ClickHouseEndpoint];
+    const shop = database([`CREATE DATABASE ${DB}`] as unknown as TemplateStringsArray);
+    const rates = table([`CREATE TABLE ${DB}.rates (code String, rate Float64) ENGINE = MergeTree ORDER BY code`] as unknown as TemplateStringsArray);
+    const build = writeBuild("repl.json", { shop, rates, linear: linear("k*x + b") });
+    const first = await apply(r1, build, { topology: "replicated" });
+    expect(first.outcome.failed).toEqual([]);
+    expect(first.sent.join("\n")).not.toMatch(/ON CLUSTER/);
+    await q(first.sent[0]!, r2);
+    await q(`SYSTEM SYNC DATABASE REPLICA ${DB}`, r2);
+    const second = await apply(r2, build, { topology: "replicated" });
+    expect(second.sent).toEqual([`CREATE FUNCTION ${LINEAR} AS (x, k, b) -> k*x + b`]);
+    expect((await q<{ y: number }>(`SELECT ${LINEAR}(2, 3, 1) AS y`, r2))[0]!.y).toBe(7);
+    expect((await apply(r1, build, { topology: "replicated" })).sent).toEqual([]);
+    expect((await apply(r2, build, { topology: "replicated" })).sent).toEqual([]);
+  }, 300_000);
+});

package/src/clickhouse/live/live.test.ts

@@ -109,10 +109,10 @@ describe("describeResources", () => {
     });
   });
 
-  test("reads the catalog in two queries, whatever the number of entities", async () => {
+  test("reads the catalog in three queries, whatever the number of entities", async () => {
     const before = ok.queries.length;
     await run(ok.url);
-    expect(ok.queries.length - before).toBe(2);
+    expect(ok.queries.length - before).toBe(3);
   });
 });