@intentius/chant-lexicon-sql 0.124.31 → 0.124.33

package/dist/clickhouse/import/ir.js

@@ -0,0 +1,160 @@
+/**
+ * ClickHouse DDL into the import IR, and back out as `database`, `table` and
+ * `view` declarations.
+ *
+ * `chant import --from <env>` reads each object's `SHOW CREATE` and `chant
+ * import schema.sql` reads a file of CREATE statements; both arrive here as
+ * statements. The generated declaration is the statement itself, the
+ * server's canonical form when it came from a server (#3046 question 5), with
+ * two changes:
+ *
+ * - a qualified name of another imported object becomes a reference to its
+ *   export (`FROM analytics.events` is `FROM ${events}`), and the database
+ *   part of an object's own name becomes a reference to the database's export,
+ *   so dependency order survives the import;
+ * - a backquoted identifier that needs no quoting is written bare, and one that
+ *   does is double-quoted, so the template has no backquotes to escape.
+ *
+ * Columns written by name inside a view's SELECT stay text: which table a
+ * bare name belongs to is the server's to resolve, and a guess would be
+ * wrong lineage. An imported view therefore records its reads, not its column
+ * lineage, until its column references are written as `${t.columns.c}`.
+ */
+import { isTrivia, tokenizeText } from "../tokens.js";
+import { parseCreate, unquote } from "../parser.js";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities.js";
+import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse.js";
+const JS_RESERVED = new Set("break case catch class const continue debugger default delete do else enum export extends false finally for function if import in instanceof new null return super switch this throw true try typeof var void while with yield let static implements interface package private protected public await".split(" "));
+function camel(name) {
+    const words = name.split(/[^A-Za-z0-9]+/).filter(Boolean);
+    if (words.length === 0)
+        return "object";
+    const s = words.map((w, i) => (i === 0 ? w[0].toLowerCase() + w.slice(1) : w[0].toUpperCase() + w.slice(1))).join("");
+    return /^[0-9]/.test(s) ? `t${s}` : s;
+}
+/**
+ * Export names for a set of objects. A table or view is its name in camel
+ * case, prefixed with its database when two databases share a name; a
+ * database is its name with `Db` after it, so `analytics` the database and
+ * `analytics` a table do not collide.
+ */
+export function exportNames(objects) {
+    const relations = objects.filter((o) => o.type !== CLICKHOUSE_ENTITY_TYPES.database);
+    const count = new Map();
+    for (const o of relations)
+        count.set(o.name, (count.get(o.name) ?? 0) + 1);
+    const used = new Set();
+    return objects.map((o) => {
+        let base = o.type === CLICKHOUSE_ENTITY_TYPES.database
+            ? `${camel(o.name)}Db`
+            : (count.get(o.name) ?? 0) > 1 && o.database
+                ? camel(`${o.database}_${o.name}`)
+                : camel(o.name);
+        if (JS_RESERVED.has(base))
+            base = `${base}Table`;
+        let name = base;
+        for (let i = 2; used.has(name); i++)
+            name = `${base}${i}`;
+        used.add(name);
+        return name;
+    });
+}
+/** Split a file of statements on top-level `;`. */
+export function splitStatements(text) {
+    const out = [];
+    let current = [];
+    const flush = () => {
+        const stmt = current.map((t) => t.text).join("").trim();
+        if (current.some((t) => !isTrivia(t)))
+            out.push(stmt);
+        current = [];
+    };
+    for (const t of tokenizeText(text, 0)) {
+        if (t.kind === "punct" && t.text === ";")
+            flush();
+        else
+            current.push(t);
+    }
+    flush();
+    return out;
+}
+const statementType = (node) => node.statement === "database"
+    ? CLICKHOUSE_ENTITY_TYPES.database
+    : node.statement === "table"
+        ? CLICKHOUSE_ENTITY_TYPES.table
+        : node.statement === "dictionary"
+            ? CLICKHOUSE_ENTITY_TYPES.dictionary
+            : node.statement === "function"
+                ? CLICKHOUSE_ENTITY_TYPES.function
+                : node.materialized
+                    ? CLICKHOUSE_ENTITY_TYPES.materializedView
+                    : CLICKHOUSE_ENTITY_TYPES.view;
+function nameOf(tokens, span) {
+    const parts = tokens
+        .slice(span.from, span.to)
+        .filter((t) => !isTrivia(t) && !(t.kind === "punct" && t.text === "."))
+        .map((t) => unquote(t.text));
+    return parts.length >= 2 ? { database: parts[parts.length - 2], name: parts[parts.length - 1] } : { name: parts[0] ?? "" };
+}
+/** Read a statement's kind and name. Throws the parser's error when it does not parse. */
+export function describeStatement(ddl) {
+    const tokens = tokenizeText(ddl, 0);
+    const node = parseCreate(tokens);
+    return { type: statementType(node), ...nameOf(tokens, node.name), ddl };
+}
+/**
+ * Leave out what the server adds and the declaration would not say: settings
+ * at the pinned server's default (`index_granularity = 8192`), and a view's
+ * column list, which the server infers from the SELECT. A server re-adds both
+ * on create, so the stripped statement creates the same object.
+ */
+export function stripServerDefaults(ddl) {
+    const tokens = tokenizeText(ddl, 0);
+    let node;
+    try {
+        node = parseCreate(tokens);
+    }
+    catch {
+        return ddl;
+    }
+    const edits = [];
+    if ((node.statement === "table" || node.statement === "view") && node.settings && node.settingsClause) {
+        const text = (s) => tokens.slice(s.from, s.to).map((t) => t.text).join("").trim();
+        const kept = node.settings.filter((s) => {
+            const def = MERGE_TREE_SETTINGS[s.key];
+            return !def || text(s.value).replace(/^'|'$/g, "") !== def.default;
+        });
+        if (kept.length !== node.settings.length) {
+            edits.push({
+                span: node.settingsClause,
+                text: kept.length === 0 ? "" : `SETTINGS ${kept.map((s) => `${s.key} = ${text(s.value)}`).join(", ")}`,
+            });
+        }
+    }
+    if (node.statement === "view" && node.columnsSpan)
+        edits.push({ span: node.columnsSpan, text: "" });
+    if (edits.length === 0)
+        return ddl;
+    edits.sort((a, b) => b.span.from - a.span.from);
+    const out = [...tokens];
+    for (const e of edits) {
+        out.splice(e.span.from, e.span.to - e.span.from, { kind: "ws", text: e.text, part: 0, start: 0, end: 0 });
+    }
+    return out
+        .map((t) => t.text)
+        .join("")
+        .split("\n")
+        .filter((line) => line.trim().length > 0)
+        .join("\n")
+        .replace(/[ \t]+$/gm, "");
+}
+/** The IR for a set of objects. */
+export function objectsToIR(objects, warnings = []) {
+    const names = exportNames(objects);
+    const resources = objects.map((o, i) => ({
+        logicalId: names[i],
+        type: o.type,
+        properties: { ...(o.database ? { database: o.database } : {}), name: o.name, ddl: o.ddl },
+    }));
+    return { resources, parameters: [], ...(warnings.length ? { warnings } : {}) };
+}

package/dist/clickhouse/import/live-export.js

@@ -0,0 +1,131 @@
+/**
+ * Live export: `chant import --from <env>` writes the server's schema as
+ * `database`, `table` and `view` declarations.
+ *
+ * Every object in the environment's databases is read with `SHOW CREATE`, the
+ * server's canonical form, which is the baseline #3046 question 5 asks for.
+ * By default what the server adds on its own is left out (settings at their
+ * default, a view's inferred column list), so the declaration says what an
+ * author would; `verbatim` keeps the statement as the server printed it.
+ *
+ * - `selector.type` is an entity type (`ClickHouse::Table`); `selector.name` is
+ *   an object name, `db.name` or `name`.
+ * - `owned` keeps the objects whose comment carries chant's ownership marker
+ *   (`../ownership.ts`), the trailer the applier stamps. The trailer itself is
+ *   never written into a declaration: `readLiveSchema` takes it off.
+ * - a migration runner's history table (`schema_migrations`,
+ *   `goose_db_version`) is left out with a warning (#3676), as on Postgres.
+ * - A SQL user-defined function belongs to no database, and every project's
+ *   functions sit side by side on a server (#3718). Without a selector, an
+ *   import adopts only the functions the imported objects use (and those
+ *   functions call), and the ones `sql.profiles.<env>.importFunctions` names;
+ *   a warning names the rest. ClickHouse stores a call as the function's body
+ *   (#3745), so a use is found by that body (`./inlined.ts`) as well as by
+ *   the name. A function carries no comment, so `owned` does
+ *   not apply to it.
+ * - An object of a kind no declaration holds (a window view, a quota, a
+ *   settings profile, a named collection, ...; `../live/undeclarable.ts`)
+ *   fails the import, which names every one of them (#3911).
+ *   `skipUndeclarable` (`--skip-undeclarable`) imports the rest and lists
+ *   them as warnings. A selector or `owned` import asks for objects chant can
+ *   hold, so it does not look.
+ */
+import { bindClickHouse } from "../live/bind.js";
+import { readLiveSchema } from "../live/catalog.js";
+import { objectsToIR, stripServerDefaults } from "./ir.js";
+import { isChantManaged } from "../ownership.js";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities.js";
+import { isTrivia, tokenizeText } from "../tokens.js";
+import { unquote } from "../parser.js";
+import { inlinedFunctions } from "./inlined.js";
+import { readUndeclarable } from "../live/undeclarable.js";
+import { refuseUndeclarable } from "../../core/undeclarable.js";
+/** The functions among `names` a statement calls: a name followed by `(`. */
+export function calledFunctions(ddl, names) {
+    const tokens = tokenizeText(ddl, 0).filter((t) => !isTrivia(t));
+    const out = new Set();
+    tokens.forEach((t, i) => {
+        if (t.kind !== "ident" && t.kind !== "qident")
+            return;
+        const next = tokens[i + 1];
+        if (next?.kind !== "punct" || next.text !== "(")
+            return;
+        const name = t.kind === "qident" ? unquote(t.text) : t.text;
+        if (names.has(name))
+            out.add(name);
+    });
+    return [...out];
+}
+/** Whether `patterns` (names, or prefixes ending in `*`) name a function. */
+const named = (patterns, name) => patterns.some((p) => (p.endsWith("*") ? name.startsWith(p.slice(0, -1)) : p === name));
+/**
+ * The functions an import adopts (#3718): those the objects call, those
+ * whose bodies the objects hold in place of a call (#3745, `./inlined.ts`),
+ * those `patterns` name, and the functions those call in turn; and the rest.
+ */
+export function scopedFunctions(objects, functions, patterns = []) {
+    const byName = new Map(functions.map((f) => [f.name, f]));
+    const names = new Set(byName.keys());
+    const adopted = new Set(functions.filter((f) => named(patterns, f.name)).map((f) => f.name));
+    for (const o of objects)
+        for (const n of calledFunctions(o.statement, names))
+            adopted.add(n);
+    for (const n of inlinedFunctions(objects.map((o) => o.statement), functions))
+        adopted.add(n);
+    const queue = [...adopted];
+    while (queue.length > 0) {
+        for (const n of calledFunctions(byName.get(queue.pop()).statement, names)) {
+            if (!adopted.has(n)) {
+                adopted.add(n);
+                queue.push(n);
+            }
+        }
+    }
+    return { adopted: functions.filter((f) => adopted.has(f.name)), skipped: functions.filter((f) => !adopted.has(f.name)).map((f) => f.name) };
+}
+export async function exportResources(options) {
+    const target = await bindClickHouse(options);
+    // A selector or an owned import asks only for what chant can hold.
+    const undeclarable = !options.selector && !options.owned ? await readUndeclarable(target) : { objects: [], unread: [] };
+    const warnings = undeclarable.unread.map((u) => `the server was not checked for objects of one kind chant cannot declare, so some may be left out: ${u}`);
+    refuseUndeclarable("clickhouse", undeclarable.objects, options.skipUndeclarable, warnings);
+    // What is refused, and with --skip-undeclarable left out, is not also read as a table (a window view is in system.tables).
+    const refused = new Set(undeclarable.objects.filter((o) => o.schema !== undefined).map((o) => `${o.schema}.${o.name}`));
+    const all = (await readLiveSchema(target)).filter((o) => !(o.database !== undefined && refused.has(`${o.database}.${o.name}`)));
+    const isFunction = (o) => o.type === CLICKHOUSE_ENTITY_TYPES.function;
+    // A selector picks what it names, functions included; without one, functions go with what calls them (#3718).
+    const live = options.selector ? all : all.filter((o) => !isFunction(o));
+    const selected = live.filter((o) => {
+        if (options.owned && !isChantManaged(o.comment))
+            return false;
+        if (options.selector?.type && o.type !== options.selector.type)
+            return false;
+        const name = options.selector?.name;
+        if (name !== undefined && name !== o.name && name !== `${o.database}.${o.name}`)
+            return false;
+        return true;
+    });
+    if (!options.selector) {
+        const { adopted, skipped } = scopedFunctions(selected.filter((o) => !o.foreign), all.filter(isFunction), target.importFunctions);
+        selected.push(...adopted);
+        if (skipped.length > 0) {
+            warnings.push(`${skipped.length} SQL function${skipped.length === 1 ? "" : "s"} on the server ${skipped.length === 1 ? "is" : "are"} not imported, since no imported object uses ${skipped.length === 1 ? "it" : "them"}: ${skipped.join(", ")}. Name ${skipped.length === 1 ? "it" : "them"} in sql.profiles.${options.environment}.importFunctions to import ${skipped.length === 1 ? "it" : "them"}.`);
+        }
+    }
+    const objects = selected
+        // The `default` database exists on every server; declaring it would make apply create what is there.
+        .filter((o) => !(o.type === "ClickHouse::Database" && o.name === "default"))
+        .filter((o) => {
+        if (!o.foreign)
+            return true;
+        warnings.push(`${o.database ? `${o.database}.` : ""}${o.name} is kept by ${o.foreign}; left out, since declaring it would have chant change what that tool owns`);
+        return false;
+    })
+        .map((o) => ({
+        type: o.type,
+        ...(o.database ? { database: o.database } : {}),
+        name: o.name,
+        ddl: options.verbatim ? o.statement : stripServerDefaults(o.statement),
+    }));
+    return objectsToIR(objects, warnings);
+}

package/dist/clickhouse/import/parser.js

@@ -0,0 +1,17 @@
+/**
+ * `chant import schema.sql` for ClickHouse: a file of CREATE statements into
+ * the import IR. A statement that is not a CREATE, or does not parse, is an
+ * error naming it (`../../files/clickhouse.ts`); none is left out.
+ */
+import { objectsToIR } from "./ir.js";
+import { readClickHouse } from "../../files/clickhouse.js";
+import { SqlFileError } from "../../files/common.js";
+export class ClickHouseSqlParser {
+    parse(content) {
+        const problems = [];
+        const objects = readClickHouse(content, { origin: "the DDL" }, problems);
+        if (problems.length > 0)
+            throw new SqlFileError("the DDL", problems);
+        return objectsToIR(objects);
+    }
+}

package/dist/clickhouse/live/bind.js

@@ -0,0 +1,116 @@
+/**
+ * Which ClickHouse server a chant environment reads: the config lookup and
+ * the binding, shared by observation and live export so both read the same
+ * server.
+ */
+import { ClickHouseQueryError, clickhouseQuery } from "../http.js";
+import { parseTopology, toTopology } from "../topology.js";
+import { credentialSource, POSTGRES_ONLY_TOKEN_SOURCES, TokenSourceError } from "../../token-source.js";
+export const isUnresolvedTarget = (v) => "reason" in v;
+/**
+ * The server an environment reads. Pure: `config` and `env` are passed in.
+ *
+ * 1. `sql.profiles.<environment>`. A profile naming a credential variable that
+ *    is not set is `no-credentials`: the profile said how to authenticate.
+ * 2. `CLICKHOUSE_URL`, with `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` when set,
+ *    and `CLICKHOUSE_TOPOLOGY` (`single`, `cluster:<name>`, `replicated`,
+ *    `cloud`).
+ * 3. Otherwise `no-binding`.
+ */
+export function resolveClickHouseTarget(input) {
+    const env = input.env ?? process.env;
+    const profile = input.environment !== undefined ? input.config?.sql?.profiles?.[input.environment] : undefined;
+    if (profile) {
+        const source = `sql.profiles.${input.environment}`;
+        const endpoint = { url: profile.url.replace(/\/+$/, "") };
+        if (profile.user) {
+            const value = env[profile.user.env];
+            if (value === undefined)
+                return { reason: "no-credentials", detail: `${source}.user names ${profile.user.env}, which is not set` };
+            endpoint.user = value;
+        }
+        const password = profile.password;
+        if (password && "env" in password) {
+            const value = env[password.env];
+            if (value === undefined)
+                return { reason: "no-credentials", detail: `${source}.password names ${password.env}, which is not set` };
+            endpoint.password = value;
+        }
+        else if (password) {
+            if (POSTGRES_ONLY_TOKEN_SOURCES.has(password.token)) {
+                return { reason: "no-credentials", detail: `${source}.password is a ${password.token} token, which only a Postgres server takes; use a command source for ClickHouse` };
+            }
+            endpoint.token = credentialSource(password, { url: profile.url, ...(endpoint.user !== undefined ? { user: endpoint.user } : {}), env, source: `${source}.password` });
+        }
+        return {
+            endpoint,
+            source,
+            ...(profile.databases ? { databases: profile.databases } : {}),
+            defaultDatabase: profile.defaultDatabase ?? "default",
+            ...(profile.topology !== undefined ? { topology: toTopology(profile.topology) } : {}),
+            ...(profile.access === true ? { access: true } : {}),
+            ...(profile.importFunctions ? { importFunctions: profile.importFunctions } : {}),
+        };
+    }
+    const url = env.CLICKHOUSE_URL;
+    if (!url) {
+        const where = input.environment !== undefined ? `sql.profiles.${input.environment} is not declared and ` : "";
+        return { reason: "no-binding", detail: `${where}CLICKHOUSE_URL is not set, so there is no ClickHouse server to read` };
+    }
+    return {
+        endpoint: {
+            url: url.replace(/\/+$/, ""),
+            ...(env.CLICKHOUSE_USER !== undefined ? { user: env.CLICKHOUSE_USER } : {}),
+            ...(env.CLICKHOUSE_PASSWORD !== undefined ? { password: env.CLICKHOUSE_PASSWORD } : {}),
+        },
+        source: "env CLICKHOUSE_URL",
+        defaultDatabase: "default",
+        ...(env.CLICKHOUSE_TOPOLOGY ? { topology: parseTopology(env.CLICKHOUSE_TOPOLOGY) } : {}),
+    };
+}
+/** A target that could not be resolved. */
+export class ClickHouseBindingError extends Error {
+    constructor(unresolved) {
+        super(unresolved.detail);
+        this.unresolved = unresolved;
+        this.name = "ClickHouseBindingError";
+    }
+}
+async function loadConfig(cwd) {
+    try {
+        const { loadChantConfig } = await import("@intentius/chant/config");
+        return (await loadChantConfig(cwd)).config;
+    }
+    catch {
+        // No project config: the environment variables still bind.
+        return undefined;
+    }
+}
+/** Resolve the environment's server. Throws {@link ClickHouseBindingError} when there is none. */
+export async function bindClickHouse(options = {}) {
+    const config = options.config ?? (await loadConfig(options.cwd ?? process.cwd()));
+    const target = resolveClickHouseTarget({ environment: options.environment, config, env: options.env });
+    if (isUnresolvedTarget(target))
+        throw new ClickHouseBindingError(target);
+    return target;
+}
+/** What a failed bind or read means, in the observation vocabulary. */
+export function classifyClickHouseFailure(err) {
+    if (err instanceof ClickHouseBindingError)
+        return { reason: err.unresolved.reason, detail: err.unresolved.detail };
+    if (err instanceof TokenSourceError)
+        return { reason: "no-credentials", detail: err.message };
+    if (err instanceof ClickHouseQueryError) {
+        // 516 AUTHENTICATION_FAILED and 497 ACCESS_DENIED arrive as 401/403 or in the message.
+        if (err.status === 401 || err.status === 403 || /AUTHENTICATION_FAILED|ACCESS_DENIED|Code: 516|Code: 497/.test(err.serverMessage)) {
+            return { reason: "no-credentials", detail: `${err.message} (the server refused the credentials)` };
+        }
+        return { reason: "read-failed", detail: err.message };
+    }
+    return { reason: "read-failed", detail: err instanceof Error ? err.message.split("\n")[0] : String(err) };
+}
+/** `SELECT version()` on the target. */
+export async function serverVersion(target) {
+    const [row] = await clickhouseQuery(target.endpoint, "SELECT version() AS v");
+    return row?.v ?? "";
+}

package/dist/clickhouse/live/catalog.js

@@ -0,0 +1,223 @@
+/**
+ * Reading a live server's schema: the databases and the tables, views,
+ * materialized views and dictionaries in them, from `system.databases`,
+ * `system.tables` and `SHOW CREATE`, and the SQL user-defined functions,
+ * from `system.functions`.
+ */
+import { clickhouseQuery } from "../http.js";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities.js";
+import { isChantWorkingObject, stripMarkerFromStatement } from "../ownership.js";
+import { foreignTool } from "../../core/foreign-tables.js";
+import { parseCreate, unquote } from "../parser.js";
+import { tokenizeText } from "../tokens.js";
+/** The server's own databases, never part of a declared schema. */
+export const SYSTEM_DATABASES = new Set(["system", "information_schema", "INFORMATION_SCHEMA"]);
+const quote = (s) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+const ident = (s) => `\`${s.replace(/`/g, "``")}\``;
+/** The entity type a `system.tables` engine stands for. */
+export function entityTypeOfEngine(engine) {
+    if (engine === "View")
+        return CLICKHOUSE_ENTITY_TYPES.view;
+    if (engine === "MaterializedView")
+        return CLICKHOUSE_ENTITY_TYPES.materializedView;
+    return CLICKHOUSE_ENTITY_TYPES.table;
+}
+/**
+ * Every object in the target's databases, sorted by database then name.
+ * A materialized view's inner table (`.inner_id.<uuid>`, `.inner.<name>`)
+ * belongs to its view and is left out, as are temporary tables. A
+ * dictionary is read (#3682) from `system.tables`, where a reader granted
+ * `SELECT` on the database sees it without a grant on `system.dictionaries`;
+ * a table with the `Dictionary` engine is a table.
+ *
+ * A table another tool keeps its migration history in (`schema_migrations`,
+ * `goose_db_version`, `../../core/foreign-tables.ts`) is read and marked with
+ * the tool, as the Postgres reader marks it (#3676): import leaves it out
+ * with a warning, and a plan names it in a hint instead of dropping it.
+ *
+ * So are chant's own working objects (`../ownership.ts`
+ * `isChantWorkingObject`): a rebuild migration's new, dual-write and retained
+ * tables, and the effect receipts database. None of them is a declaration, so
+ * a plan, an import or a prune that saw them would propose dropping, importing
+ * or pruning the very objects a rebuild in progress depends on.
+ */
+export async function readLiveSchema(target, opts = {}) {
+    const q = (sql) => clickhouseQuery(target.endpoint, sql);
+    const scope = target.databases
+        ? `IN (${target.databases.map(quote).join(", ")})`
+        : `NOT IN (${[...SYSTEM_DATABASES].map(quote).join(", ")})`;
+    const databases = await q(`SELECT name, engine, toString(uuid) AS uuid, comment FROM system.databases WHERE name ${scope} ORDER BY name`);
+    // `system.tables` shows no comment for a table over a table function, whatever it was created with: its statement has it (#3911).
+    const tables = await q(`SELECT database, name, engine, toString(uuid) AS uuid, comment, startsWith(create_table_query, 'CREATE DICTIONARY') AS dictionary, ` +
+        `if(comment = '' AND position(create_table_query, ' COMMENT ') > 0, create_table_query, '') AS proxy FROM system.tables ` +
+        `WHERE database ${scope} AND NOT is_temporary AND name NOT LIKE '.inner%' ` +
+        `ORDER BY database, name`);
+    const working = new Set(databases.filter((d) => isChantWorkingObject(d.comment)).map((d) => d.name));
+    const out = [];
+    const statement = async (kind, what) => opts.withStatements === false
+        ? ""
+        : stripMarkerFromStatement((await q(`SHOW CREATE ${kind} ${what}`))[0]?.statement ?? "");
+    for (const d of databases) {
+        if (working.has(d.name))
+            continue;
+        out.push({
+            type: CLICKHOUSE_ENTITY_TYPES.database,
+            name: d.name,
+            engine: d.engine,
+            ...(d.uuid && !/^0{8}-/.test(d.uuid) ? { uuid: d.uuid } : {}),
+            ...(d.comment ? { comment: d.comment } : {}),
+            statement: await statement("DATABASE", ident(d.name)),
+        });
+    }
+    for (const t of tables) {
+        if (t.proxy)
+            t.comment = commentOf(t.proxy) ?? t.comment;
+        if (working.has(t.database) || isChantWorkingObject(t.comment))
+            continue;
+        const type = Number(t.dictionary) === 1 ? CLICKHOUSE_ENTITY_TYPES.dictionary : entityTypeOfEngine(t.engine);
+        const foreign = type === CLICKHOUSE_ENTITY_TYPES.table ? foreignTool(t.name) : undefined;
+        out.push({
+            type,
+            database: t.database,
+            name: t.name,
+            engine: t.engine,
+            ...(t.uuid && !/^0{8}-/.test(t.uuid) ? { uuid: t.uuid } : {}),
+            ...(t.comment ? { comment: t.comment } : {}),
+            ...(foreign ? { foreign } : {}),
+            statement: await statement("TABLE", `${ident(t.database)}.${ident(t.name)}`),
+        });
+    }
+    // SQL user-defined functions belong to no database: every one is read, and a caller keeps those it declares.
+    const functions = await q(`SELECT name, create_query AS statement FROM system.functions WHERE origin = 'SQLUserDefined' ORDER BY name`);
+    for (const f of functions) {
+        out.push({ type: CLICKHOUSE_ENTITY_TYPES.function, name: f.name, engine: "", statement: opts.withStatements === false ? "" : f.statement });
+    }
+    return out;
+}
+/** A `CREATE` statement's own `COMMENT`, its value unquoted; undefined when it has none or does not parse. */
+function commentOf(statement) {
+    try {
+        const tokens = tokenizeText(statement, 0);
+        const node = parseCreate(tokens);
+        if (!node.comment)
+            return undefined;
+        const literal = tokens
+            .slice(node.comment.from, node.comment.to)
+            .map((t) => t.text)
+            .join("")
+            .trim();
+        return /^'.*'$/s.test(literal) ? literal.slice(1, -1).replace(/\\(.)/g, "$1").replace(/''/g, "'") : unquote(literal);
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * The users, roles, row policies, quotas, settings profiles and grantees'
+ * grants the build declares, as the server holds them (#3682, #3911):
+ * `SHOW CREATE USER`, `SHOW CREATE ROLE`, `SHOW CREATE ROW POLICY`,
+ * `SHOW CREATE QUOTA`, `SHOW CREATE SETTINGS PROFILE`, and for a grantee every line of
+ * `SHOW GRANTS FOR`, one per line. Nothing else is read, so a project that
+ * declares none needs no access to `system.users` and the like, and an
+ * object made by hand is never seen. One that does not exist is left out.
+ */
+export async function readLiveAccess(target, declared, opts = {}) {
+    if (declared.length === 0)
+        return [];
+    const q = (sql) => clickhouseQuery(target.endpoint, sql);
+    const first = async (sql) => (opts.withStatements === false ? "" : Object.values((await q(sql))[0] ?? {})[0] ?? "");
+    const all = async (sql) => (await q(sql)).map((r) => Object.values(r)[0] ?? "");
+    const names = (kind) => [...new Set(declared.filter((d) => d.kind === kind).map((d) => d.name))];
+    const users = names("user");
+    const roles = names("role");
+    const grantees = names("grants");
+    const principals = [...new Set([...users, ...roles, ...grantees])];
+    const list = (xs) => xs.map(quote).join(", ");
+    const existingUsers = new Set(principals.length ? (await q(`SELECT name FROM system.users WHERE name IN (${list(principals)})`)).map((r) => r.name) : []);
+    const existingRoles = new Set(principals.length ? (await q(`SELECT name FROM system.roles WHERE name IN (${list(principals)})`)).map((r) => r.name) : []);
+    const out = [];
+    for (const name of users.filter((u) => existingUsers.has(u))) {
+        out.push({ type: CLICKHOUSE_ENTITY_TYPES.user, name, engine: "", statement: await first(`SHOW CREATE USER ${ident(name)}`) });
+    }
+    for (const name of roles.filter((r) => existingRoles.has(r))) {
+        out.push({ type: CLICKHOUSE_ENTITY_TYPES.role, name, engine: "", statement: await first(`SHOW CREATE ROLE ${ident(name)}`) });
+    }
+    for (const [kind, type, table, show] of [
+        ["quota", CLICKHOUSE_ENTITY_TYPES.quota, "system.quotas", "QUOTA"],
+        ["settingsProfile", CLICKHOUSE_ENTITY_TYPES.settingsProfile, "system.settings_profiles", "SETTINGS PROFILE"],
+    ]) {
+        const wanted = names(kind);
+        if (wanted.length === 0)
+            continue;
+        const existing = new Set((await q(`SELECT name FROM ${table} WHERE name IN (${list(wanted)})`)).map((r) => r.name));
+        for (const name of wanted.filter((n) => existing.has(n))) {
+            out.push({ type, name, engine: "", statement: await first(`SHOW CREATE ${show} ${ident(name)}`) });
+        }
+    }
+    const policies = declared.filter((d) => d.kind === "rowPolicy");
+    if (policies.length > 0) {
+        const live = await q(`SELECT short_name AS name, database, table FROM system.row_policies WHERE short_name IN (${list(policies.map((p) => p.name))})`);
+        for (const p of policies) {
+            if (!live.some((l) => l.name === p.name && l.database === p.database && l.table === p.table))
+                continue;
+            out.push({
+                type: CLICKHOUSE_ENTITY_TYPES.rowPolicy,
+                name: p.name,
+                database: p.database,
+                table: p.table,
+                engine: "",
+                statement: await first(`SHOW CREATE ROW POLICY ${ident(p.name)} ON ${ident(p.database)}.${ident(p.table)}`),
+            });
+        }
+    }
+    for (const name of grantees.filter((g) => existingUsers.has(g) || existingRoles.has(g))) {
+        const lines = opts.withStatements === false ? [] : await all(`SHOW GRANTS FOR ${ident(name)}`);
+        out.push({ type: CLICKHOUSE_ENTITY_TYPES.grant, name, engine: "", statement: lines.join("\n") });
+    }
+    return out;
+}
+/** The canonical kinds a profile's `access` switch decides (#3716). */
+export const ACCESS_KINDS = new Set(["user", "role", "rowPolicy", "quota", "settingsProfile", "grants"]);
+/** The entity types a profile's `access` switch decides (#3716). */
+export const ACCESS_ENTITY_TYPES = new Set([
+    CLICKHOUSE_ENTITY_TYPES.user,
+    CLICKHOUSE_ENTITY_TYPES.role,
+    CLICKHOUSE_ENTITY_TYPES.rowPolicy,
+    CLICKHOUSE_ENTITY_TYPES.quota,
+    CLICKHOUSE_ENTITY_TYPES.settingsProfile,
+    CLICKHOUSE_ENTITY_TYPES.grant,
+]);
+/** Why an access declaration is not read, planned or applied where the profile does not manage access. */
+export const ACCESS_UNMANAGED_DETAIL = "access is not managed in this environment: set sql.profiles.<env>.access to plan and apply users, roles, row policies, quotas, settings profiles and grants";
+/** The hint a plan gives for the access declarations it left out. */
+export function accessUnmanagedHint(n, environment) {
+    return `${n} access declaration${n === 1 ? " is" : "s are"} not planned: ${environment}'s profile does not manage access (sql.profiles.<env>.access)`;
+}
+/** The access objects among declared canonical objects, for `readLiveAccess`. */
+export function declaredAccess(objects) {
+    return objects
+        .filter((o) => ACCESS_KINDS.has(o.kind))
+        .map((o) => ({ kind: o.kind, name: o.name, ...(o.database !== undefined ? { database: o.database } : {}), ...(o.table !== undefined ? { table: o.table } : {}) }));
+}
+/** The users, roles, row policies, quotas and settings profiles among declared entities, for `readLiveAccess`. */
+export function accessOf(entities, defaultDatabase) {
+    const kinds = {
+        [CLICKHOUSE_ENTITY_TYPES.user]: "user",
+        [CLICKHOUSE_ENTITY_TYPES.role]: "role",
+        [CLICKHOUSE_ENTITY_TYPES.rowPolicy]: "rowPolicy",
+        [CLICKHOUSE_ENTITY_TYPES.quota]: "quota",
+        [CLICKHOUSE_ENTITY_TYPES.settingsProfile]: "settingsProfile",
+    };
+    const out = [];
+    for (const e of entities.values()) {
+        const kind = kinds[e.entityType];
+        if (!kind)
+            continue;
+        const name = String(e.props.name ?? "");
+        if (kind === "rowPolicy")
+            out.push({ kind, name, database: typeof e.props.database === "string" ? e.props.database : defaultDatabase, table: String(e.props.table ?? "") });
+        else
+            out.push({ kind, name });
+    }
+    return out;
+}