@intentius/chant-lexicon-sql 0.118.0 → 0.120.0

package/src/clickhouse/import/import.test.ts

@@ -121,6 +121,12 @@ describe("live export", () => {
           "CREATE TABLE analytics.events\n(\n    `id` UInt64\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192\nCOMMENT 'Raw events [chant managed-by=chant stack=shop]'",
         comment: "Raw events [chant managed-by=chant stack=shop]",
       },
+      {
+        database: "analytics",
+        name: "schema_migrations",
+        engine: "MergeTree",
+        statement: "CREATE TABLE analytics.schema_migrations\n(\n    `version` Int64,\n    `dirty` UInt8,\n    `sequence` UInt64\n)\nENGINE = MergeTree\nORDER BY sequence",
+      },
     ]);
   });
   afterAll(() => server.close());
@@ -134,6 +140,14 @@ describe("live export", () => {
     expect(String(ir.resources[1]!.properties.ddl)).not.toContain("index_granularity");
   });
 
+  test("a migration runner's history table is left out with a warning (#3676)", async () => {
+    const ir = await run();
+    expect(ir.resources.map((r) => r.properties.name)).not.toContain("schema_migrations");
+    expect(ir.warnings).toEqual([
+      "analytics.schema_migrations is kept by a migration runner (Rails, golang-migrate, dbmate); left out, since declaring it would have chant change what that tool owns",
+    ]);
+  });
+
   test("verbatim keeps the statement as the server printed it", async () => {
     const ir = await run({ verbatim: true });
     expect(String(ir.resources[1]!.properties.ddl)).toContain("index_granularity = 8192");
@@ -161,3 +175,53 @@ describe("live export", () => {
     expect(new ClickHouseGenerator().generate(ir)[0]!.content).toContain("CREATE TABLE ${analyticsDb}.events");
   });
 });
+
+describe("live export of SQL functions (#3718)", () => {
+  let server: FakeServer;
+  beforeAll(async () => {
+    server = await fakeClickHouse(
+      [
+        { name: "shop", engine: "Atomic", statement: "CREATE DATABASE shop\nENGINE = Atomic" },
+        {
+          database: "shop",
+          name: "orders",
+          engine: "MergeTree",
+          statement: "CREATE TABLE shop.orders\n(\n    `id` UInt64,\n    `net` Float64 DEFAULT shop_net(id)\n)\nENGINE = MergeTree\nORDER BY id",
+        },
+      ],
+      {
+        functions: [
+          { name: "other_score", statement: "CREATE FUNCTION other_score AS x -> (x * 2)" },
+          { name: "shop_net", statement: "CREATE FUNCTION shop_net AS x -> shop_tax(x)" },
+          { name: "shop_tax", statement: "CREATE FUNCTION shop_tax AS x -> (x * 0.2)" },
+          { name: "shop_unused", statement: "CREATE FUNCTION shop_unused AS x -> x" },
+        ],
+      },
+    );
+  });
+  afterAll(() => server.close());
+
+  const run = (profile: Record<string, unknown> = {}, extra: Record<string, unknown> = {}) =>
+    exportResources({ environment: "x", config: { sql: { profiles: { x: { url: server.url, ...profile } } } } as never, env: {}, ...extra });
+
+  test("adopts the functions the imported objects call, and the ones those call; a warning names the rest", async () => {
+    const ir = await run();
+    expect(ir.resources.map((r) => r.properties.name)).toEqual(["shop", "orders", "shop_net", "shop_tax"]);
+    expect(ir.warnings).toEqual([
+      "2 SQL functions on the server are not imported, since no imported object calls them: other_score, shop_unused. Name them in sql.profiles.x.importFunctions to import them.",
+    ]);
+  });
+
+  test("importFunctions names more, by name or by prefix", async () => {
+    expect((await run({ importFunctions: ["shop_*"] })).resources.map((r) => r.properties.name)).toEqual(["shop", "orders", "shop_net", "shop_tax", "shop_unused"]);
+    const ir = await run({ importFunctions: ["other_score"] });
+    expect(ir.resources.map((r) => r.properties.name)).toContain("other_score");
+    expect(ir.warnings).toEqual([expect.stringContaining(": shop_unused.")]);
+  });
+
+  test("a selector picks a function it names", async () => {
+    const ir = await run({}, { selector: { name: "other_score" } });
+    expect(ir.resources.map((r) => r.properties.name)).toEqual(["other_score"]);
+    expect(ir.warnings).toBeUndefined();
+  });
+});

package/src/clickhouse/import/live-export.ts

@@ -13,6 +13,14 @@
  * - `owned` keeps the objects whose comment carries chant's ownership marker
  *   (`../ownership.ts`), the trailer the applier stamps. The trailer itself is
  *   never written into a declaration: `readLiveSchema` takes it off.
+ * - a migration runner's history table (`schema_migrations`,
+ *   `goose_db_version`) is left out with a warning (#3676), as on Postgres.
+ * - A SQL user-defined function belongs to no database, and every project's
+ *   functions sit side by side on a server (#3718). Without a selector, an
+ *   import adopts only the functions the imported objects call (and those
+ *   functions call), and the ones `sql.profiles.<env>.importFunctions` names;
+ *   a warning names the rest. A function carries no comment, so `owned` does
+ *   not apply to it.
  */
 
 import type { ExportedTemplate, ResourceSelector } from "@intentius/chant/lexicon";
@@ -20,6 +28,48 @@ import { bindClickHouse, type BindOptions } from "../live/bind";
 import { readLiveSchema } from "../live/catalog";
 import { objectsToIR, stripServerDefaults, type ImportedObject } from "./ir";
 import { isChantManaged } from "../ownership";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
+import { isTrivia, tokenizeText } from "../tokens";
+import { unquote } from "../parser";
+import type { LiveObject } from "../live/catalog";
+
+/** The functions among `names` a statement calls: a name followed by `(`. */
+export function calledFunctions(ddl: string, names: ReadonlySet<string>): string[] {
+  const tokens = tokenizeText(ddl, 0).filter((t) => !isTrivia(t));
+  const out = new Set<string>();
+  tokens.forEach((t, i) => {
+    if (t.kind !== "ident" && t.kind !== "qident") return;
+    const next = tokens[i + 1];
+    if (next?.kind !== "punct" || next.text !== "(") return;
+    const name = t.kind === "qident" ? unquote(t.text) : t.text;
+    if (names.has(name)) out.add(name);
+  });
+  return [...out];
+}
+
+/** Whether `patterns` (names, or prefixes ending in `*`) name a function. */
+const named = (patterns: readonly string[], name: string) => patterns.some((p) => (p.endsWith("*") ? name.startsWith(p.slice(0, -1)) : p === name));
+
+/**
+ * The functions an import adopts (#3718): those the objects call, those
+ * `patterns` name, and the functions those call in turn; and the rest.
+ */
+export function scopedFunctions(objects: readonly LiveObject[], functions: readonly LiveObject[], patterns: readonly string[] = []): { adopted: LiveObject[]; skipped: string[] } {
+  const byName = new Map(functions.map((f) => [f.name, f]));
+  const names = new Set(byName.keys());
+  const adopted = new Set<string>(functions.filter((f) => named(patterns, f.name)).map((f) => f.name));
+  for (const o of objects) for (const n of calledFunctions(o.statement, names)) adopted.add(n);
+  const queue = [...adopted];
+  while (queue.length > 0) {
+    for (const n of calledFunctions(byName.get(queue.pop()!)!.statement, names)) {
+      if (!adopted.has(n)) {
+        adopted.add(n);
+        queue.push(n);
+      }
+    }
+  }
+  return { adopted: functions.filter((f) => adopted.has(f.name)), skipped: functions.filter((f) => !adopted.has(f.name)).map((f) => f.name) };
+}
 
 export interface ExportOptions extends Omit<BindOptions, "environment"> {
   environment: string;
@@ -32,7 +82,10 @@ export interface ExportOptions extends Omit<BindOptions, "environment"> {
 
 export async function exportResources(options: ExportOptions): Promise<ExportedTemplate> {
   const target = await bindClickHouse(options);
-  const live = await readLiveSchema(target);
+  const all = await readLiveSchema(target);
+  const isFunction = (o: LiveObject) => o.type === CLICKHOUSE_ENTITY_TYPES.function;
+  // A selector picks what it names, functions included; without one, functions go with what calls them (#3718).
+  const live = options.selector ? all : all.filter((o) => !isFunction(o));
   const selected = live.filter((o) => {
     if (options.owned && !isChantManaged(o.comment)) return false;
     if (options.selector?.type && o.type !== options.selector.type) return false;
@@ -40,14 +93,29 @@ export async function exportResources(options: ExportOptions): Promise<ExportedT
     if (name !== undefined && name !== o.name && name !== `${o.database}.${o.name}`) return false;
     return true;
   });
+  const warnings: string[] = [];
+  if (!options.selector) {
+    const { adopted, skipped } = scopedFunctions(selected.filter((o) => !o.foreign), all.filter(isFunction), target.importFunctions);
+    selected.push(...adopted);
+    if (skipped.length > 0) {
+      warnings.push(
+        `${skipped.length} SQL function${skipped.length === 1 ? "" : "s"} on the server ${skipped.length === 1 ? "is" : "are"} not imported, since no imported object calls ${skipped.length === 1 ? "it" : "them"}: ${skipped.join(", ")}. Name ${skipped.length === 1 ? "it" : "them"} in sql.profiles.${options.environment}.importFunctions to import ${skipped.length === 1 ? "it" : "them"}.`,
+      );
+    }
+  }
   const objects: ImportedObject[] = selected
     // The `default` database exists on every server; declaring it would make apply create what is there.
     .filter((o) => !(o.type === "ClickHouse::Database" && o.name === "default"))
+    .filter((o) => {
+      if (!o.foreign) return true;
+      warnings.push(`${o.database ? `${o.database}.` : ""}${o.name} is kept by ${o.foreign}; left out, since declaring it would have chant change what that tool owns`);
+      return false;
+    })
     .map((o) => ({
       type: o.type,
       ...(o.database ? { database: o.database } : {}),
       name: o.name,
       ddl: options.verbatim ? o.statement : stripServerDefaults(o.statement),
     }));
-  return objectsToIR(objects) as ExportedTemplate;
+  return objectsToIR(objects, warnings) as ExportedTemplate;
 }

package/src/clickhouse/live/access.e2e.test.ts

@@ -13,7 +13,11 @@
  * - a privilege granted by hand to a declared grantee is SQLCH274 and is
  *   revoked, a declared one revoked by hand is SQLCH273 and is granted
  *   again, a changed role setting is SQLCH270, a changed policy SQLCH272;
- * - the row policy filters rows for the user.
+ *   the deep read lists both as pending, with the statements (#3733);
+ * - the row policy filters rows for the user;
+ * - with a profile that does not manage access (no `access: true`, #3716),
+ *   none of it is applied, planned or read: each access declaration is
+ *   reported filtered, and a hand change to a grant is not undone.
  *
  * Beside a Replicated database (two replicas of the pinned server in
  * Docker): access control is in no database, so the database's log does not
@@ -23,7 +27,7 @@
  * `CLICKHOUSE_PASSWORD`) names a running one, such as the one
  * `chant emulator up --lexicon sql` starts. Without it, a throwaway server
  * at the pin, skipped cleanly when Docker is not available. Users and roles
- * are global to the server, so the test names everything `chant_e2e_3682_access*`
+ * are global to the server, so the test names everything `chant_e2e_3716_access*`
  * and drops it afterwards.
  */
 
@@ -37,9 +41,10 @@ import { clickhouseImage } from "../../spec/pin";
 import { database, grant, policy, role, table, user } from "../entities";
 import { planAgainstServer } from "../plan/commands";
 import { observeResourcesDeep, sqlDeepNormalizationHooks } from "../plan/deep";
+import { describeResources } from "./describe-resources";
 import { clickhouseApply } from "../../op/activities/clickhouse-apply";
 
-const DB = "chant_e2e_3682_access";
+const DB = "chant_e2e_3716_access";
 const READER = `${DB}_reader`;
 const APP = `${DB}_app`;
 const fromEnv = process.env.CLICKHOUSE_URL;
@@ -76,13 +81,27 @@ afterAll(async () => {
   rmSync(dir, { recursive: true, force: true });
 });
 
-const envOf = (at: ClickHouseEndpoint, topology?: string) => ({
-  CLICKHOUSE_URL: at.url,
-  ...(at.user ? { CLICKHOUSE_USER: at.user } : {}),
-  ...(at.password ? { CLICKHOUSE_PASSWORD: at.password } : {}),
-  ...(topology ? { CLICKHOUSE_TOPOLOGY: topology } : {}),
+const envOf = (at: ClickHouseEndpoint) => ({
+  ...(at.user ? { CH_USER: at.user } : {}),
+  ...(at.password ? { CH_PASSWORD: at.password } : {}),
 });
 
+/** The `test` profile: the server, and whether it manages access (#3716). */
+const configOf = (at: ClickHouseEndpoint, opts: { topology?: string; access?: boolean } = {}) => ({
+  ownership: { stack: "e2e3682", env: "test" },
+  sql: {
+    profiles: {
+      test: {
+        url: at.url,
+        ...(at.user ? { user: { env: "CH_USER" } } : {}),
+        ...(at.password ? { password: { env: "CH_PASSWORD" } } : {}),
+        ...(opts.topology ? { topology: opts.topology } : {}),
+        ...(opts.access !== false ? { access: true } : {}),
+      },
+    },
+  },
+}) as never;
+
 type Entity = { entityType: string; props: { ddl: string }; dependsOn?: unknown };
 function writeBuild(file: string, declared: Record<string, Entity>): string {
   const path = join(dir, file);
@@ -91,11 +110,11 @@ function writeBuild(file: string, declared: Record<string, Entity>): string {
   return path;
 }
 
-async function apply(at: ClickHouseEndpoint, buildPath: string, topology?: string) {
+async function apply(at: ClickHouseEndpoint, buildPath: string, opts: { topology?: string; access?: boolean } = {}) {
   const sent: string[] = [];
   const outcome = await clickhouseApply({ buildPath, environment: "test" }, undefined, {
-    config: { ownership: { stack: "e2e3682", env: "test" } },
-    env: envOf(at, topology),
+    config: configOf(at, opts),
+    env: envOf(at),
     log: (l) => void (/^[A-Z]+ /.test(l) && sent.push(l)),
   });
   return { outcome, sent };
@@ -146,11 +165,12 @@ describe.skipIf(!enabled)("access control on a single node (#3682)", () => {
     expect((await q<{ s: string }>(`SELECT auth_type[1] AS s FROM system.users WHERE name = '${APP}'`))[0]!.s).toBe("sha256_password");
 
     // The server prints the policy's condition and the grants its own way: still the declarations.
-    const diff = await planAgainstServer("test", build, { config: {}, env: envOf(endpoint) });
+    const diff = await planAgainstServer("test", build, { config: configOf(endpoint), env: envOf(endpoint) });
     expect(diff.changes).toEqual([]);
     const entities = new Map(Object.entries(v1).map(([k, e]) => [k, { entityType: e.entityType, props: e.props as unknown as Record<string, unknown> }]));
-    const deep = await observeResourcesDeep({ environment: "test", entityNames: [...entities.keys()], entities, config: {}, env: envOf(endpoint) });
+    const deep = await observeResourcesDeep({ environment: "test", entityNames: [...entities.keys()], entities, config: configOf(endpoint), env: envOf(endpoint) });
     expect(Object.keys(deep.unobserved ?? {}).sort()).toEqual(["readEvents", "readerToApp"]);
+    expect(deep.pending).toBeUndefined();
     for (const k of ["reader", "app", "tenant"] as const) expect(compared(deep.resources[k]!.properties), k).toEqual(compared(v1[k].props));
 
     // The policy filters rows for the user.
@@ -161,8 +181,14 @@ describe.skipIf(!enabled)("access control on a single node (#3682)", () => {
     // Changed by hand: a privilege granted, a declared one revoked. Changed in the declaration: a setting and the condition.
     await q(`GRANT INSERT ON ${DB}.events TO ${READER}`);
     await q(`REVOKE ${READER} FROM ${APP}`);
+    // The deep read lists what an apply would grant and revoke (#3733), so an approval's plan digest moves.
+    const handMade = await observeResourcesDeep({ environment: "test", entityNames: [...entities.keys()], entities, config: configOf(endpoint), env: envOf(endpoint) });
+    expect(handMade.pending).toEqual([
+      { subject: `grants ${READER}`, change: `REVOKE INSERT ON \`${DB}\`.\`events\` FROM \`${READER}\``, entities: ["readEvents"] },
+      { subject: `grants ${APP}`, change: `GRANT \`${READER}\` TO \`${APP}\``, entities: ["readerToApp"] },
+    ]);
     const v2 = declarations({ memory: "2000000000", tenant: "b" });
-    const planned = await planAgainstServer("test", writeBuild("v2.json", v2), { config: {}, env: envOf(endpoint) });
+    const planned = await planAgainstServer("test", writeBuild("v2.json", v2), { config: configOf(endpoint), env: envOf(endpoint) });
     expect(planned.changes.map((c) => [c.object, c.field, c.rule])).toEqual([
       [`reader (role ${READER})`, "settings", "SQLCH270"],
       // Revoking the role took it off the user's default roles too.
@@ -180,8 +206,35 @@ describe.skipIf(!enabled)("access control on a single node (#3682)", () => {
       `GRANT \`${READER}\` TO \`${APP}\``,
       `ALTER USER \`${APP}\` DEFAULT ROLE ${READER}`,
     ]);
-    expect((await planAgainstServer("test", writeBuild("v2.json", v2), { config: {}, env: envOf(endpoint) })).changes).toEqual([]);
+    expect((await planAgainstServer("test", writeBuild("v2.json", v2), { config: configOf(endpoint), env: envOf(endpoint) })).changes).toEqual([]);
     expect(Number((await q<{ n: string }>(`SELECT count() AS n FROM ${DB}.events`, asApp))[0]!.n)).toBe(1);
+
+    // A profile that does not manage access (#3716): nothing access-related is read, planned or applied.
+    await q(`GRANT INSERT ON ${DB}.events TO ${READER}`);
+    const off = { access: false };
+    const v3 = declarations({ memory: "3000000000", tenant: "a" });
+    const offBuild = writeBuild("v3.json", v3);
+    const offPlan = await planAgainstServer("test", offBuild, { config: configOf(endpoint, off), env: envOf(endpoint) });
+    expect(offPlan.changes).toEqual([]);
+    expect(offPlan.hints).toContain("5 access declarations are not planned: test's profile does not manage access (sql.profiles.<env>.access)");
+    const offApply = await apply(endpoint, offBuild, off);
+    expect(offApply.outcome.failed).toEqual([]);
+    expect(offApply.sent).toEqual([]);
+    expect(offApply.outcome.notAttempted.map((n) => [n.name, n.reason])).toEqual([
+      [`role ${READER}`, "filtered"],
+      [`user ${APP}`, "filtered"],
+      [`row policy ${DB}_tenant ON ${DB}.events`, "filtered"],
+      [`grants ${READER}`, "filtered"],
+      [`grants ${APP}`, "filtered"],
+    ]);
+    expect((await q<{ s: string }>(`SHOW GRANTS FOR ${READER}`)).map((r) => Object.values(r)[0]).join("\n")).toMatch(/GRANT INSERT\b.* ON chant_e2e_3716_access\.events TO/);
+    const v3entities = new Map(Object.entries(v3).map(([k, e]) => [k, { entityType: e.entityType, props: e.props as unknown as Record<string, unknown> }]));
+    const offDeep = await observeResourcesDeep({ environment: "test", entityNames: [...v3entities.keys()], entities: v3entities, config: configOf(endpoint, off), env: envOf(endpoint) });
+    expect(Object.entries(offDeep.unobserved ?? {}).map(([k, u]) => [k, u.reason]).sort()).toEqual(
+      ["app", "readEvents", "reader", "readerToApp", "tenant"].map((k) => [k, "filtered"]),
+    );
+    const offThin = await describeResources({ environment: "test", entityNames: [...v3entities.keys()], entities: v3entities, config: configOf(endpoint, off), env: envOf(endpoint) });
+    expect(Object.keys(offThin.unobserved ?? {}).sort()).toEqual(["app", "readEvents", "reader", "readerToApp", "tenant"]);
   }, 300_000);
 });
 
@@ -198,12 +251,12 @@ describe.skipIf(!docker)("access control beside a Replicated database (#3682)",
     const [r1, r2] = cluster!.replicas as [ClickHouseEndpoint, ClickHouseEndpoint];
     const { shop, events, reader, tenant, readEvents } = declarations();
     const build = writeBuild("repl.json", { shop, events, reader, tenant, readEvents });
-    const first = await apply(r1, build, "replicated");
+    const first = await apply(r1, build, { topology: "replicated" });
     expect(first.outcome.failed).toEqual([]);
     expect(first.sent.join("\n")).not.toMatch(/ON CLUSTER/);
     await q(first.sent[0]!, r2);
     await q(`SYSTEM SYNC DATABASE REPLICA ${DB}`, r2);
-    const second = await apply(r2, build, "replicated");
+    const second = await apply(r2, build, { topology: "replicated" });
     expect(second.outcome.failed).toEqual([]);
     expect(second.outcome.applied.filter((a) => a.action === "created").map((a) => a.name)).toEqual([
       `role ${READER}`,
@@ -212,7 +265,7 @@ describe.skipIf(!docker)("access control beside a Replicated database (#3682)",
     ]);
     for (const at of [r1, r2]) {
       expect(Number((await q<{ n: string }>(`SELECT count() AS n FROM system.row_policies WHERE short_name = '${DB}_tenant'`, at))[0]!.n)).toBe(1);
-      expect((await apply(at, build, "replicated")).sent).toEqual([]);
+      expect((await apply(at, build, { topology: "replicated" })).sent).toEqual([]);
     }
   }, 300_000);
 });

package/src/clickhouse/live/bind.ts

@@ -24,6 +24,14 @@ export interface ClickHouseTarget {
    * as declared.
    */
   topology?: Topology;
+  /**
+   * Whether chant manages access here (`sql.profiles.<env>.access`, #3716):
+   * the declared users, roles, row policies and grants. Off when omitted, as
+   * on Postgres, and off for a server bound by `CLICKHOUSE_URL`.
+   */
+  access?: boolean;
+  /** `sql.profiles.<env>.importFunctions` (#3718): the functions an import adopts besides those the imported objects call. */
+  importFunctions?: string[];
 }
 
 export interface UnresolvedTarget {
@@ -75,6 +83,8 @@ export function resolveClickHouseTarget(input: {
       ...(profile.databases ? { databases: profile.databases } : {}),
       defaultDatabase: profile.defaultDatabase ?? "default",
       ...(profile.topology !== undefined ? { topology: toTopology(profile.topology) } : {}),
+      ...(profile.access === true ? { access: true } : {}),
+      ...(profile.importFunctions ? { importFunctions: profile.importFunctions } : {}),
     };
   }
   const url = env.CLICKHOUSE_URL;

package/src/clickhouse/live/catalog.ts

@@ -9,6 +9,7 @@ import { clickhouseQuery } from "../http";
 import type { ClickHouseTarget } from "./bind";
 import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "../entities";
 import { isChantWorkingObject, stripMarkerFromStatement } from "../ownership";
+import { foreignTool } from "../../core/foreign-tables";
 
 /** The server's own databases, never part of a declared schema. */
 export const SYSTEM_DATABASES = new Set(["system", "information_schema", "INFORMATION_SCHEMA"]);
@@ -26,6 +27,8 @@ export interface LiveObject {
   statement: string;
   /** A row policy's table. */
   table?: string;
+  /** The tool that keeps it, for a migration runner's history table (`../../core/foreign-tables.ts`): never chant's to import, change or drop. */
+  foreign?: string;
 }
 
 const quote = (s: string) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
@@ -46,6 +49,11 @@ export function entityTypeOfEngine(engine: string): ClickHouseEntityType {
  * `SELECT` on the database sees it without a grant on `system.dictionaries`;
  * a table with the `Dictionary` engine is a table.
  *
+ * A table another tool keeps its migration history in (`schema_migrations`,
+ * `goose_db_version`, `../../core/foreign-tables.ts`) is read and marked with
+ * the tool, as the Postgres reader marks it (#3676): import leaves it out
+ * with a warning, and a plan names it in a hint instead of dropping it.
+ *
  * So are chant's own working objects (`../ownership.ts`
  * `isChantWorkingObject`): a rebuild migration's new, dual-write and retained
  * tables, and the effect receipts database. None of them is a declaration, so
@@ -87,13 +95,16 @@ export async function readLiveSchema(target: ClickHouseTarget, opts: { withState
   }
   for (const t of tables) {
     if (working.has(t.database) || isChantWorkingObject(t.comment)) continue;
+    const type = Number(t.dictionary) === 1 ? CLICKHOUSE_ENTITY_TYPES.dictionary : entityTypeOfEngine(t.engine);
+    const foreign = type === CLICKHOUSE_ENTITY_TYPES.table ? foreignTool(t.name) : undefined;
     out.push({
-      type: Number(t.dictionary) === 1 ? CLICKHOUSE_ENTITY_TYPES.dictionary : entityTypeOfEngine(t.engine),
+      type,
       database: t.database,
       name: t.name,
       engine: t.engine,
       ...(t.uuid && !/^0{8}-/.test(t.uuid) ? { uuid: t.uuid } : {}),
       ...(t.comment ? { comment: t.comment } : {}),
+      ...(foreign ? { foreign } : {}),
       statement: await statement("TABLE", `${ident(t.database)}.${ident(t.name)}`),
     });
   }
@@ -168,7 +179,19 @@ export async function readLiveAccess(target: ClickHouseTarget, declared: readonl
   return out;
 }
 
-const ACCESS_KINDS = new Set(["user", "role", "rowPolicy", "grants"]);
+/** The canonical kinds a profile's `access` switch decides (#3716). */
+export const ACCESS_KINDS: ReadonlySet<string> = new Set(["user", "role", "rowPolicy", "grants"]);
+
+/** The entity types a profile's `access` switch decides (#3716). */
+export const ACCESS_ENTITY_TYPES: ReadonlySet<string> = new Set([CLICKHOUSE_ENTITY_TYPES.user, CLICKHOUSE_ENTITY_TYPES.role, CLICKHOUSE_ENTITY_TYPES.rowPolicy, CLICKHOUSE_ENTITY_TYPES.grant]);
+
+/** Why an access declaration is not read, planned or applied where the profile does not manage access. */
+export const ACCESS_UNMANAGED_DETAIL = "access is not managed in this environment: set sql.profiles.<env>.access to plan and apply users, roles, row policies and grants";
+
+/** The hint a plan gives for the access declarations it left out. */
+export function accessUnmanagedHint(n: number, environment: string): string {
+  return `${n} access declaration${n === 1 ? " is" : "s are"} not planned: ${environment}'s profile does not manage access (sql.profiles.<env>.access)`;
+}
 
 /** The access objects among declared canonical objects, for `readLiveAccess`. */
 export function declaredAccess(objects: ReadonlyArray<{ kind: string; name: string; database?: string; table?: string }>): DeclaredAccess[] {

package/src/clickhouse/live/describe-resources.ts

@@ -24,7 +24,7 @@ import {
   type ObserverAdapter,
 } from "@intentius/chant/observation";
 import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "./bind";
-import { accessOf, readLiveAccess, readLiveSchema, type DeclaredAccess, type LiveObject } from "./catalog";
+import { ACCESS_ENTITY_TYPES, ACCESS_UNMANAGED_DETAIL, accessOf, readLiveAccess, readLiveSchema, type DeclaredAccess, type LiveObject } from "./catalog";
 import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
 import { objectKey } from "../plan/normalize";
 import { isChantManaged, readMarker, stripMarker } from "../ownership";
@@ -54,7 +54,7 @@ function adapter(options: BindOptions & { owned?: boolean; access?: (defaultData
     async bind() {
       const target = await bindClickHouse(options);
       const byKey = new Map<string, LiveObject>();
-      const access = await readLiveAccess(target, options.access?.(target.defaultDatabase) ?? [], { withStatements: false });
+      const access = target.access === true ? await readLiveAccess(target, options.access?.(target.defaultDatabase) ?? [], { withStatements: false }) : [];
       for (const o of [...(await readLiveSchema(target, { withStatements: false })), ...access]) {
         byKey.set(objectKey(o), o);
       }
@@ -65,6 +65,9 @@ function adapter(options: BindOptions & { owned?: boolean; access?: (defaultData
       if (!entity.type.startsWith("ClickHouse::")) {
         return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
       }
+      if (target.access !== true && ACCESS_ENTITY_TYPES.has(entity.type)) {
+        return { unobserved: { reason: "filtered", detail: ACCESS_UNMANAGED_DETAIL } };
+      }
       if (entity.type === CLICKHOUSE_ENTITY_TYPES.grant) {
         return { unobserved: { reason: "unsupported-kind", detail: "a grant is compared as part of its grantees' grants, which chant sql plan reports" } };
       }

package/src/clickhouse/live/live.test.ts

@@ -60,6 +60,14 @@ describe("binding an environment to a server", () => {
     });
   });
 
+  test("access is managed only where the profile says so (#3716)", () => {
+    const on = resolveClickHouseTarget({ environment: "prod", config: { sql: { profiles: { prod: { url: "http://ch:8123", access: true } } } } as never, env: {} });
+    expect(on).toMatchObject({ access: true });
+    const off = resolveClickHouseTarget({ environment: "prod", config: { sql: { profiles: { prod: { url: "http://ch:8123" } } } } as never, env: {} });
+    expect(off).not.toHaveProperty("access");
+    expect(resolveClickHouseTarget({ env: { CLICKHOUSE_URL: "http://ch:8123" } })).not.toHaveProperty("access");
+  });
+
   test("a credential variable the profile names but nobody set is no-credentials", () => {
     const t = resolveClickHouseTarget({ environment: "prod", config: { sql: { profiles: { prod: { url: "x", password: { env: "P" } } } } }, env: {} });
     expect(isUnresolvedTarget(t) && t.reason).toBe("no-credentials");

package/src/clickhouse/plan/commands.ts

@@ -35,7 +35,7 @@ import { renderDiff } from "./report";
 import { keyedByQualifiedName, schemaFromBuildFile, schemaFromServer } from "./schema";
 import { dropFormattingOnly } from "./server-format";
 import { scopeOf } from "./normalize";
-import { declaredAccess } from "../live/catalog";
+import { ACCESS_KINDS, accessUnmanagedHint, declaredAccess } from "../live/catalog";
 import { bindClickHouse } from "../live/bind";
 import { rebuildOpSuggestions } from "./rebuild-handoff";
 import { readFileSync } from "node:fs";
@@ -120,7 +120,10 @@ export async function runDiff(ctx: CommandGroupContext): Promise<number> {
 /** The declared objects against the server, keys and labels by `database.name`. */
 export async function planAgainstServer(environment: string, buildFile: string, options: Parameters<typeof bindClickHouse>[0] = {}): Promise<SchemaDiff> {
   const target = await bindClickHouse({ ...options, environment });
-  const declared = keyedByQualifiedName(schemaFromBuildFile(buildFile, target.defaultDatabase, target.topology));
+  const all = keyedByQualifiedName(schemaFromBuildFile(buildFile, target.defaultDatabase, target.topology));
+  // Access declarations a profile does not manage are not planned, and the server's access is not read (#3716).
+  const declared = target.access === true ? all : all.filter((o) => !ACCESS_KINDS.has(o.canonical.kind));
+  const unmanaged = all.length - declared.length;
   const databases = new Set(declared.map((o) => scopeOf(o.canonical)));
   const unreadable: UnreadableEntry[] = [];
   const live = await schemaFromServer(target, databases, unreadable, declaredAccess(declared.map((o) => o.canonical)));
@@ -136,7 +139,7 @@ export async function planAgainstServer(environment: string, buildFile: string,
   return {
     changes: relabel,
     rebuilds: relabel.filter((c) => c.class === "rebuild"),
-    hints: diff.hints,
+    hints: unmanaged > 0 ? [...diff.hints, accessUnmanagedHint(unmanaged, environment)] : diff.hints,
     ...(rebuildOps.length > 0 ? { rebuildOps } : {}),
     ...(unreadable.length > 0 ? { unreadable } : {}),
   };

package/src/clickhouse/plan/deep-grants.test.ts

@@ -0,0 +1,67 @@
+/**
+ * The grants the deep read reports as pending (#3733): what an apply would
+ * grant or revoke from each declared grantee, so a privilege granted by hand
+ * after an approval moves the plan digest. The server is a stub answering the
+ * queries `readLiveAccess` sends.
+ */
+
+import { describe, expect, test, vi } from "vitest";
+import { grant } from "../entities";
+import type { ClickHouseTarget } from "../live/bind";
+
+/** What `SHOW GRANTS FOR` prints, by grantee; a grantee missing here does not exist. */
+let held: Record<string, string[]> = {};
+
+vi.mock("../http", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("../http")>();
+  return {
+    ...actual,
+    clickhouseQuery: async (_endpoint: unknown, sql: string) => {
+      if (/FROM system\.users/.test(sql)) return Object.keys(held).filter((n) => sql.includes(`'${n}'`)).map((name) => ({ name }));
+      if (/FROM system\.roles/.test(sql)) return [];
+      const show = /^SHOW GRANTS FOR `(.+)`$/.exec(sql.trim());
+      if (show) return (held[show[1]!] ?? []).map((line) => ({ [`GRANTS FOR ${show[1]}`]: line }));
+      throw new Error(`unexpected query: ${sql}`);
+    },
+  };
+});
+
+const { pendingGrants } = await import("./deep");
+
+const target: ClickHouseTarget = { endpoint: { url: "http://stub:8123" }, source: "sql.profiles.test", defaultDatabase: "default", access: true };
+
+const entities = (all: Record<string, ReturnType<typeof grant>>) =>
+  new Map(Object.entries(all).map(([k, e]) => [k, { entityType: e.entityType, props: e.props as unknown as Record<string, unknown> }]));
+
+const declared = entities({
+  readEvents: grant`GRANT SELECT ON shop.events TO app`,
+  readOrders: grant`GRANT SELECT ON shop.orders TO app`,
+});
+
+describe("pendingGrants (#3733)", () => {
+  test("a grantee holding what is declared has nothing pending", async () => {
+    held = { app: ["GRANT SELECT ON shop.events TO app", "GRANT SELECT ON shop.orders TO app"] };
+    expect(await pendingGrants(target, declared)).toEqual([]);
+  });
+
+  test("a privilege granted by hand is a REVOKE, one revoked by hand a GRANT, revokes first", async () => {
+    held = { app: ["GRANT INSERT, SELECT ON shop.events TO app"] };
+    expect(await pendingGrants(target, declared)).toEqual([
+      {
+        subject: "grants app",
+        change: "REVOKE INSERT ON `shop`.`events` FROM `app`; GRANT SELECT ON `shop`.`orders` TO `app`",
+        entities: ["readEvents", "readOrders"],
+      },
+    ]);
+  });
+
+  test("a grantee the server does not hold yet is given every declared grant", async () => {
+    held = {};
+    expect((await pendingGrants(target, declared)).map((p) => p.change)).toEqual(["GRANT SELECT ON `shop`.`events` TO `app`; GRANT SELECT ON `shop`.`orders` TO `app`"]);
+  });
+
+  test("no grant declarations read nothing", async () => {
+    held = {};
+    expect(await pendingGrants(target, new Map())).toEqual([]);
+  });
+});