@intentius/chant-lexicon-sql 0.118.1 → 0.120.0

package/src/clickhouse/import/live-export.ts

@@ -13,6 +13,14 @@
  * - `owned` keeps the objects whose comment carries chant's ownership marker
  *   (`../ownership.ts`), the trailer the applier stamps. The trailer itself is
  *   never written into a declaration: `readLiveSchema` takes it off.
+ * - a migration runner's history table (`schema_migrations`,
+ *   `goose_db_version`) is left out with a warning (#3676), as on Postgres.
+ * - A SQL user-defined function belongs to no database, and every project's
+ *   functions sit side by side on a server (#3718). Without a selector, an
+ *   import adopts only the functions the imported objects call (and those
+ *   functions call), and the ones `sql.profiles.<env>.importFunctions` names;
+ *   a warning names the rest. A function carries no comment, so `owned` does
+ *   not apply to it.
  */
 
 import type { ExportedTemplate, ResourceSelector } from "@intentius/chant/lexicon";
@@ -20,6 +28,48 @@ import { bindClickHouse, type BindOptions } from "../live/bind";
 import { readLiveSchema } from "../live/catalog";
 import { objectsToIR, stripServerDefaults, type ImportedObject } from "./ir";
 import { isChantManaged } from "../ownership";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
+import { isTrivia, tokenizeText } from "../tokens";
+import { unquote } from "../parser";
+import type { LiveObject } from "../live/catalog";
+
+/** The functions among `names` a statement calls: a name followed by `(`. */
+export function calledFunctions(ddl: string, names: ReadonlySet<string>): string[] {
+  const tokens = tokenizeText(ddl, 0).filter((t) => !isTrivia(t));
+  const out = new Set<string>();
+  tokens.forEach((t, i) => {
+    if (t.kind !== "ident" && t.kind !== "qident") return;
+    const next = tokens[i + 1];
+    if (next?.kind !== "punct" || next.text !== "(") return;
+    const name = t.kind === "qident" ? unquote(t.text) : t.text;
+    if (names.has(name)) out.add(name);
+  });
+  return [...out];
+}
+
+/** Whether `patterns` (names, or prefixes ending in `*`) name a function. */
+const named = (patterns: readonly string[], name: string) => patterns.some((p) => (p.endsWith("*") ? name.startsWith(p.slice(0, -1)) : p === name));
+
+/**
+ * The functions an import adopts (#3718): those the objects call, those
+ * `patterns` name, and the functions those call in turn; and the rest.
+ */
+export function scopedFunctions(objects: readonly LiveObject[], functions: readonly LiveObject[], patterns: readonly string[] = []): { adopted: LiveObject[]; skipped: string[] } {
+  const byName = new Map(functions.map((f) => [f.name, f]));
+  const names = new Set(byName.keys());
+  const adopted = new Set<string>(functions.filter((f) => named(patterns, f.name)).map((f) => f.name));
+  for (const o of objects) for (const n of calledFunctions(o.statement, names)) adopted.add(n);
+  const queue = [...adopted];
+  while (queue.length > 0) {
+    for (const n of calledFunctions(byName.get(queue.pop()!)!.statement, names)) {
+      if (!adopted.has(n)) {
+        adopted.add(n);
+        queue.push(n);
+      }
+    }
+  }
+  return { adopted: functions.filter((f) => adopted.has(f.name)), skipped: functions.filter((f) => !adopted.has(f.name)).map((f) => f.name) };
+}
 
 export interface ExportOptions extends Omit<BindOptions, "environment"> {
   environment: string;
@@ -32,7 +82,10 @@ export interface ExportOptions extends Omit<BindOptions, "environment"> {
 
 export async function exportResources(options: ExportOptions): Promise<ExportedTemplate> {
   const target = await bindClickHouse(options);
-  const live = await readLiveSchema(target);
+  const all = await readLiveSchema(target);
+  const isFunction = (o: LiveObject) => o.type === CLICKHOUSE_ENTITY_TYPES.function;
+  // A selector picks what it names, functions included; without one, functions go with what calls them (#3718).
+  const live = options.selector ? all : all.filter((o) => !isFunction(o));
   const selected = live.filter((o) => {
     if (options.owned && !isChantManaged(o.comment)) return false;
     if (options.selector?.type && o.type !== options.selector.type) return false;
@@ -40,14 +93,29 @@ export async function exportResources(options: ExportOptions): Promise<ExportedT
     if (name !== undefined && name !== o.name && name !== `${o.database}.${o.name}`) return false;
     return true;
   });
+  const warnings: string[] = [];
+  if (!options.selector) {
+    const { adopted, skipped } = scopedFunctions(selected.filter((o) => !o.foreign), all.filter(isFunction), target.importFunctions);
+    selected.push(...adopted);
+    if (skipped.length > 0) {
+      warnings.push(
+        `${skipped.length} SQL function${skipped.length === 1 ? "" : "s"} on the server ${skipped.length === 1 ? "is" : "are"} not imported, since no imported object calls ${skipped.length === 1 ? "it" : "them"}: ${skipped.join(", ")}. Name ${skipped.length === 1 ? "it" : "them"} in sql.profiles.${options.environment}.importFunctions to import ${skipped.length === 1 ? "it" : "them"}.`,
+      );
+    }
+  }
   const objects: ImportedObject[] = selected
     // The `default` database exists on every server; declaring it would make apply create what is there.
     .filter((o) => !(o.type === "ClickHouse::Database" && o.name === "default"))
+    .filter((o) => {
+      if (!o.foreign) return true;
+      warnings.push(`${o.database ? `${o.database}.` : ""}${o.name} is kept by ${o.foreign}; left out, since declaring it would have chant change what that tool owns`);
+      return false;
+    })
     .map((o) => ({
       type: o.type,
       ...(o.database ? { database: o.database } : {}),
       name: o.name,
       ddl: options.verbatim ? o.statement : stripServerDefaults(o.statement),
     }));
-  return objectsToIR(objects) as ExportedTemplate;
+  return objectsToIR(objects, warnings) as ExportedTemplate;
 }

package/src/clickhouse/live/access.e2e.test.ts

@@ -13,7 +13,11 @@
  * - a privilege granted by hand to a declared grantee is SQLCH274 and is
  *   revoked, a declared one revoked by hand is SQLCH273 and is granted
  *   again, a changed role setting is SQLCH270, a changed policy SQLCH272;
- * - the row policy filters rows for the user.
+ *   the deep read lists both as pending, with the statements (#3733);
+ * - the row policy filters rows for the user;
+ * - with a profile that does not manage access (no `access: true`, #3716),
+ *   none of it is applied, planned or read: each access declaration is
+ *   reported filtered, and a hand change to a grant is not undone.
  *
  * Beside a Replicated database (two replicas of the pinned server in
  * Docker): access control is in no database, so the database's log does not
@@ -23,7 +27,7 @@
  * `CLICKHOUSE_PASSWORD`) names a running one, such as the one
  * `chant emulator up --lexicon sql` starts. Without it, a throwaway server
  * at the pin, skipped cleanly when Docker is not available. Users and roles
- * are global to the server, so the test names everything `chant_e2e_3682_access*`
+ * are global to the server, so the test names everything `chant_e2e_3716_access*`
  * and drops it afterwards.
  */
 
@@ -37,9 +41,10 @@ import { clickhouseImage } from "../../spec/pin";
 import { database, grant, policy, role, table, user } from "../entities";
 import { planAgainstServer } from "../plan/commands";
 import { observeResourcesDeep, sqlDeepNormalizationHooks } from "../plan/deep";
+import { describeResources } from "./describe-resources";
 import { clickhouseApply } from "../../op/activities/clickhouse-apply";
 
-const DB = "chant_e2e_3682_access";
+const DB = "chant_e2e_3716_access";
 const READER = `${DB}_reader`;
 const APP = `${DB}_app`;
 const fromEnv = process.env.CLICKHOUSE_URL;
@@ -76,13 +81,27 @@ afterAll(async () => {
   rmSync(dir, { recursive: true, force: true });
 });
 
-const envOf = (at: ClickHouseEndpoint, topology?: string) => ({
-  CLICKHOUSE_URL: at.url,
-  ...(at.user ? { CLICKHOUSE_USER: at.user } : {}),
-  ...(at.password ? { CLICKHOUSE_PASSWORD: at.password } : {}),
-  ...(topology ? { CLICKHOUSE_TOPOLOGY: topology } : {}),
+const envOf = (at: ClickHouseEndpoint) => ({
+  ...(at.user ? { CH_USER: at.user } : {}),
+  ...(at.password ? { CH_PASSWORD: at.password } : {}),
 });
 
+/** The `test` profile: the server, and whether it manages access (#3716). */
+const configOf = (at: ClickHouseEndpoint, opts: { topology?: string; access?: boolean } = {}) => ({
+  ownership: { stack: "e2e3682", env: "test" },
+  sql: {
+    profiles: {
+      test: {
+        url: at.url,
+        ...(at.user ? { user: { env: "CH_USER" } } : {}),
+        ...(at.password ? { password: { env: "CH_PASSWORD" } } : {}),
+        ...(opts.topology ? { topology: opts.topology } : {}),
+        ...(opts.access !== false ? { access: true } : {}),
+      },
+    },
+  },
+}) as never;
+
 type Entity = { entityType: string; props: { ddl: string }; dependsOn?: unknown };
 function writeBuild(file: string, declared: Record<string, Entity>): string {
   const path = join(dir, file);
@@ -91,11 +110,11 @@ function writeBuild(file: string, declared: Record<string, Entity>): string {
   return path;
 }
 
-async function apply(at: ClickHouseEndpoint, buildPath: string, topology?: string) {
+async function apply(at: ClickHouseEndpoint, buildPath: string, opts: { topology?: string; access?: boolean } = {}) {
   const sent: string[] = [];
   const outcome = await clickhouseApply({ buildPath, environment: "test" }, undefined, {
-    config: { ownership: { stack: "e2e3682", env: "test" } },
-    env: envOf(at, topology),
+    config: configOf(at, opts),
+    env: envOf(at),
     log: (l) => void (/^[A-Z]+ /.test(l) && sent.push(l)),
   });
   return { outcome, sent };
@@ -146,11 +165,12 @@ describe.skipIf(!enabled)("access control on a single node (#3682)", () => {
     expect((await q<{ s: string }>(`SELECT auth_type[1] AS s FROM system.users WHERE name = '${APP}'`))[0]!.s).toBe("sha256_password");
 
     // The server prints the policy's condition and the grants its own way: still the declarations.
-    const diff = await planAgainstServer("test", build, { config: {}, env: envOf(endpoint) });
+    const diff = await planAgainstServer("test", build, { config: configOf(endpoint), env: envOf(endpoint) });
     expect(diff.changes).toEqual([]);
     const entities = new Map(Object.entries(v1).map(([k, e]) => [k, { entityType: e.entityType, props: e.props as unknown as Record<string, unknown> }]));
-    const deep = await observeResourcesDeep({ environment: "test", entityNames: [...entities.keys()], entities, config: {}, env: envOf(endpoint) });
+    const deep = await observeResourcesDeep({ environment: "test", entityNames: [...entities.keys()], entities, config: configOf(endpoint), env: envOf(endpoint) });
     expect(Object.keys(deep.unobserved ?? {}).sort()).toEqual(["readEvents", "readerToApp"]);
+    expect(deep.pending).toBeUndefined();
     for (const k of ["reader", "app", "tenant"] as const) expect(compared(deep.resources[k]!.properties), k).toEqual(compared(v1[k].props));
 
     // The policy filters rows for the user.
@@ -161,8 +181,14 @@ describe.skipIf(!enabled)("access control on a single node (#3682)", () => {
     // Changed by hand: a privilege granted, a declared one revoked. Changed in the declaration: a setting and the condition.
     await q(`GRANT INSERT ON ${DB}.events TO ${READER}`);
     await q(`REVOKE ${READER} FROM ${APP}`);
+    // The deep read lists what an apply would grant and revoke (#3733), so an approval's plan digest moves.
+    const handMade = await observeResourcesDeep({ environment: "test", entityNames: [...entities.keys()], entities, config: configOf(endpoint), env: envOf(endpoint) });
+    expect(handMade.pending).toEqual([
+      { subject: `grants ${READER}`, change: `REVOKE INSERT ON \`${DB}\`.\`events\` FROM \`${READER}\``, entities: ["readEvents"] },
+      { subject: `grants ${APP}`, change: `GRANT \`${READER}\` TO \`${APP}\``, entities: ["readerToApp"] },
+    ]);
     const v2 = declarations({ memory: "2000000000", tenant: "b" });
-    const planned = await planAgainstServer("test", writeBuild("v2.json", v2), { config: {}, env: envOf(endpoint) });
+    const planned = await planAgainstServer("test", writeBuild("v2.json", v2), { config: configOf(endpoint), env: envOf(endpoint) });
     expect(planned.changes.map((c) => [c.object, c.field, c.rule])).toEqual([
       [`reader (role ${READER})`, "settings", "SQLCH270"],
       // Revoking the role took it off the user's default roles too.
@@ -180,8 +206,35 @@ describe.skipIf(!enabled)("access control on a single node (#3682)", () => {
       `GRANT \`${READER}\` TO \`${APP}\``,
       `ALTER USER \`${APP}\` DEFAULT ROLE ${READER}`,
     ]);
-    expect((await planAgainstServer("test", writeBuild("v2.json", v2), { config: {}, env: envOf(endpoint) })).changes).toEqual([]);
+    expect((await planAgainstServer("test", writeBuild("v2.json", v2), { config: configOf(endpoint), env: envOf(endpoint) })).changes).toEqual([]);
     expect(Number((await q<{ n: string }>(`SELECT count() AS n FROM ${DB}.events`, asApp))[0]!.n)).toBe(1);
+
+    // A profile that does not manage access (#3716): nothing access-related is read, planned or applied.
+    await q(`GRANT INSERT ON ${DB}.events TO ${READER}`);
+    const off = { access: false };
+    const v3 = declarations({ memory: "3000000000", tenant: "a" });
+    const offBuild = writeBuild("v3.json", v3);
+    const offPlan = await planAgainstServer("test", offBuild, { config: configOf(endpoint, off), env: envOf(endpoint) });
+    expect(offPlan.changes).toEqual([]);
+    expect(offPlan.hints).toContain("5 access declarations are not planned: test's profile does not manage access (sql.profiles.<env>.access)");
+    const offApply = await apply(endpoint, offBuild, off);
+    expect(offApply.outcome.failed).toEqual([]);
+    expect(offApply.sent).toEqual([]);
+    expect(offApply.outcome.notAttempted.map((n) => [n.name, n.reason])).toEqual([
+      [`role ${READER}`, "filtered"],
+      [`user ${APP}`, "filtered"],
+      [`row policy ${DB}_tenant ON ${DB}.events`, "filtered"],
+      [`grants ${READER}`, "filtered"],
+      [`grants ${APP}`, "filtered"],
+    ]);
+    expect((await q<{ s: string }>(`SHOW GRANTS FOR ${READER}`)).map((r) => Object.values(r)[0]).join("\n")).toMatch(/GRANT INSERT\b.* ON chant_e2e_3716_access\.events TO/);
+    const v3entities = new Map(Object.entries(v3).map(([k, e]) => [k, { entityType: e.entityType, props: e.props as unknown as Record<string, unknown> }]));
+    const offDeep = await observeResourcesDeep({ environment: "test", entityNames: [...v3entities.keys()], entities: v3entities, config: configOf(endpoint, off), env: envOf(endpoint) });
+    expect(Object.entries(offDeep.unobserved ?? {}).map(([k, u]) => [k, u.reason]).sort()).toEqual(
+      ["app", "readEvents", "reader", "readerToApp", "tenant"].map((k) => [k, "filtered"]),
+    );
+    const offThin = await describeResources({ environment: "test", entityNames: [...v3entities.keys()], entities: v3entities, config: configOf(endpoint, off), env: envOf(endpoint) });
+    expect(Object.keys(offThin.unobserved ?? {}).sort()).toEqual(["app", "readEvents", "reader", "readerToApp", "tenant"]);
   }, 300_000);
 });
 
@@ -198,12 +251,12 @@ describe.skipIf(!docker)("access control beside a Replicated database (#3682)",
     const [r1, r2] = cluster!.replicas as [ClickHouseEndpoint, ClickHouseEndpoint];
     const { shop, events, reader, tenant, readEvents } = declarations();
     const build = writeBuild("repl.json", { shop, events, reader, tenant, readEvents });
-    const first = await apply(r1, build, "replicated");
+    const first = await apply(r1, build, { topology: "replicated" });
     expect(first.outcome.failed).toEqual([]);
     expect(first.sent.join("\n")).not.toMatch(/ON CLUSTER/);
     await q(first.sent[0]!, r2);
     await q(`SYSTEM SYNC DATABASE REPLICA ${DB}`, r2);
-    const second = await apply(r2, build, "replicated");
+    const second = await apply(r2, build, { topology: "replicated" });
     expect(second.outcome.failed).toEqual([]);
     expect(second.outcome.applied.filter((a) => a.action === "created").map((a) => a.name)).toEqual([
       `role ${READER}`,
@@ -212,7 +265,7 @@ describe.skipIf(!docker)("access control beside a Replicated database (#3682)",
     ]);
     for (const at of [r1, r2]) {
       expect(Number((await q<{ n: string }>(`SELECT count() AS n FROM system.row_policies WHERE short_name = '${DB}_tenant'`, at))[0]!.n)).toBe(1);
-      expect((await apply(at, build, "replicated")).sent).toEqual([]);
+      expect((await apply(at, build, { topology: "replicated" })).sent).toEqual([]);
     }
   }, 300_000);
 });

package/src/clickhouse/live/bind.ts

@@ -24,6 +24,14 @@ export interface ClickHouseTarget {
    * as declared.
    */
   topology?: Topology;
+  /**
+   * Whether chant manages access here (`sql.profiles.<env>.access`, #3716):
+   * the declared users, roles, row policies and grants. Off when omitted, as
+   * on Postgres, and off for a server bound by `CLICKHOUSE_URL`.
+   */
+  access?: boolean;
+  /** `sql.profiles.<env>.importFunctions` (#3718): the functions an import adopts besides those the imported objects call. */
+  importFunctions?: string[];
 }
 
 export interface UnresolvedTarget {
@@ -75,6 +83,8 @@ export function resolveClickHouseTarget(input: {
       ...(profile.databases ? { databases: profile.databases } : {}),
       defaultDatabase: profile.defaultDatabase ?? "default",
       ...(profile.topology !== undefined ? { topology: toTopology(profile.topology) } : {}),
+      ...(profile.access === true ? { access: true } : {}),
+      ...(profile.importFunctions ? { importFunctions: profile.importFunctions } : {}),
     };
   }
   const url = env.CLICKHOUSE_URL;

package/src/clickhouse/live/catalog.ts

@@ -9,6 +9,7 @@ import { clickhouseQuery } from "../http";
 import type { ClickHouseTarget } from "./bind";
 import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "../entities";
 import { isChantWorkingObject, stripMarkerFromStatement } from "../ownership";
+import { foreignTool } from "../../core/foreign-tables";
 
 /** The server's own databases, never part of a declared schema. */
 export const SYSTEM_DATABASES = new Set(["system", "information_schema", "INFORMATION_SCHEMA"]);
@@ -26,6 +27,8 @@ export interface LiveObject {
   statement: string;
   /** A row policy's table. */
   table?: string;
+  /** The tool that keeps it, for a migration runner's history table (`../../core/foreign-tables.ts`): never chant's to import, change or drop. */
+  foreign?: string;
 }
 
 const quote = (s: string) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
@@ -46,6 +49,11 @@ export function entityTypeOfEngine(engine: string): ClickHouseEntityType {
  * `SELECT` on the database sees it without a grant on `system.dictionaries`;
  * a table with the `Dictionary` engine is a table.
  *
+ * A table another tool keeps its migration history in (`schema_migrations`,
+ * `goose_db_version`, `../../core/foreign-tables.ts`) is read and marked with
+ * the tool, as the Postgres reader marks it (#3676): import leaves it out
+ * with a warning, and a plan names it in a hint instead of dropping it.
+ *
  * So are chant's own working objects (`../ownership.ts`
  * `isChantWorkingObject`): a rebuild migration's new, dual-write and retained
  * tables, and the effect receipts database. None of them is a declaration, so
@@ -87,13 +95,16 @@ export async function readLiveSchema(target: ClickHouseTarget, opts: { withState
   }
   for (const t of tables) {
     if (working.has(t.database) || isChantWorkingObject(t.comment)) continue;
+    const type = Number(t.dictionary) === 1 ? CLICKHOUSE_ENTITY_TYPES.dictionary : entityTypeOfEngine(t.engine);
+    const foreign = type === CLICKHOUSE_ENTITY_TYPES.table ? foreignTool(t.name) : undefined;
     out.push({
-      type: Number(t.dictionary) === 1 ? CLICKHOUSE_ENTITY_TYPES.dictionary : entityTypeOfEngine(t.engine),
+      type,
       database: t.database,
       name: t.name,
       engine: t.engine,
       ...(t.uuid && !/^0{8}-/.test(t.uuid) ? { uuid: t.uuid } : {}),
       ...(t.comment ? { comment: t.comment } : {}),
+      ...(foreign ? { foreign } : {}),
       statement: await statement("TABLE", `${ident(t.database)}.${ident(t.name)}`),
     });
   }
@@ -168,7 +179,19 @@ export async function readLiveAccess(target: ClickHouseTarget, declared: readonl
   return out;
 }
 
-const ACCESS_KINDS = new Set(["user", "role", "rowPolicy", "grants"]);
+/** The canonical kinds a profile's `access` switch decides (#3716). */
+export const ACCESS_KINDS: ReadonlySet<string> = new Set(["user", "role", "rowPolicy", "grants"]);
+
+/** The entity types a profile's `access` switch decides (#3716). */
+export const ACCESS_ENTITY_TYPES: ReadonlySet<string> = new Set([CLICKHOUSE_ENTITY_TYPES.user, CLICKHOUSE_ENTITY_TYPES.role, CLICKHOUSE_ENTITY_TYPES.rowPolicy, CLICKHOUSE_ENTITY_TYPES.grant]);
+
+/** Why an access declaration is not read, planned or applied where the profile does not manage access. */
+export const ACCESS_UNMANAGED_DETAIL = "access is not managed in this environment: set sql.profiles.<env>.access to plan and apply users, roles, row policies and grants";
+
+/** The hint a plan gives for the access declarations it left out. */
+export function accessUnmanagedHint(n: number, environment: string): string {
+  return `${n} access declaration${n === 1 ? " is" : "s are"} not planned: ${environment}'s profile does not manage access (sql.profiles.<env>.access)`;
+}
 
 /** The access objects among declared canonical objects, for `readLiveAccess`. */
 export function declaredAccess(objects: ReadonlyArray<{ kind: string; name: string; database?: string; table?: string }>): DeclaredAccess[] {

package/src/clickhouse/live/describe-resources.ts

@@ -24,7 +24,7 @@ import {
   type ObserverAdapter,
 } from "@intentius/chant/observation";
 import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "./bind";
-import { accessOf, readLiveAccess, readLiveSchema, type DeclaredAccess, type LiveObject } from "./catalog";
+import { ACCESS_ENTITY_TYPES, ACCESS_UNMANAGED_DETAIL, accessOf, readLiveAccess, readLiveSchema, type DeclaredAccess, type LiveObject } from "./catalog";
 import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
 import { objectKey } from "../plan/normalize";
 import { isChantManaged, readMarker, stripMarker } from "../ownership";
@@ -54,7 +54,7 @@ function adapter(options: BindOptions & { owned?: boolean; access?: (defaultData
     async bind() {
       const target = await bindClickHouse(options);
       const byKey = new Map<string, LiveObject>();
-      const access = await readLiveAccess(target, options.access?.(target.defaultDatabase) ?? [], { withStatements: false });
+      const access = target.access === true ? await readLiveAccess(target, options.access?.(target.defaultDatabase) ?? [], { withStatements: false }) : [];
       for (const o of [...(await readLiveSchema(target, { withStatements: false })), ...access]) {
         byKey.set(objectKey(o), o);
       }
@@ -65,6 +65,9 @@ function adapter(options: BindOptions & { owned?: boolean; access?: (defaultData
       if (!entity.type.startsWith("ClickHouse::")) {
         return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
       }
+      if (target.access !== true && ACCESS_ENTITY_TYPES.has(entity.type)) {
+        return { unobserved: { reason: "filtered", detail: ACCESS_UNMANAGED_DETAIL } };
+      }
       if (entity.type === CLICKHOUSE_ENTITY_TYPES.grant) {
         return { unobserved: { reason: "unsupported-kind", detail: "a grant is compared as part of its grantees' grants, which chant sql plan reports" } };
       }

package/src/clickhouse/live/live.test.ts

@@ -60,6 +60,14 @@ describe("binding an environment to a server", () => {
     });
   });
 
+  test("access is managed only where the profile says so (#3716)", () => {
+    const on = resolveClickHouseTarget({ environment: "prod", config: { sql: { profiles: { prod: { url: "http://ch:8123", access: true } } } } as never, env: {} });
+    expect(on).toMatchObject({ access: true });
+    const off = resolveClickHouseTarget({ environment: "prod", config: { sql: { profiles: { prod: { url: "http://ch:8123" } } } } as never, env: {} });
+    expect(off).not.toHaveProperty("access");
+    expect(resolveClickHouseTarget({ env: { CLICKHOUSE_URL: "http://ch:8123" } })).not.toHaveProperty("access");
+  });
+
   test("a credential variable the profile names but nobody set is no-credentials", () => {
     const t = resolveClickHouseTarget({ environment: "prod", config: { sql: { profiles: { prod: { url: "x", password: { env: "P" } } } } }, env: {} });
     expect(isUnresolvedTarget(t) && t.reason).toBe("no-credentials");

package/src/clickhouse/plan/commands.ts

@@ -35,7 +35,7 @@ import { renderDiff } from "./report";
 import { keyedByQualifiedName, schemaFromBuildFile, schemaFromServer } from "./schema";
 import { dropFormattingOnly } from "./server-format";
 import { scopeOf } from "./normalize";
-import { declaredAccess } from "../live/catalog";
+import { ACCESS_KINDS, accessUnmanagedHint, declaredAccess } from "../live/catalog";
 import { bindClickHouse } from "../live/bind";
 import { rebuildOpSuggestions } from "./rebuild-handoff";
 import { readFileSync } from "node:fs";
@@ -120,7 +120,10 @@ export async function runDiff(ctx: CommandGroupContext): Promise<number> {
 /** The declared objects against the server, keys and labels by `database.name`. */
 export async function planAgainstServer(environment: string, buildFile: string, options: Parameters<typeof bindClickHouse>[0] = {}): Promise<SchemaDiff> {
   const target = await bindClickHouse({ ...options, environment });
-  const declared = keyedByQualifiedName(schemaFromBuildFile(buildFile, target.defaultDatabase, target.topology));
+  const all = keyedByQualifiedName(schemaFromBuildFile(buildFile, target.defaultDatabase, target.topology));
+  // Access declarations a profile does not manage are not planned, and the server's access is not read (#3716).
+  const declared = target.access === true ? all : all.filter((o) => !ACCESS_KINDS.has(o.canonical.kind));
+  const unmanaged = all.length - declared.length;
   const databases = new Set(declared.map((o) => scopeOf(o.canonical)));
   const unreadable: UnreadableEntry[] = [];
   const live = await schemaFromServer(target, databases, unreadable, declaredAccess(declared.map((o) => o.canonical)));
@@ -136,7 +139,7 @@ export async function planAgainstServer(environment: string, buildFile: string,
   return {
     changes: relabel,
     rebuilds: relabel.filter((c) => c.class === "rebuild"),
-    hints: diff.hints,
+    hints: unmanaged > 0 ? [...diff.hints, accessUnmanagedHint(unmanaged, environment)] : diff.hints,
     ...(rebuildOps.length > 0 ? { rebuildOps } : {}),
     ...(unreadable.length > 0 ? { unreadable } : {}),
   };

package/src/clickhouse/plan/deep-grants.test.ts

@@ -0,0 +1,67 @@
+/**
+ * The grants the deep read reports as pending (#3733): what an apply would
+ * grant or revoke from each declared grantee, so a privilege granted by hand
+ * after an approval moves the plan digest. The server is a stub answering the
+ * queries `readLiveAccess` sends.
+ */
+
+import { describe, expect, test, vi } from "vitest";
+import { grant } from "../entities";
+import type { ClickHouseTarget } from "../live/bind";
+
+/** What `SHOW GRANTS FOR` prints, by grantee; a grantee missing here does not exist. */
+let held: Record<string, string[]> = {};
+
+vi.mock("../http", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("../http")>();
+  return {
+    ...actual,
+    clickhouseQuery: async (_endpoint: unknown, sql: string) => {
+      if (/FROM system\.users/.test(sql)) return Object.keys(held).filter((n) => sql.includes(`'${n}'`)).map((name) => ({ name }));
+      if (/FROM system\.roles/.test(sql)) return [];
+      const show = /^SHOW GRANTS FOR `(.+)`$/.exec(sql.trim());
+      if (show) return (held[show[1]!] ?? []).map((line) => ({ [`GRANTS FOR ${show[1]}`]: line }));
+      throw new Error(`unexpected query: ${sql}`);
+    },
+  };
+});
+
+const { pendingGrants } = await import("./deep");
+
+const target: ClickHouseTarget = { endpoint: { url: "http://stub:8123" }, source: "sql.profiles.test", defaultDatabase: "default", access: true };
+
+const entities = (all: Record<string, ReturnType<typeof grant>>) =>
+  new Map(Object.entries(all).map(([k, e]) => [k, { entityType: e.entityType, props: e.props as unknown as Record<string, unknown> }]));
+
+const declared = entities({
+  readEvents: grant`GRANT SELECT ON shop.events TO app`,
+  readOrders: grant`GRANT SELECT ON shop.orders TO app`,
+});
+
+describe("pendingGrants (#3733)", () => {
+  test("a grantee holding what is declared has nothing pending", async () => {
+    held = { app: ["GRANT SELECT ON shop.events TO app", "GRANT SELECT ON shop.orders TO app"] };
+    expect(await pendingGrants(target, declared)).toEqual([]);
+  });
+
+  test("a privilege granted by hand is a REVOKE, one revoked by hand a GRANT, revokes first", async () => {
+    held = { app: ["GRANT INSERT, SELECT ON shop.events TO app"] };
+    expect(await pendingGrants(target, declared)).toEqual([
+      {
+        subject: "grants app",
+        change: "REVOKE INSERT ON `shop`.`events` FROM `app`; GRANT SELECT ON `shop`.`orders` TO `app`",
+        entities: ["readEvents", "readOrders"],
+      },
+    ]);
+  });
+
+  test("a grantee the server does not hold yet is given every declared grant", async () => {
+    held = {};
+    expect((await pendingGrants(target, declared)).map((p) => p.change)).toEqual(["GRANT SELECT ON `shop`.`events` TO `app`; GRANT SELECT ON `shop`.`orders` TO `app`"]);
+  });
+
+  test("no grant declarations read nothing", async () => {
+    held = {};
+    expect(await pendingGrants(target, new Map())).toEqual([]);
+  });
+});

package/src/clickhouse/plan/deep.ts

@@ -17,17 +17,29 @@
  * (#3664). When a server's copy of a table differs from the profile
  * server's, that copy is what is reported, with the server in `observedOn`,
  * which the drift report prints as `seen on`.
+ *
+ * Where the profile manages access, the grants an apply would give or take
+ * from each declared grantee are reported as `pending` (#3733), the same
+ * statements `chant sql plan` plans (SQLCH273, SQLCH274), so a privilege
+ * granted or revoked by hand after an approval moves the plan digest.
  */
 
-import { deepObservation, type DeepObservationResult, type DeepResourceObservation, type DeepNormalizationHooks } from "@intentius/chant/deep-observation";
+import {
+  deepObservation,
+  type DeepObservationResult,
+  type DeepPendingChange,
+  type DeepResourceObservation,
+  type DeepNormalizationHooks,
+} from "@intentius/chant/deep-observation";
 import type { UnobservedEntity } from "@intentius/chant/lexicon";
 import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "../live/bind";
-import { accessOf, readLiveAccess, readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
+import { ACCESS_ENTITY_TYPES, ACCESS_UNMANAGED_DETAIL, accessOf, readLiveAccess, readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
 import { clickhouseQuery } from "../http";
 import { sqlString } from "../apply/statements";
 import { stripMarkerFromStatement } from "../ownership";
 import { database, dictionary, func, policy, role, table, user, view, CLICKHOUSE_ENTITY_TYPES, type ColumnDef } from "../entities";
-import { canonicalObject, objectKey, type CanonicalColumn, type CanonicalObject } from "./normalize";
+import { canonicalObject, grantsObject, objectKey, type CanonicalColumn, type CanonicalObject } from "./normalize";
+import { grantStatement, grantsByGrantee, revokeStatement, type GrantAtom } from "../access";
 import { renderFor } from "../topology";
 
 type Props = Record<string, unknown>;
@@ -207,7 +219,7 @@ export async function observeResourcesDeep(
   let live: LiveObject[];
   try {
     target = await bindClickHouse(options);
-    live = [...(await readLiveSchema(target)), ...(await readLiveAccess(target, accessOf(options.entities, target.defaultDatabase)))];
+    live = [...(await readLiveSchema(target)), ...(target.access === true ? await readLiveAccess(target, accessOf(options.entities, target.defaultDatabase)) : [])];
   } catch (err) {
     const why = classifyClickHouseFailure(err);
     for (const name of options.entityNames) {
@@ -227,17 +239,34 @@ export async function observeResourcesDeep(
     }
     return deepObservation({}, unobserved);
   }
+  const pending: DeepPendingChange[] = [];
+  if (target.access === true) {
+    try {
+      pending.push(...(await pendingGrants(target, options.entities)));
+    } catch (err) {
+      const why = classifyClickHouseFailure(err);
+      for (const name of options.entityNames) {
+        if (options.entities.get(name)?.entityType !== CLICKHOUSE_ENTITY_TYPES.grant) continue;
+        unobserved[name] = { type: CLICKHOUSE_ENTITY_TYPES.grant, reason: "read-failed", detail: `the server's grants could not be compared: ${why.detail}` };
+      }
+    }
+  }
   for (const name of options.entityNames) {
     const entity = options.entities.get(name);
     if (!entity || !entity.entityType.startsWith("ClickHouse::")) {
       unobserved[name] = { type: entity?.entityType ?? "", reason: "unsupported-kind" };
       continue;
     }
+    if (target.access !== true && ACCESS_ENTITY_TYPES.has(entity.entityType)) {
+      unobserved[name] = { type: entity.entityType, reason: "filtered", detail: ACCESS_UNMANAGED_DETAIL };
+      continue;
+    }
     if (entity.entityType === CLICKHOUSE_ENTITY_TYPES.grant) {
+      if (unobserved[name]) continue;
       unobserved[name] = {
         type: entity.entityType,
         reason: "unsupported-kind",
-        detail: "a grant is compared as part of its grantees' grants, which chant sql plan reports (SQLCH273, SQLCH274)",
+        detail: "a grant is compared as part of its grantees' grants: what an apply would grant or revoke is listed as pending (SQLCH273, SQLCH274)",
       };
       continue;
     }
@@ -280,5 +309,40 @@ export async function observeResourcesDeep(
       unobserved[name] = { type: entity.entityType, reason: "read-failed", detail: `the server's definition does not parse: ${(err as Error).message}` };
     }
   }
-  return deepObservation(resources, unobserved);
+  return deepObservation(resources, unobserved, pending);
+}
+
+/**
+ * The grants an apply would give or take from each declared grantee (#3733),
+ * as `chant sql plan` compares them: the declared grants against what
+ * `SHOW GRANTS FOR` prints, atom by atom. One change per grantee, its
+ * REVOKEs before its GRANTs as the applier sends them, with the grant
+ * declarations behind it. A grantee the server does not hold yet is given
+ * every declared grant.
+ */
+export async function pendingGrants(
+  target: ClickHouseTarget,
+  entities: ReadonlyMap<string, { entityType: string; props: Record<string, unknown> }>,
+): Promise<DeepPendingChange[]> {
+  const declared = grantsByGrantee([...entities].map(([name, e]) => ({ export: name, type: e.entityType, ddl: String(e.props.ddl ?? "") })));
+  if (declared.length === 0) return [];
+  const live = await readLiveAccess(target, declared.map((g) => ({ kind: "grants", name: g.grantee })));
+  const held = new Map(live.filter((o) => o.type === CLICKHOUSE_ENTITY_TYPES.grant).map((o) => [o.name, grantsObject(o.statement, o.name, target.defaultDatabase).access ?? {}]));
+  const atom = (json: string) => JSON.parse(json) as GrantAtom;
+  const out: DeepPendingChange[] = [];
+  for (const g of declared) {
+    const want = grantsObject(g.ddl, g.grantee, target.defaultDatabase).access ?? {};
+    const have = held.get(g.grantee) ?? {};
+    const statements = [
+      ...Object.keys(have)
+        .filter((k) => !(k in want))
+        .map((k) => revokeStatement(atom(have[k]!), g.grantee)),
+      ...Object.keys(want)
+        .filter((k) => !(k in have))
+        .map((k) => grantStatement(atom(want[k]!), g.grantee)),
+    ];
+    if (statements.length === 0) continue;
+    out.push({ subject: `grants ${g.grantee}`, change: statements.join("; "), entities: [...g.exports].sort() });
+  }
+  return out;
 }

package/src/clickhouse/plan/diff.ts

@@ -259,7 +259,10 @@ export function diffSchemas(before: readonly SchemaObject[], after: readonly Sch
   for (const m of matches) {
     if (m.kind === "created") changes.push(change(m.after.key, "object", "SQLCH200", undefined, qualified(m.after.canonical)));
     else if (m.kind === "matched") diffObject(m.after.key, m.before.canonical, m.after.canonical, changes, hints);
-    else {
+    else if (m.before.canonical.foreign) {
+      // A migration runner's history table (#3676): not chant's to drop.
+      hints.push(`${qualified(m.before.canonical)} is kept by ${m.before.canonical.foreign}; it is not chant's to drop and is left alone`);
+    } else {
       const b = m.before;
       changes.push(change(b.key, "object", "SQLCH250", qualified(b.canonical), undefined, { destructive: b.canonical.kind === "table" || b.canonical.kind === "materializedView" }));
     }