@intentius/chant-lexicon-sql 0.118.0 → 0.120.0

package/src/postgres/live/client.ts

@@ -90,3 +90,88 @@ export async function connectPostgres(endpoint: PostgresEndpoint, options: { app
     },
   };
 }
+
+/** SQLSTATE lock_not_available: `lock_timeout` passed while a statement waited for a lock. */
+export const SQLSTATE_LOCK_NOT_AVAILABLE = "55P03";
+
+/** The `lock_timeout` a statement waits under when the profile sets none (`sql.profiles.<env>.lockTimeoutMs`): the applier's and the catalog reader's. */
+export const DEFAULT_LOCK_TIMEOUT_MS = 5_000;
+
+/** A session holding ACCESS EXCLUSIVE on a relation: what a catalog read waits behind. */
+export interface LockHolder {
+  pid: number;
+  relation: string;
+  application?: string;
+  state?: string;
+  /** Seconds since the holder's transaction began. */
+  seconds?: number;
+  query?: string;
+}
+
+/**
+ * The sessions holding ACCESS EXCLUSIVE on a relation in the current database,
+ * other than this one. Only that mode conflicts with the ACCESS SHARE a
+ * catalog function (`pg_get_viewdef()`, `pg_get_indexdef()`) takes. Empty when
+ * the lookup itself fails.
+ */
+export async function accessExclusiveHolders(client: PostgresClient): Promise<LockHolder[]> {
+  try {
+    const rows = await client.query<{ pid: number; relation: string; application: string | null; state: string | null; seconds: number | null; query: string | null }>(
+      `SELECT l.pid, l.relation::pg_catalog.regclass::text AS relation, a.application_name AS application, a.state,
+              (EXTRACT(EPOCH FROM pg_catalog.now() - a.xact_start))::int AS seconds, pg_catalog.left(a.query, 120) AS query
+         FROM pg_catalog.pg_locks l LEFT JOIN pg_catalog.pg_stat_activity a ON a.pid = l.pid
+        WHERE l.locktype = 'relation' AND l.granted AND l.mode = 'AccessExclusiveLock'
+          AND l.pid <> pg_catalog.pg_backend_pid()
+          AND l.database = (SELECT oid FROM pg_catalog.pg_database WHERE datname = pg_catalog.current_database())
+        ORDER BY l.pid, 2 LIMIT 10`,
+    );
+    return rows.map((r) => ({
+      pid: Number(r.pid),
+      relation: r.relation,
+      ...(r.application ? { application: r.application } : {}),
+      ...(r.state ? { state: r.state } : {}),
+      ...(r.seconds !== null && r.seconds !== undefined ? { seconds: Number(r.seconds) } : {}),
+      ...(r.query ? { query: r.query.replace(/\s+/g, " ").trim() } : {}),
+    }));
+  } catch {
+    return [];
+  }
+}
+
+/** One line naming the lock holders, for a message. */
+export function describeLockHolders(holders: readonly LockHolder[]): string {
+  if (holders.length === 0) return "no session holds ACCESS EXCLUSIVE now; the lock was released after the wait";
+  return holders
+    .map((h) => {
+      const about = [h.application ? `application ${JSON.stringify(h.application)}` : "", h.state ?? "", h.seconds !== undefined ? `transaction open ${h.seconds}s` : "", h.query ? `last statement: ${h.query}` : ""].filter(Boolean);
+      return `${h.relation} is held in ACCESS EXCLUSIVE by pid ${h.pid}${about.length > 0 ? ` (${about.join(", ")})` : ""}`;
+    })
+    .join("; ");
+}
+
+/**
+ * A client for catalog reads: the session's `lock_timeout` is `lockTimeoutMs`
+ * (0 waits without limit, as Postgres does), so a read that needs a table
+ * another session holds in ACCESS EXCLUSIVE (a long `ALTER TABLE`, `LOCK
+ * TABLE`) fails after the wait instead of hanging. The failure names the
+ * relation and the pid holding it.
+ */
+export async function withCatalogLockTimeout(client: PostgresClient, lockTimeoutMs: number = DEFAULT_LOCK_TIMEOUT_MS): Promise<PostgresClient> {
+  await client.query("SELECT pg_catalog.set_config('lock_timeout', $1, false)", [`${lockTimeoutMs}ms`]);
+  return {
+    async query<T>(sql: string, params?: readonly unknown[]): Promise<T[]> {
+      try {
+        return await client.query<T>(sql, params);
+      } catch (err) {
+        if (!(err instanceof PostgresQueryError) || err.code !== SQLSTATE_LOCK_NOT_AVAILABLE) throw err;
+        const holders = describeLockHolders(await accessExclusiveHolders(client));
+        throw new PostgresQueryError(
+          `${err.message.split("\n")[0]}: the catalog read waited ${lockTimeoutMs}ms for a lock; ${holders}. Wait for that transaction to finish, or raise sql.profiles.<env>.lockTimeoutMs`,
+          err.code,
+          err.position,
+        );
+      }
+    },
+    end: () => client.end(),
+  };
+}

package/src/postgres/live/live.test.ts

@@ -196,3 +196,57 @@ describe("a managed provider's own objects (sql.provider)", () => {
     expect(r.resources.authUsers).toMatchObject({ ownership: "foreign", attributes: { keptBy: "Supabase" } });
   });
 });
+
+describe("catalog reads under lock_timeout (#3678)", () => {
+  /** A fake whose view read waits behind a lock and times out, and whose pg_locks names the holder. */
+  const blocked = (log: string[]) => {
+    const inner = fakeClient(CATALOG, log);
+    return {
+      async query<T>(sql: string, params?: readonly unknown[]): Promise<T[]> {
+        if (sql.includes("c.relkind IN ('v', 'm')")) {
+          log.push(sql);
+          throw new PostgresQueryError("canceling statement due to lock timeout", "55P03");
+        }
+        if (sql.includes("pg_catalog.pg_locks")) {
+          log.push(sql);
+          return [{ pid: 4242, relation: "app.orders", application: "psql", state: "idle in transaction", seconds: 12, query: "LOCK TABLE app.orders IN ACCESS EXCLUSIVE MODE" }] as T[];
+        }
+        return inner.query<T>(sql, params);
+      },
+      end: () => inner.end(),
+    };
+  };
+
+  test("the session's lock_timeout is the profile's lockTimeoutMs, else the applier's 5000 ms", async () => {
+    for (const [lockTimeoutMs, expected] of [[undefined, "5000ms"], [250, "250ms"]] as const) {
+      const log: string[] = [];
+      const params: unknown[][] = [];
+      const cfg = { sql: { profiles: { prod: { ...config.sql.profiles.prod, ...(lockTimeoutMs !== undefined ? { lockTimeoutMs } : {}) } } } };
+      await describeResources({
+        ...options({ users: DECLARED.users }),
+        config: cfg,
+        connect: async () => {
+          const c = fakeClient(CATALOG, log);
+          return { query: (sql: string, p?: readonly unknown[]) => (params.push([sql, ...(p ?? [])]), c.query(sql, p)), end: c.end } as never;
+        },
+      });
+      expect(params[0]).toEqual(["SELECT pg_catalog.set_config('lock_timeout', $1, false)", expected]);
+    }
+  });
+
+  test("a read that times out on a lock is read-failed, naming the relation and the pid holding it", async () => {
+    const log: string[] = [];
+    const r = normalizeObservation(await describeResources({ ...options({ users: DECLARED.users }), connect: async () => blocked(log) as never }));
+    const users = r.unobserved.users;
+    expect(users?.reason).toBe("read-failed");
+    expect(users?.detail).toContain("canceling statement due to lock timeout");
+    expect(users?.detail).toContain("app.orders is held in ACCESS EXCLUSIVE by pid 4242");
+    expect(users?.detail).toContain("lockTimeoutMs");
+  });
+
+  test("other failures pass through unchanged", async () => {
+    const { withCatalogLockTimeout } = await import("./client");
+    const c = await withCatalogLockTimeout({ query: async (sql: string) => { if (sql.startsWith("SELECT 1")) throw new PostgresQueryError("boom", "XX000"); return []; }, end: async () => {} }, 100);
+    await expect(c.query("SELECT 1")).rejects.toThrow(/^boom$/);
+  });
+});

package/src/postgres/live/lock-timeout.e2e.test.ts

@@ -0,0 +1,70 @@
+/**
+ * Catalog reads under `lock_timeout` (#3678). One session holds ACCESS
+ * EXCLUSIVE on a table behind a view; the observation of the view, which
+ * reads its definition with `pg_get_viewdef()`, fails after the profile's
+ * `lockTimeoutMs` instead of waiting for the lock, and names the table and the
+ * pid holding it.
+ *
+ * `CHANT_SQL_E2E_POSTGRES_URL` (with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names
+ * a running server, else a throwaway one, skipped when Docker is not
+ * available. The test makes only the schema `chant_e2e_3678`, dropped
+ * afterwards.
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { normalizeObservation } from "@intentius/chant/observation";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "../testing/server";
+import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "./client";
+import { describeResources } from "./describe-resources";
+
+const SCHEMA = "chant_e2e_3678";
+const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
+const enabled = given ? true : await dockerAvailable();
+let server: TestPostgres | undefined;
+let endpoint: PostgresEndpoint;
+let admin: PostgresClient;
+let holder: PostgresClient;
+
+describe.skipIf(!enabled)("a catalog read blocked behind ACCESS EXCLUSIVE", () => {
+  beforeAll(async () => {
+    if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
+    else {
+      server = await startTestPostgres();
+      endpoint = server.endpoint();
+    }
+    admin = await connectPostgres(endpoint);
+    await admin.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);
+    await admin.query(`CREATE SCHEMA ${SCHEMA}`);
+    await admin.query(`CREATE TABLE ${SCHEMA}.orders (id bigint PRIMARY KEY)`);
+    await admin.query(`CREATE VIEW ${SCHEMA}.recent AS SELECT id FROM ${SCHEMA}.orders`);
+    holder = await connectPostgres(endpoint, { applicationName: "chant-e2e-3678-holder" });
+    await holder.query("BEGIN");
+    await holder.query(`LOCK TABLE ${SCHEMA}.orders IN ACCESS EXCLUSIVE MODE`);
+  }, 600_000);
+
+  afterAll(async () => {
+    await holder?.query("ROLLBACK").catch(() => undefined);
+    await holder?.end();
+    await admin?.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`).catch(() => undefined);
+    await admin?.end();
+    await server?.stop();
+  });
+
+  test("fails after lockTimeoutMs, naming the relation and the pid", async () => {
+    const [{ pid }] = await holder.query<{ pid: number }>("SELECT pg_catalog.pg_backend_pid() AS pid");
+    const started = Date.now();
+    const r = normalizeObservation(await describeResources({
+      environment: "e2e",
+      config: { sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_3678_PASSWORD" }, schemas: [SCHEMA], lockTimeoutMs: 300 } } } },
+      env: { PG_E2E_3678_PASSWORD: endpoint.password ?? "" },
+      entityNames: ["recent"],
+      entities: new Map([["recent", { entityType: "Postgres::View", props: { schema: SCHEMA, name: "recent" } }]]),
+    }));
+    expect(Date.now() - started).toBeLessThan(10_000);
+    const recent = r.unobserved.recent;
+    expect(recent?.reason).toBe("read-failed");
+    expect(recent?.detail).toContain("lock timeout");
+    expect(recent?.detail).toContain(`${SCHEMA}.orders is held in ACCESS EXCLUSIVE by pid ${pid}`);
+    expect(recent?.detail).toContain("chant-e2e-3678-holder");
+  }, 60_000);
+});

package/src/postgres/plan/commands.ts

@@ -20,8 +20,7 @@ import { POSTGRES_ENTITY_TYPES } from "../entity-types";
 import { migrationOpSuggestions } from "../migrate/handoff";
 import { POSTGRES_LATEST_MAJOR } from "../../spec/postgres-pin";
 import { ACCESS_MANAGED_KINDS } from "./normalize";
-import { declaredAccess, diffAccess, predictedAccess, targetKey } from "../access/acl";
-import { readLiveAccess } from "../access/live";
+import { accessChangesAgainst } from "../access/changes";
 
 export function emitPg(diff: PgSchemaDiff, json: boolean, title: string): number {
   console.log(json ? JSON.stringify(diff, null, 2) : renderPgDiff(diff, { title }));
@@ -112,12 +111,7 @@ export async function planAgainstClient(
   const major = running ?? options.major;
   const diff = diffPgSchemas(live, declared, { major, access: false });
   if (managed) {
-    const want = declaredAccess(declared, { major: major ?? POSTGRES_LATEST_MAJOR, ...(await currentRole(client)) });
-    const have = await readLiveAccess(client, want.targets, { ...(scope ? { schemas: scope } : {}), forRoles: want.forRoles, declaredDefaults: want.targets.filter((t) => t.kind === "default") });
-    // An object this plan creates is given what the server gives a new one.
-    const missing = want.targets.filter((t) => t.kind !== "default" && t.kind !== "column" && !have.present.has(targetKey(t)));
-    const now = new Map([...have.state, ...predictedAccess(missing, have)]);
-    const access = diffAccess(now, want.state, want.exportsOf);
+    const access = await accessChangesAgainst(client, declared, { major: major ?? POSTGRES_LATEST_MAJOR, ...(scope ? { scope } : {}) });
     if (access.length > 0) {
       diff.changes.push(...access.map((a) => a.change));
       diff.access = access;
@@ -131,12 +125,6 @@ export async function planAgainstClient(
   return { declared, live, liveObjects, diff, ...(major !== undefined ? { major } : {}), ...(scope ? { scope } : {}) };
 }
 
-/** The role a session runs as: the one whose own privileges owning an object gives, and whose default privileges apply. */
-async function currentRole(client: PostgresClient): Promise<{ self?: string }> {
-  const rows = await client.query<{ self: string }>("SELECT current_user AS self").catch(() => []);
-  return rows[0]?.self ? { self: rows[0].self } : {};
-}
-
 /**
  * The declarations a plan compares: all of them where the profile manages
  * access; otherwise none of the policies, roles, grants and default

package/src/postgres/plan/deep.ts

@@ -13,7 +13,7 @@
  * of the object (the DDL text, the template source, lineage, reads).
  */
 
-import { deepObservation, type DeepObservationResult, type DeepResourceObservation } from "@intentius/chant/deep-observation";
+import { deepObservation, type DeepObservationResult, type DeepPendingChange, type DeepResourceObservation } from "@intentius/chant/deep-observation";
 import type { UnobservedEntity } from "@intentius/chant/lexicon";
 import { bindPostgres, classifyPostgresFailure, type BindOptions } from "../live/bind";
 import { liveKey, readLiveSchema, type LivePgObject } from "../live/catalog";
@@ -23,6 +23,8 @@ import { POSTGRES_ENTITY_TYPES } from "../entity-types";
 import { canonicalPgObject, columnShape, sameConstraint, type CanonicalPgObject } from "./normalize";
 import { serverNormalized } from "./server-normalize";
 import type { PostgresClient } from "../live/client";
+import { accessChangesAgainst } from "../access/changes";
+import { POSTGRES_LATEST_MAJOR } from "../../spec/postgres-pin";
 
 type Props = Record<string, unknown>;
 
@@ -214,7 +216,21 @@ export async function observeResourcesDeep(
     for (const { name, entity } of declared) unobserved[name] = { type: entity?.entityType ?? "", reason: why.reason, detail: why.detail };
     return deepObservation({}, unobserved);
   }
+  const pending: DeepPendingChange[] = [];
   try {
+    if (target.access === true) {
+      // #3706: the privilege changes an apply would make, which no one
+      // declaration's properties show, so the plan digest moves with them.
+      try {
+        pending.push(...(await pendingAccess(client, target.defaultSchema, declared, scopeFor(target, declared.filter((x) => x.entity?.entityType.startsWith("Postgres::")).map((x) => ({ type: x.entity!.entityType, props: x.entity!.props }))))));
+      } catch (err) {
+        for (const { name, entity } of declared) {
+          if (entity?.entityType === POSTGRES_ENTITY_TYPES.grant || entity?.entityType === POSTGRES_ENTITY_TYPES.defaultPrivileges) {
+            unobserved[name] = { type: entity.entityType, reason: "read-failed", detail: `the server's privileges could not be compared: ${(err as Error).message}` };
+          }
+        }
+      }
+    }
     const byKey = new Map(live.map((o) => [liveKey(o.type, o.schema, o.name, o.signature), o]));
     for (const { name, entity } of declared) {
       if (!entity || !entity.entityType.startsWith("Postgres::")) {
@@ -223,6 +239,7 @@ export async function observeResourcesDeep(
       }
       const why = accessUnobserved(entity.entityType, target.access === true);
       if (why) {
+        if (unobserved[name]) continue;
         unobserved[name] = { type: entity.entityType, ...why };
         continue;
       }
@@ -245,5 +262,30 @@ export async function observeResourcesDeep(
   } finally {
     await client.end();
   }
-  return deepObservation(resources, unobserved);
+  return deepObservation(resources, unobserved, pending);
+}
+
+/**
+ * The privilege changes an apply would make (#3706), as `chant sql plan`
+ * computes them, one per object and grantee: the statements, and the
+ * declarations behind them.
+ */
+async function pendingAccess(
+  client: PostgresClient,
+  defaultSchema: string,
+  declared: ReadonlyArray<{ name: string; entity?: { entityType: string; props: Record<string, unknown> } }>,
+  scope: string[] | undefined,
+): Promise<DeepPendingChange[]> {
+  const objects = declared
+    .filter((x) => x.entity?.entityType.startsWith("Postgres::"))
+    .map((x) => ({ key: x.name, canonical: canonicalPgObject(x.entity!.entityType, x.entity!.props, defaultSchema) }));
+  const [row] = await client.query<{ v: string }>("SELECT current_setting('server_version_num') AS v");
+  const n = Number(row?.v);
+  const major = Number.isFinite(n) && n > 0 ? Math.floor(n / 10000) : POSTGRES_LATEST_MAJOR;
+  const changes = await accessChangesAgainst(client, objects, { major, ...(scope ? { scope } : {}) });
+  return changes.map((c) => ({
+    subject: c.change.object.startsWith("acl ") ? c.change.object.slice(4) : c.change.object,
+    change: c.sql.join("; "),
+    ...(c.exports.length > 0 ? { entities: [...c.exports].sort() } : {}),
+  }));
 }

package/src/serializer.test.ts

@@ -50,6 +50,8 @@ describe("sql serializer", () => {
     expect(sql.indexOf("CREATE DATABASE")).toBeLessThan(sql.indexOf("CREATE TABLE"));
     expect(sql.indexOf("CREATE TABLE")).toBeLessThan(sql.indexOf("CREATE VIEW"));
     expect(r.verbatimFiles).toEqual([CLICKHOUSE_DDL_FILE]);
+    // DDL is copied exactly but is not a secret, so it prints with no --output (#3738).
+    expect(r.secretFiles).toBeUndefined();
   });
 
   test("10. a declarable of another lexicon is not written", () => {
@@ -98,6 +100,8 @@ describe("sql serializer, Postgres", () => {
     expect(doc(r).objects[1]).toMatchObject({ export: "users", type: "Postgres::Table", name: "users", schema: "app", sqlName: "app.users" });
     expect(r.files![POSTGRES_DDL_FILE]).toBe("CREATE SCHEMA app;\n\nCREATE TABLE app.users (id bigint PRIMARY KEY, email text NOT NULL);\n");
     expect(r.verbatimFiles).toEqual([POSTGRES_DDL_FILE]);
+    // DDL is copied exactly but is not a secret, so it prints with no --output (#3738).
+    expect(r.secretFiles).toBeUndefined();
   });
 
   test("a Postgres build records the target major: sql.postgresMajor, else the newest pinned", () => {

package/src/skills/chant-sql-postgres-plan.md

@@ -167,7 +167,7 @@ Transactions follow the class, so a long scan holds locks on one table and what
 
 A primary key or unique constraint added to an existing table goes in two steps, the unique index built `CONCURRENTLY` and then `ADD CONSTRAINT ... USING INDEX` (SQLPG221), so a busy table is never blocked for the build.
 
-Timeouts are set on every statement: `lock_timeout` 5000 ms, so a statement that cannot get its lock fails instead of queueing behind a long transaction and blocking everything behind it; `statement_timeout` 60000 ms for catalog-only statements, and 0 (no limit) for scans, rewrites and `CONCURRENTLY` builds. Set them per environment with `lockTimeoutMs`, `statementTimeoutMs` and `scanTimeoutMs` on `sql.profiles.<env>`, or per call.
+Timeouts are set on every statement: `lock_timeout` 5000 ms, so a statement that cannot get its lock fails instead of queueing behind a long transaction and blocking everything behind it; `statement_timeout` 60000 ms for catalog-only statements, and 0 (no limit) for scans, rewrites and `CONCURRENTLY` builds. Set them per environment with `lockTimeoutMs`, `statementTimeoutMs` and `scanTimeoutMs` on `sql.profiles.<env>`, or per call. The catalog read an apply makes before its first statement waits under the same `lock_timeout`: blocked behind `ACCESS EXCLUSIVE`, the apply stops with nothing written, naming the table and the pid that holds it.
 
 ```ts
 profiles: { prod: { url: "postgres://db.internal:5432/shop", lockTimeoutMs: 2000, scanTimeoutMs: 600000 } }

package/src/skills/chant-sql-rebuild.md

@@ -57,6 +57,8 @@ The swap gate is bound to a digest of the plan and the verification (row counts
 - A run stopped during the backfill (Ctrl-C, a killed job) is not a failure. The next run resumes from the receipts in `chant_receipts.receipts` on the same server (in `<db>.__chant_receipts` on a Replicated database), and a partition whose copy was cut off before its receipt is cleared and copied again, so no row is copied twice.
 - A failed step (a failed copy after its retries, a verification mismatch, a refused declaration) runs onFailure: the new table `<t>__chant_new` and the dual-write view `<t>__chant_dual` are dropped, and the next run starts from a new table. With `onFailure: "keep"` they stay, and the next run resumes; drop them by hand to start over. The original table is untouched until the swap.
 - A verification mismatch in materialized-view mode usually means rows arrived with a time before the cut-over later than `cutoverDelay`. Raise the delay and run again.
+- On a Summing, Replacing, Aggregating or Collapsing MergeTree the verification compares both tables under `FINAL`, so unmerged parts in the old table are not a mismatch. A mismatch that remains names the engine: run `OPTIMIZE TABLE <db>.<table> FINAL` and run the rebuild again.
+- A plain MergeTree rebuilt into one of those engines is compared per key of the new sorting key, on what the new engine keeps (a Summing table's sums, a Replacing table's highest version, a Collapsing table's sum of signs), so the different row counts are not a mismatch. A missing key or a changed sum still fails.
 - A backfill that stops naming writes "begun before the cut-over" waited `cutoverTimeout` for INSERTs into the old table that were still running. Let them finish, or stop them, and run again.
 - The Op refuses a database that is neither Atomic nor Replicated (`EXCHANGE TABLES` needs one of them), a table whose change is not a rebuild (use the applier), and an object already under one of its working names that is not its own. In a Replicated database it also refuses a table, or a declaration, whose engine is not `Replicated*MergeTree`: a plain MergeTree keeps different rows on each replica and one replica's copy would lose the others'.
 

package/src/skills/chant-sql.md

@@ -37,7 +37,7 @@ A dictionary is a `dictionary` template holding `CREATE DICTIONARY`, with its at
 
 A SQL user-defined function is a `func` template holding `CREATE FUNCTION name AS (x) -> expr`, with no database in its name. A changed expression is SQLCH260 (`CREATE OR REPLACE FUNCTION`). It carries no ownership marker, so chant never drops one.
 
-Users, roles, row policies and grants are `user`, `role`, `policy` and `grant` templates. Never write a password: `IDENTIFIED BY` is refused, a user declared without `IDENTIFIED` is created by the environment with its password, and chant manages the rest of it. Grants are compared per grantee: every `grant` naming a grantee adds up to its complete list, and anything else it holds is revoked (SQLCH274). None of these is ever dropped by chant.
+Users, roles, row policies and grants are `user`, `role`, `policy` and `grant` templates. Never write a password: `IDENTIFIED BY` is refused, a user declared without `IDENTIFIED` is created by the environment with its password, and chant manages the rest of it. Grants are compared per grantee: every `grant` naming a grantee adds up to its complete list, and anything else it holds is revoked (SQLCH274). None of these is ever dropped by chant. They are planned and applied only where the profile sets `access: true` (`sql.profiles.<env>.access`); elsewhere each is reported `filtered`.
 
 ## Interpolations
 

package/src/sql-files.test.ts

@@ -3,7 +3,8 @@ import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { build } from "@intentius/chant/build";
-import { readSqlFile, sqlFileDeclarations, SqlFileError } from "./sql-files";
+import { readSqlFile, sqlFileDeclarations, sqlFileEntities, SqlFileError } from "./sql-files";
+import { database, grant, policy, role, table, user } from "./clickhouse/entities";
 import { sqlPlugin } from "./plugin";
 import { sqlSerializer } from "./serializer";
 import { guessDialect, SqlFileParser } from "./import-parser";
@@ -63,6 +64,7 @@ describe("chant import of a .sql file", () => {
     expect(guessDialect("CREATE TABLE a.t (x UInt8) ENGINE = Log")).toBe("clickhouse");
     expect(guessDialect("CREATE VIEW a.v AS SELECT 1")).toBe("clickhouse");
     expect(guessDialect("CREATE FUNCTION f AS (x) -> x + 1")).toBe("clickhouse");
+    expect(guessDialect("CREATE ROW POLICY p ON a.t USING 1 TO r")).toBe("clickhouse");
     expect(guessDialect("CREATE FUNCTION app.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$")).toBe("postgres");
     expect(new SqlFileParser().parse(PG_DDL).resources.map((r) => r.type)).toEqual(["Postgres::Enum", "Postgres::Table", "Postgres::Table", "Postgres::Index"]);
   });
@@ -102,8 +104,83 @@ CREATE TABLE analytics.counts (kind String, n UInt64) ENGINE = SummingMergeTree
 CREATE MATERIALIZED VIEW analytics.counts_mv TO analytics.counts AS SELECT kind, count() AS n FROM analytics.events GROUP BY kind;
 CREATE DICTIONARY analytics.kinds (kind String, label String DEFAULT '') PRIMARY KEY kind SOURCE(clickhouse(table 'counts' db 'analytics')) LAYOUT(complex_key_hashed()) LIFETIME(300);
 CREATE FUNCTION kind_label AS (k) -> upper(k);
+CREATE ROLE reader SETTINGS max_memory_usage = 1000000000;
+CREATE USER app HOST ANY DEFAULT ROLE reader DEFAULT DATABASE analytics;
+CREATE ROW POLICY own_kinds ON analytics.events USING kind = 'a' TO reader;
+GRANT SELECT(ts, kind) ON analytics.events TO reader;
+GRANT reader TO app;
 `;
 
+describe("a file of ClickHouse access control (#3711)", () => {
+  const ACCESS = `CREATE DATABASE shop;
+CREATE TABLE shop.events (id UInt64, tenant String) ENGINE = MergeTree ORDER BY id;
+CREATE ROLE reader SETTINGS max_memory_usage = 1000000000;
+CREATE USER app HOST ANY DEFAULT ROLE reader DEFAULT DATABASE shop;
+CREATE ROW POLICY tenant_a ON shop.events USING tenant = 'a' TO reader;
+GRANT SELECT(id, tenant) ON shop.events TO reader;
+GRANT reader TO app;
+`;
+  const t = (s: string) => Object.assign([s], { raw: [s] }) as unknown as TemplateStringsArray;
+  const shopDb = database(t("CREATE DATABASE shop"));
+  const events = table`CREATE TABLE ${shopDb}.events (id UInt64, tenant String) ENGINE = MergeTree ORDER BY id`;
+  const reader = role(t("CREATE ROLE reader SETTINGS max_memory_usage = 1000000000"));
+  const app = user(t("CREATE USER app HOST ANY DEFAULT ROLE reader DEFAULT DATABASE shop"));
+  const tagged = [
+    shopDb,
+    events,
+    reader,
+    app,
+    policy`CREATE ROW POLICY tenant_a ON ${events} USING tenant = 'a' TO ${reader}`,
+    grant`GRANT SELECT(id, tenant) ON ${events} TO ${reader}`,
+    grant`GRANT reader TO ${app}`,
+  ];
+  const props = (e: unknown) => {
+    const { source: _source, ...rest } = (e as { props: Record<string, unknown> }).props;
+    return JSON.parse(JSON.stringify(rest)) as Record<string, unknown>;
+  };
+
+  test("users, roles, row policies and grants build to the declarations the tags make", () => {
+    const read = sqlFileEntities("clickhouse", [{ origin: "access.sql", ddl: ACCESS }]);
+    expect([...read.values()].map((e) => e.entityType)).toEqual(tagged.map((e) => e.entityType));
+    expect([...read.values()].map(props)).toEqual(tagged.map(props));
+    expect(readSqlFile("clickhouse", ACCESS, { origin: "access.sql" }).map((o) => [o.type, o.name])).toEqual([
+      ["ClickHouse::Database", "shop"],
+      ["ClickHouse::Table", "shop.events"],
+      ["ClickHouse::Role", "reader"],
+      ["ClickHouse::User", "app"],
+      ["ClickHouse::RowPolicy", "shop.tenant_a"],
+      ["ClickHouse::Grant", "grant select(id, tenant) on shop.events to reader"],
+      ["ClickHouse::Grant", "grant reader to app"],
+    ]);
+  });
+
+  test("with a database for unqualified names, a policy's and a grant's table are qualified", () => {
+    const objects = readSqlFile("clickhouse", "CREATE ROW POLICY p ON events USING 1 TO reader;\nGRANT SELECT ON events TO reader;\nGRANT SELECT ON *.* TO reader", { origin: "a.sql", schema: "shop" });
+    expect(objects.map((o) => o.ddl)).toEqual(["CREATE ROW POLICY p ON shop.events USING 1 TO reader", "GRANT SELECT ON shop.events TO reader", "GRANT SELECT ON *.* TO reader"]);
+  });
+
+  test("a password is refused with the tag's message, and so is a REVOKE", () => {
+    let tagMessage = "";
+    try {
+      user(t("CREATE USER app IDENTIFIED BY 'secret'"));
+    } catch (e) {
+      tagMessage = (e as Error).message;
+    }
+    expect(tagMessage).toMatch(/IDENTIFIED BY holds a password/);
+    const err = (() => {
+      try {
+        readSqlFile("clickhouse", "CREATE USER app IDENTIFIED BY 'secret';\nREVOKE SELECT ON shop.events FROM reader", { origin: "users.sql" });
+      } catch (e) {
+        return e as SqlFileError;
+      }
+      throw new Error("read");
+    })();
+    expect(err).toBeInstanceOf(SqlFileError);
+    expect(err.problems).toEqual([`${tagMessage}: CREATE USER app IDENTIFIED BY 'secret'`, expect.stringMatching(/holds a REVOKE; declare the grants a grantee keeps.*: REVOKE SELECT ON shop\.events FROM reader$/)]);
+    expect(err.message).not.toContain("not a CREATE statement");
+  });
+});
+
 describe("chant build reads the .sql files in the source directory", () => {
   const dirs: string[] = [];
   afterAll(async () => {
@@ -174,6 +251,38 @@ export const activeUsers = view\`CREATE VIEW app.active_users AS SELECT id FROM
   });
 });
 
+describe("SQL that is not schema, in a build of the project root (#3713)", () => {
+  const dirs: string[] = [];
+  afterAll(async () => {
+    for (const d of dirs) await rm(d, { recursive: true, force: true });
+  });
+
+  test("a .sql file under a migrations directory, at any depth, is not read", async () => {
+    const dir = await project("migrations", {
+      "app.ts": `import { schema, table } from "@intentius/chant-lexicon-sql/postgres";\nexport const app = schema\`CREATE SCHEMA app\`;\nexport const users = table\`CREATE TABLE app.users (id bigint)\`;\n`,
+    });
+    dirs.push(dir);
+    await mkdir(join(dir, "src", "migrations", "20261010T0000-add-email"), { recursive: true });
+    await writeFile(join(dir, "src", "migrations", "20261010T0000-add-email", "migration.sql"), "ALTER TABLE app.users ADD COLUMN email text;\n");
+    await mkdir(join(dir, "src", "prisma", "migrations", "0001_init"), { recursive: true });
+    await writeFile(join(dir, "src", "prisma", "migrations", "0001_init", "migration.sql"), "CREATE TABLE app.users (id bigint);\n");
+    const { errors, doc } = await buildProject(dir);
+    expect(errors).toEqual([]);
+    expect(doc!.applyOrder).toEqual(["app", "users"]);
+  });
+
+  test("a file named migrations.sql, or in a directory that only starts with the word, is still read", async () => {
+    const dir = await project("not-migrations", { "app.ts": `import { schema } from "@intentius/chant-lexicon-sql/postgres";\nexport const app = schema\`CREATE SCHEMA app\`;\n` });
+    dirs.push(dir);
+    await writeFile(join(dir, "src", "migrations.sql"), "CREATE TABLE app.a (id bigint);\n");
+    await mkdir(join(dir, "src", "migrations-old"), { recursive: true });
+    await writeFile(join(dir, "src", "migrations-old", "b.sql"), "CREATE TABLE app.b (id bigint);\n");
+    const { errors, doc } = await buildProject(dir);
+    expect(errors).toEqual([]);
+    expect([...doc!.applyOrder].sort()).toEqual(["a", "app", "b"]);
+  });
+});
+
 describe("references between a file's objects", () => {
   test("a bare name where SQL names an object is a reference, and a cycle is an error", async () => {
     const { sqlFileEntities } = await import("./sql-files");

package/src/sql-files.ts

@@ -15,7 +15,8 @@
  * PRIVILEGES` as an object, with `COMMENT ON`, `ALTER TABLE ... ROW LEVEL
  * SECURITY` and `ALTER TABLE ... ADD <table constraint>` folded into the
  * object they finish; ClickHouse reads `CREATE DATABASE`, `TABLE`, `VIEW`,
- * `MATERIALIZED VIEW`, `DICTIONARY` and `FUNCTION`. Any other statement, and any
+ * `MATERIALIZED VIEW`, `DICTIONARY`, `FUNCTION`, `USER`, `ROLE` and `ROW
+ * POLICY`, and `GRANT` (#3711). Any other statement, and any
  * statement that does not parse, throws a {@link SqlFileError} naming every
  * such statement: none is left out.
  *