@intentius/chant-lexicon-sql 0.118.1 → 0.121.0

package/src/postgres/access/changes.ts

@@ -0,0 +1,39 @@
+/**
+ * The privilege changes a connected server needs (#3681): what `chant sql
+ * plan` reports, and what the deep read reports as pending so the plan digest
+ * an approval binds to moves with a privilege granted or revoked by hand
+ * (#3706).
+ */
+
+import type { PostgresClient } from "../live/client";
+import type { CanonicalPgObject } from "../plan/normalize";
+import { declaredAccess, diffAccess, predictedAccess, targetKey, type AccessChange } from "./acl";
+import { readLiveAccess } from "./live";
+
+/** The role a session runs as: the one whose own privileges owning an object gives, and whose default privileges apply. */
+async function currentRole(client: PostgresClient): Promise<{ self?: string }> {
+  const rows = await client.query<{ self: string }>("SELECT current_user AS self").catch(() => []);
+  return rows[0]?.self ? { self: rows[0].self } : {};
+}
+
+/**
+ * The changes from the access the server holds to the access the
+ * declarations (keyed by export name, in the build's order) add up to. An
+ * object the server does not hold yet is taken to have what the server gives
+ * a new one. `scope` limits the default privileges read to those schemas.
+ */
+export async function accessChangesAgainst(
+  client: PostgresClient,
+  declared: ReadonlyArray<{ key: string; canonical: CanonicalPgObject }>,
+  options: { major: number; scope?: readonly string[] },
+): Promise<AccessChange[]> {
+  const want = declaredAccess(declared, { major: options.major, ...(await currentRole(client)) });
+  const have = await readLiveAccess(client, want.targets, {
+    ...(options.scope ? { schemas: options.scope } : {}),
+    forRoles: want.forRoles,
+    declaredDefaults: want.targets.filter((t) => t.kind === "default"),
+  });
+  const missing = want.targets.filter((t) => t.kind !== "default" && t.kind !== "column" && !have.present.has(targetKey(t)));
+  const now = new Map([...have.state, ...predictedAccess(missing, have)]);
+  return diffAccess(now, want.state, want.exportsOf);
+}

package/src/postgres/apply/apply.ts

@@ -70,7 +70,7 @@ import type { OwnershipMarker } from "@intentius/chant/ownership";
 import type { ApplyRef } from "@intentius/chant/apply";
 import { SqlApplyError, dependencyFailedDetail, missingDependencies, readBuildObjects, type SqlApplyOutcome } from "../../core/apply";
 import { carriesMarker, isChantManaged } from "../../core/ownership";
-import { PostgresQueryError, type PostgresClient } from "../live/client";
+import { DEFAULT_LOCK_TIMEOUT_MS, PostgresQueryError, withCatalogLockTimeout, type PostgresClient } from "../live/client";
 import type { PostgresTarget } from "../live/bind";
 import type { LivePgObject, readLiveSchema } from "../live/catalog";
 import { liveProps } from "../plan/deep";
@@ -98,7 +98,7 @@ export interface PostgresApplyTimeouts {
   scanTimeoutMs: number;
 }
 
-export const DEFAULT_POSTGRES_APPLY_TIMEOUTS: PostgresApplyTimeouts = { lockTimeoutMs: 5_000, statementTimeoutMs: 60_000, scanTimeoutMs: 0 };
+export const DEFAULT_POSTGRES_APPLY_TIMEOUTS: PostgresApplyTimeouts = { lockTimeoutMs: DEFAULT_LOCK_TIMEOUT_MS, statementTimeoutMs: 60_000, scanTimeoutMs: 0 };
 
 /** One statement as it ran: its object, the timeouts it ran with, and the transaction it was in. */
 export interface RanStatement {
@@ -254,8 +254,10 @@ export async function applyPostgres(
   };
 
   // ── Plan: the same diff `chant sql plan` shows. ──
+  // The catalog reads wait under lockTimeoutMs, as a plan's do (#3726): one blocked behind ACCESS EXCLUSIVE fails naming the holder instead of hanging.
+  const reader = await withCatalogLockTimeout(client, timeouts.lockTimeoutMs);
   const build: PgSchemaObject[] = declared.map((o) => ({ key: o.exportName, canonical: o.canonical }));
-  const plan = await planAgainstClient(client, target, build, {
+  const plan = await planAgainstClient(reader, target, build, {
     ...(opts.major !== undefined ? { major: opts.major } : {}),
     ...(opts.readLive ? { readLive: opts.readLive } : {}),
     ...(opts.serverNormalize ? { serverNormalize: opts.serverNormalize } : {}),
@@ -281,7 +283,7 @@ export async function applyPostgres(
   for (const m of matchPgObjects(plan.live, plan.declared)) if (m.kind === "matched") matchedLive.add(m.after.key);
 
   // An extension's own comment stays under the trailer when the declaration sets none.
-  const extensionComments = await extensionDefaults(client, objects.filter((o) => o.canonical.kind === "extension" && !matchedLive.has(o.key)).map((o) => o.canonical.name));
+  const extensionComments = await extensionDefaults(reader, objects.filter((o) => o.canonical.kind === "extension" && !matchedLive.has(o.key)).map((o) => o.canonical.name));
 
   // ── Each object's statements, or its verdict when it has none to send. ──
   const statements = planPgStatements({
@@ -454,7 +456,7 @@ export async function applyPostgres(
   }
   await commit();
 
-  if (managed) await applyAccess(client, declared, onServer, exportNames, { ...opts, major: plan.major ?? opts.major ?? POSTGRES_LATEST_MAJOR, ...(plan.scope ? { schemas: plan.scope } : {}) }, timeouts, outcome, log);
+  if (managed) await applyAccess(client, reader, declared, onServer, exportNames, { ...opts, major: plan.major ?? opts.major ?? POSTGRES_LATEST_MAJOR, ...(plan.scope ? { schemas: plan.scope } : {}) }, timeouts, outcome, log);
 
   if (opts.prune) await prune(client, statements.drops, liveRawByKey, opts, timeouts, outcome, log);
   if (outcome.failed.length > 0) throw new PostgresApplyError(outcome);
@@ -470,6 +472,7 @@ export async function applyPostgres(
  */
 async function applyAccess(
   client: PostgresClient,
+  reader: PostgresClient,
   declared: readonly DeclaredPgObject[],
   onServer: ReadonlySet<string>,
   exportNames: ReadonlySet<string>,
@@ -487,12 +490,12 @@ async function applyAccess(
     else ready.push(o);
   }
   const skipped = new Set(decls.filter((o) => !ready.includes(o)).map((o) => o.exportName));
-  const [me] = await client.query<{ self: string }>("SELECT current_user AS self");
+  const [me] = await reader.query<{ self: string }>("SELECT current_user AS self");
   const want = declaredAccess(
     declared.filter((o) => !skipped.has(o.exportName)).map((o) => ({ key: o.exportName, canonical: o.canonical })),
     { major: opts.major, ...(me?.self ? { self: me.self } : {}) },
   );
-  const have = await readLiveAccess(client, want.targets, {
+  const have = await readLiveAccess(reader, want.targets, {
     ...(opts.schemas ? { schemas: opts.schemas } : {}),
     forRoles: want.forRoles,
     declaredDefaults: want.targets.filter((t) => t.kind === "default"),

package/src/postgres/entities.ts

@@ -1048,7 +1048,7 @@ function grantProps(ctx: Ctx, node: GrantNode, ddl: string, source: { strings: s
       if (o.args) sqlName += `(${req(text(ctx, o.args, "objects")) ?? ""})`;
       else if (typeof t !== "string" && (t.entityType === POSTGRES_ENTITY_TYPES.function || t.entityType === POSTGRES_ENTITY_TYPES.procedure)) {
         // An interpolated routine is named by its own parameter types.
-        sqlName += `(${(t.props as RoutineProps).args.filter((a) => a.mode !== "out").map((a) => a.type).join(", ")})`;
+        sqlName += `(${routineArgTypes(t.props as RoutineProps)})`;
       } else throw new SqlTemplateError("grant", `${sqlName}: name the parameter types (${sqlName}(...)), or interpolate the routine, since overloads share a name`, 0, 0);
     }
     objectNames.push(sqlName);
@@ -1071,6 +1071,22 @@ function grantProps(ctx: Ctx, node: GrantNode, ddl: string, source: { strings: s
   }) as GrantProps;
 }
 
+/** A routine's parameter types, as `GRANT ... ON FUNCTION f(...)` names it: the `OUT` ones left out. */
+const routineArgTypes = (props: RoutineProps): string => props.args.filter((a) => a.mode !== "out").map((a) => a.type).join(", ");
+
+/** The interpolated functions and procedures not followed by their own `(...)`: the ones a grant's statement writes with the parameter types (#3707). */
+function bareRoutineSplices(tokens: readonly Token[], values: readonly unknown[]): Set<number> {
+  const out = new Set<number>();
+  tokens.forEach((t, k) => {
+    if (t.kind !== "ref") return;
+    const v = values[t.part];
+    if (!isPostgresObject(v) || (v.entityType !== POSTGRES_ENTITY_TYPES.function && v.entityType !== POSTGRES_ENTITY_TYPES.procedure)) return;
+    const next = tokens.slice(k + 1).find((x) => !isTrivia(x));
+    if (!(next?.kind === "punct" && next.text === "(")) out.add(t.part);
+  });
+  return out;
+}
+
 function build(tag: PostgresTag, strings: TemplateStringsArray | readonly string[], rawValues: readonly unknown[]): PostgresObject {
   const rawParts = templateParts(strings);
   const routine = tag === "func" || tag === "procedure";
@@ -1113,9 +1129,12 @@ function build(tag: PostgresTag, strings: TemplateStringsArray | readonly string
   const rowSecurity = rest.filter((n): n is RowSecurityNode => n.statement === "rowSecurity");
 
   const positions = regclassPositions(tokens, values);
+  const bareRoutines = tag === "grant" ? bareRoutineSplices(tokens, values) : new Set<number>();
   const render = (v: unknown, i?: number): string => {
     const at = i === undefined ? undefined : positions.get(i);
     if (at && isPostgresObject(v)) return at === "call" ? `${regclassText(v.sqlName)}::regclass` : regclassText(v.sqlName);
+    // #3707: a routine a grant interpolates is written with its parameter types, so the statement names it on its own.
+    if (i !== undefined && bareRoutines.has(i) && isPostgresObject(v)) return `${renderReference(v)}(${routineArgTypes(v.props as RoutineProps)})`;
     return renderReference(v);
   };
   const ctx: Ctx = { d: { ...POSTGRES_TEMPLATES, renderReference: render }, tokens, values, fed: values.map(() => new Set<string>()) };

package/src/postgres/live/bind.ts

@@ -6,7 +6,7 @@
 
 import type { ChantConfig } from "@intentius/chant/config";
 import type { UnobservedReason } from "@intentius/chant/observation";
-import { connectPostgres, PostgresQueryError, type PostgresClient, type PostgresEndpoint } from "./client";
+import { connectPostgres, DEFAULT_LOCK_TIMEOUT_MS, PostgresQueryError, withCatalogLockTimeout, type PostgresClient, type PostgresEndpoint } from "./client";
 import type { PostgresProvider } from "../providers/types";
 import { credentialSource, TokenSourceError } from "../../token-source";
 
@@ -139,11 +139,23 @@ export async function resolveBoundTarget(options: BindOptions = {}): Promise<Pos
   return target;
 }
 
-/** Resolve the environment's server and connect to it. The caller ends the client. */
+/**
+ * Resolve the environment's server and connect to it. The caller ends the client.
+ *
+ * The session reads under the profile's `lockTimeoutMs` (the applier's 5000 ms
+ * otherwise): a catalog read blocked behind ACCESS EXCLUSIVE fails with the
+ * relation and the pid that holds it, instead of waiting for as long as the
+ * lock is held.
+ */
 export async function bindPostgres(options: BindOptions = {}): Promise<{ target: PostgresTarget; client: PostgresClient }> {
   const target = await resolveBoundTarget(options);
-  const client = await (options.connect ?? connectPostgres)(target.endpoint);
-  return { target, client };
+  const raw = await (options.connect ?? connectPostgres)(target.endpoint);
+  try {
+    return { target, client: await withCatalogLockTimeout(raw, target.timeouts?.lockTimeoutMs ?? DEFAULT_LOCK_TIMEOUT_MS) };
+  } catch (err) {
+    await raw.end();
+    throw err;
+  }
 }
 
 /** SQLSTATEs that mean the server refused who we are: invalid_authorization_specification, invalid_password, insufficient_privilege. */

package/src/postgres/live/catalog.ts

@@ -44,26 +44,14 @@ import { quoteIdent } from "../keywords";
 import { hasChantTrailerKey, RECEIPTS_TRAILER_KEY, stripMarker } from "../../core/ownership";
 import { MIGRATION_TRAILER_KEY } from "../migrate/names";
 import { isProviderOwned, providerData } from "../providers";
+import { FOREIGN_TABLES } from "../../core/foreign-tables";
 import type { PostgresProvider } from "../providers/types";
 
 /** The server's own schemas, never part of a declared schema. */
 export const SYSTEM_SCHEMAS = ["pg_catalog", "information_schema", "pg_toast"];
 
-/** Tables another tool keeps its migration history in, by name, with the tool. */
-export const FOREIGN_TABLES: Readonly<Record<string, string>> = {
-  _prisma_migrations: "Prisma Migrate",
-  schema_migrations: "a migration runner (Rails, golang-migrate, dbmate)",
-  ar_internal_metadata: "Rails",
-  django_migrations: "Django",
-  alembic_version: "Alembic",
-  __drizzle_migrations: "drizzle-kit",
-  flyway_schema_history: "Flyway",
-  goose_db_version: "goose",
-  knex_migrations: "Knex",
-  knex_migrations_lock: "Knex",
-  SequelizeMeta: "Sequelize",
-  pgmigrations: "node-pg-migrate",
-};
+/** Tables another tool keeps its migration history in, by name, with the tool; both dialects share the list. */
+export { FOREIGN_TABLES };
 
 export interface LivePgObject {
   type: PostgresEntityType;

package/src/postgres/live/client.ts

@@ -90,3 +90,88 @@ export async function connectPostgres(endpoint: PostgresEndpoint, options: { app
     },
   };
 }
+
+/** SQLSTATE lock_not_available: `lock_timeout` passed while a statement waited for a lock. */
+export const SQLSTATE_LOCK_NOT_AVAILABLE = "55P03";
+
+/** The `lock_timeout` a statement waits under when the profile sets none (`sql.profiles.<env>.lockTimeoutMs`): the applier's and the catalog reader's. */
+export const DEFAULT_LOCK_TIMEOUT_MS = 5_000;
+
+/** A session holding ACCESS EXCLUSIVE on a relation: what a catalog read waits behind. */
+export interface LockHolder {
+  pid: number;
+  relation: string;
+  application?: string;
+  state?: string;
+  /** Seconds since the holder's transaction began. */
+  seconds?: number;
+  query?: string;
+}
+
+/**
+ * The sessions holding ACCESS EXCLUSIVE on a relation in the current database,
+ * other than this one. Only that mode conflicts with the ACCESS SHARE a
+ * catalog function (`pg_get_viewdef()`, `pg_get_indexdef()`) takes. Empty when
+ * the lookup itself fails.
+ */
+export async function accessExclusiveHolders(client: PostgresClient): Promise<LockHolder[]> {
+  try {
+    const rows = await client.query<{ pid: number; relation: string; application: string | null; state: string | null; seconds: number | null; query: string | null }>(
+      `SELECT l.pid, l.relation::pg_catalog.regclass::text AS relation, a.application_name AS application, a.state,
+              (EXTRACT(EPOCH FROM pg_catalog.now() - a.xact_start))::int AS seconds, pg_catalog.left(a.query, 120) AS query
+         FROM pg_catalog.pg_locks l LEFT JOIN pg_catalog.pg_stat_activity a ON a.pid = l.pid
+        WHERE l.locktype = 'relation' AND l.granted AND l.mode = 'AccessExclusiveLock'
+          AND l.pid <> pg_catalog.pg_backend_pid()
+          AND l.database = (SELECT oid FROM pg_catalog.pg_database WHERE datname = pg_catalog.current_database())
+        ORDER BY l.pid, 2 LIMIT 10`,
+    );
+    return rows.map((r) => ({
+      pid: Number(r.pid),
+      relation: r.relation,
+      ...(r.application ? { application: r.application } : {}),
+      ...(r.state ? { state: r.state } : {}),
+      ...(r.seconds !== null && r.seconds !== undefined ? { seconds: Number(r.seconds) } : {}),
+      ...(r.query ? { query: r.query.replace(/\s+/g, " ").trim() } : {}),
+    }));
+  } catch {
+    return [];
+  }
+}
+
+/** One line naming the lock holders, for a message. */
+export function describeLockHolders(holders: readonly LockHolder[]): string {
+  if (holders.length === 0) return "no session holds ACCESS EXCLUSIVE now; the lock was released after the wait";
+  return holders
+    .map((h) => {
+      const about = [h.application ? `application ${JSON.stringify(h.application)}` : "", h.state ?? "", h.seconds !== undefined ? `transaction open ${h.seconds}s` : "", h.query ? `last statement: ${h.query}` : ""].filter(Boolean);
+      return `${h.relation} is held in ACCESS EXCLUSIVE by pid ${h.pid}${about.length > 0 ? ` (${about.join(", ")})` : ""}`;
+    })
+    .join("; ");
+}
+
+/**
+ * A client for catalog reads: the session's `lock_timeout` is `lockTimeoutMs`
+ * (0 waits without limit, as Postgres does), so a read that needs a table
+ * another session holds in ACCESS EXCLUSIVE (a long `ALTER TABLE`, `LOCK
+ * TABLE`) fails after the wait instead of hanging. The failure names the
+ * relation and the pid holding it.
+ */
+export async function withCatalogLockTimeout(client: PostgresClient, lockTimeoutMs: number = DEFAULT_LOCK_TIMEOUT_MS): Promise<PostgresClient> {
+  await client.query("SELECT pg_catalog.set_config('lock_timeout', $1, false)", [`${lockTimeoutMs}ms`]);
+  return {
+    async query<T>(sql: string, params?: readonly unknown[]): Promise<T[]> {
+      try {
+        return await client.query<T>(sql, params);
+      } catch (err) {
+        if (!(err instanceof PostgresQueryError) || err.code !== SQLSTATE_LOCK_NOT_AVAILABLE) throw err;
+        const holders = describeLockHolders(await accessExclusiveHolders(client));
+        throw new PostgresQueryError(
+          `${err.message.split("\n")[0]}: the catalog read waited ${lockTimeoutMs}ms for a lock; ${holders}. Wait for that transaction to finish, or raise sql.profiles.<env>.lockTimeoutMs`,
+          err.code,
+          err.position,
+        );
+      }
+    },
+    end: () => client.end(),
+  };
+}

package/src/postgres/live/live.test.ts

@@ -196,3 +196,57 @@ describe("a managed provider's own objects (sql.provider)", () => {
     expect(r.resources.authUsers).toMatchObject({ ownership: "foreign", attributes: { keptBy: "Supabase" } });
   });
 });
+
+describe("catalog reads under lock_timeout (#3678)", () => {
+  /** A fake whose view read waits behind a lock and times out, and whose pg_locks names the holder. */
+  const blocked = (log: string[]) => {
+    const inner = fakeClient(CATALOG, log);
+    return {
+      async query<T>(sql: string, params?: readonly unknown[]): Promise<T[]> {
+        if (sql.includes("c.relkind IN ('v', 'm')")) {
+          log.push(sql);
+          throw new PostgresQueryError("canceling statement due to lock timeout", "55P03");
+        }
+        if (sql.includes("pg_catalog.pg_locks")) {
+          log.push(sql);
+          return [{ pid: 4242, relation: "app.orders", application: "psql", state: "idle in transaction", seconds: 12, query: "LOCK TABLE app.orders IN ACCESS EXCLUSIVE MODE" }] as T[];
+        }
+        return inner.query<T>(sql, params);
+      },
+      end: () => inner.end(),
+    };
+  };
+
+  test("the session's lock_timeout is the profile's lockTimeoutMs, else the applier's 5000 ms", async () => {
+    for (const [lockTimeoutMs, expected] of [[undefined, "5000ms"], [250, "250ms"]] as const) {
+      const log: string[] = [];
+      const params: unknown[][] = [];
+      const cfg = { sql: { profiles: { prod: { ...config.sql.profiles.prod, ...(lockTimeoutMs !== undefined ? { lockTimeoutMs } : {}) } } } };
+      await describeResources({
+        ...options({ users: DECLARED.users }),
+        config: cfg,
+        connect: async () => {
+          const c = fakeClient(CATALOG, log);
+          return { query: (sql: string, p?: readonly unknown[]) => (params.push([sql, ...(p ?? [])]), c.query(sql, p)), end: c.end } as never;
+        },
+      });
+      expect(params[0]).toEqual(["SELECT pg_catalog.set_config('lock_timeout', $1, false)", expected]);
+    }
+  });
+
+  test("a read that times out on a lock is read-failed, naming the relation and the pid holding it", async () => {
+    const log: string[] = [];
+    const r = normalizeObservation(await describeResources({ ...options({ users: DECLARED.users }), connect: async () => blocked(log) as never }));
+    const users = r.unobserved.users;
+    expect(users?.reason).toBe("read-failed");
+    expect(users?.detail).toContain("canceling statement due to lock timeout");
+    expect(users?.detail).toContain("app.orders is held in ACCESS EXCLUSIVE by pid 4242");
+    expect(users?.detail).toContain("lockTimeoutMs");
+  });
+
+  test("other failures pass through unchanged", async () => {
+    const { withCatalogLockTimeout } = await import("./client");
+    const c = await withCatalogLockTimeout({ query: async (sql: string) => { if (sql.startsWith("SELECT 1")) throw new PostgresQueryError("boom", "XX000"); return []; }, end: async () => {} }, 100);
+    await expect(c.query("SELECT 1")).rejects.toThrow(/^boom$/);
+  });
+});

package/src/postgres/live/lock-timeout.e2e.test.ts

@@ -0,0 +1,70 @@
+/**
+ * Catalog reads under `lock_timeout` (#3678). One session holds ACCESS
+ * EXCLUSIVE on a table behind a view; the observation of the view, which
+ * reads its definition with `pg_get_viewdef()`, fails after the profile's
+ * `lockTimeoutMs` instead of waiting for the lock, and names the table and the
+ * pid holding it.
+ *
+ * `CHANT_SQL_E2E_POSTGRES_URL` (with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names
+ * a running server, else a throwaway one, skipped when Docker is not
+ * available. The test makes only the schema `chant_e2e_3678`, dropped
+ * afterwards.
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { normalizeObservation } from "@intentius/chant/observation";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "../testing/server";
+import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "./client";
+import { describeResources } from "./describe-resources";
+
+const SCHEMA = "chant_e2e_3678";
+const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
+const enabled = given ? true : await dockerAvailable();
+let server: TestPostgres | undefined;
+let endpoint: PostgresEndpoint;
+let admin: PostgresClient;
+let holder: PostgresClient;
+
+describe.skipIf(!enabled)("a catalog read blocked behind ACCESS EXCLUSIVE", () => {
+  beforeAll(async () => {
+    if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
+    else {
+      server = await startTestPostgres();
+      endpoint = server.endpoint();
+    }
+    admin = await connectPostgres(endpoint);
+    await admin.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);
+    await admin.query(`CREATE SCHEMA ${SCHEMA}`);
+    await admin.query(`CREATE TABLE ${SCHEMA}.orders (id bigint PRIMARY KEY)`);
+    await admin.query(`CREATE VIEW ${SCHEMA}.recent AS SELECT id FROM ${SCHEMA}.orders`);
+    holder = await connectPostgres(endpoint, { applicationName: "chant-e2e-3678-holder" });
+    await holder.query("BEGIN");
+    await holder.query(`LOCK TABLE ${SCHEMA}.orders IN ACCESS EXCLUSIVE MODE`);
+  }, 600_000);
+
+  afterAll(async () => {
+    await holder?.query("ROLLBACK").catch(() => undefined);
+    await holder?.end();
+    await admin?.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`).catch(() => undefined);
+    await admin?.end();
+    await server?.stop();
+  });
+
+  test("fails after lockTimeoutMs, naming the relation and the pid", async () => {
+    const [{ pid }] = await holder.query<{ pid: number }>("SELECT pg_catalog.pg_backend_pid() AS pid");
+    const started = Date.now();
+    const r = normalizeObservation(await describeResources({
+      environment: "e2e",
+      config: { sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_3678_PASSWORD" }, schemas: [SCHEMA], lockTimeoutMs: 300 } } } },
+      env: { PG_E2E_3678_PASSWORD: endpoint.password ?? "" },
+      entityNames: ["recent"],
+      entities: new Map([["recent", { entityType: "Postgres::View", props: { schema: SCHEMA, name: "recent" } }]]),
+    }));
+    expect(Date.now() - started).toBeLessThan(10_000);
+    const recent = r.unobserved.recent;
+    expect(recent?.reason).toBe("read-failed");
+    expect(recent?.detail).toContain("lock timeout");
+    expect(recent?.detail).toContain(`${SCHEMA}.orders is held in ACCESS EXCLUSIVE by pid ${pid}`);
+    expect(recent?.detail).toContain("chant-e2e-3678-holder");
+  }, 60_000);
+});

package/src/postgres/plan/commands.ts

@@ -20,8 +20,7 @@ import { POSTGRES_ENTITY_TYPES } from "../entity-types";
 import { migrationOpSuggestions } from "../migrate/handoff";
 import { POSTGRES_LATEST_MAJOR } from "../../spec/postgres-pin";
 import { ACCESS_MANAGED_KINDS } from "./normalize";
-import { declaredAccess, diffAccess, predictedAccess, targetKey } from "../access/acl";
-import { readLiveAccess } from "../access/live";
+import { accessChangesAgainst } from "../access/changes";
 
 export function emitPg(diff: PgSchemaDiff, json: boolean, title: string): number {
   console.log(json ? JSON.stringify(diff, null, 2) : renderPgDiff(diff, { title }));
@@ -112,12 +111,7 @@ export async function planAgainstClient(
   const major = running ?? options.major;
   const diff = diffPgSchemas(live, declared, { major, access: false });
   if (managed) {
-    const want = declaredAccess(declared, { major: major ?? POSTGRES_LATEST_MAJOR, ...(await currentRole(client)) });
-    const have = await readLiveAccess(client, want.targets, { ...(scope ? { schemas: scope } : {}), forRoles: want.forRoles, declaredDefaults: want.targets.filter((t) => t.kind === "default") });
-    // An object this plan creates is given what the server gives a new one.
-    const missing = want.targets.filter((t) => t.kind !== "default" && t.kind !== "column" && !have.present.has(targetKey(t)));
-    const now = new Map([...have.state, ...predictedAccess(missing, have)]);
-    const access = diffAccess(now, want.state, want.exportsOf);
+    const access = await accessChangesAgainst(client, declared, { major: major ?? POSTGRES_LATEST_MAJOR, ...(scope ? { scope } : {}) });
     if (access.length > 0) {
       diff.changes.push(...access.map((a) => a.change));
       diff.access = access;
@@ -131,12 +125,6 @@ export async function planAgainstClient(
   return { declared, live, liveObjects, diff, ...(major !== undefined ? { major } : {}), ...(scope ? { scope } : {}) };
 }
 
-/** The role a session runs as: the one whose own privileges owning an object gives, and whose default privileges apply. */
-async function currentRole(client: PostgresClient): Promise<{ self?: string }> {
-  const rows = await client.query<{ self: string }>("SELECT current_user AS self").catch(() => []);
-  return rows[0]?.self ? { self: rows[0].self } : {};
-}
-
 /**
  * The declarations a plan compares: all of them where the profile manages
  * access; otherwise none of the policies, roles, grants and default

package/src/postgres/plan/deep.ts

@@ -13,7 +13,7 @@
  * of the object (the DDL text, the template source, lineage, reads).
  */
 
-import { deepObservation, type DeepObservationResult, type DeepResourceObservation } from "@intentius/chant/deep-observation";
+import { deepObservation, type DeepObservationResult, type DeepPendingChange, type DeepResourceObservation } from "@intentius/chant/deep-observation";
 import type { UnobservedEntity } from "@intentius/chant/lexicon";
 import { bindPostgres, classifyPostgresFailure, type BindOptions } from "../live/bind";
 import { liveKey, readLiveSchema, type LivePgObject } from "../live/catalog";
@@ -23,6 +23,8 @@ import { POSTGRES_ENTITY_TYPES } from "../entity-types";
 import { canonicalPgObject, columnShape, sameConstraint, type CanonicalPgObject } from "./normalize";
 import { serverNormalized } from "./server-normalize";
 import type { PostgresClient } from "../live/client";
+import { accessChangesAgainst } from "../access/changes";
+import { POSTGRES_LATEST_MAJOR } from "../../spec/postgres-pin";
 
 type Props = Record<string, unknown>;
 
@@ -214,7 +216,21 @@ export async function observeResourcesDeep(
     for (const { name, entity } of declared) unobserved[name] = { type: entity?.entityType ?? "", reason: why.reason, detail: why.detail };
     return deepObservation({}, unobserved);
   }
+  const pending: DeepPendingChange[] = [];
   try {
+    if (target.access === true) {
+      // #3706: the privilege changes an apply would make, which no one
+      // declaration's properties show, so the plan digest moves with them.
+      try {
+        pending.push(...(await pendingAccess(client, target.defaultSchema, declared, scopeFor(target, declared.filter((x) => x.entity?.entityType.startsWith("Postgres::")).map((x) => ({ type: x.entity!.entityType, props: x.entity!.props }))))));
+      } catch (err) {
+        for (const { name, entity } of declared) {
+          if (entity?.entityType === POSTGRES_ENTITY_TYPES.grant || entity?.entityType === POSTGRES_ENTITY_TYPES.defaultPrivileges) {
+            unobserved[name] = { type: entity.entityType, reason: "read-failed", detail: `the server's privileges could not be compared: ${(err as Error).message}` };
+          }
+        }
+      }
+    }
     const byKey = new Map(live.map((o) => [liveKey(o.type, o.schema, o.name, o.signature), o]));
     for (const { name, entity } of declared) {
       if (!entity || !entity.entityType.startsWith("Postgres::")) {
@@ -223,6 +239,7 @@ export async function observeResourcesDeep(
       }
       const why = accessUnobserved(entity.entityType, target.access === true);
       if (why) {
+        if (unobserved[name]) continue;
         unobserved[name] = { type: entity.entityType, ...why };
         continue;
       }
@@ -245,5 +262,30 @@ export async function observeResourcesDeep(
   } finally {
     await client.end();
   }
-  return deepObservation(resources, unobserved);
+  return deepObservation(resources, unobserved, pending);
+}
+
+/**
+ * The privilege changes an apply would make (#3706), as `chant sql plan`
+ * computes them, one per object and grantee: the statements, and the
+ * declarations behind them.
+ */
+async function pendingAccess(
+  client: PostgresClient,
+  defaultSchema: string,
+  declared: ReadonlyArray<{ name: string; entity?: { entityType: string; props: Record<string, unknown> } }>,
+  scope: string[] | undefined,
+): Promise<DeepPendingChange[]> {
+  const objects = declared
+    .filter((x) => x.entity?.entityType.startsWith("Postgres::"))
+    .map((x) => ({ key: x.name, canonical: canonicalPgObject(x.entity!.entityType, x.entity!.props, defaultSchema) }));
+  const [row] = await client.query<{ v: string }>("SELECT current_setting('server_version_num') AS v");
+  const n = Number(row?.v);
+  const major = Number.isFinite(n) && n > 0 ? Math.floor(n / 10000) : POSTGRES_LATEST_MAJOR;
+  const changes = await accessChangesAgainst(client, objects, { major, ...(scope ? { scope } : {}) });
+  return changes.map((c) => ({
+    subject: c.change.object.startsWith("acl ") ? c.change.object.slice(4) : c.change.object,
+    change: c.sql.join("; "),
+    ...(c.exports.length > 0 ? { entities: [...c.exports].sort() } : {}),
+  }));
 }

package/src/serializer.test.ts

@@ -50,6 +50,8 @@ describe("sql serializer", () => {
     expect(sql.indexOf("CREATE DATABASE")).toBeLessThan(sql.indexOf("CREATE TABLE"));
     expect(sql.indexOf("CREATE TABLE")).toBeLessThan(sql.indexOf("CREATE VIEW"));
     expect(r.verbatimFiles).toEqual([CLICKHOUSE_DDL_FILE]);
+    // DDL is copied exactly but is not a secret, so it prints with no --output (#3738).
+    expect(r.secretFiles).toBeUndefined();
   });
 
   test("10. a declarable of another lexicon is not written", () => {
@@ -98,6 +100,8 @@ describe("sql serializer, Postgres", () => {
     expect(doc(r).objects[1]).toMatchObject({ export: "users", type: "Postgres::Table", name: "users", schema: "app", sqlName: "app.users" });
     expect(r.files![POSTGRES_DDL_FILE]).toBe("CREATE SCHEMA app;\n\nCREATE TABLE app.users (id bigint PRIMARY KEY, email text NOT NULL);\n");
     expect(r.verbatimFiles).toEqual([POSTGRES_DDL_FILE]);
+    // DDL is copied exactly but is not a secret, so it prints with no --output (#3738).
+    expect(r.secretFiles).toBeUndefined();
   });
 
   test("a Postgres build records the target major: sql.postgresMajor, else the newest pinned", () => {

package/src/skills/chant-sql-postgres-plan.md

@@ -167,7 +167,7 @@ Transactions follow the class, so a long scan holds locks on one table and what
 
 A primary key or unique constraint added to an existing table goes in two steps, the unique index built `CONCURRENTLY` and then `ADD CONSTRAINT ... USING INDEX` (SQLPG221), so a busy table is never blocked for the build.
 
-Timeouts are set on every statement: `lock_timeout` 5000 ms, so a statement that cannot get its lock fails instead of queueing behind a long transaction and blocking everything behind it; `statement_timeout` 60000 ms for catalog-only statements, and 0 (no limit) for scans, rewrites and `CONCURRENTLY` builds. Set them per environment with `lockTimeoutMs`, `statementTimeoutMs` and `scanTimeoutMs` on `sql.profiles.<env>`, or per call.
+Timeouts are set on every statement: `lock_timeout` 5000 ms, so a statement that cannot get its lock fails instead of queueing behind a long transaction and blocking everything behind it; `statement_timeout` 60000 ms for catalog-only statements, and 0 (no limit) for scans, rewrites and `CONCURRENTLY` builds. Set them per environment with `lockTimeoutMs`, `statementTimeoutMs` and `scanTimeoutMs` on `sql.profiles.<env>`, or per call. The catalog read an apply makes before its first statement waits under the same `lock_timeout`: blocked behind `ACCESS EXCLUSIVE`, the apply stops with nothing written, naming the table and the pid that holds it.
 
 ```ts
 profiles: { prod: { url: "postgres://db.internal:5432/shop", lockTimeoutMs: 2000, scanTimeoutMs: 600000 } }

package/src/skills/chant-sql-rebuild.md

@@ -57,6 +57,8 @@ The swap gate is bound to a digest of the plan and the verification (row counts
 - A run stopped during the backfill (Ctrl-C, a killed job) is not a failure. The next run resumes from the receipts in `chant_receipts.receipts` on the same server (in `<db>.__chant_receipts` on a Replicated database), and a partition whose copy was cut off before its receipt is cleared and copied again, so no row is copied twice.
 - A failed step (a failed copy after its retries, a verification mismatch, a refused declaration) runs onFailure: the new table `<t>__chant_new` and the dual-write view `<t>__chant_dual` are dropped, and the next run starts from a new table. With `onFailure: "keep"` they stay, and the next run resumes; drop them by hand to start over. The original table is untouched until the swap.
 - A verification mismatch in materialized-view mode usually means rows arrived with a time before the cut-over later than `cutoverDelay`. Raise the delay and run again.
+- On a Summing, Replacing, Aggregating or Collapsing MergeTree the verification compares both tables under `FINAL`, so unmerged parts in the old table are not a mismatch. A mismatch that remains names the engine: run `OPTIMIZE TABLE <db>.<table> FINAL` and run the rebuild again.
+- A plain MergeTree rebuilt into one of those engines is compared per key of the new sorting key, on what the new engine keeps (a Summing table's sums, a Replacing table's highest version, a Collapsing table's sum of signs), so the different row counts are not a mismatch. A missing key or a changed sum still fails.
 - A backfill that stops naming writes "begun before the cut-over" waited `cutoverTimeout` for INSERTs into the old table that were still running. Let them finish, or stop them, and run again.
 - The Op refuses a database that is neither Atomic nor Replicated (`EXCHANGE TABLES` needs one of them), a table whose change is not a rebuild (use the applier), and an object already under one of its working names that is not its own. In a Replicated database it also refuses a table, or a declaration, whose engine is not `Replicated*MergeTree`: a plain MergeTree keeps different rows on each replica and one replica's copy would lose the others'.
 

package/src/skills/chant-sql.md

@@ -37,7 +37,7 @@ A dictionary is a `dictionary` template holding `CREATE DICTIONARY`, with its at
 
 A SQL user-defined function is a `func` template holding `CREATE FUNCTION name AS (x) -> expr`, with no database in its name. A changed expression is SQLCH260 (`CREATE OR REPLACE FUNCTION`). It carries no ownership marker, so chant never drops one.
 
-Users, roles, row policies and grants are `user`, `role`, `policy` and `grant` templates. Never write a password: `IDENTIFIED BY` is refused, a user declared without `IDENTIFIED` is created by the environment with its password, and chant manages the rest of it. Grants are compared per grantee: every `grant` naming a grantee adds up to its complete list, and anything else it holds is revoked (SQLCH274). None of these is ever dropped by chant.
+Users, roles, row policies and grants are `user`, `role`, `policy` and `grant` templates. Never write a password: `IDENTIFIED BY` is refused, a user declared without `IDENTIFIED` is created by the environment with its password, and chant manages the rest of it. Grants are compared per grantee: every `grant` naming a grantee adds up to its complete list, and anything else it holds is revoked (SQLCH274). None of these is ever dropped by chant. They are planned and applied only where the profile sets `access: true` (`sql.profiles.<env>.access`); elsewhere each is reported `filtered`.
 
 ## Interpolations