@intentius/chant-lexicon-sql 0.118.1 → 0.120.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/dist/clickhouse/apply/apply.d.ts +1 -1
  2. package/dist/clickhouse/apply/apply.d.ts.map +1 -1
  3. package/dist/clickhouse/apply/statements.d.ts +8 -0
  4. package/dist/clickhouse/apply/statements.d.ts.map +1 -1
  5. package/dist/clickhouse/import/generator.d.ts +1 -1
  6. package/dist/clickhouse/import/generator.d.ts.map +1 -1
  7. package/dist/clickhouse/import/live-export.d.ts +19 -0
  8. package/dist/clickhouse/import/live-export.d.ts.map +1 -1
  9. package/dist/clickhouse/live/bind.d.ts +8 -0
  10. package/dist/clickhouse/live/bind.d.ts.map +1 -1
  11. package/dist/clickhouse/live/catalog.d.ts +15 -0
  12. package/dist/clickhouse/live/catalog.d.ts.map +1 -1
  13. package/dist/clickhouse/live/describe-resources.d.ts.map +1 -1
  14. package/dist/clickhouse/plan/commands.d.ts.map +1 -1
  15. package/dist/clickhouse/plan/deep.d.ts +19 -2
  16. package/dist/clickhouse/plan/deep.d.ts.map +1 -1
  17. package/dist/clickhouse/plan/diff.d.ts.map +1 -1
  18. package/dist/clickhouse/plan/normalize.d.ts +3 -0
  19. package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
  20. package/dist/clickhouse/plan/rebuild-handoff.d.ts +2 -0
  21. package/dist/clickhouse/plan/rebuild-handoff.d.ts.map +1 -1
  22. package/dist/clickhouse/rebuild/engines.d.ts +44 -0
  23. package/dist/clickhouse/rebuild/engines.d.ts.map +1 -0
  24. package/dist/clickhouse/rebuild/verify.d.ts +25 -1
  25. package/dist/clickhouse/rebuild/verify.d.ts.map +1 -1
  26. package/dist/clickhouse/testing/fake-server.d.ts +5 -0
  27. package/dist/clickhouse/testing/fake-server.d.ts.map +1 -1
  28. package/dist/config.d.ts +32 -8
  29. package/dist/config.d.ts.map +1 -1
  30. package/dist/core/foreign-tables.d.ts +13 -0
  31. package/dist/core/foreign-tables.d.ts.map +1 -0
  32. package/dist/files/clickhouse.d.ts +7 -3
  33. package/dist/files/clickhouse.d.ts.map +1 -1
  34. package/dist/files/postgres.d.ts +3 -0
  35. package/dist/files/postgres.d.ts.map +1 -1
  36. package/dist/import-parser.d.ts.map +1 -1
  37. package/dist/integrity.json +2 -2
  38. package/dist/manifest.json +1 -1
  39. package/dist/op/activities/clickhouse-emulator.d.ts +7 -1
  40. package/dist/op/activities/clickhouse-emulator.d.ts.map +1 -1
  41. package/dist/op/activities/postgres-emulator.d.ts +8 -1
  42. package/dist/op/activities/postgres-emulator.d.ts.map +1 -1
  43. package/dist/postgres/access/changes.d.ts +23 -0
  44. package/dist/postgres/access/changes.d.ts.map +1 -0
  45. package/dist/postgres/apply/apply.d.ts.map +1 -1
  46. package/dist/postgres/entities.d.ts.map +1 -1
  47. package/dist/postgres/live/bind.d.ts +8 -1
  48. package/dist/postgres/live/bind.d.ts.map +1 -1
  49. package/dist/postgres/live/catalog.d.ts +3 -3
  50. package/dist/postgres/live/catalog.d.ts.map +1 -1
  51. package/dist/postgres/live/client.d.ts +31 -0
  52. package/dist/postgres/live/client.d.ts.map +1 -1
  53. package/dist/postgres/plan/commands.d.ts.map +1 -1
  54. package/dist/postgres/plan/deep.d.ts.map +1 -1
  55. package/dist/sql-files.d.ts +2 -1
  56. package/dist/sql-files.d.ts.map +1 -1
  57. package/package.json +2 -2
  58. package/src/clickhouse/apply/apply.ts +8 -2
  59. package/src/clickhouse/apply/statements.ts +32 -5
  60. package/src/clickhouse/import/generator.ts +8 -3
  61. package/src/clickhouse/import/import.test.ts +64 -0
  62. package/src/clickhouse/import/live-export.ts +70 -2
  63. package/src/clickhouse/live/access.e2e.test.ts +71 -18
  64. package/src/clickhouse/live/bind.ts +10 -0
  65. package/src/clickhouse/live/catalog.ts +25 -2
  66. package/src/clickhouse/live/describe-resources.ts +5 -2
  67. package/src/clickhouse/live/live.test.ts +8 -0
  68. package/src/clickhouse/plan/commands.ts +6 -3
  69. package/src/clickhouse/plan/deep-grants.test.ts +67 -0
  70. package/src/clickhouse/plan/deep.ts +70 -6
  71. package/src/clickhouse/plan/diff.ts +4 -1
  72. package/src/clickhouse/plan/normalize.ts +5 -2
  73. package/src/clickhouse/plan/plan.test.ts +9 -1
  74. package/src/clickhouse/plan/rebuild-handoff.ts +11 -1
  75. package/src/clickhouse/rebuild/engines.ts +163 -0
  76. package/src/clickhouse/rebuild/rebuild-collapsing.e2e.test.ts +120 -0
  77. package/src/clickhouse/rebuild/rebuild-into-collapsing.e2e.test.ts +198 -0
  78. package/src/clickhouse/rebuild/rebuild.test.ts +78 -0
  79. package/src/clickhouse/rebuild/verify.ts +153 -24
  80. package/src/clickhouse/testing/fake-server.ts +3 -2
  81. package/src/codegen/classifier-page.ts +1 -1
  82. package/src/config.ts +16 -4
  83. package/src/core/foreign-tables.ts +28 -0
  84. package/src/files/clickhouse.ts +71 -5
  85. package/src/files/postgres.ts +2 -1
  86. package/src/import-parser.ts +2 -0
  87. package/src/migration-statements.test.ts +33 -0
  88. package/src/migration-statements.ts +2 -0
  89. package/src/op/__snapshots__/watch-pipeline.test.ts.snap +1 -1
  90. package/src/op/activities/clickhouse-emulator.ts +22 -1
  91. package/src/op/activities/postgres-apply-lock-timeout.e2e.test.ts +87 -0
  92. package/src/op/activities/postgres-apply.test.ts +32 -0
  93. package/src/op/activities/postgres-emulator.test.ts +20 -0
  94. package/src/op/activities/postgres-emulator.ts +41 -1
  95. package/src/op/apply-gate-access-clickhouse.e2e.test.ts +195 -0
  96. package/src/op/apply-gate-access.e2e.test.ts +207 -0
  97. package/src/postgres/access/access.test.ts +26 -1
  98. package/src/postgres/access/changes.ts +39 -0
  99. package/src/postgres/apply/apply.ts +10 -7
  100. package/src/postgres/entities.ts +20 -1
  101. package/src/postgres/live/bind.ts +16 -4
  102. package/src/postgres/live/catalog.ts +3 -15
  103. package/src/postgres/live/client.ts +85 -0
  104. package/src/postgres/live/live.test.ts +54 -0
  105. package/src/postgres/live/lock-timeout.e2e.test.ts +70 -0
  106. package/src/postgres/plan/commands.ts +2 -14
  107. package/src/postgres/plan/deep.ts +44 -2
  108. package/src/serializer.test.ts +4 -0
  109. package/src/skills/chant-sql-postgres-plan.md +1 -1
  110. package/src/skills/chant-sql-rebuild.md +2 -0
  111. package/src/skills/chant-sql.md +1 -1
  112. package/src/sql-files.test.ts +78 -1
  113. package/src/sql-files.ts +2 -1
@@ -0,0 +1,207 @@
1
+ /**
2
+ * An approval does not cover a privilege granted by hand after it (#3706),
3
+ * end to end on Postgres with a profile that manages access.
4
+ *
5
+ * A project in a temporary directory declares a schema, a table and a grant
6
+ * of SELECT on it to a reader role, and a gated `ApplyOp`. The first approved
7
+ * run creates them. The next run stops at the gate; the test approves its
8
+ * plan, grants INSERT on the table to another role by hand, and runs again.
9
+ * The gate refuses with "is approved, but not for this plan", the hand grant
10
+ * is still there, and the plan the run printed lists the REVOKE an apply
11
+ * would make. Approving that plan applies it.
12
+ *
13
+ * Before #3706 the plan digest was taken over a diff that left privileges
14
+ * out (a grant was unobserved there), so the third run revoked the hand
15
+ * grant under the earlier approval.
16
+ *
17
+ * Everything is written to the schema `chant_e2e_3706` and the roles
18
+ * `chant_e2e_3706_reader` and `chant_e2e_3706_writer`, all dropped afterwards.
19
+ *
20
+ * The server: `CHANT_SQL_GATE_POSTGRES_URL` (a `postgres://user@host:port/db`
21
+ * URL, with `CHANT_SQL_GATE_POSTGRES_PASSWORD`) names a running one, such as
22
+ * the one `chant emulator up --lexicon sql` starts. Without it, a throwaway
23
+ * server at the pin, skipped cleanly when Docker is not available.
24
+ */
25
+
26
+ import { execFile } from "node:child_process";
27
+ import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
28
+ import { tmpdir } from "node:os";
29
+ import { join } from "node:path";
30
+ import { promisify } from "node:util";
31
+ import { afterAll, beforeAll, describe, expect, test } from "vitest";
32
+ import { dockerAvailable, startTestPostgres, type TestPostgres } from "../postgres/testing/server";
33
+ import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../postgres/live/client";
34
+
35
+ const exec = promisify(execFile);
36
+
37
+ const REPO = join(import.meta.dirname, "../../../..");
38
+ const BIN = join(REPO, "node_modules/.bin");
39
+ const S = "chant_e2e_3706";
40
+ const READER = "chant_e2e_3706_reader";
41
+ const WRITER = "chant_e2e_3706_writer";
42
+ const OP = "schema-apply";
43
+
44
+ const given = process.env.CHANT_SQL_GATE_POSTGRES_URL;
45
+ const enabled = given ? true : await dockerAvailable();
46
+
47
+ interface Run {
48
+ code: number;
49
+ output: string;
50
+ }
51
+
52
+ const strip = (s: string) => s.replace(/\u001b\[[0-9;]*m/g, "");
53
+
54
+ const SCHEMA = `import { schema, table, grant } from "@intentius/chant-lexicon-sql/postgres";
55
+
56
+ export const app = schema\`CREATE SCHEMA ${S}\`;
57
+
58
+ export const orders = table\`
59
+ CREATE TABLE \${app}.orders (
60
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
61
+ total numeric NOT NULL
62
+ )\`;
63
+
64
+ export const readOrders = grant\`GRANT SELECT ON TABLE \${orders} TO ${READER}\`;
65
+ `;
66
+
67
+ const OPS = `import { ApplyOp } from "@intentius/chant/op";
68
+ const { op } = ApplyOp({ name: "${OP}", env: "admin", target: "postgres", gate: { gate: "approve-schema" } });
69
+ export default op;
70
+ `;
71
+
72
+ const config = (url: string) => `import type { ChantConfig } from "@intentius/chant";
73
+ import "@intentius/chant-lexicon-sql";
74
+
75
+ export default {
76
+ lexicons: ["sql"],
77
+ ownership: { stack: "e2e-3706", env: "admin" },
78
+ sql: ${JSON.stringify({ dialect: "postgres", profiles: { admin: { url, password: { env: "ADMIN_PASSWORD" }, schemas: [S], access: true } } }, null, 2)},
79
+ } satisfies ChantConfig;
80
+ `;
81
+
82
+ /** The `--plan <digest>` the stopped run printed in its approve command. */
83
+ function approvedPlan(run: Run): string {
84
+ const m = /chant approve \S+ \S+ .*?--plan (jcs1-sha256:[0-9a-f]{64})/.exec(run.output);
85
+ if (!m) throw new Error(`the run printed no approve command with a plan:\n${run.output}`);
86
+ return m[1];
87
+ }
88
+
89
+ function gateOf(run: Run): string {
90
+ const m = /chant approve \S+ (\S+)/.exec(run.output);
91
+ if (!m) throw new Error(`the run printed no approve command:\n${run.output}`);
92
+ return m[1];
93
+ }
94
+
95
+ describe.skipIf(!enabled)("a gated ApplyOp over Postgres refuses after a privilege granted by hand (#3706)", () => {
96
+ let server: TestPostgres | undefined;
97
+ let endpoint: PostgresEndpoint;
98
+ let admin: PostgresClient;
99
+ let dir: string | undefined;
100
+ const cleanup = async () => {
101
+ await admin.query(`DROP SCHEMA IF EXISTS ${S} CASCADE`);
102
+ for (const r of [READER, WRITER]) await admin.query(`DROP ROLE IF EXISTS ${r}`);
103
+ };
104
+
105
+ const chant = async (...args: string[]): Promise<Run> => {
106
+ try {
107
+ const r = await exec(join(BIN, "chant"), args, {
108
+ cwd: dir,
109
+ env: { ...process.env, ADMIN_PASSWORD: endpoint.password ?? "", PATH: `${BIN}:${process.env.PATH ?? ""}` },
110
+ maxBuffer: 16 * 1024 * 1024,
111
+ });
112
+ return { code: 0, output: strip(`${r.stdout}${r.stderr}`) };
113
+ } catch (err) {
114
+ const e = err as { code?: number; stdout?: string; stderr?: string };
115
+ return { code: typeof e.code === "number" ? e.code : 1, output: strip(`${e.stdout ?? ""}${e.stderr ?? ""}`) };
116
+ }
117
+ };
118
+
119
+ /** Run the Op, approve the plan it stopped at, run again: the apply converges. */
120
+ const approveAndRun = async (): Promise<Run> => {
121
+ const stopped = await chant("run", OP);
122
+ const approve = await chant("approve", OP, gateOf(stopped), "--plan", approvedPlan(stopped));
123
+ expect(approve.code, approve.output).toBe(0);
124
+ const run = await chant("run", OP);
125
+ expect(run.output).toMatch(/Op "schema-apply" completed/);
126
+ return run;
127
+ };
128
+
129
+ const writerPrivileges = async () =>
130
+ (
131
+ await admin.query<{ p: string }>(
132
+ `SELECT privilege_type AS p FROM information_schema.role_table_grants WHERE table_schema = '${S}' AND table_name = 'orders' AND grantee = '${WRITER}'`,
133
+ )
134
+ ).map((r) => r.p);
135
+
136
+ beforeAll(async () => {
137
+ if (given) endpoint = { url: given, password: process.env.CHANT_SQL_GATE_POSTGRES_PASSWORD };
138
+ else {
139
+ server = await startTestPostgres();
140
+ endpoint = server.endpoint();
141
+ }
142
+ admin = await connectPostgres(endpoint);
143
+ await cleanup();
144
+ // The roles are the environment's: provisioned outside the declarations, only named in them.
145
+ for (const r of [READER, WRITER]) await admin.query(`CREATE ROLE ${r} NOLOGIN`);
146
+ dir = mkdtempSync(join(tmpdir(), "chant-sql-gate-access-"));
147
+ symlinkSync(join(REPO, "node_modules"), join(dir, "node_modules"), "dir");
148
+ const files: Record<string, string> = {
149
+ "package.json": JSON.stringify({ name: "sql-gate-access-e2e", private: true, type: "module", scripts: { build: "chant build src --lexicon sql -o dist/schema.json" } }),
150
+ ".gitignore": "node_modules\ndist\n.chant\n",
151
+ "chant.config.ts": config(endpoint.url),
152
+ "ops/apply.op.ts": OPS,
153
+ "src/schema.ts": SCHEMA,
154
+ };
155
+ for (const [path, text] of Object.entries(files)) {
156
+ mkdirSync(join(dir, path, ".."), { recursive: true });
157
+ writeFileSync(join(dir, path), text);
158
+ }
159
+ const git = (...args: string[]) => exec("git", ["-c", "user.email=e2e@chant.invalid", "-c", "user.name=e2e", ...args], { cwd: dir });
160
+ await git("init", "-q");
161
+ await git("add", "-A");
162
+ await git("commit", "-qm", "the schema");
163
+ }, 600_000);
164
+
165
+ afterAll(async () => {
166
+ if (admin) {
167
+ await cleanup().catch(() => undefined);
168
+ await admin.end();
169
+ }
170
+ await server?.stop();
171
+ if (dir) rmSync(dir, { recursive: true, force: true });
172
+ });
173
+
174
+ test("approve, grant by hand, rerun: the gate refuses and the hand grant stays", async () => {
175
+ await approveAndRun();
176
+ const readers = await admin.query<{ p: string }>(
177
+ `SELECT privilege_type AS p FROM information_schema.role_table_grants WHERE table_schema = '${S}' AND table_name = 'orders' AND grantee = '${READER}'`,
178
+ );
179
+ expect(readers.map((r) => r.p)).toEqual(["SELECT"]);
180
+
181
+ // Nothing to change: the run stops at the gate, and its plan is approved.
182
+ const first = await chant("run", OP);
183
+ expect(first.output).not.toContain("PENDING (");
184
+ const gate = gateOf(first);
185
+ const digest = approvedPlan(first);
186
+ const approve = await chant("approve", OP, gate, "--plan", digest);
187
+ expect(approve.code, approve.output).toBe(0);
188
+
189
+ await admin.query(`GRANT INSERT ON ${S}.orders TO ${WRITER}`);
190
+
191
+ const second = await chant("run", OP);
192
+ expect(second.output).toContain(`Gate "${gate}" is approved, but not for this plan.`);
193
+ expect(second.output).toContain(`approved: ${digest}`);
194
+ expect(second.output).toMatch(/PENDING \(changes an apply would make that no declared property shows\):\s+~ relation chant_e2e_3706\.orders TO chant_e2e_3706_writer: REVOKE INSERT ON TABLE chant_e2e_3706\.orders FROM chant_e2e_3706_writer/);
195
+ expect(second.output).not.toMatch(/Op "schema-apply" completed/);
196
+ expect(await writerPrivileges()).toEqual(["INSERT"]);
197
+
198
+ // Approving the plan the second run produced applies the revoke.
199
+ const fresh = approvedPlan(second);
200
+ expect(fresh).not.toBe(digest);
201
+ const reapprove = await chant("approve", OP, gate, "--plan", fresh);
202
+ expect(reapprove.code, reapprove.output).toBe(0);
203
+ const third = await chant("run", OP);
204
+ expect(third.output).toMatch(/Op "schema-apply" completed/);
205
+ expect(await writerPrivileges()).toEqual([]);
206
+ }, 600_000);
207
+ });
@@ -6,7 +6,7 @@
6
6
 
7
7
  import { describe, expect, test } from "vitest";
8
8
  import { func, grant, policy, role, schema, table } from "../entities";
9
- import { diffObject, keyedByQualifiedName, pgSchemaFromLive, type PgSchemaObject } from "../plan/schema";
9
+ import { diffObject, keyedByQualifiedName, pgSchemaFromBuildOutput, pgSchemaFromLive, type PgSchemaObject } from "../plan/schema";
10
10
  import { diffPgSchemas, matchPgObjects } from "../plan/diff";
11
11
  import { accessScoped } from "../plan/commands";
12
12
  import { alterSteps } from "../apply/statements";
@@ -21,6 +21,31 @@ const orders = table`
21
21
  ALTER TABLE ${app}.orders ENABLE ROW LEVEL SECURITY;
22
22
  ALTER TABLE ${app}.orders FORCE ROW LEVEL SECURITY`;
23
23
 
24
+ describe("a grant on an interpolated routine (#3707)", () => {
25
+ const f = func`CREATE FUNCTION ${app}.total(n int, OUT t bigint) LANGUAGE sql AS 'select 1'`;
26
+ const buildOf = (objects: Array<{ export: string; entity: { entityType: string; props: unknown } }>) =>
27
+ JSON.stringify({
28
+ dialect: "postgres",
29
+ postgresMajor: 18,
30
+ objects: objects.map((o) => ({ export: o.export, type: o.entity.entityType, ddl: (o.entity.props as { ddl: string }).ddl })),
31
+ });
32
+
33
+ test("its statement names the routine with its parameter types", () => {
34
+ expect(grant`GRANT EXECUTE ON FUNCTION ${f} TO ${reader}`.props).toMatchObject({ ddl: "GRANT EXECUTE ON FUNCTION app.total(int) TO app_reader", objectNames: ["app.total(int)"] });
35
+ // Types written after the reference are kept, not doubled.
36
+ expect((grant`GRANT EXECUTE ON FUNCTION ${f}(int) TO ${reader}`.props as { ddl: string }).ddl).toBe("GRANT EXECUTE ON FUNCTION app.total(int) TO app_reader");
37
+ });
38
+
39
+ test("against a build from before the grant, it plans as one naming the signature does", () => {
40
+ const before = pgSchemaFromBuildOutput(buildOf([{ export: "app", entity: app }, { export: "total", entity: f }]));
41
+ const plan = (g: { entityType: string; props: unknown }) =>
42
+ diffPgSchemas(before, pgSchemaFromBuildOutput(buildOf([{ export: "app", entity: app }, { export: "total", entity: f }, { export: "run", entity: g }])), { major: 18 }).changes.map((c) => [c.object, c.rule, c.after]);
43
+ const interpolated = plan(grant`GRANT EXECUTE ON FUNCTION ${f} TO ${reader}`);
44
+ expect(interpolated).toEqual(plan(grant`GRANT EXECUTE ON FUNCTION app.total(int) TO app_reader`));
45
+ expect(interpolated.map(([, rule]) => rule)).toContain("SQLPG296");
46
+ });
47
+ });
48
+
24
49
  describe("the tags", () => {
25
50
  test("a table's row-level security, a policy, a role", () => {
26
51
  expect(orders.props).toMatchObject({ rowSecurity: true, forceRowSecurity: true });
@@ -0,0 +1,39 @@
1
+ /**
2
+ * The privilege changes a connected server needs (#3681): what `chant sql
3
+ * plan` reports, and what the deep read reports as pending so the plan digest
4
+ * an approval binds to moves with a privilege granted or revoked by hand
5
+ * (#3706).
6
+ */
7
+
8
+ import type { PostgresClient } from "../live/client";
9
+ import type { CanonicalPgObject } from "../plan/normalize";
10
+ import { declaredAccess, diffAccess, predictedAccess, targetKey, type AccessChange } from "./acl";
11
+ import { readLiveAccess } from "./live";
12
+
13
+ /** The role a session runs as: the one whose own privileges owning an object gives, and whose default privileges apply. */
14
+ async function currentRole(client: PostgresClient): Promise<{ self?: string }> {
15
+ const rows = await client.query<{ self: string }>("SELECT current_user AS self").catch(() => []);
16
+ return rows[0]?.self ? { self: rows[0].self } : {};
17
+ }
18
+
19
+ /**
20
+ * The changes from the access the server holds to the access the
21
+ * declarations (keyed by export name, in the build's order) add up to. An
22
+ * object the server does not hold yet is taken to have what the server gives
23
+ * a new one. `scope` limits the default privileges read to those schemas.
24
+ */
25
+ export async function accessChangesAgainst(
26
+ client: PostgresClient,
27
+ declared: ReadonlyArray<{ key: string; canonical: CanonicalPgObject }>,
28
+ options: { major: number; scope?: readonly string[] },
29
+ ): Promise<AccessChange[]> {
30
+ const want = declaredAccess(declared, { major: options.major, ...(await currentRole(client)) });
31
+ const have = await readLiveAccess(client, want.targets, {
32
+ ...(options.scope ? { schemas: options.scope } : {}),
33
+ forRoles: want.forRoles,
34
+ declaredDefaults: want.targets.filter((t) => t.kind === "default"),
35
+ });
36
+ const missing = want.targets.filter((t) => t.kind !== "default" && t.kind !== "column" && !have.present.has(targetKey(t)));
37
+ const now = new Map([...have.state, ...predictedAccess(missing, have)]);
38
+ return diffAccess(now, want.state, want.exportsOf);
39
+ }
@@ -70,7 +70,7 @@ import type { OwnershipMarker } from "@intentius/chant/ownership";
70
70
  import type { ApplyRef } from "@intentius/chant/apply";
71
71
  import { SqlApplyError, dependencyFailedDetail, missingDependencies, readBuildObjects, type SqlApplyOutcome } from "../../core/apply";
72
72
  import { carriesMarker, isChantManaged } from "../../core/ownership";
73
- import { PostgresQueryError, type PostgresClient } from "../live/client";
73
+ import { DEFAULT_LOCK_TIMEOUT_MS, PostgresQueryError, withCatalogLockTimeout, type PostgresClient } from "../live/client";
74
74
  import type { PostgresTarget } from "../live/bind";
75
75
  import type { LivePgObject, readLiveSchema } from "../live/catalog";
76
76
  import { liveProps } from "../plan/deep";
@@ -98,7 +98,7 @@ export interface PostgresApplyTimeouts {
98
98
  scanTimeoutMs: number;
99
99
  }
100
100
 
101
- export const DEFAULT_POSTGRES_APPLY_TIMEOUTS: PostgresApplyTimeouts = { lockTimeoutMs: 5_000, statementTimeoutMs: 60_000, scanTimeoutMs: 0 };
101
+ export const DEFAULT_POSTGRES_APPLY_TIMEOUTS: PostgresApplyTimeouts = { lockTimeoutMs: DEFAULT_LOCK_TIMEOUT_MS, statementTimeoutMs: 60_000, scanTimeoutMs: 0 };
102
102
 
103
103
  /** One statement as it ran: its object, the timeouts it ran with, and the transaction it was in. */
104
104
  export interface RanStatement {
@@ -254,8 +254,10 @@ export async function applyPostgres(
254
254
  };
255
255
 
256
256
  // ── Plan: the same diff `chant sql plan` shows. ──
257
+ // The catalog reads wait under lockTimeoutMs, as a plan's do (#3726): one blocked behind ACCESS EXCLUSIVE fails naming the holder instead of hanging.
258
+ const reader = await withCatalogLockTimeout(client, timeouts.lockTimeoutMs);
257
259
  const build: PgSchemaObject[] = declared.map((o) => ({ key: o.exportName, canonical: o.canonical }));
258
- const plan = await planAgainstClient(client, target, build, {
260
+ const plan = await planAgainstClient(reader, target, build, {
259
261
  ...(opts.major !== undefined ? { major: opts.major } : {}),
260
262
  ...(opts.readLive ? { readLive: opts.readLive } : {}),
261
263
  ...(opts.serverNormalize ? { serverNormalize: opts.serverNormalize } : {}),
@@ -281,7 +283,7 @@ export async function applyPostgres(
281
283
  for (const m of matchPgObjects(plan.live, plan.declared)) if (m.kind === "matched") matchedLive.add(m.after.key);
282
284
 
283
285
  // An extension's own comment stays under the trailer when the declaration sets none.
284
- const extensionComments = await extensionDefaults(client, objects.filter((o) => o.canonical.kind === "extension" && !matchedLive.has(o.key)).map((o) => o.canonical.name));
286
+ const extensionComments = await extensionDefaults(reader, objects.filter((o) => o.canonical.kind === "extension" && !matchedLive.has(o.key)).map((o) => o.canonical.name));
285
287
 
286
288
  // ── Each object's statements, or its verdict when it has none to send. ──
287
289
  const statements = planPgStatements({
@@ -454,7 +456,7 @@ export async function applyPostgres(
454
456
  }
455
457
  await commit();
456
458
 
457
- if (managed) await applyAccess(client, declared, onServer, exportNames, { ...opts, major: plan.major ?? opts.major ?? POSTGRES_LATEST_MAJOR, ...(plan.scope ? { schemas: plan.scope } : {}) }, timeouts, outcome, log);
459
+ if (managed) await applyAccess(client, reader, declared, onServer, exportNames, { ...opts, major: plan.major ?? opts.major ?? POSTGRES_LATEST_MAJOR, ...(plan.scope ? { schemas: plan.scope } : {}) }, timeouts, outcome, log);
458
460
 
459
461
  if (opts.prune) await prune(client, statements.drops, liveRawByKey, opts, timeouts, outcome, log);
460
462
  if (outcome.failed.length > 0) throw new PostgresApplyError(outcome);
@@ -470,6 +472,7 @@ export async function applyPostgres(
470
472
  */
471
473
  async function applyAccess(
472
474
  client: PostgresClient,
475
+ reader: PostgresClient,
473
476
  declared: readonly DeclaredPgObject[],
474
477
  onServer: ReadonlySet<string>,
475
478
  exportNames: ReadonlySet<string>,
@@ -487,12 +490,12 @@ async function applyAccess(
487
490
  else ready.push(o);
488
491
  }
489
492
  const skipped = new Set(decls.filter((o) => !ready.includes(o)).map((o) => o.exportName));
490
- const [me] = await client.query<{ self: string }>("SELECT current_user AS self");
493
+ const [me] = await reader.query<{ self: string }>("SELECT current_user AS self");
491
494
  const want = declaredAccess(
492
495
  declared.filter((o) => !skipped.has(o.exportName)).map((o) => ({ key: o.exportName, canonical: o.canonical })),
493
496
  { major: opts.major, ...(me?.self ? { self: me.self } : {}) },
494
497
  );
495
- const have = await readLiveAccess(client, want.targets, {
498
+ const have = await readLiveAccess(reader, want.targets, {
496
499
  ...(opts.schemas ? { schemas: opts.schemas } : {}),
497
500
  forRoles: want.forRoles,
498
501
  declaredDefaults: want.targets.filter((t) => t.kind === "default"),
@@ -1048,7 +1048,7 @@ function grantProps(ctx: Ctx, node: GrantNode, ddl: string, source: { strings: s
1048
1048
  if (o.args) sqlName += `(${req(text(ctx, o.args, "objects")) ?? ""})`;
1049
1049
  else if (typeof t !== "string" && (t.entityType === POSTGRES_ENTITY_TYPES.function || t.entityType === POSTGRES_ENTITY_TYPES.procedure)) {
1050
1050
  // An interpolated routine is named by its own parameter types.
1051
- sqlName += `(${(t.props as RoutineProps).args.filter((a) => a.mode !== "out").map((a) => a.type).join(", ")})`;
1051
+ sqlName += `(${routineArgTypes(t.props as RoutineProps)})`;
1052
1052
  } else throw new SqlTemplateError("grant", `${sqlName}: name the parameter types (${sqlName}(...)), or interpolate the routine, since overloads share a name`, 0, 0);
1053
1053
  }
1054
1054
  objectNames.push(sqlName);
@@ -1071,6 +1071,22 @@ function grantProps(ctx: Ctx, node: GrantNode, ddl: string, source: { strings: s
1071
1071
  }) as GrantProps;
1072
1072
  }
1073
1073
 
1074
+ /** A routine's parameter types, as `GRANT ... ON FUNCTION f(...)` names it: the `OUT` ones left out. */
1075
+ const routineArgTypes = (props: RoutineProps): string => props.args.filter((a) => a.mode !== "out").map((a) => a.type).join(", ");
1076
+
1077
+ /** The interpolated functions and procedures not followed by their own `(...)`: the ones a grant's statement writes with the parameter types (#3707). */
1078
+ function bareRoutineSplices(tokens: readonly Token[], values: readonly unknown[]): Set<number> {
1079
+ const out = new Set<number>();
1080
+ tokens.forEach((t, k) => {
1081
+ if (t.kind !== "ref") return;
1082
+ const v = values[t.part];
1083
+ if (!isPostgresObject(v) || (v.entityType !== POSTGRES_ENTITY_TYPES.function && v.entityType !== POSTGRES_ENTITY_TYPES.procedure)) return;
1084
+ const next = tokens.slice(k + 1).find((x) => !isTrivia(x));
1085
+ if (!(next?.kind === "punct" && next.text === "(")) out.add(t.part);
1086
+ });
1087
+ return out;
1088
+ }
1089
+
1074
1090
  function build(tag: PostgresTag, strings: TemplateStringsArray | readonly string[], rawValues: readonly unknown[]): PostgresObject {
1075
1091
  const rawParts = templateParts(strings);
1076
1092
  const routine = tag === "func" || tag === "procedure";
@@ -1113,9 +1129,12 @@ function build(tag: PostgresTag, strings: TemplateStringsArray | readonly string
1113
1129
  const rowSecurity = rest.filter((n): n is RowSecurityNode => n.statement === "rowSecurity");
1114
1130
 
1115
1131
  const positions = regclassPositions(tokens, values);
1132
+ const bareRoutines = tag === "grant" ? bareRoutineSplices(tokens, values) : new Set<number>();
1116
1133
  const render = (v: unknown, i?: number): string => {
1117
1134
  const at = i === undefined ? undefined : positions.get(i);
1118
1135
  if (at && isPostgresObject(v)) return at === "call" ? `${regclassText(v.sqlName)}::regclass` : regclassText(v.sqlName);
1136
+ // #3707: a routine a grant interpolates is written with its parameter types, so the statement names it on its own.
1137
+ if (i !== undefined && bareRoutines.has(i) && isPostgresObject(v)) return `${renderReference(v)}(${routineArgTypes(v.props as RoutineProps)})`;
1119
1138
  return renderReference(v);
1120
1139
  };
1121
1140
  const ctx: Ctx = { d: { ...POSTGRES_TEMPLATES, renderReference: render }, tokens, values, fed: values.map(() => new Set<string>()) };
@@ -6,7 +6,7 @@
6
6
 
7
7
  import type { ChantConfig } from "@intentius/chant/config";
8
8
  import type { UnobservedReason } from "@intentius/chant/observation";
9
- import { connectPostgres, PostgresQueryError, type PostgresClient, type PostgresEndpoint } from "./client";
9
+ import { connectPostgres, DEFAULT_LOCK_TIMEOUT_MS, PostgresQueryError, withCatalogLockTimeout, type PostgresClient, type PostgresEndpoint } from "./client";
10
10
  import type { PostgresProvider } from "../providers/types";
11
11
  import { credentialSource, TokenSourceError } from "../../token-source";
12
12
 
@@ -139,11 +139,23 @@ export async function resolveBoundTarget(options: BindOptions = {}): Promise<Pos
139
139
  return target;
140
140
  }
141
141
 
142
- /** Resolve the environment's server and connect to it. The caller ends the client. */
142
+ /**
143
+ * Resolve the environment's server and connect to it. The caller ends the client.
144
+ *
145
+ * The session reads under the profile's `lockTimeoutMs` (the applier's 5000 ms
146
+ * otherwise): a catalog read blocked behind ACCESS EXCLUSIVE fails with the
147
+ * relation and the pid that holds it, instead of waiting for as long as the
148
+ * lock is held.
149
+ */
143
150
  export async function bindPostgres(options: BindOptions = {}): Promise<{ target: PostgresTarget; client: PostgresClient }> {
144
151
  const target = await resolveBoundTarget(options);
145
- const client = await (options.connect ?? connectPostgres)(target.endpoint);
146
- return { target, client };
152
+ const raw = await (options.connect ?? connectPostgres)(target.endpoint);
153
+ try {
154
+ return { target, client: await withCatalogLockTimeout(raw, target.timeouts?.lockTimeoutMs ?? DEFAULT_LOCK_TIMEOUT_MS) };
155
+ } catch (err) {
156
+ await raw.end();
157
+ throw err;
158
+ }
147
159
  }
148
160
 
149
161
  /** SQLSTATEs that mean the server refused who we are: invalid_authorization_specification, invalid_password, insufficient_privilege. */
@@ -44,26 +44,14 @@ import { quoteIdent } from "../keywords";
44
44
  import { hasChantTrailerKey, RECEIPTS_TRAILER_KEY, stripMarker } from "../../core/ownership";
45
45
  import { MIGRATION_TRAILER_KEY } from "../migrate/names";
46
46
  import { isProviderOwned, providerData } from "../providers";
47
+ import { FOREIGN_TABLES } from "../../core/foreign-tables";
47
48
  import type { PostgresProvider } from "../providers/types";
48
49
 
49
50
  /** The server's own schemas, never part of a declared schema. */
50
51
  export const SYSTEM_SCHEMAS = ["pg_catalog", "information_schema", "pg_toast"];
51
52
 
52
- /** Tables another tool keeps its migration history in, by name, with the tool. */
53
- export const FOREIGN_TABLES: Readonly<Record<string, string>> = {
54
- _prisma_migrations: "Prisma Migrate",
55
- schema_migrations: "a migration runner (Rails, golang-migrate, dbmate)",
56
- ar_internal_metadata: "Rails",
57
- django_migrations: "Django",
58
- alembic_version: "Alembic",
59
- __drizzle_migrations: "drizzle-kit",
60
- flyway_schema_history: "Flyway",
61
- goose_db_version: "goose",
62
- knex_migrations: "Knex",
63
- knex_migrations_lock: "Knex",
64
- SequelizeMeta: "Sequelize",
65
- pgmigrations: "node-pg-migrate",
66
- };
53
+ /** Tables another tool keeps its migration history in, by name, with the tool; both dialects share the list. */
54
+ export { FOREIGN_TABLES };
67
55
 
68
56
  export interface LivePgObject {
69
57
  type: PostgresEntityType;
@@ -90,3 +90,88 @@ export async function connectPostgres(endpoint: PostgresEndpoint, options: { app
90
90
  },
91
91
  };
92
92
  }
93
+
94
+ /** SQLSTATE lock_not_available: `lock_timeout` passed while a statement waited for a lock. */
95
+ export const SQLSTATE_LOCK_NOT_AVAILABLE = "55P03";
96
+
97
+ /** The `lock_timeout` a statement waits under when the profile sets none (`sql.profiles.<env>.lockTimeoutMs`): the applier's and the catalog reader's. */
98
+ export const DEFAULT_LOCK_TIMEOUT_MS = 5_000;
99
+
100
+ /** A session holding ACCESS EXCLUSIVE on a relation: what a catalog read waits behind. */
101
+ export interface LockHolder {
102
+ pid: number;
103
+ relation: string;
104
+ application?: string;
105
+ state?: string;
106
+ /** Seconds since the holder's transaction began. */
107
+ seconds?: number;
108
+ query?: string;
109
+ }
110
+
111
+ /**
112
+ * The sessions holding ACCESS EXCLUSIVE on a relation in the current database,
113
+ * other than this one. Only that mode conflicts with the ACCESS SHARE a
114
+ * catalog function (`pg_get_viewdef()`, `pg_get_indexdef()`) takes. Empty when
115
+ * the lookup itself fails.
116
+ */
117
+ export async function accessExclusiveHolders(client: PostgresClient): Promise<LockHolder[]> {
118
+ try {
119
+ const rows = await client.query<{ pid: number; relation: string; application: string | null; state: string | null; seconds: number | null; query: string | null }>(
120
+ `SELECT l.pid, l.relation::pg_catalog.regclass::text AS relation, a.application_name AS application, a.state,
121
+ (EXTRACT(EPOCH FROM pg_catalog.now() - a.xact_start))::int AS seconds, pg_catalog.left(a.query, 120) AS query
122
+ FROM pg_catalog.pg_locks l LEFT JOIN pg_catalog.pg_stat_activity a ON a.pid = l.pid
123
+ WHERE l.locktype = 'relation' AND l.granted AND l.mode = 'AccessExclusiveLock'
124
+ AND l.pid <> pg_catalog.pg_backend_pid()
125
+ AND l.database = (SELECT oid FROM pg_catalog.pg_database WHERE datname = pg_catalog.current_database())
126
+ ORDER BY l.pid, 2 LIMIT 10`,
127
+ );
128
+ return rows.map((r) => ({
129
+ pid: Number(r.pid),
130
+ relation: r.relation,
131
+ ...(r.application ? { application: r.application } : {}),
132
+ ...(r.state ? { state: r.state } : {}),
133
+ ...(r.seconds !== null && r.seconds !== undefined ? { seconds: Number(r.seconds) } : {}),
134
+ ...(r.query ? { query: r.query.replace(/\s+/g, " ").trim() } : {}),
135
+ }));
136
+ } catch {
137
+ return [];
138
+ }
139
+ }
140
+
141
+ /** One line naming the lock holders, for a message. */
142
+ export function describeLockHolders(holders: readonly LockHolder[]): string {
143
+ if (holders.length === 0) return "no session holds ACCESS EXCLUSIVE now; the lock was released after the wait";
144
+ return holders
145
+ .map((h) => {
146
+ const about = [h.application ? `application ${JSON.stringify(h.application)}` : "", h.state ?? "", h.seconds !== undefined ? `transaction open ${h.seconds}s` : "", h.query ? `last statement: ${h.query}` : ""].filter(Boolean);
147
+ return `${h.relation} is held in ACCESS EXCLUSIVE by pid ${h.pid}${about.length > 0 ? ` (${about.join(", ")})` : ""}`;
148
+ })
149
+ .join("; ");
150
+ }
151
+
152
+ /**
153
+ * A client for catalog reads: the session's `lock_timeout` is `lockTimeoutMs`
154
+ * (0 waits without limit, as Postgres does), so a read that needs a table
155
+ * another session holds in ACCESS EXCLUSIVE (a long `ALTER TABLE`, `LOCK
156
+ * TABLE`) fails after the wait instead of hanging. The failure names the
157
+ * relation and the pid holding it.
158
+ */
159
+ export async function withCatalogLockTimeout(client: PostgresClient, lockTimeoutMs: number = DEFAULT_LOCK_TIMEOUT_MS): Promise<PostgresClient> {
160
+ await client.query("SELECT pg_catalog.set_config('lock_timeout', $1, false)", [`${lockTimeoutMs}ms`]);
161
+ return {
162
+ async query<T>(sql: string, params?: readonly unknown[]): Promise<T[]> {
163
+ try {
164
+ return await client.query<T>(sql, params);
165
+ } catch (err) {
166
+ if (!(err instanceof PostgresQueryError) || err.code !== SQLSTATE_LOCK_NOT_AVAILABLE) throw err;
167
+ const holders = describeLockHolders(await accessExclusiveHolders(client));
168
+ throw new PostgresQueryError(
169
+ `${err.message.split("\n")[0]}: the catalog read waited ${lockTimeoutMs}ms for a lock; ${holders}. Wait for that transaction to finish, or raise sql.profiles.<env>.lockTimeoutMs`,
170
+ err.code,
171
+ err.position,
172
+ );
173
+ }
174
+ },
175
+ end: () => client.end(),
176
+ };
177
+ }
@@ -196,3 +196,57 @@ describe("a managed provider's own objects (sql.provider)", () => {
196
196
  expect(r.resources.authUsers).toMatchObject({ ownership: "foreign", attributes: { keptBy: "Supabase" } });
197
197
  });
198
198
  });
199
+
200
+ describe("catalog reads under lock_timeout (#3678)", () => {
201
+ /** A fake whose view read waits behind a lock and times out, and whose pg_locks names the holder. */
202
+ const blocked = (log: string[]) => {
203
+ const inner = fakeClient(CATALOG, log);
204
+ return {
205
+ async query<T>(sql: string, params?: readonly unknown[]): Promise<T[]> {
206
+ if (sql.includes("c.relkind IN ('v', 'm')")) {
207
+ log.push(sql);
208
+ throw new PostgresQueryError("canceling statement due to lock timeout", "55P03");
209
+ }
210
+ if (sql.includes("pg_catalog.pg_locks")) {
211
+ log.push(sql);
212
+ return [{ pid: 4242, relation: "app.orders", application: "psql", state: "idle in transaction", seconds: 12, query: "LOCK TABLE app.orders IN ACCESS EXCLUSIVE MODE" }] as T[];
213
+ }
214
+ return inner.query<T>(sql, params);
215
+ },
216
+ end: () => inner.end(),
217
+ };
218
+ };
219
+
220
+ test("the session's lock_timeout is the profile's lockTimeoutMs, else the applier's 5000 ms", async () => {
221
+ for (const [lockTimeoutMs, expected] of [[undefined, "5000ms"], [250, "250ms"]] as const) {
222
+ const log: string[] = [];
223
+ const params: unknown[][] = [];
224
+ const cfg = { sql: { profiles: { prod: { ...config.sql.profiles.prod, ...(lockTimeoutMs !== undefined ? { lockTimeoutMs } : {}) } } } };
225
+ await describeResources({
226
+ ...options({ users: DECLARED.users }),
227
+ config: cfg,
228
+ connect: async () => {
229
+ const c = fakeClient(CATALOG, log);
230
+ return { query: (sql: string, p?: readonly unknown[]) => (params.push([sql, ...(p ?? [])]), c.query(sql, p)), end: c.end } as never;
231
+ },
232
+ });
233
+ expect(params[0]).toEqual(["SELECT pg_catalog.set_config('lock_timeout', $1, false)", expected]);
234
+ }
235
+ });
236
+
237
+ test("a read that times out on a lock is read-failed, naming the relation and the pid holding it", async () => {
238
+ const log: string[] = [];
239
+ const r = normalizeObservation(await describeResources({ ...options({ users: DECLARED.users }), connect: async () => blocked(log) as never }));
240
+ const users = r.unobserved.users;
241
+ expect(users?.reason).toBe("read-failed");
242
+ expect(users?.detail).toContain("canceling statement due to lock timeout");
243
+ expect(users?.detail).toContain("app.orders is held in ACCESS EXCLUSIVE by pid 4242");
244
+ expect(users?.detail).toContain("lockTimeoutMs");
245
+ });
246
+
247
+ test("other failures pass through unchanged", async () => {
248
+ const { withCatalogLockTimeout } = await import("./client");
249
+ const c = await withCatalogLockTimeout({ query: async (sql: string) => { if (sql.startsWith("SELECT 1")) throw new PostgresQueryError("boom", "XX000"); return []; }, end: async () => {} }, 100);
250
+ await expect(c.query("SELECT 1")).rejects.toThrow(/^boom$/);
251
+ });
252
+ });