@intentius/chant-lexicon-sql 0.118.1 → 0.120.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/dist/clickhouse/apply/apply.d.ts +1 -1
  2. package/dist/clickhouse/apply/apply.d.ts.map +1 -1
  3. package/dist/clickhouse/apply/statements.d.ts +8 -0
  4. package/dist/clickhouse/apply/statements.d.ts.map +1 -1
  5. package/dist/clickhouse/import/generator.d.ts +1 -1
  6. package/dist/clickhouse/import/generator.d.ts.map +1 -1
  7. package/dist/clickhouse/import/live-export.d.ts +19 -0
  8. package/dist/clickhouse/import/live-export.d.ts.map +1 -1
  9. package/dist/clickhouse/live/bind.d.ts +8 -0
  10. package/dist/clickhouse/live/bind.d.ts.map +1 -1
  11. package/dist/clickhouse/live/catalog.d.ts +15 -0
  12. package/dist/clickhouse/live/catalog.d.ts.map +1 -1
  13. package/dist/clickhouse/live/describe-resources.d.ts.map +1 -1
  14. package/dist/clickhouse/plan/commands.d.ts.map +1 -1
  15. package/dist/clickhouse/plan/deep.d.ts +19 -2
  16. package/dist/clickhouse/plan/deep.d.ts.map +1 -1
  17. package/dist/clickhouse/plan/diff.d.ts.map +1 -1
  18. package/dist/clickhouse/plan/normalize.d.ts +3 -0
  19. package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
  20. package/dist/clickhouse/plan/rebuild-handoff.d.ts +2 -0
  21. package/dist/clickhouse/plan/rebuild-handoff.d.ts.map +1 -1
  22. package/dist/clickhouse/rebuild/engines.d.ts +44 -0
  23. package/dist/clickhouse/rebuild/engines.d.ts.map +1 -0
  24. package/dist/clickhouse/rebuild/verify.d.ts +25 -1
  25. package/dist/clickhouse/rebuild/verify.d.ts.map +1 -1
  26. package/dist/clickhouse/testing/fake-server.d.ts +5 -0
  27. package/dist/clickhouse/testing/fake-server.d.ts.map +1 -1
  28. package/dist/config.d.ts +32 -8
  29. package/dist/config.d.ts.map +1 -1
  30. package/dist/core/foreign-tables.d.ts +13 -0
  31. package/dist/core/foreign-tables.d.ts.map +1 -0
  32. package/dist/files/clickhouse.d.ts +7 -3
  33. package/dist/files/clickhouse.d.ts.map +1 -1
  34. package/dist/files/postgres.d.ts +3 -0
  35. package/dist/files/postgres.d.ts.map +1 -1
  36. package/dist/import-parser.d.ts.map +1 -1
  37. package/dist/integrity.json +2 -2
  38. package/dist/manifest.json +1 -1
  39. package/dist/op/activities/clickhouse-emulator.d.ts +7 -1
  40. package/dist/op/activities/clickhouse-emulator.d.ts.map +1 -1
  41. package/dist/op/activities/postgres-emulator.d.ts +8 -1
  42. package/dist/op/activities/postgres-emulator.d.ts.map +1 -1
  43. package/dist/postgres/access/changes.d.ts +23 -0
  44. package/dist/postgres/access/changes.d.ts.map +1 -0
  45. package/dist/postgres/apply/apply.d.ts.map +1 -1
  46. package/dist/postgres/entities.d.ts.map +1 -1
  47. package/dist/postgres/live/bind.d.ts +8 -1
  48. package/dist/postgres/live/bind.d.ts.map +1 -1
  49. package/dist/postgres/live/catalog.d.ts +3 -3
  50. package/dist/postgres/live/catalog.d.ts.map +1 -1
  51. package/dist/postgres/live/client.d.ts +31 -0
  52. package/dist/postgres/live/client.d.ts.map +1 -1
  53. package/dist/postgres/plan/commands.d.ts.map +1 -1
  54. package/dist/postgres/plan/deep.d.ts.map +1 -1
  55. package/dist/sql-files.d.ts +2 -1
  56. package/dist/sql-files.d.ts.map +1 -1
  57. package/package.json +2 -2
  58. package/src/clickhouse/apply/apply.ts +8 -2
  59. package/src/clickhouse/apply/statements.ts +32 -5
  60. package/src/clickhouse/import/generator.ts +8 -3
  61. package/src/clickhouse/import/import.test.ts +64 -0
  62. package/src/clickhouse/import/live-export.ts +70 -2
  63. package/src/clickhouse/live/access.e2e.test.ts +71 -18
  64. package/src/clickhouse/live/bind.ts +10 -0
  65. package/src/clickhouse/live/catalog.ts +25 -2
  66. package/src/clickhouse/live/describe-resources.ts +5 -2
  67. package/src/clickhouse/live/live.test.ts +8 -0
  68. package/src/clickhouse/plan/commands.ts +6 -3
  69. package/src/clickhouse/plan/deep-grants.test.ts +67 -0
  70. package/src/clickhouse/plan/deep.ts +70 -6
  71. package/src/clickhouse/plan/diff.ts +4 -1
  72. package/src/clickhouse/plan/normalize.ts +5 -2
  73. package/src/clickhouse/plan/plan.test.ts +9 -1
  74. package/src/clickhouse/plan/rebuild-handoff.ts +11 -1
  75. package/src/clickhouse/rebuild/engines.ts +163 -0
  76. package/src/clickhouse/rebuild/rebuild-collapsing.e2e.test.ts +120 -0
  77. package/src/clickhouse/rebuild/rebuild-into-collapsing.e2e.test.ts +198 -0
  78. package/src/clickhouse/rebuild/rebuild.test.ts +78 -0
  79. package/src/clickhouse/rebuild/verify.ts +153 -24
  80. package/src/clickhouse/testing/fake-server.ts +3 -2
  81. package/src/codegen/classifier-page.ts +1 -1
  82. package/src/config.ts +16 -4
  83. package/src/core/foreign-tables.ts +28 -0
  84. package/src/files/clickhouse.ts +71 -5
  85. package/src/files/postgres.ts +2 -1
  86. package/src/import-parser.ts +2 -0
  87. package/src/migration-statements.test.ts +33 -0
  88. package/src/migration-statements.ts +2 -0
  89. package/src/op/__snapshots__/watch-pipeline.test.ts.snap +1 -1
  90. package/src/op/activities/clickhouse-emulator.ts +22 -1
  91. package/src/op/activities/postgres-apply-lock-timeout.e2e.test.ts +87 -0
  92. package/src/op/activities/postgres-apply.test.ts +32 -0
  93. package/src/op/activities/postgres-emulator.test.ts +20 -0
  94. package/src/op/activities/postgres-emulator.ts +41 -1
  95. package/src/op/apply-gate-access-clickhouse.e2e.test.ts +195 -0
  96. package/src/op/apply-gate-access.e2e.test.ts +207 -0
  97. package/src/postgres/access/access.test.ts +26 -1
  98. package/src/postgres/access/changes.ts +39 -0
  99. package/src/postgres/apply/apply.ts +10 -7
  100. package/src/postgres/entities.ts +20 -1
  101. package/src/postgres/live/bind.ts +16 -4
  102. package/src/postgres/live/catalog.ts +3 -15
  103. package/src/postgres/live/client.ts +85 -0
  104. package/src/postgres/live/live.test.ts +54 -0
  105. package/src/postgres/live/lock-timeout.e2e.test.ts +70 -0
  106. package/src/postgres/plan/commands.ts +2 -14
  107. package/src/postgres/plan/deep.ts +44 -2
  108. package/src/serializer.test.ts +4 -0
  109. package/src/skills/chant-sql-postgres-plan.md +1 -1
  110. package/src/skills/chant-sql-rebuild.md +2 -0
  111. package/src/skills/chant-sql.md +1 -1
  112. package/src/sql-files.test.ts +78 -1
  113. package/src/sql-files.ts +2 -1
@@ -0,0 +1,70 @@
1
+ /**
2
+ * Catalog reads under `lock_timeout` (#3678). One session holds ACCESS
3
+ * EXCLUSIVE on a table behind a view; the observation of the view, which
4
+ * reads its definition with `pg_get_viewdef()`, fails after the profile's
5
+ * `lockTimeoutMs` instead of waiting for the lock, and names the table and the
6
+ * pid holding it.
7
+ *
8
+ * `CHANT_SQL_E2E_POSTGRES_URL` (with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names
9
+ * a running server, else a throwaway one, skipped when Docker is not
10
+ * available. The test makes only the schema `chant_e2e_3678`, dropped
11
+ * afterwards.
12
+ */
13
+
14
+ import { afterAll, beforeAll, describe, expect, test } from "vitest";
15
+ import { normalizeObservation } from "@intentius/chant/observation";
16
+ import { dockerAvailable, startTestPostgres, type TestPostgres } from "../testing/server";
17
+ import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "./client";
18
+ import { describeResources } from "./describe-resources";
19
+
20
+ const SCHEMA = "chant_e2e_3678";
21
+ const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
22
+ const enabled = given ? true : await dockerAvailable();
23
+ let server: TestPostgres | undefined;
24
+ let endpoint: PostgresEndpoint;
25
+ let admin: PostgresClient;
26
+ let holder: PostgresClient;
27
+
28
+ describe.skipIf(!enabled)("a catalog read blocked behind ACCESS EXCLUSIVE", () => {
29
+ beforeAll(async () => {
30
+ if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
31
+ else {
32
+ server = await startTestPostgres();
33
+ endpoint = server.endpoint();
34
+ }
35
+ admin = await connectPostgres(endpoint);
36
+ await admin.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);
37
+ await admin.query(`CREATE SCHEMA ${SCHEMA}`);
38
+ await admin.query(`CREATE TABLE ${SCHEMA}.orders (id bigint PRIMARY KEY)`);
39
+ await admin.query(`CREATE VIEW ${SCHEMA}.recent AS SELECT id FROM ${SCHEMA}.orders`);
40
+ holder = await connectPostgres(endpoint, { applicationName: "chant-e2e-3678-holder" });
41
+ await holder.query("BEGIN");
42
+ await holder.query(`LOCK TABLE ${SCHEMA}.orders IN ACCESS EXCLUSIVE MODE`);
43
+ }, 600_000);
44
+
45
+ afterAll(async () => {
46
+ await holder?.query("ROLLBACK").catch(() => undefined);
47
+ await holder?.end();
48
+ await admin?.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`).catch(() => undefined);
49
+ await admin?.end();
50
+ await server?.stop();
51
+ });
52
+
53
+ test("fails after lockTimeoutMs, naming the relation and the pid", async () => {
54
+ const [{ pid }] = await holder.query<{ pid: number }>("SELECT pg_catalog.pg_backend_pid() AS pid");
55
+ const started = Date.now();
56
+ const r = normalizeObservation(await describeResources({
57
+ environment: "e2e",
58
+ config: { sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_3678_PASSWORD" }, schemas: [SCHEMA], lockTimeoutMs: 300 } } } },
59
+ env: { PG_E2E_3678_PASSWORD: endpoint.password ?? "" },
60
+ entityNames: ["recent"],
61
+ entities: new Map([["recent", { entityType: "Postgres::View", props: { schema: SCHEMA, name: "recent" } }]]),
62
+ }));
63
+ expect(Date.now() - started).toBeLessThan(10_000);
64
+ const recent = r.unobserved.recent;
65
+ expect(recent?.reason).toBe("read-failed");
66
+ expect(recent?.detail).toContain("lock timeout");
67
+ expect(recent?.detail).toContain(`${SCHEMA}.orders is held in ACCESS EXCLUSIVE by pid ${pid}`);
68
+ expect(recent?.detail).toContain("chant-e2e-3678-holder");
69
+ }, 60_000);
70
+ });
@@ -20,8 +20,7 @@ import { POSTGRES_ENTITY_TYPES } from "../entity-types";
20
20
  import { migrationOpSuggestions } from "../migrate/handoff";
21
21
  import { POSTGRES_LATEST_MAJOR } from "../../spec/postgres-pin";
22
22
  import { ACCESS_MANAGED_KINDS } from "./normalize";
23
- import { declaredAccess, diffAccess, predictedAccess, targetKey } from "../access/acl";
24
- import { readLiveAccess } from "../access/live";
23
+ import { accessChangesAgainst } from "../access/changes";
25
24
 
26
25
  export function emitPg(diff: PgSchemaDiff, json: boolean, title: string): number {
27
26
  console.log(json ? JSON.stringify(diff, null, 2) : renderPgDiff(diff, { title }));
@@ -112,12 +111,7 @@ export async function planAgainstClient(
112
111
  const major = running ?? options.major;
113
112
  const diff = diffPgSchemas(live, declared, { major, access: false });
114
113
  if (managed) {
115
- const want = declaredAccess(declared, { major: major ?? POSTGRES_LATEST_MAJOR, ...(await currentRole(client)) });
116
- const have = await readLiveAccess(client, want.targets, { ...(scope ? { schemas: scope } : {}), forRoles: want.forRoles, declaredDefaults: want.targets.filter((t) => t.kind === "default") });
117
- // An object this plan creates is given what the server gives a new one.
118
- const missing = want.targets.filter((t) => t.kind !== "default" && t.kind !== "column" && !have.present.has(targetKey(t)));
119
- const now = new Map([...have.state, ...predictedAccess(missing, have)]);
120
- const access = diffAccess(now, want.state, want.exportsOf);
114
+ const access = await accessChangesAgainst(client, declared, { major: major ?? POSTGRES_LATEST_MAJOR, ...(scope ? { scope } : {}) });
121
115
  if (access.length > 0) {
122
116
  diff.changes.push(...access.map((a) => a.change));
123
117
  diff.access = access;
@@ -131,12 +125,6 @@ export async function planAgainstClient(
131
125
  return { declared, live, liveObjects, diff, ...(major !== undefined ? { major } : {}), ...(scope ? { scope } : {}) };
132
126
  }
133
127
 
134
- /** The role a session runs as: the one whose own privileges owning an object gives, and whose default privileges apply. */
135
- async function currentRole(client: PostgresClient): Promise<{ self?: string }> {
136
- const rows = await client.query<{ self: string }>("SELECT current_user AS self").catch(() => []);
137
- return rows[0]?.self ? { self: rows[0].self } : {};
138
- }
139
-
140
128
  /**
141
129
  * The declarations a plan compares: all of them where the profile manages
142
130
  * access; otherwise none of the policies, roles, grants and default
@@ -13,7 +13,7 @@
13
13
  * of the object (the DDL text, the template source, lineage, reads).
14
14
  */
15
15
 
16
- import { deepObservation, type DeepObservationResult, type DeepResourceObservation } from "@intentius/chant/deep-observation";
16
+ import { deepObservation, type DeepObservationResult, type DeepPendingChange, type DeepResourceObservation } from "@intentius/chant/deep-observation";
17
17
  import type { UnobservedEntity } from "@intentius/chant/lexicon";
18
18
  import { bindPostgres, classifyPostgresFailure, type BindOptions } from "../live/bind";
19
19
  import { liveKey, readLiveSchema, type LivePgObject } from "../live/catalog";
@@ -23,6 +23,8 @@ import { POSTGRES_ENTITY_TYPES } from "../entity-types";
23
23
  import { canonicalPgObject, columnShape, sameConstraint, type CanonicalPgObject } from "./normalize";
24
24
  import { serverNormalized } from "./server-normalize";
25
25
  import type { PostgresClient } from "../live/client";
26
+ import { accessChangesAgainst } from "../access/changes";
27
+ import { POSTGRES_LATEST_MAJOR } from "../../spec/postgres-pin";
26
28
 
27
29
  type Props = Record<string, unknown>;
28
30
 
@@ -214,7 +216,21 @@ export async function observeResourcesDeep(
214
216
  for (const { name, entity } of declared) unobserved[name] = { type: entity?.entityType ?? "", reason: why.reason, detail: why.detail };
215
217
  return deepObservation({}, unobserved);
216
218
  }
219
+ const pending: DeepPendingChange[] = [];
217
220
  try {
221
+ if (target.access === true) {
222
+ // #3706: the privilege changes an apply would make, which no one
223
+ // declaration's properties show, so the plan digest moves with them.
224
+ try {
225
+ pending.push(...(await pendingAccess(client, target.defaultSchema, declared, scopeFor(target, declared.filter((x) => x.entity?.entityType.startsWith("Postgres::")).map((x) => ({ type: x.entity!.entityType, props: x.entity!.props }))))));
226
+ } catch (err) {
227
+ for (const { name, entity } of declared) {
228
+ if (entity?.entityType === POSTGRES_ENTITY_TYPES.grant || entity?.entityType === POSTGRES_ENTITY_TYPES.defaultPrivileges) {
229
+ unobserved[name] = { type: entity.entityType, reason: "read-failed", detail: `the server's privileges could not be compared: ${(err as Error).message}` };
230
+ }
231
+ }
232
+ }
233
+ }
218
234
  const byKey = new Map(live.map((o) => [liveKey(o.type, o.schema, o.name, o.signature), o]));
219
235
  for (const { name, entity } of declared) {
220
236
  if (!entity || !entity.entityType.startsWith("Postgres::")) {
@@ -223,6 +239,7 @@ export async function observeResourcesDeep(
223
239
  }
224
240
  const why = accessUnobserved(entity.entityType, target.access === true);
225
241
  if (why) {
242
+ if (unobserved[name]) continue;
226
243
  unobserved[name] = { type: entity.entityType, ...why };
227
244
  continue;
228
245
  }
@@ -245,5 +262,30 @@ export async function observeResourcesDeep(
245
262
  } finally {
246
263
  await client.end();
247
264
  }
248
- return deepObservation(resources, unobserved);
265
+ return deepObservation(resources, unobserved, pending);
266
+ }
267
+
268
+ /**
269
+ * The privilege changes an apply would make (#3706), as `chant sql plan`
270
+ * computes them, one per object and grantee: the statements, and the
271
+ * declarations behind them.
272
+ */
273
+ async function pendingAccess(
274
+ client: PostgresClient,
275
+ defaultSchema: string,
276
+ declared: ReadonlyArray<{ name: string; entity?: { entityType: string; props: Record<string, unknown> } }>,
277
+ scope: string[] | undefined,
278
+ ): Promise<DeepPendingChange[]> {
279
+ const objects = declared
280
+ .filter((x) => x.entity?.entityType.startsWith("Postgres::"))
281
+ .map((x) => ({ key: x.name, canonical: canonicalPgObject(x.entity!.entityType, x.entity!.props, defaultSchema) }));
282
+ const [row] = await client.query<{ v: string }>("SELECT current_setting('server_version_num') AS v");
283
+ const n = Number(row?.v);
284
+ const major = Number.isFinite(n) && n > 0 ? Math.floor(n / 10000) : POSTGRES_LATEST_MAJOR;
285
+ const changes = await accessChangesAgainst(client, objects, { major, ...(scope ? { scope } : {}) });
286
+ return changes.map((c) => ({
287
+ subject: c.change.object.startsWith("acl ") ? c.change.object.slice(4) : c.change.object,
288
+ change: c.sql.join("; "),
289
+ ...(c.exports.length > 0 ? { entities: [...c.exports].sort() } : {}),
290
+ }));
249
291
  }
@@ -50,6 +50,8 @@ describe("sql serializer", () => {
50
50
  expect(sql.indexOf("CREATE DATABASE")).toBeLessThan(sql.indexOf("CREATE TABLE"));
51
51
  expect(sql.indexOf("CREATE TABLE")).toBeLessThan(sql.indexOf("CREATE VIEW"));
52
52
  expect(r.verbatimFiles).toEqual([CLICKHOUSE_DDL_FILE]);
53
+ // DDL is copied exactly but is not a secret, so it prints with no --output (#3738).
54
+ expect(r.secretFiles).toBeUndefined();
53
55
  });
54
56
 
55
57
  test("10. a declarable of another lexicon is not written", () => {
@@ -98,6 +100,8 @@ describe("sql serializer, Postgres", () => {
98
100
  expect(doc(r).objects[1]).toMatchObject({ export: "users", type: "Postgres::Table", name: "users", schema: "app", sqlName: "app.users" });
99
101
  expect(r.files![POSTGRES_DDL_FILE]).toBe("CREATE SCHEMA app;\n\nCREATE TABLE app.users (id bigint PRIMARY KEY, email text NOT NULL);\n");
100
102
  expect(r.verbatimFiles).toEqual([POSTGRES_DDL_FILE]);
103
+ // DDL is copied exactly but is not a secret, so it prints with no --output (#3738).
104
+ expect(r.secretFiles).toBeUndefined();
101
105
  });
102
106
 
103
107
  test("a Postgres build records the target major: sql.postgresMajor, else the newest pinned", () => {
@@ -167,7 +167,7 @@ Transactions follow the class, so a long scan holds locks on one table and what
167
167
 
168
168
  A primary key or unique constraint added to an existing table goes in two steps, the unique index built `CONCURRENTLY` and then `ADD CONSTRAINT ... USING INDEX` (SQLPG221), so a busy table is never blocked for the build.
169
169
 
170
- Timeouts are set on every statement: `lock_timeout` 5000 ms, so a statement that cannot get its lock fails instead of queueing behind a long transaction and blocking everything behind it; `statement_timeout` 60000 ms for catalog-only statements, and 0 (no limit) for scans, rewrites and `CONCURRENTLY` builds. Set them per environment with `lockTimeoutMs`, `statementTimeoutMs` and `scanTimeoutMs` on `sql.profiles.<env>`, or per call.
170
+ Timeouts are set on every statement: `lock_timeout` 5000 ms, so a statement that cannot get its lock fails instead of queueing behind a long transaction and blocking everything behind it; `statement_timeout` 60000 ms for catalog-only statements, and 0 (no limit) for scans, rewrites and `CONCURRENTLY` builds. Set them per environment with `lockTimeoutMs`, `statementTimeoutMs` and `scanTimeoutMs` on `sql.profiles.<env>`, or per call. The catalog read an apply makes before its first statement waits under the same `lock_timeout`: blocked behind `ACCESS EXCLUSIVE`, the apply stops with nothing written, naming the table and the pid that holds it.
171
171
 
172
172
  ```ts
173
173
  profiles: { prod: { url: "postgres://db.internal:5432/shop", lockTimeoutMs: 2000, scanTimeoutMs: 600000 } }
@@ -57,6 +57,8 @@ The swap gate is bound to a digest of the plan and the verification (row counts
57
57
  - A run stopped during the backfill (Ctrl-C, a killed job) is not a failure. The next run resumes from the receipts in `chant_receipts.receipts` on the same server (in `<db>.__chant_receipts` on a Replicated database), and a partition whose copy was cut off before its receipt is cleared and copied again, so no row is copied twice.
58
58
  - A failed step (a failed copy after its retries, a verification mismatch, a refused declaration) runs onFailure: the new table `<t>__chant_new` and the dual-write view `<t>__chant_dual` are dropped, and the next run starts from a new table. With `onFailure: "keep"` they stay, and the next run resumes; drop them by hand to start over. The original table is untouched until the swap.
59
59
  - A verification mismatch in materialized-view mode usually means rows arrived with a time before the cut-over later than `cutoverDelay`. Raise the delay and run again.
60
+ - On a Summing, Replacing, Aggregating or Collapsing MergeTree the verification compares both tables under `FINAL`, so unmerged parts in the old table are not a mismatch. A mismatch that remains names the engine: run `OPTIMIZE TABLE <db>.<table> FINAL` and run the rebuild again.
61
+ - A plain MergeTree rebuilt into one of those engines is compared per key of the new sorting key, on what the new engine keeps (a Summing table's sums, a Replacing table's highest version, a Collapsing table's sum of signs), so the different row counts are not a mismatch. A missing key or a changed sum still fails.
60
62
  - A backfill that stops naming writes "begun before the cut-over" waited `cutoverTimeout` for INSERTs into the old table that were still running. Let them finish, or stop them, and run again.
61
63
  - The Op refuses a database that is neither Atomic nor Replicated (`EXCHANGE TABLES` needs one of them), a table whose change is not a rebuild (use the applier), and an object already under one of its working names that is not its own. In a Replicated database it also refuses a table, or a declaration, whose engine is not `Replicated*MergeTree`: a plain MergeTree keeps different rows on each replica and one replica's copy would lose the others'.
62
64
 
@@ -37,7 +37,7 @@ A dictionary is a `dictionary` template holding `CREATE DICTIONARY`, with its at
37
37
 
38
38
  A SQL user-defined function is a `func` template holding `CREATE FUNCTION name AS (x) -> expr`, with no database in its name. A changed expression is SQLCH260 (`CREATE OR REPLACE FUNCTION`). It carries no ownership marker, so chant never drops one.
39
39
 
40
- Users, roles, row policies and grants are `user`, `role`, `policy` and `grant` templates. Never write a password: `IDENTIFIED BY` is refused, a user declared without `IDENTIFIED` is created by the environment with its password, and chant manages the rest of it. Grants are compared per grantee: every `grant` naming a grantee adds up to its complete list, and anything else it holds is revoked (SQLCH274). None of these is ever dropped by chant.
40
+ Users, roles, row policies and grants are `user`, `role`, `policy` and `grant` templates. Never write a password: `IDENTIFIED BY` is refused, a user declared without `IDENTIFIED` is created by the environment with its password, and chant manages the rest of it. Grants are compared per grantee: every `grant` naming a grantee adds up to its complete list, and anything else it holds is revoked (SQLCH274). None of these is ever dropped by chant. They are planned and applied only where the profile sets `access: true` (`sql.profiles.<env>.access`); elsewhere each is reported `filtered`.
41
41
 
42
42
  ## Interpolations
43
43
 
@@ -3,7 +3,8 @@ import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
3
3
  import { dirname, join, resolve } from "node:path";
4
4
  import { fileURLToPath } from "node:url";
5
5
  import { build } from "@intentius/chant/build";
6
- import { readSqlFile, sqlFileDeclarations, SqlFileError } from "./sql-files";
6
+ import { readSqlFile, sqlFileDeclarations, sqlFileEntities, SqlFileError } from "./sql-files";
7
+ import { database, grant, policy, role, table, user } from "./clickhouse/entities";
7
8
  import { sqlPlugin } from "./plugin";
8
9
  import { sqlSerializer } from "./serializer";
9
10
  import { guessDialect, SqlFileParser } from "./import-parser";
@@ -63,6 +64,7 @@ describe("chant import of a .sql file", () => {
63
64
  expect(guessDialect("CREATE TABLE a.t (x UInt8) ENGINE = Log")).toBe("clickhouse");
64
65
  expect(guessDialect("CREATE VIEW a.v AS SELECT 1")).toBe("clickhouse");
65
66
  expect(guessDialect("CREATE FUNCTION f AS (x) -> x + 1")).toBe("clickhouse");
67
+ expect(guessDialect("CREATE ROW POLICY p ON a.t USING 1 TO r")).toBe("clickhouse");
66
68
  expect(guessDialect("CREATE FUNCTION app.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$")).toBe("postgres");
67
69
  expect(new SqlFileParser().parse(PG_DDL).resources.map((r) => r.type)).toEqual(["Postgres::Enum", "Postgres::Table", "Postgres::Table", "Postgres::Index"]);
68
70
  });
@@ -102,8 +104,83 @@ CREATE TABLE analytics.counts (kind String, n UInt64) ENGINE = SummingMergeTree
102
104
  CREATE MATERIALIZED VIEW analytics.counts_mv TO analytics.counts AS SELECT kind, count() AS n FROM analytics.events GROUP BY kind;
103
105
  CREATE DICTIONARY analytics.kinds (kind String, label String DEFAULT '') PRIMARY KEY kind SOURCE(clickhouse(table 'counts' db 'analytics')) LAYOUT(complex_key_hashed()) LIFETIME(300);
104
106
  CREATE FUNCTION kind_label AS (k) -> upper(k);
107
+ CREATE ROLE reader SETTINGS max_memory_usage = 1000000000;
108
+ CREATE USER app HOST ANY DEFAULT ROLE reader DEFAULT DATABASE analytics;
109
+ CREATE ROW POLICY own_kinds ON analytics.events USING kind = 'a' TO reader;
110
+ GRANT SELECT(ts, kind) ON analytics.events TO reader;
111
+ GRANT reader TO app;
105
112
  `;
106
113
 
114
+ describe("a file of ClickHouse access control (#3711)", () => {
115
+ const ACCESS = `CREATE DATABASE shop;
116
+ CREATE TABLE shop.events (id UInt64, tenant String) ENGINE = MergeTree ORDER BY id;
117
+ CREATE ROLE reader SETTINGS max_memory_usage = 1000000000;
118
+ CREATE USER app HOST ANY DEFAULT ROLE reader DEFAULT DATABASE shop;
119
+ CREATE ROW POLICY tenant_a ON shop.events USING tenant = 'a' TO reader;
120
+ GRANT SELECT(id, tenant) ON shop.events TO reader;
121
+ GRANT reader TO app;
122
+ `;
123
+ const t = (s: string) => Object.assign([s], { raw: [s] }) as unknown as TemplateStringsArray;
124
+ const shopDb = database(t("CREATE DATABASE shop"));
125
+ const events = table`CREATE TABLE ${shopDb}.events (id UInt64, tenant String) ENGINE = MergeTree ORDER BY id`;
126
+ const reader = role(t("CREATE ROLE reader SETTINGS max_memory_usage = 1000000000"));
127
+ const app = user(t("CREATE USER app HOST ANY DEFAULT ROLE reader DEFAULT DATABASE shop"));
128
+ const tagged = [
129
+ shopDb,
130
+ events,
131
+ reader,
132
+ app,
133
+ policy`CREATE ROW POLICY tenant_a ON ${events} USING tenant = 'a' TO ${reader}`,
134
+ grant`GRANT SELECT(id, tenant) ON ${events} TO ${reader}`,
135
+ grant`GRANT reader TO ${app}`,
136
+ ];
137
+ const props = (e: unknown) => {
138
+ const { source: _source, ...rest } = (e as { props: Record<string, unknown> }).props;
139
+ return JSON.parse(JSON.stringify(rest)) as Record<string, unknown>;
140
+ };
141
+
142
+ test("users, roles, row policies and grants build to the declarations the tags make", () => {
143
+ const read = sqlFileEntities("clickhouse", [{ origin: "access.sql", ddl: ACCESS }]);
144
+ expect([...read.values()].map((e) => e.entityType)).toEqual(tagged.map((e) => e.entityType));
145
+ expect([...read.values()].map(props)).toEqual(tagged.map(props));
146
+ expect(readSqlFile("clickhouse", ACCESS, { origin: "access.sql" }).map((o) => [o.type, o.name])).toEqual([
147
+ ["ClickHouse::Database", "shop"],
148
+ ["ClickHouse::Table", "shop.events"],
149
+ ["ClickHouse::Role", "reader"],
150
+ ["ClickHouse::User", "app"],
151
+ ["ClickHouse::RowPolicy", "shop.tenant_a"],
152
+ ["ClickHouse::Grant", "grant select(id, tenant) on shop.events to reader"],
153
+ ["ClickHouse::Grant", "grant reader to app"],
154
+ ]);
155
+ });
156
+
157
+ test("with a database for unqualified names, a policy's and a grant's table are qualified", () => {
158
+ const objects = readSqlFile("clickhouse", "CREATE ROW POLICY p ON events USING 1 TO reader;\nGRANT SELECT ON events TO reader;\nGRANT SELECT ON *.* TO reader", { origin: "a.sql", schema: "shop" });
159
+ expect(objects.map((o) => o.ddl)).toEqual(["CREATE ROW POLICY p ON shop.events USING 1 TO reader", "GRANT SELECT ON shop.events TO reader", "GRANT SELECT ON *.* TO reader"]);
160
+ });
161
+
162
+ test("a password is refused with the tag's message, and so is a REVOKE", () => {
163
+ let tagMessage = "";
164
+ try {
165
+ user(t("CREATE USER app IDENTIFIED BY 'secret'"));
166
+ } catch (e) {
167
+ tagMessage = (e as Error).message;
168
+ }
169
+ expect(tagMessage).toMatch(/IDENTIFIED BY holds a password/);
170
+ const err = (() => {
171
+ try {
172
+ readSqlFile("clickhouse", "CREATE USER app IDENTIFIED BY 'secret';\nREVOKE SELECT ON shop.events FROM reader", { origin: "users.sql" });
173
+ } catch (e) {
174
+ return e as SqlFileError;
175
+ }
176
+ throw new Error("read");
177
+ })();
178
+ expect(err).toBeInstanceOf(SqlFileError);
179
+ expect(err.problems).toEqual([`${tagMessage}: CREATE USER app IDENTIFIED BY 'secret'`, expect.stringMatching(/holds a REVOKE; declare the grants a grantee keeps.*: REVOKE SELECT ON shop\.events FROM reader$/)]);
180
+ expect(err.message).not.toContain("not a CREATE statement");
181
+ });
182
+ });
183
+
107
184
  describe("chant build reads the .sql files in the source directory", () => {
108
185
  const dirs: string[] = [];
109
186
  afterAll(async () => {
package/src/sql-files.ts CHANGED
@@ -15,7 +15,8 @@
15
15
  * PRIVILEGES` as an object, with `COMMENT ON`, `ALTER TABLE ... ROW LEVEL
16
16
  * SECURITY` and `ALTER TABLE ... ADD <table constraint>` folded into the
17
17
  * object they finish; ClickHouse reads `CREATE DATABASE`, `TABLE`, `VIEW`,
18
- * `MATERIALIZED VIEW`, `DICTIONARY` and `FUNCTION`. Any other statement, and any
18
+ * `MATERIALIZED VIEW`, `DICTIONARY`, `FUNCTION`, `USER`, `ROLE` and `ROW
19
+ * POLICY`, and `GRANT` (#3711). Any other statement, and any
19
20
  * statement that does not parse, throws a {@link SqlFileError} naming every
20
21
  * such statement: none is left out.
21
22
  *