@intentius/chant-lexicon-sql 0.118.1 → 0.120.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clickhouse/apply/apply.d.ts +1 -1
- package/dist/clickhouse/apply/apply.d.ts.map +1 -1
- package/dist/clickhouse/apply/statements.d.ts +8 -0
- package/dist/clickhouse/apply/statements.d.ts.map +1 -1
- package/dist/clickhouse/import/generator.d.ts +1 -1
- package/dist/clickhouse/import/generator.d.ts.map +1 -1
- package/dist/clickhouse/import/live-export.d.ts +19 -0
- package/dist/clickhouse/import/live-export.d.ts.map +1 -1
- package/dist/clickhouse/live/bind.d.ts +8 -0
- package/dist/clickhouse/live/bind.d.ts.map +1 -1
- package/dist/clickhouse/live/catalog.d.ts +15 -0
- package/dist/clickhouse/live/catalog.d.ts.map +1 -1
- package/dist/clickhouse/live/describe-resources.d.ts.map +1 -1
- package/dist/clickhouse/plan/commands.d.ts.map +1 -1
- package/dist/clickhouse/plan/deep.d.ts +19 -2
- package/dist/clickhouse/plan/deep.d.ts.map +1 -1
- package/dist/clickhouse/plan/diff.d.ts.map +1 -1
- package/dist/clickhouse/plan/normalize.d.ts +3 -0
- package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
- package/dist/clickhouse/plan/rebuild-handoff.d.ts +2 -0
- package/dist/clickhouse/plan/rebuild-handoff.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/engines.d.ts +44 -0
- package/dist/clickhouse/rebuild/engines.d.ts.map +1 -0
- package/dist/clickhouse/rebuild/verify.d.ts +25 -1
- package/dist/clickhouse/rebuild/verify.d.ts.map +1 -1
- package/dist/clickhouse/testing/fake-server.d.ts +5 -0
- package/dist/clickhouse/testing/fake-server.d.ts.map +1 -1
- package/dist/config.d.ts +32 -8
- package/dist/config.d.ts.map +1 -1
- package/dist/core/foreign-tables.d.ts +13 -0
- package/dist/core/foreign-tables.d.ts.map +1 -0
- package/dist/files/clickhouse.d.ts +7 -3
- package/dist/files/clickhouse.d.ts.map +1 -1
- package/dist/files/postgres.d.ts +3 -0
- package/dist/files/postgres.d.ts.map +1 -1
- package/dist/import-parser.d.ts.map +1 -1
- package/dist/integrity.json +2 -2
- package/dist/manifest.json +1 -1
- package/dist/op/activities/clickhouse-emulator.d.ts +7 -1
- package/dist/op/activities/clickhouse-emulator.d.ts.map +1 -1
- package/dist/op/activities/postgres-emulator.d.ts +8 -1
- package/dist/op/activities/postgres-emulator.d.ts.map +1 -1
- package/dist/postgres/access/changes.d.ts +23 -0
- package/dist/postgres/access/changes.d.ts.map +1 -0
- package/dist/postgres/apply/apply.d.ts.map +1 -1
- package/dist/postgres/entities.d.ts.map +1 -1
- package/dist/postgres/live/bind.d.ts +8 -1
- package/dist/postgres/live/bind.d.ts.map +1 -1
- package/dist/postgres/live/catalog.d.ts +3 -3
- package/dist/postgres/live/catalog.d.ts.map +1 -1
- package/dist/postgres/live/client.d.ts +31 -0
- package/dist/postgres/live/client.d.ts.map +1 -1
- package/dist/postgres/plan/commands.d.ts.map +1 -1
- package/dist/postgres/plan/deep.d.ts.map +1 -1
- package/dist/sql-files.d.ts +2 -1
- package/dist/sql-files.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/clickhouse/apply/apply.ts +8 -2
- package/src/clickhouse/apply/statements.ts +32 -5
- package/src/clickhouse/import/generator.ts +8 -3
- package/src/clickhouse/import/import.test.ts +64 -0
- package/src/clickhouse/import/live-export.ts +70 -2
- package/src/clickhouse/live/access.e2e.test.ts +71 -18
- package/src/clickhouse/live/bind.ts +10 -0
- package/src/clickhouse/live/catalog.ts +25 -2
- package/src/clickhouse/live/describe-resources.ts +5 -2
- package/src/clickhouse/live/live.test.ts +8 -0
- package/src/clickhouse/plan/commands.ts +6 -3
- package/src/clickhouse/plan/deep-grants.test.ts +67 -0
- package/src/clickhouse/plan/deep.ts +70 -6
- package/src/clickhouse/plan/diff.ts +4 -1
- package/src/clickhouse/plan/normalize.ts +5 -2
- package/src/clickhouse/plan/plan.test.ts +9 -1
- package/src/clickhouse/plan/rebuild-handoff.ts +11 -1
- package/src/clickhouse/rebuild/engines.ts +163 -0
- package/src/clickhouse/rebuild/rebuild-collapsing.e2e.test.ts +120 -0
- package/src/clickhouse/rebuild/rebuild-into-collapsing.e2e.test.ts +198 -0
- package/src/clickhouse/rebuild/rebuild.test.ts +78 -0
- package/src/clickhouse/rebuild/verify.ts +153 -24
- package/src/clickhouse/testing/fake-server.ts +3 -2
- package/src/codegen/classifier-page.ts +1 -1
- package/src/config.ts +16 -4
- package/src/core/foreign-tables.ts +28 -0
- package/src/files/clickhouse.ts +71 -5
- package/src/files/postgres.ts +2 -1
- package/src/import-parser.ts +2 -0
- package/src/migration-statements.test.ts +33 -0
- package/src/migration-statements.ts +2 -0
- package/src/op/__snapshots__/watch-pipeline.test.ts.snap +1 -1
- package/src/op/activities/clickhouse-emulator.ts +22 -1
- package/src/op/activities/postgres-apply-lock-timeout.e2e.test.ts +87 -0
- package/src/op/activities/postgres-apply.test.ts +32 -0
- package/src/op/activities/postgres-emulator.test.ts +20 -0
- package/src/op/activities/postgres-emulator.ts +41 -1
- package/src/op/apply-gate-access-clickhouse.e2e.test.ts +195 -0
- package/src/op/apply-gate-access.e2e.test.ts +207 -0
- package/src/postgres/access/access.test.ts +26 -1
- package/src/postgres/access/changes.ts +39 -0
- package/src/postgres/apply/apply.ts +10 -7
- package/src/postgres/entities.ts +20 -1
- package/src/postgres/live/bind.ts +16 -4
- package/src/postgres/live/catalog.ts +3 -15
- package/src/postgres/live/client.ts +85 -0
- package/src/postgres/live/live.test.ts +54 -0
- package/src/postgres/live/lock-timeout.e2e.test.ts +70 -0
- package/src/postgres/plan/commands.ts +2 -14
- package/src/postgres/plan/deep.ts +44 -2
- package/src/serializer.test.ts +4 -0
- package/src/skills/chant-sql-postgres-plan.md +1 -1
- package/src/skills/chant-sql-rebuild.md +2 -0
- package/src/skills/chant-sql.md +1 -1
- package/src/sql-files.test.ts +78 -1
- package/src/sql-files.ts +2 -1
|
@@ -112,6 +112,8 @@ function clickhouseStatements(beforeJson: string, afterJson: string, options: Di
|
|
|
112
112
|
current: new Map(before.map((o) => [o.exportName, o.canonical])),
|
|
113
113
|
keyOf: (o) => o.exportName,
|
|
114
114
|
allowDestructive: true,
|
|
115
|
+
// A user whose password is the environment's exists by the time a migration runs: the GRANTs to it need it too (#3717).
|
|
116
|
+
environmentCreatesUsers: true,
|
|
115
117
|
topology,
|
|
116
118
|
...(options.marker ? { marker: options.marker } : {}),
|
|
117
119
|
});
|
|
@@ -62,7 +62,7 @@ jobs:
|
|
|
62
62
|
`;
|
|
63
63
|
|
|
64
64
|
exports[`generateOpsPipeline renders the sql drift watch (#3642) > GitLab: a schedule-only job with the reader's credentials and no token of its own 1`] = `
|
|
65
|
-
"# chant Ops
|
|
65
|
+
"# chant Ops — one job per Op under stage "ops", each
|
|
66
66
|
# selected by its own rules:. A merge_request_event or push job needs no
|
|
67
67
|
# setup; its rule fires on the event itself.
|
|
68
68
|
#
|
|
@@ -10,9 +10,28 @@
|
|
|
10
10
|
* `upstreamPin` already tracks the LTS tags (`-lts`).
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { emulatorLifecycle, type EmulatorCapability, type EmulatorSpec } from "@intentius/chant/op";
|
|
13
|
+
import { emulatorLifecycle, type EmulatorCapability, type EmulatorIdentity, type EmulatorSpec } from "@intentius/chant/op";
|
|
14
14
|
import { clickhouseImage } from "../../spec/pin";
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* The server's UUID (#3673), read in the container and through the endpoint
|
|
18
|
+
* `status` prints, so a server of another origin on the same port is told
|
|
19
|
+
* apart from the emulator.
|
|
20
|
+
*/
|
|
21
|
+
export const CLICKHOUSE_EMULATOR_IDENTITY: EmulatorIdentity = {
|
|
22
|
+
server: "ClickHouse",
|
|
23
|
+
command: ["clickhouse-client", "-q", "SELECT serverUUID()"],
|
|
24
|
+
async probe(endpoint) {
|
|
25
|
+
const url = new URL(endpoint);
|
|
26
|
+
url.searchParams.set("query", "SELECT serverUUID(), version() FORMAT TSV");
|
|
27
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(5_000) });
|
|
28
|
+
const body = (await res.text()).trim();
|
|
29
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}: ${body.split("\n")[0]}`);
|
|
30
|
+
const [id = "", version = "(unknown version)"] = body.split("\t");
|
|
31
|
+
return { id, label: `ClickHouse ${version}` };
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
|
|
16
35
|
export const CLICKHOUSE_EMULATOR_SPEC: EmulatorSpec = {
|
|
17
36
|
name: "chant-clickhouse",
|
|
18
37
|
image: clickhouseImage(),
|
|
@@ -21,6 +40,8 @@ export const CLICKHOUSE_EMULATOR_SPEC: EmulatorSpec = {
|
|
|
21
40
|
// The image disables network access for the passwordless default user
|
|
22
41
|
// unless this is set; a local server is reached as `default` with no password.
|
|
23
42
|
runArgs: ["-e", "CLICKHOUSE_SKIP_USER_SETUP=1"],
|
|
43
|
+
identity: CLICKHOUSE_EMULATOR_IDENTITY,
|
|
44
|
+
credentials: "user default, no password",
|
|
24
45
|
};
|
|
25
46
|
|
|
26
47
|
/** `CLICKHOUSE_URL` is the variable binding reads when no `sql.profiles.<env>` names a server. */
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The applier's catalog read under `lock_timeout` (#3726). One session holds
|
|
3
|
+
* ACCESS EXCLUSIVE on a table behind a view; `postgresApply` of a build that
|
|
4
|
+
* declares the view reads its definition first, so it stops after the
|
|
5
|
+
* profile's `lockTimeoutMs` instead of waiting for the lock, names the table
|
|
6
|
+
* and the pid holding it, and writes nothing.
|
|
7
|
+
*
|
|
8
|
+
* `CHANT_SQL_E2E_POSTGRES_URL` (with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names
|
|
9
|
+
* a running server, else a throwaway one, skipped when Docker is not
|
|
10
|
+
* available. The test makes only the schema `chant_e2e_3726`, dropped
|
|
11
|
+
* afterwards.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
15
|
+
import { tmpdir } from "node:os";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
|
18
|
+
import { dockerAvailable, startTestPostgres, type TestPostgres } from "../../postgres/testing/server";
|
|
19
|
+
import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../../postgres/live/client";
|
|
20
|
+
import { postgresApply } from "./postgres-apply";
|
|
21
|
+
|
|
22
|
+
const SCHEMA = "chant_e2e_3726";
|
|
23
|
+
const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
|
|
24
|
+
const enabled = given ? true : await dockerAvailable();
|
|
25
|
+
const dir = mkdtempSync(join(tmpdir(), "chant-pg-apply-3726-"));
|
|
26
|
+
let server: TestPostgres | undefined;
|
|
27
|
+
let endpoint: PostgresEndpoint;
|
|
28
|
+
let admin: PostgresClient;
|
|
29
|
+
let holder: PostgresClient;
|
|
30
|
+
|
|
31
|
+
describe.skipIf(!enabled)("an apply whose catalog read is blocked behind ACCESS EXCLUSIVE", () => {
|
|
32
|
+
beforeAll(async () => {
|
|
33
|
+
if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
|
|
34
|
+
else {
|
|
35
|
+
server = await startTestPostgres();
|
|
36
|
+
endpoint = server.endpoint();
|
|
37
|
+
}
|
|
38
|
+
admin = await connectPostgres(endpoint);
|
|
39
|
+
await admin.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);
|
|
40
|
+
await admin.query(`CREATE SCHEMA ${SCHEMA}`);
|
|
41
|
+
await admin.query(`CREATE TABLE ${SCHEMA}.orders (id bigint PRIMARY KEY)`);
|
|
42
|
+
await admin.query(`CREATE VIEW ${SCHEMA}.recent AS SELECT id FROM ${SCHEMA}.orders`);
|
|
43
|
+
holder = await connectPostgres(endpoint, { applicationName: "chant-e2e-3726-holder" });
|
|
44
|
+
await holder.query("BEGIN");
|
|
45
|
+
await holder.query(`LOCK TABLE ${SCHEMA}.orders IN ACCESS EXCLUSIVE MODE`);
|
|
46
|
+
}, 600_000);
|
|
47
|
+
|
|
48
|
+
afterAll(async () => {
|
|
49
|
+
await holder?.query("ROLLBACK").catch(() => undefined);
|
|
50
|
+
await holder?.end();
|
|
51
|
+
await admin?.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`).catch(() => undefined);
|
|
52
|
+
await admin?.end();
|
|
53
|
+
await server?.stop();
|
|
54
|
+
rmSync(dir, { recursive: true, force: true });
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test("stops after lockTimeoutMs, naming the table and the pid, with nothing written", async () => {
|
|
58
|
+
const [{ pid }] = await holder.query<{ pid: number }>("SELECT pg_catalog.pg_backend_pid() AS pid");
|
|
59
|
+
const buildPath = join(dir, "schema.json");
|
|
60
|
+
writeFileSync(
|
|
61
|
+
buildPath,
|
|
62
|
+
JSON.stringify({
|
|
63
|
+
dialect: "postgres",
|
|
64
|
+
applyOrder: ["app", "orders", "recent", "audit"],
|
|
65
|
+
objects: [
|
|
66
|
+
{ export: "app", type: "Postgres::Schema", dependsOn: [], ddl: `CREATE SCHEMA ${SCHEMA}` },
|
|
67
|
+
{ export: "orders", type: "Postgres::Table", dependsOn: ["app"], ddl: `CREATE TABLE ${SCHEMA}.orders (\n id bigint PRIMARY KEY\n)` },
|
|
68
|
+
{ export: "recent", type: "Postgres::View", dependsOn: ["orders"], ddl: `CREATE VIEW ${SCHEMA}.recent AS SELECT id FROM ${SCHEMA}.orders` },
|
|
69
|
+
{ export: "audit", type: "Postgres::Table", dependsOn: ["app"], ddl: `CREATE TABLE ${SCHEMA}.audit (\n id bigint\n)` },
|
|
70
|
+
],
|
|
71
|
+
}),
|
|
72
|
+
);
|
|
73
|
+
const started = Date.now();
|
|
74
|
+
const err = (await postgresApply({ buildPath, environment: "e2e" }, undefined, {
|
|
75
|
+
config: { ownership: { stack: "e2e3726", env: "e2e" }, sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_3726_PASSWORD" }, schemas: [SCHEMA], lockTimeoutMs: 300 } } } },
|
|
76
|
+
env: { PG_E2E_3726_PASSWORD: endpoint.password ?? "" },
|
|
77
|
+
log: () => undefined,
|
|
78
|
+
}).catch((e: unknown) => e)) as Error;
|
|
79
|
+
expect(Date.now() - started).toBeLessThan(10_000);
|
|
80
|
+
expect(err).toBeInstanceOf(Error);
|
|
81
|
+
expect(err.message).toContain("lock timeout: the catalog read waited 300ms for a lock");
|
|
82
|
+
expect(err.message).toContain(`${SCHEMA}.orders is held in ACCESS EXCLUSIVE by pid ${pid}`);
|
|
83
|
+
expect(err.message).toContain("chant-e2e-3726-holder");
|
|
84
|
+
const [audit] = await admin.query<{ n: number }>(`SELECT count(*)::int AS n FROM pg_catalog.pg_tables WHERE schemaname = $1 AND tablename = 'audit'`, [SCHEMA]);
|
|
85
|
+
expect(audit?.n).toBe(0);
|
|
86
|
+
}, 60_000);
|
|
87
|
+
});
|
|
@@ -8,6 +8,7 @@ import { buildMajor, postgresApply, type PostgresApplyArgs, type PostgresApplyDe
|
|
|
8
8
|
import { toApplyResult } from "./index";
|
|
9
9
|
import { writablePostgres, type StoredPgObject, type WritablePostgres } from "../../postgres/testing/writable-server";
|
|
10
10
|
import { PostgresApplyError } from "../../postgres/apply/apply";
|
|
11
|
+
import { PostgresQueryError } from "../../postgres/live/client";
|
|
11
12
|
|
|
12
13
|
const dirs: string[] = [];
|
|
13
14
|
afterAll(() => {
|
|
@@ -213,6 +214,37 @@ describe("a statement the server refuses", () => {
|
|
|
213
214
|
});
|
|
214
215
|
});
|
|
215
216
|
|
|
217
|
+
describe("the catalog read before applying (#3726)", () => {
|
|
218
|
+
test("runs under the profile's lockTimeoutMs, else 5000 ms, else the argument's", async () => {
|
|
219
|
+
for (const [profile, arg, expected] of [[undefined, undefined, "5000ms"], [250, undefined, "250ms"], [250, 75, "75ms"]] as const) {
|
|
220
|
+
const s = writablePostgres();
|
|
221
|
+
const sent: unknown[][] = [];
|
|
222
|
+
const config = { ownership: { stack: "shop", env: "prod" }, sql: { profiles: { test: { url: "postgres://fake/shop", ...(profile !== undefined ? { lockTimeoutMs: profile } : {}) } } } };
|
|
223
|
+
const client = { query: (sql: string, p?: readonly unknown[]) => (sent.push([sql, ...(p ?? [])]), s.client.query(sql)), end: s.client.end };
|
|
224
|
+
await apply(s, [APP], arg !== undefined ? { lockTimeoutMs: arg } : {}, { config, env: {}, connect: async () => client as never });
|
|
225
|
+
expect(sent[0]).toEqual(["SELECT pg_catalog.set_config('lock_timeout', $1, false)", expected]);
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
test("blocked behind ACCESS EXCLUSIVE, it stops before any write, naming the relation and the pid holding it", async () => {
|
|
230
|
+
const s = writablePostgres();
|
|
231
|
+
const client = {
|
|
232
|
+
async query<T>(sql: string): Promise<T[]> {
|
|
233
|
+
if (sql.includes("pg_get_viewdef")) throw new PostgresQueryError("canceling statement due to lock timeout", "55P03");
|
|
234
|
+
if (sql.includes("pg_catalog.pg_locks")) return [{ pid: 4242, relation: "app.users", application: "psql", state: "idle in transaction", seconds: 9, query: "LOCK TABLE app.users" }] as T[];
|
|
235
|
+
return s.client.query<T>(sql);
|
|
236
|
+
},
|
|
237
|
+
end: s.client.end,
|
|
238
|
+
};
|
|
239
|
+
const err = (await apply(s, ALL, {}, { connect: async () => client as never, readLive: async (c) => (await c.query("SELECT pg_catalog.pg_get_viewdef(1)"), []) }).catch((e: unknown) => e)) as Error;
|
|
240
|
+
expect(err).toBeInstanceOf(PostgresQueryError);
|
|
241
|
+
expect(err.message).toContain("canceling statement due to lock timeout: the catalog read waited 5000ms for a lock");
|
|
242
|
+
expect(err.message).toContain("app.users is held in ACCESS EXCLUSIVE by pid 4242");
|
|
243
|
+
expect(s.writes).toEqual([]);
|
|
244
|
+
expect(s.control).toEqual([]);
|
|
245
|
+
});
|
|
246
|
+
});
|
|
247
|
+
|
|
216
248
|
describe("the target major", () => {
|
|
217
249
|
test("is the build output's postgresMajor", () => {
|
|
218
250
|
expect(buildMajor(JSON.stringify({ dialect: "postgres", postgresMajor: 16, objects: [] }))).toBe(16);
|
|
@@ -24,3 +24,23 @@ describe("the Postgres emulator capability (#3280)", () => {
|
|
|
24
24
|
expect(lc.endpoint(5432)).toBe("postgres://localhost:5432/postgres");
|
|
25
25
|
});
|
|
26
26
|
});
|
|
27
|
+
|
|
28
|
+
describe("the sql emulators say who they are (#3673)", () => {
|
|
29
|
+
test("Postgres compares the cluster's system identifier, read over the container's socket", () => {
|
|
30
|
+
expect(POSTGRES_EMULATOR.spec.identity?.server).toBe("Postgres");
|
|
31
|
+
expect(POSTGRES_EMULATOR.spec.identity?.command).toEqual(["psql", "-U", "postgres", "-tAc", "select system_identifier from pg_control_system()"]);
|
|
32
|
+
expect(POSTGRES_EMULATOR.spec.credentials).toBe("user postgres, password chant");
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("ClickHouse compares serverUUID(), and signs in as default with no password", () => {
|
|
36
|
+
expect(CLICKHOUSE_EMULATOR.spec.identity?.server).toBe("ClickHouse");
|
|
37
|
+
expect(CLICKHOUSE_EMULATOR.spec.identity?.command).toEqual(["clickhouse-client", "-q", "SELECT serverUUID()"]);
|
|
38
|
+
expect(CLICKHOUSE_EMULATOR.spec.credentials).toBe("user default, no password");
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("a server that is not Postgres fails the Postgres probe instead of matching", async () => {
|
|
42
|
+
// Nothing listens on port 1; the probe rejects, which status reports as "does not answer as the emulator".
|
|
43
|
+
await expect(POSTGRES_EMULATOR.spec.identity!.probe("postgres://127.0.0.1:1/postgres")).rejects.toThrow();
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
|
|
@@ -15,12 +15,50 @@
|
|
|
15
15
|
* `upstreamPins`, not the emulator freshness check.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import { emulatorLifecycle, type EmulatorCapability, type EmulatorSpec } from "@intentius/chant/op";
|
|
18
|
+
import { emulatorLifecycle, type EmulatorCapability, type EmulatorIdentity, type EmulatorSpec } from "@intentius/chant/op";
|
|
19
19
|
import { POSTGRES_LATEST_MAJOR, postgresImage } from "../../spec/postgres-pin";
|
|
20
|
+
import { connectPostgres } from "../../postgres/live/client";
|
|
20
21
|
|
|
21
22
|
/** The local server's password. It is a throwaway server on localhost. */
|
|
22
23
|
export const POSTGRES_EMULATOR_PASSWORD = "chant";
|
|
23
24
|
|
|
25
|
+
/** How long `chant emulator status` waits for the server behind the endpoint. */
|
|
26
|
+
const PROBE_TIMEOUT_MS = 5_000;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The cluster's system identifier, unique per `initdb` (#3673): read in the
|
|
30
|
+
* container over its socket, and through the endpoint `status` prints. A
|
|
31
|
+
* native Postgres holding 127.0.0.1:5432 answers with its own, or refuses
|
|
32
|
+
* the emulator's sign-in.
|
|
33
|
+
*/
|
|
34
|
+
export const POSTGRES_EMULATOR_IDENTITY: EmulatorIdentity = {
|
|
35
|
+
server: "Postgres",
|
|
36
|
+
command: ["psql", "-U", "postgres", "-tAc", "select system_identifier from pg_control_system()"],
|
|
37
|
+
async probe(endpoint) {
|
|
38
|
+
const client = await withTimeout(
|
|
39
|
+
connectPostgres({ url: endpoint, user: "postgres", password: POSTGRES_EMULATOR_PASSWORD }, { applicationName: "chant emulator status" }),
|
|
40
|
+
);
|
|
41
|
+
try {
|
|
42
|
+
const [row] = await withTimeout(
|
|
43
|
+
client.query<{ id: string; version: string }>(
|
|
44
|
+
"SELECT system_identifier::text AS id, pg_catalog.current_setting('server_version') AS version FROM pg_catalog.pg_control_system()",
|
|
45
|
+
),
|
|
46
|
+
);
|
|
47
|
+
return { id: row?.id ?? "", label: `PostgreSQL ${row?.version ?? "(unknown version)"}` };
|
|
48
|
+
} finally {
|
|
49
|
+
await client.end();
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
function withTimeout<T>(promise: Promise<T>): Promise<T> {
|
|
55
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
56
|
+
const timeout = new Promise<never>((_, reject) => {
|
|
57
|
+
timer = setTimeout(() => reject(new Error(`no answer within ${PROBE_TIMEOUT_MS / 1000}s`)), PROBE_TIMEOUT_MS);
|
|
58
|
+
});
|
|
59
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
60
|
+
}
|
|
61
|
+
|
|
24
62
|
export const POSTGRES_EMULATOR_SPEC: EmulatorSpec = {
|
|
25
63
|
name: "chant-postgres",
|
|
26
64
|
image: postgresImage(POSTGRES_LATEST_MAJOR),
|
|
@@ -28,6 +66,8 @@ export const POSTGRES_EMULATOR_SPEC: EmulatorSpec = {
|
|
|
28
66
|
readyCommand: ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"],
|
|
29
67
|
endpoint: (port) => `postgres://localhost:${port}/postgres`,
|
|
30
68
|
runArgs: ["-e", `POSTGRES_PASSWORD=${POSTGRES_EMULATOR_PASSWORD}`],
|
|
69
|
+
identity: POSTGRES_EMULATOR_IDENTITY,
|
|
70
|
+
credentials: `user postgres, password ${POSTGRES_EMULATOR_PASSWORD}`,
|
|
31
71
|
};
|
|
32
72
|
|
|
33
73
|
/** `POSTGRES_URL` is the variable binding reads when no `sql.profiles.<env>` names a server. */
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An approval does not cover a ClickHouse privilege granted by hand after it
|
|
3
|
+
* (#3733), end to end with a profile that manages access. The Postgres
|
|
4
|
+
* counterpart is `./apply-gate-access.e2e.test.ts`.
|
|
5
|
+
*
|
|
6
|
+
* A project in a temporary directory declares a database, a table, a reader
|
|
7
|
+
* role and a grant of SELECT on the table to it, and a gated `ApplyOp`. The
|
|
8
|
+
* first approved run creates them. The next run stops at the gate; the test
|
|
9
|
+
* approves its plan, grants INSERT on the table to the role by hand, and runs
|
|
10
|
+
* again. The gate refuses with "is approved, but not for this plan", the hand
|
|
11
|
+
* grant is still there, and the plan the run printed lists the REVOKE an
|
|
12
|
+
* apply would make. Approving that plan applies it.
|
|
13
|
+
*
|
|
14
|
+
* Before #3733 the plan digest was taken over a diff that left grants out (a
|
|
15
|
+
* grant was unobserved there), so the third run revoked the hand grant under
|
|
16
|
+
* the earlier approval.
|
|
17
|
+
*
|
|
18
|
+
* Everything is written to the database `chant_e2e_3733` and the role
|
|
19
|
+
* `chant_e2e_3733_reader`, both dropped afterwards.
|
|
20
|
+
*
|
|
21
|
+
* The server: `CHANT_SQL_GATE_CLICKHOUSE_URL` names a running one, such as
|
|
22
|
+
* the one `chant emulator up --lexicon sql` starts. Without it, a throwaway
|
|
23
|
+
* server at the pin, skipped cleanly when Docker is not available.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { execFile } from "node:child_process";
|
|
27
|
+
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
28
|
+
import { tmpdir } from "node:os";
|
|
29
|
+
import { join } from "node:path";
|
|
30
|
+
import { promisify } from "node:util";
|
|
31
|
+
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
|
32
|
+
import { dockerAvailable, startScratchServer, type ScratchServer } from "../clickhouse/container";
|
|
33
|
+
import { clickhouseQuery, type ClickHouseEndpoint } from "../clickhouse/http";
|
|
34
|
+
import { clickhouseImage } from "../spec/pin";
|
|
35
|
+
|
|
36
|
+
const exec = promisify(execFile);
|
|
37
|
+
|
|
38
|
+
const REPO = join(import.meta.dirname, "../../../..");
|
|
39
|
+
const BIN = join(REPO, "node_modules/.bin");
|
|
40
|
+
const DB = "chant_e2e_3733";
|
|
41
|
+
const READER = "chant_e2e_3733_reader";
|
|
42
|
+
const OP = "schema-apply";
|
|
43
|
+
|
|
44
|
+
const given = process.env.CHANT_SQL_GATE_CLICKHOUSE_URL;
|
|
45
|
+
const enabled = given ? true : await dockerAvailable();
|
|
46
|
+
|
|
47
|
+
interface Run {
|
|
48
|
+
code: number;
|
|
49
|
+
output: string;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const strip = (s: string) => s.replace(/\u001b\[[0-9;]*m/g, "");
|
|
53
|
+
|
|
54
|
+
const SCHEMA = `import { database, table, role, grant } from "@intentius/chant-lexicon-sql/clickhouse";
|
|
55
|
+
|
|
56
|
+
export const db = database\`CREATE DATABASE ${DB} ENGINE = Atomic\`;
|
|
57
|
+
|
|
58
|
+
export const events = table\`
|
|
59
|
+
CREATE TABLE \${db}.events (
|
|
60
|
+
id UInt64,
|
|
61
|
+
kind String
|
|
62
|
+
)
|
|
63
|
+
ENGINE = MergeTree
|
|
64
|
+
ORDER BY id\`;
|
|
65
|
+
|
|
66
|
+
export const reader = role\`CREATE ROLE ${READER}\`;
|
|
67
|
+
|
|
68
|
+
export const readEvents = grant\`GRANT SELECT ON \${events} TO \${reader}\`;
|
|
69
|
+
`;
|
|
70
|
+
|
|
71
|
+
const OPS = `import { ApplyOp } from "@intentius/chant/op";
|
|
72
|
+
const { op } = ApplyOp({ name: "${OP}", env: "admin", target: "clickhouse", gate: { gate: "approve-schema" } });
|
|
73
|
+
export default op;
|
|
74
|
+
`;
|
|
75
|
+
|
|
76
|
+
const config = (url: string) => `import type { ChantConfig } from "@intentius/chant";
|
|
77
|
+
import "@intentius/chant-lexicon-sql";
|
|
78
|
+
|
|
79
|
+
export default {
|
|
80
|
+
lexicons: ["sql"],
|
|
81
|
+
ownership: { stack: "e2e-3733", env: "admin" },
|
|
82
|
+
sql: ${JSON.stringify({ dialect: "clickhouse", profiles: { admin: { url, databases: [DB], access: true } } }, null, 2)},
|
|
83
|
+
} satisfies ChantConfig;
|
|
84
|
+
`;
|
|
85
|
+
|
|
86
|
+
/** The `--plan <digest>` the stopped run printed in its approve command. */
|
|
87
|
+
function approvedPlan(run: Run): string {
|
|
88
|
+
const m = /chant approve \S+ \S+ .*?--plan (jcs1-sha256:[0-9a-f]{64})/.exec(run.output);
|
|
89
|
+
if (!m) throw new Error(`the run printed no approve command with a plan:\n${run.output}`);
|
|
90
|
+
return m[1];
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function gateOf(run: Run): string {
|
|
94
|
+
const m = /chant approve \S+ (\S+)/.exec(run.output);
|
|
95
|
+
if (!m) throw new Error(`the run printed no approve command:\n${run.output}`);
|
|
96
|
+
return m[1];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
describe.skipIf(!enabled)("a gated ApplyOp over ClickHouse refuses after a privilege granted by hand (#3733)", () => {
|
|
100
|
+
let server: ScratchServer | undefined;
|
|
101
|
+
let admin: ClickHouseEndpoint;
|
|
102
|
+
let dir: string | undefined;
|
|
103
|
+
const q = <T = Record<string, unknown>>(sql: string) => clickhouseQuery<T>(admin, sql);
|
|
104
|
+
const cleanup = async () => {
|
|
105
|
+
await q(`DROP ROLE IF EXISTS ${READER}`);
|
|
106
|
+
await q(`DROP DATABASE IF EXISTS ${DB} SYNC`);
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
const chant = async (...args: string[]): Promise<Run> => {
|
|
110
|
+
try {
|
|
111
|
+
const r = await exec(join(BIN, "chant"), args, {
|
|
112
|
+
cwd: dir,
|
|
113
|
+
env: { ...process.env, PATH: `${BIN}:${process.env.PATH ?? ""}` },
|
|
114
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
115
|
+
});
|
|
116
|
+
return { code: 0, output: strip(`${r.stdout}${r.stderr}`) };
|
|
117
|
+
} catch (err) {
|
|
118
|
+
const e = err as { code?: number; stdout?: string; stderr?: string };
|
|
119
|
+
return { code: typeof e.code === "number" ? e.code : 1, output: strip(`${e.stdout ?? ""}${e.stderr ?? ""}`) };
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/** What `SHOW GRANTS FOR` the reader prints, one grant per line. */
|
|
124
|
+
const readerGrants = async () => (await q<Record<string, string>>(`SHOW GRANTS FOR ${READER}`)).map((r) => Object.values(r)[0]!);
|
|
125
|
+
|
|
126
|
+
beforeAll(async () => {
|
|
127
|
+
if (given) admin = { url: given };
|
|
128
|
+
else {
|
|
129
|
+
server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-gate-access" });
|
|
130
|
+
admin = server.endpoint;
|
|
131
|
+
}
|
|
132
|
+
await cleanup();
|
|
133
|
+
dir = mkdtempSync(join(tmpdir(), "chant-sql-gate-access-ch-"));
|
|
134
|
+
symlinkSync(join(REPO, "node_modules"), join(dir, "node_modules"), "dir");
|
|
135
|
+
const files: Record<string, string> = {
|
|
136
|
+
"package.json": JSON.stringify({ name: "sql-gate-access-ch-e2e", private: true, type: "module", scripts: { build: "chant build src --lexicon sql -o dist/schema.json" } }),
|
|
137
|
+
".gitignore": "node_modules\ndist\n.chant\n",
|
|
138
|
+
"chant.config.ts": config(admin.url),
|
|
139
|
+
"ops/apply.op.ts": OPS,
|
|
140
|
+
"src/schema.ts": SCHEMA,
|
|
141
|
+
};
|
|
142
|
+
for (const [path, text] of Object.entries(files)) {
|
|
143
|
+
mkdirSync(join(dir, path, ".."), { recursive: true });
|
|
144
|
+
writeFileSync(join(dir, path), text);
|
|
145
|
+
}
|
|
146
|
+
const git = (...args: string[]) => exec("git", ["-c", "user.email=e2e@chant.invalid", "-c", "user.name=e2e", ...args], { cwd: dir });
|
|
147
|
+
await git("init", "-q");
|
|
148
|
+
await git("add", "-A");
|
|
149
|
+
await git("commit", "-qm", "the schema");
|
|
150
|
+
}, 600_000);
|
|
151
|
+
|
|
152
|
+
afterAll(async () => {
|
|
153
|
+
if (admin) await cleanup().catch(() => undefined);
|
|
154
|
+
await server?.stop();
|
|
155
|
+
if (dir) rmSync(dir, { recursive: true, force: true });
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
test("approve, grant by hand, rerun: the gate refuses and the hand grant stays", async () => {
|
|
159
|
+
// The first run creates everything once its plan is approved.
|
|
160
|
+
const created = await chant("run", OP);
|
|
161
|
+
const approveCreate = await chant("approve", OP, gateOf(created), "--plan", approvedPlan(created));
|
|
162
|
+
expect(approveCreate.code, approveCreate.output).toBe(0);
|
|
163
|
+
const applied = await chant("run", OP);
|
|
164
|
+
expect(applied.output).toMatch(/Op "schema-apply" completed/);
|
|
165
|
+
expect(await readerGrants()).toEqual([`GRANT SELECT ON ${DB}.events TO ${READER}`]);
|
|
166
|
+
|
|
167
|
+
// Nothing to change: the run stops at the gate, and its plan is approved.
|
|
168
|
+
const first = await chant("run", OP);
|
|
169
|
+
expect(first.output).not.toContain("PENDING (");
|
|
170
|
+
const gate = gateOf(first);
|
|
171
|
+
const digest = approvedPlan(first);
|
|
172
|
+
const approve = await chant("approve", OP, gate, "--plan", digest);
|
|
173
|
+
expect(approve.code, approve.output).toBe(0);
|
|
174
|
+
|
|
175
|
+
await q(`GRANT INSERT ON ${DB}.events TO ${READER}`);
|
|
176
|
+
|
|
177
|
+
const second = await chant("run", OP);
|
|
178
|
+
expect(second.output).toContain(`Gate "${gate}" is approved, but not for this plan.`);
|
|
179
|
+
expect(second.output).toContain(`approved: ${digest}`);
|
|
180
|
+
expect(second.output).toMatch(
|
|
181
|
+
/PENDING \(changes an apply would make that no declared property shows\):\s+~ grants chant_e2e_3733_reader: REVOKE INSERT ON `chant_e2e_3733`\.`events` FROM `chant_e2e_3733_reader` \[from: readEvents\]/,
|
|
182
|
+
);
|
|
183
|
+
expect(second.output).not.toMatch(/Op "schema-apply" completed/);
|
|
184
|
+
expect((await readerGrants()).join("\n")).toMatch(/GRANT INSERT, SELECT ON chant_e2e_3733\.events TO chant_e2e_3733_reader|GRANT SELECT, INSERT ON chant_e2e_3733\.events/);
|
|
185
|
+
|
|
186
|
+
// Approving the plan the second run produced applies the revoke.
|
|
187
|
+
const fresh = approvedPlan(second);
|
|
188
|
+
expect(fresh).not.toBe(digest);
|
|
189
|
+
const reapprove = await chant("approve", OP, gate, "--plan", fresh);
|
|
190
|
+
expect(reapprove.code, reapprove.output).toBe(0);
|
|
191
|
+
const third = await chant("run", OP);
|
|
192
|
+
expect(third.output).toMatch(/Op "schema-apply" completed/);
|
|
193
|
+
expect(await readerGrants()).toEqual([`GRANT SELECT ON ${DB}.events TO ${READER}`]);
|
|
194
|
+
}, 600_000);
|
|
195
|
+
});
|