@intentius/chant-lexicon-sql 0.118.0 → 0.120.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clickhouse/apply/apply.d.ts +1 -1
- package/dist/clickhouse/apply/apply.d.ts.map +1 -1
- package/dist/clickhouse/apply/statements.d.ts +8 -0
- package/dist/clickhouse/apply/statements.d.ts.map +1 -1
- package/dist/clickhouse/import/generator.d.ts +1 -1
- package/dist/clickhouse/import/generator.d.ts.map +1 -1
- package/dist/clickhouse/import/live-export.d.ts +19 -0
- package/dist/clickhouse/import/live-export.d.ts.map +1 -1
- package/dist/clickhouse/live/bind.d.ts +8 -0
- package/dist/clickhouse/live/bind.d.ts.map +1 -1
- package/dist/clickhouse/live/catalog.d.ts +15 -0
- package/dist/clickhouse/live/catalog.d.ts.map +1 -1
- package/dist/clickhouse/live/describe-resources.d.ts.map +1 -1
- package/dist/clickhouse/plan/commands.d.ts.map +1 -1
- package/dist/clickhouse/plan/deep.d.ts +19 -2
- package/dist/clickhouse/plan/deep.d.ts.map +1 -1
- package/dist/clickhouse/plan/diff.d.ts.map +1 -1
- package/dist/clickhouse/plan/normalize.d.ts +3 -0
- package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
- package/dist/clickhouse/plan/rebuild-handoff.d.ts +2 -0
- package/dist/clickhouse/plan/rebuild-handoff.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/engines.d.ts +44 -0
- package/dist/clickhouse/rebuild/engines.d.ts.map +1 -0
- package/dist/clickhouse/rebuild/verify.d.ts +25 -1
- package/dist/clickhouse/rebuild/verify.d.ts.map +1 -1
- package/dist/clickhouse/testing/fake-server.d.ts +5 -0
- package/dist/clickhouse/testing/fake-server.d.ts.map +1 -1
- package/dist/config.d.ts +32 -8
- package/dist/config.d.ts.map +1 -1
- package/dist/core/foreign-tables.d.ts +13 -0
- package/dist/core/foreign-tables.d.ts.map +1 -0
- package/dist/files/build-root.d.ts +9 -1
- package/dist/files/build-root.d.ts.map +1 -1
- package/dist/files/clickhouse.d.ts +7 -3
- package/dist/files/clickhouse.d.ts.map +1 -1
- package/dist/files/postgres.d.ts +3 -0
- package/dist/files/postgres.d.ts.map +1 -1
- package/dist/import-parser.d.ts.map +1 -1
- package/dist/integrity.json +2 -2
- package/dist/manifest.json +1 -1
- package/dist/op/activities/clickhouse-emulator.d.ts +7 -1
- package/dist/op/activities/clickhouse-emulator.d.ts.map +1 -1
- package/dist/op/activities/postgres-emulator.d.ts +8 -1
- package/dist/op/activities/postgres-emulator.d.ts.map +1 -1
- package/dist/postgres/access/changes.d.ts +23 -0
- package/dist/postgres/access/changes.d.ts.map +1 -0
- package/dist/postgres/apply/apply.d.ts.map +1 -1
- package/dist/postgres/entities.d.ts.map +1 -1
- package/dist/postgres/live/bind.d.ts +8 -1
- package/dist/postgres/live/bind.d.ts.map +1 -1
- package/dist/postgres/live/catalog.d.ts +3 -3
- package/dist/postgres/live/catalog.d.ts.map +1 -1
- package/dist/postgres/live/client.d.ts +31 -0
- package/dist/postgres/live/client.d.ts.map +1 -1
- package/dist/postgres/plan/commands.d.ts.map +1 -1
- package/dist/postgres/plan/deep.d.ts.map +1 -1
- package/dist/sql-files.d.ts +2 -1
- package/dist/sql-files.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/clickhouse/apply/apply.ts +8 -2
- package/src/clickhouse/apply/statements.ts +32 -5
- package/src/clickhouse/import/generator.ts +8 -3
- package/src/clickhouse/import/import.test.ts +64 -0
- package/src/clickhouse/import/live-export.ts +70 -2
- package/src/clickhouse/live/access.e2e.test.ts +71 -18
- package/src/clickhouse/live/bind.ts +10 -0
- package/src/clickhouse/live/catalog.ts +25 -2
- package/src/clickhouse/live/describe-resources.ts +5 -2
- package/src/clickhouse/live/live.test.ts +8 -0
- package/src/clickhouse/plan/commands.ts +6 -3
- package/src/clickhouse/plan/deep-grants.test.ts +67 -0
- package/src/clickhouse/plan/deep.ts +70 -6
- package/src/clickhouse/plan/diff.ts +4 -1
- package/src/clickhouse/plan/normalize.ts +5 -2
- package/src/clickhouse/plan/plan.test.ts +9 -1
- package/src/clickhouse/plan/rebuild-handoff.ts +11 -1
- package/src/clickhouse/rebuild/engines.ts +163 -0
- package/src/clickhouse/rebuild/rebuild-collapsing.e2e.test.ts +120 -0
- package/src/clickhouse/rebuild/rebuild-into-collapsing.e2e.test.ts +198 -0
- package/src/clickhouse/rebuild/rebuild.test.ts +78 -0
- package/src/clickhouse/rebuild/verify.ts +153 -24
- package/src/clickhouse/testing/fake-server.ts +3 -2
- package/src/codegen/classifier-page.ts +1 -1
- package/src/config.ts +16 -4
- package/src/core/foreign-tables.ts +28 -0
- package/src/files/build-root.ts +15 -3
- package/src/files/clickhouse.ts +71 -5
- package/src/files/postgres.ts +2 -1
- package/src/import-parser.ts +2 -0
- package/src/migration-statements.test.ts +33 -0
- package/src/migration-statements.ts +2 -0
- package/src/op/__snapshots__/watch-pipeline.test.ts.snap +1 -1
- package/src/op/activities/clickhouse-emulator.ts +22 -1
- package/src/op/activities/postgres-apply-lock-timeout.e2e.test.ts +87 -0
- package/src/op/activities/postgres-apply.test.ts +32 -0
- package/src/op/activities/postgres-emulator.test.ts +20 -0
- package/src/op/activities/postgres-emulator.ts +41 -1
- package/src/op/apply-gate-access-clickhouse.e2e.test.ts +195 -0
- package/src/op/apply-gate-access.e2e.test.ts +207 -0
- package/src/postgres/access/access.test.ts +26 -1
- package/src/postgres/access/changes.ts +39 -0
- package/src/postgres/apply/apply.ts +10 -7
- package/src/postgres/entities.ts +20 -1
- package/src/postgres/live/bind.ts +16 -4
- package/src/postgres/live/catalog.ts +3 -15
- package/src/postgres/live/client.ts +85 -0
- package/src/postgres/live/live.test.ts +54 -0
- package/src/postgres/live/lock-timeout.e2e.test.ts +70 -0
- package/src/postgres/plan/commands.ts +2 -14
- package/src/postgres/plan/deep.ts +44 -2
- package/src/serializer.test.ts +4 -0
- package/src/skills/chant-sql-postgres-plan.md +1 -1
- package/src/skills/chant-sql-rebuild.md +2 -0
- package/src/skills/chant-sql.md +1 -1
- package/src/sql-files.test.ts +110 -1
- package/src/sql-files.ts +2 -1
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tables another tool keeps its migration history in, by name, with the tool.
|
|
3
|
+
*
|
|
4
|
+
* Both dialects' catalog readers mark a table by one of these names as
|
|
5
|
+
* foreign: import leaves it out with a warning, and a plan names it in a hint
|
|
6
|
+
* instead of proposing a drop, since chant changing it would change what that
|
|
7
|
+
* tool owns (#3047 question 10, #3676). golang-migrate, dbmate, goose and
|
|
8
|
+
* Flyway keep the same tables on ClickHouse as on Postgres.
|
|
9
|
+
*/
|
|
10
|
+
export const FOREIGN_TABLES: Readonly<Record<string, string>> = {
|
|
11
|
+
_prisma_migrations: "Prisma Migrate",
|
|
12
|
+
schema_migrations: "a migration runner (Rails, golang-migrate, dbmate)",
|
|
13
|
+
ar_internal_metadata: "Rails",
|
|
14
|
+
django_migrations: "Django",
|
|
15
|
+
alembic_version: "Alembic",
|
|
16
|
+
__drizzle_migrations: "drizzle-kit",
|
|
17
|
+
flyway_schema_history: "Flyway",
|
|
18
|
+
goose_db_version: "goose",
|
|
19
|
+
knex_migrations: "Knex",
|
|
20
|
+
knex_migrations_lock: "Knex",
|
|
21
|
+
SequelizeMeta: "Sequelize",
|
|
22
|
+
pgmigrations: "node-pg-migrate",
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/** The tool that keeps a table by this name, or undefined when none does. */
|
|
26
|
+
export function foreignTool(name: string): string | undefined {
|
|
27
|
+
return Object.prototype.hasOwnProperty.call(FOREIGN_TABLES, name) ? FOREIGN_TABLES[name] : undefined;
|
|
28
|
+
}
|
package/src/files/build-root.ts
CHANGED
|
@@ -8,7 +8,11 @@
|
|
|
8
8
|
* git-ignored paths, child projects and the project's `exclude` globs are
|
|
9
9
|
* left out. A file whose first lines carry `-- chant-discovery-skip` (or the
|
|
10
10
|
* `/* chant-discovery-skip *\/` form) is not read, for SQL that is not schema:
|
|
11
|
-
* seed data, a migration, a query.
|
|
11
|
+
* seed data, a migration, a query. Nor is a file under a `migrations`
|
|
12
|
+
* directory (#3713): versioned migrations and an ORM's generated migrations
|
|
13
|
+
* are `ALTER`s and repeated `CREATE`s of objects declared elsewhere, and a
|
|
14
|
+
* build of the project root, as `chant lifecycle diff` and an ApplyOp's plan
|
|
15
|
+
* run with no `sourceDir`, walks into them.
|
|
12
16
|
*
|
|
13
17
|
* The dialect is the declarations': a project whose templates are Postgres
|
|
14
18
|
* reads its files as Postgres. With no templates it is `sql.dialect`, and
|
|
@@ -16,7 +20,7 @@
|
|
|
16
20
|
*/
|
|
17
21
|
|
|
18
22
|
import { readFileSync } from "node:fs";
|
|
19
|
-
import { relative, resolve } from "node:path";
|
|
23
|
+
import { relative, resolve, sep } from "node:path";
|
|
20
24
|
import type { Declarable } from "@intentius/chant/declarable";
|
|
21
25
|
import type { BuildRootContext, BuildRootContribution } from "@intentius/chant/lexicon";
|
|
22
26
|
import { findSourceFiles, hasDiscoveryMarkerSync } from "@intentius/chant/discovery/files";
|
|
@@ -36,6 +40,14 @@ export function isSqlSourceFile(name: string, full: string): boolean {
|
|
|
36
40
|
return !SQL_SKIP_MARKER.test(readHead(full, 1024));
|
|
37
41
|
}
|
|
38
42
|
|
|
43
|
+
/** A directory whose `.sql` files are migrations, never declarations. */
|
|
44
|
+
export const MIGRATIONS_DIR = "migrations";
|
|
45
|
+
|
|
46
|
+
/** Whether `file` sits under a {@link MIGRATIONS_DIR} directory below `sourceDir`. */
|
|
47
|
+
export function inMigrationsDir(sourceDir: string, file: string): boolean {
|
|
48
|
+
return relative(sourceDir, file).split(sep).slice(0, -1).includes(MIGRATIONS_DIR);
|
|
49
|
+
}
|
|
50
|
+
|
|
39
51
|
/** The dialect the project's `.sql` files are read as. */
|
|
40
52
|
export function filesDialect(entities: ReadonlyMap<string, Declarable> | undefined, config: Record<string, unknown>): SqlDialect {
|
|
41
53
|
for (const e of entities?.values() ?? []) {
|
|
@@ -54,7 +66,7 @@ export function filesDialect(entities: ReadonlyMap<string, Declarable> | undefin
|
|
|
54
66
|
/** The `buildRoots` contribution: every `.sql` file's declarations, or nothing when the source directory holds none. */
|
|
55
67
|
export async function sqlFilesBuildRoot(ctx: BuildRootContext): Promise<BuildRootContribution> {
|
|
56
68
|
const sourceDir = ctx.sourceDir ?? resolve(ctx.projectRoot, typeof ctx.config.sourceDir === "string" ? ctx.config.sourceDir : ".");
|
|
57
|
-
const files = (await findSourceFiles(sourceDir, isSqlSourceFile)).sort();
|
|
69
|
+
const files = (await findSourceFiles(sourceDir, isSqlSourceFile)).filter((f) => !inMigrationsDir(sourceDir, f)).sort();
|
|
58
70
|
if (files.length === 0) return { entities: new Map() };
|
|
59
71
|
const dialect = filesDialect(ctx.entities, ctx.config);
|
|
60
72
|
const sources = files.map((f) => ({ origin: relative(ctx.projectRoot, f) || f, ddl: readFileSync(f, "utf8") }));
|
package/src/files/clickhouse.ts
CHANGED
|
@@ -1,23 +1,79 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* A file of ClickHouse DDL as the objects it declares: one per `CREATE
|
|
3
3
|
* DATABASE`, `TABLE`, `VIEW`, `MATERIALIZED VIEW`, `DICTIONARY` or `FUNCTION`, as `chant import
|
|
4
|
-
* <file>.sql` reads them
|
|
5
|
-
*
|
|
4
|
+
* <file>.sql` reads them, and one per `CREATE USER`, `CREATE ROLE`, `CREATE
|
|
5
|
+
* ROW POLICY` and `GRANT` (#3711), each read by its tag, so what the tag
|
|
6
|
+
* refuses (`IDENTIFIED BY`, a `REVOKE`) is refused with the tag's message.
|
|
7
|
+
* Any other statement, and any statement the parser refuses, is a problem
|
|
8
|
+
* naming it.
|
|
6
9
|
*
|
|
7
10
|
* With `schema` (a database), an object's unqualified name is qualified with
|
|
8
|
-
* it
|
|
11
|
+
* it, and so is the table a row policy or a grant is on; a function, a user
|
|
12
|
+
* and a role belong to no database and keep their names.
|
|
9
13
|
*/
|
|
10
14
|
|
|
11
15
|
import type { Declarable } from "@intentius/chant/declarable";
|
|
12
16
|
import { isTrivia, tokenizeText } from "../clickhouse/tokens";
|
|
13
17
|
import { parseCreate } from "../clickhouse/parser";
|
|
14
|
-
import { database, dictionary, func, table, view } from "../clickhouse/entities";
|
|
18
|
+
import { CLICKHOUSE_ENTITY_TYPES, database, dictionary, func, grant, policy, role, table, user, view } from "../clickhouse/entities";
|
|
19
|
+
import { isAccessStatement, parseAccess, type AccessNode } from "../clickhouse/access";
|
|
20
|
+
import type { Token } from "../clickhouse/tokens";
|
|
15
21
|
import { CLICKHOUSE_TAG_OF } from "../clickhouse/import/generator";
|
|
16
22
|
import { describeStatement, type ImportedObject } from "../clickhouse/import/ir";
|
|
17
23
|
import { firstLine, quoteBare, splitTokens, tokensText, type ReadOptions } from "./common";
|
|
24
|
+
import { templateSafe, templateStrings } from "./postgres";
|
|
18
25
|
|
|
19
26
|
type Tag = (strings: TemplateStringsArray, ...values: unknown[]) => Declarable;
|
|
20
|
-
const TAGS: Record<string, Tag> = { database, table, view, dictionary, func };
|
|
27
|
+
const TAGS: Record<string, Tag> = { database, table, view, dictionary, func, user, role, policy, grant };
|
|
28
|
+
|
|
29
|
+
/** Tokens with `[db.]name`'s bare name qualified with `schema`. A `*` (all tables) is left alone. */
|
|
30
|
+
function qualifyBare(tokens: Token[], span: { from: number; to: number } | undefined, schema: string): string | undefined {
|
|
31
|
+
if (!span) return undefined;
|
|
32
|
+
const name = tokens.slice(span.from, span.to).filter((t) => !isTrivia(t));
|
|
33
|
+
const [only] = name;
|
|
34
|
+
if (!only || name.length !== 1 || (only.kind !== "ident" && only.kind !== "qident")) return undefined;
|
|
35
|
+
const at = tokens.indexOf(only);
|
|
36
|
+
return tokensText(tokens.slice(0, at)) + `${quoteBare(schema)}.` + tokensText(tokens.slice(at));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** An access statement as the object its tag declares (#3711), or the tag's refusal. */
|
|
40
|
+
function readAccess(sql: string, options: ReadOptions): ImportedObject | string {
|
|
41
|
+
let tokens = tokenizeText(sql, 0);
|
|
42
|
+
let node: AccessNode;
|
|
43
|
+
try {
|
|
44
|
+
node = parseAccess(tokens);
|
|
45
|
+
} catch (e) {
|
|
46
|
+
// What the tag refuses (a password) is refused in the tag's words.
|
|
47
|
+
const words = sql.replace(/--[^\n]*\n/g, " ").trim().split(/\s+/, 2).map((w) => w.toUpperCase());
|
|
48
|
+
const tag = words[0] === "CREATE" ? (words[1] === "USER" ? user : words[1] === "ROLE" ? role : policy) : grant;
|
|
49
|
+
try {
|
|
50
|
+
tag(templateStrings([templateSafe(sql)]));
|
|
51
|
+
} catch (refused) {
|
|
52
|
+
return refused instanceof Error ? refused.message : String(refused);
|
|
53
|
+
}
|
|
54
|
+
throw e;
|
|
55
|
+
}
|
|
56
|
+
if (options.schema) {
|
|
57
|
+
const qualified = qualifyBare(tokens, node.statement === "rowPolicy" ? node.table : node.statement === "grant" ? node.target : undefined, options.schema);
|
|
58
|
+
if (qualified !== undefined) {
|
|
59
|
+
sql = qualified;
|
|
60
|
+
tokens = tokenizeText(sql, 0);
|
|
61
|
+
node = parseAccess(tokens);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const tag = node.statement === "user" ? user : node.statement === "role" ? role : node.statement === "rowPolicy" ? policy : grant;
|
|
65
|
+
let props: { name: string; database?: string; table?: string };
|
|
66
|
+
try {
|
|
67
|
+
props = (tag(templateStrings([templateSafe(sql)])) as unknown as { props: typeof props }).props;
|
|
68
|
+
} catch (e) {
|
|
69
|
+
return e instanceof Error ? e.message : String(e);
|
|
70
|
+
}
|
|
71
|
+
if (node.statement === "user") return { type: CLICKHOUSE_ENTITY_TYPES.user, name: props.name, ddl: sql };
|
|
72
|
+
if (node.statement === "role") return { type: CLICKHOUSE_ENTITY_TYPES.role, name: props.name, ddl: sql };
|
|
73
|
+
if (node.statement === "rowPolicy") return { type: CLICKHOUSE_ENTITY_TYPES.rowPolicy, ...(props.database ? { database: props.database } : {}), name: props.name, ddl: sql };
|
|
74
|
+
// A grant has no name of its own: what it grants, on what, to whom.
|
|
75
|
+
return { type: CLICKHOUSE_ENTITY_TYPES.grant, name: `grant ${tokensText(tokens.slice(tokens.findIndex((t) => !isTrivia(t)) + 1)).replace(/\s+/g, " ").trim().toLowerCase()}`, ddl: sql };
|
|
76
|
+
}
|
|
21
77
|
|
|
22
78
|
/** The tag that declares an entity type. */
|
|
23
79
|
export function clickhouseTag(type: string): Tag | undefined {
|
|
@@ -30,6 +86,16 @@ export function readClickHouse(ddl: string, options: ReadOptions, problems: stri
|
|
|
30
86
|
const objects: ImportedObject[] = [];
|
|
31
87
|
for (const stmt of splitTokens(tokenizeText(ddl, 0), isTrivia)) {
|
|
32
88
|
let sql = tokensText(stmt);
|
|
89
|
+
if (isAccessStatement(sql)) {
|
|
90
|
+
try {
|
|
91
|
+
const read = readAccess(sql, options);
|
|
92
|
+
if (typeof read === "string") problems.push(`${read}: ${firstLine(sql)}`);
|
|
93
|
+
else objects.push(read);
|
|
94
|
+
} catch (e) {
|
|
95
|
+
problems.push(`does not parse (${e instanceof Error ? e.message : String(e)}): ${firstLine(sql)}`);
|
|
96
|
+
}
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
33
99
|
if (!/^CREATE\b/i.test(sql)) {
|
|
34
100
|
problems.push(`not a CREATE statement: ${firstLine(sql)}`);
|
|
35
101
|
continue;
|
package/src/files/postgres.ts
CHANGED
|
@@ -45,7 +45,8 @@ const isName = (t: Token | undefined): boolean => t !== undefined && (t.kind ===
|
|
|
45
45
|
const word = (t: Token | undefined, ...words: string[]): boolean => t !== undefined && t.kind === "ident" && words.includes(t.text.toUpperCase());
|
|
46
46
|
|
|
47
47
|
/** Text that sits inside a template literal: a backquote and `${` escaped. */
|
|
48
|
-
|
|
48
|
+
/** Text that sits inside a template literal: a backquote and `${` escaped. */
|
|
49
|
+
export const templateSafe = (s: string) => s.replace(/\\(?=`|\$\{)/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
|
|
49
50
|
|
|
50
51
|
interface PgFileObject {
|
|
51
52
|
node: StatementNode;
|
package/src/import-parser.ts
CHANGED
|
@@ -25,6 +25,8 @@ const CLICKHOUSE_ONLY = [
|
|
|
25
25
|
/^\s*CREATE\s+(?:DATABASE|(?:OR\s+REPLACE\s+)?DICTIONARY)\b/im,
|
|
26
26
|
// A SQL function: CREATE FUNCTION f AS (x) -> ...
|
|
27
27
|
/^\s*CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+[^\s(]+(?:\s+ON\s+CLUSTER\s+\S+)?\s+AS\s*\(/im,
|
|
28
|
+
// A row policy (#3711): Postgres says CREATE POLICY.
|
|
29
|
+
/^\s*CREATE\s+(?:OR\s+REPLACE\s+)?ROW\s+POLICY\b/im,
|
|
28
30
|
];
|
|
29
31
|
const POSTGRES_ONLY = [
|
|
30
32
|
/^\s*CREATE\s+(?:OR\s+REPLACE\s+)?(?:SCHEMA|EXTENSION|TYPE|DOMAIN|SEQUENCE|(?:UNIQUE\s+)?INDEX|(?:CONSTRAINT\s+)?TRIGGER|POLICY|ROLE|FUNCTION|PROCEDURE|(?:UNLOGGED|TEMP|TEMPORARY)\s+TABLE)\b/im,
|
|
@@ -111,6 +111,39 @@ describe("ClickHouse", () => {
|
|
|
111
111
|
expect((d.steps[0] as { detail: string }).detail).toMatch(/drop and create this object instead/);
|
|
112
112
|
});
|
|
113
113
|
|
|
114
|
+
test("a user the environment creates gets its declared clauses with ALTER USER, its password left alone (#3717)", () => {
|
|
115
|
+
const reader: Obj = { export: "reader", type: "ClickHouse::Role", ddl: "CREATE ROLE reader" };
|
|
116
|
+
const app: Obj = {
|
|
117
|
+
export: "app",
|
|
118
|
+
type: "ClickHouse::User",
|
|
119
|
+
dependsOn: ["reader", "analytics"],
|
|
120
|
+
ddl: "CREATE USER app HOST LOCAL DEFAULT ROLE reader DEFAULT DATABASE analytics SETTINGS max_memory_usage = 1000000000",
|
|
121
|
+
};
|
|
122
|
+
const readerToApp: Obj = { export: "readerToApp", type: "ClickHouse::Grant", dependsOn: ["reader", "app"], ddl: "GRANT reader TO app" };
|
|
123
|
+
const d = diffStatements(build("clickhouse", [DB]), build("clickhouse", [DB, reader, app, readerToApp]));
|
|
124
|
+
const sql = sqlOf(d);
|
|
125
|
+
expect(sql).toEqual([
|
|
126
|
+
"CREATE ROLE reader",
|
|
127
|
+
"ALTER USER `app` HOST LOCAL",
|
|
128
|
+
"ALTER USER `app` DEFAULT DATABASE analytics",
|
|
129
|
+
"ALTER USER `app` SETTINGS max_memory_usage = 1000000000",
|
|
130
|
+
"GRANT `reader` TO `app`",
|
|
131
|
+
// A default role must be held first.
|
|
132
|
+
"ALTER USER `app` DEFAULT ROLE reader",
|
|
133
|
+
]);
|
|
134
|
+
expect(sql.join("\n")).not.toMatch(/CREATE USER|IDENTIFIED/);
|
|
135
|
+
expect(d.steps.filter((s) => s.object === "app").map((s) => (s as StatementStep).rule)).toEqual(["SQLCH271", "SQLCH271", "SQLCH271"]);
|
|
136
|
+
|
|
137
|
+
// Without grants to it, the default role is set with the rest.
|
|
138
|
+
const alone = sqlOf(diffStatements(build("clickhouse", [DB, reader]), build("clickhouse", [DB, reader, app])));
|
|
139
|
+
expect(alone).toEqual([
|
|
140
|
+
"ALTER USER `app` HOST LOCAL",
|
|
141
|
+
"ALTER USER `app` DEFAULT DATABASE analytics",
|
|
142
|
+
"ALTER USER `app` SETTINGS max_memory_usage = 1000000000",
|
|
143
|
+
"ALTER USER `app` DEFAULT ROLE reader",
|
|
144
|
+
]);
|
|
145
|
+
});
|
|
146
|
+
|
|
114
147
|
test("two identical builds take no steps", () => {
|
|
115
148
|
expect(diffStatements(CH_BEFORE, CH_BEFORE)).toEqual({ dialect: "clickhouse", defaultDatabase: "default", topology: "single", steps: [], refused: false, hints: [] });
|
|
116
149
|
});
|
|
@@ -112,6 +112,8 @@ function clickhouseStatements(beforeJson: string, afterJson: string, options: Di
|
|
|
112
112
|
current: new Map(before.map((o) => [o.exportName, o.canonical])),
|
|
113
113
|
keyOf: (o) => o.exportName,
|
|
114
114
|
allowDestructive: true,
|
|
115
|
+
// A user whose password is the environment's exists by the time a migration runs: the GRANTs to it need it too (#3717).
|
|
116
|
+
environmentCreatesUsers: true,
|
|
115
117
|
topology,
|
|
116
118
|
...(options.marker ? { marker: options.marker } : {}),
|
|
117
119
|
});
|
|
@@ -62,7 +62,7 @@ jobs:
|
|
|
62
62
|
`;
|
|
63
63
|
|
|
64
64
|
exports[`generateOpsPipeline renders the sql drift watch (#3642) > GitLab: a schedule-only job with the reader's credentials and no token of its own 1`] = `
|
|
65
|
-
"# chant Ops
|
|
65
|
+
"# chant Ops — one job per Op under stage "ops", each
|
|
66
66
|
# selected by its own rules:. A merge_request_event or push job needs no
|
|
67
67
|
# setup; its rule fires on the event itself.
|
|
68
68
|
#
|
|
@@ -10,9 +10,28 @@
|
|
|
10
10
|
* `upstreamPin` already tracks the LTS tags (`-lts`).
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { emulatorLifecycle, type EmulatorCapability, type EmulatorSpec } from "@intentius/chant/op";
|
|
13
|
+
import { emulatorLifecycle, type EmulatorCapability, type EmulatorIdentity, type EmulatorSpec } from "@intentius/chant/op";
|
|
14
14
|
import { clickhouseImage } from "../../spec/pin";
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* The server's UUID (#3673), read in the container and through the endpoint
|
|
18
|
+
* `status` prints, so a server of another origin on the same port is told
|
|
19
|
+
* apart from the emulator.
|
|
20
|
+
*/
|
|
21
|
+
export const CLICKHOUSE_EMULATOR_IDENTITY: EmulatorIdentity = {
|
|
22
|
+
server: "ClickHouse",
|
|
23
|
+
command: ["clickhouse-client", "-q", "SELECT serverUUID()"],
|
|
24
|
+
async probe(endpoint) {
|
|
25
|
+
const url = new URL(endpoint);
|
|
26
|
+
url.searchParams.set("query", "SELECT serverUUID(), version() FORMAT TSV");
|
|
27
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(5_000) });
|
|
28
|
+
const body = (await res.text()).trim();
|
|
29
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}: ${body.split("\n")[0]}`);
|
|
30
|
+
const [id = "", version = "(unknown version)"] = body.split("\t");
|
|
31
|
+
return { id, label: `ClickHouse ${version}` };
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
|
|
16
35
|
export const CLICKHOUSE_EMULATOR_SPEC: EmulatorSpec = {
|
|
17
36
|
name: "chant-clickhouse",
|
|
18
37
|
image: clickhouseImage(),
|
|
@@ -21,6 +40,8 @@ export const CLICKHOUSE_EMULATOR_SPEC: EmulatorSpec = {
|
|
|
21
40
|
// The image disables network access for the passwordless default user
|
|
22
41
|
// unless this is set; a local server is reached as `default` with no password.
|
|
23
42
|
runArgs: ["-e", "CLICKHOUSE_SKIP_USER_SETUP=1"],
|
|
43
|
+
identity: CLICKHOUSE_EMULATOR_IDENTITY,
|
|
44
|
+
credentials: "user default, no password",
|
|
24
45
|
};
|
|
25
46
|
|
|
26
47
|
/** `CLICKHOUSE_URL` is the variable binding reads when no `sql.profiles.<env>` names a server. */
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The applier's catalog read under `lock_timeout` (#3726). One session holds
|
|
3
|
+
* ACCESS EXCLUSIVE on a table behind a view; `postgresApply` of a build that
|
|
4
|
+
* declares the view reads its definition first, so it stops after the
|
|
5
|
+
* profile's `lockTimeoutMs` instead of waiting for the lock, names the table
|
|
6
|
+
* and the pid holding it, and writes nothing.
|
|
7
|
+
*
|
|
8
|
+
* `CHANT_SQL_E2E_POSTGRES_URL` (with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names
|
|
9
|
+
* a running server, else a throwaway one, skipped when Docker is not
|
|
10
|
+
* available. The test makes only the schema `chant_e2e_3726`, dropped
|
|
11
|
+
* afterwards.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
15
|
+
import { tmpdir } from "node:os";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
|
18
|
+
import { dockerAvailable, startTestPostgres, type TestPostgres } from "../../postgres/testing/server";
|
|
19
|
+
import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../../postgres/live/client";
|
|
20
|
+
import { postgresApply } from "./postgres-apply";
|
|
21
|
+
|
|
22
|
+
const SCHEMA = "chant_e2e_3726";
|
|
23
|
+
const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
|
|
24
|
+
const enabled = given ? true : await dockerAvailable();
|
|
25
|
+
const dir = mkdtempSync(join(tmpdir(), "chant-pg-apply-3726-"));
|
|
26
|
+
let server: TestPostgres | undefined;
|
|
27
|
+
let endpoint: PostgresEndpoint;
|
|
28
|
+
let admin: PostgresClient;
|
|
29
|
+
let holder: PostgresClient;
|
|
30
|
+
|
|
31
|
+
describe.skipIf(!enabled)("an apply whose catalog read is blocked behind ACCESS EXCLUSIVE", () => {
|
|
32
|
+
beforeAll(async () => {
|
|
33
|
+
if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
|
|
34
|
+
else {
|
|
35
|
+
server = await startTestPostgres();
|
|
36
|
+
endpoint = server.endpoint();
|
|
37
|
+
}
|
|
38
|
+
admin = await connectPostgres(endpoint);
|
|
39
|
+
await admin.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);
|
|
40
|
+
await admin.query(`CREATE SCHEMA ${SCHEMA}`);
|
|
41
|
+
await admin.query(`CREATE TABLE ${SCHEMA}.orders (id bigint PRIMARY KEY)`);
|
|
42
|
+
await admin.query(`CREATE VIEW ${SCHEMA}.recent AS SELECT id FROM ${SCHEMA}.orders`);
|
|
43
|
+
holder = await connectPostgres(endpoint, { applicationName: "chant-e2e-3726-holder" });
|
|
44
|
+
await holder.query("BEGIN");
|
|
45
|
+
await holder.query(`LOCK TABLE ${SCHEMA}.orders IN ACCESS EXCLUSIVE MODE`);
|
|
46
|
+
}, 600_000);
|
|
47
|
+
|
|
48
|
+
afterAll(async () => {
|
|
49
|
+
await holder?.query("ROLLBACK").catch(() => undefined);
|
|
50
|
+
await holder?.end();
|
|
51
|
+
await admin?.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`).catch(() => undefined);
|
|
52
|
+
await admin?.end();
|
|
53
|
+
await server?.stop();
|
|
54
|
+
rmSync(dir, { recursive: true, force: true });
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test("stops after lockTimeoutMs, naming the table and the pid, with nothing written", async () => {
|
|
58
|
+
const [{ pid }] = await holder.query<{ pid: number }>("SELECT pg_catalog.pg_backend_pid() AS pid");
|
|
59
|
+
const buildPath = join(dir, "schema.json");
|
|
60
|
+
writeFileSync(
|
|
61
|
+
buildPath,
|
|
62
|
+
JSON.stringify({
|
|
63
|
+
dialect: "postgres",
|
|
64
|
+
applyOrder: ["app", "orders", "recent", "audit"],
|
|
65
|
+
objects: [
|
|
66
|
+
{ export: "app", type: "Postgres::Schema", dependsOn: [], ddl: `CREATE SCHEMA ${SCHEMA}` },
|
|
67
|
+
{ export: "orders", type: "Postgres::Table", dependsOn: ["app"], ddl: `CREATE TABLE ${SCHEMA}.orders (\n id bigint PRIMARY KEY\n)` },
|
|
68
|
+
{ export: "recent", type: "Postgres::View", dependsOn: ["orders"], ddl: `CREATE VIEW ${SCHEMA}.recent AS SELECT id FROM ${SCHEMA}.orders` },
|
|
69
|
+
{ export: "audit", type: "Postgres::Table", dependsOn: ["app"], ddl: `CREATE TABLE ${SCHEMA}.audit (\n id bigint\n)` },
|
|
70
|
+
],
|
|
71
|
+
}),
|
|
72
|
+
);
|
|
73
|
+
const started = Date.now();
|
|
74
|
+
const err = (await postgresApply({ buildPath, environment: "e2e" }, undefined, {
|
|
75
|
+
config: { ownership: { stack: "e2e3726", env: "e2e" }, sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_3726_PASSWORD" }, schemas: [SCHEMA], lockTimeoutMs: 300 } } } },
|
|
76
|
+
env: { PG_E2E_3726_PASSWORD: endpoint.password ?? "" },
|
|
77
|
+
log: () => undefined,
|
|
78
|
+
}).catch((e: unknown) => e)) as Error;
|
|
79
|
+
expect(Date.now() - started).toBeLessThan(10_000);
|
|
80
|
+
expect(err).toBeInstanceOf(Error);
|
|
81
|
+
expect(err.message).toContain("lock timeout: the catalog read waited 300ms for a lock");
|
|
82
|
+
expect(err.message).toContain(`${SCHEMA}.orders is held in ACCESS EXCLUSIVE by pid ${pid}`);
|
|
83
|
+
expect(err.message).toContain("chant-e2e-3726-holder");
|
|
84
|
+
const [audit] = await admin.query<{ n: number }>(`SELECT count(*)::int AS n FROM pg_catalog.pg_tables WHERE schemaname = $1 AND tablename = 'audit'`, [SCHEMA]);
|
|
85
|
+
expect(audit?.n).toBe(0);
|
|
86
|
+
}, 60_000);
|
|
87
|
+
});
|
|
@@ -8,6 +8,7 @@ import { buildMajor, postgresApply, type PostgresApplyArgs, type PostgresApplyDe
|
|
|
8
8
|
import { toApplyResult } from "./index";
|
|
9
9
|
import { writablePostgres, type StoredPgObject, type WritablePostgres } from "../../postgres/testing/writable-server";
|
|
10
10
|
import { PostgresApplyError } from "../../postgres/apply/apply";
|
|
11
|
+
import { PostgresQueryError } from "../../postgres/live/client";
|
|
11
12
|
|
|
12
13
|
const dirs: string[] = [];
|
|
13
14
|
afterAll(() => {
|
|
@@ -213,6 +214,37 @@ describe("a statement the server refuses", () => {
|
|
|
213
214
|
});
|
|
214
215
|
});
|
|
215
216
|
|
|
217
|
+
describe("the catalog read before applying (#3726)", () => {
|
|
218
|
+
test("runs under the profile's lockTimeoutMs, else 5000 ms, else the argument's", async () => {
|
|
219
|
+
for (const [profile, arg, expected] of [[undefined, undefined, "5000ms"], [250, undefined, "250ms"], [250, 75, "75ms"]] as const) {
|
|
220
|
+
const s = writablePostgres();
|
|
221
|
+
const sent: unknown[][] = [];
|
|
222
|
+
const config = { ownership: { stack: "shop", env: "prod" }, sql: { profiles: { test: { url: "postgres://fake/shop", ...(profile !== undefined ? { lockTimeoutMs: profile } : {}) } } } };
|
|
223
|
+
const client = { query: (sql: string, p?: readonly unknown[]) => (sent.push([sql, ...(p ?? [])]), s.client.query(sql)), end: s.client.end };
|
|
224
|
+
await apply(s, [APP], arg !== undefined ? { lockTimeoutMs: arg } : {}, { config, env: {}, connect: async () => client as never });
|
|
225
|
+
expect(sent[0]).toEqual(["SELECT pg_catalog.set_config('lock_timeout', $1, false)", expected]);
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
test("blocked behind ACCESS EXCLUSIVE, it stops before any write, naming the relation and the pid holding it", async () => {
|
|
230
|
+
const s = writablePostgres();
|
|
231
|
+
const client = {
|
|
232
|
+
async query<T>(sql: string): Promise<T[]> {
|
|
233
|
+
if (sql.includes("pg_get_viewdef")) throw new PostgresQueryError("canceling statement due to lock timeout", "55P03");
|
|
234
|
+
if (sql.includes("pg_catalog.pg_locks")) return [{ pid: 4242, relation: "app.users", application: "psql", state: "idle in transaction", seconds: 9, query: "LOCK TABLE app.users" }] as T[];
|
|
235
|
+
return s.client.query<T>(sql);
|
|
236
|
+
},
|
|
237
|
+
end: s.client.end,
|
|
238
|
+
};
|
|
239
|
+
const err = (await apply(s, ALL, {}, { connect: async () => client as never, readLive: async (c) => (await c.query("SELECT pg_catalog.pg_get_viewdef(1)"), []) }).catch((e: unknown) => e)) as Error;
|
|
240
|
+
expect(err).toBeInstanceOf(PostgresQueryError);
|
|
241
|
+
expect(err.message).toContain("canceling statement due to lock timeout: the catalog read waited 5000ms for a lock");
|
|
242
|
+
expect(err.message).toContain("app.users is held in ACCESS EXCLUSIVE by pid 4242");
|
|
243
|
+
expect(s.writes).toEqual([]);
|
|
244
|
+
expect(s.control).toEqual([]);
|
|
245
|
+
});
|
|
246
|
+
});
|
|
247
|
+
|
|
216
248
|
describe("the target major", () => {
|
|
217
249
|
test("is the build output's postgresMajor", () => {
|
|
218
250
|
expect(buildMajor(JSON.stringify({ dialect: "postgres", postgresMajor: 16, objects: [] }))).toBe(16);
|
|
@@ -24,3 +24,23 @@ describe("the Postgres emulator capability (#3280)", () => {
|
|
|
24
24
|
expect(lc.endpoint(5432)).toBe("postgres://localhost:5432/postgres");
|
|
25
25
|
});
|
|
26
26
|
});
|
|
27
|
+
|
|
28
|
+
describe("the sql emulators say who they are (#3673)", () => {
|
|
29
|
+
test("Postgres compares the cluster's system identifier, read over the container's socket", () => {
|
|
30
|
+
expect(POSTGRES_EMULATOR.spec.identity?.server).toBe("Postgres");
|
|
31
|
+
expect(POSTGRES_EMULATOR.spec.identity?.command).toEqual(["psql", "-U", "postgres", "-tAc", "select system_identifier from pg_control_system()"]);
|
|
32
|
+
expect(POSTGRES_EMULATOR.spec.credentials).toBe("user postgres, password chant");
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("ClickHouse compares serverUUID(), and signs in as default with no password", () => {
|
|
36
|
+
expect(CLICKHOUSE_EMULATOR.spec.identity?.server).toBe("ClickHouse");
|
|
37
|
+
expect(CLICKHOUSE_EMULATOR.spec.identity?.command).toEqual(["clickhouse-client", "-q", "SELECT serverUUID()"]);
|
|
38
|
+
expect(CLICKHOUSE_EMULATOR.spec.credentials).toBe("user default, no password");
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("a server that is not Postgres fails the Postgres probe instead of matching", async () => {
|
|
42
|
+
// Nothing listens on port 1; the probe rejects, which status reports as "does not answer as the emulator".
|
|
43
|
+
await expect(POSTGRES_EMULATOR.spec.identity!.probe("postgres://127.0.0.1:1/postgres")).rejects.toThrow();
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
|
|
@@ -15,12 +15,50 @@
|
|
|
15
15
|
* `upstreamPins`, not the emulator freshness check.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import { emulatorLifecycle, type EmulatorCapability, type EmulatorSpec } from "@intentius/chant/op";
|
|
18
|
+
import { emulatorLifecycle, type EmulatorCapability, type EmulatorIdentity, type EmulatorSpec } from "@intentius/chant/op";
|
|
19
19
|
import { POSTGRES_LATEST_MAJOR, postgresImage } from "../../spec/postgres-pin";
|
|
20
|
+
import { connectPostgres } from "../../postgres/live/client";
|
|
20
21
|
|
|
21
22
|
/** The local server's password. It is a throwaway server on localhost. */
|
|
22
23
|
export const POSTGRES_EMULATOR_PASSWORD = "chant";
|
|
23
24
|
|
|
25
|
+
/** How long `chant emulator status` waits for the server behind the endpoint. */
|
|
26
|
+
const PROBE_TIMEOUT_MS = 5_000;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The cluster's system identifier, unique per `initdb` (#3673): read in the
|
|
30
|
+
* container over its socket, and through the endpoint `status` prints. A
|
|
31
|
+
* native Postgres holding 127.0.0.1:5432 answers with its own, or refuses
|
|
32
|
+
* the emulator's sign-in.
|
|
33
|
+
*/
|
|
34
|
+
export const POSTGRES_EMULATOR_IDENTITY: EmulatorIdentity = {
|
|
35
|
+
server: "Postgres",
|
|
36
|
+
command: ["psql", "-U", "postgres", "-tAc", "select system_identifier from pg_control_system()"],
|
|
37
|
+
async probe(endpoint) {
|
|
38
|
+
const client = await withTimeout(
|
|
39
|
+
connectPostgres({ url: endpoint, user: "postgres", password: POSTGRES_EMULATOR_PASSWORD }, { applicationName: "chant emulator status" }),
|
|
40
|
+
);
|
|
41
|
+
try {
|
|
42
|
+
const [row] = await withTimeout(
|
|
43
|
+
client.query<{ id: string; version: string }>(
|
|
44
|
+
"SELECT system_identifier::text AS id, pg_catalog.current_setting('server_version') AS version FROM pg_catalog.pg_control_system()",
|
|
45
|
+
),
|
|
46
|
+
);
|
|
47
|
+
return { id: row?.id ?? "", label: `PostgreSQL ${row?.version ?? "(unknown version)"}` };
|
|
48
|
+
} finally {
|
|
49
|
+
await client.end();
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
function withTimeout<T>(promise: Promise<T>): Promise<T> {
|
|
55
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
56
|
+
const timeout = new Promise<never>((_, reject) => {
|
|
57
|
+
timer = setTimeout(() => reject(new Error(`no answer within ${PROBE_TIMEOUT_MS / 1000}s`)), PROBE_TIMEOUT_MS);
|
|
58
|
+
});
|
|
59
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
60
|
+
}
|
|
61
|
+
|
|
24
62
|
export const POSTGRES_EMULATOR_SPEC: EmulatorSpec = {
|
|
25
63
|
name: "chant-postgres",
|
|
26
64
|
image: postgresImage(POSTGRES_LATEST_MAJOR),
|
|
@@ -28,6 +66,8 @@ export const POSTGRES_EMULATOR_SPEC: EmulatorSpec = {
|
|
|
28
66
|
readyCommand: ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"],
|
|
29
67
|
endpoint: (port) => `postgres://localhost:${port}/postgres`,
|
|
30
68
|
runArgs: ["-e", `POSTGRES_PASSWORD=${POSTGRES_EMULATOR_PASSWORD}`],
|
|
69
|
+
identity: POSTGRES_EMULATOR_IDENTITY,
|
|
70
|
+
credentials: `user postgres, password ${POSTGRES_EMULATOR_PASSWORD}`,
|
|
31
71
|
};
|
|
32
72
|
|
|
33
73
|
/** `POSTGRES_URL` is the variable binding reads when no `sql.profiles.<env>` names a server. */
|