@intentius/chant-lexicon-sql 0.117.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clickhouse/access.d.ts +132 -0
- package/dist/clickhouse/access.d.ts.map +1 -0
- package/dist/clickhouse/apply/apply.d.ts.map +1 -1
- package/dist/clickhouse/apply/statements.d.ts +2 -0
- package/dist/clickhouse/apply/statements.d.ts.map +1 -1
- package/dist/clickhouse/entities.d.ts +171 -2
- package/dist/clickhouse/entities.d.ts.map +1 -1
- package/dist/clickhouse/http.d.ts +3 -0
- package/dist/clickhouse/http.d.ts.map +1 -1
- package/dist/clickhouse/import/generator.d.ts +28 -0
- package/dist/clickhouse/import/generator.d.ts.map +1 -1
- package/dist/clickhouse/import/ir.d.ts.map +1 -1
- package/dist/clickhouse/import/parser.d.ts +3 -3
- package/dist/clickhouse/import/parser.d.ts.map +1 -1
- package/dist/clickhouse/live/bind.d.ts.map +1 -1
- package/dist/clickhouse/live/catalog.d.ts +41 -6
- package/dist/clickhouse/live/catalog.d.ts.map +1 -1
- package/dist/clickhouse/live/describe-resources.d.ts +3 -2
- package/dist/clickhouse/live/describe-resources.d.ts.map +1 -1
- package/dist/clickhouse/parser.d.ts +54 -3
- package/dist/clickhouse/parser.d.ts.map +1 -1
- package/dist/clickhouse/plan/commands.d.ts +3 -1
- package/dist/clickhouse/plan/commands.d.ts.map +1 -1
- package/dist/clickhouse/plan/deep.d.ts.map +1 -1
- package/dist/clickhouse/plan/diff.d.ts +12 -0
- package/dist/clickhouse/plan/diff.d.ts.map +1 -1
- package/dist/clickhouse/plan/normalize.d.ts +61 -1
- package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
- package/dist/clickhouse/plan/report.d.ts.map +1 -1
- package/dist/clickhouse/plan/rules.d.ts +7 -0
- package/dist/clickhouse/plan/rules.d.ts.map +1 -1
- package/dist/clickhouse/plan/schema.d.ts +8 -4
- package/dist/clickhouse/plan/schema.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/backfill.d.ts +3 -1
- package/dist/clickhouse/rebuild/backfill.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/observe.d.ts +7 -3
- package/dist/clickhouse/rebuild/observe.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/op.d.ts +8 -0
- package/dist/clickhouse/rebuild/op.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/steps.d.ts +48 -0
- package/dist/clickhouse/rebuild/steps.d.ts.map +1 -1
- package/dist/clickhouse/testing/fake-server.d.ts.map +1 -1
- package/dist/clickhouse/testing/writable-server.d.ts.map +1 -1
- package/dist/clickhouse/topology.d.ts.map +1 -1
- package/dist/clickhouse.d.ts +1 -1
- package/dist/clickhouse.d.ts.map +1 -1
- package/dist/codegen/classifier-page.d.ts.map +1 -1
- package/dist/codegen/registry.d.ts.map +1 -1
- package/dist/config.d.ts +50 -7
- package/dist/config.d.ts.map +1 -1
- package/dist/core/references.d.ts +4 -2
- package/dist/core/references.d.ts.map +1 -1
- package/dist/core/statements.d.ts +11 -0
- package/dist/core/statements.d.ts.map +1 -1
- package/dist/files/build-root.d.ts +26 -0
- package/dist/files/build-root.d.ts.map +1 -0
- package/dist/files/clickhouse.d.ts +19 -0
- package/dist/files/clickhouse.d.ts.map +1 -0
- package/dist/files/common.d.ts +25 -0
- package/dist/files/common.d.ts.map +1 -0
- package/dist/files/postgres.d.ts +36 -0
- package/dist/files/postgres.d.ts.map +1 -0
- package/dist/import-parser.d.ts +25 -0
- package/dist/import-parser.d.ts.map +1 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/integrity.json +6 -6
- package/dist/lint/rules/sqlch001.d.ts +1 -1
- package/dist/lint/rules/sqlch001.d.ts.map +1 -1
- package/dist/lint/rules/sqlch002.d.ts.map +1 -1
- package/dist/lint/rules/templates.d.ts +1 -1
- package/dist/lint/rules/templates.d.ts.map +1 -1
- package/dist/manifest.json +1 -1
- package/dist/meta.json +30 -0
- package/dist/okf/index.md +7 -1
- package/dist/okf/rules/SQLCH001.md +2 -2
- package/dist/okf/types/Dictionary.md +9 -0
- package/dist/okf/types/Function.md +9 -0
- package/dist/okf/types/Grant.md +9 -0
- package/dist/okf/types/Role.md +9 -0
- package/dist/okf/types/RowPolicy.md +9 -0
- package/dist/okf/types/User.md +9 -0
- package/dist/op/activities/clickhouse-rebuild.d.ts.map +1 -1
- package/dist/op/activity-contracts.d.ts +9 -0
- package/dist/op/activity-contracts.d.ts.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/postgres/apply/statements.d.ts +68 -0
- package/dist/postgres/apply/statements.d.ts.map +1 -1
- package/dist/postgres/import/generator.d.ts +33 -0
- package/dist/postgres/import/generator.d.ts.map +1 -1
- package/dist/postgres/live/bind.d.ts.map +1 -1
- package/dist/postgres/live/client.d.ts +3 -0
- package/dist/postgres/live/client.d.ts.map +1 -1
- package/dist/rules/sqlch001.ts +26 -4
- package/dist/rules/sqlch002.ts +3 -2
- package/dist/rules/templates.ts +2 -2
- package/dist/serializer.d.ts +2 -1
- package/dist/serializer.d.ts.map +1 -1
- package/dist/sql-files.d.ts +79 -0
- package/dist/sql-files.d.ts.map +1 -0
- package/dist/token-source.d.ts +88 -0
- package/dist/token-source.d.ts.map +1 -0
- package/dist/validate.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/clickhouse/access.test.ts +203 -0
- package/src/clickhouse/access.ts +595 -0
- package/src/clickhouse/apply/apply.ts +29 -20
- package/src/clickhouse/apply/statements.ts +105 -8
- package/src/clickhouse/dictionary.test.ts +177 -0
- package/src/clickhouse/entities.ts +356 -6
- package/src/clickhouse/function.test.ts +91 -0
- package/src/clickhouse/http.ts +11 -2
- package/src/clickhouse/import/generator.ts +81 -46
- package/src/clickhouse/import/import.test.ts +9 -3
- package/src/clickhouse/import/ir.ts +6 -2
- package/src/clickhouse/import/parser.ts +10 -18
- package/src/clickhouse/live/access.e2e.test.ts +218 -0
- package/src/clickhouse/live/bind.ts +17 -6
- package/src/clickhouse/live/catalog.ts +106 -10
- package/src/clickhouse/live/describe-resources.ts +23 -10
- package/src/clickhouse/live/dictionaries.e2e.test.ts +186 -0
- package/src/clickhouse/live/functions.e2e.test.ts +155 -0
- package/src/clickhouse/live/live.test.ts +2 -2
- package/src/clickhouse/parser.ts +136 -4
- package/src/clickhouse/plan/commands.ts +18 -7
- package/src/clickhouse/plan/deep.ts +71 -11
- package/src/clickhouse/plan/diff.ts +65 -2
- package/src/clickhouse/plan/normalize.ts +200 -6
- package/src/clickhouse/plan/report.ts +14 -3
- package/src/clickhouse/plan/rules.ts +44 -1
- package/src/clickhouse/plan/schema.ts +40 -17
- package/src/clickhouse/plan/server-format.ts +1 -1
- package/src/clickhouse/rebuild/backfill.ts +5 -25
- package/src/clickhouse/rebuild/observe.ts +7 -3
- package/src/clickhouse/rebuild/op.ts +9 -0
- package/src/clickhouse/rebuild/rebuild.e2e.test.ts +62 -0
- package/src/clickhouse/rebuild/rebuild.test.ts +54 -0
- package/src/clickhouse/rebuild/steps.ts +136 -1
- package/src/clickhouse/testing/fake-server.ts +2 -0
- package/src/clickhouse/testing/writable-server.ts +2 -0
- package/src/clickhouse/topology.ts +17 -2
- package/src/clickhouse.ts +19 -0
- package/src/codegen/classifier-page.ts +2 -0
- package/src/codegen/registry.ts +6 -0
- package/src/config.ts +12 -3
- package/src/core/references.ts +5 -3
- package/src/core/statements.ts +11 -1
- package/src/files/build-root.ts +62 -0
- package/src/files/clickhouse.ts +52 -0
- package/src/files/common.ts +54 -0
- package/src/files/postgres.ts +371 -0
- package/src/generated/lexicon-sql.json +30 -0
- package/src/import-parser.ts +61 -0
- package/src/index.ts +15 -2
- package/src/lint/rules/sqlch001.ts +26 -4
- package/src/lint/rules/sqlch002.ts +3 -2
- package/src/lint/rules/templates.ts +2 -2
- package/src/lsp/template.ts +3 -3
- package/src/mcp/index.ts +6 -6
- package/src/migration-statements.test.ts +25 -0
- package/src/migration-statements.ts +1 -0
- package/src/op/activities/clickhouse-rebuild.ts +1 -0
- package/src/op/activity-contracts.ts +1 -0
- package/src/plugin.ts +29 -8
- package/src/postgres/apply/constraint-sequences.e2e.test.ts +179 -0
- package/src/postgres/apply/safe-sequences.e2e.test.ts +176 -0
- package/src/postgres/apply/statements.test.ts +138 -6
- package/src/postgres/apply/statements.ts +156 -10
- package/src/postgres/import/generator.ts +118 -45
- package/src/postgres/live/bind.ts +17 -6
- package/src/postgres/live/client.ts +10 -2
- package/src/postgres/plan/plan.test.ts +2 -2
- package/src/postgres/plan/rules.ts +5 -5
- package/src/serializer.ts +14 -9
- package/src/skills/chant-sql-postgres-plan.md +3 -3
- package/src/skills/chant-sql-rebuild.md +3 -2
- package/src/skills/chant-sql.md +7 -1
- package/src/sql-files.test.ts +188 -0
- package/src/sql-files.ts +218 -0
- package/src/token-source.e2e.test.ts +142 -0
- package/src/token-source.test.ts +119 -0
- package/src/token-source.ts +189 -0
- package/src/validate.ts +6 -0
|
@@ -24,6 +24,7 @@ import { CLASSIFIER_RULES, type ChangeClass, type ClassifierRuleId } from "../pl
|
|
|
24
24
|
import { stampedComment } from "../ownership";
|
|
25
25
|
import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "../entities";
|
|
26
26
|
import { renderFor, type Topology } from "../topology";
|
|
27
|
+
import { accessClauseText, alterAccessField, grantStatement, revokeStatement, SECRETLESS, type GrantAtom } from "../access";
|
|
27
28
|
|
|
28
29
|
/** A backquoted identifier. */
|
|
29
30
|
export const ident = (name: string): string => `\`${name.replace(/`/g, "``")}\``;
|
|
@@ -124,6 +125,11 @@ export function createStatement(
|
|
|
124
125
|
opts: { orReplace?: boolean; trailer?: Readonly<Record<string, string>> } = {},
|
|
125
126
|
): string {
|
|
126
127
|
const { tokens, node } = parsed(obj);
|
|
128
|
+
// A function has no comment to carry the marker: it is sent as declared.
|
|
129
|
+
if (node.statement === "function") {
|
|
130
|
+
const body = statementBody(tokens);
|
|
131
|
+
return opts.orReplace && !node.orReplace ? body.replace(/^(\s*(?:--[^\n]*\n\s*)*)CREATE\s+/i, "$1CREATE OR REPLACE ") : body;
|
|
132
|
+
}
|
|
127
133
|
const literal = sqlString(stampedComment(obj.canonical.comment, marker, opts.trailer));
|
|
128
134
|
let sql: string;
|
|
129
135
|
if (node.comment) {
|
|
@@ -137,7 +143,7 @@ export function createStatement(
|
|
|
137
143
|
} else {
|
|
138
144
|
sql = `${statementBody(tokens)} COMMENT ${literal}`;
|
|
139
145
|
}
|
|
140
|
-
if (opts.orReplace && node.statement === "view" && !node.orReplace) {
|
|
146
|
+
if (opts.orReplace && (node.statement === "view" || node.statement === "dictionary") && !node.orReplace) {
|
|
141
147
|
sql = sql.replace(/^(\s*(?:--[^\n]*\n\s*)*)CREATE\s+/i, "$1CREATE OR REPLACE ");
|
|
142
148
|
}
|
|
143
149
|
return sql;
|
|
@@ -154,7 +160,8 @@ export function commentStatement(obj: DeclaredObject, marker: OwnershipMarker |
|
|
|
154
160
|
/** The `DROP` for an object a prune removes. `SYNC` so the name is free when it returns. */
|
|
155
161
|
export function dropStatement(type: string, database: string | undefined, name: string): string {
|
|
156
162
|
if (type === CLICKHOUSE_ENTITY_TYPES.database) return `DROP DATABASE ${ident(name)} SYNC`;
|
|
157
|
-
|
|
163
|
+
if (type === CLICKHOUSE_ENTITY_TYPES.function) return `DROP FUNCTION ${ident(name)}`;
|
|
164
|
+
const what = type === CLICKHOUSE_ENTITY_TYPES.table ? "TABLE" : type === CLICKHOUSE_ENTITY_TYPES.dictionary ? "DICTIONARY" : "VIEW";
|
|
158
165
|
return `DROP ${what} ${qualifiedIdent(database, name)} SYNC`;
|
|
159
166
|
}
|
|
160
167
|
|
|
@@ -388,7 +395,15 @@ export function alterSteps(obj: DeclaredObject, changes: readonly Change[], opts
|
|
|
388
395
|
for (const c of changes.filter((x) => x.rule === "SQLCH230")) {
|
|
389
396
|
const from = splitQualified(c.before!);
|
|
390
397
|
const to = splitQualified(c.after!);
|
|
391
|
-
|
|
398
|
+
const what = obj.type === CLICKHOUSE_ENTITY_TYPES.dictionary ? "DICTIONARY" : "TABLE";
|
|
399
|
+
steps.push(stepFor(`RENAME ${what} ${qualifiedIdent(from.database, from.name)} TO ${qualifiedIdent(to.database, to.name)}`, "SQLCH230"));
|
|
400
|
+
}
|
|
401
|
+
if (obj.type === CLICKHOUSE_ENTITY_TYPES.function && changes.some((c) => c.rule === "SQLCH260")) {
|
|
402
|
+
steps.push(stepFor(createStatement(obj, opts.marker, { orReplace: true }), "SQLCH260"));
|
|
403
|
+
}
|
|
404
|
+
// A dictionary is replaced whole, its comment and marker with it.
|
|
405
|
+
if (obj.type === CLICKHOUSE_ENTITY_TYPES.dictionary && changes.some((c) => c.rule === "SQLCH245")) {
|
|
406
|
+
steps.push(stepFor(createStatement(obj, opts.marker, { orReplace: true }), "SQLCH245"));
|
|
392
407
|
}
|
|
393
408
|
if (obj.type === CLICKHOUSE_ENTITY_TYPES.table) steps.push(...tableSteps(obj, changes, opts.live));
|
|
394
409
|
else if (obj.type === CLICKHOUSE_ENTITY_TYPES.view || obj.type === CLICKHOUSE_ENTITY_TYPES.materializedView) steps.push(...viewSteps(obj, changes, opts.marker));
|
|
@@ -397,7 +412,10 @@ export function alterSteps(obj: DeclaredObject, changes: readonly Change[], opts
|
|
|
397
412
|
|
|
398
413
|
/** Whether the steps for these changes already set the object's comment (a plain view replaced whole). */
|
|
399
414
|
export function stepsSetComment(obj: DeclaredObject, changes: readonly Change[]): boolean {
|
|
400
|
-
return
|
|
415
|
+
return (
|
|
416
|
+
(obj.type === CLICKHOUSE_ENTITY_TYPES.view && changes.some((c) => c.rule === "SQLCH240")) ||
|
|
417
|
+
(obj.type === CLICKHOUSE_ENTITY_TYPES.dictionary && changes.some((c) => c.rule === "SQLCH245"))
|
|
418
|
+
);
|
|
401
419
|
}
|
|
402
420
|
|
|
403
421
|
// ── a schema's statements ─────────────────────────────────────────────
|
|
@@ -459,8 +477,8 @@ export interface StatementPlan {
|
|
|
459
477
|
drops: DropStatement[];
|
|
460
478
|
}
|
|
461
479
|
|
|
462
|
-
/** Drop order: what reads from a table before the table, a database last. */
|
|
463
|
-
const DROP_ORDER: Record<string, number> = { materializedView: 0, view: 1,
|
|
480
|
+
/** Drop order: what reads from a table before the table (a dictionary before its source), a database last. */
|
|
481
|
+
const DROP_ORDER: Record<string, number> = { materializedView: 0, view: 1, dictionary: 2, table: 3, database: 4, function: 5 };
|
|
464
482
|
|
|
465
483
|
/**
|
|
466
484
|
* The statements that take the current schema to the declared one, from the
|
|
@@ -481,14 +499,89 @@ export function planStatements(input: StatementPlanInput): StatementPlan {
|
|
|
481
499
|
};
|
|
482
500
|
}
|
|
483
501
|
|
|
502
|
+
const NEVER_DROPPED: ReadonlySet<string> = new Set(["function", "user", "role", "rowPolicy", "grants"]);
|
|
503
|
+
|
|
504
|
+
/** The entity types of access control (`../access.ts`): no comment, no marker, never dropped. */
|
|
505
|
+
export const ACCESS_TYPES: ReadonlySet<string> = new Set([
|
|
506
|
+
CLICKHOUSE_ENTITY_TYPES.user,
|
|
507
|
+
CLICKHOUSE_ENTITY_TYPES.role,
|
|
508
|
+
CLICKHOUSE_ENTITY_TYPES.rowPolicy,
|
|
509
|
+
CLICKHOUSE_ENTITY_TYPES.grant,
|
|
510
|
+
]);
|
|
511
|
+
|
|
512
|
+
/** What a user, role, row policy or grantee's grants take (`../access.ts`). */
|
|
513
|
+
function accessStatements(obj: DeclaredObject, mine: Change[], live: CanonicalObject | undefined, deferred: DeferredDefaultRoles): ObjectStatements {
|
|
514
|
+
const kind = obj.canonical.kind;
|
|
515
|
+
const name = obj.canonical.name;
|
|
516
|
+
if (kind === "grants") {
|
|
517
|
+
const atom = (json: string | undefined) => JSON.parse(json ?? "{}") as GrantAtom;
|
|
518
|
+
const declared = obj.canonical.access ?? {};
|
|
519
|
+
const current = live?.access ?? {};
|
|
520
|
+
const created = mine.some((c) => c.rule === "SQLCH200");
|
|
521
|
+
const revokes = created ? [] : mine.filter((c) => c.rule === "SQLCH274").map((c) => stepFor(revokeStatement(atom(current[c.before as string]), name), "SQLCH274"));
|
|
522
|
+
const grants = (created ? Object.keys(declared) : mine.filter((c) => c.rule === "SQLCH273").map((c) => c.after as string)).map((key) =>
|
|
523
|
+
stepFor(grantStatement(atom(declared[key]), name), "SQLCH273"),
|
|
524
|
+
);
|
|
525
|
+
// A user's DEFAULT ROLE names roles it must already hold: set after its grants.
|
|
526
|
+
const defaultRole = deferred.get(name);
|
|
527
|
+
const after = defaultRole ? [defaultRole.step] : [];
|
|
528
|
+
if (defaultRole) mine = [...mine, defaultRole.change];
|
|
529
|
+
return { verdict: created ? "create" : "alter", obj, changes: mine, steps: [...revokes, ...grants, ...after] };
|
|
530
|
+
}
|
|
531
|
+
const body = statementBody(tokenizeText(obj.ddl, 0));
|
|
532
|
+
if (mine.some((c) => c.rule === "SQLCH200")) {
|
|
533
|
+
const method = obj.canonical.access?.identified?.split(" ")[0];
|
|
534
|
+
if (kind === "user" && (method === undefined || !SECRETLESS.has(method))) {
|
|
535
|
+
return {
|
|
536
|
+
verdict: "withheld",
|
|
537
|
+
obj,
|
|
538
|
+
changes: mine,
|
|
539
|
+
withheld: mine,
|
|
540
|
+
detail: `the user's password is the environment's, so chant does not create it: create ${name} with its password (CREATE USER ${ident(name)} IDENTIFIED BY ...), and the next apply makes the rest of it as declared`,
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
return { verdict: "create", obj, changes: mine, steps: [stepFor(body, "SQLCH200")] };
|
|
544
|
+
}
|
|
545
|
+
if (kind === "rowPolicy") {
|
|
546
|
+
// `CREATE ROW POLICY OR REPLACE`: ClickHouse takes OR REPLACE after the kind.
|
|
547
|
+
const replace = /^\s*(?:--[^\n]*\n\s*)*CREATE\s+(?:ROW\s+)?POLICY\s+OR\s+REPLACE\b/i.test(body)
|
|
548
|
+
? body
|
|
549
|
+
: body.replace(/^(\s*(?:--[^\n]*\n\s*)*CREATE\s+(?:ROW\s+)?POLICY)\s+(?:IF\s+NOT\s+EXISTS\s+)?/i, "$1 OR REPLACE ");
|
|
550
|
+
return { verdict: "alter", obj, changes: mine, steps: mine.some((c) => c.rule === "SQLCH272") ? [stepFor(replace, "SQLCH272")] : [] };
|
|
551
|
+
}
|
|
552
|
+
const clauses = accessClauseText(obj.ddl);
|
|
553
|
+
const rule = kind === "user" ? "SQLCH271" : "SQLCH270";
|
|
554
|
+
const steps = mine
|
|
555
|
+
.filter((c) => c.rule === rule && !(kind === "user" && c.field === "defaultRole" && deferred.has(name)))
|
|
556
|
+
.map((c) => stepFor(alterAccessField(kind as "user" | "role", name, c.field, obj.canonical.access ?? {}, clauses), rule));
|
|
557
|
+
return { verdict: "alter", obj, changes: mine, steps };
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** The `ALTER USER ... DEFAULT ROLE` of each user whose grants the build also declares, by user: it runs after them. */
|
|
561
|
+
type DeferredDefaultRoles = Map<string, { step: Step; change: Change }>;
|
|
562
|
+
|
|
563
|
+
function deferredDefaultRoles(declared: readonly DeclaredObject[], byObject: ReadonlyMap<string, Change[]>, keyOf: (o: DeclaredObject) => string): DeferredDefaultRoles {
|
|
564
|
+
const out: DeferredDefaultRoles = new Map();
|
|
565
|
+
const grantees = new Set(declared.filter((o) => o.canonical.kind === "grants").map((o) => o.canonical.name));
|
|
566
|
+
for (const o of declared) {
|
|
567
|
+
if (o.canonical.kind !== "user" || !grantees.has(o.canonical.name)) continue;
|
|
568
|
+
const change = (byObject.get(keyOf(o)) ?? []).find((c) => c.rule === "SQLCH271" && c.field === "defaultRole");
|
|
569
|
+
if (!change) continue;
|
|
570
|
+
out.set(o.canonical.name, { step: stepFor(alterAccessField("user", o.canonical.name, "defaultRole", o.canonical.access ?? {}, accessClauseText(o.ddl)), "SQLCH271"), change });
|
|
571
|
+
}
|
|
572
|
+
return out;
|
|
573
|
+
}
|
|
574
|
+
|
|
484
575
|
function planDeclared(input: StatementPlanInput): StatementPlan {
|
|
485
576
|
const keyOf = input.keyOf ?? ((o: DeclaredObject) => o.key);
|
|
486
577
|
const byObject = new Map<string, Change[]>();
|
|
487
578
|
for (const c of input.changes) byObject.set(c.object, [...(byObject.get(c.object) ?? []), c]);
|
|
579
|
+
const deferred = deferredDefaultRoles(input.declared, byObject, keyOf);
|
|
488
580
|
|
|
489
581
|
const objects: ObjectStatements[] = input.declared.map((obj): ObjectStatements => {
|
|
490
582
|
const key = keyOf(obj);
|
|
491
583
|
const mine = byObject.get(key) ?? [];
|
|
584
|
+
if (ACCESS_TYPES.has(obj.type)) return accessStatements(obj, mine, input.current.get(key), deferred);
|
|
492
585
|
const refused = mine.filter(isRebuild);
|
|
493
586
|
if (refused.length > 0) return { verdict: "rebuild", obj, changes: mine, refused, detail: refusalDetail(refused, obj.key, obj.type) };
|
|
494
587
|
const destructive = mine.filter(isDestructiveAlter);
|
|
@@ -506,7 +599,9 @@ function planDeclared(input: StatementPlanInput): StatementPlan {
|
|
|
506
599
|
const renamedFrom = mine.find((c) => c.rule === "SQLCH230" || c.rule === "SQLCH231")?.before;
|
|
507
600
|
const currentKey = renamedFrom !== undefined && input.current.has(renamedFrom) ? renamedFrom : key;
|
|
508
601
|
const live = input.current.get(currentKey);
|
|
509
|
-
|
|
602
|
+
// A function has no comment, so nothing to restamp: it never carries the marker.
|
|
603
|
+
const restamp =
|
|
604
|
+
obj.type !== CLICKHOUSE_ENTITY_TYPES.function && !stepsSetComment(obj, mine) && (mine.some((c) => c.field === "comment") || !(input.carriesMarker?.(currentKey) ?? true));
|
|
510
605
|
const steps = [
|
|
511
606
|
...alterSteps(obj, mine, { ...(live ? { live } : {}), ...(input.marker ? { marker: input.marker } : {}) }),
|
|
512
607
|
...(restamp ? [stepFor(commentStatement(obj, input.marker), "SQLCH203")] : []),
|
|
@@ -519,8 +614,10 @@ function planDeclared(input: StatementPlanInput): StatementPlan {
|
|
|
519
614
|
if (c.rule !== "SQLCH250") continue;
|
|
520
615
|
const o = input.current.get(c.object);
|
|
521
616
|
if (!o) continue;
|
|
617
|
+
// A function and access control carry no marker, so chant never drops one.
|
|
618
|
+
if (NEVER_DROPPED.has(o.kind)) continue;
|
|
522
619
|
const type = CLICKHOUSE_ENTITY_TYPES[o.kind as keyof typeof CLICKHOUSE_ENTITY_TYPES];
|
|
523
|
-
const database = o.kind === "database" ? undefined : o.database;
|
|
620
|
+
const database = o.kind === "database" || o.kind === "function" ? undefined : o.database;
|
|
524
621
|
drops.push({ key: c.object, type, ...(database !== undefined ? { database } : {}), name: o.name, step: stepFor(dropStatement(type, database, o.name), "SQLCH250") });
|
|
525
622
|
}
|
|
526
623
|
drops.sort((a, b) => (DROP_ORDER[kindOf(a.type)] ?? 9) - (DROP_ORDER[kindOf(b.type)] ?? 9));
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dictionaries (#3682): the `dictionary` tag, the declaration and the
|
|
3
|
+
* server's `SHOW CREATE` compared equal, every change but a comment
|
|
4
|
+
* replacing the dictionary (SQLCH245), and the statements for each
|
|
5
|
+
* topology. `live/dictionaries.e2e.test.ts` runs them against a server.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { describe, expect, test } from "vitest";
|
|
9
|
+
import { dictionary, table, CLICKHOUSE_ENTITY_TYPES } from "./entities";
|
|
10
|
+
import { canonicalObject, canonicalDictionaryClause, canonicalLifetime } from "./plan/normalize";
|
|
11
|
+
import { diffSchemas } from "./plan/diff";
|
|
12
|
+
import { CLASSIFIER_RULES } from "./plan/rules";
|
|
13
|
+
import { alterSteps, createStatement, dropStatement, planStatements, type DeclaredObject } from "./apply/statements";
|
|
14
|
+
import { renderStatement } from "./topology";
|
|
15
|
+
import { SqlTemplateError } from "../core/template";
|
|
16
|
+
|
|
17
|
+
const DECLARED = `CREATE DICTIONARY shop.rates_dict (
|
|
18
|
+
code String,
|
|
19
|
+
rate Float64 DEFAULT 1,
|
|
20
|
+
label String DEFAULT '' EXPRESSION upper(code) INJECTIVE
|
|
21
|
+
)
|
|
22
|
+
PRIMARY KEY code
|
|
23
|
+
SOURCE(clickhouse(table 'rates' db 'shop' password 'secret'))
|
|
24
|
+
LAYOUT(complex_key_hashed())
|
|
25
|
+
LIFETIME(300)
|
|
26
|
+
COMMENT 'exchange rates'`;
|
|
27
|
+
|
|
28
|
+
// What clickhouse-server 26.8.15.10 prints for DECLARED.
|
|
29
|
+
const SHOWN = `CREATE DICTIONARY shop.rates_dict
|
|
30
|
+
(
|
|
31
|
+
\`code\` String,
|
|
32
|
+
\`rate\` Float64 DEFAULT 1,
|
|
33
|
+
\`label\` String DEFAULT '' EXPRESSION upper(code) INJECTIVE
|
|
34
|
+
)
|
|
35
|
+
PRIMARY KEY code
|
|
36
|
+
SOURCE(CLICKHOUSE(TABLE 'rates' DB 'shop' PASSWORD '[HIDDEN]'))
|
|
37
|
+
LIFETIME(MIN 0 MAX 300)
|
|
38
|
+
LAYOUT(COMPLEX_KEY_HASHED())
|
|
39
|
+
COMMENT 'exchange rates'`;
|
|
40
|
+
|
|
41
|
+
const declared = (ddl: string, exportName = "ratesDict"): DeclaredObject => {
|
|
42
|
+
const canonical = canonicalObject(ddl);
|
|
43
|
+
return { exportName, type: CLICKHOUSE_ENTITY_TYPES.dictionary, key: `${canonical.database}.${canonical.name}`, ddl, canonical, dependsOn: [] };
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
describe("the dictionary tag", () => {
|
|
47
|
+
test("parses attributes, key, source, layout, lifetime and comment into props", () => {
|
|
48
|
+
const d = dictionary([DECLARED] as unknown as TemplateStringsArray);
|
|
49
|
+
expect(d.entityType).toBe("ClickHouse::Dictionary");
|
|
50
|
+
expect(d.sqlName).toBe("shop.rates_dict");
|
|
51
|
+
expect(d.props).toMatchObject({
|
|
52
|
+
name: "rates_dict",
|
|
53
|
+
database: "shop",
|
|
54
|
+
primaryKey: "code",
|
|
55
|
+
dataSource: "clickhouse(table 'rates' db 'shop' password 'secret')",
|
|
56
|
+
layout: "complex_key_hashed()",
|
|
57
|
+
lifetime: "300",
|
|
58
|
+
comment: "exchange rates",
|
|
59
|
+
columns: [
|
|
60
|
+
{ name: "code", type: "String" },
|
|
61
|
+
{ name: "rate", type: "Float64", default: "1" },
|
|
62
|
+
{ name: "label", type: "String", default: "''", expression: "upper(code)", flags: ["INJECTIVE"] },
|
|
63
|
+
],
|
|
64
|
+
});
|
|
65
|
+
expect(Object.keys(d.columns)).toEqual(["code", "rate", "label"]);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("a range layout with its columns referenced, and settings", () => {
|
|
69
|
+
const rates = table`CREATE TABLE shop.rates (code String, rate Float64, start Date, end Date) ENGINE = MergeTree ORDER BY code`;
|
|
70
|
+
const d = dictionary`CREATE DICTIONARY shop.rates_by_day (code String, start Date, end Date, rate Float64)
|
|
71
|
+
PRIMARY KEY code SOURCE(CLICKHOUSE(TABLE 'rates' DB 'shop')) LAYOUT(COMPLEX_KEY_RANGE_HASHED())
|
|
72
|
+
RANGE(MIN ${rates.columns.start} MAX ${rates.columns.end}) LIFETIME(MIN 10 MAX 20) SETTINGS(format_csv_allow_single_quotes = 0)`;
|
|
73
|
+
expect(d.props.range).toBe("MIN start MAX end");
|
|
74
|
+
expect(d.props.settings).toEqual({ format_csv_allow_single_quotes: "0" });
|
|
75
|
+
expect(d.props.dataSource).toBe("CLICKHOUSE(TABLE 'rates' DB 'shop')");
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("a dictionary needs a key, a source and a layout; another statement names its tag", () => {
|
|
79
|
+
expect(() => dictionary(["CREATE DICTIONARY d (id UInt64) SOURCE(NULL()) LAYOUT(FLAT())"] as unknown as TemplateStringsArray)).toThrow(/PRIMARY KEY/);
|
|
80
|
+
expect(() => dictionary(["CREATE DICTIONARY d (id UInt64) PRIMARY KEY id LAYOUT(FLAT())"] as unknown as TemplateStringsArray)).toThrow(/SOURCE/);
|
|
81
|
+
expect(() => dictionary(["CREATE TABLE t (id UInt64) ENGINE = Memory"] as unknown as TemplateStringsArray)).toThrow(SqlTemplateError);
|
|
82
|
+
expect(() => table(["CREATE DICTIONARY d (id UInt64) PRIMARY KEY id SOURCE(NULL()) LAYOUT(FLAT())"] as unknown as TemplateStringsArray)).toThrow(/use the dictionary tag/);
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
describe("a dictionary compared with the server's", () => {
|
|
87
|
+
test("the declaration and what the server prints for it compare equal", () => {
|
|
88
|
+
expect(diffSchemas([{ key: "d", canonical: canonicalObject(SHOWN) }], [{ key: "d", canonical: canonicalObject(DECLARED) }]).changes).toEqual([]);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test("the server's rewrites are undone: key words upper case, a password hidden, a lifetime as MIN and MAX", () => {
|
|
92
|
+
expect(canonicalDictionaryClause("clickhouse(table src db 'x' user 'u' password 'p')")).toBe("CLICKHOUSE ( TABLE src DB 'x' USER 'u' PASSWORD '[HIDDEN]' )");
|
|
93
|
+
expect(canonicalDictionaryClause("hashed")).toBe("HASHED");
|
|
94
|
+
expect(canonicalDictionaryClause("flat(initial_array_size 100)")).toBe("FLAT ( INITIAL_ARRAY_SIZE 100 )");
|
|
95
|
+
expect(canonicalLifetime("300")).toBe("MIN 0 MAX 300");
|
|
96
|
+
expect(canonicalLifetime("max 20 min 10")).toBe("MIN 10 MAX 20");
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("every change but the comment replaces the dictionary (SQLCH245)", () => {
|
|
100
|
+
const before = canonicalObject(SHOWN);
|
|
101
|
+
const after = canonicalObject(
|
|
102
|
+
DECLARED.replace("complex_key_hashed()", "complex_key_sparse_hashed()").replace("LIFETIME(300)", "LIFETIME(600)").replace("rate Float64 DEFAULT 1", "rate Float64 DEFAULT 0").replace("'exchange rates'", "'rates'"),
|
|
103
|
+
);
|
|
104
|
+
const changes = diffSchemas([{ key: "d", canonical: before }], [{ key: "d", canonical: after }]).changes;
|
|
105
|
+
expect(changes.map((c) => [c.field, c.rule])).toEqual([
|
|
106
|
+
["comment", "SQLCH203"],
|
|
107
|
+
["columns.rate", "SQLCH245"],
|
|
108
|
+
["layout", "SQLCH245"],
|
|
109
|
+
["lifetime", "SQLCH245"],
|
|
110
|
+
]);
|
|
111
|
+
expect(CLASSIFIER_RULES.SQLCH245.class).toBe("metadata");
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
describe("a dictionary's statements", () => {
|
|
116
|
+
const marker = { stack: "shop", env: "prod" };
|
|
117
|
+
|
|
118
|
+
test("created with chant's marker in its comment, replaced whole, dropped and renamed as a dictionary", () => {
|
|
119
|
+
const obj = declared(DECLARED);
|
|
120
|
+
expect(createStatement(obj, marker)).toMatch(/^CREATE DICTIONARY shop\.rates_dict[\s\S]*COMMENT 'exchange rates \[chant[^']*stack=shop env=prod[^']*\]'$/);
|
|
121
|
+
const changes = diffSchemas([{ key: obj.key, canonical: canonicalObject(SHOWN) }], [{ key: obj.key, canonical: canonicalObject(DECLARED.replace("LIFETIME(300)", "LIFETIME(600)")) }]).changes;
|
|
122
|
+
const plan = planStatements({ declared: [declared(DECLARED.replace("LIFETIME(300)", "LIFETIME(600)"))], changes, current: new Map([[obj.key, canonicalObject(SHOWN)]]), marker });
|
|
123
|
+
const entry = plan.objects[0]!;
|
|
124
|
+
expect(entry.verdict).toBe("alter");
|
|
125
|
+
const steps = entry.verdict === "alter" ? entry.steps : [];
|
|
126
|
+
expect(steps.map((s) => s.rule)).toEqual(["SQLCH245"]);
|
|
127
|
+
expect(steps[0]!.sql).toMatch(/^CREATE OR REPLACE DICTIONARY shop\.rates_dict[\s\S]*LIFETIME\(600\)[\s\S]*stack=shop/);
|
|
128
|
+
expect(dropStatement(CLICKHOUSE_ENTITY_TYPES.dictionary, "shop", "rates_dict")).toBe("DROP DICTIONARY `shop`.`rates_dict` SYNC");
|
|
129
|
+
const renamed = alterSteps(obj, [{ object: obj.key, field: "name", rule: "SQLCH230", class: "metadata", before: "shop.old_rates", after: "shop.rates_dict" } as never]);
|
|
130
|
+
expect(renamed.map((s) => s.sql)).toEqual(["RENAME DICTIONARY `shop`.`old_rates` TO `shop`.`rates_dict`"]);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("a comment alone is restamped with ALTER, not replaced", () => {
|
|
134
|
+
const obj = declared(DECLARED.replace("'exchange rates'", "'rates'"));
|
|
135
|
+
const changes = diffSchemas([{ key: obj.key, canonical: canonicalObject(SHOWN) }], [{ key: obj.key, canonical: obj.canonical }]).changes;
|
|
136
|
+
const plan = planStatements({ declared: [obj], changes, current: new Map([[obj.key, canonicalObject(SHOWN)]]), marker });
|
|
137
|
+
const entry = plan.objects[0]!;
|
|
138
|
+
expect(entry.verdict === "alter" ? entry.steps.map((s) => s.sql) : []).toEqual([expect.stringMatching(/^ALTER TABLE `shop`\.`rates_dict` MODIFY COMMENT 'rates \[chant/)]);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test("ON CLUSTER on a cluster, none in a Replicated database or on a single node", () => {
|
|
142
|
+
const sql = createStatement(declared(DECLARED), marker);
|
|
143
|
+
expect(renderStatement(sql, { kind: "cluster", cluster: "main" })).toMatch(/^CREATE DICTIONARY shop\.rates_dict ON CLUSTER `main`\s*\(/);
|
|
144
|
+
expect(renderStatement(sql, { kind: "replicated", cluster: "main" })).not.toMatch(/ON CLUSTER/);
|
|
145
|
+
expect(renderStatement(sql, { kind: "single" })).toBe(sql);
|
|
146
|
+
expect(renderStatement(dropStatement(CLICKHOUSE_ENTITY_TYPES.dictionary, "shop", "rates_dict"), { kind: "cluster", cluster: "main" })).toBe(
|
|
147
|
+
"DROP DICTIONARY `shop`.`rates_dict` ON CLUSTER `main` SYNC",
|
|
148
|
+
);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test("dropped before the tables it reads, after the views", () => {
|
|
152
|
+
const current = new Map([
|
|
153
|
+
["shop.rates", canonicalObject("CREATE TABLE shop.rates (code String) ENGINE = MergeTree ORDER BY code")],
|
|
154
|
+
["shop.rates_dict", canonicalObject(SHOWN)],
|
|
155
|
+
["shop.v", canonicalObject("CREATE VIEW shop.v AS SELECT dictGet('shop.rates_dict', 'rate', 'x')")],
|
|
156
|
+
]);
|
|
157
|
+
const changes = diffSchemas([...current].map(([key, canonical]) => ({ key, canonical })), []).changes;
|
|
158
|
+
const plan = planStatements({ declared: [], changes, current, allowDestructive: true });
|
|
159
|
+
expect(plan.drops.map((d) => d.key)).toEqual(["shop.v", "shop.rates_dict", "shop.rates"]);
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
describe("importing a dictionary", () => {
|
|
164
|
+
test("the server's statement becomes a dictionary declaration that builds the same dictionary", async () => {
|
|
165
|
+
const { ClickHouseSqlParser } = await import("./import/parser");
|
|
166
|
+
const { ClickHouseGenerator } = await import("./import/generator");
|
|
167
|
+
const ir = new ClickHouseSqlParser().parse(`CREATE DATABASE shop;\n${SHOWN};`);
|
|
168
|
+
expect(ir.resources.map((r) => [r.logicalId, r.type])).toEqual([
|
|
169
|
+
["shopDb", "ClickHouse::Database"],
|
|
170
|
+
["ratesDict", "ClickHouse::Dictionary"],
|
|
171
|
+
]);
|
|
172
|
+
const [file] = new ClickHouseGenerator().generate(ir);
|
|
173
|
+
expect(file!.content).toContain('import { database, dictionary } from "@intentius/chant-lexicon-sql/clickhouse";');
|
|
174
|
+
expect(file!.content).toContain("export const ratesDict = dictionary`");
|
|
175
|
+
expect(file!.content).toContain("CREATE DICTIONARY ${shopDb}.rates_dict");
|
|
176
|
+
});
|
|
177
|
+
});
|