@intentius/chant-lexicon-sql 0.124.11 → 0.124.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clickhouse/apply/statements.d.ts.map +1 -1
- package/dist/clickhouse/entities.d.ts +7 -1
- package/dist/clickhouse/entities.d.ts.map +1 -1
- package/dist/clickhouse/expressions.d.ts +27 -0
- package/dist/clickhouse/expressions.d.ts.map +1 -0
- package/dist/clickhouse/parser.d.ts +12 -2
- package/dist/clickhouse/parser.d.ts.map +1 -1
- package/dist/clickhouse/plan/diff.d.ts.map +1 -1
- package/dist/clickhouse/plan/normalize.d.ts +11 -1
- package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
- package/dist/clickhouse/plan/rebuild-handoff.d.ts.map +1 -1
- package/dist/clickhouse/plan/rules.d.ts +1 -0
- package/dist/clickhouse/plan/rules.d.ts.map +1 -1
- package/dist/clickhouse.d.ts +1 -0
- package/dist/clickhouse.d.ts.map +1 -1
- package/dist/codegen/classifier-page.d.ts.map +1 -1
- package/dist/codegen/postgres-classifier-page.d.ts.map +1 -1
- package/dist/core/cursor.d.ts +41 -0
- package/dist/core/cursor.d.ts.map +1 -1
- package/dist/core/find-templates.d.ts +1 -1
- package/dist/core/find-templates.d.ts.map +1 -1
- package/dist/core/interpolation.d.ts +18 -1
- package/dist/core/interpolation.d.ts.map +1 -1
- package/dist/files/clickhouse.d.ts +2 -2
- package/dist/files/clickhouse.d.ts.map +1 -1
- package/dist/files/common.d.ts +16 -0
- package/dist/files/common.d.ts.map +1 -1
- package/dist/files/postgres.d.ts +2 -2
- package/dist/files/postgres.d.ts.map +1 -1
- package/dist/import-parser.d.ts +2 -1
- package/dist/import-parser.d.ts.map +1 -1
- package/dist/integrity.json +24 -24
- package/dist/lint/post-synth/clickhouse-helpers.d.ts +6 -19
- package/dist/lint/post-synth/clickhouse-helpers.d.ts.map +1 -1
- package/dist/lint/post-synth/postgres-helpers.d.ts +15 -55
- package/dist/lint/post-synth/postgres-helpers.d.ts.map +1 -1
- package/dist/lint/post-synth/postgres-names.d.ts +2 -2
- package/dist/lint/post-synth/postgres-names.d.ts.map +1 -1
- package/dist/lint/post-synth/sql-helpers.d.ts +5 -6
- package/dist/lint/post-synth/sql-helpers.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch102.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch103.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch104.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch105.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch106.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch107.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch111.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch112.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch113.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch114.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch115.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch120.d.ts.map +1 -1
- package/dist/lint/post-synth/sqlch125.d.ts.map +1 -1
- package/dist/lint/rules/postgres-templates.d.ts +1 -1
- package/dist/lint/rules/postgres-templates.d.ts.map +1 -1
- package/dist/lint/rules/templates.d.ts +1 -1
- package/dist/lint/rules/templates.d.ts.map +1 -1
- package/dist/lint/rules/typed-names.d.ts +1 -1
- package/dist/lint/rules/typed-names.d.ts.map +1 -1
- package/dist/manifest.json +1 -1
- package/dist/postgres/apply/statements.d.ts.map +1 -1
- package/dist/postgres/entities.d.ts +7 -1
- package/dist/postgres/entities.d.ts.map +1 -1
- package/dist/postgres/parser.d.ts +10 -2
- package/dist/postgres/parser.d.ts.map +1 -1
- package/dist/postgres/plan/diff.d.ts.map +1 -1
- package/dist/postgres/plan/normalize.d.ts.map +1 -1
- package/dist/postgres/plan/rules.d.ts +1 -0
- package/dist/postgres/plan/rules.d.ts.map +1 -1
- package/dist/postgres.d.ts +1 -0
- package/dist/postgres.d.ts.map +1 -1
- package/dist/read/clickhouse.d.ts +66 -0
- package/dist/read/clickhouse.d.ts.map +1 -0
- package/dist/read/objects.d.ts +99 -0
- package/dist/read/objects.d.ts.map +1 -0
- package/dist/read/postgres.d.ts +83 -0
- package/dist/read/postgres.d.ts.map +1 -0
- package/dist/read.d.ts +49 -0
- package/dist/read.d.ts.map +1 -0
- package/dist/rules/clickhouse-helpers.ts +20 -97
- package/dist/rules/postgres-helpers.ts +18 -53
- package/dist/rules/postgres-names.ts +6 -6
- package/dist/rules/postgres-templates.ts +1 -1
- package/dist/rules/sql-helpers.ts +8 -28
- package/dist/rules/sqlch003.ts +1 -1
- package/dist/rules/sqlch102.ts +2 -3
- package/dist/rules/sqlch103.ts +2 -3
- package/dist/rules/sqlch104.ts +2 -3
- package/dist/rules/sqlch105.ts +2 -3
- package/dist/rules/sqlch106.ts +2 -3
- package/dist/rules/sqlch107.ts +2 -3
- package/dist/rules/sqlch111.ts +2 -3
- package/dist/rules/sqlch112.ts +2 -3
- package/dist/rules/sqlch113.ts +2 -3
- package/dist/rules/sqlch114.ts +2 -3
- package/dist/rules/sqlch115.ts +2 -3
- package/dist/rules/sqlch120.ts +2 -3
- package/dist/rules/sqlch125.ts +2 -3
- package/dist/rules/sqlpg002.ts +1 -1
- package/dist/rules/templates.ts +1 -1
- package/dist/rules/typed-names.ts +1 -1
- package/dist/serializer.d.ts +11 -1
- package/dist/serializer.d.ts.map +1 -1
- package/dist/sql-files.d.ts +17 -1
- package/dist/sql-files.d.ts.map +1 -1
- package/dist/sql-types.d.ts +3 -0
- package/dist/sql-types.d.ts.map +1 -1
- package/package.json +2 -3
- package/src/clickhouse/apply/statements.ts +2 -0
- package/src/clickhouse/entities.ts +8 -1
- package/src/clickhouse/expressions.ts +108 -0
- package/src/clickhouse/parser.ts +113 -31
- package/src/clickhouse/plan/diff.ts +19 -0
- package/src/clickhouse/plan/normalize.ts +27 -2
- package/src/clickhouse/plan/rebuild-handoff.ts +2 -1
- package/src/clickhouse/plan/rules.ts +7 -0
- package/src/clickhouse/testdata/server-ddl-baseline.json +67 -0
- package/src/clickhouse.ts +10 -0
- package/src/codegen/classifier-page.ts +2 -0
- package/src/codegen/postgres-classifier-page.ts +2 -0
- package/src/core/cursor.ts +96 -0
- package/src/core/find-templates.ts +1 -1
- package/src/core/interpolation.ts +32 -1
- package/src/files/clickhouse.ts +7 -5
- package/src/files/common.ts +23 -0
- package/src/files/postgres.ts +12 -8
- package/src/import-parser.ts +2 -1
- package/src/lint/post-synth/clickhouse-helpers.ts +20 -97
- package/src/lint/post-synth/postgres-helpers.ts +18 -53
- package/src/lint/post-synth/postgres-names.ts +6 -6
- package/src/lint/post-synth/sql-helpers.ts +8 -28
- package/src/lint/post-synth/sqlch102.ts +2 -3
- package/src/lint/post-synth/sqlch103.ts +2 -3
- package/src/lint/post-synth/sqlch104.ts +2 -3
- package/src/lint/post-synth/sqlch105.ts +2 -3
- package/src/lint/post-synth/sqlch106.ts +2 -3
- package/src/lint/post-synth/sqlch107.ts +2 -3
- package/src/lint/post-synth/sqlch111.ts +2 -3
- package/src/lint/post-synth/sqlch112.ts +2 -3
- package/src/lint/post-synth/sqlch113.ts +2 -3
- package/src/lint/post-synth/sqlch114.ts +2 -3
- package/src/lint/post-synth/sqlch115.ts +2 -3
- package/src/lint/post-synth/sqlch120.ts +2 -3
- package/src/lint/post-synth/sqlch125.ts +2 -3
- package/src/lint/rules/postgres-templates.ts +1 -1
- package/src/lint/rules/sqlch003.ts +1 -1
- package/src/lint/rules/sqlpg002.ts +1 -1
- package/src/lint/rules/templates.ts +1 -1
- package/src/lint/rules/typed-names.ts +1 -1
- package/src/lint/rules/typescript-7.test.ts +101 -0
- package/src/lsp/completions.ts +1 -1
- package/src/lsp/postgres-template.ts +1 -1
- package/src/lsp/postgres.ts +1 -1
- package/src/lsp/refs.ts +1 -1
- package/src/lsp/template.ts +1 -1
- package/src/postgres/apply/statements.ts +2 -0
- package/src/postgres/entities.ts +9 -1
- package/src/postgres/parser.ts +31 -8
- package/src/postgres/plan/diff.ts +18 -0
- package/src/postgres/plan/normalize.ts +12 -0
- package/src/postgres/plan/rules.ts +7 -0
- package/src/postgres/testdata/server-ddl-baseline.json +38 -0
- package/src/postgres.ts +12 -0
- package/src/read/clickhouse.ts +137 -0
- package/src/read/objects.ts +137 -0
- package/src/read/postgres.ts +180 -0
- package/src/read-policies.e2e.test.ts +144 -0
- package/src/read.test.ts +143 -0
- package/src/read.ts +69 -0
- package/src/serializer.ts +37 -1
- package/src/sql-files.ts +27 -6
- package/src/sql-types.ts +19 -2
- package/src/unknown-clauses.test.ts +227 -0
package/src/sql-files.ts
CHANGED
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
* `MATERIALIZED VIEW`, `DICTIONARY`, `FUNCTION`, `USER`, `ROLE` and `ROW
|
|
19
19
|
* POLICY`, and `GRANT` (#3711). Any other statement, and any
|
|
20
20
|
* statement that does not parse, throws a {@link SqlFileError} naming every
|
|
21
|
-
* such statement: none is left out.
|
|
21
|
+
* such statement: none is left out. A statement that parses except for a
|
|
22
|
+
* clause the parser does not know (#3872) is read, kept as written, and the
|
|
23
|
+
* clause listed in its object's `unparsed` with the line it is on.
|
|
22
24
|
*
|
|
23
25
|
* A name is a reference where it is qualified (`app.users`), or bare where an
|
|
24
26
|
* object is named (`REFERENCES users`, `FROM users`, an index's table, a
|
|
@@ -27,7 +29,7 @@
|
|
|
27
29
|
|
|
28
30
|
import type { Declarable } from "@intentius/chant/declarable";
|
|
29
31
|
import type { SqlDialect } from "./dialects";
|
|
30
|
-
import { SqlFileError, type ReadOptions } from "./files/common";
|
|
32
|
+
import { SqlFileError, type FileLine, type ReadOptions } from "./files/common";
|
|
31
33
|
import { postgresTag, readPostgres, templateStrings } from "./files/postgres";
|
|
32
34
|
import { clickhouseTag, readClickHouse } from "./files/clickhouse";
|
|
33
35
|
import { exportNames as pgExportNames, objectsToIR as pgObjectsToIR, type ImportedPgObject } from "./postgres/import/ir";
|
|
@@ -52,6 +54,14 @@ export interface SqlFileObject {
|
|
|
52
54
|
ddl: string;
|
|
53
55
|
/** The file it came from, the `origin` it was read under. */
|
|
54
56
|
origin: string;
|
|
57
|
+
/** The line its statement starts on in the file, from 1. */
|
|
58
|
+
line?: number;
|
|
59
|
+
/**
|
|
60
|
+
* Clauses in it chant's parser does not know (#3872), each with the line
|
|
61
|
+
* it starts on: kept in `ddl` as written, for the server to check. Absent
|
|
62
|
+
* when there are none.
|
|
63
|
+
*/
|
|
64
|
+
unparsed?: Array<{ clause: string; line: number; column?: string }>;
|
|
55
65
|
}
|
|
56
66
|
|
|
57
67
|
/** A file to read: its DDL and the name errors give it. */
|
|
@@ -64,8 +74,8 @@ type Imported = ImportedPgObject | ImportedObject;
|
|
|
64
74
|
const qualifierOf = (o: Imported): string | undefined => ("schema" in o ? o.schema : "database" in o ? o.database : undefined);
|
|
65
75
|
const qualifiedName = (o: Imported): string => (qualifierOf(o) ? `${qualifierOf(o)}.${o.name}` : o.name);
|
|
66
76
|
|
|
67
|
-
function readObjects(dialect: SqlDialect, sources: readonly SqlFileSource[]): Array<Imported & { origin: string }> {
|
|
68
|
-
const out: Array<Imported & { origin: string }> = [];
|
|
77
|
+
function readObjects(dialect: SqlDialect, sources: readonly SqlFileSource[]): Array<Imported & FileLine & { origin: string }> {
|
|
78
|
+
const out: Array<Imported & FileLine & { origin: string }> = [];
|
|
69
79
|
for (const source of sources) {
|
|
70
80
|
const problems: string[] = [];
|
|
71
81
|
const objects = dialect === "postgres" ? readPostgres(source.ddl, source, problems) : readClickHouse(source.ddl, source, problems);
|
|
@@ -75,6 +85,9 @@ function readObjects(dialect: SqlDialect, sources: readonly SqlFileSource[]): Ar
|
|
|
75
85
|
return out;
|
|
76
86
|
}
|
|
77
87
|
|
|
88
|
+
/** An object's line and unknown clauses, the fields that are set. */
|
|
89
|
+
const fileMeta = (o: FileLine): FileLine => ({ ...(o.line !== undefined ? { line: o.line } : {}), ...(o.unparsed ? { unparsed: o.unparsed } : {}) });
|
|
90
|
+
|
|
78
91
|
/** Export names for objects read from files, unique across them. */
|
|
79
92
|
function namesFor(dialect: SqlDialect, objects: readonly Imported[]): string[] {
|
|
80
93
|
return dialect === "postgres" ? pgExportNames(objects as ImportedPgObject[]) : chExportNames(objects as ImportedObject[]);
|
|
@@ -84,7 +97,7 @@ function namesFor(dialect: SqlDialect, objects: readonly Imported[]): string[] {
|
|
|
84
97
|
export function readSqlFile(dialect: SqlDialect, ddl: string, options: ReadOptions): SqlFileObject[] {
|
|
85
98
|
const objects = readObjects(dialect, [{ ...options, ddl }]);
|
|
86
99
|
const names = namesFor(dialect, objects);
|
|
87
|
-
return objects.map((o, i) => ({ export: names[i]!, type: o.type, name: qualifiedName(o), ddl: o.ddl, origin: o.origin }));
|
|
100
|
+
return objects.map((o, i) => ({ export: names[i]!, type: o.type, name: qualifiedName(o), ddl: o.ddl, origin: o.origin, ...fileMeta(o) }));
|
|
88
101
|
}
|
|
89
102
|
|
|
90
103
|
/**
|
|
@@ -100,7 +113,7 @@ export function sqlFileDeclarations(dialect: SqlDialect, ddl: string, options: R
|
|
|
100
113
|
if (!file) throw new SqlFileError(options.origin, ["the DDL creates no objects"]);
|
|
101
114
|
return {
|
|
102
115
|
content: file.content,
|
|
103
|
-
objects: ir.resources.map((r, i) => ({ export: r.logicalId, type: r.type, name: qualifiedName(objects[i]!), ddl: objects[i]!.ddl, origin: options.origin })),
|
|
116
|
+
objects: ir.resources.map((r, i) => ({ export: r.logicalId, type: r.type, name: qualifiedName(objects[i]!), ddl: objects[i]!.ddl, origin: options.origin, ...fileMeta(objects[i]!) })),
|
|
104
117
|
};
|
|
105
118
|
}
|
|
106
119
|
|
|
@@ -109,7 +122,13 @@ export function sqlFileOf(entity: unknown): string | undefined {
|
|
|
109
122
|
return typeof entity === "object" && entity !== null ? ((entity as Record<symbol, unknown>)[FROM_FILE] as string | undefined) : undefined;
|
|
110
123
|
}
|
|
111
124
|
|
|
125
|
+
/** The line an entity read from a `.sql` file starts on in it. Undefined for any other entity. */
|
|
126
|
+
export function sqlFileLineOf(entity: unknown): number | undefined {
|
|
127
|
+
return typeof entity === "object" && entity !== null ? ((entity as Record<symbol, unknown>)[FROM_LINE] as number | undefined) : undefined;
|
|
128
|
+
}
|
|
129
|
+
|
|
112
130
|
const FROM_FILE = Symbol.for("chant.sql.fromFile");
|
|
131
|
+
const FROM_LINE = Symbol.for("chant.sql.fromLine");
|
|
113
132
|
|
|
114
133
|
/** A declared entity as a template target: its export, type and name. */
|
|
115
134
|
function targetOf(dialect: SqlDialect, exportName: string, entity: Declarable): PgTemplateItem | undefined {
|
|
@@ -150,6 +169,7 @@ export function sqlFileEntities(dialect: SqlDialect, sources: readonly SqlFileSo
|
|
|
150
169
|
|
|
151
170
|
const items: PgTemplateItem[] = objects.map((o, i) => ({ exportName: names[i]!, type: o.type, ...("schema" in o && o.schema ? { schema: o.schema } : {}), ...("database" in o && o.database ? { database: o.database } : {}), name: o.name, ddl: o.ddl }) as PgTemplateItem);
|
|
152
171
|
const origin = new Map(objects.map((o, i) => [names[i]!, o.origin]));
|
|
172
|
+
const lineOf = new Map(objects.map((o, i) => [names[i]!, o.line]));
|
|
153
173
|
const knownItems = [...known].map(([n, e]) => targetOf(dialect, n, e)).filter((t): t is PgTemplateItem => t !== undefined);
|
|
154
174
|
const declared = new Map(knownItems.map((t) => [`${t.type} ${qualifiedName(t as Imported)}`, t.exportName]));
|
|
155
175
|
const seen = new Map<string, string>();
|
|
@@ -186,6 +206,7 @@ export function sqlFileEntities(dialect: SqlDialect, sources: readonly SqlFileSo
|
|
|
186
206
|
try {
|
|
187
207
|
const entity = tagOf(it.type)!(templateStrings(body.parts), ...values);
|
|
188
208
|
Object.defineProperty(entity, FROM_FILE, { value: origin.get(it.exportName), enumerable: false });
|
|
209
|
+
if (lineOf.get(it.exportName) !== undefined) Object.defineProperty(entity, FROM_LINE, { value: lineOf.get(it.exportName), enumerable: false });
|
|
189
210
|
built.set(it.exportName, entity);
|
|
190
211
|
} catch (e) {
|
|
191
212
|
failed.add(it.exportName);
|
package/src/sql-types.ts
CHANGED
|
@@ -22,6 +22,9 @@
|
|
|
22
22
|
* (`./lint/post-synth/postgres-names.ts`'s `typeIdentity`), the type
|
|
23
23
|
* family in ClickHouse; `never` when it cannot be told.
|
|
24
24
|
*
|
|
25
|
+
* A column that carries a clause chant does not read (#3872) keeps the type
|
|
26
|
+
* that was read, with `nullable` and `cmp` `never`.
|
|
27
|
+
*
|
|
25
28
|
* A view's column that is one column reference (`${t.columns.c}`, `o.${t.columns.c}`)
|
|
26
29
|
* is that column's type; any other expression is `unknown`.
|
|
27
30
|
*/
|
|
@@ -108,6 +111,17 @@ interface RelationEntry {
|
|
|
108
111
|
|
|
109
112
|
const UNKNOWN_COLUMN = (name: string): ColumnEntry => ({ name, sqlType: "unknown", ts: "unknown", nullable: "never" });
|
|
110
113
|
|
|
114
|
+
/** The columns that carry a clause chant does not read (#3872). */
|
|
115
|
+
const opaqueColumns = (o: OutputObject): Set<string> =>
|
|
116
|
+
new Set(((o.unparsed as Array<{ column?: string }> | undefined) ?? []).flatMap((u) => (u.column !== undefined ? [u.column] : [])));
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* A column with a clause chant does not read (#3872): its type is what was
|
|
120
|
+
* read, but whether it can be NULL and how it compares in a key are not
|
|
121
|
+
* told, so it fits every slot rather than a guessed one.
|
|
122
|
+
*/
|
|
123
|
+
const opaque = (c: ColumnEntry): ColumnEntry => ({ name: c.name, sqlType: c.sqlType, ts: c.ts, nullable: "never" });
|
|
124
|
+
|
|
111
125
|
const posix = (p: string) => p.split(sep).join("/");
|
|
112
126
|
|
|
113
127
|
/** Where a declaration came from: its file relative to the project, and its export. */
|
|
@@ -215,13 +229,15 @@ function postgresEntries(objects: OutputObject[], major: number, entities: Reado
|
|
|
215
229
|
for (const c of pt?.columns ?? []) if (!inherited.has(c.name)) inherited.set(c.name, c.type);
|
|
216
230
|
}
|
|
217
231
|
const pk = new Set(t.primaryKey?.columns ?? []);
|
|
232
|
+
const unread = opaqueColumns(o);
|
|
218
233
|
columns = names.map((name) => {
|
|
219
234
|
const c = own.get(name);
|
|
220
235
|
const text = c?.type ?? inherited.get(name);
|
|
221
236
|
if (text === undefined) return UNKNOWN_COLUMN(name);
|
|
222
237
|
const ty = pgType(p, text);
|
|
223
238
|
const notNull = c?.notNull === true || pk.has(name) || c?.generated?.kind === "identity" || ty.notNull;
|
|
224
|
-
|
|
239
|
+
const entry: ColumnEntry = { name, sqlType: ty.sqlType, ts: ty.ts, nullable: !notNull, cmp: typeIdentity(p, text) };
|
|
240
|
+
return unread.has(name) ? opaque(entry) : entry;
|
|
225
241
|
});
|
|
226
242
|
} else {
|
|
227
243
|
columns = names.map((name) => {
|
|
@@ -265,9 +281,10 @@ function clickhouseEntries(objects: OutputObject[], entities: ReadonlyMap<string
|
|
|
265
281
|
const entity = entities.get(exportName);
|
|
266
282
|
if (!o || !entity || !CH_RELATIONS.has(o.type)) return undefined;
|
|
267
283
|
const declared = new Map(((o.columns as Array<{ name: string; type?: string; nullable?: boolean }> | undefined) ?? []).map((c) => [c.name, c]));
|
|
284
|
+
const unread = opaqueColumns(o);
|
|
268
285
|
const columns = runtimeColumns(entity).map((name) => {
|
|
269
286
|
const c = declared.get(name);
|
|
270
|
-
if (c) return chColumn(name, c.type, c.nullable);
|
|
287
|
+
if (c) return unread.has(name) ? opaque(chColumn(name, c.type, c.nullable)) : chColumn(name, c.type, c.nullable);
|
|
271
288
|
const via = passthrough(o, name, quoteIdentifier);
|
|
272
289
|
const source = via && entryOf(via.export)?.columns.find((x) => x.name === via.column);
|
|
273
290
|
return source ? { ...source, name } : UNKNOWN_COLUMN(name);
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A clause chant's parser does not know (#3872): the statement is kept as
|
|
3
|
+
* written, the build warns naming where it is, the rest of the object is
|
|
4
|
+
* read, and a plan classes a change in the clause as one to review.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { describe, expect, test } from "vitest";
|
|
8
|
+
import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
|
|
9
|
+
import { dirname, join, resolve } from "node:path";
|
|
10
|
+
import { fileURLToPath } from "node:url";
|
|
11
|
+
import { build } from "@intentius/chant/build";
|
|
12
|
+
import { tokenizeText as chTokens } from "./clickhouse/tokens";
|
|
13
|
+
import { parseCreate, type TableNode } from "./clickhouse/parser";
|
|
14
|
+
import { tokenizeText as pgTokens } from "./postgres/tokens";
|
|
15
|
+
import { parseStatements, type IndexNode, type TableNode as PgTableNode } from "./postgres/parser";
|
|
16
|
+
import { table, view, database } from "./clickhouse/entities";
|
|
17
|
+
import { table as pgTable } from "./postgres/entities";
|
|
18
|
+
import { readSqlFile, sqlFileEntities, SqlFileError } from "./sql-files";
|
|
19
|
+
import { canonicalObject } from "./clickhouse/plan/normalize";
|
|
20
|
+
import { diffSchemas } from "./clickhouse/plan/diff";
|
|
21
|
+
import { diffPgSchemas } from "./postgres/plan/diff";
|
|
22
|
+
import { diffObject, type PgSchemaObject } from "./postgres/plan/schema";
|
|
23
|
+
import { planStatements } from "./clickhouse/apply/statements";
|
|
24
|
+
import { sqlSerializer, unparsedWarnings } from "./serializer";
|
|
25
|
+
import { sqlPlugin } from "./plugin";
|
|
26
|
+
import { renderSqlTypes } from "./sql-types";
|
|
27
|
+
import type { Declarable } from "@intentius/chant/declarable";
|
|
28
|
+
|
|
29
|
+
const ch = (sql: string) => parseCreate(chTokens(sql, 0));
|
|
30
|
+
const pg = (sql: string) => parseStatements(pgTokens(sql, 0))[0]!;
|
|
31
|
+
|
|
32
|
+
describe("the ClickHouse parser keeps a clause it does not know", () => {
|
|
33
|
+
test("after the storage clauses, on a column, before a view's AS, in a database and a dictionary", () => {
|
|
34
|
+
const t = ch("CREATE TABLE a.t (x UInt64, k String ENCRYPTED WITH 'k1', y UInt8) ENGINE = MergeTree ORDER BY x PARTITION_STRATEGY = 'hourly' SETTINGS index_granularity = 8192") as TableNode;
|
|
35
|
+
expect(t.unknown?.map((u) => [u.clause, u.column])).toEqual([
|
|
36
|
+
["ENCRYPTED WITH", "k"],
|
|
37
|
+
["PARTITION_STRATEGY", undefined],
|
|
38
|
+
]);
|
|
39
|
+
// What follows the clause is still read.
|
|
40
|
+
expect(t.columns.map((c) => c.name)).toEqual(["x", "k", "y"]);
|
|
41
|
+
expect(t.settings?.map((s) => s.key)).toEqual(["index_granularity"]);
|
|
42
|
+
|
|
43
|
+
expect(ch("CREATE VIEW a.v SQL_CACHE = 'on' AS SELECT 1").unknown?.map((u) => u.clause)).toEqual(["SQL_CACHE"]);
|
|
44
|
+
expect(ch("CREATE DATABASE d ENGINE = Atomic SHARDING_MODE 'auto' COMMENT 'x'").unknown?.map((u) => u.clause)).toEqual(["SHARDING_MODE"]);
|
|
45
|
+
expect(
|
|
46
|
+
ch("CREATE DICTIONARY a.d (id UInt64, v String DEFAULT '' SPARSE) PRIMARY KEY id SOURCE(NULL()) LAYOUT(FLAT()) LIFETIME(0) WARMUP(10)").unknown?.map((u) => [u.clause, u.column]),
|
|
47
|
+
).toEqual([
|
|
48
|
+
["SPARSE", "v"],
|
|
49
|
+
["WARMUP", undefined],
|
|
50
|
+
]);
|
|
51
|
+
expect(ch("CREATE TABLE a.t (x UInt64) ENGINE = MergeTree ORDER BY x").unknown).toBeUndefined();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("and still fails on what is malformed", () => {
|
|
55
|
+
// One letter from a clause it knows: a misspelling.
|
|
56
|
+
expect(() => ch("CREATE TABLE a.t (x UInt64) ENGINE = MergeTree ORDR BY x")).toThrow(/one letter from ORDER/);
|
|
57
|
+
expect(() => ch("CREATE TABLE a.t (x UInt64 CODCE(ZSTD))")).toThrow(/one letter from CODEC/);
|
|
58
|
+
// Unbalanced parentheses, a word that is not a key word, an unknown statement kind.
|
|
59
|
+
expect(() => ch("CREATE TABLE a.t (x UInt64) ENGINE = Log FUTURE_THING (a")).toThrow(/unbalanced parentheses|expected/);
|
|
60
|
+
expect(() => ch("CREATE TABLE a.t (x UInt64) ENGINE = Log 'stray'")).toThrow(/unexpected token/);
|
|
61
|
+
expect(() => ch("CREATE NAMED COLLECTION c AS a = 1")).toThrow(/expected DATABASE, TABLE/);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
describe("the Postgres parser keeps a clause it does not know", () => {
|
|
66
|
+
test("after a table's column list, on a column, after an index's elements, before a view's AS, in a sequence", () => {
|
|
67
|
+
const t = pg("CREATE TABLE app.t (id bigint PRIMARY KEY, note text INVISIBLE, n int NOT NULL) STORAGE_CLASS 'cold' TABLESPACE fast") as PgTableNode;
|
|
68
|
+
expect(t.unknown?.map((u) => [u.clause, u.column])).toEqual([
|
|
69
|
+
["INVISIBLE", "note"],
|
|
70
|
+
["STORAGE_CLASS", undefined],
|
|
71
|
+
]);
|
|
72
|
+
expect(t.columns.map((c) => c.name)).toEqual(["id", "note", "n"]);
|
|
73
|
+
expect(t.tablespace).toBeDefined();
|
|
74
|
+
expect((pg("CREATE INDEX i ON app.t (id) INVISIBLE") as IndexNode).unknown?.map((u) => u.clause)).toEqual(["INVISIBLE"]);
|
|
75
|
+
expect(pg("CREATE VIEW app.v SECURITY_BARRIER_HINT AS SELECT 1").statement).toBe("view");
|
|
76
|
+
expect(pg("CREATE SEQUENCE app.s CYCLE SEQUENCE_KIND local").statement).toBe("sequence");
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
test("and still fails on what is malformed, and on what it refuses on purpose", () => {
|
|
80
|
+
expect(() => pg("CREATE TABLE app.t (id int) TABLESPAC fast")).toThrow(/one letter from TABLESPACE/);
|
|
81
|
+
expect(() => pg("CREATE TABLE app.t (id int NOT NUL)")).toThrow();
|
|
82
|
+
expect(() => pg("CREATE ROLE r PASSWORD 'x'")).toThrow(/password/);
|
|
83
|
+
expect(() => pg("CREATE TABLE app.t (id int) WITHOUT_SOMETHING (")).toThrow();
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
describe("a tag keeps the clause in its DDL and names it in props", () => {
|
|
88
|
+
test("ClickHouse", () => {
|
|
89
|
+
const t = table`CREATE TABLE a.t (
|
|
90
|
+
x UInt64,
|
|
91
|
+
k String ENCRYPTED WITH 'k1'
|
|
92
|
+
) ENGINE = MergeTree ORDER BY x
|
|
93
|
+
PARTITION_STRATEGY = 'hourly'`;
|
|
94
|
+
expect(t.props.ddl).toContain("PARTITION_STRATEGY = 'hourly'");
|
|
95
|
+
expect(t.props.ddl).toContain("ENCRYPTED WITH 'k1'");
|
|
96
|
+
expect(t.props.unparsed).toEqual([
|
|
97
|
+
{ clause: "ENCRYPTED WITH", text: "ENCRYPTED WITH 'k1'", column: "k", line: 3 },
|
|
98
|
+
{ clause: "PARTITION_STRATEGY", text: "PARTITION_STRATEGY = 'hourly'", line: 5 },
|
|
99
|
+
]);
|
|
100
|
+
expect(t.props.orderBy).toBe("x");
|
|
101
|
+
expect(Object.keys((t as unknown as { columns: object }).columns)).toEqual(["x", "k"]);
|
|
102
|
+
expect(database`CREATE DATABASE a`.props.unparsed).toBeUndefined();
|
|
103
|
+
expect(view`CREATE VIEW a.v AS SELECT x FROM ${t}`.props.unparsed).toBeUndefined();
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test("Postgres", () => {
|
|
107
|
+
const t = pgTable`CREATE TABLE app.t (id bigint PRIMARY KEY, note text) STORAGE_CLASS 'cold'`;
|
|
108
|
+
expect(t.props.ddl).toContain("STORAGE_CLASS 'cold'");
|
|
109
|
+
expect(t.props.unparsed).toEqual([{ clause: "STORAGE_CLASS", text: "STORAGE_CLASS 'cold'", line: 1 }]);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
const CH_FILE = `CREATE DATABASE analytics;
|
|
114
|
+
|
|
115
|
+
CREATE TABLE analytics.events (
|
|
116
|
+
ts DateTime,
|
|
117
|
+
kind String ENCRYPTED WITH 'k1'
|
|
118
|
+
) ENGINE = MergeTree ORDER BY ts
|
|
119
|
+
PARTITION_STRATEGY = 'hourly';
|
|
120
|
+
`;
|
|
121
|
+
|
|
122
|
+
describe("a .sql file", () => {
|
|
123
|
+
test("reads, with each clause and its line in the file", () => {
|
|
124
|
+
const [db, events] = readSqlFile("clickhouse", CH_FILE, { origin: "src/schema.sql" });
|
|
125
|
+
expect(db!.unparsed).toBeUndefined();
|
|
126
|
+
expect(events!.line).toBe(3);
|
|
127
|
+
expect(events!.unparsed).toEqual([
|
|
128
|
+
{ clause: "ENCRYPTED WITH", line: 5, column: "kind" },
|
|
129
|
+
{ clause: "PARTITION_STRATEGY", line: 7 },
|
|
130
|
+
]);
|
|
131
|
+
const pgObjects = readSqlFile("postgres", "CREATE SCHEMA app;\nCREATE TABLE app.t (\n id bigint\n) STORAGE_CLASS 'cold';\nCOMMENT ON TABLE app.t IS 'x';", { origin: "db.sql" });
|
|
132
|
+
expect(pgObjects[1]!.unparsed).toEqual([{ clause: "STORAGE_CLASS", line: 4 }]);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
test("builds entities whose warnings name the file and line", () => {
|
|
136
|
+
const entities = sqlFileEntities("clickhouse", [{ origin: "src/schema.sql", ddl: CH_FILE }]);
|
|
137
|
+
const warnings = unparsedWarnings(entities);
|
|
138
|
+
expect(warnings).toHaveLength(2);
|
|
139
|
+
expect(warnings[0]).toMatch(/^src\/schema\.sql:5: analytics\.events has a clause chant does not read, ENCRYPTED WITH on column kind\. It is sent as written/);
|
|
140
|
+
expect(warnings[1]).toMatch(/^src\/schema\.sql:7: analytics\.events has a clause chant does not read, PARTITION_STRATEGY\./);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("statements that are not schema still fail", () => {
|
|
144
|
+
expect(() => readSqlFile("clickhouse", "CREATE DATABASE a;\nDROP TABLE a.t", { origin: "x.sql" })).toThrow(SqlFileError);
|
|
145
|
+
expect(() => readSqlFile("postgres", "INSERT INTO app.t VALUES (1)", { origin: "x.sql" })).toThrow(/not a statement that declares an object/);
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
|
150
|
+
|
|
151
|
+
describe("a build", () => {
|
|
152
|
+
test("succeeds with a warning, and the clause is in the DDL it writes", async () => {
|
|
153
|
+
const path = join(repoRoot, ".cache", `sql-3872-${process.pid}`);
|
|
154
|
+
await rm(path, { recursive: true, force: true });
|
|
155
|
+
await mkdir(join(path, "src"), { recursive: true });
|
|
156
|
+
const dir = await realpath(path);
|
|
157
|
+
await writeFile(join(dir, "src", "schema.sql"), CH_FILE);
|
|
158
|
+
try {
|
|
159
|
+
const result = await build(join(dir, "src"), [sqlSerializer], undefined, {
|
|
160
|
+
lexicons: ["sql"],
|
|
161
|
+
intrinsics: sqlPlugin.intrinsics?.() ?? [],
|
|
162
|
+
buildRoots: [(ctx) => sqlPlugin.buildRoots!({ projectRoot: dir, config: {}, entities: ctx.entities, ...(ctx.sourceDir ? { sourceDir: ctx.sourceDir } : {}) })],
|
|
163
|
+
});
|
|
164
|
+
expect(result.errors.map((e) => e.message)).toEqual([]);
|
|
165
|
+
expect(result.warnings.filter((w) => /chant does not read/.test(w))).toHaveLength(2);
|
|
166
|
+
const out = result.outputs.get("sql");
|
|
167
|
+
const files = typeof out === "string" ? {} : (out?.files ?? {});
|
|
168
|
+
expect(String(files["clickhouse.sql"])).toContain("PARTITION_STRATEGY = 'hourly'");
|
|
169
|
+
} finally {
|
|
170
|
+
await rm(path, { recursive: true, force: true });
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
describe("typed columns", () => {
|
|
176
|
+
test("a column with a clause chant does not read keeps its type, and its nullability and key comparison are never", () => {
|
|
177
|
+
const events = table`CREATE TABLE a.events (id UInt64, k String ENCRYPTED WITH 'k1') ENGINE = MergeTree ORDER BY id`;
|
|
178
|
+
const text = renderSqlTypes(new Map([["events", events]]) as Map<string, Declarable>, { projectDir: "/" });
|
|
179
|
+
expect(text).toContain('id: Column<"UInt64", { ts: string; nullable: false; cmp: "UInt64" }>;');
|
|
180
|
+
expect(text).toContain('k: Column<"String", { ts: string; nullable: never; cmp: never }>;');
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
describe("a plan", () => {
|
|
185
|
+
const declared = (ddl: string) => ({ key: "analytics.events", exportName: "events", canonical: canonicalObject(ddl) });
|
|
186
|
+
const BASE = "CREATE TABLE analytics.events (ts DateTime) ENGINE = MergeTree ORDER BY ts";
|
|
187
|
+
|
|
188
|
+
test("ClickHouse: the same clause is no change, a changed one is SQLCH290 for review, one only the server prints is a hint", () => {
|
|
189
|
+
const same = diffSchemas([declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)], [declared(`${BASE}\nPARTITION_STRATEGY = 'hourly'`)]);
|
|
190
|
+
expect(same.changes).toEqual([]);
|
|
191
|
+
|
|
192
|
+
const changed = diffSchemas([declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)], [declared(`${BASE} PARTITION_STRATEGY = 'daily'`)]);
|
|
193
|
+
expect(changed.changes.map((c) => [c.field, c.rule, c.class])).toEqual([["unparsed.PARTITION_STRATEGY", "SQLCH290", "rebuild"]]);
|
|
194
|
+
|
|
195
|
+
const added = diffSchemas([declared(BASE)], [declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)]);
|
|
196
|
+
expect(added.changes.map((c) => c.rule)).toEqual(["SQLCH290"]);
|
|
197
|
+
|
|
198
|
+
const serverOnly = diffSchemas([declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)], [declared(BASE)]);
|
|
199
|
+
expect(serverOnly.changes).toEqual([]);
|
|
200
|
+
expect(serverOnly.hints.join("\n")).toMatch(/PARTITION_STRATEGY .* a clause chant does not read/);
|
|
201
|
+
|
|
202
|
+
// The rest of the object is still planned, and the refusal says why.
|
|
203
|
+
const both = diffSchemas([declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)], [declared(`CREATE TABLE analytics.events (ts DateTime, n UInt8) ENGINE = MergeTree ORDER BY ts PARTITION_STRATEGY = 'daily'`)]);
|
|
204
|
+
expect(both.changes.map((c) => c.rule).sort()).toEqual(["SQLCH201", "SQLCH290"]);
|
|
205
|
+
const after = declared(`CREATE TABLE analytics.events (ts DateTime, n UInt8) ENGINE = MergeTree ORDER BY ts PARTITION_STRATEGY = 'daily'`);
|
|
206
|
+
const plan = planStatements({
|
|
207
|
+
declared: [{ ...after, type: "ClickHouse::Table", ddl: "CREATE TABLE analytics.events (ts DateTime, n UInt8) ENGINE = MergeTree ORDER BY ts PARTITION_STRATEGY = 'daily'" } as never],
|
|
208
|
+
changes: both.changes,
|
|
209
|
+
current: new Map(),
|
|
210
|
+
});
|
|
211
|
+
const verdict = plan.objects[0]!;
|
|
212
|
+
expect(verdict.verdict).toBe("rebuild");
|
|
213
|
+
expect((verdict as { detail: string }).detail).toMatch(/^needs review, since chant does not read the clause/);
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
test("Postgres: a changed clause is SQLPG299, refused for review", () => {
|
|
217
|
+
const o = (ddl: string): PgSchemaObject => ({ key: "t", canonical: { ...diffObject("Postgres::Table", ddl, "public"), exportName: "t" } });
|
|
218
|
+
const before = o("CREATE TABLE app.t (id bigint PRIMARY KEY) STORAGE_CLASS 'cold'");
|
|
219
|
+
expect(diffPgSchemas([before], [o("CREATE TABLE app.t (id bigint PRIMARY KEY) STORAGE_CLASS 'cold'")]).changes).toEqual([]);
|
|
220
|
+
const d = diffPgSchemas([before], [o("CREATE TABLE app.t (id bigint PRIMARY KEY, n int) STORAGE_CLASS 'hot'")]);
|
|
221
|
+
expect(d.changes.map((c) => [c.field, c.rule, c.class])).toEqual([
|
|
222
|
+
["unparsed.STORAGE_CLASS", "SQLPG299", "expand"],
|
|
223
|
+
["columns.n", "SQLPG201", "metadata"],
|
|
224
|
+
]);
|
|
225
|
+
expect(d.refused.map((c) => c.rule)).toEqual(["SQLPG299"]);
|
|
226
|
+
});
|
|
227
|
+
});
|