@intentius/chant-lexicon-sql 0.124.11 → 0.124.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/dist/clickhouse/apply/statements.d.ts.map +1 -1
  2. package/dist/clickhouse/entities.d.ts +7 -1
  3. package/dist/clickhouse/entities.d.ts.map +1 -1
  4. package/dist/clickhouse/expressions.d.ts +27 -0
  5. package/dist/clickhouse/expressions.d.ts.map +1 -0
  6. package/dist/clickhouse/parser.d.ts +12 -2
  7. package/dist/clickhouse/parser.d.ts.map +1 -1
  8. package/dist/clickhouse/plan/diff.d.ts.map +1 -1
  9. package/dist/clickhouse/plan/normalize.d.ts +11 -1
  10. package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
  11. package/dist/clickhouse/plan/rebuild-handoff.d.ts.map +1 -1
  12. package/dist/clickhouse/plan/rules.d.ts +1 -0
  13. package/dist/clickhouse/plan/rules.d.ts.map +1 -1
  14. package/dist/clickhouse.d.ts +1 -0
  15. package/dist/clickhouse.d.ts.map +1 -1
  16. package/dist/codegen/classifier-page.d.ts.map +1 -1
  17. package/dist/codegen/postgres-classifier-page.d.ts.map +1 -1
  18. package/dist/core/cursor.d.ts +41 -0
  19. package/dist/core/cursor.d.ts.map +1 -1
  20. package/dist/core/find-templates.d.ts +1 -1
  21. package/dist/core/find-templates.d.ts.map +1 -1
  22. package/dist/core/interpolation.d.ts +18 -1
  23. package/dist/core/interpolation.d.ts.map +1 -1
  24. package/dist/files/clickhouse.d.ts +2 -2
  25. package/dist/files/clickhouse.d.ts.map +1 -1
  26. package/dist/files/common.d.ts +16 -0
  27. package/dist/files/common.d.ts.map +1 -1
  28. package/dist/files/postgres.d.ts +2 -2
  29. package/dist/files/postgres.d.ts.map +1 -1
  30. package/dist/import-parser.d.ts +2 -1
  31. package/dist/import-parser.d.ts.map +1 -1
  32. package/dist/integrity.json +24 -24
  33. package/dist/lint/post-synth/clickhouse-helpers.d.ts +6 -19
  34. package/dist/lint/post-synth/clickhouse-helpers.d.ts.map +1 -1
  35. package/dist/lint/post-synth/postgres-helpers.d.ts +15 -55
  36. package/dist/lint/post-synth/postgres-helpers.d.ts.map +1 -1
  37. package/dist/lint/post-synth/postgres-names.d.ts +2 -2
  38. package/dist/lint/post-synth/postgres-names.d.ts.map +1 -1
  39. package/dist/lint/post-synth/sql-helpers.d.ts +5 -6
  40. package/dist/lint/post-synth/sql-helpers.d.ts.map +1 -1
  41. package/dist/lint/post-synth/sqlch102.d.ts.map +1 -1
  42. package/dist/lint/post-synth/sqlch103.d.ts.map +1 -1
  43. package/dist/lint/post-synth/sqlch104.d.ts.map +1 -1
  44. package/dist/lint/post-synth/sqlch105.d.ts.map +1 -1
  45. package/dist/lint/post-synth/sqlch106.d.ts.map +1 -1
  46. package/dist/lint/post-synth/sqlch107.d.ts.map +1 -1
  47. package/dist/lint/post-synth/sqlch111.d.ts.map +1 -1
  48. package/dist/lint/post-synth/sqlch112.d.ts.map +1 -1
  49. package/dist/lint/post-synth/sqlch113.d.ts.map +1 -1
  50. package/dist/lint/post-synth/sqlch114.d.ts.map +1 -1
  51. package/dist/lint/post-synth/sqlch115.d.ts.map +1 -1
  52. package/dist/lint/post-synth/sqlch120.d.ts.map +1 -1
  53. package/dist/lint/post-synth/sqlch125.d.ts.map +1 -1
  54. package/dist/lint/rules/postgres-templates.d.ts +1 -1
  55. package/dist/lint/rules/postgres-templates.d.ts.map +1 -1
  56. package/dist/lint/rules/templates.d.ts +1 -1
  57. package/dist/lint/rules/templates.d.ts.map +1 -1
  58. package/dist/lint/rules/typed-names.d.ts +1 -1
  59. package/dist/lint/rules/typed-names.d.ts.map +1 -1
  60. package/dist/manifest.json +1 -1
  61. package/dist/postgres/apply/statements.d.ts.map +1 -1
  62. package/dist/postgres/entities.d.ts +7 -1
  63. package/dist/postgres/entities.d.ts.map +1 -1
  64. package/dist/postgres/parser.d.ts +10 -2
  65. package/dist/postgres/parser.d.ts.map +1 -1
  66. package/dist/postgres/plan/diff.d.ts.map +1 -1
  67. package/dist/postgres/plan/normalize.d.ts.map +1 -1
  68. package/dist/postgres/plan/rules.d.ts +1 -0
  69. package/dist/postgres/plan/rules.d.ts.map +1 -1
  70. package/dist/postgres.d.ts +1 -0
  71. package/dist/postgres.d.ts.map +1 -1
  72. package/dist/read/clickhouse.d.ts +66 -0
  73. package/dist/read/clickhouse.d.ts.map +1 -0
  74. package/dist/read/objects.d.ts +99 -0
  75. package/dist/read/objects.d.ts.map +1 -0
  76. package/dist/read/postgres.d.ts +83 -0
  77. package/dist/read/postgres.d.ts.map +1 -0
  78. package/dist/read.d.ts +49 -0
  79. package/dist/read.d.ts.map +1 -0
  80. package/dist/rules/clickhouse-helpers.ts +20 -97
  81. package/dist/rules/postgres-helpers.ts +18 -53
  82. package/dist/rules/postgres-names.ts +6 -6
  83. package/dist/rules/postgres-templates.ts +1 -1
  84. package/dist/rules/sql-helpers.ts +8 -28
  85. package/dist/rules/sqlch003.ts +1 -1
  86. package/dist/rules/sqlch102.ts +2 -3
  87. package/dist/rules/sqlch103.ts +2 -3
  88. package/dist/rules/sqlch104.ts +2 -3
  89. package/dist/rules/sqlch105.ts +2 -3
  90. package/dist/rules/sqlch106.ts +2 -3
  91. package/dist/rules/sqlch107.ts +2 -3
  92. package/dist/rules/sqlch111.ts +2 -3
  93. package/dist/rules/sqlch112.ts +2 -3
  94. package/dist/rules/sqlch113.ts +2 -3
  95. package/dist/rules/sqlch114.ts +2 -3
  96. package/dist/rules/sqlch115.ts +2 -3
  97. package/dist/rules/sqlch120.ts +2 -3
  98. package/dist/rules/sqlch125.ts +2 -3
  99. package/dist/rules/sqlpg002.ts +1 -1
  100. package/dist/rules/templates.ts +1 -1
  101. package/dist/rules/typed-names.ts +1 -1
  102. package/dist/serializer.d.ts +11 -1
  103. package/dist/serializer.d.ts.map +1 -1
  104. package/dist/sql-files.d.ts +17 -1
  105. package/dist/sql-files.d.ts.map +1 -1
  106. package/dist/sql-types.d.ts +3 -0
  107. package/dist/sql-types.d.ts.map +1 -1
  108. package/package.json +2 -3
  109. package/src/clickhouse/apply/statements.ts +2 -0
  110. package/src/clickhouse/entities.ts +8 -1
  111. package/src/clickhouse/expressions.ts +108 -0
  112. package/src/clickhouse/parser.ts +113 -31
  113. package/src/clickhouse/plan/diff.ts +19 -0
  114. package/src/clickhouse/plan/normalize.ts +27 -2
  115. package/src/clickhouse/plan/rebuild-handoff.ts +2 -1
  116. package/src/clickhouse/plan/rules.ts +7 -0
  117. package/src/clickhouse/testdata/server-ddl-baseline.json +67 -0
  118. package/src/clickhouse.ts +10 -0
  119. package/src/codegen/classifier-page.ts +2 -0
  120. package/src/codegen/postgres-classifier-page.ts +2 -0
  121. package/src/core/cursor.ts +96 -0
  122. package/src/core/find-templates.ts +1 -1
  123. package/src/core/interpolation.ts +32 -1
  124. package/src/files/clickhouse.ts +7 -5
  125. package/src/files/common.ts +23 -0
  126. package/src/files/postgres.ts +12 -8
  127. package/src/import-parser.ts +2 -1
  128. package/src/lint/post-synth/clickhouse-helpers.ts +20 -97
  129. package/src/lint/post-synth/postgres-helpers.ts +18 -53
  130. package/src/lint/post-synth/postgres-names.ts +6 -6
  131. package/src/lint/post-synth/sql-helpers.ts +8 -28
  132. package/src/lint/post-synth/sqlch102.ts +2 -3
  133. package/src/lint/post-synth/sqlch103.ts +2 -3
  134. package/src/lint/post-synth/sqlch104.ts +2 -3
  135. package/src/lint/post-synth/sqlch105.ts +2 -3
  136. package/src/lint/post-synth/sqlch106.ts +2 -3
  137. package/src/lint/post-synth/sqlch107.ts +2 -3
  138. package/src/lint/post-synth/sqlch111.ts +2 -3
  139. package/src/lint/post-synth/sqlch112.ts +2 -3
  140. package/src/lint/post-synth/sqlch113.ts +2 -3
  141. package/src/lint/post-synth/sqlch114.ts +2 -3
  142. package/src/lint/post-synth/sqlch115.ts +2 -3
  143. package/src/lint/post-synth/sqlch120.ts +2 -3
  144. package/src/lint/post-synth/sqlch125.ts +2 -3
  145. package/src/lint/rules/postgres-templates.ts +1 -1
  146. package/src/lint/rules/sqlch003.ts +1 -1
  147. package/src/lint/rules/sqlpg002.ts +1 -1
  148. package/src/lint/rules/templates.ts +1 -1
  149. package/src/lint/rules/typed-names.ts +1 -1
  150. package/src/lint/rules/typescript-7.test.ts +101 -0
  151. package/src/lsp/completions.ts +1 -1
  152. package/src/lsp/postgres-template.ts +1 -1
  153. package/src/lsp/postgres.ts +1 -1
  154. package/src/lsp/refs.ts +1 -1
  155. package/src/lsp/template.ts +1 -1
  156. package/src/postgres/apply/statements.ts +2 -0
  157. package/src/postgres/entities.ts +9 -1
  158. package/src/postgres/parser.ts +31 -8
  159. package/src/postgres/plan/diff.ts +18 -0
  160. package/src/postgres/plan/normalize.ts +12 -0
  161. package/src/postgres/plan/rules.ts +7 -0
  162. package/src/postgres/testdata/server-ddl-baseline.json +38 -0
  163. package/src/postgres.ts +12 -0
  164. package/src/read/clickhouse.ts +137 -0
  165. package/src/read/objects.ts +137 -0
  166. package/src/read/postgres.ts +180 -0
  167. package/src/read-policies.e2e.test.ts +144 -0
  168. package/src/read.test.ts +143 -0
  169. package/src/read.ts +69 -0
  170. package/src/serializer.ts +37 -1
  171. package/src/sql-files.ts +27 -6
  172. package/src/sql-types.ts +19 -2
  173. package/src/unknown-clauses.test.ts +227 -0
package/src/sql-files.ts CHANGED
@@ -18,7 +18,9 @@
18
18
  * `MATERIALIZED VIEW`, `DICTIONARY`, `FUNCTION`, `USER`, `ROLE` and `ROW
19
19
  * POLICY`, and `GRANT` (#3711). Any other statement, and any
20
20
  * statement that does not parse, throws a {@link SqlFileError} naming every
21
- * such statement: none is left out.
21
+ * such statement: none is left out. A statement that parses except for a
22
+ * clause the parser does not know (#3872) is read, kept as written, and the
23
+ * clause listed in its object's `unparsed` with the line it is on.
22
24
  *
23
25
  * A name is a reference where it is qualified (`app.users`), or bare where an
24
26
  * object is named (`REFERENCES users`, `FROM users`, an index's table, a
@@ -27,7 +29,7 @@
27
29
 
28
30
  import type { Declarable } from "@intentius/chant/declarable";
29
31
  import type { SqlDialect } from "./dialects";
30
- import { SqlFileError, type ReadOptions } from "./files/common";
32
+ import { SqlFileError, type FileLine, type ReadOptions } from "./files/common";
31
33
  import { postgresTag, readPostgres, templateStrings } from "./files/postgres";
32
34
  import { clickhouseTag, readClickHouse } from "./files/clickhouse";
33
35
  import { exportNames as pgExportNames, objectsToIR as pgObjectsToIR, type ImportedPgObject } from "./postgres/import/ir";
@@ -52,6 +54,14 @@ export interface SqlFileObject {
52
54
  ddl: string;
53
55
  /** The file it came from, the `origin` it was read under. */
54
56
  origin: string;
57
+ /** The line its statement starts on in the file, from 1. */
58
+ line?: number;
59
+ /**
60
+ * Clauses in it chant's parser does not know (#3872), each with the line
61
+ * it starts on: kept in `ddl` as written, for the server to check. Absent
62
+ * when there are none.
63
+ */
64
+ unparsed?: Array<{ clause: string; line: number; column?: string }>;
55
65
  }
56
66
 
57
67
  /** A file to read: its DDL and the name errors give it. */
@@ -64,8 +74,8 @@ type Imported = ImportedPgObject | ImportedObject;
64
74
  const qualifierOf = (o: Imported): string | undefined => ("schema" in o ? o.schema : "database" in o ? o.database : undefined);
65
75
  const qualifiedName = (o: Imported): string => (qualifierOf(o) ? `${qualifierOf(o)}.${o.name}` : o.name);
66
76
 
67
- function readObjects(dialect: SqlDialect, sources: readonly SqlFileSource[]): Array<Imported & { origin: string }> {
68
- const out: Array<Imported & { origin: string }> = [];
77
+ function readObjects(dialect: SqlDialect, sources: readonly SqlFileSource[]): Array<Imported & FileLine & { origin: string }> {
78
+ const out: Array<Imported & FileLine & { origin: string }> = [];
69
79
  for (const source of sources) {
70
80
  const problems: string[] = [];
71
81
  const objects = dialect === "postgres" ? readPostgres(source.ddl, source, problems) : readClickHouse(source.ddl, source, problems);
@@ -75,6 +85,9 @@ function readObjects(dialect: SqlDialect, sources: readonly SqlFileSource[]): Ar
75
85
  return out;
76
86
  }
77
87
 
88
+ /** An object's line and unknown clauses, the fields that are set. */
89
+ const fileMeta = (o: FileLine): FileLine => ({ ...(o.line !== undefined ? { line: o.line } : {}), ...(o.unparsed ? { unparsed: o.unparsed } : {}) });
90
+
78
91
  /** Export names for objects read from files, unique across them. */
79
92
  function namesFor(dialect: SqlDialect, objects: readonly Imported[]): string[] {
80
93
  return dialect === "postgres" ? pgExportNames(objects as ImportedPgObject[]) : chExportNames(objects as ImportedObject[]);
@@ -84,7 +97,7 @@ function namesFor(dialect: SqlDialect, objects: readonly Imported[]): string[] {
84
97
  export function readSqlFile(dialect: SqlDialect, ddl: string, options: ReadOptions): SqlFileObject[] {
85
98
  const objects = readObjects(dialect, [{ ...options, ddl }]);
86
99
  const names = namesFor(dialect, objects);
87
- return objects.map((o, i) => ({ export: names[i]!, type: o.type, name: qualifiedName(o), ddl: o.ddl, origin: o.origin }));
100
+ return objects.map((o, i) => ({ export: names[i]!, type: o.type, name: qualifiedName(o), ddl: o.ddl, origin: o.origin, ...fileMeta(o) }));
88
101
  }
89
102
 
90
103
  /**
@@ -100,7 +113,7 @@ export function sqlFileDeclarations(dialect: SqlDialect, ddl: string, options: R
100
113
  if (!file) throw new SqlFileError(options.origin, ["the DDL creates no objects"]);
101
114
  return {
102
115
  content: file.content,
103
- objects: ir.resources.map((r, i) => ({ export: r.logicalId, type: r.type, name: qualifiedName(objects[i]!), ddl: objects[i]!.ddl, origin: options.origin })),
116
+ objects: ir.resources.map((r, i) => ({ export: r.logicalId, type: r.type, name: qualifiedName(objects[i]!), ddl: objects[i]!.ddl, origin: options.origin, ...fileMeta(objects[i]!) })),
104
117
  };
105
118
  }
106
119
 
@@ -109,7 +122,13 @@ export function sqlFileOf(entity: unknown): string | undefined {
109
122
  return typeof entity === "object" && entity !== null ? ((entity as Record<symbol, unknown>)[FROM_FILE] as string | undefined) : undefined;
110
123
  }
111
124
 
125
+ /** The line an entity read from a `.sql` file starts on in it. Undefined for any other entity. */
126
+ export function sqlFileLineOf(entity: unknown): number | undefined {
127
+ return typeof entity === "object" && entity !== null ? ((entity as Record<symbol, unknown>)[FROM_LINE] as number | undefined) : undefined;
128
+ }
129
+
112
130
  const FROM_FILE = Symbol.for("chant.sql.fromFile");
131
+ const FROM_LINE = Symbol.for("chant.sql.fromLine");
113
132
 
114
133
  /** A declared entity as a template target: its export, type and name. */
115
134
  function targetOf(dialect: SqlDialect, exportName: string, entity: Declarable): PgTemplateItem | undefined {
@@ -150,6 +169,7 @@ export function sqlFileEntities(dialect: SqlDialect, sources: readonly SqlFileSo
150
169
 
151
170
  const items: PgTemplateItem[] = objects.map((o, i) => ({ exportName: names[i]!, type: o.type, ...("schema" in o && o.schema ? { schema: o.schema } : {}), ...("database" in o && o.database ? { database: o.database } : {}), name: o.name, ddl: o.ddl }) as PgTemplateItem);
152
171
  const origin = new Map(objects.map((o, i) => [names[i]!, o.origin]));
172
+ const lineOf = new Map(objects.map((o, i) => [names[i]!, o.line]));
153
173
  const knownItems = [...known].map(([n, e]) => targetOf(dialect, n, e)).filter((t): t is PgTemplateItem => t !== undefined);
154
174
  const declared = new Map(knownItems.map((t) => [`${t.type} ${qualifiedName(t as Imported)}`, t.exportName]));
155
175
  const seen = new Map<string, string>();
@@ -186,6 +206,7 @@ export function sqlFileEntities(dialect: SqlDialect, sources: readonly SqlFileSo
186
206
  try {
187
207
  const entity = tagOf(it.type)!(templateStrings(body.parts), ...values);
188
208
  Object.defineProperty(entity, FROM_FILE, { value: origin.get(it.exportName), enumerable: false });
209
+ if (lineOf.get(it.exportName) !== undefined) Object.defineProperty(entity, FROM_LINE, { value: lineOf.get(it.exportName), enumerable: false });
189
210
  built.set(it.exportName, entity);
190
211
  } catch (e) {
191
212
  failed.add(it.exportName);
package/src/sql-types.ts CHANGED
@@ -22,6 +22,9 @@
22
22
  * (`./lint/post-synth/postgres-names.ts`'s `typeIdentity`), the type
23
23
  * family in ClickHouse; `never` when it cannot be told.
24
24
  *
25
+ * A column that carries a clause chant does not read (#3872) keeps the type
26
+ * that was read, with `nullable` and `cmp` `never`.
27
+ *
25
28
  * A view's column that is one column reference (`${t.columns.c}`, `o.${t.columns.c}`)
26
29
  * is that column's type; any other expression is `unknown`.
27
30
  */
@@ -108,6 +111,17 @@ interface RelationEntry {
108
111
 
109
112
  const UNKNOWN_COLUMN = (name: string): ColumnEntry => ({ name, sqlType: "unknown", ts: "unknown", nullable: "never" });
110
113
 
114
+ /** The columns that carry a clause chant does not read (#3872). */
115
+ const opaqueColumns = (o: OutputObject): Set<string> =>
116
+ new Set(((o.unparsed as Array<{ column?: string }> | undefined) ?? []).flatMap((u) => (u.column !== undefined ? [u.column] : [])));
117
+
118
+ /**
119
+ * A column with a clause chant does not read (#3872): its type is what was
120
+ * read, but whether it can be NULL and how it compares in a key are not
121
+ * told, so it fits every slot rather than a guessed one.
122
+ */
123
+ const opaque = (c: ColumnEntry): ColumnEntry => ({ name: c.name, sqlType: c.sqlType, ts: c.ts, nullable: "never" });
124
+
111
125
  const posix = (p: string) => p.split(sep).join("/");
112
126
 
113
127
  /** Where a declaration came from: its file relative to the project, and its export. */
@@ -215,13 +229,15 @@ function postgresEntries(objects: OutputObject[], major: number, entities: Reado
215
229
  for (const c of pt?.columns ?? []) if (!inherited.has(c.name)) inherited.set(c.name, c.type);
216
230
  }
217
231
  const pk = new Set(t.primaryKey?.columns ?? []);
232
+ const unread = opaqueColumns(o);
218
233
  columns = names.map((name) => {
219
234
  const c = own.get(name);
220
235
  const text = c?.type ?? inherited.get(name);
221
236
  if (text === undefined) return UNKNOWN_COLUMN(name);
222
237
  const ty = pgType(p, text);
223
238
  const notNull = c?.notNull === true || pk.has(name) || c?.generated?.kind === "identity" || ty.notNull;
224
- return { name, sqlType: ty.sqlType, ts: ty.ts, nullable: !notNull, cmp: typeIdentity(p, text) };
239
+ const entry: ColumnEntry = { name, sqlType: ty.sqlType, ts: ty.ts, nullable: !notNull, cmp: typeIdentity(p, text) };
240
+ return unread.has(name) ? opaque(entry) : entry;
225
241
  });
226
242
  } else {
227
243
  columns = names.map((name) => {
@@ -265,9 +281,10 @@ function clickhouseEntries(objects: OutputObject[], entities: ReadonlyMap<string
265
281
  const entity = entities.get(exportName);
266
282
  if (!o || !entity || !CH_RELATIONS.has(o.type)) return undefined;
267
283
  const declared = new Map(((o.columns as Array<{ name: string; type?: string; nullable?: boolean }> | undefined) ?? []).map((c) => [c.name, c]));
284
+ const unread = opaqueColumns(o);
268
285
  const columns = runtimeColumns(entity).map((name) => {
269
286
  const c = declared.get(name);
270
- if (c) return chColumn(name, c.type, c.nullable);
287
+ if (c) return unread.has(name) ? opaque(chColumn(name, c.type, c.nullable)) : chColumn(name, c.type, c.nullable);
271
288
  const via = passthrough(o, name, quoteIdentifier);
272
289
  const source = via && entryOf(via.export)?.columns.find((x) => x.name === via.column);
273
290
  return source ? { ...source, name } : UNKNOWN_COLUMN(name);
@@ -0,0 +1,227 @@
1
+ /**
2
+ * A clause chant's parser does not know (#3872): the statement is kept as
3
+ * written, the build warns naming where it is, the rest of the object is
4
+ * read, and a plan classes a change in the clause as one to review.
5
+ */
6
+
7
+ import { describe, expect, test } from "vitest";
8
+ import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
9
+ import { dirname, join, resolve } from "node:path";
10
+ import { fileURLToPath } from "node:url";
11
+ import { build } from "@intentius/chant/build";
12
+ import { tokenizeText as chTokens } from "./clickhouse/tokens";
13
+ import { parseCreate, type TableNode } from "./clickhouse/parser";
14
+ import { tokenizeText as pgTokens } from "./postgres/tokens";
15
+ import { parseStatements, type IndexNode, type TableNode as PgTableNode } from "./postgres/parser";
16
+ import { table, view, database } from "./clickhouse/entities";
17
+ import { table as pgTable } from "./postgres/entities";
18
+ import { readSqlFile, sqlFileEntities, SqlFileError } from "./sql-files";
19
+ import { canonicalObject } from "./clickhouse/plan/normalize";
20
+ import { diffSchemas } from "./clickhouse/plan/diff";
21
+ import { diffPgSchemas } from "./postgres/plan/diff";
22
+ import { diffObject, type PgSchemaObject } from "./postgres/plan/schema";
23
+ import { planStatements } from "./clickhouse/apply/statements";
24
+ import { sqlSerializer, unparsedWarnings } from "./serializer";
25
+ import { sqlPlugin } from "./plugin";
26
+ import { renderSqlTypes } from "./sql-types";
27
+ import type { Declarable } from "@intentius/chant/declarable";
28
+
29
+ const ch = (sql: string) => parseCreate(chTokens(sql, 0));
30
+ const pg = (sql: string) => parseStatements(pgTokens(sql, 0))[0]!;
31
+
32
+ describe("the ClickHouse parser keeps a clause it does not know", () => {
33
+ test("after the storage clauses, on a column, before a view's AS, in a database and a dictionary", () => {
34
+ const t = ch("CREATE TABLE a.t (x UInt64, k String ENCRYPTED WITH 'k1', y UInt8) ENGINE = MergeTree ORDER BY x PARTITION_STRATEGY = 'hourly' SETTINGS index_granularity = 8192") as TableNode;
35
+ expect(t.unknown?.map((u) => [u.clause, u.column])).toEqual([
36
+ ["ENCRYPTED WITH", "k"],
37
+ ["PARTITION_STRATEGY", undefined],
38
+ ]);
39
+ // What follows the clause is still read.
40
+ expect(t.columns.map((c) => c.name)).toEqual(["x", "k", "y"]);
41
+ expect(t.settings?.map((s) => s.key)).toEqual(["index_granularity"]);
42
+
43
+ expect(ch("CREATE VIEW a.v SQL_CACHE = 'on' AS SELECT 1").unknown?.map((u) => u.clause)).toEqual(["SQL_CACHE"]);
44
+ expect(ch("CREATE DATABASE d ENGINE = Atomic SHARDING_MODE 'auto' COMMENT 'x'").unknown?.map((u) => u.clause)).toEqual(["SHARDING_MODE"]);
45
+ expect(
46
+ ch("CREATE DICTIONARY a.d (id UInt64, v String DEFAULT '' SPARSE) PRIMARY KEY id SOURCE(NULL()) LAYOUT(FLAT()) LIFETIME(0) WARMUP(10)").unknown?.map((u) => [u.clause, u.column]),
47
+ ).toEqual([
48
+ ["SPARSE", "v"],
49
+ ["WARMUP", undefined],
50
+ ]);
51
+ expect(ch("CREATE TABLE a.t (x UInt64) ENGINE = MergeTree ORDER BY x").unknown).toBeUndefined();
52
+ });
53
+
54
+ test("and still fails on what is malformed", () => {
55
+ // One letter from a clause it knows: a misspelling.
56
+ expect(() => ch("CREATE TABLE a.t (x UInt64) ENGINE = MergeTree ORDR BY x")).toThrow(/one letter from ORDER/);
57
+ expect(() => ch("CREATE TABLE a.t (x UInt64 CODCE(ZSTD))")).toThrow(/one letter from CODEC/);
58
+ // Unbalanced parentheses, a word that is not a key word, an unknown statement kind.
59
+ expect(() => ch("CREATE TABLE a.t (x UInt64) ENGINE = Log FUTURE_THING (a")).toThrow(/unbalanced parentheses|expected/);
60
+ expect(() => ch("CREATE TABLE a.t (x UInt64) ENGINE = Log 'stray'")).toThrow(/unexpected token/);
61
+ expect(() => ch("CREATE NAMED COLLECTION c AS a = 1")).toThrow(/expected DATABASE, TABLE/);
62
+ });
63
+ });
64
+
65
+ describe("the Postgres parser keeps a clause it does not know", () => {
66
+ test("after a table's column list, on a column, after an index's elements, before a view's AS, in a sequence", () => {
67
+ const t = pg("CREATE TABLE app.t (id bigint PRIMARY KEY, note text INVISIBLE, n int NOT NULL) STORAGE_CLASS 'cold' TABLESPACE fast") as PgTableNode;
68
+ expect(t.unknown?.map((u) => [u.clause, u.column])).toEqual([
69
+ ["INVISIBLE", "note"],
70
+ ["STORAGE_CLASS", undefined],
71
+ ]);
72
+ expect(t.columns.map((c) => c.name)).toEqual(["id", "note", "n"]);
73
+ expect(t.tablespace).toBeDefined();
74
+ expect((pg("CREATE INDEX i ON app.t (id) INVISIBLE") as IndexNode).unknown?.map((u) => u.clause)).toEqual(["INVISIBLE"]);
75
+ expect(pg("CREATE VIEW app.v SECURITY_BARRIER_HINT AS SELECT 1").statement).toBe("view");
76
+ expect(pg("CREATE SEQUENCE app.s CYCLE SEQUENCE_KIND local").statement).toBe("sequence");
77
+ });
78
+
79
+ test("and still fails on what is malformed, and on what it refuses on purpose", () => {
80
+ expect(() => pg("CREATE TABLE app.t (id int) TABLESPAC fast")).toThrow(/one letter from TABLESPACE/);
81
+ expect(() => pg("CREATE TABLE app.t (id int NOT NUL)")).toThrow();
82
+ expect(() => pg("CREATE ROLE r PASSWORD 'x'")).toThrow(/password/);
83
+ expect(() => pg("CREATE TABLE app.t (id int) WITHOUT_SOMETHING (")).toThrow();
84
+ });
85
+ });
86
+
87
+ describe("a tag keeps the clause in its DDL and names it in props", () => {
88
+ test("ClickHouse", () => {
89
+ const t = table`CREATE TABLE a.t (
90
+ x UInt64,
91
+ k String ENCRYPTED WITH 'k1'
92
+ ) ENGINE = MergeTree ORDER BY x
93
+ PARTITION_STRATEGY = 'hourly'`;
94
+ expect(t.props.ddl).toContain("PARTITION_STRATEGY = 'hourly'");
95
+ expect(t.props.ddl).toContain("ENCRYPTED WITH 'k1'");
96
+ expect(t.props.unparsed).toEqual([
97
+ { clause: "ENCRYPTED WITH", text: "ENCRYPTED WITH 'k1'", column: "k", line: 3 },
98
+ { clause: "PARTITION_STRATEGY", text: "PARTITION_STRATEGY = 'hourly'", line: 5 },
99
+ ]);
100
+ expect(t.props.orderBy).toBe("x");
101
+ expect(Object.keys((t as unknown as { columns: object }).columns)).toEqual(["x", "k"]);
102
+ expect(database`CREATE DATABASE a`.props.unparsed).toBeUndefined();
103
+ expect(view`CREATE VIEW a.v AS SELECT x FROM ${t}`.props.unparsed).toBeUndefined();
104
+ });
105
+
106
+ test("Postgres", () => {
107
+ const t = pgTable`CREATE TABLE app.t (id bigint PRIMARY KEY, note text) STORAGE_CLASS 'cold'`;
108
+ expect(t.props.ddl).toContain("STORAGE_CLASS 'cold'");
109
+ expect(t.props.unparsed).toEqual([{ clause: "STORAGE_CLASS", text: "STORAGE_CLASS 'cold'", line: 1 }]);
110
+ });
111
+ });
112
+
113
+ const CH_FILE = `CREATE DATABASE analytics;
114
+
115
+ CREATE TABLE analytics.events (
116
+ ts DateTime,
117
+ kind String ENCRYPTED WITH 'k1'
118
+ ) ENGINE = MergeTree ORDER BY ts
119
+ PARTITION_STRATEGY = 'hourly';
120
+ `;
121
+
122
+ describe("a .sql file", () => {
123
+ test("reads, with each clause and its line in the file", () => {
124
+ const [db, events] = readSqlFile("clickhouse", CH_FILE, { origin: "src/schema.sql" });
125
+ expect(db!.unparsed).toBeUndefined();
126
+ expect(events!.line).toBe(3);
127
+ expect(events!.unparsed).toEqual([
128
+ { clause: "ENCRYPTED WITH", line: 5, column: "kind" },
129
+ { clause: "PARTITION_STRATEGY", line: 7 },
130
+ ]);
131
+ const pgObjects = readSqlFile("postgres", "CREATE SCHEMA app;\nCREATE TABLE app.t (\n id bigint\n) STORAGE_CLASS 'cold';\nCOMMENT ON TABLE app.t IS 'x';", { origin: "db.sql" });
132
+ expect(pgObjects[1]!.unparsed).toEqual([{ clause: "STORAGE_CLASS", line: 4 }]);
133
+ });
134
+
135
+ test("builds entities whose warnings name the file and line", () => {
136
+ const entities = sqlFileEntities("clickhouse", [{ origin: "src/schema.sql", ddl: CH_FILE }]);
137
+ const warnings = unparsedWarnings(entities);
138
+ expect(warnings).toHaveLength(2);
139
+ expect(warnings[0]).toMatch(/^src\/schema\.sql:5: analytics\.events has a clause chant does not read, ENCRYPTED WITH on column kind\. It is sent as written/);
140
+ expect(warnings[1]).toMatch(/^src\/schema\.sql:7: analytics\.events has a clause chant does not read, PARTITION_STRATEGY\./);
141
+ });
142
+
143
+ test("statements that are not schema still fail", () => {
144
+ expect(() => readSqlFile("clickhouse", "CREATE DATABASE a;\nDROP TABLE a.t", { origin: "x.sql" })).toThrow(SqlFileError);
145
+ expect(() => readSqlFile("postgres", "INSERT INTO app.t VALUES (1)", { origin: "x.sql" })).toThrow(/not a statement that declares an object/);
146
+ });
147
+ });
148
+
149
+ const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
150
+
151
+ describe("a build", () => {
152
+ test("succeeds with a warning, and the clause is in the DDL it writes", async () => {
153
+ const path = join(repoRoot, ".cache", `sql-3872-${process.pid}`);
154
+ await rm(path, { recursive: true, force: true });
155
+ await mkdir(join(path, "src"), { recursive: true });
156
+ const dir = await realpath(path);
157
+ await writeFile(join(dir, "src", "schema.sql"), CH_FILE);
158
+ try {
159
+ const result = await build(join(dir, "src"), [sqlSerializer], undefined, {
160
+ lexicons: ["sql"],
161
+ intrinsics: sqlPlugin.intrinsics?.() ?? [],
162
+ buildRoots: [(ctx) => sqlPlugin.buildRoots!({ projectRoot: dir, config: {}, entities: ctx.entities, ...(ctx.sourceDir ? { sourceDir: ctx.sourceDir } : {}) })],
163
+ });
164
+ expect(result.errors.map((e) => e.message)).toEqual([]);
165
+ expect(result.warnings.filter((w) => /chant does not read/.test(w))).toHaveLength(2);
166
+ const out = result.outputs.get("sql");
167
+ const files = typeof out === "string" ? {} : (out?.files ?? {});
168
+ expect(String(files["clickhouse.sql"])).toContain("PARTITION_STRATEGY = 'hourly'");
169
+ } finally {
170
+ await rm(path, { recursive: true, force: true });
171
+ }
172
+ });
173
+ });
174
+
175
+ describe("typed columns", () => {
176
+ test("a column with a clause chant does not read keeps its type, and its nullability and key comparison are never", () => {
177
+ const events = table`CREATE TABLE a.events (id UInt64, k String ENCRYPTED WITH 'k1') ENGINE = MergeTree ORDER BY id`;
178
+ const text = renderSqlTypes(new Map([["events", events]]) as Map<string, Declarable>, { projectDir: "/" });
179
+ expect(text).toContain('id: Column<"UInt64", { ts: string; nullable: false; cmp: "UInt64" }>;');
180
+ expect(text).toContain('k: Column<"String", { ts: string; nullable: never; cmp: never }>;');
181
+ });
182
+ });
183
+
184
+ describe("a plan", () => {
185
+ const declared = (ddl: string) => ({ key: "analytics.events", exportName: "events", canonical: canonicalObject(ddl) });
186
+ const BASE = "CREATE TABLE analytics.events (ts DateTime) ENGINE = MergeTree ORDER BY ts";
187
+
188
+ test("ClickHouse: the same clause is no change, a changed one is SQLCH290 for review, one only the server prints is a hint", () => {
189
+ const same = diffSchemas([declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)], [declared(`${BASE}\nPARTITION_STRATEGY = 'hourly'`)]);
190
+ expect(same.changes).toEqual([]);
191
+
192
+ const changed = diffSchemas([declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)], [declared(`${BASE} PARTITION_STRATEGY = 'daily'`)]);
193
+ expect(changed.changes.map((c) => [c.field, c.rule, c.class])).toEqual([["unparsed.PARTITION_STRATEGY", "SQLCH290", "rebuild"]]);
194
+
195
+ const added = diffSchemas([declared(BASE)], [declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)]);
196
+ expect(added.changes.map((c) => c.rule)).toEqual(["SQLCH290"]);
197
+
198
+ const serverOnly = diffSchemas([declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)], [declared(BASE)]);
199
+ expect(serverOnly.changes).toEqual([]);
200
+ expect(serverOnly.hints.join("\n")).toMatch(/PARTITION_STRATEGY .* a clause chant does not read/);
201
+
202
+ // The rest of the object is still planned, and the refusal says why.
203
+ const both = diffSchemas([declared(`${BASE} PARTITION_STRATEGY = 'hourly'`)], [declared(`CREATE TABLE analytics.events (ts DateTime, n UInt8) ENGINE = MergeTree ORDER BY ts PARTITION_STRATEGY = 'daily'`)]);
204
+ expect(both.changes.map((c) => c.rule).sort()).toEqual(["SQLCH201", "SQLCH290"]);
205
+ const after = declared(`CREATE TABLE analytics.events (ts DateTime, n UInt8) ENGINE = MergeTree ORDER BY ts PARTITION_STRATEGY = 'daily'`);
206
+ const plan = planStatements({
207
+ declared: [{ ...after, type: "ClickHouse::Table", ddl: "CREATE TABLE analytics.events (ts DateTime, n UInt8) ENGINE = MergeTree ORDER BY ts PARTITION_STRATEGY = 'daily'" } as never],
208
+ changes: both.changes,
209
+ current: new Map(),
210
+ });
211
+ const verdict = plan.objects[0]!;
212
+ expect(verdict.verdict).toBe("rebuild");
213
+ expect((verdict as { detail: string }).detail).toMatch(/^needs review, since chant does not read the clause/);
214
+ });
215
+
216
+ test("Postgres: a changed clause is SQLPG299, refused for review", () => {
217
+ const o = (ddl: string): PgSchemaObject => ({ key: "t", canonical: { ...diffObject("Postgres::Table", ddl, "public"), exportName: "t" } });
218
+ const before = o("CREATE TABLE app.t (id bigint PRIMARY KEY) STORAGE_CLASS 'cold'");
219
+ expect(diffPgSchemas([before], [o("CREATE TABLE app.t (id bigint PRIMARY KEY) STORAGE_CLASS 'cold'")]).changes).toEqual([]);
220
+ const d = diffPgSchemas([before], [o("CREATE TABLE app.t (id bigint PRIMARY KEY, n int) STORAGE_CLASS 'hot'")]);
221
+ expect(d.changes.map((c) => [c.field, c.rule, c.class])).toEqual([
222
+ ["unparsed.STORAGE_CLASS", "SQLPG299", "expand"],
223
+ ["columns.n", "SQLPG201", "metadata"],
224
+ ]);
225
+ expect(d.refused.map((c) => c.rule)).toEqual(["SQLPG299"]);
226
+ });
227
+ });