@intentius/chant-lexicon-sql 0.116.0 → 0.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/dist/clickhouse/access.d.ts +132 -0
  2. package/dist/clickhouse/access.d.ts.map +1 -0
  3. package/dist/clickhouse/apply/apply.d.ts.map +1 -1
  4. package/dist/clickhouse/apply/statements.d.ts +2 -0
  5. package/dist/clickhouse/apply/statements.d.ts.map +1 -1
  6. package/dist/clickhouse/entities.d.ts +171 -2
  7. package/dist/clickhouse/entities.d.ts.map +1 -1
  8. package/dist/clickhouse/http.d.ts +3 -0
  9. package/dist/clickhouse/http.d.ts.map +1 -1
  10. package/dist/clickhouse/import/generator.d.ts +28 -0
  11. package/dist/clickhouse/import/generator.d.ts.map +1 -1
  12. package/dist/clickhouse/import/ir.d.ts.map +1 -1
  13. package/dist/clickhouse/import/parser.d.ts +3 -3
  14. package/dist/clickhouse/import/parser.d.ts.map +1 -1
  15. package/dist/clickhouse/live/bind.d.ts.map +1 -1
  16. package/dist/clickhouse/live/catalog.d.ts +41 -6
  17. package/dist/clickhouse/live/catalog.d.ts.map +1 -1
  18. package/dist/clickhouse/live/describe-resources.d.ts +3 -2
  19. package/dist/clickhouse/live/describe-resources.d.ts.map +1 -1
  20. package/dist/clickhouse/parser.d.ts +54 -3
  21. package/dist/clickhouse/parser.d.ts.map +1 -1
  22. package/dist/clickhouse/plan/commands.d.ts +3 -1
  23. package/dist/clickhouse/plan/commands.d.ts.map +1 -1
  24. package/dist/clickhouse/plan/deep.d.ts.map +1 -1
  25. package/dist/clickhouse/plan/diff.d.ts +12 -0
  26. package/dist/clickhouse/plan/diff.d.ts.map +1 -1
  27. package/dist/clickhouse/plan/normalize.d.ts +61 -1
  28. package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
  29. package/dist/clickhouse/plan/report.d.ts.map +1 -1
  30. package/dist/clickhouse/plan/rules.d.ts +7 -0
  31. package/dist/clickhouse/plan/rules.d.ts.map +1 -1
  32. package/dist/clickhouse/plan/schema.d.ts +8 -4
  33. package/dist/clickhouse/plan/schema.d.ts.map +1 -1
  34. package/dist/clickhouse/rebuild/backfill.d.ts +3 -1
  35. package/dist/clickhouse/rebuild/backfill.d.ts.map +1 -1
  36. package/dist/clickhouse/rebuild/observe.d.ts +7 -3
  37. package/dist/clickhouse/rebuild/observe.d.ts.map +1 -1
  38. package/dist/clickhouse/rebuild/op.d.ts +8 -0
  39. package/dist/clickhouse/rebuild/op.d.ts.map +1 -1
  40. package/dist/clickhouse/rebuild/steps.d.ts +48 -0
  41. package/dist/clickhouse/rebuild/steps.d.ts.map +1 -1
  42. package/dist/clickhouse/testing/fake-server.d.ts.map +1 -1
  43. package/dist/clickhouse/testing/writable-server.d.ts.map +1 -1
  44. package/dist/clickhouse/topology.d.ts.map +1 -1
  45. package/dist/clickhouse.d.ts +1 -1
  46. package/dist/clickhouse.d.ts.map +1 -1
  47. package/dist/codegen/classifier-page.d.ts.map +1 -1
  48. package/dist/codegen/registry.d.ts.map +1 -1
  49. package/dist/config.d.ts +70 -7
  50. package/dist/config.d.ts.map +1 -1
  51. package/dist/core/references.d.ts +4 -2
  52. package/dist/core/references.d.ts.map +1 -1
  53. package/dist/core/statements.d.ts +11 -0
  54. package/dist/core/statements.d.ts.map +1 -1
  55. package/dist/files/build-root.d.ts +26 -0
  56. package/dist/files/build-root.d.ts.map +1 -0
  57. package/dist/files/clickhouse.d.ts +19 -0
  58. package/dist/files/clickhouse.d.ts.map +1 -0
  59. package/dist/files/common.d.ts +25 -0
  60. package/dist/files/common.d.ts.map +1 -0
  61. package/dist/files/postgres.d.ts +36 -0
  62. package/dist/files/postgres.d.ts.map +1 -0
  63. package/dist/import-parser.d.ts +25 -0
  64. package/dist/import-parser.d.ts.map +1 -0
  65. package/dist/index.d.ts +2 -1
  66. package/dist/index.d.ts.map +1 -1
  67. package/dist/integrity.json +8 -8
  68. package/dist/lint/rules/sqlch001.d.ts +1 -1
  69. package/dist/lint/rules/sqlch001.d.ts.map +1 -1
  70. package/dist/lint/rules/sqlch002.d.ts.map +1 -1
  71. package/dist/lint/rules/sqlpg001.d.ts.map +1 -1
  72. package/dist/lint/rules/templates.d.ts +1 -1
  73. package/dist/lint/rules/templates.d.ts.map +1 -1
  74. package/dist/lsp/postgres-template.d.ts.map +1 -1
  75. package/dist/manifest.json +1 -1
  76. package/dist/meta.json +50 -0
  77. package/dist/okf/index.md +11 -1
  78. package/dist/okf/rules/SQLCH001.md +2 -2
  79. package/dist/okf/types/Dictionary.md +9 -0
  80. package/dist/okf/types/Function.md +9 -0
  81. package/dist/okf/types/Grant.md +9 -0
  82. package/dist/okf/types/PostgresDefaultPrivileges.md +9 -0
  83. package/dist/okf/types/PostgresGrant.md +9 -0
  84. package/dist/okf/types/PostgresPolicy.md +9 -0
  85. package/dist/okf/types/PostgresRole.md +9 -0
  86. package/dist/okf/types/Role.md +9 -0
  87. package/dist/okf/types/RowPolicy.md +9 -0
  88. package/dist/okf/types/User.md +9 -0
  89. package/dist/op/activities/clickhouse-rebuild.d.ts.map +1 -1
  90. package/dist/op/activities/postgres-migration.d.ts.map +1 -1
  91. package/dist/op/activity-contracts.d.ts +19 -0
  92. package/dist/op/activity-contracts.d.ts.map +1 -1
  93. package/dist/plugin.d.ts.map +1 -1
  94. package/dist/postgres/access/acl.d.ts +119 -0
  95. package/dist/postgres/access/acl.d.ts.map +1 -0
  96. package/dist/postgres/access/import.d.ts +13 -0
  97. package/dist/postgres/access/import.d.ts.map +1 -0
  98. package/dist/postgres/access/live.d.ts +24 -0
  99. package/dist/postgres/access/live.d.ts.map +1 -0
  100. package/dist/postgres/apply/apply.d.ts.map +1 -1
  101. package/dist/postgres/apply/statements.d.ts +71 -0
  102. package/dist/postgres/apply/statements.d.ts.map +1 -1
  103. package/dist/postgres/entities.d.ts +106 -4
  104. package/dist/postgres/entities.d.ts.map +1 -1
  105. package/dist/postgres/entity-types.d.ts +4 -0
  106. package/dist/postgres/entity-types.d.ts.map +1 -1
  107. package/dist/postgres/import/generator.d.ts +37 -1
  108. package/dist/postgres/import/generator.d.ts.map +1 -1
  109. package/dist/postgres/import/live-export.d.ts +6 -0
  110. package/dist/postgres/import/live-export.d.ts.map +1 -1
  111. package/dist/postgres/live/bind.d.ts +2 -0
  112. package/dist/postgres/live/bind.d.ts.map +1 -1
  113. package/dist/postgres/live/catalog.d.ts +11 -2
  114. package/dist/postgres/live/catalog.d.ts.map +1 -1
  115. package/dist/postgres/live/client.d.ts +3 -0
  116. package/dist/postgres/live/client.d.ts.map +1 -1
  117. package/dist/postgres/live/describe-resources.d.ts +9 -0
  118. package/dist/postgres/live/describe-resources.d.ts.map +1 -1
  119. package/dist/postgres/migrate/op.d.ts +29 -0
  120. package/dist/postgres/migrate/op.d.ts.map +1 -1
  121. package/dist/postgres/parser.d.ts +76 -3
  122. package/dist/postgres/parser.d.ts.map +1 -1
  123. package/dist/postgres/plan/commands.d.ts +12 -0
  124. package/dist/postgres/plan/commands.d.ts.map +1 -1
  125. package/dist/postgres/plan/deep.d.ts.map +1 -1
  126. package/dist/postgres/plan/diff.d.ts +10 -1
  127. package/dist/postgres/plan/diff.d.ts.map +1 -1
  128. package/dist/postgres/plan/normalize.d.ts +8 -1
  129. package/dist/postgres/plan/normalize.d.ts.map +1 -1
  130. package/dist/postgres/plan/rules.d.ts +8 -0
  131. package/dist/postgres/plan/rules.d.ts.map +1 -1
  132. package/dist/postgres/plan/schema.d.ts +2 -1
  133. package/dist/postgres/plan/schema.d.ts.map +1 -1
  134. package/dist/postgres/plan/server-normalize.d.ts +4 -1
  135. package/dist/postgres/plan/server-normalize.d.ts.map +1 -1
  136. package/dist/postgres.d.ts +1 -1
  137. package/dist/postgres.d.ts.map +1 -1
  138. package/dist/rules/postgres-templates.ts +2 -2
  139. package/dist/rules/sqlch001.ts +26 -4
  140. package/dist/rules/sqlch002.ts +3 -2
  141. package/dist/rules/sqlpg001.ts +7 -4
  142. package/dist/rules/templates.ts +2 -2
  143. package/dist/serializer.d.ts +2 -1
  144. package/dist/serializer.d.ts.map +1 -1
  145. package/dist/sql-files.d.ts +79 -0
  146. package/dist/sql-files.d.ts.map +1 -0
  147. package/dist/token-source.d.ts +88 -0
  148. package/dist/token-source.d.ts.map +1 -0
  149. package/dist/validate.d.ts.map +1 -1
  150. package/package.json +2 -2
  151. package/src/clickhouse/access.test.ts +203 -0
  152. package/src/clickhouse/access.ts +595 -0
  153. package/src/clickhouse/apply/apply.ts +29 -20
  154. package/src/clickhouse/apply/statements.ts +105 -8
  155. package/src/clickhouse/dictionary.test.ts +177 -0
  156. package/src/clickhouse/entities.ts +356 -6
  157. package/src/clickhouse/function.test.ts +91 -0
  158. package/src/clickhouse/http.ts +11 -2
  159. package/src/clickhouse/import/generator.ts +81 -46
  160. package/src/clickhouse/import/import.test.ts +9 -3
  161. package/src/clickhouse/import/ir.ts +6 -2
  162. package/src/clickhouse/import/parser.ts +10 -18
  163. package/src/clickhouse/live/access.e2e.test.ts +218 -0
  164. package/src/clickhouse/live/bind.ts +17 -6
  165. package/src/clickhouse/live/catalog.ts +106 -10
  166. package/src/clickhouse/live/describe-resources.ts +23 -10
  167. package/src/clickhouse/live/dictionaries.e2e.test.ts +186 -0
  168. package/src/clickhouse/live/functions.e2e.test.ts +155 -0
  169. package/src/clickhouse/live/live.test.ts +2 -2
  170. package/src/clickhouse/parser.ts +136 -4
  171. package/src/clickhouse/plan/commands.ts +18 -7
  172. package/src/clickhouse/plan/deep.ts +71 -11
  173. package/src/clickhouse/plan/diff.ts +65 -2
  174. package/src/clickhouse/plan/normalize.ts +200 -6
  175. package/src/clickhouse/plan/report.ts +14 -3
  176. package/src/clickhouse/plan/rules.ts +44 -1
  177. package/src/clickhouse/plan/schema.ts +40 -17
  178. package/src/clickhouse/plan/server-format.ts +1 -1
  179. package/src/clickhouse/rebuild/backfill.ts +5 -25
  180. package/src/clickhouse/rebuild/observe.ts +7 -3
  181. package/src/clickhouse/rebuild/op.ts +9 -0
  182. package/src/clickhouse/rebuild/rebuild.e2e.test.ts +62 -0
  183. package/src/clickhouse/rebuild/rebuild.test.ts +54 -0
  184. package/src/clickhouse/rebuild/steps.ts +136 -1
  185. package/src/clickhouse/testing/fake-server.ts +2 -0
  186. package/src/clickhouse/testing/writable-server.ts +2 -0
  187. package/src/clickhouse/topology.ts +17 -2
  188. package/src/clickhouse.ts +19 -0
  189. package/src/codegen/classifier-page.ts +2 -0
  190. package/src/codegen/registry.ts +10 -0
  191. package/src/config.ts +22 -3
  192. package/src/core/references.ts +5 -3
  193. package/src/core/statements.ts +11 -1
  194. package/src/files/build-root.ts +62 -0
  195. package/src/files/clickhouse.ts +52 -0
  196. package/src/files/common.ts +54 -0
  197. package/src/files/postgres.ts +371 -0
  198. package/src/generated/lexicon-sql.json +50 -0
  199. package/src/import-parser.ts +61 -0
  200. package/src/index.ts +14 -1
  201. package/src/lint/rules/postgres-templates.ts +2 -2
  202. package/src/lint/rules/sqlch001.ts +26 -4
  203. package/src/lint/rules/sqlch002.ts +3 -2
  204. package/src/lint/rules/sqlpg001.ts +7 -4
  205. package/src/lint/rules/templates.ts +2 -2
  206. package/src/lsp/postgres-template.ts +2 -0
  207. package/src/lsp/template.ts +3 -3
  208. package/src/mcp/index.ts +6 -6
  209. package/src/mcp/postgres.ts +1 -1
  210. package/src/migration-statements.test.ts +25 -0
  211. package/src/migration-statements.ts +8 -0
  212. package/src/op/activities/clickhouse-rebuild.ts +1 -0
  213. package/src/op/activities/postgres-migration.ts +27 -2
  214. package/src/op/activity-contracts.ts +2 -0
  215. package/src/plugin.ts +29 -5
  216. package/src/postgres/access/access.e2e.test.ts +191 -0
  217. package/src/postgres/access/access.test.ts +147 -0
  218. package/src/postgres/access/acl.ts +400 -0
  219. package/src/postgres/access/import.ts +64 -0
  220. package/src/postgres/access/live.ts +123 -0
  221. package/src/postgres/apply/apply.ts +117 -4
  222. package/src/postgres/apply/constraint-sequences.e2e.test.ts +179 -0
  223. package/src/postgres/apply/safe-sequences.e2e.test.ts +176 -0
  224. package/src/postgres/apply/statements.test.ts +138 -6
  225. package/src/postgres/apply/statements.ts +210 -18
  226. package/src/postgres/entities.ts +287 -11
  227. package/src/postgres/entity-types.ts +4 -0
  228. package/src/postgres/import/generator.ts +129 -46
  229. package/src/postgres/import/live-export.ts +12 -2
  230. package/src/postgres/live/bind.ts +20 -6
  231. package/src/postgres/live/catalog.ts +79 -3
  232. package/src/postgres/live/client.ts +10 -2
  233. package/src/postgres/live/describe-resources.ts +27 -3
  234. package/src/postgres/migrate/migrate.test.ts +24 -1
  235. package/src/postgres/migrate/op.ts +80 -35
  236. package/src/postgres/migrate/outer.e2e.test.ts +163 -0
  237. package/src/postgres/parser.ts +247 -8
  238. package/src/postgres/plan/commands.ts +60 -7
  239. package/src/postgres/plan/deep.ts +24 -3
  240. package/src/postgres/plan/diff.ts +71 -6
  241. package/src/postgres/plan/normalize.ts +115 -2
  242. package/src/postgres/plan/plan.test.ts +2 -2
  243. package/src/postgres/plan/rules.ts +49 -5
  244. package/src/postgres/plan/schema.ts +4 -2
  245. package/src/postgres/plan/server-normalize.ts +29 -1
  246. package/src/postgres.ts +11 -0
  247. package/src/serializer.ts +14 -9
  248. package/src/skill-defs.ts +1 -1
  249. package/src/skills/chant-sql-postgres-migration.md +3 -1
  250. package/src/skills/chant-sql-postgres-plan.md +3 -3
  251. package/src/skills/chant-sql-postgres.md +9 -2
  252. package/src/skills/chant-sql-rebuild.md +3 -2
  253. package/src/skills/chant-sql.md +7 -1
  254. package/src/sql-files.test.ts +188 -0
  255. package/src/sql-files.ts +218 -0
  256. package/src/token-source.e2e.test.ts +142 -0
  257. package/src/token-source.test.ts +119 -0
  258. package/src/token-source.ts +189 -0
  259. package/src/validate.ts +10 -0
@@ -1,7 +1,8 @@
1
1
  /**
2
2
  * A hand-written recursive-descent parser for the ClickHouse statements the
3
- * dialect declares: `CREATE DATABASE`, `CREATE TABLE`, `CREATE VIEW` and
4
- * `CREATE MATERIALIZED VIEW` (chant #3196).
3
+ * dialect declares: `CREATE DATABASE`, `CREATE TABLE`, `CREATE VIEW`,
4
+ * `CREATE MATERIALIZED VIEW` (chant #3196), `CREATE DICTIONARY` and
5
+ * `CREATE FUNCTION` (#3682).
5
6
  *
6
7
  * Statement structure is parsed; expressions are not. An expression (a
7
8
  * default, a codec, a sort key, a TTL, a view's SELECT) is kept as a span of
@@ -114,7 +115,57 @@ export interface ViewNode extends StorageNode {
114
115
  comment?: Span;
115
116
  }
116
117
 
117
- export type CreateNode = DatabaseNode | TableNode | ViewNode;
118
+ /** One attribute of a dictionary: `name Type [DEFAULT expr] [EXPRESSION expr] [HIERARCHICAL] [INJECTIVE] [IS_OBJECT_ID]`. */
119
+ export interface DictionaryAttributeNode {
120
+ name: string;
121
+ nameSpan: Span;
122
+ type: Span;
123
+ default?: Span;
124
+ expression?: Span;
125
+ /** `HIERARCHICAL`, `BIDIRECTIONAL`, `INJECTIVE`, `IS_OBJECT_ID`, upper case, as written. */
126
+ flags: string[];
127
+ }
128
+
129
+ export interface DictionaryNode {
130
+ statement: "dictionary";
131
+ orReplace: boolean;
132
+ ifNotExists: boolean;
133
+ name: Span;
134
+ onCluster?: Span;
135
+ attributes: DictionaryAttributeNode[];
136
+ /** The key columns, as written after `PRIMARY KEY`. */
137
+ primaryKey?: Span;
138
+ /** What is inside `SOURCE(...)`: `CLICKHOUSE(TABLE 'rates')`. */
139
+ source?: Span;
140
+ /** What is inside `LAYOUT(...)`: `HASHED()`. */
141
+ layout?: Span;
142
+ /** What is inside `LIFETIME(...)`: `300`, `MIN 0 MAX 300`. */
143
+ lifetime?: Span;
144
+ /** What is inside `RANGE(...)`: `MIN start MAX end`. */
145
+ range?: Span;
146
+ /** What is inside `SETTINGS(...)`. */
147
+ settings?: Span;
148
+ comment?: Span;
149
+ }
150
+
151
+ /** A SQL user-defined function: `CREATE FUNCTION name AS (x, y) -> expr`. It lives outside any database. */
152
+ export interface FunctionNode {
153
+ statement: "function";
154
+ orReplace: boolean;
155
+ ifNotExists: boolean;
156
+ name: Span;
157
+ onCluster?: Span;
158
+ /** The parameter names, each as written. */
159
+ params: Array<{ name: string; span: Span }>;
160
+ /** The whole lambda after `AS`: `(x, y) -> expr`. */
161
+ lambda: Span;
162
+ /** The expression after `->`. */
163
+ body: Span;
164
+ /** Never set: a function has no comment. Here so every node can be asked. */
165
+ comment?: undefined;
166
+ }
167
+
168
+ export type CreateNode = DatabaseNode | TableNode | ViewNode | DictionaryNode | FunctionNode;
118
169
 
119
170
  /** A token that can stand where a name goes. */
120
171
  const isNameToken = (t: Token | undefined): boolean =>
@@ -350,9 +401,11 @@ class Parser extends SqlCursor {
350
401
  }
351
402
  if (this.accept("DATABASE")) return this.database();
352
403
  if (this.accept("TABLE")) return this.table(orReplace);
404
+ if (this.accept("DICTIONARY")) return this.dictionary(orReplace);
405
+ if (this.accept("FUNCTION")) return this.function(orReplace);
353
406
  const materialized = this.accept("MATERIALIZED");
354
407
  if (this.accept("VIEW")) return this.view(orReplace, materialized);
355
- return this.fail(materialized ? "expected VIEW" : "expected DATABASE, TABLE, VIEW or MATERIALIZED VIEW");
408
+ return this.fail(materialized ? "expected VIEW" : "expected DATABASE, TABLE, VIEW, MATERIALIZED VIEW, DICTIONARY or FUNCTION");
356
409
  }
357
410
 
358
411
  private ifNotExists(): boolean {
@@ -415,6 +468,85 @@ class Parser extends SqlCursor {
415
468
  return node;
416
469
  }
417
470
 
471
+ private dictionaryAttribute(): DictionaryAttributeNode {
472
+ const { text, span } = this.name("an attribute name");
473
+ const STOP = ["DEFAULT", "EXPRESSION", "HIERARCHICAL", "BIDIRECTIONAL", "INJECTIVE", "IS_OBJECT_ID"];
474
+ const attr: DictionaryAttributeNode = { name: text, nameSpan: span, type: this.typeSpan(), flags: [] };
475
+ for (;;) {
476
+ if (this.accept("DEFAULT")) attr.default = this.expr(STOP);
477
+ else if (this.accept("EXPRESSION")) attr.expression = this.expr(STOP);
478
+ else if (kw(this.peek(), "HIERARCHICAL", "BIDIRECTIONAL", "INJECTIVE", "IS_OBJECT_ID")) attr.flags.push(this.next().text.toUpperCase());
479
+ else return attr;
480
+ }
481
+ }
482
+
483
+ private dictionary(orReplace: boolean): DictionaryNode {
484
+ const ifNotExists = this.ifNotExists();
485
+ const name = this.qualifiedName();
486
+ this.uuid();
487
+ const node: DictionaryNode = { statement: "dictionary", orReplace, ifNotExists, name, onCluster: this.onCluster(), attributes: [] };
488
+ this.expectPunct("(");
489
+ for (;;) {
490
+ node.attributes.push(this.dictionaryAttribute());
491
+ if (this.acceptPunct(",")) {
492
+ if (this.acceptPunct(")")) break;
493
+ continue;
494
+ }
495
+ this.expectPunct(")");
496
+ break;
497
+ }
498
+ const CLAUSES = ["PRIMARY", "SOURCE", "LAYOUT", "LIFETIME", "RANGE", "SETTINGS", "COMMENT"];
499
+ for (;;) {
500
+ if (kw(this.peek(), "PRIMARY") && kw(this.peek(1), "KEY")) {
501
+ this.p += 2;
502
+ node.primaryKey = this.expr(CLAUSES, false);
503
+ } else if (this.accept("SOURCE")) node.source = this.parenthesized();
504
+ else if (this.accept("LAYOUT")) node.layout = this.parenthesized();
505
+ else if (this.accept("LIFETIME")) node.lifetime = this.parenthesized();
506
+ else if (this.accept("RANGE")) node.range = this.parenthesized();
507
+ else if (this.accept("SETTINGS")) node.settings = this.parenthesized();
508
+ else if (this.accept("COMMENT")) node.comment = this.expr();
509
+ else break;
510
+ }
511
+ if (!node.primaryKey) this.fail("expected PRIMARY KEY: a dictionary needs its key columns");
512
+ if (!node.source) this.fail("expected SOURCE(...): a dictionary needs a source");
513
+ if (!node.layout) this.fail("expected LAYOUT(...): a dictionary needs a layout");
514
+ this.finish("dictionary definition");
515
+ return node;
516
+ }
517
+
518
+ private function(orReplace: boolean): FunctionNode {
519
+ const ifNotExists = this.ifNotExists();
520
+ const name = this.qualifiedName();
521
+ if (name.to - name.from > 1 && this.tokens.slice(name.from, name.to).some((t) => t.kind === "punct" && t.text === ".")) {
522
+ this.fail("a function belongs to no database: name it without one");
523
+ }
524
+ const onCluster = this.onCluster();
525
+ this.expect("AS");
526
+ const from = this.idx();
527
+ const params: FunctionNode["params"] = [];
528
+ const param = () => {
529
+ const p = this.name("a parameter name");
530
+ params.push({ name: p.text, span: p.span });
531
+ };
532
+ if (this.acceptPunct("(")) {
533
+ if (!this.acceptPunct(")")) {
534
+ for (;;) {
535
+ param();
536
+ if (this.acceptPunct(",")) continue;
537
+ this.expectPunct(")");
538
+ break;
539
+ }
540
+ }
541
+ } else param();
542
+ const arrow = this.next();
543
+ if (arrow?.kind !== "op" || arrow.text !== "->") this.fail("expected -> and the function's expression", arrow);
544
+ const body = this.expr([], false);
545
+ const lambda = this.span(from, body.refs);
546
+ this.finish("function definition");
547
+ return { statement: "function", orReplace, ifNotExists, name, onCluster, params, lambda, body };
548
+ }
549
+
418
550
  private view(orReplace: boolean, materialized: boolean): ViewNode {
419
551
  const ifNotExists = this.ifNotExists();
420
552
  const name = this.qualifiedName();
@@ -11,7 +11,9 @@
11
11
  * request's base and head, and needs no server. `plan` compares a build with
12
12
  * the server `sql.profiles.<env>` binds and asks that server's formatter about
13
13
  * expressions the rules leave different. Both exit 2 when a change needs a
14
- * rebuild, which a plan refuses to make in place, and 0 otherwise; for each
14
+ * rebuild, which a plan refuses to make in place, and `plan` also when the
15
+ * declared databases hold an object whose definition it cannot read (#3653),
16
+ * which it names; 0 otherwise. For each
15
17
  * refused table they name the `ClickHouseRebuildOp` to run instead
16
18
  * (`rebuildOps` in `--json`, `./rebuild-handoff.ts`).
17
19
  *
@@ -28,10 +30,12 @@
28
30
  */
29
31
 
30
32
  import type { CommandGroup, CommandGroupContext } from "@intentius/chant/cli/command-group";
31
- import { diffSchemas, type SchemaDiff } from "./diff";
33
+ import { diffSchemas, type SchemaDiff, type UnreadableEntry } from "./diff";
32
34
  import { renderDiff } from "./report";
33
35
  import { keyedByQualifiedName, schemaFromBuildFile, schemaFromServer } from "./schema";
34
36
  import { dropFormattingOnly } from "./server-format";
37
+ import { scopeOf } from "./normalize";
38
+ import { declaredAccess } from "../live/catalog";
35
39
  import { bindClickHouse } from "../live/bind";
36
40
  import { rebuildOpSuggestions } from "./rebuild-handoff";
37
41
  import { readFileSync } from "node:fs";
@@ -86,7 +90,7 @@ async function runDiffStatements(before: string, after: string, json: boolean, t
86
90
 
87
91
  function emit(diff: SchemaDiff, json: boolean, title: string): number {
88
92
  console.log(json ? JSON.stringify(diff, null, 2) : renderDiff(diff, { title }));
89
- return diff.rebuilds.length > 0 ? 2 : 0;
93
+ return diff.rebuilds.length > 0 || (diff.unreadable?.length ?? 0) > 0 ? 2 : 0;
90
94
  }
91
95
 
92
96
  export async function runDiff(ctx: CommandGroupContext): Promise<number> {
@@ -117,8 +121,9 @@ export async function runDiff(ctx: CommandGroupContext): Promise<number> {
117
121
  export async function planAgainstServer(environment: string, buildFile: string, options: Parameters<typeof bindClickHouse>[0] = {}): Promise<SchemaDiff> {
118
122
  const target = await bindClickHouse({ ...options, environment });
119
123
  const declared = keyedByQualifiedName(schemaFromBuildFile(buildFile, target.defaultDatabase, target.topology));
120
- const databases = new Set(declared.map((o) => o.canonical.database ?? o.canonical.name));
121
- const live = await schemaFromServer(target, databases);
124
+ const databases = new Set(declared.map((o) => scopeOf(o.canonical)));
125
+ const unreadable: UnreadableEntry[] = [];
126
+ const live = await schemaFromServer(target, databases, unreadable, declaredAccess(declared.map((o) => o.canonical)));
122
127
  const diff = diffSchemas(live, declared);
123
128
  const changes = await dropFormattingOnly(target.endpoint, diff.changes);
124
129
  const rebuildOps = rebuildOpSuggestions(
@@ -126,9 +131,15 @@ export async function planAgainstServer(environment: string, buildFile: string,
126
131
  new Map(declared.map((o) => [o.key, o.canonical])),
127
132
  environment,
128
133
  );
129
- const label = new Map(declared.map((o) => [o.key, `${o.exportName} (${o.key})`]));
134
+ const label = new Map(declared.map((o) => [o.key, o.exportName === o.key ? o.key : `${o.exportName} (${o.key})`]));
130
135
  const relabel = changes.map((c) => ({ ...c, object: label.get(c.object) ?? c.object }));
131
- return { changes: relabel, rebuilds: relabel.filter((c) => c.class === "rebuild"), hints: diff.hints, ...(rebuildOps.length > 0 ? { rebuildOps } : {}) };
136
+ return {
137
+ changes: relabel,
138
+ rebuilds: relabel.filter((c) => c.class === "rebuild"),
139
+ hints: diff.hints,
140
+ ...(rebuildOps.length > 0 ? { rebuildOps } : {}),
141
+ ...(unreadable.length > 0 ? { unreadable } : {}),
142
+ };
132
143
  }
133
144
 
134
145
  export async function runPlan(ctx: CommandGroupContext): Promise<number> {
@@ -22,12 +22,12 @@
22
22
  import { deepObservation, type DeepObservationResult, type DeepResourceObservation, type DeepNormalizationHooks } from "@intentius/chant/deep-observation";
23
23
  import type { UnobservedEntity } from "@intentius/chant/lexicon";
24
24
  import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "../live/bind";
25
- import { readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
25
+ import { accessOf, readLiveAccess, readLiveSchema, SYSTEM_DATABASES, type LiveObject } from "../live/catalog";
26
26
  import { clickhouseQuery } from "../http";
27
27
  import { sqlString } from "../apply/statements";
28
28
  import { stripMarkerFromStatement } from "../ownership";
29
- import { database, table, view, CLICKHOUSE_ENTITY_TYPES, type ColumnDef } from "../entities";
30
- import { canonicalObject, type CanonicalColumn, type CanonicalObject } from "./normalize";
29
+ import { database, dictionary, func, policy, role, table, user, view, CLICKHOUSE_ENTITY_TYPES, type ColumnDef } from "../entities";
30
+ import { canonicalObject, objectKey, type CanonicalColumn, type CanonicalObject } from "./normalize";
31
31
  import { renderFor } from "../topology";
32
32
 
33
33
  type Props = Record<string, unknown>;
@@ -40,7 +40,22 @@ export const sqlDeepNormalizationHooks: DeepNormalizationHooks = {
40
40
  };
41
41
 
42
42
  function liveProps(o: LiveObject): Props {
43
- const tag = o.type === CLICKHOUSE_ENTITY_TYPES.database ? database : o.type === CLICKHOUSE_ENTITY_TYPES.table ? table : view;
43
+ const tag =
44
+ o.type === CLICKHOUSE_ENTITY_TYPES.database
45
+ ? database
46
+ : o.type === CLICKHOUSE_ENTITY_TYPES.table
47
+ ? table
48
+ : o.type === CLICKHOUSE_ENTITY_TYPES.dictionary
49
+ ? dictionary
50
+ : o.type === CLICKHOUSE_ENTITY_TYPES.function
51
+ ? func
52
+ : o.type === CLICKHOUSE_ENTITY_TYPES.user
53
+ ? user
54
+ : o.type === CLICKHOUSE_ENTITY_TYPES.role
55
+ ? role
56
+ : o.type === CLICKHOUSE_ENTITY_TYPES.rowPolicy
57
+ ? policy
58
+ : view;
44
59
  const strings = Object.assign([o.statement], { raw: [o.statement] }) as unknown as TemplateStringsArray;
45
60
  return { ...(tag(strings).props as unknown as Props) };
46
61
  }
@@ -62,8 +77,20 @@ export function inDeclaredVocabulary(declared: Props, live: Props, d: CanonicalO
62
77
  adopt("name", d.name === l.name);
63
78
  adopt("database", d.database === l.database);
64
79
  adopt("engine", same("engine"));
65
- for (const k of ["orderBy", "primaryKey", "partitionBy", "sampleBy", "ttl", "comment", "select", "refresh"] as const) adopt(k, same(k));
80
+ for (const k of ["orderBy", "primaryKey", "partitionBy", "sampleBy", "ttl", "comment", "select", "refresh", "dataSource", "layout", "lifetime", "range"] as const) adopt(k, same(k));
66
81
  adopt("settings", same("settings"));
82
+ // A user's, role's or row policy's fields, compared as `../access.ts` reads them.
83
+ if (d.kind === "user" || d.kind === "role" || d.kind === "rowPolicy") {
84
+ for (const k of ["identified", "host", "validUntil", "defaultRole", "defaultDatabase", "grantees", "settings", "as", "for", "using", "to"]) {
85
+ adopt(k, d.access?.[k] === l.access?.[k]);
86
+ }
87
+ adopt("table", d.table === l.table);
88
+ // A user's authentication is the environment's unless the declaration says it.
89
+ if (d.kind === "user" && declared.identified === undefined) delete out.identified;
90
+ }
91
+ // A function's parameters and expression are one lambda: the same lambda is the declaration's words.
92
+ adopt("params", d.kind === "function" && same("lambda"));
93
+ adopt("body", d.kind === "function" && same("lambda"));
67
94
  adopt("indexes", same("indexes"));
68
95
  adopt("projections", same("projections"));
69
96
  adopt("constraints", same("constraints"));
@@ -72,7 +99,7 @@ export function inDeclaredVocabulary(declared: Props, live: Props, d: CanonicalO
72
99
  if (d.kind === "view" || d.kind === "materializedView") {
73
100
  // The server infers a view's columns; a declaration that lists none is not drift.
74
101
  if (declaredColumns.length === 0) out.columns = [];
75
- } else if (d.kind === "table") {
102
+ } else if (d.kind === "table" || d.kind === "dictionary") {
76
103
  const liveColumns = (live.columns as ColumnDef[] | undefined) ?? [];
77
104
  out.columns = liveColumns.map((c, i) => {
78
105
  const lc = l.columns[i];
@@ -84,6 +111,22 @@ export function inDeclaredVocabulary(declared: Props, live: Props, d: CanonicalO
84
111
  return out;
85
112
  }
86
113
 
114
+ /** The entity types that belong to no database. */
115
+ const GLOBAL_TYPES: ReadonlySet<string> = new Set([CLICKHOUSE_ENTITY_TYPES.database, CLICKHOUSE_ENTITY_TYPES.function, CLICKHOUSE_ENTITY_TYPES.user, CLICKHOUSE_ENTITY_TYPES.role]);
116
+
117
+ /** Whether the server's formatter prints two expressions the same. A formatter that cannot parse them says no. */
118
+ async function sameExpression(target: ClickHouseTarget, a: string, b: string): Promise<boolean> {
119
+ try {
120
+ const [row] = await clickhouseQuery<{ a: string; b: string }>(
121
+ target.endpoint,
122
+ `SELECT formatQuerySingleLine(${sqlString(`SELECT ${a}`)}) AS a, formatQuerySingleLine(${sqlString(`SELECT ${b}`)}) AS b`,
123
+ );
124
+ return row !== undefined && row.a === row.b;
125
+ } catch {
126
+ return false;
127
+ }
128
+ }
129
+
87
130
  /** Every server's copy of each table and view on a cluster (#3664), as `<database>.<name>` → host → statement. */
88
131
  interface ClusterCopies {
89
132
  /** The host the profile names, whose copy the rest are compared with. */
@@ -164,7 +207,7 @@ export async function observeResourcesDeep(
164
207
  let live: LiveObject[];
165
208
  try {
166
209
  target = await bindClickHouse(options);
167
- live = await readLiveSchema(target);
210
+ live = [...(await readLiveSchema(target)), ...(await readLiveAccess(target, accessOf(options.entities, target.defaultDatabase)))];
168
211
  } catch (err) {
169
212
  const why = classifyClickHouseFailure(err);
170
213
  for (const name of options.entityNames) {
@@ -172,7 +215,7 @@ export async function observeResourcesDeep(
172
215
  }
173
216
  return deepObservation({}, unobserved);
174
217
  }
175
- const byKey = new Map(live.map((o) => [`${o.database ?? ""}.${o.name}`, o]));
218
+ const byKey = new Map(live.map((o) => [objectKey(o), o]));
176
219
  let copies: ClusterCopies | undefined;
177
220
  try {
178
221
  copies = await clusterCopies(target);
@@ -190,10 +233,20 @@ export async function observeResourcesDeep(
190
233
  unobserved[name] = { type: entity?.entityType ?? "", reason: "unsupported-kind" };
191
234
  continue;
192
235
  }
236
+ if (entity.entityType === CLICKHOUSE_ENTITY_TYPES.grant) {
237
+ unobserved[name] = {
238
+ type: entity.entityType,
239
+ reason: "unsupported-kind",
240
+ detail: "a grant is compared as part of its grantees' grants, which chant sql plan reports (SQLCH273, SQLCH274)",
241
+ };
242
+ continue;
243
+ }
193
244
  const props = entity.props;
194
- const isDb = entity.entityType === CLICKHOUSE_ENTITY_TYPES.database;
245
+ const isDb = GLOBAL_TYPES.has(entity.entityType);
195
246
  const db = isDb ? undefined : typeof props.database === "string" ? props.database : target.defaultDatabase;
196
- const o = byKey.get(`${db ?? ""}.${String(props.name)}`);
247
+ const o = byKey.get(
248
+ objectKey({ type: entity.entityType, ...(db !== undefined ? { database: db } : {}), name: String(props.name), ...(typeof props.table === "string" ? { table: props.table } : {}) }),
249
+ );
197
250
  if (!o) continue;
198
251
  try {
199
252
  const d = canonicalObject(renderFor(String(props.ddl), target.topology), target.defaultDatabase);
@@ -210,7 +263,14 @@ export async function observeResourcesDeep(
210
263
  };
211
264
  continue;
212
265
  }
213
- const l = canonicalObject(o.statement, target.defaultDatabase);
266
+ let l = canonicalObject(o.statement, target.defaultDatabase);
267
+ // The server prints a function's expression its own way (`k*x + b` as `((k * x) + b)`): its formatter says whether the two are one.
268
+ if (d.kind === "function" && d.lambda !== l.lambda && (await sameExpression(target, d.lambda ?? "", l.lambda ?? ""))) l = { ...l, lambda: d.lambda! };
269
+ // A row policy's condition, the same way.
270
+ const using = d.access?.using;
271
+ if (d.kind === "rowPolicy" && using !== undefined && using !== l.access?.using && (await sameExpression(target, using, l.access?.using ?? ""))) {
272
+ l = { ...l, access: { ...l.access, using } };
273
+ }
214
274
  resources[name] = {
215
275
  type: o.type,
216
276
  physicalId,
@@ -17,7 +17,7 @@
17
17
  */
18
18
 
19
19
  import { CLASSIFIER_RULES, type ChangeClass, type ClassifierRuleId } from "./rules";
20
- import type { CanonicalObject } from "./normalize";
20
+ import type { CanonicalColumn, CanonicalObject } from "./normalize";
21
21
  import type { RebuildOpSuggestion } from "./rebuild-handoff";
22
22
  import { MERGE_TREE_SETTINGS } from "../../generated/clickhouse";
23
23
  import { classifiedChange, type ChangeSet, type ClassifiedChange } from "../../core/classifier";
@@ -32,6 +32,19 @@ export interface SchemaDiff extends ChangeSet<Change> {
32
32
  rebuilds: Change[];
33
33
  /** For each table refused as a rebuild, the rebuild migration Op to run instead (`./rebuild-handoff.ts`). */
34
34
  rebuildOps?: RebuildOpSuggestion[];
35
+ /**
36
+ * Objects on the server, in the declared databases, whose definition chant
37
+ * cannot read (#3653): a plan names each one and fails rather than leave
38
+ * it out.
39
+ */
40
+ unreadable?: UnreadableEntry[];
41
+ }
42
+
43
+ /** An object a plan could not read: `database.name`, its type and why. */
44
+ export interface UnreadableEntry {
45
+ object: string;
46
+ type: string;
47
+ reason: string;
35
48
  }
36
49
 
37
50
  const qualified = (o: CanonicalObject) => (o.database ? `${o.database}.${o.name}` : o.name);
@@ -133,6 +146,38 @@ function diffObject(key: string, before: CanonicalObject, after: CanonicalObject
133
146
  return;
134
147
  }
135
148
 
149
+ if (before.kind === "user" || before.kind === "role" || before.kind === "rowPolicy") {
150
+ const rule = before.kind === "user" ? "SQLCH271" : before.kind === "role" ? "SQLCH270" : "SQLCH272";
151
+ const b = before.access ?? {};
152
+ const a = after.access ?? {};
153
+ const order = ["identified", "host", "validUntil", "defaultRole", "defaultDatabase", "grantees", "settings", "as", "for", "using", "to"];
154
+ const fields = [...new Set([...Object.keys(b), ...Object.keys(a)])].sort((x, y) => order.indexOf(x) - order.indexOf(y));
155
+ for (const field of fields) {
156
+ // A user's authentication is compared only when the declaration says it.
157
+ if (before.kind === "user" && field === "identified" && a.identified === undefined) continue;
158
+ if (b[field] !== a[field]) out.push(change(key, field, rule, b[field], a[field]));
159
+ }
160
+ return;
161
+ }
162
+
163
+ if (before.kind === "grants") {
164
+ const b = before.access ?? {};
165
+ const a = after.access ?? {};
166
+ for (const atom of Object.keys(b)) if (!(atom in a)) out.push(change(key, `grants.${atom}`, "SQLCH274", atom, undefined));
167
+ for (const atom of Object.keys(a)) if (!(atom in b)) out.push(change(key, `grants.${atom}`, "SQLCH273", undefined, atom));
168
+ return;
169
+ }
170
+
171
+ if (before.kind === "function") {
172
+ if (before.lambda !== after.lambda) out.push(change(key, "lambda", "SQLCH260", before.lambda, after.lambda));
173
+ return;
174
+ }
175
+
176
+ if (before.kind === "dictionary") {
177
+ diffDictionary(key, before, after, out);
178
+ return;
179
+ }
180
+
136
181
  if (before.kind === "view") {
137
182
  if (before.select !== after.select) out.push(change(key, "select", "SQLCH240", before.select, after.select));
138
183
  return;
@@ -180,6 +225,23 @@ function diffObject(key: string, before: CanonicalObject, after: CanonicalObject
180
225
  diffMap(key, "constraints", before.constraints, after.constraints, "SQLCH215", out);
181
226
  }
182
227
 
228
+ /** A dictionary: every change but its comment replaces it (SQLCH245). */
229
+ function diffDictionary(key: string, before: CanonicalObject, after: CanonicalObject, out: Change[]): void {
230
+ const attribute = (c: CanonicalColumn) =>
231
+ [c.type, c.defaultExpr !== undefined ? `DEFAULT ${c.defaultExpr}` : "", c.expression !== undefined ? `EXPRESSION ${c.expression}` : "", c.flags ?? ""].filter(Boolean).join(" ");
232
+ const b = new Map(before.columns.map((c) => [c.name, attribute(c)]));
233
+ const a = new Map(after.columns.map((c) => [c.name, attribute(c)]));
234
+ for (const name of new Set([...b.keys(), ...a.keys()])) {
235
+ if (b.get(name) !== a.get(name)) out.push(change(key, `columns.${name}`, "SQLCH245", b.get(name), a.get(name)));
236
+ }
237
+ const order = (o: CanonicalObject) => o.columns.map((c) => c.name).join(", ");
238
+ if (order(before) !== order(after) && [...a.keys()].every((n) => b.has(n)) && a.size === b.size) out.push(change(key, "columns.order", "SQLCH245", order(before), order(after)));
239
+ for (const f of ["primaryKey", "dataSource", "layout", "lifetime", "range"] as const) {
240
+ if (before[f] !== after[f]) out.push(change(key, f, "SQLCH245", before[f], after[f]));
241
+ }
242
+ for (const name of changedEntries(before.settings, after.settings)) out.push(change(key, `settings.${name}`, "SQLCH245", before.settings[name], after.settings[name]));
243
+ }
244
+
183
245
  /**
184
246
  * Diff two schemas. `before` and `after` are matched by key; an `after`
185
247
  * object whose key is new but whose `previously` names a `before` object
@@ -190,7 +252,8 @@ export function diffSchemas(before: readonly SchemaObject[], after: readonly Sch
190
252
  const hints: string[] = [];
191
253
  const matches = matchByIdentity(before, after, {
192
254
  qualified,
193
- previously: (o) => o.previously,
255
+ // A function, a user, a role, a row policy and a grantee's grants are not renamed: one under a new name is a create.
256
+ previously: (o) => (o.kind === "function" || o.kind === "user" || o.kind === "role" || o.kind === "rowPolicy" || o.kind === "grants" ? undefined : o.previously),
194
257
  previousNames: (o, prev) => [prev, `${o.database}.${prev}`],
195
258
  });
196
259
  for (const m of matches) {