@intentius/chant-lexicon-sql 0.112.0 → 0.113.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/clickhouse/apply/apply.d.ts.map +1 -1
  2. package/dist/clickhouse/apply/statements.d.ts +77 -1
  3. package/dist/clickhouse/apply/statements.d.ts.map +1 -1
  4. package/dist/clickhouse/plan/commands.d.ts +6 -1
  5. package/dist/clickhouse/plan/commands.d.ts.map +1 -1
  6. package/dist/clickhouse/topology.d.ts +100 -0
  7. package/dist/clickhouse/topology.d.ts.map +1 -0
  8. package/dist/codegen/classifier-page.d.ts.map +1 -1
  9. package/dist/codegen/postgres-classifier-page.d.ts.map +1 -1
  10. package/dist/core/statements.d.ts +104 -0
  11. package/dist/core/statements.d.ts.map +1 -0
  12. package/dist/index.d.ts +1 -0
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/integrity.json +2 -2
  15. package/dist/manifest.json +1 -1
  16. package/dist/migration-statements.d.ts +55 -0
  17. package/dist/migration-statements.d.ts.map +1 -0
  18. package/dist/postgres/apply/apply.d.ts.map +1 -1
  19. package/dist/postgres/apply/statements.d.ts +91 -1
  20. package/dist/postgres/apply/statements.d.ts.map +1 -1
  21. package/package.json +2 -2
  22. package/src/clickhouse/apply/apply.ts +25 -53
  23. package/src/clickhouse/apply/statements.ts +149 -31
  24. package/src/clickhouse/plan/commands.ts +30 -6
  25. package/src/clickhouse/topology.e2e.test.ts +100 -0
  26. package/src/clickhouse/topology.test.ts +276 -0
  27. package/src/clickhouse/topology.ts +451 -0
  28. package/src/codegen/classifier-page.ts +17 -0
  29. package/src/codegen/postgres-classifier-page.ts +17 -0
  30. package/src/core/statements.ts +127 -0
  31. package/src/index.ts +15 -0
  32. package/src/migration-statements.test.ts +265 -0
  33. package/src/migration-statements.ts +252 -0
  34. package/src/op/__snapshots__/watch-pipeline.test.ts.snap +93 -0
  35. package/src/op/watch-drift.e2e.test.ts +302 -0
  36. package/src/op/watch-pipeline.test.ts +113 -0
  37. package/src/postgres/apply/apply.ts +38 -96
  38. package/src/postgres/apply/statements.ts +145 -2
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@intentius/chant-lexicon-sql",
3
- "version": "0.112.0",
3
+ "version": "0.113.0",
4
4
  "description": "SQL lexicon for chant — database schema as typed declarations, one dialect at a time; ClickHouse first, with types read from a pinned server's catalog",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://intentius.io/chant",
@@ -70,7 +70,7 @@
70
70
  "typescript": "^5.9.3"
71
71
  },
72
72
  "peerDependencies": {
73
- "@intentius/chant": "^0.112.0",
73
+ "@intentius/chant": "^0.113.0",
74
74
  "typescript": "^5.9.3"
75
75
  }
76
76
  }
@@ -36,7 +36,6 @@ import type { OwnershipMarker } from "@intentius/chant/ownership";
36
36
  import type { ApplyRef } from "@intentius/chant/apply";
37
37
  import {
38
38
  SqlApplyError,
39
- changesByObject,
40
39
  dependencyFailedDetail,
41
40
  missingDependencies,
42
41
  readBuildObjects,
@@ -50,19 +49,7 @@ import { diffSchemas, type Change } from "../plan/diff";
50
49
  import { dropFormattingOnly } from "../plan/server-format";
51
50
  import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "../entities";
52
51
  import { carriesMarker, isChantManaged } from "../ownership";
53
- import {
54
- alterSteps,
55
- commentStatement,
56
- createStatement,
57
- dropStatement,
58
- isDestructiveAlter,
59
- isRebuild,
60
- refusalDetail,
61
- sqlString,
62
- stepsSetComment,
63
- type DeclaredObject,
64
- type Step,
65
- } from "./statements";
52
+ import { planStatements, sqlString, type DeclaredObject, type DropStatement, type Step } from "./statements";
66
53
  import { waitForMutations } from "./mutations";
67
54
 
68
55
  export type { FailedObject } from "../../core/apply";
@@ -117,14 +104,6 @@ export class ClickHouseApplyError extends SqlApplyError<ClickHouseApplyOutcome>
117
104
 
118
105
  const liveKey = (o: LiveObject) => (o.database !== undefined ? `${o.database}.${o.name}` : o.name);
119
106
 
120
- /** Drop order: what reads from a table before the table, a database last. */
121
- const DROP_ORDER: Record<string, number> = {
122
- [CLICKHOUSE_ENTITY_TYPES.materializedView]: 0,
123
- [CLICKHOUSE_ENTITY_TYPES.view]: 1,
124
- [CLICKHOUSE_ENTITY_TYPES.table]: 2,
125
- [CLICKHOUSE_ENTITY_TYPES.database]: 3,
126
- };
127
-
128
107
  /**
129
108
  * The server's objects in the databases the declarations use, as the diff
130
109
  * compares them. The `default` database itself is never part of a schema.
@@ -160,7 +139,6 @@ export async function applyClickHouse(target: ClickHouseTarget, declared: readon
160
139
  const { objects: liveObjects, canonical: live } = await liveSchema(target, declared);
161
140
  const liveByKey = new Map(liveObjects.map((o) => [liveKey(o), o]));
162
141
  const changes = await plannedChanges(target, declared, live);
163
- const byObject = changesByObject(changes);
164
142
 
165
143
  const outcome: ClickHouseApplyOutcome = { target: target.endpoint.url, source: target.source, applied: [], pruned: [], notAttempted: [], failed: [] };
166
144
  /** Export names whose object is on the server. */
@@ -181,35 +159,33 @@ export async function applyClickHouse(target: ClickHouseTarget, declared: readon
181
159
  }
182
160
  };
183
161
 
184
- for (const obj of declared) {
162
+ const plan = planStatements({
163
+ declared,
164
+ changes,
165
+ current: live,
166
+ allowDestructive: opts.prune === true,
167
+ carriesMarker: (key) => carriesMarker(liveByKey.get(key)?.comment, opts.marker),
168
+ ...(opts.marker ? { marker: opts.marker } : {}),
169
+ });
170
+
171
+ for (const entry of plan.objects) {
172
+ const obj = entry.obj;
185
173
  const ref = { kind: obj.type, name: obj.key };
186
- const mine = byObject.get(obj.key) ?? [];
187
- const created = mine.some((c) => c.rule === "SQLCH200");
188
- const renamedFrom = mine.find((c) => c.rule === "SQLCH230" || c.rule === "SQLCH231")?.before;
174
+ const renamedFrom = entry.changes.find((c) => c.rule === "SQLCH230" || c.rule === "SQLCH231")?.before;
189
175
  const liveObject = liveByKey.get(renamedFrom ?? obj.key);
190
- const liveCanonical = live.get(renamedFrom ?? obj.key);
191
176
 
192
- const refused = mine.filter(isRebuild);
193
- if (refused.length > 0) {
177
+ if (entry.verdict === "rebuild") {
194
178
  if (liveObject) onServer.add(obj.exportName);
195
- outcome.notAttempted.push({ ...ref, reason: "unsupported-kind", detail: refusalDetail(refused, obj.key, obj.type) });
179
+ outcome.notAttempted.push({ ...ref, reason: "unsupported-kind", detail: entry.detail });
196
180
  continue;
197
181
  }
198
- const destructive = mine.filter(isDestructiveAlter);
199
- if (destructive.length > 0 && !opts.prune) {
182
+ if (entry.verdict === "withheld") {
200
183
  onServer.add(obj.exportName);
201
- outcome.notAttempted.push({
202
- ...ref,
203
- reason: "filtered",
204
- detail: `drops ${destructive.map((c) => c.field).join(", ")}, which destroys the data in it; an apply that may delete (prune, ApplyOp delete "owned-only" or "gated") makes it`,
205
- });
184
+ outcome.notAttempted.push({ ...ref, reason: "filtered", detail: entry.detail });
206
185
  continue;
207
186
  }
208
-
209
- const restamp = !created && !stepsSetComment(obj, mine) && (mine.some((c) => c.field === "comment") || !carriesMarker(liveObject?.comment, opts.marker));
210
- const steps: Step[] = created
211
- ? [{ sql: createStatement(obj, opts.marker), rewrite: false }]
212
- : [...alterSteps(obj, mine, { ...(liveCanonical ? { live: liveCanonical } : {}), ...(opts.marker ? { marker: opts.marker } : {}) }), ...(restamp ? [{ sql: commentStatement(obj, opts.marker), rewrite: false }] : [])];
187
+ const created = entry.verdict === "create";
188
+ const steps = entry.steps;
213
189
 
214
190
  if (steps.length === 0) {
215
191
  onServer.add(obj.exportName);
@@ -235,26 +211,22 @@ export async function applyClickHouse(target: ClickHouseTarget, declared: readon
235
211
  }
236
212
  }
237
213
 
238
- if (opts.prune) await prune(target, changes, liveByKey, opts, outcome, log);
214
+ if (opts.prune) await prune(target, plan.drops, liveByKey, opts, outcome, log);
239
215
  if (outcome.failed.length > 0) throw new ClickHouseApplyError(outcome);
240
216
  return outcome;
241
217
  }
242
218
 
243
219
  async function prune(
244
220
  target: ClickHouseTarget,
245
- changes: readonly Change[],
221
+ drops: readonly DropStatement[],
246
222
  liveByKey: Map<string, LiveObject>,
247
223
  opts: ClickHouseApplyOptions,
248
224
  outcome: ClickHouseApplyOutcome,
249
225
  log: (line: string) => void,
250
226
  ): Promise<void> {
251
- const orphans = changes
252
- .filter((c) => c.rule === "SQLCH250")
253
- .map((c) => liveByKey.get(c.object))
254
- .filter((o): o is LiveObject => o !== undefined)
255
- .sort((a, b) => (DROP_ORDER[a.type] ?? 9) - (DROP_ORDER[b.type] ?? 9));
256
-
257
- for (const o of orphans) {
227
+ for (const drop of drops) {
228
+ const o = liveByKey.get(drop.key);
229
+ if (!o) continue;
258
230
  const ref = { kind: o.type, name: liveKey(o) };
259
231
  if (!isChantManaged(o.comment)) continue; // not chant's: never touched, never reported
260
232
  if (!opts.marker?.stack) {
@@ -275,7 +247,7 @@ async function prune(
275
247
  continue;
276
248
  }
277
249
  }
278
- const sql = dropStatement(o.type, o.database, o.name);
250
+ const sql = drop.step.sql;
279
251
  opts.signal?.throwIfAborted();
280
252
  log(sql);
281
253
  await clickhouseQuery(target.endpoint, sql);
@@ -20,7 +20,7 @@ import { isTrivia, tokenizeText, type Token } from "../tokens";
20
20
  import { parseCreate, type CreateNode, type Span, type TableNode, type ViewNode } from "../parser";
21
21
  import { canonicalObject, type CanonicalObject } from "../plan/normalize";
22
22
  import type { Change } from "../plan/diff";
23
- import { CLASSIFIER_RULES } from "../plan/rules";
23
+ import { CLASSIFIER_RULES, type ChangeClass, type ClassifierRuleId } from "../plan/rules";
24
24
  import { stampedComment } from "../ownership";
25
25
  import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "../entities";
26
26
 
@@ -53,11 +53,21 @@ export interface DeclaredObject {
53
53
  dependsOn: string[];
54
54
  }
55
55
 
56
- /** One statement to send, and whether it starts a background rewrite to wait on. */
56
+ /** One statement to send, the change it makes, and whether it starts a background rewrite to wait on. */
57
57
  export interface Step {
58
58
  sql: string;
59
59
  /** A mutation follows on the server; wait on `system.mutations` before the next step. */
60
60
  rewrite: boolean;
61
+ /** The classifier rule of the change it makes. */
62
+ rule: ClassifierRuleId;
63
+ /** That rule's class. */
64
+ class: ChangeClass;
65
+ }
66
+
67
+ /** The step for a statement that makes a change under `rule`: a rewrite-class rule waits on its mutation. */
68
+ export function stepFor(sql: string, rule: ClassifierRuleId): Step {
69
+ const cls = CLASSIFIER_RULES[rule].class;
70
+ return { sql, rewrite: cls === "rewrite", rule, class: cls };
61
71
  }
62
72
 
63
73
  interface Parsed {
@@ -259,7 +269,7 @@ function reorderSteps(t: string, tokens: Token[], node: TableNode, live: Canonic
259
269
  if (have === want) continue;
260
270
  current.splice(at, 1);
261
271
  current.splice(want === undefined ? 0 : current.indexOf(want) + 1, 0, name);
262
- steps.push({ sql: `ALTER TABLE ${t} MODIFY COLUMN ${ident(name)} ${columnType(tokens, node, name)} ${want === undefined ? "FIRST" : `AFTER ${ident(want)}`}`, rewrite: false });
272
+ steps.push(stepFor(`ALTER TABLE ${t} MODIFY COLUMN ${ident(name)} ${columnType(tokens, node, name)} ${want === undefined ? "FIRST" : `AFTER ${ident(want)}`}`, "SQLCH209"));
263
273
  }
264
274
  return steps;
265
275
  }
@@ -267,11 +277,11 @@ function reorderSteps(t: string, tokens: Token[], node: TableNode, live: Canonic
267
277
  function tableSteps(obj: DeclaredObject, changes: readonly Change[], live: CanonicalObject | undefined): Step[] {
268
278
  const { tokens, node } = parsed(obj) as { tokens: Token[]; node: TableNode };
269
279
  const t = objectIdent(obj);
270
- const alter = (clause: string, rewrite = false): Step => ({ sql: `ALTER TABLE ${t} ${clause}`, rewrite });
280
+ const alter = (clause: string, rule: ClassifierRuleId): Step => stepFor(`ALTER TABLE ${t} ${clause}`, rule);
271
281
  const steps: Step[] = [];
272
282
  const of = (rule: string) => changes.filter((c) => c.rule === rule);
273
283
 
274
- for (const c of of("SQLCH212")) steps.push(alter(`RENAME COLUMN ${ident(c.before!)} TO ${ident(c.after!)}`));
284
+ for (const c of of("SQLCH212")) steps.push(alter(`RENAME COLUMN ${ident(c.before!)} TO ${ident(c.after!)}`, "SQLCH212"));
275
285
 
276
286
  // Added columns, in declared order, each at its declared place. Appending
277
287
  // new columns to the sorting key has to happen in the same ALTER as adding
@@ -281,10 +291,10 @@ function tableSteps(obj: DeclaredObject, changes: readonly Change[], live: Canon
281
291
  .sort((a, b) => node.columns.findIndex((c) => c.name === a) - node.columns.findIndex((c) => c.name === b));
282
292
  const adds = added.map((name) => `ADD COLUMN ${columnDefinition(tokens, node, name)} ${position(node, name)}`);
283
293
  const sortKey = of("SQLCH216")[0];
284
- if (sortKey) steps.push(alter([...adds, `MODIFY ORDER BY ${text(tokens, node.orderBy)}`].join(", ")));
285
- else for (const a of adds) steps.push(alter(a));
294
+ if (sortKey) steps.push(alter([...adds, `MODIFY ORDER BY ${text(tokens, node.orderBy)}`].join(", "), "SQLCH216"));
295
+ else for (const a of adds) steps.push(alter(a, "SQLCH201"));
286
296
 
287
- for (const c of of("SQLCH202")) steps.push(alter(`DROP COLUMN ${ident(columnField(c.field)!.column)}`));
297
+ for (const c of of("SQLCH202")) steps.push(alter(`DROP COLUMN ${ident(columnField(c.field)!.column)}`, "SQLCH202"));
288
298
 
289
299
  for (const c of changes) {
290
300
  const f = columnField(c.field);
@@ -293,59 +303,56 @@ function tableSteps(obj: DeclaredObject, changes: readonly Change[], live: Canon
293
303
  const name = ident(f.column);
294
304
  switch (c.rule) {
295
305
  case "SQLCH210":
296
- steps.push(alter(`MODIFY COLUMN ${name} ${columnType(tokens, node, f.column)}`, true));
306
+ steps.push(alter(`MODIFY COLUMN ${name} ${columnType(tokens, node, f.column)}`, c.rule));
297
307
  break;
298
308
  case "SQLCH207":
299
- if (col.default) steps.push(alter(`MODIFY COLUMN ${name} ${col.default.kind}${col.default.expr ? ` ${text(tokens, col.default.expr)}` : ""}`));
300
- else steps.push(alter(`MODIFY COLUMN ${name} REMOVE ${(c.before ?? "DEFAULT").split(" ")[0]}`));
309
+ if (col.default) steps.push(alter(`MODIFY COLUMN ${name} ${col.default.kind}${col.default.expr ? ` ${text(tokens, col.default.expr)}` : ""}`, c.rule));
310
+ else steps.push(alter(`MODIFY COLUMN ${name} REMOVE ${(c.before ?? "DEFAULT").split(" ")[0]}`, c.rule));
301
311
  break;
302
312
  case "SQLCH208":
303
- steps.push(alter(col.codec ? `MODIFY COLUMN ${name} CODEC(${text(tokens, col.codec)})` : `MODIFY COLUMN ${name} REMOVE CODEC`));
313
+ steps.push(alter(col.codec ? `MODIFY COLUMN ${name} CODEC(${text(tokens, col.codec)})` : `MODIFY COLUMN ${name} REMOVE CODEC`, c.rule));
304
314
  break;
305
315
  case "SQLCH205":
306
- steps.push(alter(col.ttl ? `MODIFY COLUMN ${name} TTL ${text(tokens, col.ttl)}` : `MODIFY COLUMN ${name} REMOVE TTL`, true));
316
+ steps.push(alter(col.ttl ? `MODIFY COLUMN ${name} TTL ${text(tokens, col.ttl)}` : `MODIFY COLUMN ${name} REMOVE TTL`, c.rule));
307
317
  break;
308
318
  case "SQLCH203":
309
- steps.push(alter(`COMMENT COLUMN ${name} ${sqlString(obj.canonical.columns.find((x) => x.name === f.column)?.comment ?? "")}`));
319
+ steps.push(alter(`COMMENT COLUMN ${name} ${sqlString(obj.canonical.columns.find((x) => x.name === f.column)?.comment ?? "")}`, c.rule));
310
320
  break;
311
321
  }
312
322
  }
313
323
 
314
324
  for (const c of of("SQLCH204")) {
315
325
  const name = c.field.slice("indexes.".length);
316
- if (c.before !== undefined) steps.push(alter(`DROP INDEX ${ident(name)}`));
326
+ if (c.before !== undefined) steps.push(alter(`DROP INDEX ${ident(name)}`, c.rule));
317
327
  const ix = node.indexes.find((x) => x.name === name);
318
328
  if (ix) {
319
329
  steps.push(
320
- alter(`ADD INDEX ${ident(name)} ${text(tokens, ix.expr)} TYPE ${text(tokens, ix.type)}${ix.granularity ? ` GRANULARITY ${text(tokens, ix.granularity)}` : ""}`),
330
+ alter(`ADD INDEX ${ident(name)} ${text(tokens, ix.expr)} TYPE ${text(tokens, ix.type)}${ix.granularity ? ` GRANULARITY ${text(tokens, ix.granularity)}` : ""}`, c.rule),
321
331
  );
322
332
  }
323
333
  }
324
334
  for (const c of of("SQLCH214")) {
325
335
  const name = c.field.slice("projections.".length);
326
- if (c.before !== undefined) steps.push(alter(`DROP PROJECTION ${ident(name)}`));
336
+ if (c.before !== undefined) steps.push(alter(`DROP PROJECTION ${ident(name)}`, c.rule));
327
337
  const p = node.projections.find((x) => x.name === name);
328
- if (p) steps.push(alter(`ADD PROJECTION ${ident(name)} (${text(tokens, p.body)})`));
338
+ if (p) steps.push(alter(`ADD PROJECTION ${ident(name)} (${text(tokens, p.body)})`, c.rule));
329
339
  }
330
340
  for (const c of of("SQLCH215")) {
331
341
  const name = c.field.slice("constraints.".length);
332
- if (c.before !== undefined) steps.push(alter(`DROP CONSTRAINT ${ident(name)}`));
342
+ if (c.before !== undefined) steps.push(alter(`DROP CONSTRAINT ${ident(name)}`, c.rule));
333
343
  const k = node.constraints.find((x) => x.name === name);
334
- if (k) steps.push(alter(`ADD CONSTRAINT ${ident(name)} ${k.kind} ${text(tokens, k.expr)}`));
344
+ if (k) steps.push(alter(`ADD CONSTRAINT ${ident(name)} ${k.kind} ${text(tokens, k.expr)}`, c.rule));
335
345
  }
336
346
 
337
347
  for (const c of of("SQLCH205").filter((x) => x.field === "ttl")) {
338
- steps.push(alter(node.ttl ? `MODIFY TTL ${text(tokens, node.ttl)}` : "REMOVE TTL", true));
339
- }
340
- for (const c of of("SQLCH217")) {
341
- void c;
342
- steps.push(alter(node.sampleBy ? `MODIFY SAMPLE BY ${text(tokens, node.sampleBy)}` : "REMOVE SAMPLE BY"));
348
+ steps.push(alter(node.ttl ? `MODIFY TTL ${text(tokens, node.ttl)}` : "REMOVE TTL", c.rule));
343
349
  }
350
+ for (const c of of("SQLCH217")) steps.push(alter(node.sampleBy ? `MODIFY SAMPLE BY ${text(tokens, node.sampleBy)}` : "REMOVE SAMPLE BY", c.rule));
344
351
  for (const c of of("SQLCH206")) {
345
352
  const key = c.field.slice("settings.".length);
346
353
  const s = node.settings?.find((x) => x.key === key);
347
354
  // A setting declared at its default compares as absent: set it as written.
348
- steps.push(alter(s ? `MODIFY SETTING ${key} = ${text(tokens, s.value)}` : `RESET SETTING ${key}`));
355
+ steps.push(alter(s ? `MODIFY SETTING ${key} = ${text(tokens, s.value)}` : `RESET SETTING ${key}`, c.rule));
349
356
  }
350
357
 
351
358
  if (of("SQLCH209").length > 0) steps.push(...reorderSteps(t, tokens, node, live, changes));
@@ -357,11 +364,11 @@ function viewSteps(obj: DeclaredObject, changes: readonly Change[], marker: Owne
357
364
  const t = objectIdent(obj);
358
365
  const steps: Step[] = [];
359
366
  if (!node.materialized) {
360
- if (changes.some((c) => c.rule === "SQLCH240")) steps.push({ sql: createStatement(obj, marker, { orReplace: true }), rewrite: false });
367
+ if (changes.some((c) => c.rule === "SQLCH240")) steps.push(stepFor(createStatement(obj, marker, { orReplace: true }), "SQLCH240"));
361
368
  return steps;
362
369
  }
363
- if (changes.some((c) => c.rule === "SQLCH241")) steps.push({ sql: `ALTER TABLE ${t} MODIFY QUERY ${text(tokens, node.select)}`, rewrite: false });
364
- if (changes.some((c) => c.rule === "SQLCH244") && node.refresh) steps.push({ sql: `ALTER TABLE ${t} MODIFY REFRESH ${text(tokens, node.refresh)}`, rewrite: false });
370
+ if (changes.some((c) => c.rule === "SQLCH241")) steps.push(stepFor(`ALTER TABLE ${t} MODIFY QUERY ${text(tokens, node.select)}`, "SQLCH241"));
371
+ if (changes.some((c) => c.rule === "SQLCH244") && node.refresh) steps.push(stepFor(`ALTER TABLE ${t} MODIFY REFRESH ${text(tokens, node.refresh)}`, "SQLCH244"));
365
372
  return steps;
366
373
  }
367
374
 
@@ -376,11 +383,11 @@ function viewSteps(obj: DeclaredObject, changes: readonly Change[], marker: Owne
376
383
  */
377
384
  export function alterSteps(obj: DeclaredObject, changes: readonly Change[], opts: { live?: CanonicalObject; marker?: OwnershipMarker } = {}): Step[] {
378
385
  const steps: Step[] = [];
379
- for (const c of changes.filter((x) => x.rule === "SQLCH231")) steps.push({ sql: `RENAME DATABASE ${ident(c.before!)} TO ${ident(c.after!)}`, rewrite: false });
386
+ for (const c of changes.filter((x) => x.rule === "SQLCH231")) steps.push(stepFor(`RENAME DATABASE ${ident(c.before!)} TO ${ident(c.after!)}`, "SQLCH231"));
380
387
  for (const c of changes.filter((x) => x.rule === "SQLCH230")) {
381
388
  const from = splitQualified(c.before!);
382
389
  const to = splitQualified(c.after!);
383
- steps.push({ sql: `RENAME TABLE ${qualifiedIdent(from.database, from.name)} TO ${qualifiedIdent(to.database, to.name)}`, rewrite: false });
390
+ steps.push(stepFor(`RENAME TABLE ${qualifiedIdent(from.database, from.name)} TO ${qualifiedIdent(to.database, to.name)}`, "SQLCH230"));
384
391
  }
385
392
  if (obj.type === CLICKHOUSE_ENTITY_TYPES.table) steps.push(...tableSteps(obj, changes, opts.live));
386
393
  else if (obj.type === CLICKHOUSE_ENTITY_TYPES.view || obj.type === CLICKHOUSE_ENTITY_TYPES.materializedView) steps.push(...viewSteps(obj, changes, opts.marker));
@@ -391,3 +398,114 @@ export function alterSteps(obj: DeclaredObject, changes: readonly Change[], opts
391
398
  export function stepsSetComment(obj: DeclaredObject, changes: readonly Change[]): boolean {
392
399
  return obj.type === CLICKHOUSE_ENTITY_TYPES.view && changes.some((c) => c.rule === "SQLCH240");
393
400
  }
401
+
402
+ // ── a schema's statements ─────────────────────────────────────────────
403
+
404
+ /**
405
+ * What one declared object takes, as the applier and the offline renderer
406
+ * (`../../migration-statements.ts`) both see it:
407
+ *
408
+ * - `create` or `alter`: the statements, in the order they run. An `alter`
409
+ * with none is an object already as declared.
410
+ * - `rebuild`: a change ALTER cannot make; nothing is sent for the object and
411
+ * the rebuild migration Op makes it (`detail` names the Op).
412
+ * - `withheld`: a column drop, which destroys data, in a plan not allowed to
413
+ * delete.
414
+ */
415
+ export type ObjectStatements =
416
+ | { verdict: "create" | "alter"; obj: DeclaredObject; changes: Change[]; steps: Step[] }
417
+ | { verdict: "rebuild"; obj: DeclaredObject; changes: Change[]; refused: Change[]; detail: string }
418
+ | { verdict: "withheld"; obj: DeclaredObject; changes: Change[]; withheld: Change[]; detail: string };
419
+
420
+ /** An object a schema no longer declares, and the `DROP` for it. */
421
+ export interface DropStatement {
422
+ /** The key the changes name it by. */
423
+ key: string;
424
+ type: ClickHouseEntityType;
425
+ database?: string;
426
+ name: string;
427
+ step: Step;
428
+ }
429
+
430
+ export interface StatementPlanInput {
431
+ /** The declared objects, in creation order. */
432
+ declared: readonly DeclaredObject[];
433
+ /** The classified changes from the current schema to the declared one. */
434
+ changes: readonly Change[];
435
+ /** The current definitions, keyed as the changes key objects. */
436
+ current: ReadonlyMap<string, CanonicalObject>;
437
+ /** How the changes key a declared object: `database.name` against a server (the default), the export name between two builds. */
438
+ keyOf?: (obj: DeclaredObject) => string;
439
+ /** The marker every `CREATE` and restamped comment carries. */
440
+ marker?: OwnershipMarker;
441
+ /** Column drops are planned, not withheld. Default: off. */
442
+ allowDestructive?: boolean;
443
+ /** Whether the current object (by its key in `current`) carries the marker already; one that does not is restamped. Default: it does. */
444
+ carriesMarker?: (currentKey: string) => boolean;
445
+ }
446
+
447
+ export interface StatementPlan {
448
+ /** One entry per declared object, in creation order. */
449
+ objects: ObjectStatements[];
450
+ /** The objects the declarations no longer hold, in drop order: what reads from a table first, a database last. */
451
+ drops: DropStatement[];
452
+ }
453
+
454
+ /** Drop order: what reads from a table before the table, a database last. */
455
+ const DROP_ORDER: Record<string, number> = { materializedView: 0, view: 1, table: 2, database: 3 };
456
+
457
+ /**
458
+ * The statements that take the current schema to the declared one, from the
459
+ * classified changes between them. Pure: nothing is read or sent. The applier
460
+ * (`./apply.ts`) runs the steps against a server; `diffStatements`
461
+ * (`../../migration-statements.ts`) renders the same steps for a migration
462
+ * file. Anything that changes the SQL sent for a change belongs here, so
463
+ * both see it.
464
+ */
465
+ export function planStatements(input: StatementPlanInput): StatementPlan {
466
+ const keyOf = input.keyOf ?? ((o: DeclaredObject) => o.key);
467
+ const byObject = new Map<string, Change[]>();
468
+ for (const c of input.changes) byObject.set(c.object, [...(byObject.get(c.object) ?? []), c]);
469
+
470
+ const objects: ObjectStatements[] = input.declared.map((obj): ObjectStatements => {
471
+ const key = keyOf(obj);
472
+ const mine = byObject.get(key) ?? [];
473
+ const refused = mine.filter(isRebuild);
474
+ if (refused.length > 0) return { verdict: "rebuild", obj, changes: mine, refused, detail: refusalDetail(refused, obj.key, obj.type) };
475
+ const destructive = mine.filter(isDestructiveAlter);
476
+ if (destructive.length > 0 && !input.allowDestructive) {
477
+ return {
478
+ verdict: "withheld",
479
+ obj,
480
+ changes: mine,
481
+ withheld: destructive,
482
+ detail: `drops ${destructive.map((c) => c.field).join(", ")}, which destroys the data in it; an apply that may delete (prune, ApplyOp delete "owned-only" or "gated") makes it`,
483
+ };
484
+ }
485
+ if (mine.some((c) => c.rule === "SQLCH200")) return { verdict: "create", obj, changes: mine, steps: [stepFor(createStatement(obj, input.marker), "SQLCH200")] };
486
+
487
+ const renamedFrom = mine.find((c) => c.rule === "SQLCH230" || c.rule === "SQLCH231")?.before;
488
+ const currentKey = renamedFrom !== undefined && input.current.has(renamedFrom) ? renamedFrom : key;
489
+ const live = input.current.get(currentKey);
490
+ const restamp = !stepsSetComment(obj, mine) && (mine.some((c) => c.field === "comment") || !(input.carriesMarker?.(currentKey) ?? true));
491
+ const steps = [
492
+ ...alterSteps(obj, mine, { ...(live ? { live } : {}), ...(input.marker ? { marker: input.marker } : {}) }),
493
+ ...(restamp ? [stepFor(commentStatement(obj, input.marker), "SQLCH203")] : []),
494
+ ];
495
+ return { verdict: "alter", obj, changes: mine, steps };
496
+ });
497
+
498
+ const drops: DropStatement[] = [];
499
+ for (const c of input.changes) {
500
+ if (c.rule !== "SQLCH250") continue;
501
+ const o = input.current.get(c.object);
502
+ if (!o) continue;
503
+ const type = CLICKHOUSE_ENTITY_TYPES[o.kind as keyof typeof CLICKHOUSE_ENTITY_TYPES];
504
+ const database = o.kind === "database" ? undefined : o.database;
505
+ drops.push({ key: c.object, type, ...(database !== undefined ? { database } : {}), name: o.name, step: stepFor(dropStatement(type, database, o.name), "SQLCH250") });
506
+ }
507
+ drops.sort((a, b) => (DROP_ORDER[kindOf(a.type)] ?? 9) - (DROP_ORDER[kindOf(b.type)] ?? 9));
508
+ return { objects, drops };
509
+ }
510
+
511
+ const kindOf = (type: string): string => Object.entries(CLICKHOUSE_ENTITY_TYPES).find(([, t]) => t === type)?.[0] ?? "table";
@@ -4,7 +4,7 @@
4
4
  * says `"dialect": "postgres"`) or a Postgres binding goes to
5
5
  * `../../postgres/plan/commands.ts`.
6
6
  *
7
- * chant sql diff <before.json> <after.json> [--json]
7
+ * chant sql diff <before.json> <after.json> [--json] [--statements]
8
8
  * chant sql plan <env> <build.json> [--json]
9
9
  *
10
10
  * `diff` is offline: it compares two `chant build` outputs, say a pull
@@ -14,6 +14,11 @@
14
14
  * rebuild, which a plan refuses to make in place, and 0 otherwise; for each
15
15
  * refused table they name the `ClickHouseRebuildOp` to run instead
16
16
  * (`rebuildOps` in `--json`, `./rebuild-handoff.ts`).
17
+ *
18
+ * `diff --statements` prints the statements instead, for a migration file:
19
+ * each one the applier would send, with its change's rule and class, and a
20
+ * refused change as the Op that makes it (`../../migration-statements.ts`).
21
+ * The marker in each `CREATE` is the project's `ownership` config's.
17
22
  */
18
23
 
19
24
  import type { CommandGroup, CommandGroupContext } from "@intentius/chant/cli/command-group";
@@ -34,8 +39,26 @@ function outputDialect(path: string): string | undefined {
34
39
  }
35
40
  }
36
41
 
37
- function split(args: string[]): { positional: string[]; json: boolean } {
38
- return { positional: args.filter((a) => !a.startsWith("--")), json: args.includes("--json") };
42
+ function split(args: string[]): { positional: string[]; json: boolean; statements: boolean } {
43
+ return { positional: args.filter((a) => !a.startsWith("--")), json: args.includes("--json"), statements: args.includes("--statements") };
44
+ }
45
+
46
+ /** `chant sql diff --statements`: the statements between two builds, as SQL with comments or as JSON. Exits 2 when a step is an Op or manual. */
47
+ async function runDiffStatements(before: string, after: string, json: boolean): Promise<number> {
48
+ const { diffStatements, renderStatements } = await import("../../migration-statements");
49
+ const { resolveOwnershipMarker } = await import("../../core/apply");
50
+ let marker;
51
+ try {
52
+ const { loadChantConfig } = await import("@intentius/chant/config");
53
+ const config = await loadChantConfig(process.cwd()).then((r) => r.config).catch(() => undefined);
54
+ marker = resolveOwnershipMarker({}, config, "chant sql diff --statements");
55
+ } catch (err) {
56
+ console.error(err instanceof Error ? err.message : String(err));
57
+ return 1;
58
+ }
59
+ const doc = diffStatements(readFileSync(before, "utf-8"), readFileSync(after, "utf-8"), marker ? { marker } : {});
60
+ console.log(json ? JSON.stringify(doc, null, 2) : renderStatements(doc));
61
+ return doc.refused ? 2 : 0;
39
62
  }
40
63
 
41
64
  function emit(diff: SchemaDiff, json: boolean, title: string): number {
@@ -44,11 +67,12 @@ function emit(diff: SchemaDiff, json: boolean, title: string): number {
44
67
  }
45
68
 
46
69
  export async function runDiff(ctx: CommandGroupContext): Promise<number> {
47
- const { positional, json } = split(ctx.rawArgs);
70
+ const { positional, json, statements } = split(ctx.rawArgs);
48
71
  if (positional.length !== 2) {
49
- console.error("usage: chant sql diff <before.json> <after.json> [--json]");
72
+ console.error("usage: chant sql diff <before.json> <after.json> [--json] [--statements]");
50
73
  return 1;
51
74
  }
75
+ if (statements) return runDiffStatements(positional[0]!, positional[1]!, json);
52
76
  if (outputDialect(positional[1]!) === "postgres") {
53
77
  const { diffPgBuildFiles, emitPg, projectMajor } = await import("../../postgres/plan/commands");
54
78
  return emitPg(diffPgBuildFiles(positional[0]!, positional[1]!, await projectMajor()), json, `${positional[0]} -> ${positional[1]}`);
@@ -95,7 +119,7 @@ export const sqlCommands: CommandGroup = {
95
119
  name: "sql",
96
120
  description: "Schema changes, classified: between two builds, or a build and a live server",
97
121
  commands: [
98
- { name: "diff", description: "Classify the change between two chant build outputs (offline)", handler: runDiff },
122
+ { name: "diff", description: "Classify the change between two chant build outputs (offline); --statements prints the statements it takes", handler: runDiff },
99
123
  { name: "plan", description: "Classify the change from an environment's server to a chant build output", handler: runPlan },
100
124
  ],
101
125
  };
@@ -0,0 +1,100 @@
1
+ /**
2
+ * The single-node rendering against a server (#3645): one migration source
3
+ * written for a cluster (`ON CLUSTER`, `Replicated*MergeTree` with Keeper
4
+ * paths, a `Replicated` database) is refused by a single node as written
5
+ * and applies once rendered for it, with the plain MergeTree family.
6
+ *
7
+ * Runs against `CLICKHOUSE_URL` when it is set (the emulator, `chant
8
+ * emulator up --lexicon sql`), else a scratch server of the pinned image.
9
+ * Everything it writes is in the database `yodel_3645`. Needs one or the
10
+ * other; skips cleanly without Docker.
11
+ */
12
+
13
+ import { afterAll, beforeAll, describe, expect, test } from "vitest";
14
+ import { dockerAvailable, startScratchServer, type ScratchServer } from "./container";
15
+ import { clickhouseQuery, type ClickHouseEndpoint } from "./http";
16
+ import { clickhouseImage } from "../spec/pin";
17
+ import { renderStatement, type Topology } from "./topology";
18
+
19
+ const DB = "yodel_3645";
20
+ const fromEnv = process.env.CLICKHOUSE_URL;
21
+ const enabled = fromEnv !== undefined || (await dockerAvailable());
22
+ let server: ScratchServer | undefined;
23
+ let endpoint: ClickHouseEndpoint;
24
+
25
+ const q = <T = Record<string, unknown>>(sql: string) => clickhouseQuery<T>(endpoint, sql);
26
+
27
+ beforeAll(async () => {
28
+ if (!enabled) return;
29
+ if (fromEnv !== undefined) {
30
+ endpoint = {
31
+ url: fromEnv,
32
+ ...(process.env.CLICKHOUSE_USER ? { user: process.env.CLICKHOUSE_USER } : {}),
33
+ ...(process.env.CLICKHOUSE_PASSWORD ? { password: process.env.CLICKHOUSE_PASSWORD } : {}),
34
+ };
35
+ } else {
36
+ server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-topology" });
37
+ endpoint = server.endpoint;
38
+ }
39
+ await q(`DROP DATABASE IF EXISTS ${DB} SYNC`);
40
+ }, 600_000);
41
+
42
+ afterAll(async () => {
43
+ if (enabled) await q(`DROP DATABASE IF EXISTS ${DB} SYNC`).catch(() => undefined);
44
+ await server?.stop();
45
+ });
46
+
47
+ const SINGLE: Topology = { kind: "single" };
48
+
49
+ /** One migration, written for a cluster named `main`. */
50
+ const SOURCE = [
51
+ `CREATE DATABASE ${DB} ON CLUSTER main ENGINE = Replicated('/clickhouse/databases/${DB}', '{shard}', '{replica}') COMMENT 'topology e2e'`,
52
+ `CREATE TABLE ${DB}.events ON CLUSTER main (
53
+ id UInt64,
54
+ ts DateTime,
55
+ kind LowCardinality(String)
56
+ ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/${DB}/events', '{replica}', ts)
57
+ ORDER BY id`,
58
+ `CREATE MATERIALIZED VIEW ${DB}.by_kind ON CLUSTER main
59
+ ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/${DB}/by_kind', '{replica}')
60
+ ORDER BY kind
61
+ AS SELECT kind, count() AS n FROM ${DB}.events GROUP BY kind`,
62
+ `CREATE TABLE ${DB}.staging ON CLUSTER main (id UInt64, ts DateTime, kind LowCardinality(String)) ENGINE = ReplicatedReplacingMergeTree(ts) ORDER BY id`,
63
+ `ALTER TABLE ${DB}.events ON CLUSTER main ADD COLUMN source String DEFAULT 'web' AFTER kind`,
64
+ `ALTER TABLE ${DB}.staging ON CLUSTER main ADD COLUMN source String DEFAULT 'web' AFTER kind`,
65
+ `ALTER TABLE ${DB}.events ON CLUSTER main MODIFY COMMENT 'events'`,
66
+ `EXCHANGE TABLES ${DB}.events AND ${DB}.staging ON CLUSTER main`,
67
+ `RENAME TABLE ${DB}.staging TO ${DB}.events_old ON CLUSTER main`,
68
+ `DROP TABLE ${DB}.events_old ON CLUSTER main SYNC`,
69
+ ];
70
+
71
+ describe.skipIf(!enabled)("the single-node rendering against a server (#3645)", () => {
72
+ test("the source as written for a cluster is refused by a single node", async () => {
73
+ await expect(q(SOURCE[0]!)).rejects.toThrow();
74
+ await expect(q(renderStatement(SOURCE[0]!, SINGLE))).resolves.toEqual([]);
75
+ // The database is there now; the table as written still fails, on its ON CLUSTER or its Keeper path.
76
+ await expect(q(SOURCE[1]!)).rejects.toThrow();
77
+ await q(`DROP DATABASE ${DB} SYNC`);
78
+ });
79
+
80
+ test("every statement rendered for a single node applies, with the plain MergeTree family", async () => {
81
+ for (const sql of SOURCE) {
82
+ const rendered = renderStatement(sql, SINGLE);
83
+ expect(rendered).not.toMatch(/ON CLUSTER|Replicated/);
84
+ await q(rendered);
85
+ }
86
+ const db = await q<{ engine: string }>(`SELECT engine FROM system.databases WHERE name = '${DB}'`);
87
+ expect(db).toEqual([{ engine: "Atomic" }]);
88
+ const tables = await q<{ name: string; engine: string }>(
89
+ `SELECT name, engine FROM system.tables WHERE database = '${DB}' AND NOT startsWith(name, '.inner') ORDER BY name`,
90
+ );
91
+ expect(tables).toEqual([
92
+ { name: "by_kind", engine: "MaterializedView" },
93
+ { name: "events", engine: "ReplacingMergeTree" },
94
+ ]);
95
+ const inner = await q<{ engine: string }>(`SELECT engine FROM system.tables WHERE database = '${DB}' AND startsWith(name, '.inner')`);
96
+ expect(inner).toEqual([{ engine: "SummingMergeTree" }]);
97
+ const cols = await q<{ name: string }>(`SELECT name FROM system.columns WHERE database = '${DB}' AND table = 'events' ORDER BY position`);
98
+ expect(cols.map((c) => c.name)).toEqual(["id", "ts", "kind", "source"]);
99
+ });
100
+ });