@intentius/chant-lexicon-sql 0.112.1 → 0.114.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/clickhouse/apply/apply.d.ts +6 -1
  2. package/dist/clickhouse/apply/apply.d.ts.map +1 -1
  3. package/dist/clickhouse/apply/statements.d.ts +85 -1
  4. package/dist/clickhouse/apply/statements.d.ts.map +1 -1
  5. package/dist/clickhouse/live/bind.d.ts +11 -1
  6. package/dist/clickhouse/live/bind.d.ts.map +1 -1
  7. package/dist/clickhouse/plan/commands.d.ts +12 -1
  8. package/dist/clickhouse/plan/commands.d.ts.map +1 -1
  9. package/dist/clickhouse/plan/deep.d.ts.map +1 -1
  10. package/dist/clickhouse/plan/diff.d.ts.map +1 -1
  11. package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
  12. package/dist/clickhouse/plan/schema.d.ts +4 -3
  13. package/dist/clickhouse/plan/schema.d.ts.map +1 -1
  14. package/dist/clickhouse/rebuild/backfill.d.ts.map +1 -1
  15. package/dist/clickhouse/rebuild/receipts.d.ts +5 -0
  16. package/dist/clickhouse/rebuild/receipts.d.ts.map +1 -1
  17. package/dist/clickhouse/rebuild/steps.d.ts.map +1 -1
  18. package/dist/clickhouse/topology.d.ts +112 -0
  19. package/dist/clickhouse/topology.d.ts.map +1 -0
  20. package/dist/clickhouse.d.ts +1 -0
  21. package/dist/clickhouse.d.ts.map +1 -1
  22. package/dist/codegen/classifier-page.d.ts.map +1 -1
  23. package/dist/codegen/postgres-classifier-page.d.ts.map +1 -1
  24. package/dist/config.d.ts +46 -0
  25. package/dist/config.d.ts.map +1 -1
  26. package/dist/core/statements.d.ts +106 -0
  27. package/dist/core/statements.d.ts.map +1 -0
  28. package/dist/index.d.ts +1 -0
  29. package/dist/index.d.ts.map +1 -1
  30. package/dist/integrity.json +2 -2
  31. package/dist/manifest.json +1 -1
  32. package/dist/migration-statements.d.ts +63 -0
  33. package/dist/migration-statements.d.ts.map +1 -0
  34. package/dist/postgres/apply/apply.d.ts.map +1 -1
  35. package/dist/postgres/apply/statements.d.ts +91 -1
  36. package/dist/postgres/apply/statements.d.ts.map +1 -1
  37. package/package.json +2 -2
  38. package/src/clickhouse/apply/apply.ts +35 -56
  39. package/src/clickhouse/apply/statements.ts +168 -31
  40. package/src/clickhouse/live/bind.ts +13 -1
  41. package/src/clickhouse/live/live.test.ts +10 -0
  42. package/src/clickhouse/plan/commands.ts +63 -9
  43. package/src/clickhouse/plan/deep.ts +2 -1
  44. package/src/clickhouse/plan/diff.ts +3 -1
  45. package/src/clickhouse/plan/normalize.ts +11 -2
  46. package/src/clickhouse/plan/schema.ts +6 -5
  47. package/src/clickhouse/rebuild/backfill.ts +1 -0
  48. package/src/clickhouse/rebuild/receipts.ts +14 -4
  49. package/src/clickhouse/rebuild/steps.ts +8 -5
  50. package/src/clickhouse/topology-replicated.e2e.test.ts +125 -0
  51. package/src/clickhouse/topology.e2e.test.ts +183 -0
  52. package/src/clickhouse/topology.test.ts +309 -0
  53. package/src/clickhouse/topology.ts +477 -0
  54. package/src/clickhouse.ts +16 -0
  55. package/src/codegen/classifier-page.ts +19 -0
  56. package/src/codegen/postgres-classifier-page.ts +17 -0
  57. package/src/config.ts +33 -0
  58. package/src/core/statements.ts +129 -0
  59. package/src/index.ts +15 -0
  60. package/src/migration-statements.test.ts +384 -0
  61. package/src/migration-statements.ts +262 -0
  62. package/src/op/__snapshots__/watch-pipeline.test.ts.snap +93 -0
  63. package/src/op/activities/clickhouse-apply.ts +1 -1
  64. package/src/op/activities/clickhouse-rebuild.ts +1 -1
  65. package/src/op/apply-gate-create.e2e.test.ts +280 -0
  66. package/src/op/watch-drift.e2e.test.ts +302 -0
  67. package/src/op/watch-pipeline.test.ts +113 -0
  68. package/src/plugin.test.ts +9 -0
  69. package/src/postgres/apply/apply.ts +38 -96
  70. package/src/postgres/apply/statements.ts +145 -2
@@ -0,0 +1,262 @@
1
+ /**
2
+ * A schema change's statements, rendered offline (#3644): for two `chant
3
+ * build` outputs, the ordered statements that take the first to the second,
4
+ * each with its change's class and rule, for a migration file. No server is
5
+ * read.
6
+ *
7
+ * The statements are the applier's: each dialect's `planStatements` /
8
+ * `planPgStatements` (`./clickhouse/apply/statements.ts`,
9
+ * `./postgres/apply/statements.ts`) turns classified changes into steps, and
10
+ * both the applier and this renderer call it. What differs is only where the
11
+ * current schema comes from: here the older build, keyed by export name as
12
+ * `chant sql diff` keys it; in the applier the server's catalog.
13
+ *
14
+ * A change the applier refuses in place is a step naming the Op that makes
15
+ * it, never DDL: a ClickHouse rebuild is `ClickHouseRebuildOp`, a Postgres
16
+ * column rename or type change across kinds is `PostgresMigrationOp`. One no
17
+ * Op makes is a manual step. Column and object drops are statements, marked
18
+ * destructive.
19
+ *
20
+ * Where the applier asks the server something, the renderer cannot:
21
+ *
22
+ * - ClickHouse: the applier also asks the server's formatter whether a
23
+ * difference the rules leave is only formatting; the offline diff is the
24
+ * rules' alone, as `chant sql diff` is. Before dropping a database it
25
+ * checks the database is empty.
26
+ * - Postgres: the applier asks the server to normalize expressions the rules
27
+ * leave different, and keeps an extension's own comment (from its control
28
+ * file) under the trailer when the declaration sets none.
29
+ * - Both: the applier restamps an object whose comment lost chant's marker;
30
+ * offline, the older build's objects carry it.
31
+ */
32
+
33
+ import type { OwnershipMarker } from "@intentius/chant/ownership";
34
+ import { statementChange, type ManualStep, type MigrationStep, type SchemaStatements, type StepObject } from "./core/statements";
35
+ import { declaredObjects as clickhouseDeclared } from "./clickhouse/apply/apply";
36
+ import { planStatements, type DeclaredObject } from "./clickhouse/apply/statements";
37
+ import { topologyLabel, type Topology } from "./clickhouse/topology";
38
+ import { diffSchemas, type Change } from "./clickhouse/plan/diff";
39
+ import { rebuildOpSuggestions } from "./clickhouse/plan/rebuild-handoff";
40
+ import { declaredObjects as postgresDeclared } from "./postgres/apply/apply";
41
+ import { planPgStatements, type DeclaredPgObject } from "./postgres/apply/statements";
42
+ import { diffPgSchemas, type PgChange } from "./postgres/plan/diff";
43
+ import { pgBuildMajor } from "./postgres/plan/schema";
44
+ import { migrationOpSuggestions, migrationTarget } from "./postgres/migrate/handoff";
45
+ import { POSTGRES_ENTITY_TYPES } from "./postgres/entity-types";
46
+ import { POSTGRES_LATEST_MAJOR } from "./spec/postgres-pin";
47
+
48
+ export type { ManualStep, MigrationStep, OpStep, SchemaStatements, StatementChange, StatementStep, StepObject } from "./core/statements";
49
+ export { renderStatements } from "./core/statements";
50
+
51
+ export interface DiffStatementsOptions {
52
+ /** The ownership marker every `CREATE` and restamped comment carries, as the applier stamps it. Default: `managed-by=chant` alone. */
53
+ marker?: OwnershipMarker;
54
+ /** The environment an Op step's suggested declaration names. Default: `<env>`. */
55
+ env?: string;
56
+ /** ClickHouse: the database a bare name means. Default: `default`. */
57
+ defaultDatabase?: string;
58
+ /**
59
+ * ClickHouse: the topology to render the statements for
60
+ * (`./clickhouse/topology.ts`): `ON CLUSTER` and the engine, on both
61
+ * builds' declarations before they are compared, so the steps are the ones
62
+ * the applier sends to an environment with that topology. Default: `single`.
63
+ */
64
+ topology?: Topology;
65
+ /** Postgres: the schema a bare name means. Default: `public`. */
66
+ defaultSchema?: string;
67
+ /** Postgres: the major to classify for. Default: the newer build's `postgresMajor`, else the older's, else the newest pinned. */
68
+ major?: number;
69
+ }
70
+
71
+ type BuildInput = string | object;
72
+
73
+ const text = (b: BuildInput): string => (typeof b === "string" ? b : JSON.stringify(b));
74
+
75
+ /** The dialect a build output names, at its root or under `sql`. */
76
+ function dialectOf(json: string): string | undefined {
77
+ const raw = JSON.parse(json) as { dialect?: unknown; sql?: { dialect?: unknown } };
78
+ const d = raw?.sql && typeof raw.sql === "object" ? raw.sql.dialect : raw?.dialect;
79
+ return typeof d === "string" ? d : undefined;
80
+ }
81
+
82
+ /**
83
+ * The statements that take the schema `before` builds to the one `after`
84
+ * builds. Each is a `chant build` output of the sql lexicon, as JSON text or
85
+ * parsed. Both must be the same dialect.
86
+ */
87
+ export function diffStatements(before: BuildInput, after: BuildInput, options: DiffStatementsOptions = {}): SchemaStatements {
88
+ const b = text(before);
89
+ const a = text(after);
90
+ const dialect = dialectOf(a);
91
+ if (dialectOf(b) !== dialect) throw new Error(`diffStatements: the builds are different dialects (${dialectOf(b) ?? "none"} and ${dialect ?? "none"})`);
92
+ if (dialect === "postgres") return postgresStatements(b, a, options);
93
+ if (dialect === "clickhouse") return clickhouseStatements(b, a, options);
94
+ throw new Error(`diffStatements: not a sql lexicon build output (dialect ${dialect ?? "missing"})`);
95
+ }
96
+
97
+ const at = (object: string, type: string, name: string): StepObject => ({ object, type, name });
98
+
99
+ // ── ClickHouse ─────────────────────────────────────────────────────────
100
+
101
+ function clickhouseStatements(beforeJson: string, afterJson: string, options: DiffStatementsOptions): SchemaStatements {
102
+ const defaultDatabase = options.defaultDatabase ?? "default";
103
+ const env = options.env ?? "<env>";
104
+ const topology: Topology = options.topology ?? { kind: "single" };
105
+ const before = clickhouseDeclared(beforeJson, defaultDatabase, topology);
106
+ const after = clickhouseDeclared(afterJson, defaultDatabase, topology);
107
+ const keyed = (objs: readonly DeclaredObject[]) => objs.map((o) => ({ key: o.exportName, canonical: o.canonical }));
108
+ const diff = diffSchemas(keyed(before), keyed(after));
109
+ const plan = planStatements({
110
+ declared: after,
111
+ changes: diff.changes,
112
+ current: new Map(before.map((o) => [o.exportName, o.canonical])),
113
+ keyOf: (o) => o.exportName,
114
+ allowDestructive: true,
115
+ topology,
116
+ ...(options.marker ? { marker: options.marker } : {}),
117
+ });
118
+ const declaredByExport = new Map(after.map((o) => [o.exportName, o.canonical]));
119
+ const isDestructive = (changes: readonly Change[], rule: string) => changes.some((c) => c.rule === rule && c.destructive);
120
+
121
+ const steps: MigrationStep[] = [];
122
+ for (const entry of plan.objects) {
123
+ const where = at(entry.obj.exportName, entry.obj.type, entry.obj.key);
124
+ if (entry.verdict === "rebuild") {
125
+ const [op] = rebuildOpSuggestions({ ...diff, rebuilds: entry.refused }, declaredByExport, env);
126
+ if (op) {
127
+ steps.push({
128
+ kind: "op",
129
+ ...where,
130
+ op: "ClickHouseRebuildOp",
131
+ importPath: "@intentius/chant-lexicon-sql/clickhouse",
132
+ options: { name: op.name, env: op.env, table: op.table, dualWrite: op.dualWrite },
133
+ declaration: op.declaration,
134
+ // The rebuild creates the new table from the declaration, so it makes every change to it.
135
+ changes: entry.changes.map(statementChange),
136
+ detail: entry.detail,
137
+ });
138
+ } else steps.push({ kind: "manual", ...where, changes: entry.refused.map(statementChange), detail: entry.detail });
139
+ continue;
140
+ }
141
+ if (entry.verdict === "withheld") continue; // not reached: destructive changes are planned
142
+ for (const s of entry.steps) {
143
+ steps.push({
144
+ kind: "statement",
145
+ ...where,
146
+ sql: s.sql,
147
+ rule: s.rule,
148
+ class: s.class,
149
+ transactional: false,
150
+ ...(s.rewrite ? { waitsForMutation: true } : {}),
151
+ ...(s.rule === "SQLCH202" || isDestructive(entry.changes, s.rule) ? { destructive: true } : {}),
152
+ });
153
+ }
154
+ }
155
+ const beforeByExport = new Map(before.map((o) => [o.exportName, o]));
156
+ for (const d of plan.drops) {
157
+ const o = beforeByExport.get(d.key);
158
+ const change = diff.changes.find((c) => c.object === d.key && c.rule === "SQLCH250");
159
+ steps.push({
160
+ kind: "statement",
161
+ ...at(d.key, d.type, o?.key ?? d.name),
162
+ sql: d.step.sql,
163
+ rule: d.step.rule,
164
+ class: d.step.class,
165
+ transactional: false,
166
+ ...(change?.destructive ? { destructive: true } : {}),
167
+ });
168
+ }
169
+ return { dialect: "clickhouse", defaultDatabase, topology: topologyLabel(topology), steps, refused: steps.some((s) => s.kind !== "statement"), hints: diff.hints };
170
+ }
171
+
172
+ // ── Postgres ───────────────────────────────────────────────────────────
173
+
174
+ function postgresStatements(beforeJson: string, afterJson: string, options: DiffStatementsOptions): SchemaStatements {
175
+ const defaultSchema = options.defaultSchema ?? "public";
176
+ const env = options.env ?? "<env>";
177
+ const major = options.major ?? buildMajor(afterJson) ?? buildMajor(beforeJson) ?? POSTGRES_LATEST_MAJOR;
178
+ const before = postgresDeclared(beforeJson, defaultSchema);
179
+ const after = postgresDeclared(afterJson, defaultSchema);
180
+ const keyed = (objs: readonly DeclaredPgObject[]) => objs.map((o) => ({ key: o.exportName, canonical: o.canonical }));
181
+ const current = keyed(before);
182
+ const diff = diffPgSchemas(current, keyed(after), { major });
183
+ const plan = planPgStatements({
184
+ declared: after,
185
+ changes: diff.changes,
186
+ current,
187
+ keyOf: (o) => o.exportName,
188
+ major,
189
+ allowDestructive: true,
190
+ ...(options.marker ? { marker: options.marker } : {}),
191
+ });
192
+ const isDestructive = (changes: readonly PgChange[], rule: string | undefined) => changes.some((c) => c.rule === rule && c.destructive);
193
+
194
+ const steps: MigrationStep[] = [];
195
+ for (const entry of plan.objects) {
196
+ const where = at(entry.obj.exportName, entry.obj.type, entry.obj.name);
197
+ switch (entry.verdict) {
198
+ case "refused": {
199
+ const handled = new Set<PgChange>();
200
+ for (const op of migrationOpSuggestions(entry.refused, new Map([[entry.obj.exportName, entry.obj.canonical]]), env, defaultSchema)) {
201
+ const made = entry.refused.filter((c) => {
202
+ const t = migrationTarget(c, entry.obj.canonical, defaultSchema);
203
+ return t !== undefined && t.table === op.table && t.column === op.column;
204
+ });
205
+ for (const c of made) handled.add(c);
206
+ steps.push({
207
+ kind: "op",
208
+ ...where,
209
+ op: "PostgresMigrationOp",
210
+ importPath: "@intentius/chant-lexicon-sql/postgres",
211
+ options: { name: op.name, env: op.env, table: op.table, column: op.column },
212
+ declaration: op.declaration,
213
+ changes: made.map(statementChange),
214
+ detail: entry.detail,
215
+ });
216
+ }
217
+ const rest = entry.changes.filter((c) => !handled.has(c));
218
+ if (rest.length > 0) steps.push(manual(where, rest, handled.size > 0 ? `the applier sends nothing for ${entry.obj.name} while a change to it is refused; these changes are not made by the Op: ${entry.detail}` : entry.detail));
219
+ continue;
220
+ }
221
+ case "unsupported":
222
+ steps.push(manual(where, entry.changes, entry.detail));
223
+ continue;
224
+ case "foreign":
225
+ case "withheld":
226
+ continue; // not reached offline: a build declares no foreign object, and destructive changes are planned
227
+ }
228
+ for (const s of entry.steps) {
229
+ steps.push({
230
+ kind: "statement",
231
+ ...where,
232
+ sql: s.sql,
233
+ rule: s.rule ?? (s.class === "create" ? "SQLPG200" : (entry.changes[0]?.rule ?? "SQLPG200")),
234
+ class: s.class,
235
+ transactional: s.transactional,
236
+ ...(s.rule === "SQLPG204" || isDestructive(entry.changes, s.rule) ? { destructive: true } : {}),
237
+ });
238
+ }
239
+ }
240
+ const beforeByExport = new Map(before.map((o) => [o.exportName, o]));
241
+ for (const d of plan.drops) {
242
+ const o = beforeByExport.get(d.key);
243
+ const change = diff.changes.find((c) => c.object === d.key && (c.rule === "SQLPG270" || c.rule === "SQLPG242"));
244
+ steps.push({
245
+ kind: "statement",
246
+ ...at(d.key, o?.type ?? POSTGRES_ENTITY_TYPES[d.kind], o?.name ?? d.name),
247
+ sql: d.step.sql,
248
+ rule: d.step.rule ?? "SQLPG270",
249
+ class: d.step.class,
250
+ transactional: d.step.transactional,
251
+ ...(change?.destructive ? { destructive: true } : {}),
252
+ });
253
+ }
254
+ return { dialect: "postgres", defaultSchema, major, steps, refused: steps.some((s) => s.kind !== "statement"), hints: diff.hints };
255
+ }
256
+
257
+ const manual = (where: StepObject, changes: readonly PgChange[], detail: string): ManualStep => ({ kind: "manual", ...where, changes: changes.map(statementChange), detail });
258
+
259
+ function buildMajor(json: string): number | undefined {
260
+ const raw = JSON.parse(json) as { sql?: unknown };
261
+ return pgBuildMajor(raw.sql && typeof raw.sql === "object" ? JSON.stringify(raw.sql) : json);
262
+ }
@@ -0,0 +1,93 @@
1
+ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
2
+
3
+ exports[`generateOpsPipeline renders the sql drift watch (#3642) > Forgejo: the Op's own cron and the reader's credentials; the runner's token takes no permissions block 1`] = `
4
+ "name: schema-watch
5
+
6
+ on:
7
+ schedule:
8
+ - cron: '17 * * * *'
9
+ workflow_dispatch: {}
10
+
11
+ concurrency:
12
+ group: schema-watch
13
+ cancel-in-progress: false
14
+
15
+ jobs:
16
+ schema-watch:
17
+ runs-on: docker
18
+ container: node:22-slim
19
+ env:
20
+ CLICKHOUSE_READER_USER: chant_reader
21
+ CLICKHOUSE_READER_PASSWORD: '\${{ secrets.CLICKHOUSE_READER_PASSWORD }}'
22
+ steps:
23
+ - uses: https://code.forgejo.org/actions/checkout@v7
24
+ - run: npm ci
25
+ - run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
26
+ - run: chant run schema-watch
27
+ env:
28
+ GITHUB_TOKEN: '\${{ github.token }}'
29
+ "
30
+ `;
31
+
32
+ exports[`generateOpsPipeline renders the sql drift watch (#3642) > GitHub: the Op's own cron, a read-only token and the reader's credentials 1`] = `
33
+ "name: schema-watch
34
+
35
+ on:
36
+ schedule:
37
+ - cron: '17 * * * *'
38
+ workflow_dispatch: {}
39
+
40
+ concurrency:
41
+ group: schema-watch
42
+ cancel-in-progress: false
43
+
44
+ permissions:
45
+ contents: read
46
+
47
+ jobs:
48
+ schema-watch:
49
+ runs-on: ubuntu-latest
50
+ container: node:22-slim
51
+ env:
52
+ CLICKHOUSE_READER_USER: chant_reader
53
+ CLICKHOUSE_READER_PASSWORD: '\${{ secrets.CLICKHOUSE_READER_PASSWORD }}'
54
+ steps:
55
+ - uses: actions/checkout@v7
56
+ - run: npm ci
57
+ - run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
58
+ - run: chant run schema-watch
59
+ env:
60
+ GITHUB_TOKEN: '\${{ github.token }}'
61
+ "
62
+ `;
63
+
64
+ exports[`generateOpsPipeline renders the sql drift watch (#3642) > GitLab: a schedule-only job with the reader's credentials and no token of its own 1`] = `
65
+ "# chant Ops (#927, #2084, #2293) — one job per Op under stage "ops", each
66
+ # selected by its own rules:. A merge_request_event or push job needs no
67
+ # setup; its rule fires on the event itself.
68
+ #
69
+ # GitLab has no in-file cron. Create one Pipeline Schedule per cron Op
70
+ # below (Settings > CI/CD > Schedules): set its cron to the value noted
71
+ # here and its CHANT_SCHEDULED_OP CI/CD variable to the Op's name, so only
72
+ # that job runs on that schedule.
73
+ #
74
+ # schema-watch: cron "17 * * * *", CHANT_SCHEDULED_OP="schema-watch", finding-mode report
75
+
76
+ stages:
77
+ - ops
78
+
79
+ schema-watch:
80
+ stage: ops
81
+ image: node:22-slim
82
+ resource_group: schema-watch
83
+ variables:
84
+ CLICKHOUSE_READER_USER: chant_reader
85
+ CLICKHOUSE_READER_PASSWORD: '$CLICKHOUSE_READER_PASSWORD'
86
+ rules:
87
+ - if: '$CI_PIPELINE_SOURCE == "schedule" && $CHANT_SCHEDULED_OP == "schema-watch"'
88
+ script:
89
+ - npm ci
90
+ - export PATH="$PWD/node_modules/.bin:$PATH"
91
+ - chant run schema-watch
92
+ "
93
+ `;
@@ -84,7 +84,7 @@ export async function clickhouseApply(args: ClickHouseApplyArgs, signal?: AbortS
84
84
  if (!(err instanceof ClickHouseBindingError)) throw err;
85
85
  return refused(declaredObjects(json), err.unresolved.reason, err.unresolved.detail);
86
86
  }
87
- const declared = declaredObjects(json, target.defaultDatabase);
87
+ const declared = declaredObjects(json, target.defaultDatabase, target.topology);
88
88
  let outcome: ClickHouseApplyOutcome;
89
89
  try {
90
90
  outcome = await applyClickHouse(target, declared, {
@@ -59,7 +59,7 @@ export async function rebuildRun(args: ClickHouseRebuildArgs, signal: AbortSigna
59
59
  const config = deps.config ?? (await loadConfig(cwd));
60
60
  const marker = resolveMarker(args, config);
61
61
  const target = await bindClickHouse({ ...(args.environment !== undefined ? { environment: args.environment } : {}), config: config ?? {}, ...(deps.env ? { env: deps.env } : {}) });
62
- const declared = declaredTable(declaredObjects(readFileSync(resolve(cwd, args.buildPath), "utf8"), target.defaultDatabase), args.table);
62
+ const declared = declaredTable(declaredObjects(readFileSync(resolve(cwd, args.buildPath), "utf8"), target.defaultDatabase, target.topology), args.table);
63
63
  const runId = currentOpRun()?.runId;
64
64
  return {
65
65
  target,
@@ -0,0 +1,280 @@
1
+ /**
2
+ * An approval for a plan that creates an object does not survive an edit to
3
+ * that object (#3652), end to end on both dialects.
4
+ *
5
+ * For each dialect, a project in a temporary directory declares a table that
6
+ * does not exist yet and a gated `ApplyOp`. `chant run` stops at the gate and
7
+ * prints the `chant approve ... --plan <digest>` that answers it; the test
8
+ * approves that digest, adds a column to the declared table, and runs again.
9
+ * The gate refuses with "is approved, but not for this plan" and nothing is
10
+ * created. Approving the new plan then applies the edited table.
11
+ *
12
+ * Before #3652 the plan digest was taken over a diff that listed the table by
13
+ * name only, so the second run applied the edited table under the first
14
+ * approval.
15
+ *
16
+ * Everything is written to the database (ClickHouse) or schema (Postgres)
17
+ * `yodel_3652`.
18
+ *
19
+ * The servers: `CHANT_SQL_GATE_CLICKHOUSE_URL` and
20
+ * `CHANT_SQL_GATE_POSTGRES_URL` (a `postgres://user@host:port/db` URL, with
21
+ * `CHANT_SQL_GATE_POSTGRES_PASSWORD`) name running ones, such as the servers
22
+ * `chant emulator up --lexicon sql` starts. Without them, each dialect starts
23
+ * a throwaway server at the same pin, and skips cleanly when Docker is not
24
+ * available.
25
+ */
26
+
27
+ import { execFile } from "node:child_process";
28
+ import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
29
+ import { tmpdir } from "node:os";
30
+ import { join } from "node:path";
31
+ import { promisify } from "node:util";
32
+ import { afterAll, beforeAll, describe, expect, test } from "vitest";
33
+ import { dockerAvailable, startScratchServer, type ScratchServer } from "../clickhouse/container";
34
+ import { clickhouseQuery, type ClickHouseEndpoint } from "../clickhouse/http";
35
+ import { clickhouseImage } from "../spec/pin";
36
+ import { startTestPostgres, type TestPostgres } from "../postgres/testing/server";
37
+ import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../postgres/live/client";
38
+
39
+ const exec = promisify(execFile);
40
+
41
+ const REPO = join(import.meta.dirname, "../../../..");
42
+ const BIN = join(REPO, "node_modules/.bin");
43
+ const NAME = "yodel_3652";
44
+ const OP = "schema-apply";
45
+
46
+ const givenClickHouse = process.env.CHANT_SQL_GATE_CLICKHOUSE_URL;
47
+ const givenPostgres = process.env.CHANT_SQL_GATE_POSTGRES_URL;
48
+ const docker = givenClickHouse && givenPostgres ? false : await dockerAvailable();
49
+
50
+ interface Run {
51
+ code: number;
52
+ output: string;
53
+ }
54
+
55
+ const strip = (s: string) => s.replace(/\u001b\[[0-9;]*m/g, "");
56
+
57
+ /**
58
+ * A project directory chant can run in: the repository's `node_modules`
59
+ * linked in, so `@intentius/chant` and the sql lexicon resolve, and a git
60
+ * repository, which the gate ledger and snapshots are written to.
61
+ */
62
+ async function project(files: Record<string, string>, env: Record<string, string>) {
63
+ const dir = mkdtempSync(join(tmpdir(), "chant-sql-gate-"));
64
+ symlinkSync(join(REPO, "node_modules"), join(dir, "node_modules"), "dir");
65
+ const write = (all: Record<string, string>) => {
66
+ for (const [path, text] of Object.entries(all)) {
67
+ mkdirSync(join(dir, path, ".."), { recursive: true });
68
+ writeFileSync(join(dir, path), text);
69
+ }
70
+ };
71
+ write({
72
+ "package.json": JSON.stringify({ name: "sql-gate-e2e", private: true, type: "module", scripts: { build: "chant build src --lexicon sql -o dist/schema.json" } }),
73
+ ".gitignore": "node_modules\ndist\n.chant\n",
74
+ ...files,
75
+ });
76
+ const git = (...args: string[]) => exec("git", ["-c", "user.email=e2e@chant.invalid", "-c", "user.name=e2e", ...args], { cwd: dir });
77
+ await git("init", "-q");
78
+ await git("add", "-A");
79
+ await git("commit", "-qm", "the schema");
80
+ const chant = async (...args: string[]): Promise<Run> => {
81
+ try {
82
+ const r = await exec(join(BIN, "chant"), args, {
83
+ cwd: dir,
84
+ env: { ...process.env, ...env, PATH: `${BIN}:${process.env.PATH ?? ""}` },
85
+ maxBuffer: 16 * 1024 * 1024,
86
+ });
87
+ return { code: 0, output: strip(`${r.stdout}${r.stderr}`) };
88
+ } catch (err) {
89
+ const e = err as { code?: number; stdout?: string; stderr?: string };
90
+ return { code: typeof e.code === "number" ? e.code : 1, output: strip(`${e.stdout ?? ""}${e.stderr ?? ""}`) };
91
+ }
92
+ };
93
+ return {
94
+ dir,
95
+ chant,
96
+ async edit(path: string, text: string) {
97
+ write({ [path]: text });
98
+ await git("commit", "-qam", `edit ${path}`);
99
+ },
100
+ };
101
+ }
102
+
103
+ /** The `--plan <digest>` the stopped run printed in its approve command. */
104
+ function approvedPlan(run: Run): string {
105
+ const m = /chant approve \S+ \S+ .*?--plan (jcs1-sha256:[0-9a-f]{64})/.exec(run.output);
106
+ if (!m) throw new Error(`the run printed no approve command with a plan:\n${run.output}`);
107
+ return m[1];
108
+ }
109
+
110
+ function gateOf(run: Run): string {
111
+ const m = /chant approve \S+ (\S+)/.exec(run.output);
112
+ if (!m) throw new Error(`the run printed no approve command:\n${run.output}`);
113
+ return m[1];
114
+ }
115
+
116
+ const OPS = (target: string) => ({
117
+ "ops/apply.op.ts": `import { ApplyOp } from "@intentius/chant/op";
118
+ const { op } = ApplyOp({ name: "${OP}", env: "admin", target: "${target}", gate: { gate: "approve-schema" } });
119
+ export default op;
120
+ `,
121
+ });
122
+
123
+ const config = (sql: Record<string, unknown>) => `import type { ChantConfig } from "@intentius/chant";
124
+ import "@intentius/chant-lexicon-sql";
125
+
126
+ export default {
127
+ lexicons: ["sql"],
128
+ ownership: { stack: "yodel-3652", env: "admin" },
129
+ sql: ${JSON.stringify(sql, null, 2)},
130
+ } satisfies ChantConfig;
131
+ `;
132
+
133
+ /**
134
+ * The shared scenario: approve a create, edit the declared object, rerun.
135
+ * `columns` reads the live table's column names.
136
+ */
137
+ async function approveEditRerun(
138
+ p: Awaited<ReturnType<typeof project>>,
139
+ edited: { path: string; text: string },
140
+ columns: () => Promise<string[]>,
141
+ ): Promise<void> {
142
+ const first = await p.chant("run", OP);
143
+ expect(first.output).toMatch(/MISSING \(declared, provider reports not in cloud\):[\s\S]*- events .*\[definition [0-9a-f]{64}\]/);
144
+ const gate = gateOf(first);
145
+ const digest = approvedPlan(first);
146
+
147
+ const approve = await p.chant("approve", OP, gate, "--plan", digest);
148
+ expect(approve.code, approve.output).toBe(0);
149
+
150
+ await p.edit(edited.path, edited.text);
151
+
152
+ const second = await p.chant("run", OP);
153
+ expect(second.output).toContain(`Gate "${gate}" is approved, but not for this plan.`);
154
+ expect(second.output).toContain(`approved: ${digest}`);
155
+ expect(second.output).not.toMatch(/Op "schema-apply" completed/);
156
+ expect(await columns()).toEqual([]);
157
+
158
+ // Approving the plan the second run produced applies the edited table.
159
+ const fresh = approvedPlan(second);
160
+ expect(fresh).not.toBe(digest);
161
+ const reapprove = await p.chant("approve", OP, gate, "--plan", fresh);
162
+ expect(reapprove.code, reapprove.output).toBe(0);
163
+ const third = await p.chant("run", OP);
164
+ expect(third.output).toMatch(/Op "schema-apply" completed/);
165
+ expect(await columns()).toContain("note");
166
+ }
167
+
168
+ // ── ClickHouse ──────────────────────────────────────────────────────────────
169
+
170
+ const CH_SCHEMA = (extra: string) => `import { database, table } from "@intentius/chant-lexicon-sql/clickhouse";
171
+
172
+ export const db = database\`CREATE DATABASE ${NAME} ENGINE = Atomic COMMENT 'gate binds creates'\`;
173
+
174
+ export const events = table\`
175
+ CREATE TABLE \${db}.events (
176
+ user_id UInt64,
177
+ ts DateTime${extra}
178
+ )
179
+ ENGINE = MergeTree
180
+ ORDER BY (user_id, ts)
181
+ COMMENT 'Raw events'\`;
182
+ `;
183
+
184
+ describe.skipIf(!givenClickHouse && !docker)("a gated ApplyOp over ClickHouse refuses an edited create (#3652)", () => {
185
+ let server: ScratchServer | undefined;
186
+ let admin: ClickHouseEndpoint;
187
+ let p: Awaited<ReturnType<typeof project>>;
188
+ const q = (sql: string) => clickhouseQuery(admin, sql);
189
+
190
+ beforeAll(async () => {
191
+ if (givenClickHouse) admin = { url: givenClickHouse };
192
+ else {
193
+ server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-gate" });
194
+ admin = server.endpoint;
195
+ }
196
+ await q(`DROP DATABASE IF EXISTS ${NAME} SYNC`);
197
+ p = await project(
198
+ {
199
+ ...OPS("clickhouse"),
200
+ "chant.config.ts": config({ dialect: "clickhouse", profiles: { admin: { url: admin.url, databases: [NAME] } } }),
201
+ "src/schema.ts": CH_SCHEMA(""),
202
+ },
203
+ {},
204
+ );
205
+ }, 600_000);
206
+
207
+ afterAll(async () => {
208
+ if (admin) await q(`DROP DATABASE IF EXISTS ${NAME} SYNC`).catch(() => undefined);
209
+ await server?.stop();
210
+ if (p) rmSync(p.dir, { recursive: true, force: true });
211
+ });
212
+
213
+ test("approve a create, add a column, rerun: the gate refuses", async () => {
214
+ await approveEditRerun(p, { path: "src/schema.ts", text: CH_SCHEMA(",\n note String") }, async () => {
215
+ const rows = await clickhouseQuery<{ name: string }>(admin, `SELECT name FROM system.columns WHERE database = '${NAME}' AND table = 'events' ORDER BY position`);
216
+ return rows.map((r) => r.name);
217
+ });
218
+ }, 600_000);
219
+ });
220
+
221
+ // ── Postgres ────────────────────────────────────────────────────────────────
222
+
223
+ const PG_SCHEMA = (extra: string) => `import { schema, table } from "@intentius/chant-lexicon-sql/postgres";
224
+
225
+ export const app = schema\`CREATE SCHEMA ${NAME};
226
+ COMMENT ON SCHEMA ${NAME} IS 'gate binds creates'\`;
227
+
228
+ export const events = table\`
229
+ CREATE TABLE \${app}.events (
230
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
231
+ kind text NOT NULL${extra}
232
+ )\`;
233
+ `;
234
+
235
+ describe.skipIf(!givenPostgres && !docker)("a gated ApplyOp over Postgres refuses an edited create (#3652)", () => {
236
+ let server: TestPostgres | undefined;
237
+ let endpoint: PostgresEndpoint;
238
+ let admin: PostgresClient;
239
+ let p: Awaited<ReturnType<typeof project>>;
240
+ const q = (sql: string) => admin.query(sql);
241
+
242
+ beforeAll(async () => {
243
+ if (givenPostgres) endpoint = { url: givenPostgres, password: process.env.CHANT_SQL_GATE_POSTGRES_PASSWORD };
244
+ else {
245
+ server = await startTestPostgres();
246
+ endpoint = server.endpoint();
247
+ }
248
+ admin = await connectPostgres(endpoint);
249
+ await q(`DROP SCHEMA IF EXISTS ${NAME} CASCADE`);
250
+ p = await project(
251
+ {
252
+ ...OPS("postgres"),
253
+ "chant.config.ts": config({
254
+ dialect: "postgres",
255
+ profiles: { admin: { url: endpoint.url, password: { env: "ADMIN_PASSWORD" }, schemas: [NAME] } },
256
+ }),
257
+ "src/schema.ts": PG_SCHEMA(""),
258
+ },
259
+ { ADMIN_PASSWORD: endpoint.password ?? "" },
260
+ );
261
+ }, 600_000);
262
+
263
+ afterAll(async () => {
264
+ if (admin) {
265
+ await q(`DROP SCHEMA IF EXISTS ${NAME} CASCADE`).catch(() => undefined);
266
+ await admin.end();
267
+ }
268
+ await server?.stop();
269
+ if (p) rmSync(p.dir, { recursive: true, force: true });
270
+ });
271
+
272
+ test("approve a create, add a column, rerun: the gate refuses", async () => {
273
+ await approveEditRerun(p, { path: "src/schema.ts", text: PG_SCHEMA(",\n note text") }, async () => {
274
+ const rows = await admin.query<{ column_name: string }>(
275
+ `SELECT column_name FROM information_schema.columns WHERE table_schema = '${NAME}' AND table_name = 'events' ORDER BY ordinal_position`,
276
+ );
277
+ return rows.map((r) => r.column_name);
278
+ });
279
+ }, 600_000);
280
+ });