@intentius/chant-lexicon-sql 0.112.1 → 0.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clickhouse/apply/apply.d.ts +6 -1
- package/dist/clickhouse/apply/apply.d.ts.map +1 -1
- package/dist/clickhouse/apply/statements.d.ts +85 -1
- package/dist/clickhouse/apply/statements.d.ts.map +1 -1
- package/dist/clickhouse/live/bind.d.ts +11 -1
- package/dist/clickhouse/live/bind.d.ts.map +1 -1
- package/dist/clickhouse/plan/commands.d.ts +12 -1
- package/dist/clickhouse/plan/commands.d.ts.map +1 -1
- package/dist/clickhouse/plan/deep.d.ts.map +1 -1
- package/dist/clickhouse/plan/diff.d.ts.map +1 -1
- package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
- package/dist/clickhouse/plan/schema.d.ts +4 -3
- package/dist/clickhouse/plan/schema.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/backfill.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/receipts.d.ts +5 -0
- package/dist/clickhouse/rebuild/receipts.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/steps.d.ts.map +1 -1
- package/dist/clickhouse/topology.d.ts +112 -0
- package/dist/clickhouse/topology.d.ts.map +1 -0
- package/dist/clickhouse.d.ts +1 -0
- package/dist/clickhouse.d.ts.map +1 -1
- package/dist/codegen/classifier-page.d.ts.map +1 -1
- package/dist/codegen/postgres-classifier-page.d.ts.map +1 -1
- package/dist/config.d.ts +46 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/core/statements.d.ts +106 -0
- package/dist/core/statements.d.ts.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/integrity.json +2 -2
- package/dist/manifest.json +1 -1
- package/dist/migration-statements.d.ts +63 -0
- package/dist/migration-statements.d.ts.map +1 -0
- package/dist/postgres/apply/apply.d.ts.map +1 -1
- package/dist/postgres/apply/statements.d.ts +91 -1
- package/dist/postgres/apply/statements.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/clickhouse/apply/apply.ts +35 -56
- package/src/clickhouse/apply/statements.ts +168 -31
- package/src/clickhouse/live/bind.ts +13 -1
- package/src/clickhouse/live/live.test.ts +10 -0
- package/src/clickhouse/plan/commands.ts +63 -9
- package/src/clickhouse/plan/deep.ts +2 -1
- package/src/clickhouse/plan/diff.ts +3 -1
- package/src/clickhouse/plan/normalize.ts +11 -2
- package/src/clickhouse/plan/schema.ts +6 -5
- package/src/clickhouse/rebuild/backfill.ts +1 -0
- package/src/clickhouse/rebuild/receipts.ts +14 -4
- package/src/clickhouse/rebuild/steps.ts +8 -5
- package/src/clickhouse/topology-replicated.e2e.test.ts +125 -0
- package/src/clickhouse/topology.e2e.test.ts +183 -0
- package/src/clickhouse/topology.test.ts +309 -0
- package/src/clickhouse/topology.ts +477 -0
- package/src/clickhouse.ts +16 -0
- package/src/codegen/classifier-page.ts +19 -0
- package/src/codegen/postgres-classifier-page.ts +17 -0
- package/src/config.ts +33 -0
- package/src/core/statements.ts +129 -0
- package/src/index.ts +15 -0
- package/src/migration-statements.test.ts +384 -0
- package/src/migration-statements.ts +262 -0
- package/src/op/__snapshots__/watch-pipeline.test.ts.snap +93 -0
- package/src/op/activities/clickhouse-apply.ts +1 -1
- package/src/op/activities/clickhouse-rebuild.ts +1 -1
- package/src/op/apply-gate-create.e2e.test.ts +280 -0
- package/src/op/watch-drift.e2e.test.ts +302 -0
- package/src/op/watch-pipeline.test.ts +113 -0
- package/src/plugin.test.ts +9 -0
- package/src/postgres/apply/apply.ts +38 -96
- package/src/postgres/apply/statements.ts +145 -2
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A schema change's statements, rendered offline (#3644): for two `chant
|
|
3
|
+
* build` outputs, the ordered statements that take the first to the second,
|
|
4
|
+
* each with its change's class and rule, for a migration file. No server is
|
|
5
|
+
* read.
|
|
6
|
+
*
|
|
7
|
+
* The statements are the applier's: each dialect's `planStatements` /
|
|
8
|
+
* `planPgStatements` (`./clickhouse/apply/statements.ts`,
|
|
9
|
+
* `./postgres/apply/statements.ts`) turns classified changes into steps, and
|
|
10
|
+
* both the applier and this renderer call it. What differs is only where the
|
|
11
|
+
* current schema comes from: here the older build, keyed by export name as
|
|
12
|
+
* `chant sql diff` keys it; in the applier the server's catalog.
|
|
13
|
+
*
|
|
14
|
+
* A change the applier refuses in place is a step naming the Op that makes
|
|
15
|
+
* it, never DDL: a ClickHouse rebuild is `ClickHouseRebuildOp`, a Postgres
|
|
16
|
+
* column rename or type change across kinds is `PostgresMigrationOp`. One no
|
|
17
|
+
* Op makes is a manual step. Column and object drops are statements, marked
|
|
18
|
+
* destructive.
|
|
19
|
+
*
|
|
20
|
+
* Where the applier asks the server something, the renderer cannot:
|
|
21
|
+
*
|
|
22
|
+
* - ClickHouse: the applier also asks the server's formatter whether a
|
|
23
|
+
* difference the rules leave is only formatting; the offline diff is the
|
|
24
|
+
* rules' alone, as `chant sql diff` is. Before dropping a database it
|
|
25
|
+
* checks the database is empty.
|
|
26
|
+
* - Postgres: the applier asks the server to normalize expressions the rules
|
|
27
|
+
* leave different, and keeps an extension's own comment (from its control
|
|
28
|
+
* file) under the trailer when the declaration sets none.
|
|
29
|
+
* - Both: the applier restamps an object whose comment lost chant's marker;
|
|
30
|
+
* offline, the older build's objects carry it.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import type { OwnershipMarker } from "@intentius/chant/ownership";
|
|
34
|
+
import { statementChange, type ManualStep, type MigrationStep, type SchemaStatements, type StepObject } from "./core/statements";
|
|
35
|
+
import { declaredObjects as clickhouseDeclared } from "./clickhouse/apply/apply";
|
|
36
|
+
import { planStatements, type DeclaredObject } from "./clickhouse/apply/statements";
|
|
37
|
+
import { topologyLabel, type Topology } from "./clickhouse/topology";
|
|
38
|
+
import { diffSchemas, type Change } from "./clickhouse/plan/diff";
|
|
39
|
+
import { rebuildOpSuggestions } from "./clickhouse/plan/rebuild-handoff";
|
|
40
|
+
import { declaredObjects as postgresDeclared } from "./postgres/apply/apply";
|
|
41
|
+
import { planPgStatements, type DeclaredPgObject } from "./postgres/apply/statements";
|
|
42
|
+
import { diffPgSchemas, type PgChange } from "./postgres/plan/diff";
|
|
43
|
+
import { pgBuildMajor } from "./postgres/plan/schema";
|
|
44
|
+
import { migrationOpSuggestions, migrationTarget } from "./postgres/migrate/handoff";
|
|
45
|
+
import { POSTGRES_ENTITY_TYPES } from "./postgres/entity-types";
|
|
46
|
+
import { POSTGRES_LATEST_MAJOR } from "./spec/postgres-pin";
|
|
47
|
+
|
|
48
|
+
export type { ManualStep, MigrationStep, OpStep, SchemaStatements, StatementChange, StatementStep, StepObject } from "./core/statements";
|
|
49
|
+
export { renderStatements } from "./core/statements";
|
|
50
|
+
|
|
51
|
+
export interface DiffStatementsOptions {
|
|
52
|
+
/** The ownership marker every `CREATE` and restamped comment carries, as the applier stamps it. Default: `managed-by=chant` alone. */
|
|
53
|
+
marker?: OwnershipMarker;
|
|
54
|
+
/** The environment an Op step's suggested declaration names. Default: `<env>`. */
|
|
55
|
+
env?: string;
|
|
56
|
+
/** ClickHouse: the database a bare name means. Default: `default`. */
|
|
57
|
+
defaultDatabase?: string;
|
|
58
|
+
/**
|
|
59
|
+
* ClickHouse: the topology to render the statements for
|
|
60
|
+
* (`./clickhouse/topology.ts`): `ON CLUSTER` and the engine, on both
|
|
61
|
+
* builds' declarations before they are compared, so the steps are the ones
|
|
62
|
+
* the applier sends to an environment with that topology. Default: `single`.
|
|
63
|
+
*/
|
|
64
|
+
topology?: Topology;
|
|
65
|
+
/** Postgres: the schema a bare name means. Default: `public`. */
|
|
66
|
+
defaultSchema?: string;
|
|
67
|
+
/** Postgres: the major to classify for. Default: the newer build's `postgresMajor`, else the older's, else the newest pinned. */
|
|
68
|
+
major?: number;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
type BuildInput = string | object;
|
|
72
|
+
|
|
73
|
+
const text = (b: BuildInput): string => (typeof b === "string" ? b : JSON.stringify(b));
|
|
74
|
+
|
|
75
|
+
/** The dialect a build output names, at its root or under `sql`. */
|
|
76
|
+
function dialectOf(json: string): string | undefined {
|
|
77
|
+
const raw = JSON.parse(json) as { dialect?: unknown; sql?: { dialect?: unknown } };
|
|
78
|
+
const d = raw?.sql && typeof raw.sql === "object" ? raw.sql.dialect : raw?.dialect;
|
|
79
|
+
return typeof d === "string" ? d : undefined;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The statements that take the schema `before` builds to the one `after`
|
|
84
|
+
* builds. Each is a `chant build` output of the sql lexicon, as JSON text or
|
|
85
|
+
* parsed. Both must be the same dialect.
|
|
86
|
+
*/
|
|
87
|
+
export function diffStatements(before: BuildInput, after: BuildInput, options: DiffStatementsOptions = {}): SchemaStatements {
|
|
88
|
+
const b = text(before);
|
|
89
|
+
const a = text(after);
|
|
90
|
+
const dialect = dialectOf(a);
|
|
91
|
+
if (dialectOf(b) !== dialect) throw new Error(`diffStatements: the builds are different dialects (${dialectOf(b) ?? "none"} and ${dialect ?? "none"})`);
|
|
92
|
+
if (dialect === "postgres") return postgresStatements(b, a, options);
|
|
93
|
+
if (dialect === "clickhouse") return clickhouseStatements(b, a, options);
|
|
94
|
+
throw new Error(`diffStatements: not a sql lexicon build output (dialect ${dialect ?? "missing"})`);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const at = (object: string, type: string, name: string): StepObject => ({ object, type, name });
|
|
98
|
+
|
|
99
|
+
// ── ClickHouse ─────────────────────────────────────────────────────────
|
|
100
|
+
|
|
101
|
+
function clickhouseStatements(beforeJson: string, afterJson: string, options: DiffStatementsOptions): SchemaStatements {
|
|
102
|
+
const defaultDatabase = options.defaultDatabase ?? "default";
|
|
103
|
+
const env = options.env ?? "<env>";
|
|
104
|
+
const topology: Topology = options.topology ?? { kind: "single" };
|
|
105
|
+
const before = clickhouseDeclared(beforeJson, defaultDatabase, topology);
|
|
106
|
+
const after = clickhouseDeclared(afterJson, defaultDatabase, topology);
|
|
107
|
+
const keyed = (objs: readonly DeclaredObject[]) => objs.map((o) => ({ key: o.exportName, canonical: o.canonical }));
|
|
108
|
+
const diff = diffSchemas(keyed(before), keyed(after));
|
|
109
|
+
const plan = planStatements({
|
|
110
|
+
declared: after,
|
|
111
|
+
changes: diff.changes,
|
|
112
|
+
current: new Map(before.map((o) => [o.exportName, o.canonical])),
|
|
113
|
+
keyOf: (o) => o.exportName,
|
|
114
|
+
allowDestructive: true,
|
|
115
|
+
topology,
|
|
116
|
+
...(options.marker ? { marker: options.marker } : {}),
|
|
117
|
+
});
|
|
118
|
+
const declaredByExport = new Map(after.map((o) => [o.exportName, o.canonical]));
|
|
119
|
+
const isDestructive = (changes: readonly Change[], rule: string) => changes.some((c) => c.rule === rule && c.destructive);
|
|
120
|
+
|
|
121
|
+
const steps: MigrationStep[] = [];
|
|
122
|
+
for (const entry of plan.objects) {
|
|
123
|
+
const where = at(entry.obj.exportName, entry.obj.type, entry.obj.key);
|
|
124
|
+
if (entry.verdict === "rebuild") {
|
|
125
|
+
const [op] = rebuildOpSuggestions({ ...diff, rebuilds: entry.refused }, declaredByExport, env);
|
|
126
|
+
if (op) {
|
|
127
|
+
steps.push({
|
|
128
|
+
kind: "op",
|
|
129
|
+
...where,
|
|
130
|
+
op: "ClickHouseRebuildOp",
|
|
131
|
+
importPath: "@intentius/chant-lexicon-sql/clickhouse",
|
|
132
|
+
options: { name: op.name, env: op.env, table: op.table, dualWrite: op.dualWrite },
|
|
133
|
+
declaration: op.declaration,
|
|
134
|
+
// The rebuild creates the new table from the declaration, so it makes every change to it.
|
|
135
|
+
changes: entry.changes.map(statementChange),
|
|
136
|
+
detail: entry.detail,
|
|
137
|
+
});
|
|
138
|
+
} else steps.push({ kind: "manual", ...where, changes: entry.refused.map(statementChange), detail: entry.detail });
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (entry.verdict === "withheld") continue; // not reached: destructive changes are planned
|
|
142
|
+
for (const s of entry.steps) {
|
|
143
|
+
steps.push({
|
|
144
|
+
kind: "statement",
|
|
145
|
+
...where,
|
|
146
|
+
sql: s.sql,
|
|
147
|
+
rule: s.rule,
|
|
148
|
+
class: s.class,
|
|
149
|
+
transactional: false,
|
|
150
|
+
...(s.rewrite ? { waitsForMutation: true } : {}),
|
|
151
|
+
...(s.rule === "SQLCH202" || isDestructive(entry.changes, s.rule) ? { destructive: true } : {}),
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
const beforeByExport = new Map(before.map((o) => [o.exportName, o]));
|
|
156
|
+
for (const d of plan.drops) {
|
|
157
|
+
const o = beforeByExport.get(d.key);
|
|
158
|
+
const change = diff.changes.find((c) => c.object === d.key && c.rule === "SQLCH250");
|
|
159
|
+
steps.push({
|
|
160
|
+
kind: "statement",
|
|
161
|
+
...at(d.key, d.type, o?.key ?? d.name),
|
|
162
|
+
sql: d.step.sql,
|
|
163
|
+
rule: d.step.rule,
|
|
164
|
+
class: d.step.class,
|
|
165
|
+
transactional: false,
|
|
166
|
+
...(change?.destructive ? { destructive: true } : {}),
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
return { dialect: "clickhouse", defaultDatabase, topology: topologyLabel(topology), steps, refused: steps.some((s) => s.kind !== "statement"), hints: diff.hints };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ── Postgres ───────────────────────────────────────────────────────────
|
|
173
|
+
|
|
174
|
+
function postgresStatements(beforeJson: string, afterJson: string, options: DiffStatementsOptions): SchemaStatements {
|
|
175
|
+
const defaultSchema = options.defaultSchema ?? "public";
|
|
176
|
+
const env = options.env ?? "<env>";
|
|
177
|
+
const major = options.major ?? buildMajor(afterJson) ?? buildMajor(beforeJson) ?? POSTGRES_LATEST_MAJOR;
|
|
178
|
+
const before = postgresDeclared(beforeJson, defaultSchema);
|
|
179
|
+
const after = postgresDeclared(afterJson, defaultSchema);
|
|
180
|
+
const keyed = (objs: readonly DeclaredPgObject[]) => objs.map((o) => ({ key: o.exportName, canonical: o.canonical }));
|
|
181
|
+
const current = keyed(before);
|
|
182
|
+
const diff = diffPgSchemas(current, keyed(after), { major });
|
|
183
|
+
const plan = planPgStatements({
|
|
184
|
+
declared: after,
|
|
185
|
+
changes: diff.changes,
|
|
186
|
+
current,
|
|
187
|
+
keyOf: (o) => o.exportName,
|
|
188
|
+
major,
|
|
189
|
+
allowDestructive: true,
|
|
190
|
+
...(options.marker ? { marker: options.marker } : {}),
|
|
191
|
+
});
|
|
192
|
+
const isDestructive = (changes: readonly PgChange[], rule: string | undefined) => changes.some((c) => c.rule === rule && c.destructive);
|
|
193
|
+
|
|
194
|
+
const steps: MigrationStep[] = [];
|
|
195
|
+
for (const entry of plan.objects) {
|
|
196
|
+
const where = at(entry.obj.exportName, entry.obj.type, entry.obj.name);
|
|
197
|
+
switch (entry.verdict) {
|
|
198
|
+
case "refused": {
|
|
199
|
+
const handled = new Set<PgChange>();
|
|
200
|
+
for (const op of migrationOpSuggestions(entry.refused, new Map([[entry.obj.exportName, entry.obj.canonical]]), env, defaultSchema)) {
|
|
201
|
+
const made = entry.refused.filter((c) => {
|
|
202
|
+
const t = migrationTarget(c, entry.obj.canonical, defaultSchema);
|
|
203
|
+
return t !== undefined && t.table === op.table && t.column === op.column;
|
|
204
|
+
});
|
|
205
|
+
for (const c of made) handled.add(c);
|
|
206
|
+
steps.push({
|
|
207
|
+
kind: "op",
|
|
208
|
+
...where,
|
|
209
|
+
op: "PostgresMigrationOp",
|
|
210
|
+
importPath: "@intentius/chant-lexicon-sql/postgres",
|
|
211
|
+
options: { name: op.name, env: op.env, table: op.table, column: op.column },
|
|
212
|
+
declaration: op.declaration,
|
|
213
|
+
changes: made.map(statementChange),
|
|
214
|
+
detail: entry.detail,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
const rest = entry.changes.filter((c) => !handled.has(c));
|
|
218
|
+
if (rest.length > 0) steps.push(manual(where, rest, handled.size > 0 ? `the applier sends nothing for ${entry.obj.name} while a change to it is refused; these changes are not made by the Op: ${entry.detail}` : entry.detail));
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
case "unsupported":
|
|
222
|
+
steps.push(manual(where, entry.changes, entry.detail));
|
|
223
|
+
continue;
|
|
224
|
+
case "foreign":
|
|
225
|
+
case "withheld":
|
|
226
|
+
continue; // not reached offline: a build declares no foreign object, and destructive changes are planned
|
|
227
|
+
}
|
|
228
|
+
for (const s of entry.steps) {
|
|
229
|
+
steps.push({
|
|
230
|
+
kind: "statement",
|
|
231
|
+
...where,
|
|
232
|
+
sql: s.sql,
|
|
233
|
+
rule: s.rule ?? (s.class === "create" ? "SQLPG200" : (entry.changes[0]?.rule ?? "SQLPG200")),
|
|
234
|
+
class: s.class,
|
|
235
|
+
transactional: s.transactional,
|
|
236
|
+
...(s.rule === "SQLPG204" || isDestructive(entry.changes, s.rule) ? { destructive: true } : {}),
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
const beforeByExport = new Map(before.map((o) => [o.exportName, o]));
|
|
241
|
+
for (const d of plan.drops) {
|
|
242
|
+
const o = beforeByExport.get(d.key);
|
|
243
|
+
const change = diff.changes.find((c) => c.object === d.key && (c.rule === "SQLPG270" || c.rule === "SQLPG242"));
|
|
244
|
+
steps.push({
|
|
245
|
+
kind: "statement",
|
|
246
|
+
...at(d.key, o?.type ?? POSTGRES_ENTITY_TYPES[d.kind], o?.name ?? d.name),
|
|
247
|
+
sql: d.step.sql,
|
|
248
|
+
rule: d.step.rule ?? "SQLPG270",
|
|
249
|
+
class: d.step.class,
|
|
250
|
+
transactional: d.step.transactional,
|
|
251
|
+
...(change?.destructive ? { destructive: true } : {}),
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
return { dialect: "postgres", defaultSchema, major, steps, refused: steps.some((s) => s.kind !== "statement"), hints: diff.hints };
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const manual = (where: StepObject, changes: readonly PgChange[], detail: string): ManualStep => ({ kind: "manual", ...where, changes: changes.map(statementChange), detail });
|
|
258
|
+
|
|
259
|
+
function buildMajor(json: string): number | undefined {
|
|
260
|
+
const raw = JSON.parse(json) as { sql?: unknown };
|
|
261
|
+
return pgBuildMajor(raw.sql && typeof raw.sql === "object" ? JSON.stringify(raw.sql) : json);
|
|
262
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
|
|
2
|
+
|
|
3
|
+
exports[`generateOpsPipeline renders the sql drift watch (#3642) > Forgejo: the Op's own cron and the reader's credentials; the runner's token takes no permissions block 1`] = `
|
|
4
|
+
"name: schema-watch
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
schedule:
|
|
8
|
+
- cron: '17 * * * *'
|
|
9
|
+
workflow_dispatch: {}
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: schema-watch
|
|
13
|
+
cancel-in-progress: false
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
schema-watch:
|
|
17
|
+
runs-on: docker
|
|
18
|
+
container: node:22-slim
|
|
19
|
+
env:
|
|
20
|
+
CLICKHOUSE_READER_USER: chant_reader
|
|
21
|
+
CLICKHOUSE_READER_PASSWORD: '\${{ secrets.CLICKHOUSE_READER_PASSWORD }}'
|
|
22
|
+
steps:
|
|
23
|
+
- uses: https://code.forgejo.org/actions/checkout@v7
|
|
24
|
+
- run: npm ci
|
|
25
|
+
- run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
|
|
26
|
+
- run: chant run schema-watch
|
|
27
|
+
env:
|
|
28
|
+
GITHUB_TOKEN: '\${{ github.token }}'
|
|
29
|
+
"
|
|
30
|
+
`;
|
|
31
|
+
|
|
32
|
+
exports[`generateOpsPipeline renders the sql drift watch (#3642) > GitHub: the Op's own cron, a read-only token and the reader's credentials 1`] = `
|
|
33
|
+
"name: schema-watch
|
|
34
|
+
|
|
35
|
+
on:
|
|
36
|
+
schedule:
|
|
37
|
+
- cron: '17 * * * *'
|
|
38
|
+
workflow_dispatch: {}
|
|
39
|
+
|
|
40
|
+
concurrency:
|
|
41
|
+
group: schema-watch
|
|
42
|
+
cancel-in-progress: false
|
|
43
|
+
|
|
44
|
+
permissions:
|
|
45
|
+
contents: read
|
|
46
|
+
|
|
47
|
+
jobs:
|
|
48
|
+
schema-watch:
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
container: node:22-slim
|
|
51
|
+
env:
|
|
52
|
+
CLICKHOUSE_READER_USER: chant_reader
|
|
53
|
+
CLICKHOUSE_READER_PASSWORD: '\${{ secrets.CLICKHOUSE_READER_PASSWORD }}'
|
|
54
|
+
steps:
|
|
55
|
+
- uses: actions/checkout@v7
|
|
56
|
+
- run: npm ci
|
|
57
|
+
- run: echo "$PWD/node_modules/.bin" >> "$GITHUB_PATH"
|
|
58
|
+
- run: chant run schema-watch
|
|
59
|
+
env:
|
|
60
|
+
GITHUB_TOKEN: '\${{ github.token }}'
|
|
61
|
+
"
|
|
62
|
+
`;
|
|
63
|
+
|
|
64
|
+
exports[`generateOpsPipeline renders the sql drift watch (#3642) > GitLab: a schedule-only job with the reader's credentials and no token of its own 1`] = `
|
|
65
|
+
"# chant Ops (#927, #2084, #2293) — one job per Op under stage "ops", each
|
|
66
|
+
# selected by its own rules:. A merge_request_event or push job needs no
|
|
67
|
+
# setup; its rule fires on the event itself.
|
|
68
|
+
#
|
|
69
|
+
# GitLab has no in-file cron. Create one Pipeline Schedule per cron Op
|
|
70
|
+
# below (Settings > CI/CD > Schedules): set its cron to the value noted
|
|
71
|
+
# here and its CHANT_SCHEDULED_OP CI/CD variable to the Op's name, so only
|
|
72
|
+
# that job runs on that schedule.
|
|
73
|
+
#
|
|
74
|
+
# schema-watch: cron "17 * * * *", CHANT_SCHEDULED_OP="schema-watch", finding-mode report
|
|
75
|
+
|
|
76
|
+
stages:
|
|
77
|
+
- ops
|
|
78
|
+
|
|
79
|
+
schema-watch:
|
|
80
|
+
stage: ops
|
|
81
|
+
image: node:22-slim
|
|
82
|
+
resource_group: schema-watch
|
|
83
|
+
variables:
|
|
84
|
+
CLICKHOUSE_READER_USER: chant_reader
|
|
85
|
+
CLICKHOUSE_READER_PASSWORD: '$CLICKHOUSE_READER_PASSWORD'
|
|
86
|
+
rules:
|
|
87
|
+
- if: '$CI_PIPELINE_SOURCE == "schedule" && $CHANT_SCHEDULED_OP == "schema-watch"'
|
|
88
|
+
script:
|
|
89
|
+
- npm ci
|
|
90
|
+
- export PATH="$PWD/node_modules/.bin:$PATH"
|
|
91
|
+
- chant run schema-watch
|
|
92
|
+
"
|
|
93
|
+
`;
|
|
@@ -84,7 +84,7 @@ export async function clickhouseApply(args: ClickHouseApplyArgs, signal?: AbortS
|
|
|
84
84
|
if (!(err instanceof ClickHouseBindingError)) throw err;
|
|
85
85
|
return refused(declaredObjects(json), err.unresolved.reason, err.unresolved.detail);
|
|
86
86
|
}
|
|
87
|
-
const declared = declaredObjects(json, target.defaultDatabase);
|
|
87
|
+
const declared = declaredObjects(json, target.defaultDatabase, target.topology);
|
|
88
88
|
let outcome: ClickHouseApplyOutcome;
|
|
89
89
|
try {
|
|
90
90
|
outcome = await applyClickHouse(target, declared, {
|
|
@@ -59,7 +59,7 @@ export async function rebuildRun(args: ClickHouseRebuildArgs, signal: AbortSigna
|
|
|
59
59
|
const config = deps.config ?? (await loadConfig(cwd));
|
|
60
60
|
const marker = resolveMarker(args, config);
|
|
61
61
|
const target = await bindClickHouse({ ...(args.environment !== undefined ? { environment: args.environment } : {}), config: config ?? {}, ...(deps.env ? { env: deps.env } : {}) });
|
|
62
|
-
const declared = declaredTable(declaredObjects(readFileSync(resolve(cwd, args.buildPath), "utf8"), target.defaultDatabase), args.table);
|
|
62
|
+
const declared = declaredTable(declaredObjects(readFileSync(resolve(cwd, args.buildPath), "utf8"), target.defaultDatabase, target.topology), args.table);
|
|
63
63
|
const runId = currentOpRun()?.runId;
|
|
64
64
|
return {
|
|
65
65
|
target,
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An approval for a plan that creates an object does not survive an edit to
|
|
3
|
+
* that object (#3652), end to end on both dialects.
|
|
4
|
+
*
|
|
5
|
+
* For each dialect, a project in a temporary directory declares a table that
|
|
6
|
+
* does not exist yet and a gated `ApplyOp`. `chant run` stops at the gate and
|
|
7
|
+
* prints the `chant approve ... --plan <digest>` that answers it; the test
|
|
8
|
+
* approves that digest, adds a column to the declared table, and runs again.
|
|
9
|
+
* The gate refuses with "is approved, but not for this plan" and nothing is
|
|
10
|
+
* created. Approving the new plan then applies the edited table.
|
|
11
|
+
*
|
|
12
|
+
* Before #3652 the plan digest was taken over a diff that listed the table by
|
|
13
|
+
* name only, so the second run applied the edited table under the first
|
|
14
|
+
* approval.
|
|
15
|
+
*
|
|
16
|
+
* Everything is written to the database (ClickHouse) or schema (Postgres)
|
|
17
|
+
* `yodel_3652`.
|
|
18
|
+
*
|
|
19
|
+
* The servers: `CHANT_SQL_GATE_CLICKHOUSE_URL` and
|
|
20
|
+
* `CHANT_SQL_GATE_POSTGRES_URL` (a `postgres://user@host:port/db` URL, with
|
|
21
|
+
* `CHANT_SQL_GATE_POSTGRES_PASSWORD`) name running ones, such as the servers
|
|
22
|
+
* `chant emulator up --lexicon sql` starts. Without them, each dialect starts
|
|
23
|
+
* a throwaway server at the same pin, and skips cleanly when Docker is not
|
|
24
|
+
* available.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { execFile } from "node:child_process";
|
|
28
|
+
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
29
|
+
import { tmpdir } from "node:os";
|
|
30
|
+
import { join } from "node:path";
|
|
31
|
+
import { promisify } from "node:util";
|
|
32
|
+
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
|
33
|
+
import { dockerAvailable, startScratchServer, type ScratchServer } from "../clickhouse/container";
|
|
34
|
+
import { clickhouseQuery, type ClickHouseEndpoint } from "../clickhouse/http";
|
|
35
|
+
import { clickhouseImage } from "../spec/pin";
|
|
36
|
+
import { startTestPostgres, type TestPostgres } from "../postgres/testing/server";
|
|
37
|
+
import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../postgres/live/client";
|
|
38
|
+
|
|
39
|
+
const exec = promisify(execFile);
|
|
40
|
+
|
|
41
|
+
const REPO = join(import.meta.dirname, "../../../..");
|
|
42
|
+
const BIN = join(REPO, "node_modules/.bin");
|
|
43
|
+
const NAME = "yodel_3652";
|
|
44
|
+
const OP = "schema-apply";
|
|
45
|
+
|
|
46
|
+
const givenClickHouse = process.env.CHANT_SQL_GATE_CLICKHOUSE_URL;
|
|
47
|
+
const givenPostgres = process.env.CHANT_SQL_GATE_POSTGRES_URL;
|
|
48
|
+
const docker = givenClickHouse && givenPostgres ? false : await dockerAvailable();
|
|
49
|
+
|
|
50
|
+
interface Run {
|
|
51
|
+
code: number;
|
|
52
|
+
output: string;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const strip = (s: string) => s.replace(/\u001b\[[0-9;]*m/g, "");
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* A project directory chant can run in: the repository's `node_modules`
|
|
59
|
+
* linked in, so `@intentius/chant` and the sql lexicon resolve, and a git
|
|
60
|
+
* repository, which the gate ledger and snapshots are written to.
|
|
61
|
+
*/
|
|
62
|
+
async function project(files: Record<string, string>, env: Record<string, string>) {
|
|
63
|
+
const dir = mkdtempSync(join(tmpdir(), "chant-sql-gate-"));
|
|
64
|
+
symlinkSync(join(REPO, "node_modules"), join(dir, "node_modules"), "dir");
|
|
65
|
+
const write = (all: Record<string, string>) => {
|
|
66
|
+
for (const [path, text] of Object.entries(all)) {
|
|
67
|
+
mkdirSync(join(dir, path, ".."), { recursive: true });
|
|
68
|
+
writeFileSync(join(dir, path), text);
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
write({
|
|
72
|
+
"package.json": JSON.stringify({ name: "sql-gate-e2e", private: true, type: "module", scripts: { build: "chant build src --lexicon sql -o dist/schema.json" } }),
|
|
73
|
+
".gitignore": "node_modules\ndist\n.chant\n",
|
|
74
|
+
...files,
|
|
75
|
+
});
|
|
76
|
+
const git = (...args: string[]) => exec("git", ["-c", "user.email=e2e@chant.invalid", "-c", "user.name=e2e", ...args], { cwd: dir });
|
|
77
|
+
await git("init", "-q");
|
|
78
|
+
await git("add", "-A");
|
|
79
|
+
await git("commit", "-qm", "the schema");
|
|
80
|
+
const chant = async (...args: string[]): Promise<Run> => {
|
|
81
|
+
try {
|
|
82
|
+
const r = await exec(join(BIN, "chant"), args, {
|
|
83
|
+
cwd: dir,
|
|
84
|
+
env: { ...process.env, ...env, PATH: `${BIN}:${process.env.PATH ?? ""}` },
|
|
85
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
86
|
+
});
|
|
87
|
+
return { code: 0, output: strip(`${r.stdout}${r.stderr}`) };
|
|
88
|
+
} catch (err) {
|
|
89
|
+
const e = err as { code?: number; stdout?: string; stderr?: string };
|
|
90
|
+
return { code: typeof e.code === "number" ? e.code : 1, output: strip(`${e.stdout ?? ""}${e.stderr ?? ""}`) };
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
return {
|
|
94
|
+
dir,
|
|
95
|
+
chant,
|
|
96
|
+
async edit(path: string, text: string) {
|
|
97
|
+
write({ [path]: text });
|
|
98
|
+
await git("commit", "-qam", `edit ${path}`);
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The `--plan <digest>` the stopped run printed in its approve command. */
|
|
104
|
+
function approvedPlan(run: Run): string {
|
|
105
|
+
const m = /chant approve \S+ \S+ .*?--plan (jcs1-sha256:[0-9a-f]{64})/.exec(run.output);
|
|
106
|
+
if (!m) throw new Error(`the run printed no approve command with a plan:\n${run.output}`);
|
|
107
|
+
return m[1];
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function gateOf(run: Run): string {
|
|
111
|
+
const m = /chant approve \S+ (\S+)/.exec(run.output);
|
|
112
|
+
if (!m) throw new Error(`the run printed no approve command:\n${run.output}`);
|
|
113
|
+
return m[1];
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const OPS = (target: string) => ({
|
|
117
|
+
"ops/apply.op.ts": `import { ApplyOp } from "@intentius/chant/op";
|
|
118
|
+
const { op } = ApplyOp({ name: "${OP}", env: "admin", target: "${target}", gate: { gate: "approve-schema" } });
|
|
119
|
+
export default op;
|
|
120
|
+
`,
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
const config = (sql: Record<string, unknown>) => `import type { ChantConfig } from "@intentius/chant";
|
|
124
|
+
import "@intentius/chant-lexicon-sql";
|
|
125
|
+
|
|
126
|
+
export default {
|
|
127
|
+
lexicons: ["sql"],
|
|
128
|
+
ownership: { stack: "yodel-3652", env: "admin" },
|
|
129
|
+
sql: ${JSON.stringify(sql, null, 2)},
|
|
130
|
+
} satisfies ChantConfig;
|
|
131
|
+
`;
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* The shared scenario: approve a create, edit the declared object, rerun.
|
|
135
|
+
* `columns` reads the live table's column names.
|
|
136
|
+
*/
|
|
137
|
+
async function approveEditRerun(
|
|
138
|
+
p: Awaited<ReturnType<typeof project>>,
|
|
139
|
+
edited: { path: string; text: string },
|
|
140
|
+
columns: () => Promise<string[]>,
|
|
141
|
+
): Promise<void> {
|
|
142
|
+
const first = await p.chant("run", OP);
|
|
143
|
+
expect(first.output).toMatch(/MISSING \(declared, provider reports not in cloud\):[\s\S]*- events .*\[definition [0-9a-f]{64}\]/);
|
|
144
|
+
const gate = gateOf(first);
|
|
145
|
+
const digest = approvedPlan(first);
|
|
146
|
+
|
|
147
|
+
const approve = await p.chant("approve", OP, gate, "--plan", digest);
|
|
148
|
+
expect(approve.code, approve.output).toBe(0);
|
|
149
|
+
|
|
150
|
+
await p.edit(edited.path, edited.text);
|
|
151
|
+
|
|
152
|
+
const second = await p.chant("run", OP);
|
|
153
|
+
expect(second.output).toContain(`Gate "${gate}" is approved, but not for this plan.`);
|
|
154
|
+
expect(second.output).toContain(`approved: ${digest}`);
|
|
155
|
+
expect(second.output).not.toMatch(/Op "schema-apply" completed/);
|
|
156
|
+
expect(await columns()).toEqual([]);
|
|
157
|
+
|
|
158
|
+
// Approving the plan the second run produced applies the edited table.
|
|
159
|
+
const fresh = approvedPlan(second);
|
|
160
|
+
expect(fresh).not.toBe(digest);
|
|
161
|
+
const reapprove = await p.chant("approve", OP, gate, "--plan", fresh);
|
|
162
|
+
expect(reapprove.code, reapprove.output).toBe(0);
|
|
163
|
+
const third = await p.chant("run", OP);
|
|
164
|
+
expect(third.output).toMatch(/Op "schema-apply" completed/);
|
|
165
|
+
expect(await columns()).toContain("note");
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// ── ClickHouse ──────────────────────────────────────────────────────────────
|
|
169
|
+
|
|
170
|
+
const CH_SCHEMA = (extra: string) => `import { database, table } from "@intentius/chant-lexicon-sql/clickhouse";
|
|
171
|
+
|
|
172
|
+
export const db = database\`CREATE DATABASE ${NAME} ENGINE = Atomic COMMENT 'gate binds creates'\`;
|
|
173
|
+
|
|
174
|
+
export const events = table\`
|
|
175
|
+
CREATE TABLE \${db}.events (
|
|
176
|
+
user_id UInt64,
|
|
177
|
+
ts DateTime${extra}
|
|
178
|
+
)
|
|
179
|
+
ENGINE = MergeTree
|
|
180
|
+
ORDER BY (user_id, ts)
|
|
181
|
+
COMMENT 'Raw events'\`;
|
|
182
|
+
`;
|
|
183
|
+
|
|
184
|
+
describe.skipIf(!givenClickHouse && !docker)("a gated ApplyOp over ClickHouse refuses an edited create (#3652)", () => {
|
|
185
|
+
let server: ScratchServer | undefined;
|
|
186
|
+
let admin: ClickHouseEndpoint;
|
|
187
|
+
let p: Awaited<ReturnType<typeof project>>;
|
|
188
|
+
const q = (sql: string) => clickhouseQuery(admin, sql);
|
|
189
|
+
|
|
190
|
+
beforeAll(async () => {
|
|
191
|
+
if (givenClickHouse) admin = { url: givenClickHouse };
|
|
192
|
+
else {
|
|
193
|
+
server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-gate" });
|
|
194
|
+
admin = server.endpoint;
|
|
195
|
+
}
|
|
196
|
+
await q(`DROP DATABASE IF EXISTS ${NAME} SYNC`);
|
|
197
|
+
p = await project(
|
|
198
|
+
{
|
|
199
|
+
...OPS("clickhouse"),
|
|
200
|
+
"chant.config.ts": config({ dialect: "clickhouse", profiles: { admin: { url: admin.url, databases: [NAME] } } }),
|
|
201
|
+
"src/schema.ts": CH_SCHEMA(""),
|
|
202
|
+
},
|
|
203
|
+
{},
|
|
204
|
+
);
|
|
205
|
+
}, 600_000);
|
|
206
|
+
|
|
207
|
+
afterAll(async () => {
|
|
208
|
+
if (admin) await q(`DROP DATABASE IF EXISTS ${NAME} SYNC`).catch(() => undefined);
|
|
209
|
+
await server?.stop();
|
|
210
|
+
if (p) rmSync(p.dir, { recursive: true, force: true });
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
test("approve a create, add a column, rerun: the gate refuses", async () => {
|
|
214
|
+
await approveEditRerun(p, { path: "src/schema.ts", text: CH_SCHEMA(",\n note String") }, async () => {
|
|
215
|
+
const rows = await clickhouseQuery<{ name: string }>(admin, `SELECT name FROM system.columns WHERE database = '${NAME}' AND table = 'events' ORDER BY position`);
|
|
216
|
+
return rows.map((r) => r.name);
|
|
217
|
+
});
|
|
218
|
+
}, 600_000);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
// ── Postgres ────────────────────────────────────────────────────────────────
|
|
222
|
+
|
|
223
|
+
const PG_SCHEMA = (extra: string) => `import { schema, table } from "@intentius/chant-lexicon-sql/postgres";
|
|
224
|
+
|
|
225
|
+
export const app = schema\`CREATE SCHEMA ${NAME};
|
|
226
|
+
COMMENT ON SCHEMA ${NAME} IS 'gate binds creates'\`;
|
|
227
|
+
|
|
228
|
+
export const events = table\`
|
|
229
|
+
CREATE TABLE \${app}.events (
|
|
230
|
+
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
231
|
+
kind text NOT NULL${extra}
|
|
232
|
+
)\`;
|
|
233
|
+
`;
|
|
234
|
+
|
|
235
|
+
describe.skipIf(!givenPostgres && !docker)("a gated ApplyOp over Postgres refuses an edited create (#3652)", () => {
|
|
236
|
+
let server: TestPostgres | undefined;
|
|
237
|
+
let endpoint: PostgresEndpoint;
|
|
238
|
+
let admin: PostgresClient;
|
|
239
|
+
let p: Awaited<ReturnType<typeof project>>;
|
|
240
|
+
const q = (sql: string) => admin.query(sql);
|
|
241
|
+
|
|
242
|
+
beforeAll(async () => {
|
|
243
|
+
if (givenPostgres) endpoint = { url: givenPostgres, password: process.env.CHANT_SQL_GATE_POSTGRES_PASSWORD };
|
|
244
|
+
else {
|
|
245
|
+
server = await startTestPostgres();
|
|
246
|
+
endpoint = server.endpoint();
|
|
247
|
+
}
|
|
248
|
+
admin = await connectPostgres(endpoint);
|
|
249
|
+
await q(`DROP SCHEMA IF EXISTS ${NAME} CASCADE`);
|
|
250
|
+
p = await project(
|
|
251
|
+
{
|
|
252
|
+
...OPS("postgres"),
|
|
253
|
+
"chant.config.ts": config({
|
|
254
|
+
dialect: "postgres",
|
|
255
|
+
profiles: { admin: { url: endpoint.url, password: { env: "ADMIN_PASSWORD" }, schemas: [NAME] } },
|
|
256
|
+
}),
|
|
257
|
+
"src/schema.ts": PG_SCHEMA(""),
|
|
258
|
+
},
|
|
259
|
+
{ ADMIN_PASSWORD: endpoint.password ?? "" },
|
|
260
|
+
);
|
|
261
|
+
}, 600_000);
|
|
262
|
+
|
|
263
|
+
afterAll(async () => {
|
|
264
|
+
if (admin) {
|
|
265
|
+
await q(`DROP SCHEMA IF EXISTS ${NAME} CASCADE`).catch(() => undefined);
|
|
266
|
+
await admin.end();
|
|
267
|
+
}
|
|
268
|
+
await server?.stop();
|
|
269
|
+
if (p) rmSync(p.dir, { recursive: true, force: true });
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
test("approve a create, add a column, rerun: the gate refuses", async () => {
|
|
273
|
+
await approveEditRerun(p, { path: "src/schema.ts", text: PG_SCHEMA(",\n note text") }, async () => {
|
|
274
|
+
const rows = await admin.query<{ column_name: string }>(
|
|
275
|
+
`SELECT column_name FROM information_schema.columns WHERE table_schema = '${NAME}' AND table_name = 'events' ORDER BY ordinal_position`,
|
|
276
|
+
);
|
|
277
|
+
return rows.map((r) => r.column_name);
|
|
278
|
+
});
|
|
279
|
+
}, 600_000);
|
|
280
|
+
});
|