@intentius/chant-lexicon-sql 0.124.54 → 0.124.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capability-plugin.d.ts +160 -0
- package/dist/capability-plugin.d.ts.map +1 -0
- package/dist/capability-plugin.js +370 -0
- package/dist/change-set.d.ts +35 -0
- package/dist/change-set.d.ts.map +1 -1
- package/dist/change-set.js +43 -0
- package/dist/ci-setup.d.ts +34 -0
- package/dist/ci-setup.d.ts.map +1 -0
- package/dist/ci-setup.js +106 -0
- package/dist/clickhouse/access.d.ts +20 -0
- package/dist/clickhouse/access.d.ts.map +1 -1
- package/dist/clickhouse/access.js +42 -1
- package/dist/clickhouse/live/describe-resources.d.ts +3 -1
- package/dist/clickhouse/live/describe-resources.d.ts.map +1 -1
- package/dist/clickhouse/live/describe-resources.js +31 -3
- package/dist/clickhouse/plan/deep.d.ts +42 -11
- package/dist/clickhouse/plan/deep.d.ts.map +1 -1
- package/dist/clickhouse/plan/deep.js +104 -33
- package/dist/exit-codes.d.ts +69 -0
- package/dist/exit-codes.d.ts.map +1 -0
- package/dist/exit-codes.js +105 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -0
- package/dist/integrity.json +2 -2
- package/dist/lock-grain.d.ts +27 -0
- package/dist/lock-grain.d.ts.map +1 -0
- package/dist/lock-grain.js +13 -0
- package/dist/manifest.json +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin.js +8 -0
- package/dist/postgres/testing/migration.d.ts +2 -0
- package/dist/postgres/testing/migration.d.ts.map +1 -1
- package/dist/token-source.d.ts +28 -0
- package/dist/token-source.d.ts.map +1 -1
- package/dist/token-source.js +70 -1
- package/dist/yodel-run.d.ts +58 -0
- package/dist/yodel-run.d.ts.map +1 -0
- package/dist/yodel-run.js +145 -0
- package/package.json +2 -2
- package/src/capability-plugin.test.ts +317 -0
- package/src/capability-plugin.ts +515 -0
- package/src/change-set.ts +74 -0
- package/src/ci-setup.test.ts +67 -0
- package/src/ci-setup.ts +116 -0
- package/src/clickhouse/access.ts +49 -2
- package/src/clickhouse/live/access.e2e.test.ts +8 -7
- package/src/clickhouse/live/describe-resources.ts +27 -3
- package/src/clickhouse/plan/deep-grants-drift.test.ts +151 -0
- package/src/clickhouse/plan/deep-grants.test.ts +10 -5
- package/src/clickhouse/plan/deep.ts +103 -32
- package/src/exit-codes.test.ts +46 -0
- package/src/exit-codes.ts +140 -0
- package/src/index.ts +28 -0
- package/src/lock-grain.test.ts +57 -0
- package/src/lock-grain.ts +37 -0
- package/src/plugin.ts +9 -0
- package/src/testdata/fake-yodel.mjs +58 -0
- package/src/token-source-traded.test.ts +75 -0
- package/src/token-source.ts +74 -1
- package/src/yodel-run.ts +175 -0
|
@@ -18,10 +18,15 @@
|
|
|
18
18
|
* server's, that copy is what is reported, with the server in `observedOn`,
|
|
19
19
|
* which the drift report prints as `seen on`.
|
|
20
20
|
*
|
|
21
|
-
* Where the profile manages access,
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
21
|
+
* Where the profile manages access, each grant declaration is read as
|
|
22
|
+
* `grants.<grantee>.<atom>` (#4582): a privilege or role it declares that the
|
|
23
|
+
* grantee does not hold (revoked by hand, SQLCH273) is `absent` drift on that
|
|
24
|
+
* declaration. A grant declaration whose grantee the server does not have is
|
|
25
|
+
* not read here; the thin diff reports it missing. A privilege a grantee holds
|
|
26
|
+
* that no declaration gives (granted by hand, SQLCH274) belongs to no one
|
|
27
|
+
* declaration, so it is reported as `pending` (#3733) with every statement an
|
|
28
|
+
* apply would send that grantee, as `chant sql plan` plans them. Either way a
|
|
29
|
+
* privilege granted or revoked by hand after an approval moves the plan digest.
|
|
25
30
|
*/
|
|
26
31
|
|
|
27
32
|
import {
|
|
@@ -39,7 +44,7 @@ import { sqlString } from "../apply/statements";
|
|
|
39
44
|
import { stripMarkerFromStatement } from "../ownership";
|
|
40
45
|
import { database, dictionary, func, policy, quota, role, settingsProfile, table, user, view, CLICKHOUSE_ENTITY_TYPES, type ColumnDef } from "../entities";
|
|
41
46
|
import { canonicalObject, grantsObject, objectKey, type CanonicalColumn, type CanonicalObject } from "./normalize";
|
|
42
|
-
import { grantStatement, grantsByGrantee, revokeStatement, type GrantAtom } from "../access";
|
|
47
|
+
import { atomKey, declaredGrantAtoms, grantStatement, grantsByGrantee, holdsAtom, revokeStatement, type GrantAtom } from "../access";
|
|
43
48
|
import { renderFor } from "../topology";
|
|
44
49
|
|
|
45
50
|
type Props = Record<string, unknown>;
|
|
@@ -47,8 +52,48 @@ type Props = Record<string, unknown>;
|
|
|
47
52
|
// `concurrently` is how a Postgres index is created, not a property of it.
|
|
48
53
|
const PRUNED = new Set(["ddl", "source", "lineage", "reads", "to", "orReplace", "ifNotExists", "concurrently"]);
|
|
49
54
|
|
|
55
|
+
/** What a held grant atom reads as, on both sides of the diff. */
|
|
56
|
+
const GRANTED = "granted";
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* A grant declaration's tree as the diff compares it (#4582):
|
|
60
|
+
* `{ grants: { <grantee>: { <atom>: "granted" } } }`, each atom as the
|
|
61
|
+
* declaration spells it. The reader writes the live side in the same shape,
|
|
62
|
+
* with the atoms the grantee holds. Undefined when the statement does not
|
|
63
|
+
* parse as a grant.
|
|
64
|
+
*/
|
|
65
|
+
export function declaredGrantTree(ddl: string): Props | undefined {
|
|
66
|
+
try {
|
|
67
|
+
const grants: Record<string, Record<string, string>> = {};
|
|
68
|
+
for (const { grantee, key } of declaredGrantAtoms(ddl)) (grants[grantee] ??= {})[key] = GRANTED;
|
|
69
|
+
return { grants };
|
|
70
|
+
} catch {
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A grant declaration's live tree (#4582): each atom it declares that the
|
|
77
|
+
* grantee holds, keyed as {@link declaredGrantTree} keys it. Undefined when
|
|
78
|
+
* the server does not have one of its grantees.
|
|
79
|
+
*/
|
|
80
|
+
export function liveGrantTree(ddl: string, held: ReadonlyMap<string, readonly GrantAtom[]>, defaultDatabase: string): Props | undefined {
|
|
81
|
+
const grants: Record<string, Record<string, string>> = {};
|
|
82
|
+
for (const { grantee, key, atom } of declaredGrantAtoms(ddl, defaultDatabase)) {
|
|
83
|
+
const have = held.get(grantee);
|
|
84
|
+
if (!have) return undefined;
|
|
85
|
+
if (holdsAtom(have, atom)) (grants[grantee] ??= {})[key] = GRANTED;
|
|
86
|
+
}
|
|
87
|
+
return Object.keys(grants).length > 0 ? { grants } : {};
|
|
88
|
+
}
|
|
89
|
+
|
|
50
90
|
export const sqlDeepNormalizationHooks: DeepNormalizationHooks = {
|
|
51
91
|
prune: (node) => PRUNED.has(node.pattern.split(/[.[]/)[0]!),
|
|
92
|
+
// A grant declaration is compared atom by atom (#4582); the reader writes the live side in this shape.
|
|
93
|
+
prepare: (tree, { entityType, side }) => {
|
|
94
|
+
if (entityType !== CLICKHOUSE_ENTITY_TYPES.grant || side !== "declared" || typeof tree.ddl !== "string") return tree;
|
|
95
|
+
return declaredGrantTree(tree.ddl) ?? tree;
|
|
96
|
+
},
|
|
52
97
|
};
|
|
53
98
|
|
|
54
99
|
function liveProps(o: LiveObject): Props {
|
|
@@ -259,9 +304,11 @@ export async function observeResourcesDeep(
|
|
|
259
304
|
return deepObservation({}, unobserved);
|
|
260
305
|
}
|
|
261
306
|
const pending: DeepPendingChange[] = [];
|
|
307
|
+
let held: Map<string, GrantAtom[]> | undefined;
|
|
262
308
|
if (target.access === true) {
|
|
263
309
|
try {
|
|
264
|
-
|
|
310
|
+
held = await heldGrants(target, options.entities);
|
|
311
|
+
pending.push(...(await pendingGrants(target, options.entities, held)));
|
|
265
312
|
} catch (err) {
|
|
266
313
|
const why = classifyClickHouseFailure(err);
|
|
267
314
|
for (const name of options.entityNames) {
|
|
@@ -281,12 +328,15 @@ export async function observeResourcesDeep(
|
|
|
281
328
|
continue;
|
|
282
329
|
}
|
|
283
330
|
if (entity.entityType === CLICKHOUSE_ENTITY_TYPES.grant) {
|
|
284
|
-
if (unobserved[name]) continue;
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
331
|
+
if (unobserved[name] || !held) continue;
|
|
332
|
+
const ddl = String(entity.props.ddl ?? "");
|
|
333
|
+
try {
|
|
334
|
+
const properties = liveGrantTree(ddl, held, target.defaultDatabase);
|
|
335
|
+
// A grantee the server does not have: the thin diff reports the declaration missing.
|
|
336
|
+
if (properties) resources[name] = { type: entity.entityType, physicalId: `grants ${[...new Set(declaredGrantAtoms(ddl).map((a) => a.grantee))].join(", ")}`, properties };
|
|
337
|
+
} catch (err) {
|
|
338
|
+
unobserved[name] = { type: entity.entityType, reason: "read-failed", detail: `the grant does not parse: ${(err as Error).message}` };
|
|
339
|
+
}
|
|
290
340
|
continue;
|
|
291
341
|
}
|
|
292
342
|
const props = entity.props;
|
|
@@ -331,37 +381,58 @@ export async function observeResourcesDeep(
|
|
|
331
381
|
return deepObservation(resources, unobserved, pending);
|
|
332
382
|
}
|
|
333
383
|
|
|
384
|
+
const grantDeclarations = (entities: ReadonlyMap<string, { entityType: string; props: Record<string, unknown> }>) =>
|
|
385
|
+
grantsByGrantee([...entities].map(([name, e]) => ({ export: name, type: e.entityType, ddl: String(e.props.ddl ?? "") })));
|
|
386
|
+
|
|
334
387
|
/**
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
*
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
388
|
+
* What the server holds for each declared grantee, as `SHOW GRANTS FOR`
|
|
389
|
+
* prints it, folded into atoms. A grantee the server does not have is left
|
|
390
|
+
* out.
|
|
391
|
+
*/
|
|
392
|
+
export async function heldGrants(target: ClickHouseTarget, entities: ReadonlyMap<string, { entityType: string; props: Record<string, unknown> }>): Promise<Map<string, GrantAtom[]>> {
|
|
393
|
+
const declared = grantDeclarations(entities);
|
|
394
|
+
if (declared.length === 0) return new Map();
|
|
395
|
+
const live = await readLiveAccess(target, declared.map((g) => ({ kind: "grants", name: g.grantee })));
|
|
396
|
+
return new Map(
|
|
397
|
+
live
|
|
398
|
+
.filter((o) => o.type === CLICKHOUSE_ENTITY_TYPES.grant)
|
|
399
|
+
.map((o) => [o.name, Object.values(grantsObject(o.statement, o.name, target.defaultDatabase).access ?? {}).map((json) => JSON.parse(json) as GrantAtom)]),
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* The grantees holding a privilege or role no declaration gives them
|
|
405
|
+
* (#3733), as `chant sql plan` compares them: the declared grants against
|
|
406
|
+
* what `SHOW GRANTS FOR` prints, atom by atom. One change per such grantee,
|
|
407
|
+
* every statement an apply would send it, its REVOKEs before its GRANTs as
|
|
408
|
+
* the applier sends them, with the grant declarations behind it. A grantee
|
|
409
|
+
* that only lacks declared grants has nothing pending: each declaration
|
|
410
|
+
* reports those as `absent` drift of its own (#4582). A grantee the server
|
|
411
|
+
* does not have is the thin diff's missing.
|
|
341
412
|
*/
|
|
342
413
|
export async function pendingGrants(
|
|
343
414
|
target: ClickHouseTarget,
|
|
344
415
|
entities: ReadonlyMap<string, { entityType: string; props: Record<string, unknown> }>,
|
|
416
|
+
held?: ReadonlyMap<string, readonly GrantAtom[]>,
|
|
345
417
|
): Promise<DeepPendingChange[]> {
|
|
346
|
-
const declared =
|
|
418
|
+
const declared = grantDeclarations(entities);
|
|
347
419
|
if (declared.length === 0) return [];
|
|
348
|
-
const live =
|
|
349
|
-
const held = new Map(live.filter((o) => o.type === CLICKHOUSE_ENTITY_TYPES.grant).map((o) => [o.name, grantsObject(o.statement, o.name, target.defaultDatabase).access ?? {}]));
|
|
420
|
+
const live = held ?? (await heldGrants(target, entities));
|
|
350
421
|
const atom = (json: string) => JSON.parse(json) as GrantAtom;
|
|
351
422
|
const out: DeepPendingChange[] = [];
|
|
352
423
|
for (const g of declared) {
|
|
424
|
+
const atoms = live.get(g.grantee);
|
|
425
|
+
if (!atoms) continue;
|
|
426
|
+
const have = Object.fromEntries(atoms.map((a) => [atomKey(a), JSON.stringify(a)]));
|
|
353
427
|
const want = grantsObject(g.ddl, g.grantee, target.defaultDatabase).access ?? {};
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
];
|
|
363
|
-
if (statements.length === 0) continue;
|
|
364
|
-
out.push({ subject: `grants ${g.grantee}`, change: statements.join("; "), entities: [...g.exports].sort() });
|
|
428
|
+
const revokes = Object.keys(have)
|
|
429
|
+
.filter((k) => !(k in want))
|
|
430
|
+
.map((k) => revokeStatement(atom(have[k]!), g.grantee));
|
|
431
|
+
if (revokes.length === 0) continue;
|
|
432
|
+
const grants = Object.keys(want)
|
|
433
|
+
.filter((k) => !(k in have))
|
|
434
|
+
.map((k) => grantStatement(atom(want[k]!), g.grantee));
|
|
435
|
+
out.push({ subject: `grants ${g.grantee}`, change: [...revokes, ...grants].join("; "), entities: [...g.exports].sort() });
|
|
365
436
|
}
|
|
366
437
|
return out;
|
|
367
438
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { describe, expect, test } from "vitest";
|
|
2
|
+
import { CHANT_EXIT } from "@intentius/chant/exit-codes";
|
|
3
|
+
import { sqlExitOf, sqlPlanMoved, YODEL_LEGACY_EXIT, yodelExitOf } from "./exit-codes";
|
|
4
|
+
|
|
5
|
+
describe("SQL engine outcomes on chant's table (#4570)", () => {
|
|
6
|
+
test("a moved wave plan and an approval of another digest are moved", () => {
|
|
7
|
+
expect(sqlExitOf("plan-moved")).toBe(CHANT_EXIT.moved);
|
|
8
|
+
expect(sqlExitOf("approval-moved")).toBe(CHANT_EXIT.moved);
|
|
9
|
+
expect(sqlPlanMoved("plan-moved")).toBe(true);
|
|
10
|
+
expect(sqlPlanMoved("checksum-mismatch")).toBe(false);
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
test("denial, lock, promotion and drift each have their own row", () => {
|
|
14
|
+
expect(sqlExitOf("policy-denied")).toBe(CHANT_EXIT.denied);
|
|
15
|
+
expect(sqlExitOf("lock-held")).toBe(CHANT_EXIT.held);
|
|
16
|
+
expect(sqlExitOf("promotion-unmet")).toBe(CHANT_EXIT.waiting);
|
|
17
|
+
expect(sqlExitOf("pending")).toBe(CHANT_EXIT.waiting);
|
|
18
|
+
expect(sqlExitOf("drift")).toBe(CHANT_EXIT.drift);
|
|
19
|
+
expect(sqlExitOf("stood-down")).toBe(CHANT_EXIT.superseded);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("a refusal no approval fixes is failed, lint errors included", () => {
|
|
23
|
+
for (const o of ["checksum-mismatch", "fork", "out-of-order", "directory", "shared-identity", "reader-write", "lint-errors"] as const) {
|
|
24
|
+
expect(sqlExitOf(o)).toBe(CHANT_EXIT.failed);
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
describe("yodel exit statuses (#4570)", () => {
|
|
30
|
+
test("a yodel on chant's table passes through", () => {
|
|
31
|
+
for (const code of Object.values(CHANT_EXIT)) expect(yodelExitOf({ code })).toBe(code);
|
|
32
|
+
expect(yodelExitOf({ code: 99 })).toBe(CHANT_EXIT.failed);
|
|
33
|
+
expect(yodelExitOf({ code: null })).toBe(CHANT_EXIT.failed);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test("a legacy yodel's codes are read by meaning", () => {
|
|
37
|
+
const legacy = true;
|
|
38
|
+
expect(yodelExitOf({ code: YODEL_LEGACY_EXIT.needsOp, command: "drift", legacy })).toBe(CHANT_EXIT.drift);
|
|
39
|
+
expect(yodelExitOf({ code: YODEL_LEGACY_EXIT.needsOp, command: "apply", legacy })).toBe(CHANT_EXIT.waiting);
|
|
40
|
+
expect(yodelExitOf({ code: YODEL_LEGACY_EXIT.outstanding, command: "lint", legacy })).toBe(CHANT_EXIT.failed);
|
|
41
|
+
expect(yodelExitOf({ code: YODEL_LEGACY_EXIT.outstanding, command: "status", legacy })).toBe(CHANT_EXIT.waiting);
|
|
42
|
+
expect(yodelExitOf({ code: YODEL_LEGACY_EXIT.refused, command: "apply", legacy })).toBe(CHANT_EXIT.failed);
|
|
43
|
+
expect(yodelExitOf({ code: YODEL_LEGACY_EXIT.locked, legacy })).toBe(CHANT_EXIT.held);
|
|
44
|
+
expect(yodelExitOf({ code: 0, legacy })).toBe(CHANT_EXIT.done);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQL engine outcomes mapped onto chant's exit-code table (#4553, `CHANT_EXIT`
|
|
3
|
+
* in `@intentius/chant/exit-codes`), for the versioned migration engine chant
|
|
4
|
+
* runs through the sql capability (sql-yodeler, `yodel`).
|
|
5
|
+
*
|
|
6
|
+
* sql-yodeler from sql-yodeler#288 on exits by chant's table, so its codes
|
|
7
|
+
* pass through. An older yodel has its own table, where 3 means anything
|
|
8
|
+
* outstanding (lint errors included) and 4 every refusal; {@link yodelExitOf}
|
|
9
|
+
* reads one of those with the command it came from.
|
|
10
|
+
*
|
|
11
|
+
* | Engine outcome | Table |
|
|
12
|
+
* |---|---|
|
|
13
|
+
* | applied, nothing pending, stood down at plan time | `done` |
|
|
14
|
+
* | a wave plan that moved (`WavePlanMoved`), an approval of another digest | `moved` |
|
|
15
|
+
* | a policy rule denies the plan and no override counts | `denied` |
|
|
16
|
+
* | another apply holds the history lock | `held` |
|
|
17
|
+
* | a newer commit on the branch applies the same environment (stand-down) | `superseded` |
|
|
18
|
+
* | an apply stopped at its gate, pending migrations under `yodel status`, a migration the required environment has not applied, a change that needs an Op the engine does not run, a manual step | `waiting` |
|
|
19
|
+
* | drift found by `yodel drift` | `drift` |
|
|
20
|
+
* | a checksum mismatch, a fork, an out-of-order migration, an applied migration gone from the directory, reader and writer one identity, a write run with the reader, lint errors | `failed` |
|
|
21
|
+
*/
|
|
22
|
+
import { CHANT_EXIT, type ChantExitCode } from "@intentius/chant/exit-codes";
|
|
23
|
+
|
|
24
|
+
/** An outcome of the versioned migration engine, by name. */
|
|
25
|
+
export type SqlEngineOutcome =
|
|
26
|
+
| "applied"
|
|
27
|
+
| "unchanged"
|
|
28
|
+
| "plan-moved"
|
|
29
|
+
| "approval-moved"
|
|
30
|
+
| "policy-denied"
|
|
31
|
+
| "lock-held"
|
|
32
|
+
| "stood-down"
|
|
33
|
+
| "gated"
|
|
34
|
+
| "pending"
|
|
35
|
+
| "promotion-unmet"
|
|
36
|
+
| "needs-op"
|
|
37
|
+
| "manual-step"
|
|
38
|
+
| "drift"
|
|
39
|
+
| "checksum-mismatch"
|
|
40
|
+
| "fork"
|
|
41
|
+
| "out-of-order"
|
|
42
|
+
| "directory"
|
|
43
|
+
| "shared-identity"
|
|
44
|
+
| "reader-write"
|
|
45
|
+
| "lint-errors"
|
|
46
|
+
| "failed";
|
|
47
|
+
|
|
48
|
+
const OUTCOME_EXIT: Record<SqlEngineOutcome, ChantExitCode> = {
|
|
49
|
+
applied: CHANT_EXIT.done,
|
|
50
|
+
unchanged: CHANT_EXIT.done,
|
|
51
|
+
"plan-moved": CHANT_EXIT.moved,
|
|
52
|
+
"approval-moved": CHANT_EXIT.moved,
|
|
53
|
+
"policy-denied": CHANT_EXIT.denied,
|
|
54
|
+
"lock-held": CHANT_EXIT.held,
|
|
55
|
+
"stood-down": CHANT_EXIT.superseded,
|
|
56
|
+
gated: CHANT_EXIT.waiting,
|
|
57
|
+
pending: CHANT_EXIT.waiting,
|
|
58
|
+
"promotion-unmet": CHANT_EXIT.waiting,
|
|
59
|
+
"needs-op": CHANT_EXIT.waiting,
|
|
60
|
+
"manual-step": CHANT_EXIT.waiting,
|
|
61
|
+
drift: CHANT_EXIT.drift,
|
|
62
|
+
"checksum-mismatch": CHANT_EXIT.failed,
|
|
63
|
+
fork: CHANT_EXIT.failed,
|
|
64
|
+
"out-of-order": CHANT_EXIT.failed,
|
|
65
|
+
directory: CHANT_EXIT.failed,
|
|
66
|
+
"shared-identity": CHANT_EXIT.failed,
|
|
67
|
+
"reader-write": CHANT_EXIT.failed,
|
|
68
|
+
"lint-errors": CHANT_EXIT.failed,
|
|
69
|
+
failed: CHANT_EXIT.failed,
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The table's code for an engine outcome. A refusal no approval fixes is
|
|
74
|
+
* `failed`; one a person can clear (a gate, an unmet promotion) is `waiting`.
|
|
75
|
+
*/
|
|
76
|
+
export function sqlExitOf(outcome: SqlEngineOutcome): ChantExitCode {
|
|
77
|
+
return OUTCOME_EXIT[outcome];
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Whether `outcome` is a plan that moved since its approval. */
|
|
81
|
+
export function sqlPlanMoved(outcome: SqlEngineOutcome): boolean {
|
|
82
|
+
return sqlExitOf(outcome) === CHANT_EXIT.moved;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** yodel's own exit table before sql-yodeler#288 (its `src/exit.ts` up to 0.10). */
|
|
86
|
+
export const YODEL_LEGACY_EXIT = {
|
|
87
|
+
ok: 0,
|
|
88
|
+
error: 1,
|
|
89
|
+
/** Something yodel will not do itself; drift, for `yodel drift`. */
|
|
90
|
+
needsOp: 2,
|
|
91
|
+
/** Something outstanding: a gate, pending migrations, lint errors. */
|
|
92
|
+
outstanding: 3,
|
|
93
|
+
/** Refused, nothing done: every refusal, policy denials and unmet promotions included. */
|
|
94
|
+
refused: 4,
|
|
95
|
+
locked: 5,
|
|
96
|
+
} as const;
|
|
97
|
+
|
|
98
|
+
/** A `yodel` exit status and the command it came from. */
|
|
99
|
+
export interface YodelExitSource {
|
|
100
|
+
code: number | null | undefined;
|
|
101
|
+
/** The yodel verb (`apply`, `status`, `lint`, `drift`, ...). */
|
|
102
|
+
command?: string;
|
|
103
|
+
/** The yodel predates sql-yodeler#288 and exits by {@link YODEL_LEGACY_EXIT}. */
|
|
104
|
+
legacy?: boolean;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The table's code for a `yodel` exit status. A yodel on chant's table passes
|
|
109
|
+
* through (a code outside the table is `failed`). A legacy yodel's codes are
|
|
110
|
+
* read by meaning:
|
|
111
|
+
*
|
|
112
|
+
* | legacy yodel | Table |
|
|
113
|
+
* |---|---|
|
|
114
|
+
* | 0 | `done` |
|
|
115
|
+
* | 2 from `yodel drift` | `drift` |
|
|
116
|
+
* | 2 from any other verb, a change that needs an Op or a manual step | `waiting` |
|
|
117
|
+
* | 3 from `yodel lint` | `failed` (sql-yodeler#285 ruling C) |
|
|
118
|
+
* | 3 from any other verb | `waiting` |
|
|
119
|
+
* | 4, any refusal | `failed`: the legacy code does not tell a policy denial or an unmet promotion apart |
|
|
120
|
+
* | 5 | `held` |
|
|
121
|
+
* | 1, anything else | `failed` |
|
|
122
|
+
*/
|
|
123
|
+
export function yodelExitOf(source: YodelExitSource): ChantExitCode {
|
|
124
|
+
const { code, command, legacy } = source;
|
|
125
|
+
if (code === 0) return CHANT_EXIT.done;
|
|
126
|
+
if (!legacy) {
|
|
127
|
+
for (const value of Object.values(CHANT_EXIT)) if (value === code) return value;
|
|
128
|
+
return CHANT_EXIT.failed;
|
|
129
|
+
}
|
|
130
|
+
switch (code) {
|
|
131
|
+
case YODEL_LEGACY_EXIT.needsOp:
|
|
132
|
+
return command === "drift" ? CHANT_EXIT.drift : CHANT_EXIT.waiting;
|
|
133
|
+
case YODEL_LEGACY_EXIT.outstanding:
|
|
134
|
+
return command === "lint" ? CHANT_EXIT.failed : CHANT_EXIT.waiting;
|
|
135
|
+
case YODEL_LEGACY_EXIT.locked:
|
|
136
|
+
return CHANT_EXIT.held;
|
|
137
|
+
default:
|
|
138
|
+
return CHANT_EXIT.failed;
|
|
139
|
+
}
|
|
140
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -47,3 +47,31 @@ export {
|
|
|
47
47
|
type SqlFileOptions,
|
|
48
48
|
type SqlFileSource,
|
|
49
49
|
} from "./sql-files";
|
|
50
|
+
|
|
51
|
+
// SQL engine and yodel outcomes on chant's exit-code table (#4570); also at
|
|
52
|
+
// `@intentius/chant-lexicon-sql/exit-codes`.
|
|
53
|
+
export { sqlExitOf, sqlPlanMoved, yodelExitOf, YODEL_LEGACY_EXIT, type SqlEngineOutcome, type YodelExitSource } from "./exit-codes";
|
|
54
|
+
|
|
55
|
+
// The `sql-migrate` and `sql-apply` capabilities (#4569), contributed to core's
|
|
56
|
+
// capability-plugin seam: core loads `sqlCapabilityPlugin` when a project's
|
|
57
|
+
// chant.config lists this lexicon, so op waves and pr-plan plan and apply a
|
|
58
|
+
// SQL member through chant. sql-yodeler's engine runs by a fixed argv list
|
|
59
|
+
// (`YODEL_COMMANDS`).
|
|
60
|
+
export {
|
|
61
|
+
sqlCapabilityPlugin,
|
|
62
|
+
sqlMigrateCapability,
|
|
63
|
+
sqlApplyCapability,
|
|
64
|
+
createSqlMigrateCapability,
|
|
65
|
+
createSqlApplyCapability,
|
|
66
|
+
SqlPlanMovedError,
|
|
67
|
+
YodelRunError,
|
|
68
|
+
SQL_VERB_FAMILIES,
|
|
69
|
+
type SqlMigrateStepInput,
|
|
70
|
+
type SqlMigrateStepOutput,
|
|
71
|
+
type SqlMigrateArtifact,
|
|
72
|
+
type SqlApplyStepInput,
|
|
73
|
+
type SqlApplyStepOutput,
|
|
74
|
+
} from "./capability-plugin";
|
|
75
|
+
// The lock grain of a SQL member (#4571): target, with resource refused by name.
|
|
76
|
+
export { sqlDefaultLockGrain, sqlLockGrainRefusal, type SqlLockGrainKind } from "./lock-grain";
|
|
77
|
+
export { YODEL_COMMANDS, YODEL_OPTIONAL_FLAGS, runAllowedYodel, allowedYodelArgv, type YodelRunner, type YodelRun } from "./yodel-run";
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The lock grain of a SQL member (#4571): both kinds default to `target`,
|
|
3
|
+
* refuse `resource` naming the engine, and leave `repo` alone. Resolved
|
|
4
|
+
* through core's `resolvePlannedLock`, as pr-apply and op waves resolve it.
|
|
5
|
+
*/
|
|
6
|
+
import { describe, expect, test } from "vitest";
|
|
7
|
+
import { LockGrainRefusedError, resolvePlannedLock } from "@intentius/chant/lifecycle/lock-rows";
|
|
8
|
+
import { createSqlApplyCapability, createSqlMigrateCapability } from "./capability-plugin";
|
|
9
|
+
import { sqlDefaultLockGrain, sqlLockGrainRefusal } from "./lock-grain";
|
|
10
|
+
|
|
11
|
+
const migrate = createSqlMigrateCapability({ run: async () => ({ code: 0, stdout: "", stderr: "" }) });
|
|
12
|
+
const apply = createSqlApplyCapability({
|
|
13
|
+
plan: async (_input, dialect) => ({ dialect, changes: [], refused: [], holes: [], problems: [] }),
|
|
14
|
+
apply: async () => ({ applied: [], pruned: [], notAttempted: [] }),
|
|
15
|
+
});
|
|
16
|
+
const migrateInput = { environment: "prod", spec: "yodel-waves.json" };
|
|
17
|
+
const applyInput = { environment: "prod", build: "dist/schema.json" };
|
|
18
|
+
const plan = { member: "prod" };
|
|
19
|
+
|
|
20
|
+
describe("the SQL lock grain (#4571)", () => {
|
|
21
|
+
test("the default is target, the environment's history", () => {
|
|
22
|
+
expect(sqlDefaultLockGrain()).toBe("target");
|
|
23
|
+
expect(migrate.defaultLockGrain?.(migrateInput)).toBe("target");
|
|
24
|
+
expect(apply.defaultLockGrain?.(applyInput)).toBe("target");
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("an undeclared member resolves to target from the lexicon, with no rows", async () => {
|
|
28
|
+
for (const [cap, input] of [
|
|
29
|
+
[migrate, migrateInput],
|
|
30
|
+
[apply, applyInput],
|
|
31
|
+
] as const) {
|
|
32
|
+
const lock = await resolvePlannedLock(cap, input, plan, undefined);
|
|
33
|
+
expect(lock).toEqual({ grain: "target", source: "lexicon", rows: [] });
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("resource is refused by name, for both kinds", async () => {
|
|
38
|
+
expect(sqlLockGrainRefusal("sql-migrate", "resource")).toMatch(/SQL Yodeler.*KeeperMap.*advisory lock/);
|
|
39
|
+
expect(sqlLockGrainRefusal("sql-apply", "resource")).toMatch(/creation order/);
|
|
40
|
+
const refused = await resolvePlannedLock(migrate, migrateInput, plan, "resource").catch((e: unknown) => e);
|
|
41
|
+
expect(refused).toBeInstanceOf(LockGrainRefusedError);
|
|
42
|
+
expect((refused as Error).message).toMatch(/^"prod": the resource lock grain is refused: SQL Yodeler/);
|
|
43
|
+
await expect(resolvePlannedLock(apply, applyInput, plan, "resource")).rejects.toThrow(/the resource lock grain is refused: the sql lexicon's applier/);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("target and repo, declared, are taken as declared", async () => {
|
|
47
|
+
expect(sqlLockGrainRefusal("sql-migrate", "target")).toBeUndefined();
|
|
48
|
+
expect(sqlLockGrainRefusal("sql-apply", "repo")).toBeUndefined();
|
|
49
|
+
expect(await resolvePlannedLock(migrate, migrateInput, plan, "repo")).toEqual({ grain: "repo", source: "declared", rows: [] });
|
|
50
|
+
expect(await resolvePlannedLock(apply, applyInput, plan, "target")).toEqual({ grain: "target", source: "declared", rows: [] });
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("neither kind names rows", () => {
|
|
54
|
+
expect(migrate.lockRows).toBeUndefined();
|
|
55
|
+
expect(apply.lockRows).toBeUndefined();
|
|
56
|
+
});
|
|
57
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The lock grain of a SQL member (#4571, on #4552's hooks).
|
|
3
|
+
*
|
|
4
|
+
* A SQL member's grain is `target`: one environment's history. Two applies
|
|
5
|
+
* of one environment are kept apart by the environment's own lock, which
|
|
6
|
+
* sql-yodeler takes for every apply (a KeeperMap row on ClickHouse, a
|
|
7
|
+
* session advisory lock on Postgres), and by chant's target lock on the
|
|
8
|
+
* environment. `sql-apply` sends a whole build in creation order, so it
|
|
9
|
+
* holds the environment too.
|
|
10
|
+
*
|
|
11
|
+
* `resource` is refused by name, as it is on Terraform and stock Pulumi: a
|
|
12
|
+
* migration chain applies in order, and a row per object would let two
|
|
13
|
+
* applies of one chain interleave. Neither kind answers `lockRows`. `repo`
|
|
14
|
+
* is the pipeline's, and any member may take it.
|
|
15
|
+
*
|
|
16
|
+
* sql-yodeler's rendered apply jobs set `APPLY_LOCK` (`YODEL_LOCK_WAIT`,
|
|
17
|
+
* `YODEL_STAND_DOWN`, src/ci/render.ts:323) for the engine's lock wait and
|
|
18
|
+
* its stand-down; dropping it for these hooks is sql-yodeler#293's.
|
|
19
|
+
*/
|
|
20
|
+
import type { LockGrain } from "@intentius/chant/lifecycle/lock-rows";
|
|
21
|
+
|
|
22
|
+
/** The capability kinds this lexicon gives a grain to. */
|
|
23
|
+
export type SqlLockGrainKind = "sql-migrate" | "sql-apply";
|
|
24
|
+
|
|
25
|
+
/** The grain a SQL member takes when `ci.lockGrain` sets none: always `target`. */
|
|
26
|
+
export function sqlDefaultLockGrain(): LockGrain {
|
|
27
|
+
return "target";
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Why a SQL member cannot take `grain`, naming the engine; undefined for `target` and `repo`. */
|
|
31
|
+
export function sqlLockGrainRefusal(kind: SqlLockGrainKind, grain: LockGrain): string | undefined {
|
|
32
|
+
if (grain !== "resource") return undefined;
|
|
33
|
+
if (kind === "sql-migrate") {
|
|
34
|
+
return "SQL Yodeler applies an environment's migrations in chain order under the environment's own lock (a KeeperMap row on ClickHouse, a session advisory lock on Postgres), so nothing in it is per resource (ci.lockGrain: target)";
|
|
35
|
+
}
|
|
36
|
+
return "the sql lexicon's applier sends a whole build in creation order, since its objects depend on each other, so nothing in it is per resource (ci.lockGrain: target)";
|
|
37
|
+
}
|
package/src/plugin.ts
CHANGED
|
@@ -33,6 +33,7 @@ import { looksLikePostgres } from "./migrate/from-postgres/detect";
|
|
|
33
33
|
import { sqlWorkspaceOutputs } from "./workspace-outputs";
|
|
34
34
|
import { CDC_TEMPLATE, DEFAULT_TEMPLATE, EVENTS_TEMPLATE, POSTGRES_EVENTS_TEMPLATE, POSTGRES_TEMPLATE, POSTGRES_TENANT_TEMPLATE } from "./init-templates";
|
|
35
35
|
import { compositeCatalog } from "./composites/catalog";
|
|
36
|
+
import { sqlCiServices, sqlCiSetup } from "./ci-setup";
|
|
36
37
|
|
|
37
38
|
export const sqlPlugin: LexiconPlugin = {
|
|
38
39
|
name: "sql",
|
|
@@ -45,6 +46,14 @@ export const sqlPlugin: LexiconPlugin = {
|
|
|
45
46
|
/** chant's marker is a trailer on the object's comment (#3208, ./clickhouse/ownership.ts). */
|
|
46
47
|
ownershipChannel: SQL_OWNERSHIP_CHANNEL,
|
|
47
48
|
|
|
49
|
+
// #4572: the setup lines for the pull request pipeline's default image, and the replay server per dialect.
|
|
50
|
+
ciSetup(ctx: { dir: string }): string[] {
|
|
51
|
+
return sqlCiSetup(ctx);
|
|
52
|
+
},
|
|
53
|
+
ciServices(ctx: { dir: string }) {
|
|
54
|
+
return sqlCiServices(ctx);
|
|
55
|
+
},
|
|
56
|
+
|
|
48
57
|
async generate(options?: { verbose?: boolean }): Promise<void> {
|
|
49
58
|
const { generate, writeGeneratedFiles } = await import("./codegen/generate");
|
|
50
59
|
writeGeneratedFiles(await generate(options));
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// A fake `yodel` for the sql-migrate tests (#4569). It answers the three
|
|
3
|
+
// commands chant runs, writes the files sql-yodeler would, and appends each
|
|
4
|
+
// call (argv, and whether the writer was in its environment) to
|
|
5
|
+
// $FAKE_YODEL_LOG. $FAKE_YODEL_APPLY_EXIT sets the apply's exit status, and
|
|
6
|
+
// $FAKE_YODEL_WAVE_DIGEST the wave plan's digest.
|
|
7
|
+
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
|
8
|
+
import { dirname, resolve } from "node:path";
|
|
9
|
+
|
|
10
|
+
const args = process.argv.slice(2);
|
|
11
|
+
const at = (flag) => args[args.indexOf(flag) + 1];
|
|
12
|
+
const write = (file, doc) => {
|
|
13
|
+
const path = resolve(file);
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
15
|
+
writeFileSync(path, `${JSON.stringify(doc, null, 2)}\n`);
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
if (process.env.FAKE_YODEL_LOG) {
|
|
19
|
+
appendFileSync(
|
|
20
|
+
process.env.FAKE_YODEL_LOG,
|
|
21
|
+
`${JSON.stringify({ args, role: process.env.YODEL_ROLE ?? null, writer: process.env.DEV_POSTGRES_WRITER_PASSWORD ?? null })}\n`,
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const env = args[1];
|
|
26
|
+
if (args[0] === "plan") {
|
|
27
|
+
write(at("--part"), {
|
|
28
|
+
member: { member: "app", lexicon: "sql", planner: "sql", scope: env, status: "planned", planDigest: "jcs1-sha256:migrations", holes: [] },
|
|
29
|
+
entries: [
|
|
30
|
+
{ member: "app", lexicon: "sql", planner: "sql", address: "app.users", type: "Postgres::Table", action: "update", disruption: "rolling", scope: env, attributes: [{ path: "columns.email", after: "text" }] },
|
|
31
|
+
{ member: "app", lexicon: "sql", planner: "sql", address: "app.audit", type: "Postgres::Table", action: "create", scope: env, attributes: [] },
|
|
32
|
+
],
|
|
33
|
+
});
|
|
34
|
+
process.exit(0);
|
|
35
|
+
}
|
|
36
|
+
if (args[0] === "apply" && args[2] === "--wave-plan") {
|
|
37
|
+
write(at("--wave-plan"), {
|
|
38
|
+
planDigest: process.env.FAKE_YODEL_WAVE_DIGEST ?? "jcs1-sha256:wave",
|
|
39
|
+
destructive: true,
|
|
40
|
+
empty: false,
|
|
41
|
+
environment: env,
|
|
42
|
+
migrations: "jcs1-sha256:migrations",
|
|
43
|
+
gate: "on-destructive",
|
|
44
|
+
base: "0".repeat(40),
|
|
45
|
+
pending: ["0002_users_email"],
|
|
46
|
+
destructiveSteps: [{ migration: "0002_users_email", step: 0, what: "ALTER TABLE app.users", why: "pg-rewrite, pg-lock" }],
|
|
47
|
+
requires: [],
|
|
48
|
+
});
|
|
49
|
+
process.exit(0);
|
|
50
|
+
}
|
|
51
|
+
if (args[0] === "apply" && args[2] === "--wave-apply") {
|
|
52
|
+
const code = Number(process.env.FAKE_YODEL_APPLY_EXIT ?? "0");
|
|
53
|
+
if (code === 0) process.stdout.write("Applied: 0002_users_email\n");
|
|
54
|
+
else process.stderr.write(`refused: ${env}'s wave plan moved since the wave was decided; nothing was applied. Run the wave again.\n`);
|
|
55
|
+
process.exit(code);
|
|
56
|
+
}
|
|
57
|
+
process.stderr.write(`fake yodel: no answer for ${args.join(" ")}\n`);
|
|
58
|
+
process.exit(1);
|