@intentius/chant-lexicon-sql 0.116.0 → 0.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/codegen/registry.d.ts.map +1 -1
- package/dist/config.d.ts +20 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/integrity.json +5 -5
- package/dist/lint/rules/sqlpg001.d.ts.map +1 -1
- package/dist/lsp/postgres-template.d.ts.map +1 -1
- package/dist/manifest.json +1 -1
- package/dist/meta.json +20 -0
- package/dist/okf/index.md +4 -0
- package/dist/okf/types/PostgresDefaultPrivileges.md +9 -0
- package/dist/okf/types/PostgresGrant.md +9 -0
- package/dist/okf/types/PostgresPolicy.md +9 -0
- package/dist/okf/types/PostgresRole.md +9 -0
- package/dist/op/activities/postgres-migration.d.ts.map +1 -1
- package/dist/op/activity-contracts.d.ts +10 -0
- package/dist/op/activity-contracts.d.ts.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/postgres/access/acl.d.ts +119 -0
- package/dist/postgres/access/acl.d.ts.map +1 -0
- package/dist/postgres/access/import.d.ts +13 -0
- package/dist/postgres/access/import.d.ts.map +1 -0
- package/dist/postgres/access/live.d.ts +24 -0
- package/dist/postgres/access/live.d.ts.map +1 -0
- package/dist/postgres/apply/apply.d.ts.map +1 -1
- package/dist/postgres/apply/statements.d.ts +3 -0
- package/dist/postgres/apply/statements.d.ts.map +1 -1
- package/dist/postgres/entities.d.ts +106 -4
- package/dist/postgres/entities.d.ts.map +1 -1
- package/dist/postgres/entity-types.d.ts +4 -0
- package/dist/postgres/entity-types.d.ts.map +1 -1
- package/dist/postgres/import/generator.d.ts +4 -1
- package/dist/postgres/import/generator.d.ts.map +1 -1
- package/dist/postgres/import/live-export.d.ts +6 -0
- package/dist/postgres/import/live-export.d.ts.map +1 -1
- package/dist/postgres/live/bind.d.ts +2 -0
- package/dist/postgres/live/bind.d.ts.map +1 -1
- package/dist/postgres/live/catalog.d.ts +11 -2
- package/dist/postgres/live/catalog.d.ts.map +1 -1
- package/dist/postgres/live/describe-resources.d.ts +9 -0
- package/dist/postgres/live/describe-resources.d.ts.map +1 -1
- package/dist/postgres/migrate/op.d.ts +29 -0
- package/dist/postgres/migrate/op.d.ts.map +1 -1
- package/dist/postgres/parser.d.ts +76 -3
- package/dist/postgres/parser.d.ts.map +1 -1
- package/dist/postgres/plan/commands.d.ts +12 -0
- package/dist/postgres/plan/commands.d.ts.map +1 -1
- package/dist/postgres/plan/deep.d.ts.map +1 -1
- package/dist/postgres/plan/diff.d.ts +10 -1
- package/dist/postgres/plan/diff.d.ts.map +1 -1
- package/dist/postgres/plan/normalize.d.ts +8 -1
- package/dist/postgres/plan/normalize.d.ts.map +1 -1
- package/dist/postgres/plan/rules.d.ts +8 -0
- package/dist/postgres/plan/rules.d.ts.map +1 -1
- package/dist/postgres/plan/schema.d.ts +2 -1
- package/dist/postgres/plan/schema.d.ts.map +1 -1
- package/dist/postgres/plan/server-normalize.d.ts +4 -1
- package/dist/postgres/plan/server-normalize.d.ts.map +1 -1
- package/dist/postgres.d.ts +1 -1
- package/dist/postgres.d.ts.map +1 -1
- package/dist/rules/postgres-templates.ts +2 -2
- package/dist/rules/sqlpg001.ts +7 -4
- package/dist/validate.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/codegen/registry.ts +4 -0
- package/src/config.ts +10 -0
- package/src/generated/lexicon-sql.json +20 -0
- package/src/index.ts +1 -1
- package/src/lint/rules/postgres-templates.ts +2 -2
- package/src/lint/rules/sqlpg001.ts +7 -4
- package/src/lsp/postgres-template.ts +2 -0
- package/src/mcp/postgres.ts +1 -1
- package/src/migration-statements.ts +7 -0
- package/src/op/activities/postgres-migration.ts +27 -2
- package/src/op/activity-contracts.ts +1 -0
- package/src/plugin.ts +3 -0
- package/src/postgres/access/access.e2e.test.ts +191 -0
- package/src/postgres/access/access.test.ts +147 -0
- package/src/postgres/access/acl.ts +400 -0
- package/src/postgres/access/import.ts +64 -0
- package/src/postgres/access/live.ts +123 -0
- package/src/postgres/apply/apply.ts +117 -4
- package/src/postgres/apply/statements.ts +55 -9
- package/src/postgres/entities.ts +287 -11
- package/src/postgres/entity-types.ts +4 -0
- package/src/postgres/import/generator.ts +14 -4
- package/src/postgres/import/live-export.ts +12 -2
- package/src/postgres/live/bind.ts +3 -0
- package/src/postgres/live/catalog.ts +79 -3
- package/src/postgres/live/describe-resources.ts +27 -3
- package/src/postgres/migrate/migrate.test.ts +24 -1
- package/src/postgres/migrate/op.ts +80 -35
- package/src/postgres/migrate/outer.e2e.test.ts +163 -0
- package/src/postgres/parser.ts +247 -8
- package/src/postgres/plan/commands.ts +60 -7
- package/src/postgres/plan/deep.ts +24 -3
- package/src/postgres/plan/diff.ts +71 -6
- package/src/postgres/plan/normalize.ts +115 -2
- package/src/postgres/plan/rules.ts +44 -0
- package/src/postgres/plan/schema.ts +4 -2
- package/src/postgres/plan/server-normalize.ts +29 -1
- package/src/postgres.ts +11 -0
- package/src/skill-defs.ts +1 -1
- package/src/skills/chant-sql-postgres-migration.md +3 -1
- package/src/skills/chant-sql-postgres.md +9 -2
- package/src/validate.ts +4 -0
|
@@ -41,15 +41,30 @@ interface Bound {
|
|
|
41
41
|
*/
|
|
42
42
|
export function declaredAddress(entity: { type: string; props: Record<string, unknown> }, defaultSchema: string): { schema?: string; name: string; signature?: string } {
|
|
43
43
|
const name = String(entity.props.name ?? "");
|
|
44
|
-
if (entity.type === POSTGRES_ENTITY_TYPES.schema || entity.type === POSTGRES_ENTITY_TYPES.extension) return { name };
|
|
44
|
+
if (entity.type === POSTGRES_ENTITY_TYPES.schema || entity.type === POSTGRES_ENTITY_TYPES.extension || entity.type === POSTGRES_ENTITY_TYPES.role) return { name };
|
|
45
45
|
const schema = typeof entity.props.schema === "string" ? entity.props.schema : identValue(defaultSchema);
|
|
46
|
-
if (entity.type === POSTGRES_ENTITY_TYPES.function || entity.type === POSTGRES_ENTITY_TYPES.procedure || entity.type === POSTGRES_ENTITY_TYPES.trigger) {
|
|
46
|
+
if (entity.type === POSTGRES_ENTITY_TYPES.function || entity.type === POSTGRES_ENTITY_TYPES.procedure || entity.type === POSTGRES_ENTITY_TYPES.trigger || entity.type === POSTGRES_ENTITY_TYPES.policy) {
|
|
47
47
|
const signature = canonicalPgObject(entity.type, entity.props, defaultSchema).signature;
|
|
48
48
|
return { schema, name, ...(signature !== undefined ? { signature } : {}) };
|
|
49
49
|
}
|
|
50
50
|
return { schema, name };
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
/**
|
|
54
|
+
* Why an access declaration is not observed one by one: a grant or default
|
|
55
|
+
* privileges, which `chant sql plan` compares as the access they add up to;
|
|
56
|
+
* a policy or role where the profile does not manage access.
|
|
57
|
+
*/
|
|
58
|
+
export function accessUnobserved(type: string, managed: boolean): { reason: "unsupported-kind" | "filtered"; detail: string } | undefined {
|
|
59
|
+
if (type === POSTGRES_ENTITY_TYPES.grant || type === POSTGRES_ENTITY_TYPES.defaultPrivileges) {
|
|
60
|
+
return { reason: "unsupported-kind", detail: "a grant is compared as the privileges the declarations add up to, by chant sql plan and the applier, not one statement at a time" };
|
|
61
|
+
}
|
|
62
|
+
if (!managed && (type === POSTGRES_ENTITY_TYPES.policy || type === POSTGRES_ENTITY_TYPES.role)) {
|
|
63
|
+
return { reason: "filtered", detail: "access is not managed in this environment (sql.profiles.<env>.access)" };
|
|
64
|
+
}
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
|
|
53
68
|
/** A schema scope for the catalog read: the profile's schemas, else every schema the declarations name and the default one. */
|
|
54
69
|
export function scopeFor(target: PostgresTarget, entities: ReadonlyArray<{ type: string; props: Record<string, unknown> }>): string[] | undefined {
|
|
55
70
|
if (target.schemas) return target.schemas;
|
|
@@ -68,7 +83,14 @@ function adapter(options: BindOptions & { owned?: boolean; declared: DeclaredEnt
|
|
|
68
83
|
async bind() {
|
|
69
84
|
const { target, client } = await bindPostgres(options);
|
|
70
85
|
try {
|
|
71
|
-
const live = markProviderOwned(
|
|
86
|
+
const live = markProviderOwned(
|
|
87
|
+
await readLiveSchema(client, {
|
|
88
|
+
schemas: scopeFor(target, options.declared.filter((d) => d.type.startsWith("Postgres::"))),
|
|
89
|
+
access: target.access === true,
|
|
90
|
+
roles: options.declared.filter((d) => d.type === POSTGRES_ENTITY_TYPES.role).map((d) => String(d.props.name)),
|
|
91
|
+
}),
|
|
92
|
+
target.provider,
|
|
93
|
+
);
|
|
72
94
|
return { target, byKey: new Map(live.map((o) => [liveKey(o.type, o.schema, o.name, o.signature), o])) };
|
|
73
95
|
} finally {
|
|
74
96
|
await client.end();
|
|
@@ -77,6 +99,8 @@ function adapter(options: BindOptions & { owned?: boolean; declared: DeclaredEnt
|
|
|
77
99
|
classifyBindFailure: (err) => classifyPostgresFailure(err),
|
|
78
100
|
async read({ target, byKey }, entity): Promise<EntityObservation> {
|
|
79
101
|
if (!entity.type.startsWith("Postgres::")) return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
|
|
102
|
+
const why = accessUnobserved(entity.type, target.access === true);
|
|
103
|
+
if (why) return { unobserved: why };
|
|
80
104
|
const { schema, name, signature } = declaredAddress(entity, target.defaultSchema);
|
|
81
105
|
const queried = `${redactUrl(target.endpoint.url)} ${schema ? `${schema}.` : ""}${name}${signature ?? ""}`;
|
|
82
106
|
const live = byKey.get(liveKey(entity.type, schema, name, signature));
|
|
@@ -72,7 +72,30 @@ describe("PostgresMigrationOp", () => {
|
|
|
72
72
|
expect(Object.keys(g.approval.context)).toEqual(["verifiedRows", "mismatched"]);
|
|
73
73
|
});
|
|
74
74
|
|
|
75
|
+
test('gates: "outer" leaves out both approval gates and the Contract phase, and keeps the verification (#3687)', () => {
|
|
76
|
+
const props = propsOf({ ...BASE, gates: "outer" });
|
|
77
|
+
expect(props.phases.map((p) => p.name)).toEqual(["Build", "Plan", "Expand", "Dual write", "Backfill", "Carry over", "Verify", "Switch", "Retain"]);
|
|
78
|
+
expect(props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "gate")).toEqual([]);
|
|
79
|
+
const fns = props.phases.flatMap((p) => p.steps).map((s) => (s as { fn?: string }).fn);
|
|
80
|
+
expect(fns.indexOf("postgresMigrationVerify")).toBeLessThan(fns.indexOf("postgresMigrationSwitch"));
|
|
81
|
+
expect(fns).not.toContain("postgresMigrationContract");
|
|
82
|
+
expect(props.onFailure?.map((p) => p.steps.map((s) => (s as { fn?: string }).fn))).toEqual([["postgresMigrationCompensate"]]);
|
|
83
|
+
expect(props.overview).toContain("the caller's approval");
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test('onFailure: "keep" has no onFailure and tells every step so (#3687)', () => {
|
|
87
|
+
const props = propsOf({ ...BASE, onFailure: "keep" });
|
|
88
|
+
expect(props.onFailure).toBeUndefined();
|
|
89
|
+
const steps = props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "activity" && (s as { fn: string }).fn.startsWith("postgresMigration")) as Array<{ args: Record<string, unknown> }>;
|
|
90
|
+
expect(steps).toHaveLength(9);
|
|
91
|
+
for (const s of steps) expect(s.args.keepOnFailure).toBe(true);
|
|
92
|
+
expect(props.phases.map((p) => p.name)).toContain("Approve");
|
|
93
|
+
expect((propsOf(BASE).phases[1]!.steps[0] as { args: Record<string, unknown> }).args).not.toHaveProperty("keepOnFailure");
|
|
94
|
+
});
|
|
95
|
+
|
|
75
96
|
test("bad configuration fails when the Op is built", () => {
|
|
97
|
+
expect(() => PostgresMigrationOp({ ...BASE, gates: "none" as never })).toThrow(/gates must be/);
|
|
98
|
+
expect(() => PostgresMigrationOp({ ...BASE, onFailure: "retry" as never })).toThrow(/onFailure must be/);
|
|
76
99
|
expect(() => PostgresMigrationOp({ ...BASE, table: "app.users; DROP" })).toThrow(/table must be/);
|
|
77
100
|
expect(() => PostgresMigrationOp({ ...BASE, column: "" })).toThrow(/column must be/);
|
|
78
101
|
expect(() => PostgresMigrationOp({ ...BASE, batchSize: 0 })).toThrow(/batchSize/);
|
|
@@ -95,7 +118,7 @@ describe("PostgresMigrationOp", () => {
|
|
|
95
118
|
ownershipEnv: "prod",
|
|
96
119
|
build: false,
|
|
97
120
|
};
|
|
98
|
-
for (const config of [BASE, full, { ...BASE, replicationLag: false as const }]) {
|
|
121
|
+
for (const config of [BASE, full, { ...BASE, replicationLag: false as const }, { ...BASE, gates: "outer" as const, onFailure: "keep" as const }]) {
|
|
99
122
|
const props = propsOf(config);
|
|
100
123
|
expect(validateActivitySteps(props, merged)).toEqual([]);
|
|
101
124
|
expect(validateStepOutputRefs(props, merged)).toEqual([]);
|
|
@@ -35,6 +35,12 @@
|
|
|
35
35
|
* machine) is not a failure, runs no onFailure, and the next run resumes the
|
|
36
36
|
* backfill from its receipts; a step that fails runs onFailure, and the next
|
|
37
37
|
* run starts again from a new column.
|
|
38
|
+
*
|
|
39
|
+
* `gates: "outer"` (#3687) leaves out both approval gates and the Contract
|
|
40
|
+
* phase, for a caller that runs the Op under an approval of its own: the run
|
|
41
|
+
* goes from Plan to Retain, verification included. `onFailure: "keep"`
|
|
42
|
+
* leaves out the onFailure, so a failed run's next run resumes the backfill
|
|
43
|
+
* from its receipts instead of starting again.
|
|
38
44
|
*/
|
|
39
45
|
|
|
40
46
|
import { Op, activity, gate, phase, stepOutput, type GateApproval } from "@intentius/chant/op";
|
|
@@ -98,6 +104,27 @@ export interface PostgresMigrationOpConfig {
|
|
|
98
104
|
stack?: string;
|
|
99
105
|
/** Ownership env. Default: `ownership.env` in `chant.config.ts`. */
|
|
100
106
|
ownershipEnv?: string;
|
|
107
|
+
/**
|
|
108
|
+
* Whose approval the switch and the contract wait for. `"own"` (the
|
|
109
|
+
* default): the Op's two gates. `"outer"`: an approval the caller already
|
|
110
|
+
* holds for a larger change that runs this Op as one of its steps. The
|
|
111
|
+
* switch gate, the contract gate and the Contract phase are left out, so
|
|
112
|
+
* the Op ends at Retain with the old column kept until its retention date.
|
|
113
|
+
* Drop it after that date, or run the Op with `"own"`: its switch gate has
|
|
114
|
+
* nothing left to switch, and its contract gate binds that old column. The
|
|
115
|
+
* verification still runs, and any difference fails the run with nothing
|
|
116
|
+
* switched.
|
|
117
|
+
*/
|
|
118
|
+
gates?: "own" | "outer";
|
|
119
|
+
/**
|
|
120
|
+
* What a failed run does with what the expand added. `"drop"` (the
|
|
121
|
+
* default): onFailure drops the new column, the dual-write trigger and the
|
|
122
|
+
* receipts, and the next run starts again from a new column. `"keep"`:
|
|
123
|
+
* there is no onFailure, and the next run resumes the backfill from its
|
|
124
|
+
* per-batch receipts. To start again instead, run the Op once with
|
|
125
|
+
* `"drop"`.
|
|
126
|
+
*/
|
|
127
|
+
onFailure?: "drop" | "keep";
|
|
101
128
|
}
|
|
102
129
|
|
|
103
130
|
export interface PostgresMigrationOpResources {
|
|
@@ -119,6 +146,8 @@ export interface PostgresMigrationArgs {
|
|
|
119
146
|
stack?: string;
|
|
120
147
|
ownershipEnv?: string;
|
|
121
148
|
cwd?: string;
|
|
149
|
+
/** A failed run keeps what the expand added (`onFailure: "keep"`): a refusal says how to start again. */
|
|
150
|
+
keepOnFailure?: boolean;
|
|
122
151
|
}
|
|
123
152
|
|
|
124
153
|
const IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$|^"(?:[^"]|"")+"$/;
|
|
@@ -134,6 +163,14 @@ export function PostgresMigrationOp(config: PostgresMigrationOpConfig): Postgres
|
|
|
134
163
|
}
|
|
135
164
|
if (config.replicationLag) for (const d of [config.replicationLag.max, config.replicationLag.wait]) if (d !== undefined) parseDuration(d);
|
|
136
165
|
if (config.retain !== undefined) parseDuration(config.retain);
|
|
166
|
+
if (config.gates !== undefined && config.gates !== "own" && config.gates !== "outer") {
|
|
167
|
+
throw new Error(`PostgresMigrationOp "${config.name}": gates must be "own" or "outer"`);
|
|
168
|
+
}
|
|
169
|
+
if (config.onFailure !== undefined && config.onFailure !== "drop" && config.onFailure !== "keep") {
|
|
170
|
+
throw new Error(`PostgresMigrationOp "${config.name}": onFailure must be "drop" or "keep"`);
|
|
171
|
+
}
|
|
172
|
+
const outer = config.gates === "outer";
|
|
173
|
+
const keep = config.onFailure === "keep";
|
|
137
174
|
|
|
138
175
|
const args: PostgresMigrationArgs = {
|
|
139
176
|
table: config.table,
|
|
@@ -149,6 +186,7 @@ export function PostgresMigrationOp(config: PostgresMigrationOpConfig): Postgres
|
|
|
149
186
|
...(config.stack ? { stack: config.stack } : {}),
|
|
150
187
|
...(config.ownershipEnv ? { ownershipEnv: config.ownershipEnv } : {}),
|
|
151
188
|
...(config.path && config.path !== "." ? { cwd: config.path } : {}),
|
|
189
|
+
...(keep ? { keepOnFailure: true } : {}),
|
|
152
190
|
};
|
|
153
191
|
const a = args as unknown as Record<string, unknown>;
|
|
154
192
|
const switchGate = config.gate?.gate ?? `approve-${config.name}`;
|
|
@@ -159,6 +197,33 @@ export function PostgresMigrationOp(config: PostgresMigrationOpConfig): Postgres
|
|
|
159
197
|
const step = (fn: string, extra: Partial<StepDefinition & { id: string; outcomeAttribute: unknown; profile: string; timeout: string }> = {}): StepDefinition =>
|
|
160
198
|
({ ...activity(fn, a), ...extra }) as StepDefinition;
|
|
161
199
|
|
|
200
|
+
// The switch waits for this gate, unless the caller holds the approval (`gates: "outer"`).
|
|
201
|
+
const approve = phase("Approve", [
|
|
202
|
+
gate(switchGate, {
|
|
203
|
+
...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
|
|
204
|
+
plan: stepOutput("verify", "planDigest"),
|
|
205
|
+
description:
|
|
206
|
+
config.gate?.description ??
|
|
207
|
+
`Approve switching ${what} on ${config.env}: the run record's Verification line has what this approval binds; for a rename, readers move to the new name once it is switched`,
|
|
208
|
+
...(config.gate?.approval
|
|
209
|
+
? {
|
|
210
|
+
approval: {
|
|
211
|
+
...config.gate.approval,
|
|
212
|
+
...(policy
|
|
213
|
+
? {
|
|
214
|
+
context: {
|
|
215
|
+
verifiedRows: stepOutput("verify", "rows"),
|
|
216
|
+
mismatched: stepOutput("verify", "mismatched"),
|
|
217
|
+
...config.gate.approval.context,
|
|
218
|
+
},
|
|
219
|
+
}
|
|
220
|
+
: {}),
|
|
221
|
+
},
|
|
222
|
+
}
|
|
223
|
+
: {}),
|
|
224
|
+
}),
|
|
225
|
+
]);
|
|
226
|
+
|
|
162
227
|
const phases = [
|
|
163
228
|
...(config.build === false ? [] : [phase("Build", [activity("chantBuild", { path: config.path ?? "." })])]),
|
|
164
229
|
phase("Plan", [step("postgresMigrationPlan", { id: "plan", outcomeAttribute: [{ name: "MigrationState", from: "state" }, { name: "Change", from: "change" }] })]),
|
|
@@ -194,49 +259,29 @@ export function PostgresMigrationOp(config: PostgresMigrationOpConfig): Postgres
|
|
|
194
259
|
],
|
|
195
260
|
}),
|
|
196
261
|
]),
|
|
197
|
-
|
|
198
|
-
gate(switchGate, {
|
|
199
|
-
...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
|
|
200
|
-
plan: stepOutput("verify", "planDigest"),
|
|
201
|
-
description:
|
|
202
|
-
config.gate?.description ??
|
|
203
|
-
`Approve switching ${what} on ${config.env}: the run record's Verification line has what this approval binds; for a rename, readers move to the new name once it is switched`,
|
|
204
|
-
...(config.gate?.approval
|
|
205
|
-
? {
|
|
206
|
-
approval: {
|
|
207
|
-
...config.gate.approval,
|
|
208
|
-
...(policy
|
|
209
|
-
? {
|
|
210
|
-
context: {
|
|
211
|
-
verifiedRows: stepOutput("verify", "rows"),
|
|
212
|
-
mismatched: stepOutput("verify", "mismatched"),
|
|
213
|
-
...config.gate.approval.context,
|
|
214
|
-
},
|
|
215
|
-
}
|
|
216
|
-
: {}),
|
|
217
|
-
},
|
|
218
|
-
}
|
|
219
|
-
: {}),
|
|
220
|
-
}),
|
|
221
|
-
]),
|
|
262
|
+
...(outer ? [] : [approve]),
|
|
222
263
|
phase("Switch", [step("postgresMigrationSwitch", { timeout: "6h", outcomeAttribute: [{ name: "OldColumn", from: "oldColumn" }] })]),
|
|
223
264
|
phase("Retain", [step("postgresMigrationRetain", { id: "retain", outcomeAttribute: { name: "RetainUntil", from: "retainUntil" } })]),
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
265
|
+
...(outer
|
|
266
|
+
? []
|
|
267
|
+
: [
|
|
268
|
+
phase("Approve contract", [
|
|
269
|
+
gate(contractGate, {
|
|
270
|
+
...(config.contractGate?.timeout ? { timeout: config.contractGate.timeout } : {}),
|
|
271
|
+
plan: stepOutput("retain", "contractDigest"),
|
|
272
|
+
description: config.contractGate?.description ?? `Approve dropping the old column of ${what} once its retention has passed and no reader uses it`,
|
|
273
|
+
}),
|
|
274
|
+
]),
|
|
275
|
+
phase("Contract", [step("postgresMigrationContract", { outcomeAttribute: { name: "Dropped", from: "dropped" } })]),
|
|
276
|
+
]),
|
|
232
277
|
];
|
|
233
278
|
|
|
234
279
|
const op = Op({
|
|
235
280
|
name: config.name,
|
|
236
|
-
overview: `Migrate ${what} on ${config.env} as expand and contract: new column, dual write, backfill, verify, gated switch, gated contract`,
|
|
281
|
+
overview: `Migrate ${what} on ${config.env} as expand and contract: new column, dual write, backfill, verify, ${outer ? "switch under the caller's approval" : "gated switch, gated contract"}`,
|
|
237
282
|
labels: { Migration: "true", Env: config.env, Table: config.table, Column: config.column },
|
|
238
283
|
phases,
|
|
239
|
-
onFailure: [phase("Compensate", [step("postgresMigrationCompensate")])],
|
|
284
|
+
...(keep ? {} : { onFailure: [phase("Compensate", [step("postgresMigrationCompensate")])] }),
|
|
240
285
|
});
|
|
241
286
|
return { op };
|
|
242
287
|
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `PostgresMigrationOp` under an outer approval with what the expand added
|
|
3
|
+
* kept on failure (#3687): a backfill that fails part way leaves the new
|
|
4
|
+
* column, its dual write and its receipts; the next run skips the batches
|
|
5
|
+
* already filled, verifies and switches with no gate, and keeps the old
|
|
6
|
+
* column; a later run with the Op's own gates stops at its gates. A refusal
|
|
7
|
+
* says how to start again, since nothing drops what was kept.
|
|
8
|
+
*
|
|
9
|
+
* The server: `CHANT_SQL_E2E_POSTGRES_URL` (a `postgres://user@host:port/db`
|
|
10
|
+
* URL, with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names a running one, such as
|
|
11
|
+
* the one `chant emulator up --lexicon sql` starts. Without it, a throwaway
|
|
12
|
+
* server at the pin, skipped cleanly when Docker is not available. The test
|
|
13
|
+
* writes only to the schema `chant_e2e_3687`, dropped afterwards.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
17
|
+
import { tmpdir } from "node:os";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
|
20
|
+
import { OpRunFailure } from "@intentius/chant/op";
|
|
21
|
+
import { dockerAvailable, startTestPostgres, type TestPostgres } from "../testing/server";
|
|
22
|
+
import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../live/client";
|
|
23
|
+
import { postgresApply } from "../../op/activities/postgres-apply";
|
|
24
|
+
import * as m from "../../op/activities/postgres-migration";
|
|
25
|
+
import type { PostgresMigrationArgs, PostgresMigrationOpConfig } from "./op";
|
|
26
|
+
import { ApprovingLedger, runMigrationOp, runOutcome } from "../testing/migration";
|
|
27
|
+
import { POSTGRES_RECEIPTS_TABLE } from "./receipts";
|
|
28
|
+
|
|
29
|
+
const S = "chant_e2e_3687";
|
|
30
|
+
const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
|
|
31
|
+
const enabled = given ? true : await dockerAvailable();
|
|
32
|
+
const dir = mkdtempSync(join(tmpdir(), "chant-pg-outer-"));
|
|
33
|
+
let server: TestPostgres | undefined;
|
|
34
|
+
let endpoint: PostgresEndpoint;
|
|
35
|
+
let admin: PostgresClient;
|
|
36
|
+
|
|
37
|
+
beforeAll(async () => {
|
|
38
|
+
if (!enabled) return;
|
|
39
|
+
if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
|
|
40
|
+
else {
|
|
41
|
+
server = await startTestPostgres();
|
|
42
|
+
endpoint = server.endpoint();
|
|
43
|
+
}
|
|
44
|
+
admin = await connectPostgres(endpoint);
|
|
45
|
+
await admin.query(`DROP SCHEMA IF EXISTS ${S} CASCADE`);
|
|
46
|
+
}, 600_000);
|
|
47
|
+
|
|
48
|
+
afterAll(async () => {
|
|
49
|
+
if (enabled) {
|
|
50
|
+
await admin?.query(`DROP SCHEMA IF EXISTS ${S} CASCADE`).catch(() => undefined);
|
|
51
|
+
await admin?.end();
|
|
52
|
+
}
|
|
53
|
+
await server?.stop();
|
|
54
|
+
rmSync(dir, { recursive: true, force: true });
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
interface Obj {
|
|
58
|
+
export: string;
|
|
59
|
+
type: string;
|
|
60
|
+
ddl: string;
|
|
61
|
+
dependsOn?: string[];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function writeBuild(file: string, objects: Obj[]): string {
|
|
65
|
+
writeFileSync(join(dir, file), JSON.stringify({ dialect: "postgres", postgresMajor: 18, applyOrder: objects.map((o) => o.export), objects: objects.map((o) => ({ dependsOn: [], ...o })) }));
|
|
66
|
+
return file;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const MARKER = { stack: "e2e3687", env: "test" };
|
|
70
|
+
const profile = () => ({
|
|
71
|
+
config: { ownership: MARKER, sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_PASSWORD" }, schemas: [S] } } } },
|
|
72
|
+
env: { PG_E2E_PASSWORD: endpoint.password ?? "" },
|
|
73
|
+
});
|
|
74
|
+
const deps = (extra: Partial<m.PostgresMigrationDeps> = {}): m.PostgresMigrationDeps => ({ ...profile(), log: () => undefined, ...extra });
|
|
75
|
+
const apply = (file: string) => postgresApply({ buildPath: join(dir, file), environment: "e2e" }, undefined, { ...profile(), log: () => undefined });
|
|
76
|
+
const one = async <T>(sql: string): Promise<T> => (await admin.query<T>(sql))[0]!;
|
|
77
|
+
const columnsOf = async (table: string) =>
|
|
78
|
+
(await admin.query<{ name: string }>(`SELECT attname AS name FROM pg_attribute WHERE attrelid = '${table}'::regclass AND attnum > 0 AND NOT attisdropped ORDER BY attnum`)).map((r) => r.name);
|
|
79
|
+
const receipts = async () => (await one<{ n: number }>(`SELECT count(*)::int AS n FROM ${S}.${POSTGRES_RECEIPTS_TABLE} WHERE address LIKE 'e2e3687/test/migrate/${S}.events.kind/%' AND address NOT LIKE '%/batches'`)).n;
|
|
80
|
+
|
|
81
|
+
describe.skipIf(!enabled)("under an outer approval, a failed migration keeps its new column and the rerun resumes (#3687)", () => {
|
|
82
|
+
const SCHEMA: Obj = { export: "app", type: "Postgres::Schema", ddl: `CREATE SCHEMA ${S}` };
|
|
83
|
+
const V1: Obj = { export: "events", type: "Postgres::Table", dependsOn: ["app"], ddl: `CREATE TABLE ${S}.events (id bigint PRIMARY KEY, kind varchar(20) NOT NULL)` };
|
|
84
|
+
const V2: Obj = { ...V1, ddl: `CREATE TABLE ${S}.events (id bigint PRIMARY KEY, kind varchar(10) NOT NULL)` };
|
|
85
|
+
const config = (): PostgresMigrationOpConfig => ({
|
|
86
|
+
name: "migrate-events-kind",
|
|
87
|
+
env: "e2e",
|
|
88
|
+
table: `${S}.events`,
|
|
89
|
+
column: "kind",
|
|
90
|
+
build: false,
|
|
91
|
+
path: dir,
|
|
92
|
+
output: "events-v2.json",
|
|
93
|
+
retain: "0s",
|
|
94
|
+
batchSize: 100,
|
|
95
|
+
stack: MARKER.stack,
|
|
96
|
+
ownershipEnv: MARKER.env,
|
|
97
|
+
gates: "outer",
|
|
98
|
+
onFailure: "keep",
|
|
99
|
+
});
|
|
100
|
+
const args = (): PostgresMigrationArgs => ({ table: `${S}.events`, column: "kind", buildPath: "events-v2.json", environment: "e2e", batchSize: 100, stack: MARKER.stack, ownershipEnv: MARKER.env, cwd: dir });
|
|
101
|
+
|
|
102
|
+
test("the failed run keeps the new column and its receipts; the rerun skips what was filled, switches without a gate and keeps the old column", async () => {
|
|
103
|
+
await apply(writeBuild("events-v1.json", [SCHEMA, V1]));
|
|
104
|
+
await admin.query(`INSERT INTO ${S}.events SELECT g, CASE WHEN g % 2 = 0 THEN 'click' ELSE 'view' END FROM generate_series(0, 599) g`);
|
|
105
|
+
writeBuild("events-v2.json", [SCHEMA, V2]);
|
|
106
|
+
|
|
107
|
+
// A refusal says how to start again when nothing drops what was kept; without keepOnFailure it does not.
|
|
108
|
+
await expect(m.postgresMigrationBackfill({ ...args(), keepOnFailure: true }, undefined, deps())).rejects.toThrow(
|
|
109
|
+
/the Expand phase adds it\. This migration keeps what the expand added on failure \(onFailure: "keep"\), so nothing drops it: to start again from a new column, run the Op once with onFailure: "drop"/,
|
|
110
|
+
);
|
|
111
|
+
await expect(m.postgresMigrationBackfill(args(), undefined, deps())).rejects.not.toThrow(/onFailure: "keep"/);
|
|
112
|
+
|
|
113
|
+
// Every attempt of the backfill step fails after its batch: three attempts, three batches filled, then the run fails.
|
|
114
|
+
let filled = 0;
|
|
115
|
+
const ledger = new ApprovingLedger();
|
|
116
|
+
const failure = await runMigrationOp(config(), ledger.port, () =>
|
|
117
|
+
deps({
|
|
118
|
+
backfill: {
|
|
119
|
+
afterBatch: () => {
|
|
120
|
+
filled++;
|
|
121
|
+
throw new Error(`the backfill failed after ${filled} batch(es)`);
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
}),
|
|
125
|
+
).then(
|
|
126
|
+
() => undefined,
|
|
127
|
+
(e: unknown) => e,
|
|
128
|
+
);
|
|
129
|
+
expect(failure).toBeInstanceOf(OpRunFailure);
|
|
130
|
+
const records = (failure as OpRunFailure).result.records;
|
|
131
|
+
expect(records.find((x) => x.fn === "postgresMigrationBackfill")?.error).toMatch(/the backfill failed after 3 batch/);
|
|
132
|
+
// No onFailure ran: the new column, the dual write and the receipts are there.
|
|
133
|
+
expect(records.find((x) => x.fn === "postgresMigrationCompensate")).toBeUndefined();
|
|
134
|
+
expect(await columnsOf(`${S}.events`)).toEqual(["id", "kind", "kind__chant_new"]);
|
|
135
|
+
expect(await one(`SELECT count(*)::int AS n FROM pg_trigger WHERE tgrelid = '${S}.events'::regclass AND tgname = 'kind__chant_sync'`)).toEqual({ n: 1 });
|
|
136
|
+
expect(await one(`SELECT count(*)::int AS n FROM ${S}.events WHERE kind__chant_new IS NOT NULL`)).toEqual({ n: 300 });
|
|
137
|
+
expect(await receipts()).toBe(3);
|
|
138
|
+
|
|
139
|
+
// The rerun resumes: three batches skipped by receipt, three filled, verified and switched, with no gate.
|
|
140
|
+
const r = await runMigrationOp(config(), ledger.port, () => deps());
|
|
141
|
+
expect(r.status).toBe("ok");
|
|
142
|
+
expect(ledger.port.appended).toEqual([]);
|
|
143
|
+
expect(runOutcome(r, "Skipped")).toBe(3);
|
|
144
|
+
expect(runOutcome(r, "Filled")).toBe(3);
|
|
145
|
+
expect(runOutcome(r, "VerifiedRows")).toBe(600);
|
|
146
|
+
expect(r.records.map((x) => x.fn)).not.toContain("postgresMigrationContract");
|
|
147
|
+
expect(await one(`SELECT format_type(atttypid, atttypmod) AS t FROM pg_attribute WHERE attrelid = '${S}.events'::regclass AND attname = 'kind'`)).toEqual({ t: "character varying(10)" });
|
|
148
|
+
// The old column is kept: dropping it is a run with the Op's own gates.
|
|
149
|
+
expect(await columnsOf(`${S}.events`)).toEqual(["id", "kind__chant_old", "kind"]);
|
|
150
|
+
|
|
151
|
+
// Run with its own gates, the Op has nothing left to switch, and its contract gate binds that old column.
|
|
152
|
+
const own = new ApprovingLedger();
|
|
153
|
+
const first = await runMigrationOp({ ...config(), gates: "own" }, own.port, () => deps());
|
|
154
|
+
expect(first.status).toBe("gated");
|
|
155
|
+
expect(first.gate).toMatchObject({ gate: "approve-migrate-events-kind" });
|
|
156
|
+
expect(runOutcome(first, "MigrationState")).toBe("switched");
|
|
157
|
+
own.approveLast();
|
|
158
|
+
const second = await runMigrationOp({ ...config(), gates: "own" }, own.port, () => deps());
|
|
159
|
+
expect(second.status).toBe("gated");
|
|
160
|
+
expect(second.gate).toMatchObject({ gate: "approve-migrate-events-kind-contract" });
|
|
161
|
+
expect(await columnsOf(`${S}.events`)).toEqual(["id", "kind__chant_old", "kind"]);
|
|
162
|
+
}, 300_000);
|
|
163
|
+
});
|