@intentius/chant-lexicon-sql 0.116.0 → 0.117.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/codegen/registry.d.ts.map +1 -1
  2. package/dist/config.d.ts +20 -0
  3. package/dist/config.d.ts.map +1 -1
  4. package/dist/index.d.ts +1 -1
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/integrity.json +5 -5
  7. package/dist/lint/rules/sqlpg001.d.ts.map +1 -1
  8. package/dist/lsp/postgres-template.d.ts.map +1 -1
  9. package/dist/manifest.json +1 -1
  10. package/dist/meta.json +20 -0
  11. package/dist/okf/index.md +4 -0
  12. package/dist/okf/types/PostgresDefaultPrivileges.md +9 -0
  13. package/dist/okf/types/PostgresGrant.md +9 -0
  14. package/dist/okf/types/PostgresPolicy.md +9 -0
  15. package/dist/okf/types/PostgresRole.md +9 -0
  16. package/dist/op/activities/postgres-migration.d.ts.map +1 -1
  17. package/dist/op/activity-contracts.d.ts +10 -0
  18. package/dist/op/activity-contracts.d.ts.map +1 -1
  19. package/dist/plugin.d.ts.map +1 -1
  20. package/dist/postgres/access/acl.d.ts +119 -0
  21. package/dist/postgres/access/acl.d.ts.map +1 -0
  22. package/dist/postgres/access/import.d.ts +13 -0
  23. package/dist/postgres/access/import.d.ts.map +1 -0
  24. package/dist/postgres/access/live.d.ts +24 -0
  25. package/dist/postgres/access/live.d.ts.map +1 -0
  26. package/dist/postgres/apply/apply.d.ts.map +1 -1
  27. package/dist/postgres/apply/statements.d.ts +3 -0
  28. package/dist/postgres/apply/statements.d.ts.map +1 -1
  29. package/dist/postgres/entities.d.ts +106 -4
  30. package/dist/postgres/entities.d.ts.map +1 -1
  31. package/dist/postgres/entity-types.d.ts +4 -0
  32. package/dist/postgres/entity-types.d.ts.map +1 -1
  33. package/dist/postgres/import/generator.d.ts +4 -1
  34. package/dist/postgres/import/generator.d.ts.map +1 -1
  35. package/dist/postgres/import/live-export.d.ts +6 -0
  36. package/dist/postgres/import/live-export.d.ts.map +1 -1
  37. package/dist/postgres/live/bind.d.ts +2 -0
  38. package/dist/postgres/live/bind.d.ts.map +1 -1
  39. package/dist/postgres/live/catalog.d.ts +11 -2
  40. package/dist/postgres/live/catalog.d.ts.map +1 -1
  41. package/dist/postgres/live/describe-resources.d.ts +9 -0
  42. package/dist/postgres/live/describe-resources.d.ts.map +1 -1
  43. package/dist/postgres/migrate/op.d.ts +29 -0
  44. package/dist/postgres/migrate/op.d.ts.map +1 -1
  45. package/dist/postgres/parser.d.ts +76 -3
  46. package/dist/postgres/parser.d.ts.map +1 -1
  47. package/dist/postgres/plan/commands.d.ts +12 -0
  48. package/dist/postgres/plan/commands.d.ts.map +1 -1
  49. package/dist/postgres/plan/deep.d.ts.map +1 -1
  50. package/dist/postgres/plan/diff.d.ts +10 -1
  51. package/dist/postgres/plan/diff.d.ts.map +1 -1
  52. package/dist/postgres/plan/normalize.d.ts +8 -1
  53. package/dist/postgres/plan/normalize.d.ts.map +1 -1
  54. package/dist/postgres/plan/rules.d.ts +8 -0
  55. package/dist/postgres/plan/rules.d.ts.map +1 -1
  56. package/dist/postgres/plan/schema.d.ts +2 -1
  57. package/dist/postgres/plan/schema.d.ts.map +1 -1
  58. package/dist/postgres/plan/server-normalize.d.ts +4 -1
  59. package/dist/postgres/plan/server-normalize.d.ts.map +1 -1
  60. package/dist/postgres.d.ts +1 -1
  61. package/dist/postgres.d.ts.map +1 -1
  62. package/dist/rules/postgres-templates.ts +2 -2
  63. package/dist/rules/sqlpg001.ts +7 -4
  64. package/dist/validate.d.ts.map +1 -1
  65. package/package.json +2 -2
  66. package/src/codegen/registry.ts +4 -0
  67. package/src/config.ts +10 -0
  68. package/src/generated/lexicon-sql.json +20 -0
  69. package/src/index.ts +1 -1
  70. package/src/lint/rules/postgres-templates.ts +2 -2
  71. package/src/lint/rules/sqlpg001.ts +7 -4
  72. package/src/lsp/postgres-template.ts +2 -0
  73. package/src/mcp/postgres.ts +1 -1
  74. package/src/migration-statements.ts +7 -0
  75. package/src/op/activities/postgres-migration.ts +27 -2
  76. package/src/op/activity-contracts.ts +1 -0
  77. package/src/plugin.ts +3 -0
  78. package/src/postgres/access/access.e2e.test.ts +191 -0
  79. package/src/postgres/access/access.test.ts +147 -0
  80. package/src/postgres/access/acl.ts +400 -0
  81. package/src/postgres/access/import.ts +64 -0
  82. package/src/postgres/access/live.ts +123 -0
  83. package/src/postgres/apply/apply.ts +117 -4
  84. package/src/postgres/apply/statements.ts +55 -9
  85. package/src/postgres/entities.ts +287 -11
  86. package/src/postgres/entity-types.ts +4 -0
  87. package/src/postgres/import/generator.ts +14 -4
  88. package/src/postgres/import/live-export.ts +12 -2
  89. package/src/postgres/live/bind.ts +3 -0
  90. package/src/postgres/live/catalog.ts +79 -3
  91. package/src/postgres/live/describe-resources.ts +27 -3
  92. package/src/postgres/migrate/migrate.test.ts +24 -1
  93. package/src/postgres/migrate/op.ts +80 -35
  94. package/src/postgres/migrate/outer.e2e.test.ts +163 -0
  95. package/src/postgres/parser.ts +247 -8
  96. package/src/postgres/plan/commands.ts +60 -7
  97. package/src/postgres/plan/deep.ts +24 -3
  98. package/src/postgres/plan/diff.ts +71 -6
  99. package/src/postgres/plan/normalize.ts +115 -2
  100. package/src/postgres/plan/rules.ts +44 -0
  101. package/src/postgres/plan/schema.ts +4 -2
  102. package/src/postgres/plan/server-normalize.ts +29 -1
  103. package/src/postgres.ts +11 -0
  104. package/src/skill-defs.ts +1 -1
  105. package/src/skills/chant-sql-postgres-migration.md +3 -1
  106. package/src/skills/chant-sql-postgres.md +9 -2
  107. package/src/validate.ts +4 -0
@@ -41,15 +41,30 @@ interface Bound {
41
41
  */
42
42
  export function declaredAddress(entity: { type: string; props: Record<string, unknown> }, defaultSchema: string): { schema?: string; name: string; signature?: string } {
43
43
  const name = String(entity.props.name ?? "");
44
- if (entity.type === POSTGRES_ENTITY_TYPES.schema || entity.type === POSTGRES_ENTITY_TYPES.extension) return { name };
44
+ if (entity.type === POSTGRES_ENTITY_TYPES.schema || entity.type === POSTGRES_ENTITY_TYPES.extension || entity.type === POSTGRES_ENTITY_TYPES.role) return { name };
45
45
  const schema = typeof entity.props.schema === "string" ? entity.props.schema : identValue(defaultSchema);
46
- if (entity.type === POSTGRES_ENTITY_TYPES.function || entity.type === POSTGRES_ENTITY_TYPES.procedure || entity.type === POSTGRES_ENTITY_TYPES.trigger) {
46
+ if (entity.type === POSTGRES_ENTITY_TYPES.function || entity.type === POSTGRES_ENTITY_TYPES.procedure || entity.type === POSTGRES_ENTITY_TYPES.trigger || entity.type === POSTGRES_ENTITY_TYPES.policy) {
47
47
  const signature = canonicalPgObject(entity.type, entity.props, defaultSchema).signature;
48
48
  return { schema, name, ...(signature !== undefined ? { signature } : {}) };
49
49
  }
50
50
  return { schema, name };
51
51
  }
52
52
 
53
+ /**
54
+ * Why an access declaration is not observed one by one: a grant or default
55
+ * privileges, which `chant sql plan` compares as the access they add up to;
56
+ * a policy or role where the profile does not manage access.
57
+ */
58
+ export function accessUnobserved(type: string, managed: boolean): { reason: "unsupported-kind" | "filtered"; detail: string } | undefined {
59
+ if (type === POSTGRES_ENTITY_TYPES.grant || type === POSTGRES_ENTITY_TYPES.defaultPrivileges) {
60
+ return { reason: "unsupported-kind", detail: "a grant is compared as the privileges the declarations add up to, by chant sql plan and the applier, not one statement at a time" };
61
+ }
62
+ if (!managed && (type === POSTGRES_ENTITY_TYPES.policy || type === POSTGRES_ENTITY_TYPES.role)) {
63
+ return { reason: "filtered", detail: "access is not managed in this environment (sql.profiles.<env>.access)" };
64
+ }
65
+ return undefined;
66
+ }
67
+
53
68
  /** A schema scope for the catalog read: the profile's schemas, else every schema the declarations name and the default one. */
54
69
  export function scopeFor(target: PostgresTarget, entities: ReadonlyArray<{ type: string; props: Record<string, unknown> }>): string[] | undefined {
55
70
  if (target.schemas) return target.schemas;
@@ -68,7 +83,14 @@ function adapter(options: BindOptions & { owned?: boolean; declared: DeclaredEnt
68
83
  async bind() {
69
84
  const { target, client } = await bindPostgres(options);
70
85
  try {
71
- const live = markProviderOwned(await readLiveSchema(client, { schemas: scopeFor(target, options.declared.filter((d) => d.type.startsWith("Postgres::"))) }), target.provider);
86
+ const live = markProviderOwned(
87
+ await readLiveSchema(client, {
88
+ schemas: scopeFor(target, options.declared.filter((d) => d.type.startsWith("Postgres::"))),
89
+ access: target.access === true,
90
+ roles: options.declared.filter((d) => d.type === POSTGRES_ENTITY_TYPES.role).map((d) => String(d.props.name)),
91
+ }),
92
+ target.provider,
93
+ );
72
94
  return { target, byKey: new Map(live.map((o) => [liveKey(o.type, o.schema, o.name, o.signature), o])) };
73
95
  } finally {
74
96
  await client.end();
@@ -77,6 +99,8 @@ function adapter(options: BindOptions & { owned?: boolean; declared: DeclaredEnt
77
99
  classifyBindFailure: (err) => classifyPostgresFailure(err),
78
100
  async read({ target, byKey }, entity): Promise<EntityObservation> {
79
101
  if (!entity.type.startsWith("Postgres::")) return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
102
+ const why = accessUnobserved(entity.type, target.access === true);
103
+ if (why) return { unobserved: why };
80
104
  const { schema, name, signature } = declaredAddress(entity, target.defaultSchema);
81
105
  const queried = `${redactUrl(target.endpoint.url)} ${schema ? `${schema}.` : ""}${name}${signature ?? ""}`;
82
106
  const live = byKey.get(liveKey(entity.type, schema, name, signature));
@@ -72,7 +72,30 @@ describe("PostgresMigrationOp", () => {
72
72
  expect(Object.keys(g.approval.context)).toEqual(["verifiedRows", "mismatched"]);
73
73
  });
74
74
 
75
+ test('gates: "outer" leaves out both approval gates and the Contract phase, and keeps the verification (#3687)', () => {
76
+ const props = propsOf({ ...BASE, gates: "outer" });
77
+ expect(props.phases.map((p) => p.name)).toEqual(["Build", "Plan", "Expand", "Dual write", "Backfill", "Carry over", "Verify", "Switch", "Retain"]);
78
+ expect(props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "gate")).toEqual([]);
79
+ const fns = props.phases.flatMap((p) => p.steps).map((s) => (s as { fn?: string }).fn);
80
+ expect(fns.indexOf("postgresMigrationVerify")).toBeLessThan(fns.indexOf("postgresMigrationSwitch"));
81
+ expect(fns).not.toContain("postgresMigrationContract");
82
+ expect(props.onFailure?.map((p) => p.steps.map((s) => (s as { fn?: string }).fn))).toEqual([["postgresMigrationCompensate"]]);
83
+ expect(props.overview).toContain("the caller's approval");
84
+ });
85
+
86
+ test('onFailure: "keep" has no onFailure and tells every step so (#3687)', () => {
87
+ const props = propsOf({ ...BASE, onFailure: "keep" });
88
+ expect(props.onFailure).toBeUndefined();
89
+ const steps = props.phases.flatMap((p) => p.steps).filter((s) => s.kind === "activity" && (s as { fn: string }).fn.startsWith("postgresMigration")) as Array<{ args: Record<string, unknown> }>;
90
+ expect(steps).toHaveLength(9);
91
+ for (const s of steps) expect(s.args.keepOnFailure).toBe(true);
92
+ expect(props.phases.map((p) => p.name)).toContain("Approve");
93
+ expect((propsOf(BASE).phases[1]!.steps[0] as { args: Record<string, unknown> }).args).not.toHaveProperty("keepOnFailure");
94
+ });
95
+
75
96
  test("bad configuration fails when the Op is built", () => {
97
+ expect(() => PostgresMigrationOp({ ...BASE, gates: "none" as never })).toThrow(/gates must be/);
98
+ expect(() => PostgresMigrationOp({ ...BASE, onFailure: "retry" as never })).toThrow(/onFailure must be/);
76
99
  expect(() => PostgresMigrationOp({ ...BASE, table: "app.users; DROP" })).toThrow(/table must be/);
77
100
  expect(() => PostgresMigrationOp({ ...BASE, column: "" })).toThrow(/column must be/);
78
101
  expect(() => PostgresMigrationOp({ ...BASE, batchSize: 0 })).toThrow(/batchSize/);
@@ -95,7 +118,7 @@ describe("PostgresMigrationOp", () => {
95
118
  ownershipEnv: "prod",
96
119
  build: false,
97
120
  };
98
- for (const config of [BASE, full, { ...BASE, replicationLag: false as const }]) {
121
+ for (const config of [BASE, full, { ...BASE, replicationLag: false as const }, { ...BASE, gates: "outer" as const, onFailure: "keep" as const }]) {
99
122
  const props = propsOf(config);
100
123
  expect(validateActivitySteps(props, merged)).toEqual([]);
101
124
  expect(validateStepOutputRefs(props, merged)).toEqual([]);
@@ -35,6 +35,12 @@
35
35
  * machine) is not a failure, runs no onFailure, and the next run resumes the
36
36
  * backfill from its receipts; a step that fails runs onFailure, and the next
37
37
  * run starts again from a new column.
38
+ *
39
+ * `gates: "outer"` (#3687) leaves out both approval gates and the Contract
40
+ * phase, for a caller that runs the Op under an approval of its own: the run
41
+ * goes from Plan to Retain, verification included. `onFailure: "keep"`
42
+ * leaves out the onFailure, so a failed run's next run resumes the backfill
43
+ * from its receipts instead of starting again.
38
44
  */
39
45
 
40
46
  import { Op, activity, gate, phase, stepOutput, type GateApproval } from "@intentius/chant/op";
@@ -98,6 +104,27 @@ export interface PostgresMigrationOpConfig {
98
104
  stack?: string;
99
105
  /** Ownership env. Default: `ownership.env` in `chant.config.ts`. */
100
106
  ownershipEnv?: string;
107
+ /**
108
+ * Whose approval the switch and the contract wait for. `"own"` (the
109
+ * default): the Op's two gates. `"outer"`: an approval the caller already
110
+ * holds for a larger change that runs this Op as one of its steps. The
111
+ * switch gate, the contract gate and the Contract phase are left out, so
112
+ * the Op ends at Retain with the old column kept until its retention date.
113
+ * Drop it after that date, or run the Op with `"own"`: its switch gate has
114
+ * nothing left to switch, and its contract gate binds that old column. The
115
+ * verification still runs, and any difference fails the run with nothing
116
+ * switched.
117
+ */
118
+ gates?: "own" | "outer";
119
+ /**
120
+ * What a failed run does with what the expand added. `"drop"` (the
121
+ * default): onFailure drops the new column, the dual-write trigger and the
122
+ * receipts, and the next run starts again from a new column. `"keep"`:
123
+ * there is no onFailure, and the next run resumes the backfill from its
124
+ * per-batch receipts. To start again instead, run the Op once with
125
+ * `"drop"`.
126
+ */
127
+ onFailure?: "drop" | "keep";
101
128
  }
102
129
 
103
130
  export interface PostgresMigrationOpResources {
@@ -119,6 +146,8 @@ export interface PostgresMigrationArgs {
119
146
  stack?: string;
120
147
  ownershipEnv?: string;
121
148
  cwd?: string;
149
+ /** A failed run keeps what the expand added (`onFailure: "keep"`): a refusal says how to start again. */
150
+ keepOnFailure?: boolean;
122
151
  }
123
152
 
124
153
  const IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$|^"(?:[^"]|"")+"$/;
@@ -134,6 +163,14 @@ export function PostgresMigrationOp(config: PostgresMigrationOpConfig): Postgres
134
163
  }
135
164
  if (config.replicationLag) for (const d of [config.replicationLag.max, config.replicationLag.wait]) if (d !== undefined) parseDuration(d);
136
165
  if (config.retain !== undefined) parseDuration(config.retain);
166
+ if (config.gates !== undefined && config.gates !== "own" && config.gates !== "outer") {
167
+ throw new Error(`PostgresMigrationOp "${config.name}": gates must be "own" or "outer"`);
168
+ }
169
+ if (config.onFailure !== undefined && config.onFailure !== "drop" && config.onFailure !== "keep") {
170
+ throw new Error(`PostgresMigrationOp "${config.name}": onFailure must be "drop" or "keep"`);
171
+ }
172
+ const outer = config.gates === "outer";
173
+ const keep = config.onFailure === "keep";
137
174
 
138
175
  const args: PostgresMigrationArgs = {
139
176
  table: config.table,
@@ -149,6 +186,7 @@ export function PostgresMigrationOp(config: PostgresMigrationOpConfig): Postgres
149
186
  ...(config.stack ? { stack: config.stack } : {}),
150
187
  ...(config.ownershipEnv ? { ownershipEnv: config.ownershipEnv } : {}),
151
188
  ...(config.path && config.path !== "." ? { cwd: config.path } : {}),
189
+ ...(keep ? { keepOnFailure: true } : {}),
152
190
  };
153
191
  const a = args as unknown as Record<string, unknown>;
154
192
  const switchGate = config.gate?.gate ?? `approve-${config.name}`;
@@ -159,6 +197,33 @@ export function PostgresMigrationOp(config: PostgresMigrationOpConfig): Postgres
159
197
  const step = (fn: string, extra: Partial<StepDefinition & { id: string; outcomeAttribute: unknown; profile: string; timeout: string }> = {}): StepDefinition =>
160
198
  ({ ...activity(fn, a), ...extra }) as StepDefinition;
161
199
 
200
+ // The switch waits for this gate, unless the caller holds the approval (`gates: "outer"`).
201
+ const approve = phase("Approve", [
202
+ gate(switchGate, {
203
+ ...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
204
+ plan: stepOutput("verify", "planDigest"),
205
+ description:
206
+ config.gate?.description ??
207
+ `Approve switching ${what} on ${config.env}: the run record's Verification line has what this approval binds; for a rename, readers move to the new name once it is switched`,
208
+ ...(config.gate?.approval
209
+ ? {
210
+ approval: {
211
+ ...config.gate.approval,
212
+ ...(policy
213
+ ? {
214
+ context: {
215
+ verifiedRows: stepOutput("verify", "rows"),
216
+ mismatched: stepOutput("verify", "mismatched"),
217
+ ...config.gate.approval.context,
218
+ },
219
+ }
220
+ : {}),
221
+ },
222
+ }
223
+ : {}),
224
+ }),
225
+ ]);
226
+
162
227
  const phases = [
163
228
  ...(config.build === false ? [] : [phase("Build", [activity("chantBuild", { path: config.path ?? "." })])]),
164
229
  phase("Plan", [step("postgresMigrationPlan", { id: "plan", outcomeAttribute: [{ name: "MigrationState", from: "state" }, { name: "Change", from: "change" }] })]),
@@ -194,49 +259,29 @@ export function PostgresMigrationOp(config: PostgresMigrationOpConfig): Postgres
194
259
  ],
195
260
  }),
196
261
  ]),
197
- phase("Approve", [
198
- gate(switchGate, {
199
- ...(config.gate?.timeout ? { timeout: config.gate.timeout } : {}),
200
- plan: stepOutput("verify", "planDigest"),
201
- description:
202
- config.gate?.description ??
203
- `Approve switching ${what} on ${config.env}: the run record's Verification line has what this approval binds; for a rename, readers move to the new name once it is switched`,
204
- ...(config.gate?.approval
205
- ? {
206
- approval: {
207
- ...config.gate.approval,
208
- ...(policy
209
- ? {
210
- context: {
211
- verifiedRows: stepOutput("verify", "rows"),
212
- mismatched: stepOutput("verify", "mismatched"),
213
- ...config.gate.approval.context,
214
- },
215
- }
216
- : {}),
217
- },
218
- }
219
- : {}),
220
- }),
221
- ]),
262
+ ...(outer ? [] : [approve]),
222
263
  phase("Switch", [step("postgresMigrationSwitch", { timeout: "6h", outcomeAttribute: [{ name: "OldColumn", from: "oldColumn" }] })]),
223
264
  phase("Retain", [step("postgresMigrationRetain", { id: "retain", outcomeAttribute: { name: "RetainUntil", from: "retainUntil" } })]),
224
- phase("Approve contract", [
225
- gate(contractGate, {
226
- ...(config.contractGate?.timeout ? { timeout: config.contractGate.timeout } : {}),
227
- plan: stepOutput("retain", "contractDigest"),
228
- description: config.contractGate?.description ?? `Approve dropping the old column of ${what} once its retention has passed and no reader uses it`,
229
- }),
230
- ]),
231
- phase("Contract", [step("postgresMigrationContract", { outcomeAttribute: { name: "Dropped", from: "dropped" } })]),
265
+ ...(outer
266
+ ? []
267
+ : [
268
+ phase("Approve contract", [
269
+ gate(contractGate, {
270
+ ...(config.contractGate?.timeout ? { timeout: config.contractGate.timeout } : {}),
271
+ plan: stepOutput("retain", "contractDigest"),
272
+ description: config.contractGate?.description ?? `Approve dropping the old column of ${what} once its retention has passed and no reader uses it`,
273
+ }),
274
+ ]),
275
+ phase("Contract", [step("postgresMigrationContract", { outcomeAttribute: { name: "Dropped", from: "dropped" } })]),
276
+ ]),
232
277
  ];
233
278
 
234
279
  const op = Op({
235
280
  name: config.name,
236
- overview: `Migrate ${what} on ${config.env} as expand and contract: new column, dual write, backfill, verify, gated switch, gated contract`,
281
+ overview: `Migrate ${what} on ${config.env} as expand and contract: new column, dual write, backfill, verify, ${outer ? "switch under the caller's approval" : "gated switch, gated contract"}`,
237
282
  labels: { Migration: "true", Env: config.env, Table: config.table, Column: config.column },
238
283
  phases,
239
- onFailure: [phase("Compensate", [step("postgresMigrationCompensate")])],
284
+ ...(keep ? {} : { onFailure: [phase("Compensate", [step("postgresMigrationCompensate")])] }),
240
285
  });
241
286
  return { op };
242
287
  }
@@ -0,0 +1,163 @@
1
+ /**
2
+ * `PostgresMigrationOp` under an outer approval with what the expand added
3
+ * kept on failure (#3687): a backfill that fails part way leaves the new
4
+ * column, its dual write and its receipts; the next run skips the batches
5
+ * already filled, verifies and switches with no gate, and keeps the old
6
+ * column; a later run with the Op's own gates stops at its gates. A refusal
7
+ * says how to start again, since nothing drops what was kept.
8
+ *
9
+ * The server: `CHANT_SQL_E2E_POSTGRES_URL` (a `postgres://user@host:port/db`
10
+ * URL, with `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names a running one, such as
11
+ * the one `chant emulator up --lexicon sql` starts. Without it, a throwaway
12
+ * server at the pin, skipped cleanly when Docker is not available. The test
13
+ * writes only to the schema `chant_e2e_3687`, dropped afterwards.
14
+ */
15
+
16
+ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
17
+ import { tmpdir } from "node:os";
18
+ import { join } from "node:path";
19
+ import { afterAll, beforeAll, describe, expect, test } from "vitest";
20
+ import { OpRunFailure } from "@intentius/chant/op";
21
+ import { dockerAvailable, startTestPostgres, type TestPostgres } from "../testing/server";
22
+ import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../live/client";
23
+ import { postgresApply } from "../../op/activities/postgres-apply";
24
+ import * as m from "../../op/activities/postgres-migration";
25
+ import type { PostgresMigrationArgs, PostgresMigrationOpConfig } from "./op";
26
+ import { ApprovingLedger, runMigrationOp, runOutcome } from "../testing/migration";
27
+ import { POSTGRES_RECEIPTS_TABLE } from "./receipts";
28
+
29
+ const S = "chant_e2e_3687";
30
+ const given = process.env.CHANT_SQL_E2E_POSTGRES_URL;
31
+ const enabled = given ? true : await dockerAvailable();
32
+ const dir = mkdtempSync(join(tmpdir(), "chant-pg-outer-"));
33
+ let server: TestPostgres | undefined;
34
+ let endpoint: PostgresEndpoint;
35
+ let admin: PostgresClient;
36
+
37
+ beforeAll(async () => {
38
+ if (!enabled) return;
39
+ if (given) endpoint = { url: given, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
40
+ else {
41
+ server = await startTestPostgres();
42
+ endpoint = server.endpoint();
43
+ }
44
+ admin = await connectPostgres(endpoint);
45
+ await admin.query(`DROP SCHEMA IF EXISTS ${S} CASCADE`);
46
+ }, 600_000);
47
+
48
+ afterAll(async () => {
49
+ if (enabled) {
50
+ await admin?.query(`DROP SCHEMA IF EXISTS ${S} CASCADE`).catch(() => undefined);
51
+ await admin?.end();
52
+ }
53
+ await server?.stop();
54
+ rmSync(dir, { recursive: true, force: true });
55
+ });
56
+
57
+ interface Obj {
58
+ export: string;
59
+ type: string;
60
+ ddl: string;
61
+ dependsOn?: string[];
62
+ }
63
+
64
+ function writeBuild(file: string, objects: Obj[]): string {
65
+ writeFileSync(join(dir, file), JSON.stringify({ dialect: "postgres", postgresMajor: 18, applyOrder: objects.map((o) => o.export), objects: objects.map((o) => ({ dependsOn: [], ...o })) }));
66
+ return file;
67
+ }
68
+
69
+ const MARKER = { stack: "e2e3687", env: "test" };
70
+ const profile = () => ({
71
+ config: { ownership: MARKER, sql: { profiles: { e2e: { url: endpoint.url, password: { env: "PG_E2E_PASSWORD" }, schemas: [S] } } } },
72
+ env: { PG_E2E_PASSWORD: endpoint.password ?? "" },
73
+ });
74
+ const deps = (extra: Partial<m.PostgresMigrationDeps> = {}): m.PostgresMigrationDeps => ({ ...profile(), log: () => undefined, ...extra });
75
+ const apply = (file: string) => postgresApply({ buildPath: join(dir, file), environment: "e2e" }, undefined, { ...profile(), log: () => undefined });
76
+ const one = async <T>(sql: string): Promise<T> => (await admin.query<T>(sql))[0]!;
77
+ const columnsOf = async (table: string) =>
78
+ (await admin.query<{ name: string }>(`SELECT attname AS name FROM pg_attribute WHERE attrelid = '${table}'::regclass AND attnum > 0 AND NOT attisdropped ORDER BY attnum`)).map((r) => r.name);
79
+ const receipts = async () => (await one<{ n: number }>(`SELECT count(*)::int AS n FROM ${S}.${POSTGRES_RECEIPTS_TABLE} WHERE address LIKE 'e2e3687/test/migrate/${S}.events.kind/%' AND address NOT LIKE '%/batches'`)).n;
80
+
81
+ describe.skipIf(!enabled)("under an outer approval, a failed migration keeps its new column and the rerun resumes (#3687)", () => {
82
+ const SCHEMA: Obj = { export: "app", type: "Postgres::Schema", ddl: `CREATE SCHEMA ${S}` };
83
+ const V1: Obj = { export: "events", type: "Postgres::Table", dependsOn: ["app"], ddl: `CREATE TABLE ${S}.events (id bigint PRIMARY KEY, kind varchar(20) NOT NULL)` };
84
+ const V2: Obj = { ...V1, ddl: `CREATE TABLE ${S}.events (id bigint PRIMARY KEY, kind varchar(10) NOT NULL)` };
85
+ const config = (): PostgresMigrationOpConfig => ({
86
+ name: "migrate-events-kind",
87
+ env: "e2e",
88
+ table: `${S}.events`,
89
+ column: "kind",
90
+ build: false,
91
+ path: dir,
92
+ output: "events-v2.json",
93
+ retain: "0s",
94
+ batchSize: 100,
95
+ stack: MARKER.stack,
96
+ ownershipEnv: MARKER.env,
97
+ gates: "outer",
98
+ onFailure: "keep",
99
+ });
100
+ const args = (): PostgresMigrationArgs => ({ table: `${S}.events`, column: "kind", buildPath: "events-v2.json", environment: "e2e", batchSize: 100, stack: MARKER.stack, ownershipEnv: MARKER.env, cwd: dir });
101
+
102
+ test("the failed run keeps the new column and its receipts; the rerun skips what was filled, switches without a gate and keeps the old column", async () => {
103
+ await apply(writeBuild("events-v1.json", [SCHEMA, V1]));
104
+ await admin.query(`INSERT INTO ${S}.events SELECT g, CASE WHEN g % 2 = 0 THEN 'click' ELSE 'view' END FROM generate_series(0, 599) g`);
105
+ writeBuild("events-v2.json", [SCHEMA, V2]);
106
+
107
+ // A refusal says how to start again when nothing drops what was kept; without keepOnFailure it does not.
108
+ await expect(m.postgresMigrationBackfill({ ...args(), keepOnFailure: true }, undefined, deps())).rejects.toThrow(
109
+ /the Expand phase adds it\. This migration keeps what the expand added on failure \(onFailure: "keep"\), so nothing drops it: to start again from a new column, run the Op once with onFailure: "drop"/,
110
+ );
111
+ await expect(m.postgresMigrationBackfill(args(), undefined, deps())).rejects.not.toThrow(/onFailure: "keep"/);
112
+
113
+ // Every attempt of the backfill step fails after its batch: three attempts, three batches filled, then the run fails.
114
+ let filled = 0;
115
+ const ledger = new ApprovingLedger();
116
+ const failure = await runMigrationOp(config(), ledger.port, () =>
117
+ deps({
118
+ backfill: {
119
+ afterBatch: () => {
120
+ filled++;
121
+ throw new Error(`the backfill failed after ${filled} batch(es)`);
122
+ },
123
+ },
124
+ }),
125
+ ).then(
126
+ () => undefined,
127
+ (e: unknown) => e,
128
+ );
129
+ expect(failure).toBeInstanceOf(OpRunFailure);
130
+ const records = (failure as OpRunFailure).result.records;
131
+ expect(records.find((x) => x.fn === "postgresMigrationBackfill")?.error).toMatch(/the backfill failed after 3 batch/);
132
+ // No onFailure ran: the new column, the dual write and the receipts are there.
133
+ expect(records.find((x) => x.fn === "postgresMigrationCompensate")).toBeUndefined();
134
+ expect(await columnsOf(`${S}.events`)).toEqual(["id", "kind", "kind__chant_new"]);
135
+ expect(await one(`SELECT count(*)::int AS n FROM pg_trigger WHERE tgrelid = '${S}.events'::regclass AND tgname = 'kind__chant_sync'`)).toEqual({ n: 1 });
136
+ expect(await one(`SELECT count(*)::int AS n FROM ${S}.events WHERE kind__chant_new IS NOT NULL`)).toEqual({ n: 300 });
137
+ expect(await receipts()).toBe(3);
138
+
139
+ // The rerun resumes: three batches skipped by receipt, three filled, verified and switched, with no gate.
140
+ const r = await runMigrationOp(config(), ledger.port, () => deps());
141
+ expect(r.status).toBe("ok");
142
+ expect(ledger.port.appended).toEqual([]);
143
+ expect(runOutcome(r, "Skipped")).toBe(3);
144
+ expect(runOutcome(r, "Filled")).toBe(3);
145
+ expect(runOutcome(r, "VerifiedRows")).toBe(600);
146
+ expect(r.records.map((x) => x.fn)).not.toContain("postgresMigrationContract");
147
+ expect(await one(`SELECT format_type(atttypid, atttypmod) AS t FROM pg_attribute WHERE attrelid = '${S}.events'::regclass AND attname = 'kind'`)).toEqual({ t: "character varying(10)" });
148
+ // The old column is kept: dropping it is a run with the Op's own gates.
149
+ expect(await columnsOf(`${S}.events`)).toEqual(["id", "kind__chant_old", "kind"]);
150
+
151
+ // Run with its own gates, the Op has nothing left to switch, and its contract gate binds that old column.
152
+ const own = new ApprovingLedger();
153
+ const first = await runMigrationOp({ ...config(), gates: "own" }, own.port, () => deps());
154
+ expect(first.status).toBe("gated");
155
+ expect(first.gate).toMatchObject({ gate: "approve-migrate-events-kind" });
156
+ expect(runOutcome(first, "MigrationState")).toBe("switched");
157
+ own.approveLast();
158
+ const second = await runMigrationOp({ ...config(), gates: "own" }, own.port, () => deps());
159
+ expect(second.status).toBe("gated");
160
+ expect(second.gate).toMatchObject({ gate: "approve-migrate-events-kind-contract" });
161
+ expect(await columnsOf(`${S}.events`)).toEqual(["id", "kind__chant_old", "kind"]);
162
+ }, 300_000);
163
+ });