@ultimat3/db 29.3.2 → 30.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CLAUDE.md +78 -116
  2. package/README.md +199 -16
  3. package/package.json +13 -2
  4. package/src/app-relation.ts +7 -2
  5. package/src/catalog-fold.ts +5 -0
  6. package/src/catalog-objects.ts +20 -24
  7. package/src/catalog-policies.ts +81 -0
  8. package/src/catalog-relations.ts +31 -10
  9. package/src/catalog.ts +40 -1
  10. package/src/check-ddl.ts +34 -7
  11. package/src/client.ts +132 -48
  12. package/src/column-alter.ts +42 -1
  13. package/src/concurrent-index.ts +157 -0
  14. package/src/concurrent-plan.ts +294 -0
  15. package/src/connection-url.ts +16 -4
  16. package/src/db-diagnostics-deadline.ts +43 -0
  17. package/src/db-diagnostics-redact.ts +58 -0
  18. package/src/db-diagnostics-rows.ts +166 -0
  19. package/src/db-diagnostics-sql.ts +92 -0
  20. package/src/db-diagnostics.ts +178 -0
  21. package/src/destructive.ts +21 -35
  22. package/src/diagnostics-entry.ts +15 -0
  23. package/src/drift-errors.ts +9 -4
  24. package/src/drift-findings.ts +47 -62
  25. package/src/drift-missing.ts +203 -0
  26. package/src/drift-row-security.ts +88 -0
  27. package/src/drift-triggers.ts +54 -0
  28. package/src/drift.ts +88 -21
  29. package/src/entity-shape.ts +75 -0
  30. package/src/errors.ts +80 -9
  31. package/src/expected-loop.ts +34 -4
  32. package/src/failure-store-pg.ts +157 -0
  33. package/src/foreign-key-plan.ts +75 -23
  34. package/src/foreign-key.ts +76 -5
  35. package/src/generate-counter.ts +85 -0
  36. package/src/generate-history.ts +142 -0
  37. package/src/generate-row-security.ts +51 -0
  38. package/src/generate-snapshot.ts +82 -0
  39. package/src/generate-steps.ts +99 -0
  40. package/src/generate-triggers.ts +119 -0
  41. package/src/generate-view.ts +97 -0
  42. package/src/generate.ts +128 -89
  43. package/src/index-ddl.ts +23 -35
  44. package/src/index-plan.ts +35 -14
  45. package/src/index.ts +136 -6
  46. package/src/introspect-catalog.ts +26 -6
  47. package/src/introspect-rows.ts +82 -0
  48. package/src/introspect.ts +108 -50
  49. package/src/keyset.ts +182 -0
  50. package/src/lifecycle-errors.ts +15 -0
  51. package/src/local-settings.ts +38 -0
  52. package/src/marker-line.ts +47 -0
  53. package/src/migrate-lock-retry.ts +60 -0
  54. package/src/migrate-rollback.ts +45 -6
  55. package/src/migrate.ts +67 -69
  56. package/src/migration-errors.ts +48 -1
  57. package/src/migration-ledger.ts +38 -4
  58. package/src/migration-step.ts +192 -0
  59. package/src/partition-order.ts +56 -0
  60. package/src/partitioned-tables.ts +24 -0
  61. package/src/pool-gate.ts +120 -0
  62. package/src/pool-listeners.ts +143 -0
  63. package/src/pool-pins.ts +68 -0
  64. package/src/pool-profile.ts +23 -7
  65. package/src/pool-reserve.ts +11 -4
  66. package/src/pool-retire.ts +22 -0
  67. package/src/pool-sweep.ts +115 -0
  68. package/src/pool-timer.ts +10 -0
  69. package/src/pooler-fix.ts +53 -0
  70. package/src/pooler.ts +161 -0
  71. package/src/read-transaction.ts +59 -0
  72. package/src/readonly-query.ts +4 -4
  73. package/src/replica-client.ts +54 -22
  74. package/src/replica-position.ts +89 -0
  75. package/src/replica-scope.ts +25 -4
  76. package/src/retired.ts +131 -0
  77. package/src/retype-dependents.ts +2 -0
  78. package/src/rolling-errors.ts +97 -0
  79. package/src/rolling.ts +212 -0
  80. package/src/row-secure-client.ts +54 -0
  81. package/src/row-security-catalog.ts +26 -0
  82. package/src/row-security-migrator.ts +48 -0
  83. package/src/row-security-plan.ts +58 -0
  84. package/src/row-security-role.ts +60 -0
  85. package/src/row-security-roles.ts +64 -0
  86. package/src/row-security.ts +59 -0
  87. package/src/same-tenant.ts +173 -0
  88. package/src/schema-dump-policies.ts +26 -0
  89. package/src/schema-dump-table.ts +21 -6
  90. package/src/schema-dump.ts +13 -2
  91. package/src/snapshot-parse.ts +42 -2
  92. package/src/sqlstate.ts +4 -0
  93. package/src/statement-bound.ts +82 -0
  94. package/src/transaction-errors.ts +14 -0
  95. package/src/transaction-guard.ts +40 -0
  96. package/src/transaction-settings.ts +24 -0
  97. package/src/transaction.ts +74 -19
  98. package/src/ungeneratable.ts +30 -0
package/src/generate.ts CHANGED
@@ -4,14 +4,19 @@
4
4
  // field. Every generated migration must be reversible; a drop that loses data refuses instead.
5
5
 
6
6
  import { systemClock } from '@ultimat3/core';
7
- import { checkClauses, checkPlan, declaredChecks } from './check-ddl';
8
- import { alterColumnInPlace } from './column-alter';
7
+ import { checkClauses, checkPlan } from './check-ddl';
8
+ import { alterColumnInPlace, retypeOverDefault, wantedDefault } from './column-alter';
9
9
  import { defaultExpression } from './column-default';
10
- import { isDestructiveMigration } from './destructive';
10
+ import { type ConcurrentPlan, emptyConcurrentPlan } from './concurrent-plan';
11
11
  import { dropOrder } from './drop-order';
12
12
  import type { ColumnDescriptionLike, EntityDescriptionLike } from './entity-shape';
13
- import { type ConstraintPlans, foreignKeyPlan, foreignKeysOf, type Plan } from './foreign-key-plan';
13
+ import { type ConstraintPlans, foreignKeyPlan, type Plan } from './foreign-key-plan';
14
14
  import { appendOnlyBackfillRefused, appendOnlyPlan } from './generate-append-only';
15
+ import { type RowSecurityOption, rowSecurityInput } from './generate-row-security';
16
+ import { snapshotOf } from './generate-snapshot';
17
+ import { assembleSteps } from './generate-steps';
18
+ import { recordTriggers, triggerPlan } from './generate-triggers';
19
+ import { splitViews, viewPlan } from './generate-view';
15
20
  import type { Regeneration } from './generated-column';
16
21
  import { generatedClause, isGenerated, regenerate } from './generated-column';
17
22
  import { createIndex, impliedByColumnClause } from './index-ddl';
@@ -27,12 +32,16 @@ import { migrationIrreversible } from './migration-errors';
27
32
  import { addChangedKey, dropChangedKey, migrationNameArg } from './primary-key';
28
33
  import type { ReplicaIdentityInput } from './replica-identity';
29
34
  import { replicaIdentityFullAfter, replicaIdentityPlan } from './replica-identity';
35
+ import { contractsMarkerFor, recordRetired, retirePlan } from './retired';
30
36
  import type { MovedAside } from './retype-dependents';
31
37
  import { moveDependentsAside } from './retype-dependents';
32
38
  import { moveKeysAside, retypedColumns, retypedIn } from './retype-keys';
39
+ import { columnDropNotRetired } from './rolling-errors';
40
+ import { rowSecurityAfter, rowSecurityPlan } from './row-security-plan';
41
+ import { withSameTenantIndexes } from './same-tenant';
33
42
  import { identifier } from './sql';
34
43
  import { sqlType } from './sql-type';
35
- import { type UnrenderedDeclaration, unrenderedComment, unrenderedOf } from './unrendered';
44
+ import { type UnrenderedDeclaration, unrenderedOf } from './unrendered';
36
45
 
37
46
  function columnClause(column: ColumnDescriptionLike): string {
38
47
  // The generation clause sits directly after the type, and `generatedClause` refuses the pairs
@@ -59,67 +68,6 @@ function columnClause(column: ColumnDescriptionLike): string {
59
68
  return parts.join(' ');
60
69
  }
61
70
 
62
- /** No table subscribed to, which is what every caller outside `generateMigration` describes. */
63
- const NONE: ReadonlySet<string> = new Set();
64
-
65
- export function snapshotOf(
66
- entities: readonly EntityDescriptionLike[],
67
- replicaIdentityFull: ReadonlySet<string> = NONE,
68
- ): SchemaDescription {
69
- const tables = [...entities]
70
- .sort((a, b) => (a.table < b.table ? -1 : 1))
71
- .map((entity): TableDescription => {
72
- const columns: ColumnDescription[] = [...entity.columns]
73
- .sort((a, b) => (a.column < b.column ? -1 : 1))
74
- .map((column, index) => ({
75
- name: column.column,
76
- dataType: sqlType(column.kind),
77
- nullable: !column.notNull,
78
- default: defaultExpression(column),
79
- position: index + 1,
80
- // Only when one was declared — absent stays absent, so no snapshot written before this
81
- // field existed gains a key and no app's sidecar regenerates over a fact already true.
82
- ...(column.generated === undefined ? {} : { generated: column.generated }),
83
- }));
84
- const checks = declaredChecks(entity);
85
- return {
86
- schema: 'public',
87
- name: entity.table,
88
- columns,
89
- primaryKey: [...entity.primaryKey],
90
- // Whole, never partly: a snapshot that recorded the name and dropped the predicate made
91
- // the next generation blind to a `where` or an `order` changing, and a partial index
92
- // silently kept as a total one is a constraint the entity no longer declares.
93
- //
94
- // `declaredIndexes`, so a `unique` invariant's index is recorded exactly like an entity's
95
- // own — a statement emitted and not recorded is `42P07` on the very next `x db gen`.
96
- indexes: declaredIndexes(entity).map((index) => ({
97
- name: index.name,
98
- columns: [...index.columns],
99
- unique: index.unique,
100
- primary: false,
101
- where: index.where,
102
- order: index.order,
103
- // Only when one was declared. Writing `using: 'btree'` out for every index would rewrite
104
- // every sidecar in every app on the next `x db gen` — a diff on every file for a fact
105
- // that was already true, which `indexMethodOf` reads out of the absence anyway.
106
- ...(index.using === undefined ? {} : { using: index.using }),
107
- })),
108
- foreignKeys: foreignKeysOf(entity),
109
- // Absent, never `[]`, on a table declaring none: a sidecar that predates this field must
110
- // read as "nothing recorded" so the next generation adds the constraints the database is
111
- // genuinely missing — the rule `using` and `generated` already state one field up.
112
- ...(checks.length === 0 ? {} : { checks }),
113
- // The same rule once more: `true` or absent, never `false`. This is what makes the ALTER
114
- // beside it a one-time statement rather than a line every `x db gen` writes again.
115
- ...(replicaIdentityFull.has(entity.table) ? { replicaIdentityFull: true as const } : {}),
116
- // `true` or absent once more — the record that makes the trigger a one-time statement.
117
- ...(entity.appendOnly === true ? { appendOnly: true as const } : {}),
118
- };
119
- });
120
- return { tables };
121
- }
122
-
123
71
  function createTable(entity: EntityDescriptionLike): readonly string[] {
124
72
  const clauses = entity.columns.map(columnClause);
125
73
  if (entity.primaryKey.length > 0) {
@@ -176,8 +124,8 @@ function retypeColumn(
176
124
  const alter = (type: string): string =>
177
125
  `alter table ${identifier(table).text} alter column ${identifier(column.column).text} ` +
178
126
  `type ${type} using ${identifier(column.column).text}::${type};`;
179
- plan.up.push(alter(wanted));
180
- plan.down.push(alter(recorded.dataType));
127
+ retypeOverDefault(table, column, recorded, plan, alter(wanted), alter(recorded.dataType));
128
+ moved.defaults.add(column.column);
181
129
  return 'altered';
182
130
  }
183
131
 
@@ -186,6 +134,7 @@ function diffTable(
186
134
  live: TableDescription,
187
135
  plan: Plan,
188
136
  retyped: ReadonlySet<string>,
137
+ concurrent: ConcurrentPlan,
189
138
  ): void {
190
139
  const existing = new Map(live.columns.map((column) => [column.name, column]));
191
140
  const added = new Set<string>();
@@ -193,16 +142,27 @@ function diffTable(
193
142
  // over it has to be stated again even though its own definition never moved.
194
143
  const rebuilt = new Set<string>();
195
144
  // What a retype dropped ahead of itself, read by the two arms below.
196
- const moved: MovedAside = { indexes: new Set(), checks: new Set() };
145
+ const moved: MovedAside = { indexes: new Set(), checks: new Set(), defaults: new Set() };
197
146
  for (const column of entity.columns) {
198
147
  const recorded = existing.get(column.column);
199
148
  if (recorded !== undefined) {
200
149
  if (retypeColumn(live, column, recorded, plan, moved, retyped) === 'rebuilt') {
201
150
  rebuilt.add(column.column);
202
151
  } else {
203
- // After any retype, so a new default is set against the column's new type. A rebuilt
204
- // column was re-added carrying its whole clause, so it has nothing left to move.
205
- alterColumnInPlace(entity, column, recorded, plan);
152
+ // After any retype, so nullability moves on the column's new type. A rebuilt column was
153
+ // re-added carrying its whole clause, so it has nothing left to move. A retype already
154
+ // moved the default, across the type (`column-alter.ts`).
155
+ alterColumnInPlace(
156
+ entity,
157
+ column,
158
+ moved.defaults.has(column.column)
159
+ ? {
160
+ ...recorded,
161
+ default: wantedDefault(column, recorded),
162
+ }
163
+ : recorded,
164
+ plan,
165
+ );
206
166
  }
207
167
  continue;
208
168
  }
@@ -233,15 +193,17 @@ function diffTable(
233
193
 
234
194
  // Both directions, in `index-plan.ts`: a recorded index the entity no longer declares is DROPPED
235
195
  // there, which is the arm this loop did not have for as long as it lived here.
236
- indexPlan(entity, live, plan, { added, rebuilt, moved: moved.indexes });
196
+ indexPlan(entity, live, plan, { added, rebuilt, moved: moved.indexes, concurrent });
237
197
 
238
198
  // Last: a CHECK may read a column this migration just added, and `add constraint` on a column
239
199
  // that does not exist yet is `42703`. `check-ddl.ts` owns which of them move; `rebuilt` because a
240
200
  // column dropped and re-added lost its constraint while the snapshot still records it, and
241
201
  // `moved.checks` because a retype already dropped the ones written against the old type.
242
- checkPlan(entity, live, plan, rebuilt, moved.checks);
202
+ checkPlan(entity, live, plan, rebuilt, moved.checks, concurrent);
243
203
  }
244
204
 
205
+ export { snapshotOf } from './generate-snapshot';
206
+
245
207
  export interface GenerateOptions {
246
208
  readonly entities: readonly EntityDescriptionLike[];
247
209
  /** The schema migrations already declare — `expectedSchema(migrations, ledger)`. */
@@ -250,6 +212,12 @@ export interface GenerateOptions {
250
212
  readonly now?: Date | undefined;
251
213
  /** Allow a DROP COLUMN whose down cannot restore the data. `x db gen --allow-destructive`. */
252
214
  readonly allowDestructive?: boolean | undefined;
215
+ /**
216
+ * Drop a column that was never retired, for an app that stops before it starts: one instance, or
217
+ * `strategy: Recreate`. `x db gen --rolling-unsafe`. Consent to the outage and not to the loss,
218
+ * so `allowDestructive` is still asked of it.
219
+ */
220
+ readonly rollingUnsafe?: boolean | undefined;
253
221
  /**
254
222
  * Tables a live query subscribes to, so logical replication carries the old row —
255
223
  * `@ultimat3/realtime` refuses a subscription without it. Absent or empty emits nothing.
@@ -261,6 +229,12 @@ export interface GenerateOptions {
261
229
  * is the one caller that can answer it.
262
230
  */
263
231
  readonly replicaIdentityFull?: readonly string[] | undefined;
232
+ /**
233
+ * `database.rowSecurity` and each tenant table's column (`describeTenancy()`): a parameter for
234
+ * `replicaIdentityFull`'s reason — a config fact no entity carries. Absent is off, so a recorded
235
+ * policy is dropped: `@ultimat3/cli`'s `db-generate.ts` passes it on every run.
236
+ */
237
+ readonly rowSecurity?: RowSecurityOption | undefined;
264
238
  }
265
239
 
266
240
  export interface GeneratedMigration {
@@ -276,6 +250,11 @@ export interface GeneratedMigration {
276
250
  * cannot be written unmarked and then refused by `x verify` for lacking the mark.
277
251
  */
278
252
  readonly destructive: boolean;
253
+ /**
254
+ * `true` when `--rolling-unsafe` let this `up` drop a column it never retired. Absent otherwise.
255
+ * `x db gen` writes it as `-- rolling-unsafe: true`, the line the `drift` step's rail reads.
256
+ */
257
+ readonly rollingUnsafe?: boolean | undefined;
279
258
  /**
280
259
  * Every declaration this generator could not write down. Empty on a migration that carries the
281
260
  * whole schema, which is what makes it readable as a verdict rather than as noise — and the same
@@ -283,6 +262,18 @@ export interface GeneratedMigration {
283
262
  * (`x db gen`) and a reviewer reading the committed file are looking at one answer.
284
263
  */
285
264
  readonly unrendered: readonly UnrenderedDeclaration[];
265
+ /**
266
+ * `false` when `up` opens with `-- transaction: false` and the runner applies it statement by
267
+ * statement outside a transaction — the step holding every concurrent index build and every
268
+ * index removal (`concurrent-plan.ts`).
269
+ */
270
+ readonly transactional: boolean;
271
+ /**
272
+ * The non-transactional step this diff ALSO needs, when it needs both: `<id>_concurrently`,
273
+ * sorting right after this one, written as a file of its own. Absent when the diff holds only
274
+ * one kind — a diff that only builds or drops indexes IS the non-transactional step.
275
+ */
276
+ readonly followUp?: GeneratedMigration | undefined;
286
277
  }
287
278
 
288
279
  export function migrationStamp(now: Date): string {
@@ -296,9 +287,16 @@ export function slugify(name: string): string {
296
287
  .replace(/^_+|_+$/g, '');
297
288
  }
298
289
 
299
- export function generateMigration(options: GenerateOptions): GeneratedMigration {
290
+ export function generateMigration(given: GenerateOptions): GeneratedMigration {
291
+ // One list for every arm below: a target's same-tenant index is declared by whoever points at it,
292
+ // and neither `createTable`, `diffTable` nor a snapshot may hold a different answer.
293
+ // Views leave here, before any table arm: none of them is a table (`generate-view.ts`).
294
+ const { tables, views } = splitViews(given.entities);
295
+ const options = { ...given, entities: withSameTenantIndexes(tables, given.current) };
300
296
  const current = options.current ?? { tables: [] };
301
297
  const plan: Plan = { up: [], down: [] };
298
+ // Applied AFTER `plan`, outside its transaction, in a step of its own (`concurrent-plan.ts`).
299
+ const concurrent = emptyConcurrentPlan();
302
300
  // Merged into `plan` once every table statement is in, never interleaved with them.
303
301
  const constraints: Plan = { up: [], down: [] };
304
302
  // Merged in BEFORE them, for the mirror-image reason: a key still pointing at a table this
@@ -320,11 +318,11 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
320
318
  // and the key that a retype of its column breaks is recorded on whichever table OWNS the key.
321
319
  const retyped = retypedColumns(options.entities, current);
322
320
  const predropped = moveKeysAside(current, retyped, doomed, preAlters);
323
- const plans: ConstraintPlans = { constraints, preDrops, doomed, predropped };
321
+ const plans: ConstraintPlans = { constraints, preDrops, doomed, predropped, concurrent };
322
+ let droppedUnretired = false;
324
323
 
325
324
  for (const entity of options.entities) {
326
325
  const live = findTable(current, entity.table);
327
- foreignKeyPlan(entity, live, plans);
328
326
  if (live === undefined) {
329
327
  plan.up.push(...createTable(entity));
330
328
  plan.down.push(`drop table ${identifier(entity.table).text};`);
@@ -333,17 +331,36 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
333
331
  }
334
332
  // The key first and last, around every column statement of the table (`primary-key.ts`): the
335
333
  // snapshot below records `entity.primaryKey`, and until this arm existed nothing produced it.
334
+ retirePlan(entity, live, plan, options.entities);
336
335
  dropChangedKey(entity, live, current, plan, options.name);
337
- diffTable(entity, live, plan, retypedIn(retyped, entity.table));
338
- const kept = new Set(entity.columns.map((column) => column.column));
336
+ diffTable(entity, live, plan, retypedIn(retyped, entity.table), concurrent);
337
+ const kept = new Set([
338
+ ...entity.columns.map((column) => column.column),
339
+ ...(entity.retired ?? []),
340
+ ]);
339
341
  for (const column of live.columns) {
340
342
  if (kept.has(column.name)) continue;
343
+ // Order matters: the release boundary first, with the flag or without it. Retiring is what
344
+ // earns the next question; it is never consent to losing the rows, so that one still stands.
345
+ if (column.retired !== true && options.rollingUnsafe !== true) {
346
+ throw columnDropNotRetired(
347
+ entity.name,
348
+ entity.table,
349
+ column.name,
350
+ migrationNameArg(options.name),
351
+ );
352
+ }
341
353
  if (options.allowDestructive !== true) {
342
354
  throw migrationIrreversible(
343
355
  `dropping "${entity.table}"."${column.name}" discards its rows and cannot be undone`,
344
- `x db gen ${migrationNameArg(options.name)} --allow-destructive # or keep the column and deprecate it`,
356
+ `x db gen ${migrationNameArg(options.name)} --allow-destructive # or put "${column.name}" back in columns and keep it`,
345
357
  );
346
358
  }
359
+ if (column.retired === true) {
360
+ plan.up.push(contractsMarkerFor(entity.table, column.name));
361
+ } else {
362
+ droppedUnretired = true;
363
+ }
347
364
  plan.up.push(
348
365
  `alter table ${identifier(entity.table).text} drop column ${identifier(column.name).text};`,
349
366
  );
@@ -356,6 +373,11 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
356
373
  addChangedKey(entity, live, plan, options.name);
357
374
  }
358
375
 
376
+ // After every table's diff, so the deferred step already says which target indexes it builds.
377
+ for (const entity of options.entities) {
378
+ foreignKeyPlan(entity, findTable(current, entity.table), plans);
379
+ }
380
+
359
381
  const order = dropOrder(current.tables.filter((table) => !wanted.has(table.name)));
360
382
  for (const table of order.tables) {
361
383
  if (options.allowDestructive !== true) {
@@ -381,6 +403,11 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
381
403
 
382
404
  // After every table exists, so a `create table` in this same migration is already above it.
383
405
  appendOnlyPlan(plan, options.entities, current, created);
406
+ // History, its guard and counters, the same way: after every table they name exists.
407
+ triggerPlan(plan, options.entities, current);
408
+ // Then each tenant table's policy, both directions (`row-security-plan.ts`), on tables that exist.
409
+ const rowSecurity = rowSecurityInput(options.entities, current, options.rowSecurity);
410
+ rowSecurityPlan(plan, rowSecurity);
384
411
 
385
412
  // Dead last in `up`, and it is the only placement that is right for every arm: the table has to
386
413
  // exist, and a `create table` in this same migration is the reason it might not. It is ordered
@@ -408,19 +435,31 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
408
435
  // `current`, not the entities alone: an `assert` whose CHECK a previous migration recorded is a
409
436
  // loss only because THIS plan drops it, and the recorded schema is the only thing that knows.
410
437
  const unrendered = unrenderedOf(options.entities, current);
411
- const body = [...preAlters.up, ...plan.up].join('\n');
412
- const up = body.length === 0 ? body : unrenderedComment(unrendered) + body;
413
- return {
438
+ // Dropped ahead of every table statement and created after the last one: a view compiled against
439
+ // a column this migration retypes or drops is `0A000`/`2BP01` otherwise.
440
+ const viewed = viewPlan(views, current, [...preAlters.up, ...plan.up]);
441
+ return assembleSteps({
414
442
  id,
415
443
  name: options.name,
416
- fileName: `migrations/${id}.sql`,
417
- up,
444
+ up: [...viewed.before.up, ...preAlters.up, ...plan.up, ...viewed.after.up],
418
445
  // Reverse order: the last thing created is the first thing dropped. `preAlters` goes in at the
419
446
  // FRONT here precisely so reversal puts it last — a key is added back only once both of its
420
447
  // ends have been retyped back, which is every other statement in the script.
421
- down: [...preAlters.down, ...plan.down].reverse().join('\n'),
422
- snapshot: snapshotOf(options.entities, replicaIdentityFullAfter(replicaIdentity)),
423
- destructive: isDestructiveMigration(up),
448
+ down: [...viewed.before.down, ...preAlters.down, ...plan.down, ...viewed.after.down].reverse(),
449
+ concurrent,
450
+ snapshot: recordTriggers(
451
+ recordRetired(
452
+ snapshotOf(
453
+ [...options.entities, ...views],
454
+ replicaIdentityFullAfter(replicaIdentity),
455
+ rowSecurityAfter(rowSecurity),
456
+ ),
457
+ options.entities,
458
+ current,
459
+ ),
460
+ options.entities,
461
+ ),
424
462
  unrendered,
425
- };
463
+ ...(droppedUnretired ? { rollingUnsafe: true } : {}),
464
+ });
426
465
  }
package/src/index-ddl.ts CHANGED
@@ -83,8 +83,17 @@ function indexPredicateSql(index: IndexDescriptionLike): string {
83
83
  * Every part of the declaration reaches the statement: the whole column list in its declared
84
84
  * order, the direction when one was asked for, and the predicate that makes it partial. A part
85
85
  * dropped here is a constraint the database does not hold or an index the planner cannot use.
86
+ *
87
+ * `concurrently` writes `create index concurrently if not exists` — the spelling the
88
+ * non-transactional step needs (`concurrent-index.ts`): re-running a step that failed after this
89
+ * build finished must find it and move on, and one that failed DURING it left an INVALID index the
90
+ * runner drops first (`migration-step.ts`). Off, the statement is byte-for-byte what shipped before.
86
91
  */
87
- export function createIndex(table: string, index: IndexDescriptionLike): string {
92
+ export function createIndex(
93
+ table: string,
94
+ index: IndexDescriptionLike,
95
+ concurrently = false,
96
+ ): string {
88
97
  assert(
89
98
  index.columns.length > 0,
90
99
  `index "${index.name}" on "${table}" names no columns`,
@@ -106,7 +115,9 @@ export function createIndex(table: string, index: IndexDescriptionLike): string
106
115
  `index "${index.name}" on "${table}" is ${method} and ${index.order}; only a btree orders its keys`,
107
116
  `indexes: [{ on: ['<column>'], using: '${method}' }] # drop order, or drop using`,
108
117
  );
109
- const kind = index.unique ? 'create unique index' : 'create index';
118
+ const kind =
119
+ (index.unique ? 'create unique index' : 'create index') +
120
+ (concurrently ? ' concurrently if not exists' : '');
110
121
  const direction = indexOrderSql(index);
111
122
  const columns = index.columns
112
123
  .map((column) => `${identifier(column).text}${direction}`)
@@ -124,6 +135,15 @@ export function createIndex(table: string, index: IndexDescriptionLike): string
124
135
  /** `drop index "n";` — the one spelling, so a drop and its recreate cannot name it differently. */
125
136
  export const dropIndex = (name: string): string => `drop index ${identifier(name).text};`;
126
137
 
138
+ /**
139
+ * `drop index concurrently if exists "n";` — the non-transactional drop. `if exists` because a
140
+ * re-run of a step that already dropped it must move on, and because it also removes an INVALID
141
+ * index a failed concurrent build left behind. Postgres refuses it on a constraint's index (2BP01),
142
+ * so a recorded index that may be one is unhooked first by `dropRecordedConcurrently`.
143
+ */
144
+ export const dropIndexConcurrently = (name: string): string =>
145
+ `drop index concurrently if exists ${identifier(name).text};`;
146
+
127
147
  /**
128
148
  * Whether Postgres could be backing this RECORDED index with a UNIQUE constraint rather than
129
149
  * holding it as an index of its own — which decides the only question `dropIndex` cannot answer.
@@ -131,7 +151,7 @@ export const dropIndex = (name: string): string => `drop index ${identifier(name
131
151
  * A UNIQUE constraint's index is unique, total, unordered and btree; `add constraint … unique`
132
152
  * and a `unique` column clause can produce nothing else. So an index missing any one of those is
133
153
  * provably an index, and `drop index` on it is right. Everything else is genuinely ambiguous —
134
- * see `dropRecordedIndex`.
154
+ * see `dropRecordedConcurrently` (`concurrent-plan.ts`).
135
155
  */
136
156
  export function mayBeConstraintBacked(index: IndexDescription): boolean {
137
157
  return (
@@ -143,38 +163,6 @@ export function mayBeConstraintBacked(index: IndexDescription): boolean {
143
163
  );
144
164
  }
145
165
 
146
- /**
147
- * Remove a RECORDED index whose kind this generator cannot know, in statements that are correct
148
- * on both databases it cannot tell apart.
149
- *
150
- * `TableDescription` carries no discriminator, and it cannot be given one that would help: the
151
- * SAME declaration reaches the server as a CONSTRAINT or as an INDEX depending on which migration
152
- * created it. A `unique` column on a table `createTable` writes emits `create table … slug text
153
- * unique`, and Postgres backs that with a constraint named `posts_slug_key`; the same column
154
- * gaining `unique` later takes `diffTable`'s `create unique index "posts_slug_key"` and is a plain
155
- * index. `snapshotOf` records both as `{ unique: true, primary: false }`, and every sidecar already
156
- * on disk was written that way — a new field could not classify one of them retroactively.
157
- *
158
- * Measured on 18.4 (`index-removal.live.test.ts`), which is why the pair and not a guess:
159
- *
160
- * | statement | on a constraint's index | on a plain index |
161
- * |--------------------------------------------|-------------------------|------------------|
162
- * | `drop index "n"` | **2BP01** | ok |
163
- * | `drop index if exists "n"` | **2BP01** — not suppressed | ok |
164
- * | `alter table … drop constraint if exists` | drops it, index and all | notice, no-op |
165
- *
166
- * Constraint first, then the index: reversed, the `drop index` reaches a constraint's index and is
167
- * the 2BP01 this exists to avoid. Both halves carry `if exists`, so whichever one did nothing says
168
- * so with a notice rather than 42704.
169
- */
170
- export function dropRecordedIndex(table: string, index: IndexDescription): readonly string[] {
171
- if (!mayBeConstraintBacked(index)) return [dropIndex(index.name)];
172
- return [
173
- `alter table ${identifier(table).text} drop constraint if exists ${identifier(index.name).text};`,
174
- `drop index if exists ${identifier(index.name).text};`,
175
- ];
176
- }
177
-
178
166
  /**
179
167
  * A RECORDED index as a declaration this generator can emit again — `declaredMethod`, never a
180
168
  * cast: the recorded side is typed open because the catalog shares the shape, and a method this
package/src/index-plan.ts CHANGED
@@ -16,16 +16,15 @@
16
16
  // declaration is already broken — the key has nothing to point at — and the failure arrives with
17
17
  // the server's own words naming both ends.
18
18
 
19
+ import {
20
+ buildConcurrently,
21
+ type ConcurrentPlan,
22
+ redefineConcurrently,
23
+ removeConcurrently,
24
+ } from './concurrent-plan';
19
25
  import type { EntityDescriptionLike } from './entity-shape';
20
26
  import type { Plan } from './foreign-key-plan';
21
- import {
22
- asDeclared,
23
- createIndex,
24
- dropIndex,
25
- dropRecordedIndex,
26
- impliedByColumnClause,
27
- redefineIndex,
28
- } from './index-ddl';
27
+ import { createIndex, dropIndex, impliedByColumnClause, redefineIndex } from './index-ddl';
29
28
  import type { TableDescription } from './introspect';
30
29
  import { declaredIndexes } from './invariant-ddl';
31
30
 
@@ -40,6 +39,11 @@ export interface IndexPlanContext {
40
39
  readonly rebuilt: ReadonlySet<string>;
41
40
  /** Indexes a retype already dropped ahead of its ALTER (`moveDependentsAside`). */
42
41
  readonly moved: ReadonlySet<string>;
42
+ /**
43
+ * The non-transactional step (`concurrent-plan.ts`): every `concurrently: true` build and every
44
+ * removal lands here, outside the transaction `plan` is applied in.
45
+ */
46
+ readonly concurrent: ConcurrentPlan;
43
47
  }
44
48
 
45
49
  /**
@@ -68,19 +72,31 @@ export function indexPlan(
68
72
  // `redefineIndex` sees a definition that never moved and would emit nothing at all.
69
73
  const gone =
70
74
  context.moved.has(index.name) || index.columns.some((each) => context.rebuilt.has(each));
75
+ // `concurrently` only where the table is live and the index is the only thing moving: a
76
+ // rebuilt or retyped column already holds ACCESS EXCLUSIVE inside the transaction, so a
77
+ // concurrent rebuild after it buys nothing and leaves the table without the index between.
78
+ const concurrently = index.concurrently === true && !gone;
71
79
  if (recorded !== undefined && !gone) {
72
- redefineIndex(entity.table, index, recorded, plan);
80
+ if (concurrently) {
81
+ redefineConcurrently(entity.table, index, recorded, context.concurrent);
82
+ } else {
83
+ redefineIndex(entity.table, index, recorded, plan);
84
+ }
73
85
  continue;
74
86
  }
75
87
  // `added` only: an index over a column that was already there is implied by no clause this
76
88
  // migration emits, so it still needs a statement of its own.
77
89
  if (impliedByColumnClause(entity, index, context.added)) continue;
90
+ if (concurrently) {
91
+ buildConcurrently(entity.table, index, context.concurrent);
92
+ continue;
93
+ }
78
94
  plan.up.push(createIndex(entity.table, index));
79
95
  // The plain drop, always: this migration CREATED it, with `create index`, so it is an index
80
- // and never a constraint's — `dropRecordedIndex`'s ambiguity is about the recorded side only.
96
+ // and never a constraint's — the ambiguity is about the recorded side only.
81
97
  plan.down.push(dropIndex(index.name));
82
98
  }
83
- removeUndeclared(entity, live, plan, context, declared);
99
+ removeUndeclared(entity, live, context, declared);
84
100
  }
85
101
 
86
102
  /**
@@ -88,6 +104,13 @@ export function indexPlan(
88
104
  * the SNAPSHOT recorded, never from what the entity declares, since the entity is precisely what
89
105
  * stopped describing it. The rule the retype path already states.
90
106
  *
107
+ * ALWAYS concurrently, in the non-transactional step (`removeConcurrently`), whether or not the
108
+ * index was built that way: a plain `drop index` takes ACCESS EXCLUSIVE on the TABLE, so every
109
+ * read and write of a hot table queued behind it. Expand → contract: the drop — and the
110
+ * constraint unhook a UNIQUE shape needs — lands in the step's CONTRACT phase, after everything the
111
+ * transactional step and the step's own builds added, so a renamed index exists twice for a moment
112
+ * and never zero times. Nothing records how an index was built, so the drop cannot depend on it.
113
+ *
91
114
  * Four names are skipped, and each one is a statement Postgres would refuse or repeat:
92
115
  *
93
116
  * | skipped | because |
@@ -103,7 +126,6 @@ export function indexPlan(
103
126
  function removeUndeclared(
104
127
  entity: EntityDescriptionLike,
105
128
  live: TableDescription,
106
- plan: Plan,
107
129
  context: IndexPlanContext,
108
130
  declared: ReadonlySet<string>,
109
131
  ): void {
@@ -113,7 +135,6 @@ function removeUndeclared(
113
135
  if (context.moved.has(recorded.name)) continue;
114
136
  if (recorded.columns.some((column) => context.rebuilt.has(column))) continue;
115
137
  if (!recorded.columns.every((column) => columns.has(column))) continue;
116
- plan.up.push(...dropRecordedIndex(live.name, recorded));
117
- plan.down.push(createIndex(live.name, asDeclared(recorded)));
138
+ removeConcurrently(live.name, recorded, context.concurrent);
118
139
  }
119
140
  }