@ultimat3/db 29.3.2 → 29.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +16 -18
- package/README.md +36 -1
- package/package.json +2 -2
- package/src/concurrent-index.ts +137 -0
- package/src/concurrent-plan.ts +176 -0
- package/src/destructive.ts +4 -32
- package/src/entity-shape.ts +8 -0
- package/src/errors.ts +3 -0
- package/src/generate-steps.ts +94 -0
- package/src/generate.ts +25 -12
- package/src/index-ddl.ts +23 -35
- package/src/index-plan.ts +35 -14
- package/src/index.ts +7 -0
- package/src/marker-line.ts +39 -0
- package/src/migrate.ts +22 -65
- package/src/migration-errors.ts +21 -0
- package/src/migration-step.ts +164 -0
package/src/generate.ts
CHANGED
|
@@ -7,11 +7,12 @@ import { systemClock } from '@ultimat3/core';
|
|
|
7
7
|
import { checkClauses, checkPlan, declaredChecks } from './check-ddl';
|
|
8
8
|
import { alterColumnInPlace } from './column-alter';
|
|
9
9
|
import { defaultExpression } from './column-default';
|
|
10
|
-
import {
|
|
10
|
+
import { type ConcurrentPlan, emptyConcurrentPlan } from './concurrent-plan';
|
|
11
11
|
import { dropOrder } from './drop-order';
|
|
12
12
|
import type { ColumnDescriptionLike, EntityDescriptionLike } from './entity-shape';
|
|
13
13
|
import { type ConstraintPlans, foreignKeyPlan, foreignKeysOf, type Plan } from './foreign-key-plan';
|
|
14
14
|
import { appendOnlyBackfillRefused, appendOnlyPlan } from './generate-append-only';
|
|
15
|
+
import { assembleSteps } from './generate-steps';
|
|
15
16
|
import type { Regeneration } from './generated-column';
|
|
16
17
|
import { generatedClause, isGenerated, regenerate } from './generated-column';
|
|
17
18
|
import { createIndex, impliedByColumnClause } from './index-ddl';
|
|
@@ -32,7 +33,7 @@ import { moveDependentsAside } from './retype-dependents';
|
|
|
32
33
|
import { moveKeysAside, retypedColumns, retypedIn } from './retype-keys';
|
|
33
34
|
import { identifier } from './sql';
|
|
34
35
|
import { sqlType } from './sql-type';
|
|
35
|
-
import { type UnrenderedDeclaration,
|
|
36
|
+
import { type UnrenderedDeclaration, unrenderedOf } from './unrendered';
|
|
36
37
|
|
|
37
38
|
function columnClause(column: ColumnDescriptionLike): string {
|
|
38
39
|
// The generation clause sits directly after the type, and `generatedClause` refuses the pairs
|
|
@@ -186,6 +187,7 @@ function diffTable(
|
|
|
186
187
|
live: TableDescription,
|
|
187
188
|
plan: Plan,
|
|
188
189
|
retyped: ReadonlySet<string>,
|
|
190
|
+
concurrent: ConcurrentPlan,
|
|
189
191
|
): void {
|
|
190
192
|
const existing = new Map(live.columns.map((column) => [column.name, column]));
|
|
191
193
|
const added = new Set<string>();
|
|
@@ -233,7 +235,7 @@ function diffTable(
|
|
|
233
235
|
|
|
234
236
|
// Both directions, in `index-plan.ts`: a recorded index the entity no longer declares is DROPPED
|
|
235
237
|
// there, which is the arm this loop did not have for as long as it lived here.
|
|
236
|
-
indexPlan(entity, live, plan, { added, rebuilt, moved: moved.indexes });
|
|
238
|
+
indexPlan(entity, live, plan, { added, rebuilt, moved: moved.indexes, concurrent });
|
|
237
239
|
|
|
238
240
|
// Last: a CHECK may read a column this migration just added, and `add constraint` on a column
|
|
239
241
|
// that does not exist yet is `42703`. `check-ddl.ts` owns which of them move; `rebuilt` because a
|
|
@@ -283,6 +285,18 @@ export interface GeneratedMigration {
|
|
|
283
285
|
* (`x db gen`) and a reviewer reading the committed file are looking at one answer.
|
|
284
286
|
*/
|
|
285
287
|
readonly unrendered: readonly UnrenderedDeclaration[];
|
|
288
|
+
/**
|
|
289
|
+
* `false` when `up` opens with `-- transaction: false` and the runner applies it statement by
|
|
290
|
+
* statement outside a transaction — the step holding every concurrent index build and every
|
|
291
|
+
* index removal (`concurrent-plan.ts`).
|
|
292
|
+
*/
|
|
293
|
+
readonly transactional: boolean;
|
|
294
|
+
/**
|
|
295
|
+
* The non-transactional step this diff ALSO needs, when it needs both: `<id>_concurrently`,
|
|
296
|
+
* sorting right after this one, written as a file of its own. Absent when the diff holds only
|
|
297
|
+
* one kind — a diff that only builds or drops indexes IS the non-transactional step.
|
|
298
|
+
*/
|
|
299
|
+
readonly followUp?: GeneratedMigration | undefined;
|
|
286
300
|
}
|
|
287
301
|
|
|
288
302
|
export function migrationStamp(now: Date): string {
|
|
@@ -299,6 +313,8 @@ export function slugify(name: string): string {
|
|
|
299
313
|
export function generateMigration(options: GenerateOptions): GeneratedMigration {
|
|
300
314
|
const current = options.current ?? { tables: [] };
|
|
301
315
|
const plan: Plan = { up: [], down: [] };
|
|
316
|
+
// Applied AFTER `plan`, outside its transaction, in a step of its own (`concurrent-plan.ts`).
|
|
317
|
+
const concurrent = emptyConcurrentPlan();
|
|
302
318
|
// Merged into `plan` once every table statement is in, never interleaved with them.
|
|
303
319
|
const constraints: Plan = { up: [], down: [] };
|
|
304
320
|
// Merged in BEFORE them, for the mirror-image reason: a key still pointing at a table this
|
|
@@ -334,7 +350,7 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
|
|
|
334
350
|
// The key first and last, around every column statement of the table (`primary-key.ts`): the
|
|
335
351
|
// snapshot below records `entity.primaryKey`, and until this arm existed nothing produced it.
|
|
336
352
|
dropChangedKey(entity, live, current, plan, options.name);
|
|
337
|
-
diffTable(entity, live, plan, retypedIn(retyped, entity.table));
|
|
353
|
+
diffTable(entity, live, plan, retypedIn(retyped, entity.table), concurrent);
|
|
338
354
|
const kept = new Set(entity.columns.map((column) => column.column));
|
|
339
355
|
for (const column of live.columns) {
|
|
340
356
|
if (kept.has(column.name)) continue;
|
|
@@ -408,19 +424,16 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
|
|
|
408
424
|
// `current`, not the entities alone: an `assert` whose CHECK a previous migration recorded is a
|
|
409
425
|
// loss only because THIS plan drops it, and the recorded schema is the only thing that knows.
|
|
410
426
|
const unrendered = unrenderedOf(options.entities, current);
|
|
411
|
-
|
|
412
|
-
const up = body.length === 0 ? body : unrenderedComment(unrendered) + body;
|
|
413
|
-
return {
|
|
427
|
+
return assembleSteps({
|
|
414
428
|
id,
|
|
415
429
|
name: options.name,
|
|
416
|
-
|
|
417
|
-
up,
|
|
430
|
+
up: [...preAlters.up, ...plan.up],
|
|
418
431
|
// Reverse order: the last thing created is the first thing dropped. `preAlters` goes in at the
|
|
419
432
|
// FRONT here precisely so reversal puts it last — a key is added back only once both of its
|
|
420
433
|
// ends have been retyped back, which is every other statement in the script.
|
|
421
|
-
down: [...preAlters.down, ...plan.down].reverse()
|
|
434
|
+
down: [...preAlters.down, ...plan.down].reverse(),
|
|
435
|
+
concurrent,
|
|
422
436
|
snapshot: snapshotOf(options.entities, replicaIdentityFullAfter(replicaIdentity)),
|
|
423
|
-
destructive: isDestructiveMigration(up),
|
|
424
437
|
unrendered,
|
|
425
|
-
};
|
|
438
|
+
});
|
|
426
439
|
}
|
package/src/index-ddl.ts
CHANGED
|
@@ -83,8 +83,17 @@ function indexPredicateSql(index: IndexDescriptionLike): string {
|
|
|
83
83
|
* Every part of the declaration reaches the statement: the whole column list in its declared
|
|
84
84
|
* order, the direction when one was asked for, and the predicate that makes it partial. A part
|
|
85
85
|
* dropped here is a constraint the database does not hold or an index the planner cannot use.
|
|
86
|
+
*
|
|
87
|
+
* `concurrently` writes `create index concurrently if not exists` — the spelling the
|
|
88
|
+
* non-transactional step needs (`concurrent-index.ts`): re-running a step that failed after this
|
|
89
|
+
* build finished must find it and move on, and one that failed DURING it left an INVALID index the
|
|
90
|
+
* runner drops first (`migration-step.ts`). Off, the statement is byte-for-byte what shipped before.
|
|
86
91
|
*/
|
|
87
|
-
export function createIndex(
|
|
92
|
+
export function createIndex(
|
|
93
|
+
table: string,
|
|
94
|
+
index: IndexDescriptionLike,
|
|
95
|
+
concurrently = false,
|
|
96
|
+
): string {
|
|
88
97
|
assert(
|
|
89
98
|
index.columns.length > 0,
|
|
90
99
|
`index "${index.name}" on "${table}" names no columns`,
|
|
@@ -106,7 +115,9 @@ export function createIndex(table: string, index: IndexDescriptionLike): string
|
|
|
106
115
|
`index "${index.name}" on "${table}" is ${method} and ${index.order}; only a btree orders its keys`,
|
|
107
116
|
`indexes: [{ on: ['<column>'], using: '${method}' }] # drop order, or drop using`,
|
|
108
117
|
);
|
|
109
|
-
const kind =
|
|
118
|
+
const kind =
|
|
119
|
+
(index.unique ? 'create unique index' : 'create index') +
|
|
120
|
+
(concurrently ? ' concurrently if not exists' : '');
|
|
110
121
|
const direction = indexOrderSql(index);
|
|
111
122
|
const columns = index.columns
|
|
112
123
|
.map((column) => `${identifier(column).text}${direction}`)
|
|
@@ -124,6 +135,15 @@ export function createIndex(table: string, index: IndexDescriptionLike): string
|
|
|
124
135
|
/** `drop index "n";` — the one spelling, so a drop and its recreate cannot name it differently. */
|
|
125
136
|
export const dropIndex = (name: string): string => `drop index ${identifier(name).text};`;
|
|
126
137
|
|
|
138
|
+
/**
|
|
139
|
+
* `drop index concurrently if exists "n";` — the non-transactional drop. `if exists` because a
|
|
140
|
+
* re-run of a step that already dropped it must move on, and because it also removes an INVALID
|
|
141
|
+
* index a failed concurrent build left behind. Postgres refuses it on a constraint's index (2BP01),
|
|
142
|
+
* so a recorded index that may be one is unhooked first by `dropRecordedConcurrently`.
|
|
143
|
+
*/
|
|
144
|
+
export const dropIndexConcurrently = (name: string): string =>
|
|
145
|
+
`drop index concurrently if exists ${identifier(name).text};`;
|
|
146
|
+
|
|
127
147
|
/**
|
|
128
148
|
* Whether Postgres could be backing this RECORDED index with a UNIQUE constraint rather than
|
|
129
149
|
* holding it as an index of its own — which decides the only question `dropIndex` cannot answer.
|
|
@@ -131,7 +151,7 @@ export const dropIndex = (name: string): string => `drop index ${identifier(name
|
|
|
131
151
|
* A UNIQUE constraint's index is unique, total, unordered and btree; `add constraint … unique`
|
|
132
152
|
* and a `unique` column clause can produce nothing else. So an index missing any one of those is
|
|
133
153
|
* provably an index, and `drop index` on it is right. Everything else is genuinely ambiguous —
|
|
134
|
-
* see `
|
|
154
|
+
* see `dropRecordedConcurrently` (`concurrent-plan.ts`).
|
|
135
155
|
*/
|
|
136
156
|
export function mayBeConstraintBacked(index: IndexDescription): boolean {
|
|
137
157
|
return (
|
|
@@ -143,38 +163,6 @@ export function mayBeConstraintBacked(index: IndexDescription): boolean {
|
|
|
143
163
|
);
|
|
144
164
|
}
|
|
145
165
|
|
|
146
|
-
/**
|
|
147
|
-
* Remove a RECORDED index whose kind this generator cannot know, in statements that are correct
|
|
148
|
-
* on both databases it cannot tell apart.
|
|
149
|
-
*
|
|
150
|
-
* `TableDescription` carries no discriminator, and it cannot be given one that would help: the
|
|
151
|
-
* SAME declaration reaches the server as a CONSTRAINT or as an INDEX depending on which migration
|
|
152
|
-
* created it. A `unique` column on a table `createTable` writes emits `create table … slug text
|
|
153
|
-
* unique`, and Postgres backs that with a constraint named `posts_slug_key`; the same column
|
|
154
|
-
* gaining `unique` later takes `diffTable`'s `create unique index "posts_slug_key"` and is a plain
|
|
155
|
-
* index. `snapshotOf` records both as `{ unique: true, primary: false }`, and every sidecar already
|
|
156
|
-
* on disk was written that way — a new field could not classify one of them retroactively.
|
|
157
|
-
*
|
|
158
|
-
* Measured on 18.4 (`index-removal.live.test.ts`), which is why the pair and not a guess:
|
|
159
|
-
*
|
|
160
|
-
* | statement | on a constraint's index | on a plain index |
|
|
161
|
-
* |--------------------------------------------|-------------------------|------------------|
|
|
162
|
-
* | `drop index "n"` | **2BP01** | ok |
|
|
163
|
-
* | `drop index if exists "n"` | **2BP01** — not suppressed | ok |
|
|
164
|
-
* | `alter table … drop constraint if exists` | drops it, index and all | notice, no-op |
|
|
165
|
-
*
|
|
166
|
-
* Constraint first, then the index: reversed, the `drop index` reaches a constraint's index and is
|
|
167
|
-
* the 2BP01 this exists to avoid. Both halves carry `if exists`, so whichever one did nothing says
|
|
168
|
-
* so with a notice rather than 42704.
|
|
169
|
-
*/
|
|
170
|
-
export function dropRecordedIndex(table: string, index: IndexDescription): readonly string[] {
|
|
171
|
-
if (!mayBeConstraintBacked(index)) return [dropIndex(index.name)];
|
|
172
|
-
return [
|
|
173
|
-
`alter table ${identifier(table).text} drop constraint if exists ${identifier(index.name).text};`,
|
|
174
|
-
`drop index if exists ${identifier(index.name).text};`,
|
|
175
|
-
];
|
|
176
|
-
}
|
|
177
|
-
|
|
178
166
|
/**
|
|
179
167
|
* A RECORDED index as a declaration this generator can emit again — `declaredMethod`, never a
|
|
180
168
|
* cast: the recorded side is typed open because the catalog shares the shape, and a method this
|
package/src/index-plan.ts
CHANGED
|
@@ -16,16 +16,15 @@
|
|
|
16
16
|
// declaration is already broken — the key has nothing to point at — and the failure arrives with
|
|
17
17
|
// the server's own words naming both ends.
|
|
18
18
|
|
|
19
|
+
import {
|
|
20
|
+
buildConcurrently,
|
|
21
|
+
type ConcurrentPlan,
|
|
22
|
+
redefineConcurrently,
|
|
23
|
+
removeConcurrently,
|
|
24
|
+
} from './concurrent-plan';
|
|
19
25
|
import type { EntityDescriptionLike } from './entity-shape';
|
|
20
26
|
import type { Plan } from './foreign-key-plan';
|
|
21
|
-
import {
|
|
22
|
-
asDeclared,
|
|
23
|
-
createIndex,
|
|
24
|
-
dropIndex,
|
|
25
|
-
dropRecordedIndex,
|
|
26
|
-
impliedByColumnClause,
|
|
27
|
-
redefineIndex,
|
|
28
|
-
} from './index-ddl';
|
|
27
|
+
import { createIndex, dropIndex, impliedByColumnClause, redefineIndex } from './index-ddl';
|
|
29
28
|
import type { TableDescription } from './introspect';
|
|
30
29
|
import { declaredIndexes } from './invariant-ddl';
|
|
31
30
|
|
|
@@ -40,6 +39,11 @@ export interface IndexPlanContext {
|
|
|
40
39
|
readonly rebuilt: ReadonlySet<string>;
|
|
41
40
|
/** Indexes a retype already dropped ahead of its ALTER (`moveDependentsAside`). */
|
|
42
41
|
readonly moved: ReadonlySet<string>;
|
|
42
|
+
/**
|
|
43
|
+
* The non-transactional step (`concurrent-plan.ts`): every `concurrently: true` build and every
|
|
44
|
+
* removal lands here, outside the transaction `plan` is applied in.
|
|
45
|
+
*/
|
|
46
|
+
readonly concurrent: ConcurrentPlan;
|
|
43
47
|
}
|
|
44
48
|
|
|
45
49
|
/**
|
|
@@ -68,19 +72,31 @@ export function indexPlan(
|
|
|
68
72
|
// `redefineIndex` sees a definition that never moved and would emit nothing at all.
|
|
69
73
|
const gone =
|
|
70
74
|
context.moved.has(index.name) || index.columns.some((each) => context.rebuilt.has(each));
|
|
75
|
+
// `concurrently` only where the table is live and the index is the only thing moving: a
|
|
76
|
+
// rebuilt or retyped column already holds ACCESS EXCLUSIVE inside the transaction, so a
|
|
77
|
+
// concurrent rebuild after it buys nothing and leaves the table without the index between.
|
|
78
|
+
const concurrently = index.concurrently === true && !gone;
|
|
71
79
|
if (recorded !== undefined && !gone) {
|
|
72
|
-
|
|
80
|
+
if (concurrently) {
|
|
81
|
+
redefineConcurrently(entity.table, index, recorded, context.concurrent);
|
|
82
|
+
} else {
|
|
83
|
+
redefineIndex(entity.table, index, recorded, plan);
|
|
84
|
+
}
|
|
73
85
|
continue;
|
|
74
86
|
}
|
|
75
87
|
// `added` only: an index over a column that was already there is implied by no clause this
|
|
76
88
|
// migration emits, so it still needs a statement of its own.
|
|
77
89
|
if (impliedByColumnClause(entity, index, context.added)) continue;
|
|
90
|
+
if (concurrently) {
|
|
91
|
+
buildConcurrently(entity.table, index, context.concurrent);
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
78
94
|
plan.up.push(createIndex(entity.table, index));
|
|
79
95
|
// The plain drop, always: this migration CREATED it, with `create index`, so it is an index
|
|
80
|
-
// and never a constraint's —
|
|
96
|
+
// and never a constraint's — the ambiguity is about the recorded side only.
|
|
81
97
|
plan.down.push(dropIndex(index.name));
|
|
82
98
|
}
|
|
83
|
-
removeUndeclared(entity, live,
|
|
99
|
+
removeUndeclared(entity, live, context, declared);
|
|
84
100
|
}
|
|
85
101
|
|
|
86
102
|
/**
|
|
@@ -88,6 +104,13 @@ export function indexPlan(
|
|
|
88
104
|
* the SNAPSHOT recorded, never from what the entity declares, since the entity is precisely what
|
|
89
105
|
* stopped describing it. The rule the retype path already states.
|
|
90
106
|
*
|
|
107
|
+
* ALWAYS concurrently, in the non-transactional step (`removeConcurrently`), whether or not the
|
|
108
|
+
* index was built that way: a plain `drop index` takes ACCESS EXCLUSIVE on the TABLE, so every
|
|
109
|
+
* read and write of a hot table queued behind it. Expand → contract: the drop — and the
|
|
110
|
+
* constraint unhook a UNIQUE shape needs — lands in the step's CONTRACT phase, after everything the
|
|
111
|
+
* transactional step and the step's own builds added, so a renamed index exists twice for a moment
|
|
112
|
+
* and never zero times. Nothing records how an index was built, so the drop cannot depend on it.
|
|
113
|
+
*
|
|
91
114
|
* Four names are skipped, and each one is a statement Postgres would refuse or repeat:
|
|
92
115
|
*
|
|
93
116
|
* | skipped | because |
|
|
@@ -103,7 +126,6 @@ export function indexPlan(
|
|
|
103
126
|
function removeUndeclared(
|
|
104
127
|
entity: EntityDescriptionLike,
|
|
105
128
|
live: TableDescription,
|
|
106
|
-
plan: Plan,
|
|
107
129
|
context: IndexPlanContext,
|
|
108
130
|
declared: ReadonlySet<string>,
|
|
109
131
|
): void {
|
|
@@ -113,7 +135,6 @@ function removeUndeclared(
|
|
|
113
135
|
if (context.moved.has(recorded.name)) continue;
|
|
114
136
|
if (recorded.columns.some((column) => context.rebuilt.has(column))) continue;
|
|
115
137
|
if (!recorded.columns.every((column) => columns.has(column))) continue;
|
|
116
|
-
|
|
117
|
-
plan.down.push(createIndex(live.name, asDeclared(recorded)));
|
|
138
|
+
removeConcurrently(live.name, recorded, context.concurrent);
|
|
118
139
|
}
|
|
119
140
|
}
|
package/src/index.ts
CHANGED
|
@@ -30,6 +30,12 @@ export type {
|
|
|
30
30
|
export { baseClient, db, isReservable, postgresClient, setDbClient } from './client';
|
|
31
31
|
export type { ColumnDefaultLike } from './column-default';
|
|
32
32
|
export { defaultExpression } from './column-default';
|
|
33
|
+
export {
|
|
34
|
+
concurrentIndexTarget,
|
|
35
|
+
isConcurrentIndexStatement,
|
|
36
|
+
isNonTransactional,
|
|
37
|
+
NO_TRANSACTION_MARKER,
|
|
38
|
+
} from './concurrent-index';
|
|
33
39
|
export { dbExecutor } from './db-executor';
|
|
34
40
|
export type { DbHealthReport } from './db-health';
|
|
35
41
|
export { checkDb } from './db-health';
|
|
@@ -140,6 +146,7 @@ export {
|
|
|
140
146
|
} from './migrate';
|
|
141
147
|
export {
|
|
142
148
|
migrateConcurrent,
|
|
149
|
+
migrationConcurrentInTransaction,
|
|
143
150
|
migrationConflict,
|
|
144
151
|
migrationDestructive,
|
|
145
152
|
migrationIrreversible,
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Single responsibility: whether a migration DECLARES something with a marker line — a top-level
|
|
2
|
+
// `--` comment of its own, and nothing that merely looks like one. Two markers ask this question
|
|
3
|
+
// (`-- destructive: true`, `-- transaction: false`), so the scan is written once.
|
|
4
|
+
|
|
5
|
+
import { noiseAt } from './sql-scan';
|
|
6
|
+
|
|
7
|
+
/** Whether only spaces and tabs separate `index` from the start of its line. */
|
|
8
|
+
function startsLine(sql: string, index: number): boolean {
|
|
9
|
+
for (let at = index - 1; at >= 0; at -= 1) {
|
|
10
|
+
const char = sql[at];
|
|
11
|
+
if (char === '\n') return true;
|
|
12
|
+
if (char !== ' ' && char !== '\t') return false;
|
|
13
|
+
}
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Declared only where a reader would see it: a top-level `--` line of its own, whose WHOLE text
|
|
19
|
+
* `marker` matches.
|
|
20
|
+
*
|
|
21
|
+
* A regex over the raw file matched a marker inside `/* … *\/` and inside a dollar-quoted body,
|
|
22
|
+
* where it declares nothing and no reviewer reading the diff would call it a declaration. Scanning
|
|
23
|
+
* is what tells a comment from a comment about a comment; a marker is a lexical fact, not a
|
|
24
|
+
* substring.
|
|
25
|
+
*/
|
|
26
|
+
export function hasMarkerLine(sql: string, marker: RegExp): boolean {
|
|
27
|
+
let index = 0;
|
|
28
|
+
while (index < sql.length) {
|
|
29
|
+
const noise = noiseAt(sql, index);
|
|
30
|
+
if (noise === null) {
|
|
31
|
+
index += 1;
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
const text = sql.slice(index, noise.end);
|
|
35
|
+
if (noise.kind === 'line-comment' && startsLine(sql, index) && marker.test(text)) return true;
|
|
36
|
+
index = noise.end;
|
|
37
|
+
}
|
|
38
|
+
return false;
|
|
39
|
+
}
|
package/src/migrate.ts
CHANGED
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
|
|
6
6
|
import { appVersion, finiteCount, logger, renderFixShellArg } from '@ultimat3/core';
|
|
7
7
|
import { baseClient, type DbClient, type DbConnection, isReservable } from './client';
|
|
8
|
-
import { refuseDependentViews } from './dependent-view';
|
|
9
8
|
import { expectedQueryLoop } from './expected-loop';
|
|
10
9
|
import { ledgerAheadOfBuild } from './migrate-rollback';
|
|
11
10
|
import { migrateConcurrent, migrationConflict, rollbackStepsInvalid } from './migration-errors';
|
|
@@ -18,10 +17,9 @@ import {
|
|
|
18
17
|
pendingMigrations,
|
|
19
18
|
readLedger,
|
|
20
19
|
} from './migration-ledger';
|
|
20
|
+
import { applyMigrationStep } from './migration-step';
|
|
21
21
|
import { poolProfileFor } from './pool-profile';
|
|
22
22
|
import { raw, sql } from './sql';
|
|
23
|
-
import { statementsOf } from './statement-split';
|
|
24
|
-
import { withTransaction } from './transaction';
|
|
25
23
|
|
|
26
24
|
export type { LedgerRow, Migration } from './migration-ledger';
|
|
27
25
|
export {
|
|
@@ -175,43 +173,6 @@ async function withAdvisoryLock<T>(
|
|
|
175
173
|
}
|
|
176
174
|
}
|
|
177
175
|
|
|
178
|
-
/**
|
|
179
|
-
* A migration's `up` and `down` are **scripts**, and one send is one statement — because the two
|
|
180
|
-
* drivers disagree about anything else. PGlite's `query()` is the extended protocol always and
|
|
181
|
-
* answers `cannot insert multiple commands into a prepared statement`; `Bun.SQL.unsafe` degrades
|
|
182
|
-
* to the simple protocol when no value is bound and applies the same script, which is a fact about
|
|
183
|
-
* bun 1.3.14 and not a contract. `createTable` emits the table *and* every index it carries, and
|
|
184
|
-
* `x dev` runs on the embedded driver, so the refusing side was the common path.
|
|
185
|
-
*
|
|
186
|
-
* No `expectedQueryLoop` of its own: both call sites already run inside the one declared for the
|
|
187
|
-
* migration loop, and nesting here would replace that reason with a narrower one for no gain. An
|
|
188
|
-
* empty script sends nothing at all, which is how a no-op migration reaches its ledger row.
|
|
189
|
-
*/
|
|
190
|
-
async function applyScript(tx: DbClient, script: string): Promise<void> {
|
|
191
|
-
for (const statement of statementsOf(script)) await tx.execute(raw(statement));
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
/**
|
|
195
|
-
* Bound how long this migration will queue behind a lock it cannot take.
|
|
196
|
-
*
|
|
197
|
-
* `alter table … add column` needs `ACCESS EXCLUSIVE`. A long `SELECT` holding `ACCESS SHARE`
|
|
198
|
-
* makes it wait — and because Postgres' lock queue is FIFO, **every subsequent query on that table
|
|
199
|
-
* queues behind the ALTER**. The `migrate` profile runs `statement_timeout = 0` deliberately, so
|
|
200
|
-
* without this the migrator waits forever and the app is down on one table for as long as the
|
|
201
|
-
* reporting query runs. `lock_timeout` bounds the *wait* alone and never the work, which is why it
|
|
202
|
-
* is the right knob where `statement_timeout` is not.
|
|
203
|
-
*
|
|
204
|
-
* `SET LOCAL`, inside the migration's own transaction: it reverts at COMMIT, so a value chosen for
|
|
205
|
-
* DDL never leaks onto the session the ledger insert or the next migration runs on. A 0 disables
|
|
206
|
-
* it, exactly like `statementTimeoutMs`. The failure it produces is `55P03`, typed as
|
|
207
|
-
* `X_DB_LOCK_TIMEOUT` by `driverError` with the `pg_stat_activity` read as its fix.
|
|
208
|
-
*/
|
|
209
|
-
async function setLockTimeout(tx: DbClient, lockTimeoutMs: number): Promise<void> {
|
|
210
|
-
if (lockTimeoutMs <= 0) return;
|
|
211
|
-
// `SET LOCAL` takes no parameter placeholder, and the value is a validated integer of ours.
|
|
212
|
-
await tx.execute(raw(`SET LOCAL lock_timeout = ${Math.round(lockTimeoutMs)}`));
|
|
213
|
-
}
|
|
214
|
-
|
|
215
176
|
/**
|
|
216
177
|
* `migrate` role's profile whatever role is running, because the statement is a migration whatever
|
|
217
178
|
* process issues it: `x db migrate` from a laptop, `x dev`'s boot and `ROLE=migrate` all take the
|
|
@@ -271,27 +232,23 @@ export async function migrate(options: MigrateOptions): Promise<MigrationReport>
|
|
|
271
232
|
const done: AppliedMigration[] = [];
|
|
272
233
|
for (const migration of pending) {
|
|
273
234
|
const at = performance.now();
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
await applyScript(tx, migration.up);
|
|
235
|
+
// Inside a transaction, or — a migration marked `-- transaction: false` — statement by
|
|
236
|
+
// statement outside one (`migration-step.ts`). Either way the ledger row is written
|
|
237
|
+
// only after the last statement succeeded, on the lock's own session.
|
|
238
|
+
await applyMigrationStep(session, {
|
|
239
|
+
id: migration.id,
|
|
240
|
+
up: migration.up,
|
|
241
|
+
script: migration.up,
|
|
242
|
+
lockTimeoutMs,
|
|
243
|
+
record: async (client) => {
|
|
284
244
|
const durationMs = Math.round(performance.now() - at);
|
|
285
|
-
await
|
|
245
|
+
await client.execute(sql`
|
|
286
246
|
insert into ${raw(LEDGER_TABLE)} (id, name, checksum, app_version, duration_ms)
|
|
287
247
|
values (${migration.id}, ${migration.name}, ${migrationChecksum(migration)},
|
|
288
248
|
${appVersion}, ${durationMs})
|
|
289
249
|
`);
|
|
290
250
|
},
|
|
291
|
-
|
|
292
|
-
// the lock at all, and on `ROLE=migrate` there is no other connection to apply it on.
|
|
293
|
-
{ client: session },
|
|
294
|
-
);
|
|
251
|
+
});
|
|
295
252
|
done.push({
|
|
296
253
|
id: migration.id,
|
|
297
254
|
name: migration.name,
|
|
@@ -369,17 +326,17 @@ export async function rollback(options: RollbackOptions): Promise<readonly strin
|
|
|
369
326
|
'and roll back there — its down SQL exists nowhere else',
|
|
370
327
|
);
|
|
371
328
|
}
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
329
|
+
// The marker on `up` governs `down` too: the reversal of a concurrent build is a
|
|
330
|
+
// concurrent drop, refused inside a transaction just the same.
|
|
331
|
+
await applyMigrationStep(session, {
|
|
332
|
+
id: migration.id,
|
|
333
|
+
up: migration.up,
|
|
334
|
+
script: migration.down,
|
|
335
|
+
lockTimeoutMs,
|
|
336
|
+
record: async (client) => {
|
|
337
|
+
await client.execute(sql`delete from ${raw(LEDGER_TABLE)} where id = ${row.id}`);
|
|
380
338
|
},
|
|
381
|
-
|
|
382
|
-
);
|
|
339
|
+
});
|
|
383
340
|
reverted.push(row.id);
|
|
384
341
|
}
|
|
385
342
|
return reverted;
|
package/src/migration-errors.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { renderFixShellArg } from '@ultimat3/core';
|
|
9
9
|
import { DESTRUCTIVE_CAUSE, DESTRUCTIVE_MARKER, type DestructiveStatement } from './destructive';
|
|
10
10
|
import { DbError } from './errors';
|
|
11
|
+
import { statementExcerpt } from './statement-excerpt';
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* The migration advisory lock was still held when the wait ran out. `pg_advisory_lock` blocks with
|
|
@@ -155,3 +156,23 @@ export const migrationDestructive = (
|
|
|
155
156
|
fix: `add the line "${DESTRUCTIVE_MARKER}" to ${file}, or regenerate it: x db gen "<name>" --allow-destructive`,
|
|
156
157
|
meta: { file, kind: first.kind, statements: more + 1 },
|
|
157
158
|
});
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* A concurrent index statement in a migration the runner would wrap in a transaction, refused
|
|
162
|
+
* BEFORE `BEGIN` — Postgres answers `25001` only after the transaction is open, in words that name
|
|
163
|
+
* neither the migration nor the marker that fixes it. `-- transaction: false` is the marker
|
|
164
|
+
* (`concurrent-index.ts`); the path is the one directory `x db gen` writes, `packages/db/migrations`.
|
|
165
|
+
* The id is FILENAME text, so it reaches the command through `renderFixShellArg`.
|
|
166
|
+
*/
|
|
167
|
+
export const migrationConcurrentInTransaction = (id: string, statement: string): DbError =>
|
|
168
|
+
new DbError({
|
|
169
|
+
code: 'X_MIGRATION_CONCURRENT_IN_TRANSACTION',
|
|
170
|
+
cause:
|
|
171
|
+
`migration "${id}" runs inside a transaction and holds a statement Postgres refuses there ` +
|
|
172
|
+
`(25001): ${statementExcerpt(statement)}`,
|
|
173
|
+
// The marker is spelled out rather than spliced: it is `NO_TRANSACTION_MARKER`, and
|
|
174
|
+
// `concurrent-index.ts` imports nothing from here, but a constant in the command position is
|
|
175
|
+
// one the shell-argument screen cannot tell from caller input.
|
|
176
|
+
fix: `sed -i '1i -- transaction: false' packages/db/migrations/${renderFixShellArg(id, '<id>')}.sql # before it is applied — or move the statement into a migration of its own`,
|
|
177
|
+
meta: { id },
|
|
178
|
+
});
|