@ultimat3/db 11.3.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +171 -3
- package/README.md +74 -5
- package/package.json +2 -2
- package/src/app-relation.ts +57 -0
- package/src/client.ts +7 -3
- package/src/default-client.ts +35 -0
- package/src/drift.ts +18 -0
- package/src/entity-shape.ts +20 -0
- package/src/errors.ts +16 -0
- package/src/generate.ts +68 -8
- package/src/generated-column.ts +110 -0
- package/src/index-method.ts +75 -0
- package/src/index.ts +18 -0
- package/src/introspect.ts +42 -6
- package/src/replica-client.ts +137 -0
- package/src/replica-route.ts +60 -0
- package/src/replica-scope.ts +54 -0
- package/src/snapshot-parse.ts +8 -1
- package/src/transaction.ts +6 -0
package/src/generate.ts
CHANGED
|
@@ -13,6 +13,9 @@ import type {
|
|
|
13
13
|
} from './entity-shape';
|
|
14
14
|
import { migrationIrreversible } from './errors';
|
|
15
15
|
import { type ConstraintPlans, foreignKeyPlan, foreignKeysOf, type Plan } from './foreign-key-plan';
|
|
16
|
+
import type { Regeneration } from './generated-column';
|
|
17
|
+
import { generatedClause, isGenerated, regenerate } from './generated-column';
|
|
18
|
+
import { declaredMethod, indexMethodOf, indexMethodSql } from './index-method';
|
|
16
19
|
import {
|
|
17
20
|
type ColumnDescription,
|
|
18
21
|
findTable,
|
|
@@ -53,7 +56,10 @@ function defaultExpression(column: ColumnDescriptionLike): string | null {
|
|
|
53
56
|
}
|
|
54
57
|
|
|
55
58
|
function columnClause(column: ColumnDescriptionLike): string {
|
|
56
|
-
|
|
59
|
+
// The generation clause sits directly after the type, and `generatedClause` refuses the pairs
|
|
60
|
+
// Postgres has no column for. Every other part below is unchanged and unreachable for a
|
|
61
|
+
// generated column: it may carry no default, and `hasDefault` is what the refusal reads.
|
|
62
|
+
const parts = [`"${column.column}"`, `${sqlType(column.kind)}${generatedClause(column)}`];
|
|
57
63
|
const expression = defaultExpression(column);
|
|
58
64
|
if (expression !== null) parts.push(`default ${expression}`);
|
|
59
65
|
if (column.notNull) parts.push('not null');
|
|
@@ -105,6 +111,9 @@ export function snapshotOf(entities: readonly EntityDescriptionLike[]): SchemaDe
|
|
|
105
111
|
nullable: !column.notNull,
|
|
106
112
|
default: defaultExpression(column),
|
|
107
113
|
position: index + 1,
|
|
114
|
+
// Only when one was declared — absent stays absent, so no snapshot written before this
|
|
115
|
+
// field existed gains a key and no app's sidecar regenerates over a fact already true.
|
|
116
|
+
...(column.generated === undefined ? {} : { generated: column.generated }),
|
|
108
117
|
}));
|
|
109
118
|
return {
|
|
110
119
|
schema: 'public',
|
|
@@ -121,6 +130,10 @@ export function snapshotOf(entities: readonly EntityDescriptionLike[]): SchemaDe
|
|
|
121
130
|
primary: false,
|
|
122
131
|
where: index.where,
|
|
123
132
|
order: index.order,
|
|
133
|
+
// Only when one was declared. Writing `using: 'btree'` out for every index would rewrite
|
|
134
|
+
// every sidecar in every app on the next `x db gen` — a diff on every file for a fact
|
|
135
|
+
// that was already true, which `indexMethodOf` reads out of the absence anyway.
|
|
136
|
+
...(index.using === undefined ? {} : { using: index.using }),
|
|
124
137
|
})),
|
|
125
138
|
foreignKeys: foreignKeysOf(entity),
|
|
126
139
|
};
|
|
@@ -154,11 +167,30 @@ function createIndex(table: string, index: IndexDescriptionLike): string {
|
|
|
154
167
|
`index "${index.name}" on "${table}" names no columns`,
|
|
155
168
|
`indexes: [{ on: ['<column>'] }] # name the columns in the entity(), then x db gen`,
|
|
156
169
|
);
|
|
170
|
+
const method = index.using ?? 'btree';
|
|
171
|
+
// Two rules Postgres has and a declaration can break, refused here rather than at migrate time:
|
|
172
|
+
// GIN supports neither a unique index nor an ASC/DESC option, and either one reaches the server
|
|
173
|
+
// as a syntax error inside `ROLE=migrate` — a release phase that fails with the server's words
|
|
174
|
+
// and none of the entity's. `X_INVARIANT` for the reason `createIndex` already uses it on an
|
|
175
|
+
// index naming no columns: a declaration this build cannot honour is refused, never reinterpreted.
|
|
176
|
+
assert(
|
|
177
|
+
method === 'btree' || !index.unique,
|
|
178
|
+
`index "${index.name}" on "${table}" is unique and ${method}; Postgres has no unique ${method} index`,
|
|
179
|
+
`indexes: [{ on: ['<column>'], using: '${method}' }] # drop unique, or drop using`,
|
|
180
|
+
);
|
|
181
|
+
assert(
|
|
182
|
+
method === 'btree' || index.order === null,
|
|
183
|
+
`index "${index.name}" on "${table}" is ${method} and ${index.order}; only a btree orders its keys`,
|
|
184
|
+
`indexes: [{ on: ['<column>'], using: '${method}' }] # drop order, or drop using`,
|
|
185
|
+
);
|
|
157
186
|
const kind = index.unique ? 'create unique index' : 'create index';
|
|
158
187
|
const direction = index.order === null ? '' : ` ${index.order}`;
|
|
159
188
|
const columns = index.columns.map((column) => `"${column}"${direction}`).join(', ');
|
|
160
189
|
const predicate = index.where === null ? '' : ` where (${index.where})`;
|
|
161
|
-
|
|
190
|
+
// Re-derived from the closed set, never spliced: `indexMethodSql` answers `''` for a btree, so
|
|
191
|
+
// an index that declared no method emits the statement this generator always emitted, byte for
|
|
192
|
+
// byte, and one that declared a method Postgres does not have is refused instead of built.
|
|
193
|
+
return `${kind} "${index.name}" on "${table}"${indexMethodSql(method)} (${columns})${predicate};`;
|
|
162
194
|
}
|
|
163
195
|
|
|
164
196
|
/**
|
|
@@ -173,19 +205,31 @@ function retypeColumn(
|
|
|
173
205
|
column: ColumnDescriptionLike,
|
|
174
206
|
recorded: ColumnDescription,
|
|
175
207
|
plan: Plan,
|
|
176
|
-
):
|
|
208
|
+
): Regeneration {
|
|
177
209
|
const wanted = sqlType(column.kind);
|
|
178
|
-
|
|
210
|
+
// A generated column moves by its own rules — see `generated-column.ts`. Asked whenever EITHER
|
|
211
|
+
// side is one, because becoming generated and ceasing to be are both changes with a statement.
|
|
212
|
+
if (isGenerated(column) || recorded.generated !== undefined) {
|
|
213
|
+
return regenerate(table, column, wanted, recorded, plan);
|
|
214
|
+
}
|
|
215
|
+
if (recorded.dataType === wanted) return 'unchanged';
|
|
179
216
|
const alter = (type: string): string =>
|
|
180
217
|
`alter table "${table}" alter column "${column.column}" type ${type} ` +
|
|
181
218
|
`using "${column.column}"::${type};`;
|
|
182
219
|
plan.up.push(alter(wanted));
|
|
183
220
|
plan.down.push(alter(recorded.dataType));
|
|
221
|
+
return 'altered';
|
|
184
222
|
}
|
|
185
223
|
|
|
186
224
|
/** The parts of an index Postgres cannot alter in place — every one of them is a rebuild. */
|
|
187
225
|
function indexShape(index: IndexDescriptionLike | IndexDescription): string {
|
|
188
|
-
return JSON.stringify([
|
|
226
|
+
return JSON.stringify([
|
|
227
|
+
[...index.columns],
|
|
228
|
+
index.unique,
|
|
229
|
+
index.where,
|
|
230
|
+
index.order ?? null,
|
|
231
|
+
indexMethodOf(index),
|
|
232
|
+
]);
|
|
189
233
|
}
|
|
190
234
|
|
|
191
235
|
/**
|
|
@@ -216,6 +260,10 @@ function redefineIndex(
|
|
|
216
260
|
unique: recorded.unique,
|
|
217
261
|
where: recorded.where,
|
|
218
262
|
order: recorded.order,
|
|
263
|
+
// `declaredMethod`, never a cast: `recorded` is a snapshot's, typed open because the catalog
|
|
264
|
+
// shares the shape, and a method this generator cannot emit must refuse rather than be
|
|
265
|
+
// rebuilt as a btree — a `down` that recreates the wrong structure is worse than none.
|
|
266
|
+
...(recorded.using === undefined ? {} : { using: declaredMethod(recorded.using) }),
|
|
219
267
|
}),
|
|
220
268
|
`drop index "${index.name}";`,
|
|
221
269
|
);
|
|
@@ -224,16 +272,26 @@ function redefineIndex(
|
|
|
224
272
|
function diffTable(entity: EntityDescriptionLike, live: TableDescription, plan: Plan): void {
|
|
225
273
|
const existing = new Map(live.columns.map((column) => [column.name, column]));
|
|
226
274
|
const added = new Set<string>();
|
|
275
|
+
// A column `regenerate` had to replace outright: `add column` implies no index, so every index
|
|
276
|
+
// over it has to be stated again even though its own definition never moved.
|
|
277
|
+
const rebuilt = new Set<string>();
|
|
227
278
|
for (const column of entity.columns) {
|
|
228
279
|
const recorded = existing.get(column.column);
|
|
229
280
|
if (recorded !== undefined) {
|
|
230
|
-
retypeColumn(entity.table, column, recorded, plan)
|
|
281
|
+
if (retypeColumn(entity.table, column, recorded, plan) === 'rebuilt') {
|
|
282
|
+
rebuilt.add(column.column);
|
|
283
|
+
}
|
|
231
284
|
continue;
|
|
232
285
|
}
|
|
233
286
|
added.add(column.column);
|
|
234
287
|
// A NOT NULL add with no default cannot succeed on a populated table; emit it nullable and
|
|
235
288
|
// leave the agent the exact follow-up rather than a migration that fails at 3am.
|
|
236
|
-
|
|
289
|
+
//
|
|
290
|
+
// A GENERATED column is the exception and not a special case of it: the database computes it
|
|
291
|
+
// for every existing row inside the same `add column`, so it lands NOT NULL and populated in
|
|
292
|
+
// one statement — measured. Emitting it nullable would leave a `-- backfill` comment naming a
|
|
293
|
+
// step nobody can perform, since a generated column cannot be written to.
|
|
294
|
+
const nullable = column.notNull && !isGenerated(column) && defaultExpression(column) === null;
|
|
237
295
|
const clause = nullable ? columnClause({ ...column, notNull: false }) : columnClause(column);
|
|
238
296
|
plan.up.push(`alter table "${entity.table}" add column ${clause};`);
|
|
239
297
|
if (nullable) {
|
|
@@ -248,7 +306,9 @@ function diffTable(entity: EntityDescriptionLike, live: TableDescription, plan:
|
|
|
248
306
|
const indexed = new Map(live.indexes.map((index) => [index.name, index]));
|
|
249
307
|
for (const index of entity.indexes) {
|
|
250
308
|
const recorded = indexed.get(index.name);
|
|
251
|
-
|
|
309
|
+
// A rebuilt column took its indexes down with it, so this one is CREATED rather than compared:
|
|
310
|
+
// `redefineIndex` sees a definition that never moved and would emit nothing at all.
|
|
311
|
+
if (recorded !== undefined && !index.columns.some((column) => rebuilt.has(column))) {
|
|
252
312
|
redefineIndex(entity.table, index, recorded, plan);
|
|
253
313
|
continue;
|
|
254
314
|
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// Single responsibility: what a column the DATABASE computes contributes to DDL, and what changes
|
|
2
|
+
// to one a migration may emit. Split from `generate.ts` because Postgres treats a generated column
|
|
3
|
+
// as a different thing at every step — its clause, its retype, its NOT NULL add and the way its
|
|
4
|
+
// expression moves are each a rule of their own, and none of them is the ordinary column's.
|
|
5
|
+
|
|
6
|
+
import { assert } from '@ultimat3/core';
|
|
7
|
+
import type { ColumnDescriptionLike } from './entity-shape';
|
|
8
|
+
import type { Plan } from './foreign-key-plan';
|
|
9
|
+
import type { ColumnDescription } from './introspect';
|
|
10
|
+
|
|
11
|
+
/** How a column that moved was brought back into line — what the caller has to do next, if anything. */
|
|
12
|
+
export type Regeneration = 'unchanged' | 'altered' | 'rebuilt';
|
|
13
|
+
|
|
14
|
+
const alterColumn = (table: string, column: string): string =>
|
|
15
|
+
`alter table "${table}" alter column "${column}"`;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The `generated always as (…) stored` clause, or `''` for every ordinary column — so a description
|
|
19
|
+
* written before this field existed emits the statement it always emitted, byte for byte.
|
|
20
|
+
*
|
|
21
|
+
* The two rules Postgres has about the pair are refused HERE, where the entity is still named, for
|
|
22
|
+
* the reason `createIndex` refuses a unique GIN in the same file: an unguarded generator writes DDL
|
|
23
|
+
* whose first reader is `ROLE=migrate`, and the server's message carries none of the declaration's
|
|
24
|
+
* words. A column may not be both DEFAULTED and GENERATED (`42601` — a generated column's value IS
|
|
25
|
+
* its expression), and an empty expression is not one.
|
|
26
|
+
*/
|
|
27
|
+
export function generatedClause(column: ColumnDescriptionLike): string {
|
|
28
|
+
const expression = column.generated;
|
|
29
|
+
if (expression === undefined || expression === null) return '';
|
|
30
|
+
assert(
|
|
31
|
+
!column.hasDefault,
|
|
32
|
+
`column "${column.column}" is declared both generated and defaulted, and Postgres has neither`,
|
|
33
|
+
`drop the default from "${column.property}" — a generated column's value is its expression, computed on every write`,
|
|
34
|
+
);
|
|
35
|
+
assert(
|
|
36
|
+
expression.trim().length > 0,
|
|
37
|
+
`column "${column.column}" is generated by an empty expression`,
|
|
38
|
+
`give "${column.property}" an expression, or drop the generated declaration`,
|
|
39
|
+
);
|
|
40
|
+
return ` generated always as (${expression}) stored`;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export const isGenerated = (column: ColumnDescriptionLike): boolean =>
|
|
44
|
+
typeof column.generated === 'string';
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* A generated column whose TYPE or whose EXPRESSION moved, brought into line without rebuilding it.
|
|
48
|
+
*
|
|
49
|
+
* `set expression as (…)` (Postgres 17) rewrites the table and recomputes every row, and the
|
|
50
|
+
* column's indexes survive — measured. Drop-and-recreate was the alternative and is worse in two
|
|
51
|
+
* ways that matter: dropping the column takes its indexes with it and nothing in this diff puts
|
|
52
|
+
* them back, and `alter table … drop column` is what `destructive.ts` reads as a data loss, so
|
|
53
|
+
* every expression change would have carried `-- destructive: true` on a migration that loses
|
|
54
|
+
* nothing. A marker on a migration that destroys nothing is a marker reviewers learn to ignore.
|
|
55
|
+
*
|
|
56
|
+
* The retype carries no `using`: Postgres refuses one on a generated column outright ("column … is
|
|
57
|
+
* a generated column"), which is exactly the statement `retypeColumn` emits for every other column
|
|
58
|
+
* — and there is nothing to convert, because the expression produces the new type itself.
|
|
59
|
+
*
|
|
60
|
+
* Two transitions this cannot express, and both are refused rather than half-emitted:
|
|
61
|
+
* plain → generated (there is no `set expression` for a column that has none) and a column whose
|
|
62
|
+
* recorded expression is unknown. `drop expression` is the one that HAS a statement, and it is the
|
|
63
|
+
* generated → plain direction, which keeps the values it computed.
|
|
64
|
+
*/
|
|
65
|
+
export function regenerate(
|
|
66
|
+
table: string,
|
|
67
|
+
column: ColumnDescriptionLike,
|
|
68
|
+
wantedType: string,
|
|
69
|
+
recorded: ColumnDescription,
|
|
70
|
+
plan: Plan,
|
|
71
|
+
): Regeneration {
|
|
72
|
+
const wanted = column.generated ?? null;
|
|
73
|
+
const held = recorded.generated ?? null;
|
|
74
|
+
if (wanted === null && held === null) return 'unchanged';
|
|
75
|
+
// Generated -> plain: the column keeps every value it computed and simply stops being derived.
|
|
76
|
+
if (wanted === null) {
|
|
77
|
+
plan.up.push(`${alterColumn(table, column.column)} drop expression;`);
|
|
78
|
+
plan.down.push(`${alterColumn(table, column.column)} set expression as (${held ?? ''});`);
|
|
79
|
+
return 'altered';
|
|
80
|
+
}
|
|
81
|
+
// Plain -> generated: `set expression` needs a column that already has one, so this is the whole
|
|
82
|
+
// column again. Reported as `rebuilt` so the caller can put the indexes back — an `add column`
|
|
83
|
+
// implies none of them.
|
|
84
|
+
if (held === null) {
|
|
85
|
+
plan.up.push(
|
|
86
|
+
`alter table "${table}" drop column "${column.column}";`,
|
|
87
|
+
`alter table "${table}" add column "${column.column}" ${wantedType}` +
|
|
88
|
+
`${generatedClause(column)}${column.notNull ? ' not null' : ''};`,
|
|
89
|
+
);
|
|
90
|
+
// Pushed forwards and read backwards — `down` is reversed at assembly.
|
|
91
|
+
plan.down.push(
|
|
92
|
+
`alter table "${table}" add column "${column.column}" ${recorded.dataType};` +
|
|
93
|
+
' -- was not a generated column',
|
|
94
|
+
`alter table "${table}" drop column "${column.column}";`,
|
|
95
|
+
);
|
|
96
|
+
return 'rebuilt';
|
|
97
|
+
}
|
|
98
|
+
let moved = false;
|
|
99
|
+
if (recorded.dataType !== wantedType) {
|
|
100
|
+
plan.up.push(`${alterColumn(table, column.column)} type ${wantedType};`);
|
|
101
|
+
plan.down.push(`${alterColumn(table, column.column)} type ${recorded.dataType};`);
|
|
102
|
+
moved = true;
|
|
103
|
+
}
|
|
104
|
+
if (held !== wanted) {
|
|
105
|
+
plan.up.push(`${alterColumn(table, column.column)} set expression as (${wanted});`);
|
|
106
|
+
plan.down.push(`${alterColumn(table, column.column)} set expression as (${held});`);
|
|
107
|
+
moved = true;
|
|
108
|
+
}
|
|
109
|
+
return moved ? 'altered' : 'unchanged';
|
|
110
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Single responsibility: an index's access method — the closed set an entity may declare, the one
|
|
2
|
+
// normalisation both sides of a comparison pass through, and the DDL fragment. Its own file for the
|
|
3
|
+
// reason `foreign-key.ts` holds `onDeleteRule`: a generator and a detector that disagreed about
|
|
4
|
+
// what "the default" is would report drift on a database that is exactly right.
|
|
5
|
+
|
|
6
|
+
import { indexMethodInvalid } from './errors';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The methods an entity may declare. Two members, deliberately: `btree` is what every index has
|
|
10
|
+
* always been, and `gin` is the one with a caller — `@>` / `<@` / `&&` / `?` on a `json()` or
|
|
11
|
+
* `arrayOf()` column is a sequential scan without it.
|
|
12
|
+
*
|
|
13
|
+
* `gist`, `brin`, `hash` and `spgist` are legitimate Postgres methods and are **not** here, because
|
|
14
|
+
* nothing declares one and each brings a rule of its own that would have to be enforced with no
|
|
15
|
+
* caller to test it — `hash` and `brin` cannot be unique, `gist` needs `btree_gist` to be, and none
|
|
16
|
+
* of the three accepts `asc`/`desc`. Adding a member later is additive; shipping four that nobody
|
|
17
|
+
* uses is four ways for a first caller to be silently wrong. A method the catalog reports and this
|
|
18
|
+
* set does not carry is still READ and still compared — see `indexMethodOf`.
|
|
19
|
+
*/
|
|
20
|
+
export const INDEX_METHODS = ['btree', 'gin'] as const;
|
|
21
|
+
|
|
22
|
+
export type IndexMethod = (typeof INDEX_METHODS)[number];
|
|
23
|
+
|
|
24
|
+
export function isIndexMethod(value: unknown): value is IndexMethod {
|
|
25
|
+
return typeof value === 'string' && (INDEX_METHODS as readonly string[]).includes(value);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* What method this index is on, whichever side it came from. `undefined` is `btree` — Postgres'
|
|
30
|
+
* own default, which nothing writes out, which every index created before this existed is, and
|
|
31
|
+
* which is therefore what a snapshot recorded before it carried the field at all.
|
|
32
|
+
*
|
|
33
|
+
* The CATALOG's answer is passed through verbatim, `gist` and an extension's own access method
|
|
34
|
+
* included: the live side is whatever `pg_am` said, and folding an unknown name into `btree` would
|
|
35
|
+
* hide exactly the difference an operator needs to see.
|
|
36
|
+
*/
|
|
37
|
+
export function indexMethodOf(index: { readonly using?: string | undefined }): string {
|
|
38
|
+
return index.using ?? 'btree';
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The closed-set reading of a method that arrived on the OPEN side — a catalog row or a snapshot
|
|
43
|
+
* this generator did not write. `undefined` for absent, and a **refusal** for anything the set does
|
|
44
|
+
* not carry, never a silent fall back to `btree`: the one caller is `redefineIndex`, whose `down`
|
|
45
|
+
* recreates the index a previous migration recorded, and a `gist` quietly rebuilt as a btree is a
|
|
46
|
+
* rollback that leaves the database in a state no migration describes.
|
|
47
|
+
*/
|
|
48
|
+
export function declaredMethod(using: string | undefined): IndexMethod | undefined {
|
|
49
|
+
if (using === undefined) return undefined;
|
|
50
|
+
if (!isIndexMethod(using)) throw indexMethodInvalid(using);
|
|
51
|
+
return using;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The clause, or `''` for a btree — so an index that declared nothing emits the statement it always
|
|
56
|
+
* did, byte for byte.
|
|
57
|
+
*
|
|
58
|
+
* The literal is **re-derived from the set, never spliced from the input**, the same shape
|
|
59
|
+
* `isolationMode` uses for `BEGIN`. The type is not the guard: this value reaches `create index …`
|
|
60
|
+
* as text from an entity declaration, a config or a generator, and `using ${method}` on an operand
|
|
61
|
+
* TypeScript never saw is the identical hole to the one `columnName` carried — a name that closed
|
|
62
|
+
* the parenthesis and opened a second command.
|
|
63
|
+
*/
|
|
64
|
+
export function indexMethodSql(method: IndexMethod): string {
|
|
65
|
+
switch (method) {
|
|
66
|
+
case 'btree':
|
|
67
|
+
return '';
|
|
68
|
+
case 'gin':
|
|
69
|
+
return ' using gin';
|
|
70
|
+
default: {
|
|
71
|
+
const unhandled: never = method;
|
|
72
|
+
throw indexMethodInvalid(unhandled);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -32,6 +32,7 @@ export {
|
|
|
32
32
|
poolProfileFor,
|
|
33
33
|
setDbClient,
|
|
34
34
|
} from './client';
|
|
35
|
+
export { defaultClient, REPLICA_URL_ENV } from './default-client';
|
|
35
36
|
export type { DestructiveKind, DestructiveStatement } from './destructive';
|
|
36
37
|
export {
|
|
37
38
|
DESTRUCTIVE_CAUSE,
|
|
@@ -86,6 +87,14 @@ export type { RecordedStatement, RecordingClient, StubResponse } from './fake';
|
|
|
86
87
|
export { createRecordingClient } from './fake';
|
|
87
88
|
export type { GeneratedMigration, GenerateOptions } from './generate';
|
|
88
89
|
export { generateMigration, migrationStamp, slugify, snapshotOf } from './generate';
|
|
90
|
+
export type { IndexMethod } from './index-method';
|
|
91
|
+
export {
|
|
92
|
+
declaredMethod,
|
|
93
|
+
INDEX_METHODS,
|
|
94
|
+
indexMethodOf,
|
|
95
|
+
indexMethodSql,
|
|
96
|
+
isIndexMethod,
|
|
97
|
+
} from './index-method';
|
|
89
98
|
export type {
|
|
90
99
|
ColumnDescription,
|
|
91
100
|
ForeignKeyDescription,
|
|
@@ -142,6 +151,15 @@ export type { ReadOnlyQueryOptions, ReadOnlyQueryResult } from './readonly-query
|
|
|
142
151
|
export { READONLY_TIMEOUT_MS, readOnlyQuery } from './readonly-query';
|
|
143
152
|
export type { ReadOnlyRoleOptions } from './readonly-role';
|
|
144
153
|
export { ensureReadOnlyRole, grantReadOnlySql, READONLY_ROLE } from './readonly-role';
|
|
154
|
+
export type {
|
|
155
|
+
ReplicaStats,
|
|
156
|
+
ReplicatedClient,
|
|
157
|
+
ReplicatedClientOptions,
|
|
158
|
+
} from './replica-client';
|
|
159
|
+
export { BREAKER_COOLDOWN_MS, BREAKER_FAILURES, replicatedClient } from './replica-client';
|
|
160
|
+
export { type DbNode, isPlainRead } from './replica-route';
|
|
161
|
+
export type { ReplicaScope } from './replica-scope';
|
|
162
|
+
export { markScopeWrote, replicaScope, withReplicaReads } from './replica-scope';
|
|
145
163
|
export { snapshotJson } from './snapshot-json';
|
|
146
164
|
export { parseSnapshot } from './snapshot-parse';
|
|
147
165
|
export type { SqlFragment } from './sql';
|
package/src/introspect.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// Single responsibility: read the live schema out of `information_schema` / `pg_catalog` into a
|
|
2
|
-
// plain, sortable description
|
|
3
|
-
//
|
|
4
|
-
// and deterministically ordered
|
|
2
|
+
// plain, sortable description — app tables only, never a relation an extension owns and never a
|
|
3
|
+
// view (`app-relation.ts`). `checkDrift` is its one shipped consumer, so what it omits a deploy
|
|
4
|
+
// cannot refuse over. Keep it JSON-safe and deterministically ordered: it is diffed and serialised.
|
|
5
5
|
|
|
6
|
+
import { nonAppRelations } from './app-relation';
|
|
6
7
|
import { type DbClient, db } from './client';
|
|
7
8
|
import { sql } from './sql';
|
|
8
9
|
|
|
@@ -13,6 +14,15 @@ export interface ColumnDescription {
|
|
|
13
14
|
readonly nullable: boolean;
|
|
14
15
|
readonly default: string | null;
|
|
15
16
|
readonly position: number;
|
|
17
|
+
/**
|
|
18
|
+
* The generation expression, as the SNAPSHOT spells it. Absent for an ordinary column and absent
|
|
19
|
+
* for every row this module reads out of the live catalog — deliberately: Postgres stores its own
|
|
20
|
+
* rewriting of the expression (`COALESCE(title, ''::text)` for `coalesce("title", '')`), so a
|
|
21
|
+
* catalog value could never compare equal to a generated one, and drift would report a correct
|
|
22
|
+
* database forever. Both sides of the diff that DOES read it — `x db gen`'s — are generated
|
|
23
|
+
* spellings, which is the same rule `IndexDescription.where` states one field down.
|
|
24
|
+
*/
|
|
25
|
+
readonly generated?: string | undefined;
|
|
16
26
|
}
|
|
17
27
|
|
|
18
28
|
export interface IndexDescription {
|
|
@@ -29,6 +39,14 @@ export interface IndexDescription {
|
|
|
29
39
|
readonly where: string | null;
|
|
30
40
|
/** `desc` only when every key column is descending; `null` is Postgres' own default. */
|
|
31
41
|
readonly order: 'asc' | 'desc' | null;
|
|
42
|
+
/**
|
|
43
|
+
* The access method as `pg_am` names it — `btree`, `gin`, `gist`, or an extension's own. Read
|
|
44
|
+
* OPEN rather than as the closed set an entity may declare: the live side is whatever the
|
|
45
|
+
* catalog said, and a `gist` folded into `btree` is the difference drift exists to report.
|
|
46
|
+
* Absent means `btree` — a snapshot written before this field existed carries no method, and
|
|
47
|
+
* every index it recorded was one. `indexMethodOf()` is the one reader of that rule.
|
|
48
|
+
*/
|
|
49
|
+
readonly using?: string | undefined;
|
|
32
50
|
}
|
|
33
51
|
|
|
34
52
|
export interface ForeignKeyDescription {
|
|
@@ -55,7 +73,13 @@ export interface SchemaDescription {
|
|
|
55
73
|
export interface IntrospectOptions {
|
|
56
74
|
readonly client?: DbClient | undefined;
|
|
57
75
|
readonly schema?: string | undefined;
|
|
58
|
-
/**
|
|
76
|
+
/**
|
|
77
|
+
* The ledger is framework bookkeeping, not user schema — excluded so it never reads as drift.
|
|
78
|
+
*
|
|
79
|
+
* Replaces the default (`['x_migrations']`) rather than adding to it. It never replaces the set
|
|
80
|
+
* `nonAppRelations()` derives: an extension's relations are not app schema in any deployment,
|
|
81
|
+
* so that exclusion is not a caller's to switch off.
|
|
82
|
+
*/
|
|
59
83
|
readonly exclude?: readonly string[] | undefined;
|
|
60
84
|
}
|
|
61
85
|
|
|
@@ -76,6 +100,7 @@ interface IndexRow {
|
|
|
76
100
|
readonly columns: readonly string[];
|
|
77
101
|
readonly predicate: string | null;
|
|
78
102
|
readonly descending: boolean;
|
|
103
|
+
readonly method?: string | undefined;
|
|
79
104
|
}
|
|
80
105
|
|
|
81
106
|
interface ForeignKeyRow {
|
|
@@ -92,7 +117,13 @@ const byName = (a: { name: string }, b: { name: string }): number => (a.name < b
|
|
|
92
117
|
export async function introspect(options: IntrospectOptions = {}): Promise<SchemaDescription> {
|
|
93
118
|
const client = options.client ?? db();
|
|
94
119
|
const schema = options.schema ?? 'public';
|
|
95
|
-
|
|
120
|
+
// Asked first, and unconditionally: everything below reads `information_schema`, which admits a
|
|
121
|
+
// view and an extension's own tables alongside the app's. Merged into `excluded` rather than
|
|
122
|
+
// filtered afterwards so one deny list feeds the whole fold.
|
|
123
|
+
const excluded = [
|
|
124
|
+
...(options.exclude ?? ['x_migrations']),
|
|
125
|
+
...(await nonAppRelations(client, schema)),
|
|
126
|
+
];
|
|
96
127
|
|
|
97
128
|
const columns = await client.query<ColumnRow>(sql`
|
|
98
129
|
select table_name, column_name, data_type, is_nullable, column_default, ordinal_position
|
|
@@ -112,16 +143,18 @@ export async function introspect(options: IntrospectOptions = {}): Promise<Schem
|
|
|
112
143
|
ix.indisunique as is_unique,
|
|
113
144
|
ix.indisprimary as is_primary,
|
|
114
145
|
pg_get_expr(ix.indpred, ix.indrelid) as predicate,
|
|
146
|
+
am.amname as method,
|
|
115
147
|
array_agg(a.attname order by k.ord) as columns,
|
|
116
148
|
bool_and((ix.indoption[k.ord - 1] & 1) = 1) as descending
|
|
117
149
|
from pg_class t
|
|
118
150
|
join pg_namespace n on n.oid = t.relnamespace
|
|
119
151
|
join pg_index ix on ix.indrelid = t.oid
|
|
120
152
|
join pg_class i on i.oid = ix.indexrelid
|
|
153
|
+
join pg_am am on am.oid = i.relam
|
|
121
154
|
cross join lateral unnest(ix.indkey::smallint[]) with ordinality as k(attnum, ord)
|
|
122
155
|
join pg_attribute a on a.attrelid = t.oid and a.attnum = k.attnum
|
|
123
156
|
where n.nspname = ${schema} and t.relkind = 'r' and k.ord <= ix.indnkeyatts
|
|
124
|
-
group by t.relname, i.relname, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid
|
|
157
|
+
group by t.relname, i.relname, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid, am.amname
|
|
125
158
|
order by t.relname, i.relname
|
|
126
159
|
`);
|
|
127
160
|
|
|
@@ -175,6 +208,9 @@ export function buildSchema(
|
|
|
175
208
|
primary: row.is_primary,
|
|
176
209
|
where: row.predicate,
|
|
177
210
|
order: row.descending ? ('desc' as const) : null,
|
|
211
|
+
// `exactOptionalPropertyTypes`: a row with no method is a stub's, and absent must stay
|
|
212
|
+
// absent rather than becoming an explicit `undefined` a strict comparison can see.
|
|
213
|
+
...(row.method === undefined ? {} : { using: row.method }),
|
|
178
214
|
}))
|
|
179
215
|
.sort(byName);
|
|
180
216
|
return {
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// Single responsibility: one `DbClient` over a primary and a read replica. It decides nothing about
|
|
2
|
+
// SQL (`replica-route.ts`) and nothing about scope (`replica-scope.ts`) — what lives here is which
|
|
3
|
+
// handle a statement is sent on, what happens when the replica will not answer, and the counters
|
|
4
|
+
// that make both visible to a test that cannot scrape a metrics endpoint.
|
|
5
|
+
|
|
6
|
+
import { type Clock, logger, renderThrowable, systemClock } from '@ultimat3/core';
|
|
7
|
+
import { type DbClient, type DbConnection, isReservable, type ReservableClient } from './client';
|
|
8
|
+
import { isPlainRead } from './replica-route';
|
|
9
|
+
import { markScopeWrote, replicaScope } from './replica-scope';
|
|
10
|
+
import type { SqlFragment } from './sql';
|
|
11
|
+
|
|
12
|
+
export interface ReplicaStats {
|
|
13
|
+
/** Statements the replica answered. */
|
|
14
|
+
readonly replica: number;
|
|
15
|
+
/** Statements sent to the primary, fallbacks included. */
|
|
16
|
+
readonly primary: number;
|
|
17
|
+
/** Replica attempts that failed and were re-run on the primary. */
|
|
18
|
+
readonly fallbacks: number;
|
|
19
|
+
/** True while the breaker is parked and every read is going to the primary. */
|
|
20
|
+
readonly parked: boolean;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface ReplicatedClientOptions {
|
|
24
|
+
/** Consecutive replica failures before it is parked. */
|
|
25
|
+
readonly breakerFailures?: number | undefined;
|
|
26
|
+
readonly breakerCooldownMs?: number | undefined;
|
|
27
|
+
/** Injection seam; production passes neither. */
|
|
28
|
+
readonly clock?: Clock | undefined;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface ReplicatedClient extends DbClient {
|
|
32
|
+
readonly stats: ReplicaStats;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Three in a row, then a ten-second rest — an outage costs 3 doubled reads, not every read. */
|
|
36
|
+
export const BREAKER_FAILURES = 3;
|
|
37
|
+
export const BREAKER_COOLDOWN_MS = 10_000;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* `primary` answers everything that is not provably a replica-safe read inside an open
|
|
41
|
+
* `withReplicaReads` scope. Reservations are ALWAYS the primary's: `withTransaction` pins a
|
|
42
|
+
* connection through `reserve()`, and a BEGIN that landed on a standby is not a transaction, it is
|
|
43
|
+
* `25006` on the first write inside it.
|
|
44
|
+
*
|
|
45
|
+
* `reserve` is present only when the primary has one, so `isReservable()` keeps answering about the
|
|
46
|
+
* database rather than about this wrapper — a wrapper that always exposed `reserve` would make
|
|
47
|
+
* `runRoot` pin a connection out of a client that cannot pin, and one that never exposed it would
|
|
48
|
+
* make `runRoot` run BEGIN, the statements and COMMIT on three different pooled connections.
|
|
49
|
+
*/
|
|
50
|
+
export function replicatedClient(
|
|
51
|
+
primary: DbClient,
|
|
52
|
+
replica: DbClient,
|
|
53
|
+
options: ReplicatedClientOptions = {},
|
|
54
|
+
): ReplicatedClient {
|
|
55
|
+
const clock = options.clock ?? systemClock;
|
|
56
|
+
const limit = options.breakerFailures ?? BREAKER_FAILURES;
|
|
57
|
+
const cooldown = options.breakerCooldownMs ?? BREAKER_COOLDOWN_MS;
|
|
58
|
+
let replicaCount = 0;
|
|
59
|
+
let primaryCount = 0;
|
|
60
|
+
let fallbackCount = 0;
|
|
61
|
+
let consecutiveFailures = 0;
|
|
62
|
+
let parkedUntil = 0;
|
|
63
|
+
|
|
64
|
+
/** Monotonic, never wall clock: a leap second or an NTP step must not un-park the breaker. */
|
|
65
|
+
function parked(): boolean {
|
|
66
|
+
return clock.monotonic() < parkedUntil;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function nodeIsReplica(text: string): boolean {
|
|
70
|
+
const scope = replicaScope();
|
|
71
|
+
// No scope: nobody declared these reads replica-safe, so this is a single-pool client.
|
|
72
|
+
if (scope === undefined) return false;
|
|
73
|
+
if (!isPlainRead(text)) {
|
|
74
|
+
markScopeWrote();
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
// Read-your-writes, and the reason the flag is on a mutable scope value rather than computed
|
|
78
|
+
// per statement: once this scope has written, every later read in it is the primary's. A
|
|
79
|
+
// replica is behind by an unbounded amount — streaming lag is not a number this tier can know
|
|
80
|
+
// — so "the row I just inserted" is the one question a standby is guaranteed to answer wrong.
|
|
81
|
+
if (scope.wrote) return false;
|
|
82
|
+
return !parked();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function send<T>(fragment: SqlFragment, on: (client: DbClient) => Promise<T>): Promise<T> {
|
|
86
|
+
if (!nodeIsReplica(fragment.text)) {
|
|
87
|
+
primaryCount += 1;
|
|
88
|
+
return on(primary);
|
|
89
|
+
}
|
|
90
|
+
try {
|
|
91
|
+
const answer = await on(replica);
|
|
92
|
+
replicaCount += 1;
|
|
93
|
+
consecutiveFailures = 0;
|
|
94
|
+
return answer;
|
|
95
|
+
} catch (error) {
|
|
96
|
+
// Re-running is exactly-once, not at-least-once: only `isPlainRead` statements reach here,
|
|
97
|
+
// and a statement a standby refused (`25006`) never executed. A replica outage therefore
|
|
98
|
+
// costs latency and never an answer — which is the whole point, since a read replica is a
|
|
99
|
+
// capacity tier and must not become a new way for the app to be down.
|
|
100
|
+
consecutiveFailures += 1;
|
|
101
|
+
if (consecutiveFailures >= limit) parkedUntil = clock.monotonic() + cooldown;
|
|
102
|
+
fallbackCount += 1;
|
|
103
|
+
// `renderThrowable`, never `${error}`: a driver error's `message` getter is app code.
|
|
104
|
+
logger.warn('db.replica_fallback', {
|
|
105
|
+
error: renderThrowable(error),
|
|
106
|
+
consecutiveFailures,
|
|
107
|
+
parked: parked(),
|
|
108
|
+
});
|
|
109
|
+
primaryCount += 1;
|
|
110
|
+
return on(primary);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const base: ReplicatedClient = {
|
|
115
|
+
get stats(): ReplicaStats {
|
|
116
|
+
return {
|
|
117
|
+
replica: replicaCount,
|
|
118
|
+
primary: primaryCount,
|
|
119
|
+
fallbacks: fallbackCount,
|
|
120
|
+
parked: parked(),
|
|
121
|
+
};
|
|
122
|
+
},
|
|
123
|
+
query: <T>(fragment: SqlFragment) => send(fragment, (client) => client.query<T>(fragment)),
|
|
124
|
+
one: <T>(fragment: SqlFragment) => send(fragment, (client) => client.one<T>(fragment)),
|
|
125
|
+
execute: (fragment: SqlFragment) => send(fragment, (client) => client.execute(fragment)),
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
if (!isReservable(primary)) return base;
|
|
129
|
+
const reservable: ReplicatedClient & ReservableClient = {
|
|
130
|
+
...base,
|
|
131
|
+
get stats(): ReplicaStats {
|
|
132
|
+
return base.stats;
|
|
133
|
+
},
|
|
134
|
+
reserve: (): Promise<DbConnection> => primary.reserve(),
|
|
135
|
+
};
|
|
136
|
+
return reservable;
|
|
137
|
+
}
|