@ultimat3/db 16.0.0 → 18.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +170 -12
- package/README.md +4 -3
- package/package.json +3 -3
- package/src/array-parameter.ts +91 -0
- package/src/bun-sql.ts +32 -0
- package/src/client.ts +17 -321
- package/src/connection-url.ts +102 -0
- package/src/db-health.ts +31 -0
- package/src/default-client.ts +2 -1
- package/src/drift-errors.ts +36 -0
- package/src/drift-findings.ts +63 -6
- package/src/drift-fixtures.ts +23 -0
- package/src/errors.ts +0 -9
- package/src/generate.ts +40 -2
- package/src/index.ts +15 -15
- package/src/introspect.ts +16 -0
- package/src/migrate.ts +19 -11
- package/src/pool-profile.ts +125 -0
- package/src/pool-reserve.ts +50 -0
- package/src/readonly-query.ts +21 -8
- package/src/replica-client.ts +16 -3
- package/src/replica-identity.ts +84 -0
- package/src/snapshot-parse.ts +11 -2
- package/src/sql.ts +36 -0
- package/src/statement-funnel.ts +101 -0
- package/src/ungeneratable.ts +11 -0
package/src/drift-findings.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import { onDeleteRule, rebuildForeignKey } from './foreign-key';
|
|
15
15
|
import type { CheckDescription, ForeignKeyDescription } from './introspect';
|
|
16
16
|
import type { Migration } from './migrate';
|
|
17
|
+
import { shellInertIdentifier } from './sql';
|
|
17
18
|
|
|
18
19
|
export type DriftKind =
|
|
19
20
|
| 'unexpected-column'
|
|
@@ -41,6 +42,15 @@ export interface DriftReport {
|
|
|
41
42
|
readonly differences: readonly DriftDifference[];
|
|
42
43
|
}
|
|
43
44
|
|
|
45
|
+
/**
|
|
46
|
+
* The one `fix:` here whose second layer no quoting closes. `x db gen "add C"` puts the column
|
|
47
|
+
* inside SHELL DOUBLE QUOTES, where `$(…)` and a backtick substitute before `x` is reached at all
|
|
48
|
+
* — and the argument is a migration DESCRIPTION, not an identifier, so there is no quoted form
|
|
49
|
+
* that would make a hostile name safe to pass. A name `shellInertIdentifier` (`sql.ts`) refuses
|
|
50
|
+
* is therefore left out of the command rather than escaped into it: the command still runs and
|
|
51
|
+
* still generates the migration, and the name is read off `cause` and `column`, which are prose
|
|
52
|
+
* nobody pastes.
|
|
53
|
+
*/
|
|
44
54
|
export function unexpectedColumn(table: string, column: string): DriftDifference {
|
|
45
55
|
return {
|
|
46
56
|
kind: 'unexpected-column',
|
|
@@ -48,7 +58,12 @@ export function unexpectedColumn(table: string, column: string): DriftDifference
|
|
|
48
58
|
column,
|
|
49
59
|
// Pinned by the contract. Do not reword without changing docs/errors/X_DB_DRIFT.
|
|
50
60
|
cause: `table "${table}" has column "${column}" not present in any migration`,
|
|
51
|
-
fix:
|
|
61
|
+
fix:
|
|
62
|
+
shellInertIdentifier(column) === null
|
|
63
|
+
? 'x db gen "add the undeclared column" # the live column name carries a backtick, a ' +
|
|
64
|
+
'dollar sign, a quote, a backslash or whitespace, so it is in the cause and not in ' +
|
|
65
|
+
'this command'
|
|
66
|
+
: `x db gen "add ${column}"`,
|
|
52
67
|
};
|
|
53
68
|
}
|
|
54
69
|
|
|
@@ -82,6 +97,8 @@ export function changedColumn(
|
|
|
82
97
|
liveNullable: boolean,
|
|
83
98
|
): DriftDifference {
|
|
84
99
|
const clause = liveNullable ? 'set not null' : 'drop not null';
|
|
100
|
+
const relation = shellInertIdentifier(table);
|
|
101
|
+
const attribute = shellInertIdentifier(column);
|
|
85
102
|
return {
|
|
86
103
|
kind: 'changed-column',
|
|
87
104
|
table,
|
|
@@ -89,19 +106,47 @@ export function changedColumn(
|
|
|
89
106
|
cause: liveNullable
|
|
90
107
|
? `table "${table}" allows NULL in column "${column}" that migrations declare not null`
|
|
91
108
|
: `table "${table}" forbids NULL in column "${column}" that migrations declare nullable`,
|
|
109
|
+
// Both identifiers are the catalog's, so both go through the one screen. A refusal names the
|
|
110
|
+
// column as the thing it could not spell, which is what tells this line apart from
|
|
111
|
+
// `missingCheck`'s refusal in a report that carries both.
|
|
92
112
|
fix:
|
|
93
|
-
|
|
94
|
-
|
|
113
|
+
relation === null || attribute === null
|
|
114
|
+
? `${clause} on the column named in this difference, in a new migration, then ` +
|
|
115
|
+
'x db migrate — its table or column name carries a backtick, a dollar sign, a quote, ' +
|
|
116
|
+
'a backslash or whitespace, so no statement here can spell it'
|
|
117
|
+
: `alter table ${relation} alter column ${attribute} ${clause}; # in a new migration` +
|
|
118
|
+
(liveNullable ? ' — backfill the existing NULLs first' : ''),
|
|
95
119
|
};
|
|
96
120
|
}
|
|
97
121
|
|
|
122
|
+
/**
|
|
123
|
+
* The `fix:` names the two edits that actually resolve this, and neither is `x db gen` (issue
|
|
124
|
+
* #345). That command diffs the ENTITY REGISTRY against the newest snapshot, and a table nothing
|
|
125
|
+
* declares is absent from both sides of that diff — so it wrote an EMPTY migration, and the
|
|
126
|
+
* generator's own empty-diff branch writes no file at all, leaving the reader with nothing to run
|
|
127
|
+
* and the same finding on the next deploy.
|
|
128
|
+
*
|
|
129
|
+
* What is left once `@ultimat3/cli`'s `acceptCreatedTables` has run is a table no migration's SQL
|
|
130
|
+
* creates and no entity declares — so either a migration should claim it (`if not exists`, because
|
|
131
|
+
* the relation is already there, and `x db migrate` then accepts a table its own SQL creates), or
|
|
132
|
+
* nothing owns it and it should not be in this schema. No migration PATH is named: where an app
|
|
133
|
+
* keeps its migrations is the CLI's fact, not this package's.
|
|
134
|
+
*/
|
|
98
135
|
export function unexpectedTable(table: string): DriftDifference {
|
|
136
|
+
const name = shellInertIdentifier(table);
|
|
99
137
|
return {
|
|
100
138
|
kind: 'unexpected-table',
|
|
101
139
|
table,
|
|
102
140
|
column: null,
|
|
103
141
|
cause: `table "${table}" is not present in any migration`,
|
|
104
|
-
fix:
|
|
142
|
+
fix:
|
|
143
|
+
name === null
|
|
144
|
+
? 'claim it in a migration with create table if not exists, or drop it by hand — its ' +
|
|
145
|
+
'table name carries a backtick, a dollar sign, a quote, a backslash or whitespace, so ' +
|
|
146
|
+
'no statement here can spell it'
|
|
147
|
+
: `put a create table if not exists ${name} (…) statement in a migration — x db migrate ` +
|
|
148
|
+
'then accepts a table its own SQL creates — or, if nothing owns it, run ' +
|
|
149
|
+
`drop table ${name}; inside psql "$DATABASE_URL"`,
|
|
105
150
|
};
|
|
106
151
|
}
|
|
107
152
|
|
|
@@ -174,6 +219,8 @@ export function changedIndex(table: string, index: string, detail: string): Drif
|
|
|
174
219
|
* the predicate, which is what makes an executable fix possible at all.
|
|
175
220
|
*/
|
|
176
221
|
export function missingCheck(table: string, check: CheckDescription): DriftDifference {
|
|
222
|
+
const relation = shellInertIdentifier(table);
|
|
223
|
+
const constraint = shellInertIdentifier(check.name);
|
|
177
224
|
return {
|
|
178
225
|
kind: 'missing-check',
|
|
179
226
|
table,
|
|
@@ -183,9 +230,19 @@ export function missingCheck(table: string, check: CheckDescription): DriftDiffe
|
|
|
183
230
|
// banned advice word the `errors` gate demands a command beside: writing the migration is half
|
|
184
231
|
// the repair and applying it is the other half, and `changedColumn`'s bare `# in a new
|
|
185
232
|
// migration` leaves the second half to be guessed.
|
|
233
|
+
//
|
|
234
|
+
// Both NAMES go through the one screen; the EXPRESSION deliberately does not, and cannot. It
|
|
235
|
+
// is a predicate, so no screen could accept `status in ('draft', 'published')` and reject a
|
|
236
|
+
// second statement — and it is the DECLARED side's own text, out of the author's migration,
|
|
237
|
+
// where both names are the catalog's and a sidecar's. Narrower than "this line is safe", and
|
|
238
|
+
// it is the honest claim.
|
|
186
239
|
fix:
|
|
187
|
-
|
|
188
|
-
|
|
240
|
+
relation === null || constraint === null
|
|
241
|
+
? 'add the constraint named in this difference back in a new migration, then ' +
|
|
242
|
+
'x db migrate — its table or constraint name carries a backtick, a dollar sign, a ' +
|
|
243
|
+
'quote, a backslash or whitespace, so no statement here can spell it'
|
|
244
|
+
: `alter table ${relation} add constraint ${constraint} ` +
|
|
245
|
+
`check (${check.expression}); # in a new migration, then x db migrate`,
|
|
189
246
|
};
|
|
190
247
|
}
|
|
191
248
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// TEST-ONLY. The two builders every drift suite compares with — a `TableDescription` of text
|
|
2
|
+
// columns and the `SchemaDescription` around it. One copy, because three suites arguing about
|
|
3
|
+
// schemas built differently would each be judging a different fixture. Never exported from
|
|
4
|
+
// `index.ts`.
|
|
5
|
+
|
|
6
|
+
import type { SchemaDescription, TableDescription } from './introspect';
|
|
7
|
+
|
|
8
|
+
export const table = (name: string, columns: readonly string[]): TableDescription => ({
|
|
9
|
+
schema: 'public',
|
|
10
|
+
name,
|
|
11
|
+
columns: columns.map((column, index) => ({
|
|
12
|
+
name: column,
|
|
13
|
+
dataType: 'text',
|
|
14
|
+
nullable: true,
|
|
15
|
+
default: null,
|
|
16
|
+
position: index + 1,
|
|
17
|
+
})),
|
|
18
|
+
primaryKey: ['id'],
|
|
19
|
+
indexes: [],
|
|
20
|
+
foreignKeys: [],
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
export const schema = (...tables: readonly TableDescription[]): SchemaDescription => ({ tables });
|
package/src/errors.ts
CHANGED
|
@@ -285,15 +285,6 @@ export const serializationExhausted = (attempts: number, sourceError: unknown):
|
|
|
285
285
|
sourceError,
|
|
286
286
|
});
|
|
287
287
|
|
|
288
|
-
/** The contract's pinned wording. Mirror of `@ultimat3/entity`'s `dbDrift()` — keep in sync. */
|
|
289
|
-
export const dbDrift = (tableName: string, columnName: string): DbError =>
|
|
290
|
-
new DbError({
|
|
291
|
-
code: 'X_DB_DRIFT',
|
|
292
|
-
cause: `table "${tableName}" has column "${columnName}" not present in any migration`,
|
|
293
|
-
fix: `x db gen "add ${columnName}"`,
|
|
294
|
-
meta: { table: tableName, column: columnName },
|
|
295
|
-
});
|
|
296
|
-
|
|
297
288
|
export const sqlUnsafe = (received: string, position: number): DbError =>
|
|
298
289
|
new DbError({
|
|
299
290
|
code: 'X_SQL_UNSAFE',
|
package/src/generate.ts
CHANGED
|
@@ -22,6 +22,8 @@ import {
|
|
|
22
22
|
} from './introspect';
|
|
23
23
|
import { declaredIndexes } from './invariant-ddl';
|
|
24
24
|
import { migrationIrreversible } from './migration-errors';
|
|
25
|
+
import type { ReplicaIdentityInput } from './replica-identity';
|
|
26
|
+
import { replicaIdentityFullAfter, replicaIdentityPlan } from './replica-identity';
|
|
25
27
|
import type { MovedAside } from './retype-dependents';
|
|
26
28
|
import { moveDependentsAside } from './retype-dependents';
|
|
27
29
|
import { moveKeysAside, retypedColumns, retypedIn } from './retype-keys';
|
|
@@ -54,7 +56,13 @@ function columnClause(column: ColumnDescriptionLike): string {
|
|
|
54
56
|
return parts.join(' ');
|
|
55
57
|
}
|
|
56
58
|
|
|
57
|
-
|
|
59
|
+
/** No table subscribed to, which is what every caller outside `generateMigration` describes. */
|
|
60
|
+
const NONE: ReadonlySet<string> = new Set();
|
|
61
|
+
|
|
62
|
+
export function snapshotOf(
|
|
63
|
+
entities: readonly EntityDescriptionLike[],
|
|
64
|
+
replicaIdentityFull: ReadonlySet<string> = NONE,
|
|
65
|
+
): SchemaDescription {
|
|
58
66
|
const tables = [...entities]
|
|
59
67
|
.sort((a, b) => (a.table < b.table ? -1 : 1))
|
|
60
68
|
.map((entity): TableDescription => {
|
|
@@ -99,6 +107,9 @@ export function snapshotOf(entities: readonly EntityDescriptionLike[]): SchemaDe
|
|
|
99
107
|
// read as "nothing recorded" so the next generation adds the constraints the database is
|
|
100
108
|
// genuinely missing — the rule `using` and `generated` already state one field up.
|
|
101
109
|
...(checks.length === 0 ? {} : { checks }),
|
|
110
|
+
// The same rule once more: `true` or absent, never `false`. This is what makes the ALTER
|
|
111
|
+
// beside it a one-time statement rather than a line every `x db gen` writes again.
|
|
112
|
+
...(replicaIdentityFull.has(entity.table) ? { replicaIdentityFull: true as const } : {}),
|
|
102
113
|
};
|
|
103
114
|
});
|
|
104
115
|
return { tables };
|
|
@@ -227,6 +238,17 @@ export interface GenerateOptions {
|
|
|
227
238
|
readonly now?: Date | undefined;
|
|
228
239
|
/** Allow a DROP COLUMN whose down cannot restore the data. `x db gen --allow-destructive`. */
|
|
229
240
|
readonly allowDestructive?: boolean | undefined;
|
|
241
|
+
/**
|
|
242
|
+
* Tables a live query subscribes to, so logical replication carries the old row —
|
|
243
|
+
* `@ultimat3/realtime` refuses a subscription without it. Absent or empty emits nothing.
|
|
244
|
+
*
|
|
245
|
+
* A parameter and never an `EntityDescriptionLike` field: which tables need it is DECLARED by
|
|
246
|
+
* each `live: true` query's `subscribes:`, never derived — the relation is a literal inside the
|
|
247
|
+
* query's `sql:` callback, which nothing can invoke without valid input. This package is tier 1
|
|
248
|
+
* and can see neither the manifest nor `@ultimat3/query`, so `@ultimat3/cli`'s `db-generate.ts`
|
|
249
|
+
* is the one caller that can answer it.
|
|
250
|
+
*/
|
|
251
|
+
readonly replicaIdentityFull?: readonly string[] | undefined;
|
|
230
252
|
}
|
|
231
253
|
|
|
232
254
|
export interface GeneratedMigration {
|
|
@@ -275,6 +297,9 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
|
|
|
275
297
|
// of one key can move in two different entities' diffs (`retype-keys.ts`).
|
|
276
298
|
const preAlters: Plan = { up: [], down: [] };
|
|
277
299
|
const wanted = new Set(options.entities.map((entity) => entity.table));
|
|
300
|
+
// The tables this run brings into being, read by the replica-identity arm below: their `down` is
|
|
301
|
+
// already `drop table`, so reverting the identity ahead of it performs nothing.
|
|
302
|
+
const created = new Set<string>();
|
|
278
303
|
|
|
279
304
|
const doomed = new Set(
|
|
280
305
|
current.tables.filter((table) => !wanted.has(table.name)).map((table) => table.name),
|
|
@@ -291,6 +316,7 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
|
|
|
291
316
|
if (live === undefined) {
|
|
292
317
|
plan.up.push(...createTable(entity));
|
|
293
318
|
plan.down.push(`drop table ${identifier(entity.table).text};`);
|
|
319
|
+
created.add(entity.table);
|
|
294
320
|
continue;
|
|
295
321
|
}
|
|
296
322
|
diffTable(entity, live, plan, retypedIn(retyped, entity.table));
|
|
@@ -337,6 +363,18 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
|
|
|
337
363
|
plan.up.push(...constraints.up);
|
|
338
364
|
plan.down.push(...constraints.down);
|
|
339
365
|
|
|
366
|
+
// Dead last in `up`, and it is the only placement that is right for every arm: the table has to
|
|
367
|
+
// exist, and a `create table` in this same migration is the reason it might not. It is ordered
|
|
368
|
+
// against nothing else — replica identity constrains no column, no index and no constraint — so
|
|
369
|
+
// the end is where it can never be read as depending on a statement above it.
|
|
370
|
+
const replicaIdentity: ReplicaIdentityInput = {
|
|
371
|
+
wanted: options.replicaIdentityFull,
|
|
372
|
+
declared: wanted,
|
|
373
|
+
created,
|
|
374
|
+
current,
|
|
375
|
+
};
|
|
376
|
+
replicaIdentityPlan(plan, replicaIdentity);
|
|
377
|
+
|
|
340
378
|
const id = `${migrationStamp(options.now ?? systemClock.now())}_${slugify(options.name)}`;
|
|
341
379
|
// At the TOP of `up`, so what is MISSING is the first thing read — and a line comment, so it is
|
|
342
380
|
// noise to every reader that matters: `statementsOf` drops a chunk of comments alone,
|
|
@@ -362,7 +400,7 @@ export function generateMigration(options: GenerateOptions): GeneratedMigration
|
|
|
362
400
|
// FRONT here precisely so reversal puts it last — a key is added back only once both of its
|
|
363
401
|
// ends have been retyped back, which is every other statement in the script.
|
|
364
402
|
down: [...preAlters.down, ...plan.down].reverse().join('\n'),
|
|
365
|
-
snapshot: snapshotOf(options.entities),
|
|
403
|
+
snapshot: snapshotOf(options.entities, replicaIdentityFullAfter(replicaIdentity)),
|
|
366
404
|
destructive: isDestructive(up),
|
|
367
405
|
unrendered,
|
|
368
406
|
};
|
package/src/index.ts
CHANGED
|
@@ -23,25 +23,15 @@ export {
|
|
|
23
23
|
export type {
|
|
24
24
|
DbClient,
|
|
25
25
|
DbConnection,
|
|
26
|
-
DbHealthReport,
|
|
27
|
-
PoolProfile,
|
|
28
26
|
PostgresClient,
|
|
29
27
|
PostgresClientOptions,
|
|
30
28
|
ReservableClient,
|
|
31
29
|
} from './client';
|
|
32
|
-
export {
|
|
33
|
-
baseClient,
|
|
34
|
-
checkDb,
|
|
35
|
-
createPostgresClient,
|
|
36
|
-
db,
|
|
37
|
-
isReservable,
|
|
38
|
-
POOL_MAX_ENV,
|
|
39
|
-
POOL_PROFILES,
|
|
40
|
-
poolProfileFor,
|
|
41
|
-
setDbClient,
|
|
42
|
-
} from './client';
|
|
30
|
+
export { baseClient, createPostgresClient, db, isReservable, setDbClient } from './client';
|
|
43
31
|
export type { ColumnDefaultLike } from './column-default';
|
|
44
32
|
export { defaultExpression } from './column-default';
|
|
33
|
+
export type { DbHealthReport } from './db-health';
|
|
34
|
+
export { checkDb } from './db-health';
|
|
45
35
|
export { defaultClient, REPLICA_URL_ENV } from './default-client';
|
|
46
36
|
export type { DestructiveKind, DestructiveStatement } from './destructive';
|
|
47
37
|
export {
|
|
@@ -62,6 +52,7 @@ export {
|
|
|
62
52
|
expectedSchema,
|
|
63
53
|
FRAMEWORK_TABLE_PREFIX,
|
|
64
54
|
} from './drift';
|
|
55
|
+
export { dbDrift } from './drift-errors';
|
|
65
56
|
export type {
|
|
66
57
|
ColumnDescriptionLike,
|
|
67
58
|
EntityDescriptionLike,
|
|
@@ -76,7 +67,6 @@ export {
|
|
|
76
67
|
DB_ERROR_RETRY,
|
|
77
68
|
DB_ERROR_TITLES,
|
|
78
69
|
DbError,
|
|
79
|
-
dbDrift,
|
|
80
70
|
dbNotImplemented,
|
|
81
71
|
dbUnavailable,
|
|
82
72
|
driverError,
|
|
@@ -169,6 +159,8 @@ export {
|
|
|
169
159
|
} from './pglite';
|
|
170
160
|
export type { PgliteBranchInfo, PgliteBranchOptions } from './pglite-branch';
|
|
171
161
|
export { branchPglite, pgliteBranchDir } from './pglite-branch';
|
|
162
|
+
export type { PoolProfile } from './pool-profile';
|
|
163
|
+
export { POOL_MAX_ENV, POOL_PROFILES, poolProfileFor } from './pool-profile';
|
|
172
164
|
export type { ReadOnlyQueryOptions, ReadOnlyQueryResult } from './readonly-query';
|
|
173
165
|
export { READONLY_TIMEOUT_MS, readOnlyQuery } from './readonly-query';
|
|
174
166
|
export type { ReadOnlyRoleOptions } from './readonly-role';
|
|
@@ -185,7 +177,15 @@ export { markScopeWrote, replicaScope, withReplicaReads } from './replica-scope'
|
|
|
185
177
|
export { snapshotJson } from './snapshot-json';
|
|
186
178
|
export { parseSnapshot } from './snapshot-parse';
|
|
187
179
|
export type { SqlFragment } from './sql';
|
|
188
|
-
export {
|
|
180
|
+
export {
|
|
181
|
+
identifier,
|
|
182
|
+
isSqlFragment,
|
|
183
|
+
join,
|
|
184
|
+
literal,
|
|
185
|
+
raw,
|
|
186
|
+
shellInertIdentifier,
|
|
187
|
+
sql,
|
|
188
|
+
} from './sql';
|
|
189
189
|
export { stripSqlNoise } from './sql-noise';
|
|
190
190
|
export type { DbSqlStateCode } from './sqlstate';
|
|
191
191
|
export { DB_SQLSTATE_CODES, isRetryableState, SQLSTATE, sqlState, sqlStateCode } from './sqlstate';
|
package/src/introspect.ts
CHANGED
|
@@ -105,6 +105,22 @@ export interface TableDescription {
|
|
|
105
105
|
* read one instead of reporting every declared constraint as missing.
|
|
106
106
|
*/
|
|
107
107
|
readonly checkNames?: readonly string[] | undefined;
|
|
108
|
+
/**
|
|
109
|
+
* That a migration set `replica identity full` on this table — what `@ultimat3/realtime` requires
|
|
110
|
+
* of every table a live query subscribes to, and the only reason this generator emits it.
|
|
111
|
+
*
|
|
112
|
+
* `true` or absent, never `false`, and the literal type is what enforces it. Absent means
|
|
113
|
+
* *nothing recorded*: a sidecar written before this field existed, exactly as `checks` absent
|
|
114
|
+
* means "no constraint was recorded" rather than "none is declared". Writing `false` onto every
|
|
115
|
+
* table an app has would rewrite every sidecar in the tree on the next `x db gen` for a fact that
|
|
116
|
+
* was already true — the argument `IndexDescription.using` makes about `btree`.
|
|
117
|
+
*
|
|
118
|
+
* `introspect()` never answers it. The catalog's half is `pg_class.relreplident`, which
|
|
119
|
+
* `@ultimat3/realtime`'s preflight already reads at the only moment it matters; a second reader
|
|
120
|
+
* here would let a `diffSchema` compare a declaration against a catalog value, which is the
|
|
121
|
+
* mistake `checks` and `checkNames` exist as two fields to prevent.
|
|
122
|
+
*/
|
|
123
|
+
readonly replicaIdentityFull?: true | undefined;
|
|
108
124
|
}
|
|
109
125
|
|
|
110
126
|
export interface SchemaDescription {
|
package/src/migrate.ts
CHANGED
|
@@ -3,18 +3,13 @@
|
|
|
3
3
|
// app-version fence is the `migrate` role's contract — a pod must refuse to migrate a database
|
|
4
4
|
// another build already owns, because the alternative is two schemas racing during a rollout.
|
|
5
5
|
|
|
6
|
-
import { appVersion } from '@ultimat3/core';
|
|
7
|
-
import {
|
|
8
|
-
baseClient,
|
|
9
|
-
type DbClient,
|
|
10
|
-
type DbConnection,
|
|
11
|
-
isReservable,
|
|
12
|
-
poolProfileFor,
|
|
13
|
-
} from './client';
|
|
6
|
+
import { appVersion, finiteCount } from '@ultimat3/core';
|
|
7
|
+
import { baseClient, type DbClient, type DbConnection, isReservable } from './client';
|
|
14
8
|
import { refuseDependentViews } from './dependent-view';
|
|
15
9
|
import { expectedQueryLoop } from './expected-loop';
|
|
16
10
|
import type { SchemaDescription } from './introspect';
|
|
17
11
|
import { migrateConcurrent, migrationConflict, rollbackStepsInvalid } from './migration-errors';
|
|
12
|
+
import { poolProfileFor } from './pool-profile';
|
|
18
13
|
import { raw, sql } from './sql';
|
|
19
14
|
import { SQLSTATE, sqlState } from './sqlstate';
|
|
20
15
|
import { statementsOf } from './statement-split';
|
|
@@ -237,6 +232,9 @@ async function acquireLock(session: DbClient, waitMs: number): Promise<void> {
|
|
|
237
232
|
sql`select pg_try_advisory_lock(${MIGRATION_LOCK_KEY}) as locked`,
|
|
238
233
|
);
|
|
239
234
|
if (row?.locked === true) return;
|
|
235
|
+
// `waitMs` is screened at both call sites: `NaN - elapsed` is `NaN`, `NaN <= 0` is false
|
|
236
|
+
// and `Bun.sleep(Math.min(POLL, NaN))` does not sleep — so an unbounded wait is not what a
|
|
237
|
+
// non-finite one produced, a tight spin re-taking `pg_try_advisory_lock` was.
|
|
240
238
|
const remaining = waitMs - (performance.now() - started);
|
|
241
239
|
if (remaining <= 0) {
|
|
242
240
|
throw migrateConcurrent(MIGRATION_LOCK_KEY, Math.round(performance.now() - started));
|
|
@@ -317,7 +315,17 @@ async function setLockTimeout(tx: DbTx, lockTimeoutMs: number): Promise<void> {
|
|
|
317
315
|
* same `ACCESS EXCLUSIVE` locks against the same tables.
|
|
318
316
|
*/
|
|
319
317
|
function migrationLockTimeoutMs(explicit: number | undefined): number {
|
|
320
|
-
|
|
318
|
+
// Screened here, where BOTH `migrate()` and `rollback()` resolve it, and before either takes the
|
|
319
|
+
// advisory lock. `lockWaitMs` beside it was screened from the start and this one was not, so
|
|
320
|
+
// `Number(process.env.…)` on an unset variable travelled all the way to `SET LOCAL lock_timeout
|
|
321
|
+
// = NaN`, which the server rejects inside the migration's own transaction — the option that was
|
|
322
|
+
// wrong appears nowhere in what the deploy prints. `finiteCount`, never a second bound checker.
|
|
323
|
+
return finiteCount(
|
|
324
|
+
'migrate',
|
|
325
|
+
'lockTimeoutMs',
|
|
326
|
+
explicit ?? poolProfileFor('migrate').lockTimeoutMs,
|
|
327
|
+
0,
|
|
328
|
+
);
|
|
321
329
|
}
|
|
322
330
|
|
|
323
331
|
export async function migrate(options: MigrateOptions): Promise<MigrationReport> {
|
|
@@ -329,7 +337,7 @@ export async function migrate(options: MigrateOptions): Promise<MigrationReport>
|
|
|
329
337
|
return withAdvisoryLock(
|
|
330
338
|
client,
|
|
331
339
|
options.lock !== false,
|
|
332
|
-
options.lockWaitMs ?? MIGRATION_LOCK_WAIT_MS,
|
|
340
|
+
finiteCount('migrate', 'lockWaitMs', options.lockWaitMs ?? MIGRATION_LOCK_WAIT_MS, 0),
|
|
333
341
|
async (session) => {
|
|
334
342
|
await ensureLedger(session);
|
|
335
343
|
const ledger = await readLedger(session);
|
|
@@ -419,7 +427,7 @@ export async function rollback(options: RollbackOptions): Promise<readonly strin
|
|
|
419
427
|
return withAdvisoryLock(
|
|
420
428
|
client,
|
|
421
429
|
options.lock !== false,
|
|
422
|
-
options.lockWaitMs ?? MIGRATION_LOCK_WAIT_MS,
|
|
430
|
+
finiteCount('migrate', 'lockWaitMs', options.lockWaitMs ?? MIGRATION_LOCK_WAIT_MS, 0),
|
|
423
431
|
async (session) => {
|
|
424
432
|
const ledger = await readLedger(session);
|
|
425
433
|
const targets = [...ledger].reverse().slice(0, steps);
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// Single responsibility: the five numbers a Postgres pool runs on — the per-role defaults, the one
|
|
2
|
+
// environment override an operator may layer over them, and the screen every resolved profile
|
|
3
|
+
// passes. Split from `client.ts`, which now owns connecting and nothing about sizing.
|
|
4
|
+
|
|
5
|
+
import { assert, type Role, resolveRole } from '@ultimat3/core';
|
|
6
|
+
import { poolMaxInvalid } from './errors';
|
|
7
|
+
|
|
8
|
+
export interface PoolProfile {
|
|
9
|
+
readonly max: number;
|
|
10
|
+
/** 0 disables the timeout — only `migrate`, which is allowed to take as long as it takes. */
|
|
11
|
+
readonly statementTimeoutMs: number;
|
|
12
|
+
readonly idleTimeoutMs: number;
|
|
13
|
+
/**
|
|
14
|
+
* How long a statement may **wait for a lock** before `55P03`, distinct from how long it may run.
|
|
15
|
+
* 0 everywhere but `migrate`, which is the only role that takes `ACCESS EXCLUSIVE`: an `alter
|
|
16
|
+
* table` queued behind a long `SELECT` puts every later query on that table behind it too,
|
|
17
|
+
* because Postgres' lock queue is FIFO — and `migrate` runs `statement_timeout = 0`, so nothing
|
|
18
|
+
* else would ever end the wait. Read by `migrate()` as a `SET LOCAL`, never by the pool.
|
|
19
|
+
*/
|
|
20
|
+
readonly lockTimeoutMs: number;
|
|
21
|
+
/**
|
|
22
|
+
* How long `reserve()` may wait for a free connection before `X_DB_POOL_EXHAUSTED`. 0 waits
|
|
23
|
+
* forever, which is what a run-once role wants and what a request-serving one must never do:
|
|
24
|
+
* queueing turns exhaustion into a hang, `/readyz`'s `select 1` joins the same queue, the kubelet
|
|
25
|
+
* kills the pod, and the replacement inherits the same saturated database.
|
|
26
|
+
*/
|
|
27
|
+
readonly acquireTimeoutMs: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Sized per role because the failure modes differ: RPS bursts vs. queue depth vs. run-once. */
|
|
31
|
+
export const POOL_PROFILES = Object.freeze<Record<Role, PoolProfile>>({
|
|
32
|
+
web: {
|
|
33
|
+
max: 20,
|
|
34
|
+
statementTimeoutMs: 10_000,
|
|
35
|
+
idleTimeoutMs: 30_000,
|
|
36
|
+
lockTimeoutMs: 0,
|
|
37
|
+
acquireTimeoutMs: 5_000,
|
|
38
|
+
},
|
|
39
|
+
sync: {
|
|
40
|
+
max: 10,
|
|
41
|
+
statementTimeoutMs: 10_000,
|
|
42
|
+
idleTimeoutMs: 60_000,
|
|
43
|
+
lockTimeoutMs: 0,
|
|
44
|
+
acquireTimeoutMs: 5_000,
|
|
45
|
+
},
|
|
46
|
+
worker: {
|
|
47
|
+
max: 8,
|
|
48
|
+
statementTimeoutMs: 120_000,
|
|
49
|
+
idleTimeoutMs: 30_000,
|
|
50
|
+
lockTimeoutMs: 0,
|
|
51
|
+
acquireTimeoutMs: 10_000,
|
|
52
|
+
},
|
|
53
|
+
scheduler: {
|
|
54
|
+
max: 2,
|
|
55
|
+
statementTimeoutMs: 15_000,
|
|
56
|
+
idleTimeoutMs: 60_000,
|
|
57
|
+
lockTimeoutMs: 0,
|
|
58
|
+
acquireTimeoutMs: 10_000,
|
|
59
|
+
},
|
|
60
|
+
// `migrate` waits: its pool is `max: 1` and the advisory-lock pin holds it for the whole run, so
|
|
61
|
+
// a deadline here would refuse the migration's own session. The wait that needed bounding is the
|
|
62
|
+
// advisory lock's, and `MIGRATION_LOCK_WAIT_MS` bounds it.
|
|
63
|
+
migrate: {
|
|
64
|
+
max: 1,
|
|
65
|
+
statementTimeoutMs: 0,
|
|
66
|
+
idleTimeoutMs: 10_000,
|
|
67
|
+
lockTimeoutMs: 3_000,
|
|
68
|
+
acquireTimeoutMs: 0,
|
|
69
|
+
},
|
|
70
|
+
replicator: {
|
|
71
|
+
max: 4,
|
|
72
|
+
statementTimeoutMs: 0,
|
|
73
|
+
idleTimeoutMs: 60_000,
|
|
74
|
+
lockTimeoutMs: 0,
|
|
75
|
+
acquireTimeoutMs: 0,
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
export function poolProfileFor(role: Role = resolveRole()): PoolProfile {
|
|
80
|
+
return POOL_PROFILES[role];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The one pool knob an operator can turn without a rebuild. Layered over the role default. */
|
|
84
|
+
export const POOL_MAX_ENV = 'DATABASE_POOL_MAX';
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* `DATABASE_POOL_MAX`, or nothing. `POOL_PROFILES` is frozen into the build, so before this the
|
|
88
|
+
* only way to change a fleet's connection count was to ship a new image — and 400 `web` pods at
|
|
89
|
+
* `max: 20` is 8,000 backends against a `max_connections` of 450. An unparseable value **refuses**
|
|
90
|
+
* rather than falling back: a fleet that ignored the number it was given is the failure the
|
|
91
|
+
* variable exists to prevent, and it would only be found in `pg_stat_activity` at 3am.
|
|
92
|
+
*/
|
|
93
|
+
export function poolMaxFromEnv(): Partial<PoolProfile> {
|
|
94
|
+
const raw = process.env[POOL_MAX_ENV];
|
|
95
|
+
if (raw === undefined || raw.trim() === '') return {};
|
|
96
|
+
const max = Number(raw);
|
|
97
|
+
if (!Number.isSafeInteger(max) || max < 1) throw poolMaxInvalid(raw);
|
|
98
|
+
return { max };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The five numbers a pool runs on, screened on the MERGED profile — an override is spread over a
|
|
103
|
+
* role default the caller never restated, so the resolved object is the only one that can be
|
|
104
|
+
* judged. Every one of them is a plausible `Number(process.env.…)`, which is `NaN` for an unset
|
|
105
|
+
* variable and not nullish, so `??` and the spread both keep it. None of the five then fails
|
|
106
|
+
* loudly: `idleTimeout: NaN` goes to `Bun.SQL`, `statement_timeout=NaN` goes into the libpq
|
|
107
|
+
* options string for the SERVER to reject on connect, and a timer given `NaN` fires at 1ms in this
|
|
108
|
+
* Bun — so a pool with free connections reports itself exhausted. `0` stays legal for the three
|
|
109
|
+
* budgets that document it as "no bound"; `max` is at least one connection, or nothing can run.
|
|
110
|
+
*/
|
|
111
|
+
export function assertPoolProfile(profile: PoolProfile): PoolProfile {
|
|
112
|
+
const whole = (option: string, value: number, min: 0 | 1): void => {
|
|
113
|
+
assert(
|
|
114
|
+
Number.isSafeInteger(value) && value >= min,
|
|
115
|
+
`pool profile ${option} is ${String(value)}; it must be a whole number of ${min === 1 ? 'at least 1' : '0 or more, where 0 is the documented "no bound"'}`,
|
|
116
|
+
`pass a whole number for ${option} in createPostgresClient({ profile }), and parse an environment value first — Number(process.env.DATABASE_${option.toUpperCase()} ?? '') is NaN when the variable is unset`,
|
|
117
|
+
);
|
|
118
|
+
};
|
|
119
|
+
whole('max', profile.max, 1);
|
|
120
|
+
whole('statementTimeoutMs', profile.statementTimeoutMs, 0);
|
|
121
|
+
whole('idleTimeoutMs', profile.idleTimeoutMs, 0);
|
|
122
|
+
whole('lockTimeoutMs', profile.lockTimeoutMs, 0);
|
|
123
|
+
whole('acquireTimeoutMs', profile.acquireTimeoutMs, 0);
|
|
124
|
+
return profile;
|
|
125
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// Single responsibility: pinning a connection out of the pool under the profile's acquire deadline,
|
|
2
|
+
// and giving back a reservation that arrives after the deadline has passed. Split from `client.ts`,
|
|
3
|
+
// which now asks for a pin rather than owning what "waited too long" means.
|
|
4
|
+
|
|
5
|
+
import type { BunSqlDriver, BunSqlReserved } from './bun-sql';
|
|
6
|
+
import { poolAcquireTimeout } from './errors';
|
|
7
|
+
import type { PoolProfile } from './pool-profile';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* `pool.reserve()` under a deadline. Without one an exhausted pool does not fail, it **queues** —
|
|
11
|
+
* so a slow endpoint filling all 20 slots turns every later request, `/readyz`'s `select 1`
|
|
12
|
+
* included, into a wait with no end and no error, and the pod is killed for being unready rather
|
|
13
|
+
* than answering 503 for the requests it cannot serve.
|
|
14
|
+
*
|
|
15
|
+
* The losing reservation is released, never dropped: the pool hands out a connection whenever one
|
|
16
|
+
* frees, deadline or no deadline, and a pin nobody holds is a connection nobody gets back. That is
|
|
17
|
+
* the whole reason this is not a bare `Promise.race`.
|
|
18
|
+
*/
|
|
19
|
+
export async function reserveWithin(
|
|
20
|
+
pool: Pick<BunSqlDriver, 'reserve'>,
|
|
21
|
+
profile: PoolProfile,
|
|
22
|
+
): Promise<BunSqlReserved> {
|
|
23
|
+
const budget = profile.acquireTimeoutMs;
|
|
24
|
+
if (budget <= 0) return pool.reserve();
|
|
25
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
26
|
+
let expired = false;
|
|
27
|
+
const pending = pool.reserve();
|
|
28
|
+
try {
|
|
29
|
+
return await Promise.race([
|
|
30
|
+
pending,
|
|
31
|
+
new Promise<never>((_resolve, reject) => {
|
|
32
|
+
timer = setTimeout(() => {
|
|
33
|
+
expired = true;
|
|
34
|
+
reject(poolAcquireTimeout(budget, profile.max));
|
|
35
|
+
}, budget);
|
|
36
|
+
// The deadline must not be what keeps a finished process alive.
|
|
37
|
+
timer.unref?.();
|
|
38
|
+
}),
|
|
39
|
+
]);
|
|
40
|
+
} finally {
|
|
41
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
42
|
+
// Attached unconditionally so a rejection arriving after we gave up is handled, not unhandled.
|
|
43
|
+
void pending.then(
|
|
44
|
+
(late) => {
|
|
45
|
+
if (expired) late.release();
|
|
46
|
+
},
|
|
47
|
+
() => undefined,
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
}
|
package/src/readonly-query.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// enforcing read-only beats a regex, and the timeout bounds a runaway scan the agent never
|
|
4
4
|
// meant to ask for.
|
|
5
5
|
|
|
6
|
+
import { finiteCount } from '@ultimat3/core';
|
|
6
7
|
import { baseClient, type DbClient, type DbConnection, isReservable } from './client';
|
|
7
8
|
import { multipleStatements } from './errors';
|
|
8
9
|
import { identifier, raw, sql } from './sql';
|
|
@@ -89,6 +90,26 @@ export async function readOnlyQuery<T>(
|
|
|
89
90
|
const statements = statementsOf(statement);
|
|
90
91
|
if (statements.length > 1) throw multipleStatements(statement, statements.length);
|
|
91
92
|
|
|
93
|
+
// Decided before anything is opened, for the reason `rollback({ steps })` screens before it
|
|
94
|
+
// takes the advisory lock: a value this build cannot honour is not a fact about the pool. It
|
|
95
|
+
// was computed after `reserve()` and after `BEGIN READ ONLY`, so an unbounded `timeoutMs`
|
|
96
|
+
// against an exhausted pool answered with the pool's error — or waited for a connection it was
|
|
97
|
+
// never going to use — in place of the `X_INVARIANT` naming the option that was wrong.
|
|
98
|
+
//
|
|
99
|
+
// Clamped and truncated to an integer: the result is a JS number the caller never touches
|
|
100
|
+
// as text, so there is nothing here for `raw()` to inject — `SET LOCAL` can't bind `$n`
|
|
101
|
+
// parameters, which is why this can't go through `sql` the normal, parameterised way.
|
|
102
|
+
// REFUSED rather than normalised: `NaN` used to take the default silently, so a config typo
|
|
103
|
+
// ran under a timeout nobody wrote. Only an explicit 0 disables the layer, which is why the
|
|
104
|
+
// floor is 0 and not 1; the hour ceiling is a clamp on a number that IS one.
|
|
105
|
+
const asked = finiteCount(
|
|
106
|
+
'readonlyQuery',
|
|
107
|
+
'timeoutMs',
|
|
108
|
+
options.timeoutMs ?? READONLY_TIMEOUT_MS,
|
|
109
|
+
0,
|
|
110
|
+
);
|
|
111
|
+
const ms = Math.min(3_600_000, asked);
|
|
112
|
+
|
|
92
113
|
const client = options.client ?? baseClient();
|
|
93
114
|
// A pooled BEGIN that lands on a different physical connection than the query that follows is
|
|
94
115
|
// not a transaction at all, so a reservable client must pin one connection for the sequence.
|
|
@@ -104,14 +125,6 @@ export async function readOnlyQuery<T>(
|
|
|
104
125
|
await connection.execute(raw('BEGIN READ ONLY'));
|
|
105
126
|
guards.push('txn:read-only');
|
|
106
127
|
|
|
107
|
-
// Clamped and truncated to an integer: the result is a JS number the caller never touches
|
|
108
|
-
// as text, so there is nothing here for `raw()` to inject — `SET LOCAL` can't bind `$n`
|
|
109
|
-
// parameters, which is why this can't go through `sql` the normal, parameterised way.
|
|
110
|
-
// NaN normalises to the default first: only an explicit 0 disables the timeout, and
|
|
111
|
-
// `Math.min(3_600_000, NaN)` is NaN, which would fail `ms > 0` and silently skip the layer.
|
|
112
|
-
const asked = options.timeoutMs ?? READONLY_TIMEOUT_MS;
|
|
113
|
-
const requested = Number.isNaN(asked) ? READONLY_TIMEOUT_MS : asked;
|
|
114
|
-
const ms = Math.max(0, Math.min(3_600_000, Math.trunc(requested)));
|
|
115
128
|
if (ms > 0) {
|
|
116
129
|
// `LOCAL`, so the setting dies with the transaction — one agent read must not re-time
|
|
117
130
|
// every request the pool serves afterwards. Caveat: embedded PGlite applies the setting
|