@ultimat3/db 29.4.0 → 30.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +74 -110
- package/README.md +163 -15
- package/package.json +13 -2
- package/src/app-relation.ts +7 -2
- package/src/catalog-fold.ts +5 -0
- package/src/catalog-objects.ts +20 -24
- package/src/catalog-policies.ts +81 -0
- package/src/catalog-relations.ts +31 -10
- package/src/catalog.ts +40 -1
- package/src/check-ddl.ts +34 -7
- package/src/client.ts +132 -48
- package/src/column-alter.ts +42 -1
- package/src/concurrent-index.ts +20 -0
- package/src/concurrent-plan.ts +135 -17
- package/src/connection-url.ts +16 -4
- package/src/db-diagnostics-deadline.ts +43 -0
- package/src/db-diagnostics-redact.ts +58 -0
- package/src/db-diagnostics-rows.ts +166 -0
- package/src/db-diagnostics-sql.ts +92 -0
- package/src/db-diagnostics.ts +178 -0
- package/src/destructive.ts +17 -3
- package/src/diagnostics-entry.ts +15 -0
- package/src/drift-errors.ts +9 -4
- package/src/drift-findings.ts +47 -62
- package/src/drift-missing.ts +203 -0
- package/src/drift-row-security.ts +88 -0
- package/src/drift-triggers.ts +54 -0
- package/src/drift.ts +88 -21
- package/src/entity-shape.ts +67 -0
- package/src/errors.ts +77 -9
- package/src/expected-loop.ts +34 -4
- package/src/failure-store-pg.ts +157 -0
- package/src/foreign-key-plan.ts +75 -23
- package/src/foreign-key.ts +76 -5
- package/src/generate-counter.ts +85 -0
- package/src/generate-history.ts +142 -0
- package/src/generate-row-security.ts +51 -0
- package/src/generate-snapshot.ts +82 -0
- package/src/generate-steps.ts +6 -1
- package/src/generate-triggers.ts +119 -0
- package/src/generate-view.ts +97 -0
- package/src/generate.ts +105 -79
- package/src/index.ts +129 -6
- package/src/introspect-catalog.ts +26 -6
- package/src/introspect-rows.ts +82 -0
- package/src/introspect.ts +108 -50
- package/src/keyset.ts +182 -0
- package/src/lifecycle-errors.ts +15 -0
- package/src/local-settings.ts +38 -0
- package/src/marker-line.ts +12 -4
- package/src/migrate-lock-retry.ts +60 -0
- package/src/migrate-rollback.ts +45 -6
- package/src/migrate.ts +45 -4
- package/src/migration-errors.ts +27 -1
- package/src/migration-ledger.ts +38 -4
- package/src/migration-step.ts +50 -22
- package/src/partition-order.ts +56 -0
- package/src/partitioned-tables.ts +24 -0
- package/src/pool-gate.ts +120 -0
- package/src/pool-listeners.ts +143 -0
- package/src/pool-pins.ts +68 -0
- package/src/pool-profile.ts +23 -7
- package/src/pool-reserve.ts +11 -4
- package/src/pool-retire.ts +22 -0
- package/src/pool-sweep.ts +115 -0
- package/src/pool-timer.ts +10 -0
- package/src/pooler-fix.ts +53 -0
- package/src/pooler.ts +161 -0
- package/src/read-transaction.ts +59 -0
- package/src/readonly-query.ts +4 -4
- package/src/replica-client.ts +54 -22
- package/src/replica-position.ts +89 -0
- package/src/replica-scope.ts +25 -4
- package/src/retired.ts +131 -0
- package/src/retype-dependents.ts +2 -0
- package/src/rolling-errors.ts +97 -0
- package/src/rolling.ts +212 -0
- package/src/row-secure-client.ts +54 -0
- package/src/row-security-catalog.ts +26 -0
- package/src/row-security-migrator.ts +48 -0
- package/src/row-security-plan.ts +58 -0
- package/src/row-security-role.ts +60 -0
- package/src/row-security-roles.ts +64 -0
- package/src/row-security.ts +59 -0
- package/src/same-tenant.ts +173 -0
- package/src/schema-dump-policies.ts +26 -0
- package/src/schema-dump-table.ts +21 -6
- package/src/schema-dump.ts +13 -2
- package/src/snapshot-parse.ts +42 -2
- package/src/sqlstate.ts +4 -0
- package/src/statement-bound.ts +82 -0
- package/src/transaction-errors.ts +14 -0
- package/src/transaction-guard.ts +40 -0
- package/src/transaction-settings.ts +24 -0
- package/src/transaction.ts +74 -19
- package/src/ungeneratable.ts +30 -0
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Single responsibility: the catalog ROWS `introspectSchema()` reads, and the pure helpers that turn
|
|
2
|
+
// one into a description field — a key's rule with its list, a table's `NOT VALID` names. Split
|
|
3
|
+
// from `introspect.ts`, which issues the queries and folds the rows into tables.
|
|
4
|
+
|
|
5
|
+
export interface ColumnRow {
|
|
6
|
+
readonly table_name: string;
|
|
7
|
+
readonly column_name: string;
|
|
8
|
+
readonly data_type: string;
|
|
9
|
+
readonly is_nullable: string;
|
|
10
|
+
readonly column_default: string | null;
|
|
11
|
+
readonly ordinal_position: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface IndexRow {
|
|
15
|
+
readonly table_name: string;
|
|
16
|
+
readonly index_name: string;
|
|
17
|
+
readonly is_unique: boolean;
|
|
18
|
+
readonly is_primary: boolean;
|
|
19
|
+
readonly columns: readonly string[];
|
|
20
|
+
readonly predicate: string | null;
|
|
21
|
+
readonly descending: boolean;
|
|
22
|
+
readonly method?: string | undefined;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface ForeignKeyRow {
|
|
26
|
+
readonly table_name: string;
|
|
27
|
+
readonly constraint_name: string;
|
|
28
|
+
readonly columns: readonly string[];
|
|
29
|
+
readonly referenced_table: string;
|
|
30
|
+
readonly referenced_columns: readonly string[];
|
|
31
|
+
readonly on_delete: string | null;
|
|
32
|
+
/** `set null (a, b)`'s list as the server prints it. Postgres 15+; `null` for any other rule. */
|
|
33
|
+
readonly set_null_columns?: string | null | undefined;
|
|
34
|
+
readonly validated?: boolean | undefined;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* A key's rule in the spelling a description holds, list included. The list reads the definition
|
|
39
|
+
* text, not `confdelsetcols`, which is a column only Postgres 15+ has: this query runs against
|
|
40
|
+
* whatever an operator points it at. The server quotes a reserved word and a description never
|
|
41
|
+
* does, so each name is unquoted or `user` is a permanent `changed-foreign-key`.
|
|
42
|
+
*/
|
|
43
|
+
const unquotedName = (name: string): string => {
|
|
44
|
+
const trimmed = name.trim();
|
|
45
|
+
return trimmed.startsWith('"') && trimmed.endsWith('"')
|
|
46
|
+
? trimmed.slice(1, -1).replaceAll('""', '"')
|
|
47
|
+
: trimmed;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
export const onDeleteOf = (row: ForeignKeyRow): string | null =>
|
|
51
|
+
row.set_null_columns == null
|
|
52
|
+
? row.on_delete
|
|
53
|
+
: `set null (${row.set_null_columns.split(',').map(unquotedName).join(', ')})`;
|
|
54
|
+
|
|
55
|
+
/** A CHECK constraint's NAME. There is deliberately no column for its definition — see `checkNames`. */
|
|
56
|
+
export interface CheckRow {
|
|
57
|
+
readonly table_name: string;
|
|
58
|
+
readonly constraint_name: string;
|
|
59
|
+
readonly validated?: boolean | undefined;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** An enabled trigger's NAME — the half of `appendOnly` the catalog can answer. */
|
|
63
|
+
export interface TriggerNameRow {
|
|
64
|
+
readonly table_name: string;
|
|
65
|
+
readonly trigger_name: string;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Absent when none: `false` only, so a row a fake client wrote without the field is not one. */
|
|
69
|
+
export const notValidOf = (
|
|
70
|
+
rows: readonly {
|
|
71
|
+
readonly table_name: string;
|
|
72
|
+
readonly constraint_name: string;
|
|
73
|
+
readonly validated?: boolean | undefined;
|
|
74
|
+
}[],
|
|
75
|
+
table: string,
|
|
76
|
+
): { readonly notValidNames?: readonly string[] } => {
|
|
77
|
+
const names = rows
|
|
78
|
+
.filter((row) => row.table_name === table && row.validated === false)
|
|
79
|
+
.map((row) => row.constraint_name)
|
|
80
|
+
.sort();
|
|
81
|
+
return names.length === 0 ? {} : { notValidNames: names };
|
|
82
|
+
};
|
package/src/introspect.ts
CHANGED
|
@@ -5,6 +5,17 @@
|
|
|
5
5
|
|
|
6
6
|
import { nonAppRelations } from './app-relation';
|
|
7
7
|
import { type DbClient, db } from './client';
|
|
8
|
+
import {
|
|
9
|
+
type CheckRow,
|
|
10
|
+
type ColumnRow,
|
|
11
|
+
type ForeignKeyRow,
|
|
12
|
+
type IndexRow,
|
|
13
|
+
notValidOf,
|
|
14
|
+
onDeleteOf,
|
|
15
|
+
type TriggerNameRow,
|
|
16
|
+
} from './introspect-rows';
|
|
17
|
+
import { partitionedTables } from './partitioned-tables';
|
|
18
|
+
import { type PolicyNameRow, policyNameRows } from './row-security-catalog';
|
|
8
19
|
import { sql } from './sql';
|
|
9
20
|
|
|
10
21
|
export interface ColumnDescription {
|
|
@@ -23,6 +34,12 @@ export interface ColumnDescription {
|
|
|
23
34
|
* spellings, which is the same rule `IndexDescription.where` states one field down.
|
|
24
35
|
*/
|
|
25
36
|
readonly generated?: string | undefined;
|
|
37
|
+
/**
|
|
38
|
+
* `true` on a column an entity lists under `retired` and a migration has marked (`retired.ts`):
|
|
39
|
+
* old pods may still read it, so it stays until a later migration drops it. Absent otherwise, and
|
|
40
|
+
* always absent on a catalog read: the database has no such idea.
|
|
41
|
+
*/
|
|
42
|
+
readonly retired?: true | undefined;
|
|
26
43
|
}
|
|
27
44
|
|
|
28
45
|
export interface IndexDescription {
|
|
@@ -109,6 +126,11 @@ export interface TableDescription {
|
|
|
109
126
|
* read one instead of reporting every declared constraint as missing.
|
|
110
127
|
*/
|
|
111
128
|
readonly checkNames?: readonly string[] | undefined;
|
|
129
|
+
/**
|
|
130
|
+
* The constraints the catalog holds `NOT VALID` (`convalidated`): rows that predate one are
|
|
131
|
+
* unchecked. A catalog reading, absent when there are none, and never on a sidecar.
|
|
132
|
+
*/
|
|
133
|
+
readonly notValidNames?: readonly string[] | undefined;
|
|
112
134
|
/**
|
|
113
135
|
* That a migration set `replica identity full` on this table — what `@ultimat3/realtime` requires
|
|
114
136
|
* of every table a live query subscribes to, and the only reason this generator emits it.
|
|
@@ -138,10 +160,57 @@ export interface TableDescription {
|
|
|
138
160
|
* Written by `introspectSchema()` always, `[]` included; absent means nobody asked the catalog.
|
|
139
161
|
*/
|
|
140
162
|
readonly triggerNames?: readonly string[] | undefined;
|
|
163
|
+
/**
|
|
164
|
+
* That the catalog holds this table as a PARTITIONED parent (`relkind = 'p'`). A catalog reading,
|
|
165
|
+
* `true` or absent, never on a sidecar: a partitioned table is declared by a hand edit to the
|
|
166
|
+
* generated `create table`, which no entity records, and drift compares what still holds on the
|
|
167
|
+
* parent — columns, keys, indexes. Absent on every ordinary table, so an unpartitioned app's
|
|
168
|
+
* descriptions are unchanged. Its partitions are Postgres' and appear in no description.
|
|
169
|
+
*/
|
|
170
|
+
readonly partitioned?: true | undefined;
|
|
171
|
+
/**
|
|
172
|
+
* That a migration gave this table the tenant policy, enabled and forced (`row-security-plan.ts`).
|
|
173
|
+
* The SNAPSHOT's half, `true` or absent for `appendOnly`'s reason; the catalog's is `policyNames`.
|
|
174
|
+
*/
|
|
175
|
+
readonly rowSecurity?: true | undefined;
|
|
176
|
+
/**
|
|
177
|
+
* The CATALOG's half of `rowSecurity`: the policies on the table when its row security is enabled
|
|
178
|
+
* and forced, `[]` otherwise (`row-security-catalog.ts`). Absent means nobody asked the catalog.
|
|
179
|
+
*/
|
|
180
|
+
readonly policyNames?: readonly string[] | undefined;
|
|
181
|
+
/**
|
|
182
|
+
* The lifecycle triggers a migration installed on this table — history, its guard, counters
|
|
183
|
+
* (`generate-triggers.ts`). The SNAPSHOT's half, absent when none; the catalog's is `triggerNames`.
|
|
184
|
+
*/
|
|
185
|
+
readonly triggers?: readonly TriggerRecord[] | undefined;
|
|
186
|
+
/**
|
|
187
|
+
* That the entity declares `retain` (`drift-triggers.ts` refuses it on a partitioned parent).
|
|
188
|
+
* SNAPSHOT-only, `true` or absent for `appendOnly`'s reason.
|
|
189
|
+
*/
|
|
190
|
+
readonly retain?: true | undefined;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** One installed trigger: its name, and the statements that install it — the drift repair too. */
|
|
194
|
+
export interface TriggerRecord {
|
|
195
|
+
readonly name: string;
|
|
196
|
+
readonly install: readonly string[];
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* A view `x db gen` wrote (`generate-view.ts`): SNAPSHOT-only, as `checks` is declared-only — the
|
|
201
|
+
* catalog reading never fills it (`app-relation.ts` keeps every view out of `tables`), so drift
|
|
202
|
+
* never reports one. `create` is the statements verbatim, which is what the next diff compares.
|
|
203
|
+
*/
|
|
204
|
+
export interface ViewDescription {
|
|
205
|
+
readonly name: string;
|
|
206
|
+
readonly materialized?: true | undefined;
|
|
207
|
+
readonly create: readonly string[];
|
|
141
208
|
}
|
|
142
209
|
|
|
143
210
|
export interface SchemaDescription {
|
|
144
211
|
readonly tables: readonly TableDescription[];
|
|
212
|
+
/** Absent on every snapshot written before views existed, and on every catalog reading. */
|
|
213
|
+
readonly views?: readonly ViewDescription[] | undefined;
|
|
145
214
|
}
|
|
146
215
|
|
|
147
216
|
export interface IntrospectOptions {
|
|
@@ -157,47 +226,6 @@ export interface IntrospectOptions {
|
|
|
157
226
|
readonly exclude?: readonly string[] | undefined;
|
|
158
227
|
}
|
|
159
228
|
|
|
160
|
-
interface ColumnRow {
|
|
161
|
-
readonly table_name: string;
|
|
162
|
-
readonly column_name: string;
|
|
163
|
-
readonly data_type: string;
|
|
164
|
-
readonly is_nullable: string;
|
|
165
|
-
readonly column_default: string | null;
|
|
166
|
-
readonly ordinal_position: number;
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
interface IndexRow {
|
|
170
|
-
readonly table_name: string;
|
|
171
|
-
readonly index_name: string;
|
|
172
|
-
readonly is_unique: boolean;
|
|
173
|
-
readonly is_primary: boolean;
|
|
174
|
-
readonly columns: readonly string[];
|
|
175
|
-
readonly predicate: string | null;
|
|
176
|
-
readonly descending: boolean;
|
|
177
|
-
readonly method?: string | undefined;
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
interface ForeignKeyRow {
|
|
181
|
-
readonly table_name: string;
|
|
182
|
-
readonly constraint_name: string;
|
|
183
|
-
readonly columns: readonly string[];
|
|
184
|
-
readonly referenced_table: string;
|
|
185
|
-
readonly referenced_columns: readonly string[];
|
|
186
|
-
readonly on_delete: string | null;
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/** A CHECK constraint's NAME. There is deliberately no column for its definition — see `checkNames`. */
|
|
190
|
-
interface CheckRow {
|
|
191
|
-
readonly table_name: string;
|
|
192
|
-
readonly constraint_name: string;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
/** An enabled trigger's NAME — the half of `appendOnly` the catalog can answer. */
|
|
196
|
-
interface TriggerNameRow {
|
|
197
|
-
readonly table_name: string;
|
|
198
|
-
readonly trigger_name: string;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
229
|
const byName = (a: { name: string }, b: { name: string }): number => (a.name < b.name ? -1 : 1);
|
|
202
230
|
|
|
203
231
|
export async function introspectSchema(
|
|
@@ -272,7 +300,7 @@ export async function introspectSchema(
|
|
|
272
300
|
join pg_am am on am.oid = i.relam
|
|
273
301
|
cross join lateral unnest(ix.indkey::smallint[]) with ordinality as k(attnum, ord)
|
|
274
302
|
left join pg_attribute a on a.attrelid = t.oid and a.attnum = k.attnum and k.attnum > 0
|
|
275
|
-
where n.nspname = ${schema} and t.relkind
|
|
303
|
+
where n.nspname = ${schema} and t.relkind in ('r', 'p') and k.ord <= ix.indnkeyatts
|
|
276
304
|
group by
|
|
277
305
|
t.relname, i.relname, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid,
|
|
278
306
|
ix.indexrelid, am.amname
|
|
@@ -284,6 +312,8 @@ export async function introspectSchema(
|
|
|
284
312
|
// two-column key came back as four source columns against four referenced ones, duplicated and
|
|
285
313
|
// misaligned. Ordered by `k.ord` (the constraint's own key position), never by `attnum`, for the
|
|
286
314
|
// same reason `indkey` orders the index query: `references t (y, x)` is not `references t (x, y)`.
|
|
315
|
+
// `conparentid = 0`: a key into a partitioned table makes Postgres add one clone per partition
|
|
316
|
+
// (`notes_event_fkey_1`, `_2`, …), each a fact of the partitioning and never of a migration.
|
|
287
317
|
const foreignKeys = await client.query<ForeignKeyRow>(sql`
|
|
288
318
|
select
|
|
289
319
|
src.relname as table_name,
|
|
@@ -291,7 +321,9 @@ export async function introspectSchema(
|
|
|
291
321
|
array_agg(sa.attname order by k.ord) as columns,
|
|
292
322
|
tgt.relname as referenced_table,
|
|
293
323
|
array_agg(ta.attname order by k.ord) as referenced_columns,
|
|
294
|
-
c.confdeltype as on_delete
|
|
324
|
+
c.confdeltype as on_delete,
|
|
325
|
+
substring(pg_get_constraintdef(c.oid) from 'ON DELETE SET NULL \\(([^)]*)\\)') as set_null_columns,
|
|
326
|
+
c.convalidated as validated
|
|
295
327
|
from pg_constraint c
|
|
296
328
|
join pg_class src on src.oid = c.conrelid
|
|
297
329
|
join pg_class tgt on tgt.oid = c.confrelid
|
|
@@ -299,8 +331,8 @@ export async function introspectSchema(
|
|
|
299
331
|
cross join lateral unnest(c.conkey, c.confkey) with ordinality as k(src_attnum, tgt_attnum, ord)
|
|
300
332
|
join pg_attribute sa on sa.attrelid = src.oid and sa.attnum = k.src_attnum
|
|
301
333
|
join pg_attribute ta on ta.attrelid = tgt.oid and ta.attnum = k.tgt_attnum
|
|
302
|
-
where c.contype = 'f' and n.nspname = ${schema}
|
|
303
|
-
group by src.relname, c.conname, tgt.relname, c.confdeltype
|
|
334
|
+
where c.contype = 'f' and n.nspname = ${schema} and c.conparentid = 0
|
|
335
|
+
group by c.oid, src.relname, c.conname, tgt.relname, c.confdeltype, c.convalidated
|
|
304
336
|
order by src.relname, c.conname
|
|
305
337
|
`);
|
|
306
338
|
|
|
@@ -310,11 +342,11 @@ export async function introspectSchema(
|
|
|
310
342
|
// a correct database as wrong. `contype = 'c'` is the CHECK constraints alone — Postgres 17
|
|
311
343
|
// onwards records a NOT NULL as `'n'`, and a domain's as `'c'` on the domain rather than here.
|
|
312
344
|
const checks = await client.query<CheckRow>(sql`
|
|
313
|
-
select src.relname as table_name, c.conname as constraint_name
|
|
345
|
+
select src.relname as table_name, c.conname as constraint_name, c.convalidated as validated
|
|
314
346
|
from pg_constraint c
|
|
315
347
|
join pg_class src on src.oid = c.conrelid
|
|
316
348
|
join pg_namespace n on n.oid = src.relnamespace
|
|
317
|
-
where c.contype = 'c' and n.nspname = ${schema} and src.relkind
|
|
349
|
+
where c.contype = 'c' and n.nspname = ${schema} and src.relkind in ('r', 'p')
|
|
318
350
|
order by src.relname, c.conname
|
|
319
351
|
`);
|
|
320
352
|
|
|
@@ -326,11 +358,24 @@ export async function introspectSchema(
|
|
|
326
358
|
join pg_class src on src.oid = t.tgrelid
|
|
327
359
|
join pg_namespace n on n.oid = src.relnamespace
|
|
328
360
|
where not t.tgisinternal and t.tgenabled in ('O', 'A')
|
|
329
|
-
and n.nspname = ${schema} and src.relkind
|
|
361
|
+
and n.nspname = ${schema} and src.relkind in ('r', 'p')
|
|
330
362
|
order by src.relname, t.tgname
|
|
331
363
|
`);
|
|
332
364
|
|
|
333
|
-
|
|
365
|
+
const partitioned = await partitionedTables(client, schema);
|
|
366
|
+
const policies = await policyNameRows(client, schema);
|
|
367
|
+
|
|
368
|
+
return buildSchema(
|
|
369
|
+
schema,
|
|
370
|
+
excluded,
|
|
371
|
+
columns,
|
|
372
|
+
indexes,
|
|
373
|
+
foreignKeys,
|
|
374
|
+
checks,
|
|
375
|
+
triggers,
|
|
376
|
+
partitioned,
|
|
377
|
+
policies,
|
|
378
|
+
);
|
|
334
379
|
}
|
|
335
380
|
|
|
336
381
|
/** Pure, so the row -> description mapping is testable without a database. */
|
|
@@ -343,6 +388,9 @@ export function buildSchema(
|
|
|
343
388
|
checks: readonly CheckRow[] = [],
|
|
344
389
|
/** Absent leaves `triggerNames` off every table: nobody asked the catalog. */
|
|
345
390
|
triggers?: readonly TriggerNameRow[],
|
|
391
|
+
partitioned: readonly string[] = [],
|
|
392
|
+
/** Absent leaves `policyNames` off every table, as `triggers` does `triggerNames`. */
|
|
393
|
+
policies?: readonly PolicyNameRow[],
|
|
346
394
|
): SchemaDescription {
|
|
347
395
|
const names = [...new Set(columns.map((row) => row.table_name))]
|
|
348
396
|
.filter((name) => !excluded.includes(name))
|
|
@@ -385,7 +433,7 @@ export function buildSchema(
|
|
|
385
433
|
columns: [...row.columns],
|
|
386
434
|
referencedTable: row.referenced_table,
|
|
387
435
|
referencedColumns: [...row.referenced_columns],
|
|
388
|
-
onDelete: row
|
|
436
|
+
onDelete: onDeleteOf(row),
|
|
389
437
|
}))
|
|
390
438
|
.sort(byName),
|
|
391
439
|
// Always written, `[]` included: this reading of a table HAS asked the catalog, and absence
|
|
@@ -394,6 +442,7 @@ export function buildSchema(
|
|
|
394
442
|
.filter((row) => row.table_name === name)
|
|
395
443
|
.map((row) => row.constraint_name)
|
|
396
444
|
.sort(),
|
|
445
|
+
...notValidOf([...foreignKeys, ...checks], name),
|
|
397
446
|
...(triggers === undefined
|
|
398
447
|
? {}
|
|
399
448
|
: {
|
|
@@ -402,6 +451,15 @@ export function buildSchema(
|
|
|
402
451
|
.map((row) => row.trigger_name)
|
|
403
452
|
.sort(),
|
|
404
453
|
}),
|
|
454
|
+
...(partitioned.includes(name) ? { partitioned: true as const } : {}),
|
|
455
|
+
...(policies === undefined
|
|
456
|
+
? {}
|
|
457
|
+
: {
|
|
458
|
+
policyNames: policies
|
|
459
|
+
.filter((row) => row.table_name === name)
|
|
460
|
+
.map((row) => row.policy_name)
|
|
461
|
+
.sort(),
|
|
462
|
+
}),
|
|
405
463
|
};
|
|
406
464
|
});
|
|
407
465
|
|
package/src/keyset.ts
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// Single responsibility: keyset paging for a raw read — a join the typed handle cannot express.
|
|
2
|
+
// The handle's `.after()` keeps its own seek; this is the same position, carried as the column's
|
|
3
|
+
// TEXT and bound back cast by kind, so a microsecond the JS `Date` would drop is never dropped.
|
|
4
|
+
|
|
5
|
+
import type { Page } from '@ultimat3/core';
|
|
6
|
+
import {
|
|
7
|
+
assert,
|
|
8
|
+
CursorInvalidError,
|
|
9
|
+
decodeCursor,
|
|
10
|
+
encodeCursor,
|
|
11
|
+
finiteCount,
|
|
12
|
+
pageOf,
|
|
13
|
+
} from '@ultimat3/core';
|
|
14
|
+
import { DbError } from './errors';
|
|
15
|
+
import { join, raw, type SqlFragment, sql } from './sql';
|
|
16
|
+
|
|
17
|
+
/** The kinds a position can be bound back as, each with its own cast. A closed list. */
|
|
18
|
+
export type KeysetKind = 'timestamptz' | 'uuid' | 'text' | 'integer' | 'bigint';
|
|
19
|
+
|
|
20
|
+
export interface KeysetKey {
|
|
21
|
+
/** The ordered expression as written in the statement: `raw('p.created_at')`, or a `sql` one. */
|
|
22
|
+
readonly column: SqlFragment;
|
|
23
|
+
readonly kind: KeysetKind;
|
|
24
|
+
readonly direction: 'asc' | 'desc';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface KeysetOptions {
|
|
28
|
+
/** This read and its arguments: a cursor from another scope is `X_CURSOR_INVALID`. */
|
|
29
|
+
readonly scope: string;
|
|
30
|
+
/** The order, last key the tiebreak. Every key must be not null on every row. */
|
|
31
|
+
readonly keys: readonly KeysetKey[];
|
|
32
|
+
/** The previous page's `nextCursor`; `null` for the first page. */
|
|
33
|
+
readonly after: string | null;
|
|
34
|
+
readonly limit: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A row as `page` answers it: the `_keyset_<n>` aliases `select` added are gone. */
|
|
38
|
+
export type KeysetRow<Row> = Omit<Row, `_keyset_${number}`>;
|
|
39
|
+
|
|
40
|
+
/** Four fragments to splice — `select …, ${k.select}`, `where ${k.where}`, `order by`, `limit`. */
|
|
41
|
+
export interface Keyset {
|
|
42
|
+
readonly select: SqlFragment;
|
|
43
|
+
readonly where: SqlFragment;
|
|
44
|
+
readonly orderBy: SqlFragment;
|
|
45
|
+
/** `limit + 1`: the extra row is how `page` knows there is a next page. */
|
|
46
|
+
readonly limit: SqlFragment;
|
|
47
|
+
/** The rows the statement answered, cut to `limit`, the key aliases stripped. */
|
|
48
|
+
page<Row extends object>(rows: readonly Row[]): Page<KeysetRow<Row>>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const CASTS: ReadonlyMap<KeysetKind, string> = new Map<KeysetKind, string>([
|
|
52
|
+
['timestamptz', 'timestamptz'],
|
|
53
|
+
['uuid', 'uuid'],
|
|
54
|
+
['text', 'text'],
|
|
55
|
+
['integer', 'integer'],
|
|
56
|
+
['bigint', 'bigint'],
|
|
57
|
+
]);
|
|
58
|
+
|
|
59
|
+
const NULL_KEY_FIX =
|
|
60
|
+
"keyset({ keys: [{ column: sql`coalesce(p.published_at, p.created_at)`, kind: 'timestamptz', direction: 'desc' }, …] }) # or order by a not-null column";
|
|
61
|
+
|
|
62
|
+
const aliasOf = (index: number): string => `_keyset_${String(index)}`;
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The scope the cursor is signed under: the caller's, plus the order itself — so a cursor minted
|
|
66
|
+
* under `desc` cannot seek an `asc` read, nor a `uuid` key be bound back as `text`. Hashed: the
|
|
67
|
+
* key expressions are the statement's, not the client's to read.
|
|
68
|
+
*/
|
|
69
|
+
const scopeOf = (scope: string, keys: readonly KeysetKey[]): string => {
|
|
70
|
+
const order = keys
|
|
71
|
+
.map(
|
|
72
|
+
(key) =>
|
|
73
|
+
`${key.column.text}${JSON.stringify(key.column.values.map(String))} ${key.kind} ${key.direction}`,
|
|
74
|
+
)
|
|
75
|
+
.join(',');
|
|
76
|
+
const digest = new Bun.CryptoHasher('sha256').update(`${scope}|${order}`).digest('hex');
|
|
77
|
+
return `keyset:${digest.slice(0, 16)}`;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
const bind = (key: KeysetKey, text: string): SqlFragment =>
|
|
81
|
+
sql`${text}::${raw(CASTS.get(key.kind) ?? 'text')}`;
|
|
82
|
+
|
|
83
|
+
const past = (key: KeysetKey, value: SqlFragment): SqlFragment =>
|
|
84
|
+
sql`${key.column} ${raw(key.direction === 'desc' ? '<' : '>')} ${value}`;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Uniform direction is ONE row comparison, the shape Postgres pushes into a multicolumn index as
|
|
88
|
+
* an Index Cond. A mixed order has none, so it is the or-chain — the same split as the entity
|
|
89
|
+
* driver's `seekSql`. No null branch in either: `page` refuses a null key before it is a cursor.
|
|
90
|
+
*/
|
|
91
|
+
const seek = (keys: readonly KeysetKey[], position: readonly string[]): SqlFragment => {
|
|
92
|
+
const binds = (): SqlFragment[] => keys.map((key, index) => bind(key, position[index] ?? ''));
|
|
93
|
+
const [first] = keys;
|
|
94
|
+
if (first === undefined) return sql`true`;
|
|
95
|
+
if (keys.length === 1) return past(first, bind(first, position[0] ?? ''));
|
|
96
|
+
if (keys.every((key) => key.direction === first.direction)) {
|
|
97
|
+
const op = raw(first.direction === 'desc' ? '<' : '>');
|
|
98
|
+
return sql`(${join(keys.map((key) => key.column))}) ${op} (${join(binds())})`;
|
|
99
|
+
}
|
|
100
|
+
const terms = keys.map((key, index) => {
|
|
101
|
+
const equal = keys
|
|
102
|
+
.slice(0, index)
|
|
103
|
+
.map((prior, at) => sql`${prior.column} = ${bind(prior, position[at] ?? '')}`);
|
|
104
|
+
const term = past(key, bind(key, position[index] ?? ''));
|
|
105
|
+
return equal.length === 0 ? term : sql`(${join([...equal, term], ' and ')})`;
|
|
106
|
+
});
|
|
107
|
+
return sql`(${join(terms, ' or ')})`;
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
const positionOf = (cursor: string, scope: string, keys: readonly KeysetKey[]): string[] => {
|
|
111
|
+
const { key } = decodeCursor(cursor, scope);
|
|
112
|
+
if (key.length !== keys.length || !key.every((part) => typeof part === 'string')) {
|
|
113
|
+
throw new CursorInvalidError(
|
|
114
|
+
`it carries ${key.length} key values, this order needs ${keys.length}`,
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
return key as string[];
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
/** Every key of `row`, as the text the server answered — refused when one is null. */
|
|
121
|
+
const keyText = (row: object, keys: readonly KeysetKey[], index: number): string[] =>
|
|
122
|
+
keys.map((key, at) => {
|
|
123
|
+
const value: unknown = Reflect.get(row, aliasOf(at));
|
|
124
|
+
if (typeof value === 'string') return value;
|
|
125
|
+
if (value === null) {
|
|
126
|
+
throw new DbError({
|
|
127
|
+
code: 'X_KEYSET_NULL_KEY',
|
|
128
|
+
cause: `row ${String(index)} holds null in keyset key ${String(at)} (${key.column.text}): a row comparison has no null order, so that row and every one like it would fall off the next page`,
|
|
129
|
+
fix: NULL_KEY_FIX,
|
|
130
|
+
meta: { key: at, row: index },
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
throw new DbError({
|
|
134
|
+
code: 'X_INVARIANT',
|
|
135
|
+
cause: `row ${String(index)} carries no ${aliasOf(at)}: the statement did not select keyset()'s keys`,
|
|
136
|
+
fix: "db().query(sql`select p.*, <k.select> from … where <k.where> order by <k.orderBy> limit <k.limit>`) # each <…> an interpolation of keyset()'s answer",
|
|
137
|
+
meta: { key: at, row: index },
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
const stripped = <Row extends object>(row: Row, aliases: ReadonlySet<string>): KeysetRow<Row> =>
|
|
142
|
+
Object.fromEntries(Object.entries(row).filter(([name]) => !aliases.has(name))) as KeysetRow<Row>;
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* A keyset page over any statement. The key is selected as `col::text` and bound back as
|
|
146
|
+
* `$n::<kind>`: exact, no arithmetic, and the `order by` stays on the bare column, so an index on
|
|
147
|
+
* `(created_at desc, id desc)` answers it as an Index Only Scan.
|
|
148
|
+
*/
|
|
149
|
+
export function keyset(options: KeysetOptions): Keyset {
|
|
150
|
+
const { keys } = options;
|
|
151
|
+
assert(
|
|
152
|
+
keys.length > 0,
|
|
153
|
+
'keyset() was given no keys, so it has no order to page by',
|
|
154
|
+
"keyset({ keys: [{ column: raw('p.created_at'), kind: 'timestamptz', direction: 'desc' }, { column: raw('p.id'), kind: 'uuid', direction: 'desc' }], … })",
|
|
155
|
+
);
|
|
156
|
+
assert(
|
|
157
|
+
options.scope.length > 0,
|
|
158
|
+
'keyset() was given an empty scope, so a cursor from any other raw read would page this one',
|
|
159
|
+
"keyset({ scope: 'posts.feed', … }) # one name per read, plus its filter arguments",
|
|
160
|
+
);
|
|
161
|
+
const limit = finiteCount('keyset', 'limit', options.limit, 1);
|
|
162
|
+
const scope = scopeOf(options.scope, keys);
|
|
163
|
+
const where =
|
|
164
|
+
options.after === null ? sql`true` : seek(keys, positionOf(options.after, scope, keys));
|
|
165
|
+
return {
|
|
166
|
+
select: join(keys.map((key, at) => sql`${key.column}::text as ${raw(`"${aliasOf(at)}"`)}`)),
|
|
167
|
+
where,
|
|
168
|
+
orderBy: join(keys.map((key) => sql`${key.column} ${raw(key.direction)}`)),
|
|
169
|
+
limit: sql`${limit + 1}`,
|
|
170
|
+
page<Row extends object>(rows: readonly Row[]): Page<KeysetRow<Row>> {
|
|
171
|
+
const positions = rows.map((row, index) => keyText(row, keys, index));
|
|
172
|
+
const aliases = new Set(keys.map((_key, at) => aliasOf(at)));
|
|
173
|
+
const kept = rows.slice(0, limit).map((row) => stripped(row, aliases));
|
|
174
|
+
const last = positions[limit - 1];
|
|
175
|
+
const next =
|
|
176
|
+
rows.length > limit && last !== undefined
|
|
177
|
+
? encodeCursor({ scope, key: last, id: '' })
|
|
178
|
+
: null;
|
|
179
|
+
return pageOf(kept, next);
|
|
180
|
+
},
|
|
181
|
+
};
|
|
182
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// Single responsibility: the refusal `entity({ retain })` raises at declaration. Here because the
|
|
2
|
+
// code is this package's — the drift step raises it too, for a partitioned table — and
|
|
3
|
+
// `@ultimat3/entity` (tier 2) reaches down for it rather than registering the same code twice.
|
|
4
|
+
|
|
5
|
+
import { renderFixShellArg } from '@ultimat3/core';
|
|
6
|
+
import { DbError } from './errors';
|
|
7
|
+
|
|
8
|
+
/** `retain` with no window, beside `appendOnly`, or otherwise unsweepable. */
|
|
9
|
+
export const retainInvalid = (entity: string, reason: string): DbError =>
|
|
10
|
+
new DbError({
|
|
11
|
+
code: 'X_ENTITY_RETAIN_INVALID',
|
|
12
|
+
cause: `entity "${entity}" declares retain, and ${reason}`,
|
|
13
|
+
fix: `x db gen "retention" # after entity('${renderFixShellArg(entity, '<entity>')}', { retain: { after: '90d', on: 'createdAt' } }) names a window and appendOnly is gone`,
|
|
14
|
+
meta: { entity },
|
|
15
|
+
});
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Single responsibility: the framework's one writer of a transaction-local setting. Every
|
|
2
|
+
// `set_config` the framework sends is this file's: parameter-bound and always local, so a value
|
|
3
|
+
// ends with the transaction that set it and a pooled connection hands the next caller nothing.
|
|
4
|
+
|
|
5
|
+
import type { DbClient } from './client';
|
|
6
|
+
import { sql } from './sql';
|
|
7
|
+
|
|
8
|
+
/** A setting's name and its value, both text — what `set_config` takes. */
|
|
9
|
+
export type LocalSettings = Readonly<Record<string, string>>;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* `select set_config($1, $2, true)` per setting, on `tx`. `true` is "local": the value lives until
|
|
13
|
+
* the transaction ends, which is the only lifetime safe behind a transaction-mode pooler. Never
|
|
14
|
+
* `set`/`set local` text: those take no parameters, so a value would have to be spliced in.
|
|
15
|
+
*/
|
|
16
|
+
export async function setLocal(tx: DbClient, settings: LocalSettings): Promise<void> {
|
|
17
|
+
for (const [name, value] of Object.entries(settings)) {
|
|
18
|
+
await tx.execute(sql`select set_config(${name}, ${value}, true)`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface SettingRow {
|
|
23
|
+
readonly value: string | null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* What each named setting reads as now, on `tx`, for a caller that must put it back. An unset custom
|
|
28
|
+
* setting (`x.tenant` before anything set it) reads `''`, which `set_config` restores to the same
|
|
29
|
+
* "unset" every policy reads through `nullif(…, '')`.
|
|
30
|
+
*/
|
|
31
|
+
export async function readLocal(tx: DbClient, names: readonly string[]): Promise<LocalSettings> {
|
|
32
|
+
const prior = new Map<string, string>();
|
|
33
|
+
for (const name of names) {
|
|
34
|
+
const row = await tx.one<SettingRow>(sql`select current_setting(${name}, true) as value`);
|
|
35
|
+
prior.set(name, row?.value ?? '');
|
|
36
|
+
}
|
|
37
|
+
return Object.fromEntries(prior);
|
|
38
|
+
}
|
package/src/marker-line.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Single responsibility: whether a migration DECLARES something with a marker line — a top-level
|
|
2
|
-
// `--` comment of its own, and nothing that merely looks like one.
|
|
3
|
-
// (`-- destructive: true`, `-- transaction: false`), so the scan is written once.
|
|
2
|
+
// `--` comment of its own, and nothing that merely looks like one. Several markers ask this question
|
|
3
|
+
// (`-- destructive: true`, `-- transaction: false`, `rolling.ts`'s), so the scan is written once.
|
|
4
4
|
|
|
5
5
|
import { noiseAt } from './sql-scan';
|
|
6
6
|
|
|
@@ -24,6 +24,12 @@ function startsLine(sql: string, index: number): boolean {
|
|
|
24
24
|
* substring.
|
|
25
25
|
*/
|
|
26
26
|
export function hasMarkerLine(sql: string, marker: RegExp): boolean {
|
|
27
|
+
return markerLines(sql, marker).length > 0;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Every such line's text, in source order: a marker that names something is read off these. */
|
|
31
|
+
export function markerLines(sql: string, marker: RegExp): readonly string[] {
|
|
32
|
+
const found: string[] = [];
|
|
27
33
|
let index = 0;
|
|
28
34
|
while (index < sql.length) {
|
|
29
35
|
const noise = noiseAt(sql, index);
|
|
@@ -32,8 +38,10 @@ export function hasMarkerLine(sql: string, marker: RegExp): boolean {
|
|
|
32
38
|
continue;
|
|
33
39
|
}
|
|
34
40
|
const text = sql.slice(index, noise.end);
|
|
35
|
-
if (noise.kind === 'line-comment' && startsLine(sql, index) && marker.test(text))
|
|
41
|
+
if (noise.kind === 'line-comment' && startsLine(sql, index) && marker.test(text)) {
|
|
42
|
+
found.push(text);
|
|
43
|
+
}
|
|
36
44
|
index = noise.end;
|
|
37
45
|
}
|
|
38
|
-
return
|
|
46
|
+
return found;
|
|
39
47
|
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Single responsibility: send one unit of a migration again when `lock_timeout` fires. A deploy that
|
|
2
|
+
// dies because one reader held a table for 5 s is a wedged rollout for a thing that heals by itself.
|
|
3
|
+
// `migration-step.ts` picks the unit: a transactional step whole, since it rolled back whole, and a
|
|
4
|
+
// non-transactional step one statement at a time, since what already ran stays run.
|
|
5
|
+
|
|
6
|
+
import { isUltimateError, jitterStatedDelay, logger, type Random } from '@ultimat3/core';
|
|
7
|
+
import { DbError } from './errors';
|
|
8
|
+
|
|
9
|
+
/** Retries after the first attempt, so six tries: at 3 s each, near half a minute of patience. */
|
|
10
|
+
export const MIGRATION_LOCK_RETRIES = 5;
|
|
11
|
+
|
|
12
|
+
export interface LockRetryPolicy {
|
|
13
|
+
readonly retries: number;
|
|
14
|
+
readonly lockTimeoutMs: number;
|
|
15
|
+
/** Seams; a test asserts the schedule rather than sleeping through it. */
|
|
16
|
+
readonly sleep?: ((ms: number) => Promise<void>) | undefined;
|
|
17
|
+
readonly random?: Random | undefined;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* At least one `lockTimeoutMs` between attempts. While the ALTER queued, every query on that table
|
|
22
|
+
* queued behind it, and going straight back in would stack the same queue on a table that has had
|
|
23
|
+
* no time to serve it. Jitter upward only, so that floor is a floor.
|
|
24
|
+
*/
|
|
25
|
+
export async function withLockRetry(
|
|
26
|
+
id: string,
|
|
27
|
+
policy: LockRetryPolicy,
|
|
28
|
+
attempt: () => Promise<void>,
|
|
29
|
+
): Promise<void> {
|
|
30
|
+
const sleep = policy.sleep ?? Bun.sleep;
|
|
31
|
+
const random = policy.random ?? Math.random;
|
|
32
|
+
for (let done = 1; ; done += 1) {
|
|
33
|
+
try {
|
|
34
|
+
await attempt();
|
|
35
|
+
return;
|
|
36
|
+
} catch (error) {
|
|
37
|
+
if (!isUltimateError(error) || error.code !== 'X_DB_LOCK_TIMEOUT') throw error;
|
|
38
|
+
const { cause, fix } = error;
|
|
39
|
+
if (done > policy.retries) {
|
|
40
|
+
throw new DbError({
|
|
41
|
+
code: 'X_DB_LOCK_TIMEOUT',
|
|
42
|
+
cause: `migration "${id}" gave up after ${done} attempts, ${policy.lockTimeoutMs} ms of lock wait each: ${cause}`,
|
|
43
|
+
fix,
|
|
44
|
+
meta: { id, attempts: done },
|
|
45
|
+
sourceError: error,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
const waitMs = jitterStatedDelay(policy.lockTimeoutMs, policy.lockTimeoutMs, random);
|
|
49
|
+
logger.warn('ultimate migrate lock wait timed out, trying again', {
|
|
50
|
+
id,
|
|
51
|
+
attempt: done,
|
|
52
|
+
of: policy.retries + 1,
|
|
53
|
+
waitMs,
|
|
54
|
+
cause,
|
|
55
|
+
fix,
|
|
56
|
+
});
|
|
57
|
+
await sleep(waitMs);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|