@ultimat3/db 29.3.2 → 30.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CLAUDE.md +78 -116
  2. package/README.md +199 -16
  3. package/package.json +13 -2
  4. package/src/app-relation.ts +7 -2
  5. package/src/catalog-fold.ts +5 -0
  6. package/src/catalog-objects.ts +20 -24
  7. package/src/catalog-policies.ts +81 -0
  8. package/src/catalog-relations.ts +31 -10
  9. package/src/catalog.ts +40 -1
  10. package/src/check-ddl.ts +34 -7
  11. package/src/client.ts +132 -48
  12. package/src/column-alter.ts +42 -1
  13. package/src/concurrent-index.ts +157 -0
  14. package/src/concurrent-plan.ts +294 -0
  15. package/src/connection-url.ts +16 -4
  16. package/src/db-diagnostics-deadline.ts +43 -0
  17. package/src/db-diagnostics-redact.ts +58 -0
  18. package/src/db-diagnostics-rows.ts +166 -0
  19. package/src/db-diagnostics-sql.ts +92 -0
  20. package/src/db-diagnostics.ts +178 -0
  21. package/src/destructive.ts +21 -35
  22. package/src/diagnostics-entry.ts +15 -0
  23. package/src/drift-errors.ts +9 -4
  24. package/src/drift-findings.ts +47 -62
  25. package/src/drift-missing.ts +203 -0
  26. package/src/drift-row-security.ts +88 -0
  27. package/src/drift-triggers.ts +54 -0
  28. package/src/drift.ts +88 -21
  29. package/src/entity-shape.ts +75 -0
  30. package/src/errors.ts +80 -9
  31. package/src/expected-loop.ts +34 -4
  32. package/src/failure-store-pg.ts +157 -0
  33. package/src/foreign-key-plan.ts +75 -23
  34. package/src/foreign-key.ts +76 -5
  35. package/src/generate-counter.ts +85 -0
  36. package/src/generate-history.ts +142 -0
  37. package/src/generate-row-security.ts +51 -0
  38. package/src/generate-snapshot.ts +82 -0
  39. package/src/generate-steps.ts +99 -0
  40. package/src/generate-triggers.ts +119 -0
  41. package/src/generate-view.ts +97 -0
  42. package/src/generate.ts +128 -89
  43. package/src/index-ddl.ts +23 -35
  44. package/src/index-plan.ts +35 -14
  45. package/src/index.ts +136 -6
  46. package/src/introspect-catalog.ts +26 -6
  47. package/src/introspect-rows.ts +82 -0
  48. package/src/introspect.ts +108 -50
  49. package/src/keyset.ts +182 -0
  50. package/src/lifecycle-errors.ts +15 -0
  51. package/src/local-settings.ts +38 -0
  52. package/src/marker-line.ts +47 -0
  53. package/src/migrate-lock-retry.ts +60 -0
  54. package/src/migrate-rollback.ts +45 -6
  55. package/src/migrate.ts +67 -69
  56. package/src/migration-errors.ts +48 -1
  57. package/src/migration-ledger.ts +38 -4
  58. package/src/migration-step.ts +192 -0
  59. package/src/partition-order.ts +56 -0
  60. package/src/partitioned-tables.ts +24 -0
  61. package/src/pool-gate.ts +120 -0
  62. package/src/pool-listeners.ts +143 -0
  63. package/src/pool-pins.ts +68 -0
  64. package/src/pool-profile.ts +23 -7
  65. package/src/pool-reserve.ts +11 -4
  66. package/src/pool-retire.ts +22 -0
  67. package/src/pool-sweep.ts +115 -0
  68. package/src/pool-timer.ts +10 -0
  69. package/src/pooler-fix.ts +53 -0
  70. package/src/pooler.ts +161 -0
  71. package/src/read-transaction.ts +59 -0
  72. package/src/readonly-query.ts +4 -4
  73. package/src/replica-client.ts +54 -22
  74. package/src/replica-position.ts +89 -0
  75. package/src/replica-scope.ts +25 -4
  76. package/src/retired.ts +131 -0
  77. package/src/retype-dependents.ts +2 -0
  78. package/src/rolling-errors.ts +97 -0
  79. package/src/rolling.ts +212 -0
  80. package/src/row-secure-client.ts +54 -0
  81. package/src/row-security-catalog.ts +26 -0
  82. package/src/row-security-migrator.ts +48 -0
  83. package/src/row-security-plan.ts +58 -0
  84. package/src/row-security-role.ts +60 -0
  85. package/src/row-security-roles.ts +64 -0
  86. package/src/row-security.ts +59 -0
  87. package/src/same-tenant.ts +173 -0
  88. package/src/schema-dump-policies.ts +26 -0
  89. package/src/schema-dump-table.ts +21 -6
  90. package/src/schema-dump.ts +13 -2
  91. package/src/snapshot-parse.ts +42 -2
  92. package/src/sqlstate.ts +4 -0
  93. package/src/statement-bound.ts +82 -0
  94. package/src/transaction-errors.ts +14 -0
  95. package/src/transaction-guard.ts +40 -0
  96. package/src/transaction-settings.ts +24 -0
  97. package/src/transaction.ts +74 -19
  98. package/src/ungeneratable.ts +30 -0
@@ -0,0 +1,81 @@
1
+ // Single responsibility: row security as the catalog holds it — which tables enable or force it,
2
+ // and every policy on them, clause by clause — read and folded for the schema dump. A dump without
3
+ // them loads a test database that refuses nothing, so a policy would be untested and prod-only.
4
+
5
+ import type { CatalogPolicy, CatalogRowSecurity } from './catalog';
6
+ import { by } from './catalog-fold';
7
+ import { notExtensionOwned } from './catalog-relations';
8
+ import type { DbClient } from './client';
9
+ import { sql } from './sql';
10
+
11
+ export interface RowSecurityRow {
12
+ readonly table_name: string;
13
+ readonly enabled: boolean;
14
+ readonly forced: boolean;
15
+ }
16
+
17
+ export interface PolicyRow {
18
+ readonly table_name: string;
19
+ readonly name: string;
20
+ readonly permissive: boolean;
21
+ readonly command: string;
22
+ readonly roles: readonly string[];
23
+ readonly using: string | null;
24
+ readonly with_check: string | null;
25
+ }
26
+
27
+ export const rowSecurityRows = (
28
+ client: DbClient,
29
+ schema: string,
30
+ ): Promise<readonly RowSecurityRow[]> =>
31
+ client.query<RowSecurityRow>(sql`
32
+ select c.relname as table_name, c.relrowsecurity as enabled, c.relforcerowsecurity as forced
33
+ from pg_class c
34
+ join pg_namespace n on n.oid = c.relnamespace
35
+ where n.nspname = ${schema} and (c.relrowsecurity or c.relforcerowsecurity)
36
+ and ${notExtensionOwned('pg_class', 'c.oid')}
37
+ `);
38
+
39
+ /** `polroles` holds `0` for `public`, which no `pg_roles` row answers; each name is resolved here. */
40
+ export const policyRows = (client: DbClient, schema: string): Promise<readonly PolicyRow[]> =>
41
+ client.query<PolicyRow>(sql`
42
+ select
43
+ c.relname as table_name,
44
+ p.polname as name,
45
+ p.polpermissive as permissive,
46
+ case p.polcmd when 'r' then 'select' when 'a' then 'insert' when 'w' then 'update'
47
+ when 'd' then 'delete' else 'all' end as command,
48
+ array(
49
+ select case when r.oid = 0 then 'public' else (select rolname from pg_roles where oid = r.oid) end
50
+ from unnest(p.polroles) as r(oid)
51
+ ) as roles,
52
+ pg_get_expr(p.polqual, p.polrelid) as using,
53
+ pg_get_expr(p.polwithcheck, p.polrelid) as with_check
54
+ from pg_policy p
55
+ join pg_class c on c.oid = p.polrelid
56
+ join pg_namespace n on n.oid = c.relnamespace
57
+ where n.nspname = ${schema} and ${notExtensionOwned('pg_class', 'c.oid')}
58
+ `);
59
+
60
+ export const rowSecurityOf = (rows: readonly RowSecurityRow[]): readonly CatalogRowSecurity[] =>
61
+ rows
62
+ .map((row) => ({ table: row.table_name, enabled: row.enabled, forced: row.forced }))
63
+ .sort(by((entry) => entry.table));
64
+
65
+ export const policiesOf = (rows: readonly PolicyRow[]): readonly CatalogPolicy[] =>
66
+ rows
67
+ .map((row) => ({
68
+ table: row.table_name,
69
+ name: row.name,
70
+ permissive: row.permissive,
71
+ command: row.command,
72
+ roles: [...row.roles].sort(by((role) => role)),
73
+ using: row.using,
74
+ withCheck: row.with_check,
75
+ }))
76
+ .sort(
77
+ by(
78
+ (policy) => policy.table,
79
+ (policy) => policy.name,
80
+ ),
81
+ );
@@ -26,6 +26,11 @@ export interface TableRow {
26
26
  readonly replica_identity: string;
27
27
  readonly replica_index: string | null;
28
28
  readonly options: string | null;
29
+ /** `pg_get_partkeydef` — `RANGE (at)` — on a partitioned table; `null` on any other. */
30
+ readonly partition_key: string | null;
31
+ /** The table this one is a partition of, and its bound (`FOR VALUES …`, `DEFAULT`); else `null`. */
32
+ readonly partition_of: string | null;
33
+ readonly partition_bound: string | null;
29
34
  }
30
35
 
31
36
  export interface ColumnRow {
@@ -69,9 +74,10 @@ export interface IndexRow {
69
74
  }
70
75
 
71
76
  /**
72
- * Plain tables only. A partition, an inheritance child and a partitioned parent are `relkind`
73
- * `r`/`p` rows a `create table (…)` cannot rebuild, so they are named by `unrenderedRows`
74
- * (`catalog-objects.ts`) instead of rendered wrong here.
77
+ * Plain tables, partitioned tables and their partitions. An inheritance child that is not a
78
+ * partition is a row a `create table (…)` cannot rebuild, so it is named by `unrenderedRows`
79
+ * (`catalog-objects.ts`) instead of rendered wrong here. A partition's bound is the server's own
80
+ * `pg_get_expr` text: a `timestamptz` bound is spelled in the session's `TimeZone`.
75
81
  */
76
82
  export const tableRows = (client: DbClient, schema: string): Promise<readonly TableRow[]> =>
77
83
  client.query<TableRow>(sql`
@@ -85,13 +91,20 @@ export const tableRows = (client: DbClient, schema: string): Promise<readonly Ta
85
91
  join pg_class i on i.oid = x.indexrelid
86
92
  where x.indrelid = c.oid and x.indisreplident
87
93
  ) as replica_index,
88
- array_to_string(c.reloptions, ', ') as options
94
+ array_to_string(c.reloptions, ', ') as options,
95
+ pg_get_partkeydef(c.oid) as partition_key,
96
+ (
97
+ select p.relname
98
+ from pg_inherits h
99
+ join pg_class p on p.oid = h.inhparent
100
+ where h.inhrelid = c.oid and c.relispartition
101
+ ) as partition_of,
102
+ case when c.relispartition then pg_get_expr(c.relpartbound, c.oid) end as partition_bound
89
103
  from pg_class c
90
104
  join pg_namespace n on n.oid = c.relnamespace
91
105
  where n.nspname = ${schema}
92
- and c.relkind = 'r'
93
- and not c.relispartition
94
- and not exists (select 1 from pg_inherits h where h.inhrelid = c.oid)
106
+ and c.relkind in ('r', 'p')
107
+ and (c.relispartition or not exists (select 1 from pg_inherits h where h.inhrelid = c.oid))
95
108
  and ${notExtensionOwned('pg_class', 'c.oid')}
96
109
  `);
97
110
 
@@ -114,13 +127,15 @@ export const columnRows = (client: DbClient, schema: string): Promise<readonly C
114
127
  join pg_type t on t.oid = a.atttypid
115
128
  left join pg_attrdef d on d.adrelid = a.attrelid and d.adnum = a.attnum
116
129
  left join pg_collation co on co.oid = a.attcollation
117
- where n.nspname = ${schema} and c.relkind = 'r' and a.attnum > 0 and not a.attisdropped
130
+ where n.nspname = ${schema} and c.relkind in ('r', 'p') and a.attnum > 0 and not a.attisdropped
118
131
  `);
119
132
 
120
133
  /**
121
134
  * Every constraint a table carries, foreign keys included — the fold routes those to their own
122
135
  * directory. `contype = 'n'` (a NOT NULL recorded as a constraint, Postgres 18) is left out: the
123
136
  * column's own `not null` already says it, and reading it would make the dump differ by server.
137
+ * A constraint a partition holds from its parent — a key's clone (`conparentid`), an inherited
138
+ * check (`not conislocal`) — is left out too: the parent's own statement recreates it.
124
139
  */
125
140
  export const constraintRows = (
126
141
  client: DbClient,
@@ -136,6 +151,7 @@ export const constraintRows = (
136
151
  join pg_class c on c.oid = k.conrelid
137
152
  join pg_namespace n on n.oid = c.relnamespace
138
153
  where n.nspname = ${schema} and k.contype in ('p', 'u', 'c', 'x', 'f')
154
+ and k.conparentid = 0 and k.conislocal
139
155
  `);
140
156
 
141
157
  /** `::text` on every bound: a `bigint` is a string on one driver and a number on the other. */
@@ -164,7 +180,11 @@ export const sequenceRows = (client: DbClient, schema: string): Promise<readonly
164
180
  where n.nspname = ${schema} and ${notExtensionOwned('pg_class', 'c.oid')}
165
181
  `);
166
182
 
167
- /** Indexes no constraint backs, on a table or a materialized view. */
183
+ /**
184
+ * Indexes no constraint backs, on a table, a partitioned table or a materialized view. A
185
+ * partition's index ATTACHED to its parent's (a `pg_inherits` row of its own) is left out: the
186
+ * parent's index, built on load, builds it.
187
+ */
168
188
  export const indexRows = (client: DbClient, schema: string): Promise<readonly IndexRow[]> =>
169
189
  client.query<IndexRow>(sql`
170
190
  select
@@ -176,9 +196,10 @@ export const indexRows = (client: DbClient, schema: string): Promise<readonly In
176
196
  join pg_class t on t.oid = x.indrelid
177
197
  join pg_namespace n on n.oid = t.relnamespace
178
198
  where n.nspname = ${schema}
179
- and t.relkind in ('r', 'm')
199
+ and t.relkind in ('r', 'm', 'p')
180
200
  and not exists (
181
201
  select 1 from pg_constraint k
182
202
  where k.conindid = x.indexrelid and k.contype in ('p', 'u', 'x')
183
203
  )
204
+ and not exists (select 1 from pg_inherits h where h.inhrelid = x.indexrelid)
184
205
  `);
package/src/catalog.ts CHANGED
@@ -90,6 +90,14 @@ export interface CatalogTable {
90
90
  readonly constraints: readonly CatalogConstraint[];
91
91
  readonly sequences: readonly CatalogOwnedSequence[];
92
92
  readonly replicaIdentity: CatalogReplicaIdentity;
93
+ /** `pg_get_partkeydef` — `RANGE (at)` — on a partitioned table. Absent on any other. */
94
+ readonly partitionBy?: string | undefined;
95
+ /**
96
+ * On a partition: the table it is a partition of and its bound as the server spells it
97
+ * (`FOR VALUES FROM (…) TO (…)`, `DEFAULT`). Absent on any other. A partition's columns and the
98
+ * constraints it holds from its parent are the parent's, so the dump spells neither.
99
+ */
100
+ readonly partitionOf?: { readonly table: string; readonly bound: string } | undefined;
93
101
  }
94
102
 
95
103
  /** An index no constraint backs; a constraint's own index is created by the constraint. */
@@ -111,6 +119,11 @@ export interface CatalogView {
111
119
  readonly options: string | null;
112
120
  /** `pg_get_viewdef` verbatim, trailing `;` and all. */
113
121
  readonly definition: string;
122
+ /**
123
+ * The output columns, in order. Never rendered — the definition already says them — and read by
124
+ * the `drift` step against a view entity's declared `columns`. Absent on a hand-built catalog.
125
+ */
126
+ readonly columns?: readonly string[] | undefined;
114
127
  }
115
128
 
116
129
  export interface CatalogFunction {
@@ -130,9 +143,31 @@ export interface CatalogTrigger {
130
143
  readonly enabled: string;
131
144
  }
132
145
 
146
+ /** A table whose row security is enabled or forced — two flags, each its own `alter table`. */
147
+ export interface CatalogRowSecurity {
148
+ readonly table: string;
149
+ readonly enabled: boolean;
150
+ /** `force`: the table's owner is bound too. Without it an owner sees every row. */
151
+ readonly forced: boolean;
152
+ }
153
+
154
+ /** A row-security policy, every clause spelled: a clause left out is a default two servers may differ on. */
155
+ export interface CatalogPolicy {
156
+ readonly table: string;
157
+ readonly name: string;
158
+ readonly permissive: boolean;
159
+ /** `all`, `select`, `insert`, `update` or `delete`. */
160
+ readonly command: string;
161
+ /** Role names, `public` for the `0` grantee; sorted. */
162
+ readonly roles: readonly string[];
163
+ /** `pg_get_expr` verbatim, or `null` when the clause is absent. */
164
+ readonly using: string | null;
165
+ readonly withCheck: string | null;
166
+ }
167
+
133
168
  /**
134
169
  * An object in the schema that the dump has no statement for. Named rather than dropped: a dump
135
- * that silently omitted a row-security policy would load into a database that is not the one the
170
+ * that silently omitted an object would load into a database that is not the one the
136
171
  * migrations build, and the load-equals-replay check could not see it, because both sides of that
137
172
  * comparison would be blind in the same place.
138
173
  */
@@ -155,6 +190,8 @@ export interface CatalogDescription {
155
190
  readonly views: readonly CatalogView[];
156
191
  readonly functions: readonly CatalogFunction[];
157
192
  readonly triggers: readonly CatalogTrigger[];
193
+ readonly rowSecurity: readonly CatalogRowSecurity[];
194
+ readonly policies: readonly CatalogPolicy[];
158
195
  readonly unrendered: readonly CatalogUnrendered[];
159
196
  }
160
197
 
@@ -170,5 +207,7 @@ export const emptyCatalog = (schema = 'public'): CatalogDescription => ({
170
207
  views: [],
171
208
  functions: [],
172
209
  triggers: [],
210
+ rowSecurity: [],
211
+ policies: [],
173
212
  unrendered: [],
174
213
  });
package/src/check-ddl.ts CHANGED
@@ -11,6 +11,7 @@
11
11
  // is `42710` on the very next generation — a migration nobody can apply.
12
12
 
13
13
  import { assert } from '@ultimat3/core';
14
+ import { type ConcurrentPlan, deferValidate } from './concurrent-plan';
14
15
  import type { ColumnDescriptionLike, EntityDescriptionLike } from './entity-shape';
15
16
  import type { CheckDescription, TableDescription } from './introspect';
16
17
  import { invariantChecks, isIdentifier, MAX_IDENTIFIER_BYTES } from './invariant-ddl';
@@ -32,6 +33,9 @@ import { statementsOf } from './statement-split';
32
33
  */
33
34
  const spellColumnCheckName = (table: string, column: string): string => `${table}_${column}_check`;
34
35
 
36
+ /** The spelling, for a reader asking about a recorded column (`retired.ts`). Validates nothing. */
37
+ export const columnCheckNameOf = spellColumnCheckName;
38
+
35
39
  /**
36
40
  * Whether a RECORDED constraint is one of THIS entity's columns' own CHECKs. Validates nothing and
37
41
  * never throws, exactly as `namesConstraint` does not, because its caller is a REPORTER
@@ -131,6 +135,10 @@ export const addCheck = (table: string, check: CheckDescription): string =>
131
135
  `alter table ${identifier(table).text} add constraint ${identifier(check.name).text} ` +
132
136
  `check (${check.expression});`;
133
137
 
138
+ /** `not valid`: new writes are held to it at once, and the scan runs later (`deferValidate`). */
139
+ const addCheckNotValid = (table: string, check: CheckDescription): string =>
140
+ `${addCheck(table, check).slice(0, -1)} not valid;`;
141
+
134
142
  export const dropCheck = (table: string, name: string): string =>
135
143
  `alter table ${identifier(table).text} drop constraint ${identifier(name).text};`;
136
144
 
@@ -144,9 +152,13 @@ export const dropCheck = (table: string, name: string): string =>
144
152
  * `add constraint` is `42710` on the first, inside `ROLE=migrate`, with the server's words and none
145
153
  * of the entity's. `drop constraint if exists` costs a notice on the second and repairs the first.
146
154
  */
147
- const rebuildCheck = (table: string, check: CheckDescription): readonly string[] => [
155
+ const rebuildCheck = (
156
+ table: string,
157
+ check: CheckDescription,
158
+ add: typeof addCheck,
159
+ ): readonly string[] => [
148
160
  `alter table ${identifier(table).text} drop constraint if exists ${identifier(check.name).text};`,
149
- addCheck(table, check),
161
+ add(table, check),
150
162
  ];
151
163
 
152
164
  /**
@@ -171,6 +183,14 @@ const rebuildCheck = (table: string, check: CheckDescription): readonly string[]
171
183
  * `drop constraint` on it a second time is `42704`, and its `down` belongs to the retype that
172
184
  * moved it. Keyed by name and not by column because an INVARIANT's check reads a column without
173
185
  * being derived from one, which is exactly the constraint `examples/dummy` retypes under.
186
+ *
187
+ * `concurrent` is what makes this safe on a live table: a CHECK needs no index, so it can always be
188
+ * added `not valid` in the transaction, scanning nothing, with its `validate constraint` in the
189
+ * deferred step. A name that was rebuilt or moved aside stays plain: that column's rewrite already
190
+ * holds ACCESS EXCLUSIVE, and a window with an unvalidated constraint would buy nothing. Omitted,
191
+ * every constraint takes the plain form, which is what a caller outside `generateMigration` gets.
192
+ *
193
+ * A column `retired` keeps its own check recorded and untouched until the drop takes both.
174
194
  */
175
195
  export function checkPlan(
176
196
  entity: EntityDescriptionLike,
@@ -178,6 +198,7 @@ export function checkPlan(
178
198
  plan: { up: string[]; down: string[] },
179
199
  rebuilt: ReadonlySet<string> = new Set(),
180
200
  predropped: ReadonlySet<string> = new Set(),
201
+ concurrent?: ConcurrentPlan,
181
202
  ): void {
182
203
  const recorded = new Map((live.checks ?? []).map((check) => [check.name, check]));
183
204
  const present = new Set(live.columns.map((column) => column.name));
@@ -202,22 +223,28 @@ export function checkPlan(
202
223
  for (const check of wanted) {
203
224
  const gone = dropped.has(check.name) || predropped.has(check.name);
204
225
  const held = gone ? undefined : recorded.get(check.name);
226
+ if (held !== undefined && held.expression === check.expression) continue;
227
+ const online = concurrent !== undefined && !gone;
228
+ const add = online ? addCheckNotValid : addCheck;
229
+ if (online) deferValidate(entity.table, check.name, concurrent);
205
230
  if (held === undefined) {
206
231
  plan.up.push(
207
232
  ...(exposed.has(check.name) && !predropped.has(check.name)
208
- ? rebuildCheck(entity.table, check)
209
- : [addCheck(entity.table, check)]),
233
+ ? rebuildCheck(entity.table, check, add)
234
+ : [add(entity.table, check)]),
210
235
  );
211
236
  plan.down.push(dropCheck(entity.table, check.name));
212
237
  continue;
213
238
  }
214
- if (held.expression === check.expression) continue;
215
- plan.up.push(dropCheck(entity.table, check.name), addCheck(entity.table, check));
239
+ plan.up.push(dropCheck(entity.table, check.name), add(entity.table, check));
216
240
  plan.down.push(addCheck(entity.table, held), dropCheck(entity.table, check.name));
217
241
  }
218
242
  const declared = new Set(wanted.map((check) => check.name));
243
+ const retired = new Set(
244
+ (entity.retired ?? []).map((column) => spellColumnCheckName(entity.table, column)),
245
+ );
219
246
  for (const check of live.checks ?? []) {
220
- if (declared.has(check.name) || predropped.has(check.name)) continue;
247
+ if (declared.has(check.name) || predropped.has(check.name) || retired.has(check.name)) continue;
221
248
  plan.up.push(dropCheck(entity.table, check.name));
222
249
  plan.down.push(addCheck(entity.table, check));
223
250
  }
package/src/client.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  // `Bun.SQL` slice in `bun-sql.ts` and the observed statement funnel in `statement-funnel.ts`, so
5
5
  // importing this module never opens a socket.
6
6
 
7
- import { logger, type Role, renderThrowable, resolveRole } from '@ultimat3/core';
7
+ import { logger, type Role, resolveRole, type Scheduler } from '@ultimat3/core';
8
8
  import {
9
9
  type BunSqlDriver,
10
10
  type BunSqlReserved,
@@ -16,16 +16,17 @@ import { connectionUrl } from './connection-url';
16
16
  // Deliberate cycle, the same shape as `client.ts ⇄ transaction.ts`: nothing here is referenced at
17
17
  // module evaluation, and both sides are `function` declarations, so hoisting covers the TDZ.
18
18
  import { defaultClient } from './default-client';
19
- import { DbError, drainTimeout, driverError } from './errors';
20
- import {
21
- assertListenChannel,
22
- type DbSubscription,
23
- type ListeningClient,
24
- listenUnsupported,
25
- } from './listen';
26
- import { type PoolDemand, trackPool } from './pool-gauge';
19
+ import { DbError, dbUnavailable, drainTimeout, driverError } from './errors';
20
+ import type { DbSubscription, ListeningClient } from './listen';
21
+ import { freeSlot, type GatedPool, poolGate, takeSlot } from './pool-gate';
22
+ import { trackPool } from './pool-gauge';
23
+ import { poolListeners } from './pool-listeners';
24
+ import { noteAnswered, pinBound, pinDeadlineMs } from './pool-pins';
27
25
  import { assertPoolProfile, type PoolProfile, poolProfileFor } from './pool-profile';
28
26
  import { reserveWithin } from './pool-reserve';
27
+ import { closeRetiredPool } from './pool-retire';
28
+ import { poolSweep, UNANSWERED_GRACE_MS } from './pool-sweep';
29
+ import { type PoolerMode, poolerFromEnv, poolerSessionRequired, poolerWaitFor } from './pooler';
29
30
  import { type SqlFragment, sql } from './sql';
30
31
  import { affectedBy, rowsOf, runOn } from './statement-funnel';
31
32
  import { currentTx } from './transaction';
@@ -60,17 +61,44 @@ export interface PostgresClientOptions {
60
61
  readonly role?: Role | undefined;
61
62
  readonly profile?: Partial<PoolProfile> | undefined;
62
63
  readonly applicationName?: string | undefined;
64
+ /**
65
+ * The environment this pool is configured from: `DATABASE_POOLER`, `ROLE` and, with no `url`,
66
+ * `DATABASE_URL`. `process.env` when absent. One source per client, so a caller handed an
67
+ * environment cannot get its URL from it and its pooler from the process; `client.pooler` is
68
+ * then the one answer to "is a pooler in the path", for everything that asks afterwards.
69
+ */
70
+ readonly env?: Readonly<Record<string, string | undefined>> | undefined;
71
+ /**
72
+ * How long a statement may queue in the pooler for a backend before it counts as unanswered,
73
+ * added to the deadline only when a pooler is named. Default `POOLER_WAIT_MS`, PgBouncer's
74
+ * `query_wait_timeout`; match it to the pooler's own if that one is set.
75
+ */
76
+ readonly poolerWaitMs?: number | undefined;
77
+ /** Injection seam for the pool's deadlines; production passes none. */
78
+ readonly schedule?: Scheduler | undefined;
63
79
  }
64
80
 
65
81
  export interface PostgresClient extends ReservableClient, ListeningClient {
66
82
  readonly profile: PoolProfile;
83
+ readonly pooler: PoolerMode | undefined;
67
84
  ping(): Promise<void>;
68
85
  close(): Promise<void>;
69
86
  }
70
87
 
88
+ const PIN_RETIRED =
89
+ 'the connection this transaction holds stopped answering and was retired; nothing more is sent on it';
90
+
71
91
  /** Lazily connects: the pool opens on the first statement, never at import. */
72
92
  export function postgresClient(options: PostgresClientOptions = {}): PostgresClient {
73
- const role = options.role ?? resolveRole();
93
+ const env = options.env ?? process.env;
94
+ const role = options.role ?? resolveRole({ env });
95
+ const pooler = poolerFromEnv(env);
96
+ const poolerWaitMs = poolerWaitFor(pooler, options.poolerWaitMs);
97
+ // Here, before a socket: through a transaction pooler `pg_try_advisory_lock` answers true to two
98
+ // migrators on one backend, and a statement could not tell either of them so.
99
+ if (pooler === 'transaction' && (role === 'migrate' || role === 'replicator')) {
100
+ throw poolerSessionRequired(`ROLE=${role}`);
101
+ }
74
102
  const profile: PoolProfile = assertPoolProfile({
75
103
  ...poolProfileFor(role),
76
104
  ...(options.profile ?? {}),
@@ -79,28 +107,64 @@ export function postgresClient(options: PostgresClientOptions = {}): PostgresCli
79
107
  // as ONE value: a pool still draining after `close()` settles its own work against its own
80
108
  // counter, never against the pool that replaced it. Built only once the driver exists, so a
81
109
  // connection string that cannot be built registers no pool in `db_pool_max`.
82
- let driver: { readonly pool: BunSqlDriver; readonly demand: PoolDemand } | undefined;
110
+ // The gate rides on it for the same reason: a pin held on a retired pool is no slot of the new one.
111
+ type LivePool = GatedPool & { readonly pool: BunSqlDriver };
112
+ let driver: LivePool | undefined;
113
+ const listeners = poolListeners(() => connect().pool, options.schedule);
114
+ // Every statement past it is rejected and its pool retired: pooled, or on a pin against the bound
115
+ // in force on that connection (`pool-pins.ts`). `0` (`migrate`, `replicator`): none.
116
+ // Behind a pooler the server's bound starts only once a backend is assigned.
117
+ const beyondBoundMs = UNANSWERED_GRACE_MS + poolerWaitMs;
118
+ const sweep =
119
+ profile.statementTimeoutMs === 0
120
+ ? undefined
121
+ : poolSweep<LivePool>({
122
+ deadlineMs: profile.statementTimeoutMs + beyondBoundMs,
123
+ onMiss: retire,
124
+ schedule: options.schedule,
125
+ });
83
126
 
84
- function connect(): { readonly pool: BunSqlDriver; readonly demand: PoolDemand } {
127
+ function connect(): LivePool {
85
128
  if (driver !== undefined) return driver;
86
- const url = connectionUrl(options, profile);
129
+ const url = connectionUrl(
130
+ { ...options, url: options.url ?? env['DATABASE_URL'], pooler },
131
+ profile,
132
+ );
87
133
  const Factory = bunSqlFactory();
88
- driver = { pool: new Factory(url, bunSqlPoolOptions(profile)), demand: trackPool(profile.max) };
134
+ driver = {
135
+ pool: new Factory(url, bunSqlPoolOptions(profile)),
136
+ demand: trackPool(profile.max),
137
+ gate: poolGate(profile.max, options.schedule),
138
+ };
89
139
  return driver;
90
140
  }
91
141
 
142
+ /** A statement went unanswered on `missed`. Its stragglers miss too, and retire nothing newer. */
143
+ function retire(missed: LivePool, excerpt: string, waitedMs: number): void {
144
+ if (driver !== missed) return;
145
+ // Read-then-clear, as `close()` does: the next `connect()` opens a fresh pool, never this one.
146
+ driver = undefined;
147
+ missed.demand.close();
148
+ logger.error('db.pool.retired', { statement: excerpt, waitedMs, role });
149
+ closeRetiredPool(missed.pool);
150
+ missed.gate.handOverWaiting();
151
+ listeners.redial();
152
+ }
153
+
92
154
  async function run(fragment: SqlFragment): Promise<unknown> {
93
- const { pool, demand } = connect();
94
- demand.enter();
155
+ const slot = takeSlot(connect, profile.acquireTimeoutMs);
156
+ const live = slot instanceof Promise ? await slot : slot;
95
157
  try {
96
- return await runOn(pool, fragment);
158
+ const sending = runOn(live.pool, fragment);
159
+ return await (sweep === undefined ? sending : sweep.track(live, fragment.text, sending));
97
160
  } finally {
98
- demand.leave();
161
+ freeSlot(live);
99
162
  }
100
163
  }
101
164
 
102
165
  const client: PostgresClient = {
103
166
  profile,
167
+ pooler,
104
168
  async query<T>(fragment: SqlFragment): Promise<readonly T[]> {
105
169
  return rowsOf<T>(await run(fragment));
106
170
  },
@@ -116,14 +180,20 @@ export function postgresClient(options: PostgresClientOptions = {}): PostgresCli
116
180
  // (`ERR_POSTGRES_UNSAFE_TRANSACTION`), and a BEGIN that landed on a different connection
117
181
  // than the statement after it would not be a transaction at all — which is exactly what
118
182
  // `withTransaction` and `readOnlyQuery` depend on being true.
119
- const { pool, demand } = connect();
183
+ // One gate slot for the life of the pin, so one number counts statements and pins alike.
184
+ const budget = profile.acquireTimeoutMs;
185
+ const started = performance.now();
186
+ const live = await takeSlot(connect, budget);
120
187
  let reserved: BunSqlReserved;
121
- // Counted from the ASK: a pin queued behind a full pool is exactly what `waiting` reports.
122
- demand.enter();
123
188
  try {
124
- reserved = await reserveWithin(pool, profile);
189
+ // `reserveWithin` keeps its own timer for `Bun.SQL`'s queue, on what the gate left of ONE
190
+ // budget: two full timers made `web` wait 10 s for a pin promised in 5. At least 1 ms, since
191
+ // a budget of 0 means "wait forever" there.
192
+ const left =
193
+ budget === 0 ? 0 : Math.max(1, Math.round(budget - (performance.now() - started)));
194
+ reserved = await reserveWithin(live.pool, profile, left);
125
195
  } catch (error) {
126
- demand.leave();
196
+ freeSlot(live);
127
197
  // Acquiring the pin is the one step that runs outside `runOn`, so an exhausted or
128
198
  // unreachable pool would escape as an untyped driver error — and `readOnlyQuery` reaches
129
199
  // this line before its first statement, which is how MCP ends up returning something
@@ -133,20 +203,38 @@ export function postgresClient(options: PostgresClientOptions = {}): PostgresCli
133
203
  throw driverError('could not reserve a connection from the pool', error);
134
204
  }
135
205
  let held = true;
206
+ const pin = pinBound(profile.statementTimeoutMs);
136
207
  // Direct only while the pin is held. `release()` hands this physical connection back, and
137
208
  // the pool may already have given it to another unit of work mid-transaction — a statement
138
209
  // issued on the stale handle would land inside theirs, committed or rolled back with it and
139
210
  // no error anywhere to explain the row. So a late statement takes its own connection out of
140
211
  // the pool, exactly like any other caller. Same rule as `pglite.ts`, one driver down.
141
212
  const on = (fragment: SqlFragment): Promise<unknown> =>
142
- held ? runOn(reserved, fragment) : run(fragment);
213
+ held ? onPinned(fragment) : run(fragment);
214
+ // A miss retired the pool; this connection is never written to again, so the transaction's
215
+ // ROLLBACK is refused here rather than waiting out a second bound on the same dead path.
216
+ const onPinned = async (fragment: SqlFragment): Promise<unknown> => {
217
+ if (pin.dead) throw dbUnavailable(PIN_RETIRED);
218
+ const sending = runOn(reserved, fragment);
219
+ const deadlineMs = pinDeadlineMs(pin, beyondBoundMs);
220
+ try {
221
+ const answer = await (sweep === undefined
222
+ ? sending
223
+ : sweep.track(live, fragment.text, sending, deadlineMs));
224
+ noteAnswered(pin, fragment, profile.statementTimeoutMs);
225
+ return answer;
226
+ } catch (error) {
227
+ if (error instanceof DbError && error.code === 'X_DB_UNAVAILABLE') pin.dead = true;
228
+ throw error;
229
+ }
230
+ };
143
231
  // Idempotent because two owners already exist on one exit path: `withTransaction` releases
144
232
  // in a `finally` and `[Symbol.dispose]` fires on the same scope. A second `release()` on a
145
233
  // handle already back in the pool frees whoever holds that connection now.
146
234
  const release = (): void => {
147
235
  if (!held) return;
148
236
  held = false;
149
- demand.leave();
237
+ freeSlot(live);
150
238
  // Total by construction — `releaseReserved` owns the reason (`bun-sql.ts`).
151
239
  releaseReserved(reserved);
152
240
  };
@@ -158,29 +246,8 @@ export function postgresClient(options: PostgresClientOptions = {}): PostgresCli
158
246
  [Symbol.dispose]: release,
159
247
  };
160
248
  },
161
- async listen(channel, onNotify, onListening): Promise<DbSubscription> {
162
- assertListenChannel(channel);
163
- const { pool } = connect();
164
- if (pool.listen === undefined) throw listenUnsupported('this Bun.SQL');
165
- let held: { unlisten(): Promise<void> };
166
- try {
167
- // The driver's own session, never a pin out of the pool: a reserved connection holds the
168
- // LISTEN and surfaces no notification, and it would cost the pool a slot for good.
169
- held = await pool.listen(channel, onNotify, onListening);
170
- } catch (error) {
171
- throw driverError(`LISTEN ${channel}`, error);
172
- }
173
- let ended: Promise<void> | undefined;
174
- return {
175
- unlisten: () => {
176
- // Best-effort, the rule `releaseReserved` states: the session this would end may be
177
- // gone with the pool already, and that is the outcome asked for.
178
- ended ??= held.unlisten().catch((error: unknown) => {
179
- logger.debug('db.unlisten_failed', { error: renderThrowable(error) });
180
- });
181
- return ended;
182
- },
183
- };
249
+ listen(channel, onNotify, onListening): Promise<DbSubscription> {
250
+ return listeners.listen(channel, onNotify, onListening);
184
251
  },
185
252
  async ping(): Promise<void> {
186
253
  await client.query(sql`select 1`);
@@ -193,7 +260,12 @@ export function postgresClient(options: PostgresClientOptions = {}): PostgresCli
193
260
  // rejection still reaches the caller — a shutdown that could not drain wants to know.
194
261
  const closing = driver;
195
262
  driver = undefined;
263
+ listeners.drop();
196
264
  if (closing === undefined) return;
265
+ // A queued statement must not outlive this and open a pool in a process shutting down.
266
+ closing.gate.refuseWaiting(
267
+ dbUnavailable('the pool closed while this statement waited for a slot'),
268
+ );
197
269
  // Only THIS pool's counter leaves the totals; its in-flight work settles against it.
198
270
  closing.demand.close();
199
271
  const { pool } = closing;
@@ -236,10 +308,22 @@ export function postgresClient(options: PostgresClientOptions = {}): PostgresCli
236
308
  }
237
309
 
238
310
  let ambient: DbClient | undefined;
311
+ // Separate from `ambient`, which the lazy default also fills: a unit test that sent one statement
312
+ // through `db()` must not flip `@ultimat3/entity`'s `testDriver()` onto a pool nobody asked for.
313
+ let installed = false;
239
314
 
240
315
  /** Test/dev seam. `setDbClient(undefined)` restores lazy construction from `DATABASE_URL`. */
241
316
  export function setDbClient(client: DbClient | undefined): void {
242
317
  ambient = client;
318
+ installed = client !== undefined;
319
+ }
320
+
321
+ /**
322
+ * Whether `setDbClient` installed a client that is still in place. Asking never builds the lazy
323
+ * default — `testDriver()` asks on every repository call of every unit test.
324
+ */
325
+ export function dbClientInstalled(): boolean {
326
+ return installed;
243
327
  }
244
328
 
245
329
  /**