@ultimat3/db 11.3.0 → 12.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CLAUDE.md CHANGED
@@ -369,6 +369,56 @@ anonymous SQL, never as a `job` statement, and will keep seeing it that way unti
369
369
  own pair through `driver-pg.ts` the way `postgresRepo` now threads entity's — that is still future
370
370
  work, not something this change reaches.
371
371
 
372
+ **An index's ACCESS METHOD is carried end to end, `As of 2026-08-24`, and it had to land here
373
+ before `@ultimat3/entity` could declare it.** `@>` / `<@` / `&&` / `?` on a `json()` or `arrayOf()`
374
+ column is a sequential scan without a GIN index. `IndexInit.using` on the entity side while this
375
+ package ignored it would emit a **btree for a declared GIN index** — a declared-and-never-wired key,
376
+ which is the defect class this release exists to eliminate and strictly worse than the missing
377
+ capability. So the method reaches all four places or none: `createIndex` emits it, `snapshotOf`
378
+ records it, `indexShape` rebuilds on it, and `compareIndexes` reports it.
379
+
380
+ `index-method.ts` is the vocabulary, its own file for the reason `foreign-key.ts` holds
381
+ `onDeleteRule`: a generator and a detector that disagreed about what "the default" is would report
382
+ drift on a database that is exactly right. Four rules.
383
+
384
+ **The set is closed at two — `btree` and `gin`.** `gist`, `brin`, `hash` and `spgist` are legitimate
385
+ and are deliberately absent: nothing declares one, and each brings a rule that would have to be
386
+ enforced with no caller to test it (`hash` and `brin` cannot be unique, `gist` needs `btree_gist` to
387
+ be, none of the three accepts `asc`/`desc`). Adding a member later is additive; shipping four nobody
388
+ uses is four ways for a first caller to be silently wrong.
389
+
390
+ **Declared is CLOSED, live is OPEN.** `IndexDescriptionLike.using` is `IndexMethod | undefined` —
391
+ what an entity may ask for. `IndexDescription.using` is `string | undefined` — whatever `pg_am`
392
+ answered, `gist` and an extension's own access method included. Folding an unknown catalog name into
393
+ `btree` would hide exactly the difference drift exists to report, so `indexMethodOf` passes the live
394
+ side through verbatim and `declaredMethod` is the one place the open reading is narrowed back — a
395
+ **refusal**, never a silent fall back, because its one caller is `redefineIndex`'s `down` and a
396
+ `gist` quietly rebuilt as a btree is a rollback leaving a state no migration describes.
397
+
398
+ **Absent is `btree`, on both sides, through one function.** `indexMethodOf` is that function.
399
+ Postgres' default is written out by nobody, every index created before this existed is one, and
400
+ every sidecar written before the field is silent about it — so `snapshotOf` records `using` only
401
+ when one was declared. Writing `'btree'` out for every index would rewrite every sidecar in every
402
+ app on the next `x db gen`, a diff on every file for a fact that was already true.
403
+
404
+ **The literal is re-derived from the set, never spliced from the input** — `indexMethodSql` is a
405
+ `switch` whose `default` arm is `never` and throws `indexMethodInvalid` (`X_SQL_UNSAFE`, the code
406
+ `isolationLevelInvalid` and `branchNameInvalid` already use for a value spliced into a statement).
407
+ The type is not the guard: this value arrives from an entity declaration, a config or a hand-edited
408
+ snapshot, and `using ${method}` on an operand TypeScript never saw is the **identical hole** to the
409
+ one `columnName` carried when it was `meta.name ?? snake(property)` with only the second branch
410
+ validated — a name that closed the parenthesis and opened a second command, measured through
411
+ `generateMigration`. `create index "x" on "t" using gin ("c") where (...)` also refuses a unique or
412
+ an ordered GIN through core's `assert` (`X_INVARIANT`), the discipline `createIndex` already applies
413
+ to an index naming no columns: Postgres has neither, and a syntax error inside `ROLE=migrate` fails
414
+ the release phase with the server's words and none of the entity's.
415
+
416
+ `introspect()` reads the method from `pg_am` joined through `pg_class.relam`, and
417
+ `introspect-embedded.test.ts` is where that is pinned — a recording client can pin the SQL text and
418
+ nothing more, and a query that silently returned no method would read as `btree` everywhere and make
419
+ drift blind to the one case `using` exists for. Measured on PGlite: a real `using gin` index reads
420
+ back `gin`, the btree beside it and the primary key's own index read back `btree`.
421
+
372
422
  **`generate.ts` reads an index, it never re-derives one.** `EntityDescriptionLike.indexes` carries
373
423
  `columns`, `unique`, `where` and `order`, and `createIndex` writes every one of them out. It used to
374
424
  carry names alone and `parseIndexName` recovered the column list from the `<table>_<a>_<b>_idx`
@@ -502,6 +552,62 @@ override — `x_migrations.app_version` and `@ultimat3/jobs`' `x_backfills.app_v
502
552
  durable columns an operator reads side by side, and `jobs` cannot import this package for the
503
553
  answer, so the key has one reader at tier 0 rather than one per writer.
504
554
 
555
+ **Read replicas are opt-in twice, and the second opt-in is the correctness argument, `As of
556
+ 2026-08-24`.** A replica pool exists when `DATABASE_REPLICA_URL` names one (`default-client.ts`);
557
+ a read is *offered* to it only inside `withReplicaReads(fn)` (`replica-scope.ts`). With no scope
558
+ open nothing routes and the client is byte-identical to the single-pool one it has always been —
559
+ which is what makes "nobody adopted it yet" today's behaviour rather than a wrong answer.
560
+
561
+ The scope is what closes **read-your-writes**, and the reason it is a scope and not a request id is
562
+ worth writing down because the request id is the obvious answer and it does not work. `Ctx.requestId`
563
+ IS reachable from here — `@ultimat3/http`'s pipeline opens `runWithContext` around every request
564
+ (`packages/http/src/pipeline.ts`), `withChildContext` may not change the id, and `tryUseContext()` is
565
+ tier 0 — but nothing tells tier 1 when a request ENDED. A `Map<requestId, wrote>` therefore only
566
+ grows, ~100 bytes a request forever, and every eviction policy that forgets a request which WROTE
567
+ serves it a stale row on its next read. That is a data-correctness bug strictly worse than the
568
+ capacity problem replicas exist to solve, so the marker lives on a mutable value on an async context
569
+ (`ReplicaScope.wrote`, the same shape as `TxState.live`) whose lifetime somebody else already owns.
570
+
571
+ **`withTransaction` is on the primary structurally, not by rule.** `runRoot` pins a connection
572
+ through `reserve()`, and `replicatedClient` delegates `reserve()` to the primary and exposes it only
573
+ when the primary has one — so BEGIN, every statement and COMMIT are one connection on one server.
574
+ `isReservable` therefore has to keep answering about the DATABASE and not about the wrapper: a
575
+ wrapper that always exposed `reserve` makes `runRoot` pin a client that cannot pin, and one that
576
+ never exposed it makes `runRoot` run BEGIN, the body and COMMIT on three different pooled
577
+ connections. What `runRoot` adds is one line — `markScopeWrote()` unless `readOnly: true` — because
578
+ its statements go through a reservation and never through the router, so the scope could not
579
+ otherwise see that the request has written.
580
+
581
+ **`isPlainRead` is an allow-list, and that inversion is why it is not the lexer this file forbids.**
582
+ `readonly.ts` was deleted for defaulting to PERMISSION: a 22-word deny-list that read
583
+ `select pg_sleep(60)` as safe. This one defaults to the primary — a statement shape nobody
584
+ anticipated costs a replica opportunity and never an answer. **`statementKind()` is not the
585
+ authority and must not become it**: it calls `with … update … returning` a read, which is right for
586
+ an N+1 report and catastrophic for a routing decision, and `replica-route.test.ts` asserts the
587
+ disagreement so the two can never be collapsed. Three refusals earn their line — a locking read
588
+ (`for update`/`for share`; a standby cannot take the row lock), `select … into` (it creates a
589
+ table), and the functions a word boundary cannot reach (`pg_advisory_lock`, `set_config`,
590
+ `nextval`), which a standby ANSWERS rather than refusing, so the server cannot be the safety net for
591
+ those the way it is for a real write.
592
+
593
+ **A misroute fails loudly and repairs itself; a replica outage costs latency and never an answer.**
594
+ A statement a standby refuses (`25006`) never executed, and only `isPlainRead` statements are ever
595
+ sent there, so re-running one on the primary is exactly-once rather than at-least-once — which is
596
+ what makes the blanket fallback in `replica-client.ts` safe. The breaker is what stops that from
597
+ doubling every read during an outage: three consecutive failures park the replica for ten seconds,
598
+ counted on `Clock.monotonic()` so an NTP step cannot un-park it. `ReplicaStats` is exposed on the
599
+ client for a test that cannot scrape, the same reason `@ultimat3/realtime` exposes
600
+ `droppedChannelFrames`, and each fallback logs `db.replica_fallback` with `renderThrowable(error)`.
601
+
602
+ **The URL must name a read-only standby**, and nothing here can check it. The `25006` refusal is the
603
+ whole safety net under a text classifier that cannot be complete; pointed at a writable node, a
604
+ misroute becomes a write on the wrong server with nothing anywhere to report it.
605
+
606
+ **Nothing opens `withReplicaReads` per request yet.** The scope, the client and the wiring are tier
607
+ 1 and land here first; the adopter is one call in `@ultimat3/http`'s pipeline (or an app's own
608
+ handler), and until it exists no production traffic is routed. That is the tier rule working —
609
+ lowest tier first, consumers after — not an omission.
610
+
505
611
  `checkDrift()` is the **post-migrate verification** and the only drift question that needs a
506
612
  database: the live catalog against the ledger the run just wrote. It is asked where a connection is
507
613
  open — `@ultimat3/cli`'s `runMigrations`, which is `x db migrate`, `x db reset` and `ROLE=migrate`
@@ -548,7 +654,7 @@ The `fix:` is the `alter table … set not null` itself and deliberately not `x
548
654
  never emitted one and would answer with an empty migration.
549
655
 
550
656
  `compareTable` judges **declared** indexes: one the migrations name and the catalog does not hold is
551
- `missing-index`, and one whose column list or uniqueness moved is `changed-index` — which is what
657
+ `missing-index`, and one whose access method, column list or uniqueness moved is `changed-index` — which is what
552
658
  catches a composite index rebuilt with its columns the other way round while the column diff said
553
659
  `ok: true`. A live index no snapshot names is deliberately **not** reported: Postgres creates one for
554
660
  every primary key and every unique constraint, so counting those is eight findings against a correct
@@ -681,7 +787,39 @@ exists` at boot, declared by no migration and carried in no snapshot, so counted
681
787
  are eight `unexpected-table` findings against a correct database. The prefix is the rule, not a
682
788
  list, so a table a future package adds needs no second declaration here. `introspect()` keeps its
683
789
  narrower default (`x_migrations` alone) because the admin schema view and the MCP `schema.describe`
684
- tool legitimately show `x_users` — only drift wants the whole namespace gone.
790
+ tool legitimately show `x_users` — only drift wants the whole namespace gone. That last sentence is
791
+ a *reservation*, not a description, `As of 2026-08-24`: nothing outside this package imports
792
+ `introspect()` today, and `schema.describe` (`@ultimat3/mcp`'s `dev-server.ts`) answers from the
793
+ entity registry.
794
+
795
+ **`app-relation.ts` is the other half, and it is ownership, never a name — issue #340,
796
+ `As of 2026-08-24`.** `pg_stat_statements` is a view an extension owns, the CNPG/RDS/Supabase
797
+ default puts it in `public` of every database, and the drift audit after `ROLE=migrate` reported it
798
+ as `unexpected-table` with `x db gen "add pg_stat_statements"` as the fix — so every deploy of the
799
+ demo app failed terminally for 16 hours, and following the fix would have written an extension's
800
+ internal view into the app's migration set. `nonAppRelations(client, schema)` names what
801
+ `introspect()` must not see, and `introspect()` merges it into `excluded` **unconditionally**: an
802
+ explicit `exclude` replaces the `x_migrations` default, never this set, because an extension's
803
+ relations are not app schema in any deployment and that is not a caller's to switch off.
804
+
805
+ Two disqualifications, one question. **Extension ownership is read out of `pg_depend`**
806
+ (`deptype = 'e'`, `refclassid = 'pg_extension'`) — Postgres' own record, and the only rule that
807
+ generalises: a `pg_*` prefix would have covered the reported view and missed `postgis`'
808
+ `spatial_ref_sys`, `timescaledb`'s catalog, and `pg_stat_statements`' own `pg_stat_statements_info`
809
+ sibling, which is a real `relkind = 'r'` table. **A view, a materialised view and a foreign table
810
+ are not tables**, whoever made them: measured on PGlite, a plain `create view` reaches
811
+ `information_schema.columns` while the index query already fences on `relkind = 'r'`, so one arrived
812
+ as a table with columns, no primary key and no indexes — a `TableDescription` that cannot be true,
813
+ and a finding no author could clear because no snapshot records a view. Excluding by NAME is safe
814
+ because `pg_class` names are unique within a namespace.
815
+
816
+ Nothing else in the audit had the same hole. An extension cannot own a **column** of a table it does
817
+ not own — `alter extension … add` has no `COLUMN` form — so `unexpected-column` is unreachable that
818
+ way. **Types and enums** are never compared (`compareTable` reads nullability and existence, never
819
+ the type). **Indexes and foreign keys** are judged on the declared side only, so an extension's
820
+ index on an app table was already silent. `introspect-embedded.test.ts` proves the predicate against
821
+ a real catalog by writing the exact `pg_depend` row `create extension` writes; a recording client
822
+ can only pin the SQL text, which is what `app-relation.test.ts` does.
685
823
 
686
824
  The `X_DB_DRIFT` rendering in `drift.ts` and the title in `DB_ERROR_TITLES` are pinned by the
687
825
  framework contract and duplicated in `@ultimat3/entity`. Change them together or not at all.
package/README.md CHANGED
@@ -13,7 +13,7 @@ boundary stays thin.
13
13
  ## Public API
14
14
 
15
15
  ```ts
16
- import { db, sql, raw, withTransaction, currentTx, readOnly, setDbClient } from '@ultimat3/db';
16
+ import { db, sql, raw, withTransaction, currentTx, setDbClient } from '@ultimat3/db';
17
17
 
18
18
  const rows = await db().query<Post>(sql`select * from posts where org_id = ${orgId}`);
19
19
 
@@ -32,7 +32,11 @@ await withTransaction(async (tx) => {
32
32
  | `sqlState()` / `sqlStateCode()` / `isRetryableState()` / `SQLSTATE` | `As of 2026-08`: the SQLSTATE a driver error carries, and the closed table from it to a code. `Bun.SQL` puts it on `errno`; PGlite puts it on `code`; **one** reader answers for both |
33
33
  | `migrate()` / `rollback()` / `readLedger()` | the `x_migrations` ledger |
34
34
  | `statementsOf()` | `As of 2026-08`: a SQL script → the statements a driver sends one at a time. One send is one statement, so `migrate()` splits with this — a `;` inside a literal, an identifier, a dollar-quoted body or a comment is data |
35
- | `checkDrift()` / `diffSchema()` / `assertNoDrift()` | drift, with a `--json` report. `checkDrift()` is the **post-migrate verification** — the live database against the ledger: columns, declared indexes (columns, uniqueness, direction, and whether a predicate is there at all never its text) and declared foreign keys, matched on where the key points and not on its constraint name, with the `on delete` rule compared through one normalisation `As of 2026-08-19` |
35
+ | `withReplicaReads()` / `replicaScope()` | `As of 2026-08-24`: the scope inside which a plain read may be served by a replicauntil it writes, after which every read in it is the primary's. No scope open, nothing routes |
36
+ | `replicatedClient()` / `ReplicaStats` / `REPLICA_URL_ENV` | `As of 2026-08-24`: one `DbClient` over a primary and a standby. `baseClient()` builds one when `DATABASE_REPLICA_URL` is set and the single-pool client when it is not |
37
+ | `INDEX_METHODS` / `IndexMethod` / `indexMethodOf()` / `indexMethodSql()` / `declaredMethod()` / `isIndexMethod()` | `As of 2026-08-24`: an index's access method — `btree` or `gin`, closed. Absent is `btree`, the live side is read open (whatever `pg_am` said), and the DDL literal is re-derived from the set rather than spliced from the input |
38
+ | `isPlainRead()` | `As of 2026-08-24`: whether a statement may leave the primary. An allow-list — everything it cannot vouch for is the primary's |
39
+ | `checkDrift()` / `diffSchema()` / `assertNoDrift()` | drift, with a `--json` report. `checkDrift()` is the **post-migrate verification** — the live database against the ledger: columns, declared indexes (access method `As of 2026-08-24`, columns, uniqueness, direction, and whether a predicate is there at all — never its text) and declared foreign keys, matched on where the key points and not on its constraint name, with the `on delete` rule compared through one normalisation `As of 2026-08-19` |
36
40
  | `declaredSchema()` / `expectedSchema()` | `As of 2026-08`: the schema the migrations write down, or `undefined` when the newest one carries no snapshot — never an older snapshot standing in for it |
37
41
  | `parseSnapshot()` | `As of 2026-08`: a `<id>.snapshot.json` sidecar validated to the last nested field, or `undefined`. `{"tables":[null]}` is valid JSON and is not a schema |
38
42
  | `snapshotJson()` | `As of 2026-08`: the sidecar's **bytes** — the JSON Biome would have printed, trailing newline included. The one writer of a `<id>.snapshot.json`, because `JSON.stringify(…, null, 2)` is not formatter-clean and an app's `lint` step rejected the file `x db gen` had just written |
@@ -41,7 +45,7 @@ await withTransaction(async (tx) => {
41
45
  | `generateMigration()` | `x db gen "<name>"` — reversible up/down SQL, and `destructive` for the marker the file must carry. `As of 2026-08` a foreign key is its own `alter table … add constraint`, emitted after every table statement: inline, a `references()` had to point at a table entity registration order happened to create first, and `down` had to drop them in an order it did not control. `As of 2026-08-19` a **removed** `references()` emits its `drop constraint` (it emitted nothing, and the snapshot then denied a constraint the database still held), a changed `onDelete` is a drop-and-add rebuild, and a declared `on delete` rule reaches the clause at all |
42
46
  | `destructiveStatements()` / `hasDestructiveMarker()` / `isDestructive()` / `DESTRUCTIVE_MARKER` | `As of 2026-08`: the destructive-SQL rail — does this `up` drop, truncate or retype, and does the file declare it with `-- destructive: true`? One classifier, read by `x db gen` when it writes the marker and by `x verify` when it demands one |
43
47
  | `stripSqlNoise()` | comments, literals, dollar-quoted bodies and quoted identifiers blanked **in source order**, so a reader sees the operation and not the prose. Shared by `readOnlyQuery()` and the destructive rail |
44
- | `introspect()` | live schema → `SchemaDescription` |
48
+ | `introspect()` | live schema → `SchemaDescription`. **App tables only**, `As of 2026-08-24`: a relation an extension owns (`pg_depend`, `deptype = 'e'`) and anything that is not an ordinary or partitioned table are excluded before the fold, and an explicit `exclude` cannot bring them back |
45
49
  | `createBranch()` / `dropBranch()` / `reapBranches()` | copy-on-write branch databases. `As of 2026-08-19` the marker comment records the **base** as well as the instant (`ultimate:branch:<base>:<iso>`, on `BranchInfo.base`), and `reapBranches()` sweeps only branches of the database it is connected to — one Postgres hosting two Ultimate apps used to mean one app's nightly reap dropped the other's branches. A pre-3.x marker records no base and is skipped, never dropped |
46
50
  | `createPgliteClient()` / `branchPglite()` | the embedded database — Postgres in this process |
47
51
  | `ensureReadOnlyRole()` / `grantReadOnlySql()` / `READONLY_ROLE` | a `NOLOGIN`, SELECT-only Postgres role — layer 1 of `db.query`'s defence |
@@ -98,6 +102,62 @@ statement's own error. Every caller handles that failure; nothing here swallows
98
102
  `ensureReadOnlyRole()` — otherwise a table created later is not selectable by the role, and
99
103
  layer 1 covers only what existed at grant time.
100
104
 
105
+ ## Index access methods
106
+
107
+ `btree` is the default and is never written out; `gin` is what makes `@>` / `<@` / `&&` / `?` on a
108
+ `json()` or `arrayOf()` column an index lookup rather than a sequential scan.
109
+
110
+ | | |
111
+ |---|---|
112
+ | declared | `IndexDescriptionLike.using` — `'btree' \| 'gin'`, closed. Anything else is `X_SQL_UNSAFE` before it reaches DDL |
113
+ | absent | `btree`, on every side, through `indexMethodOf()` — so a snapshot written before the field existed reads as what it always was |
114
+ | emitted | `create index "posts_tags_idx" on "posts" using gin ("tags");` — and `create index "posts_tags_idx" on "posts" ("tags");` when no method was declared, byte for byte what shipped before |
115
+ | refused | a unique GIN and an ordered GIN (`X_INVARIANT`) — Postgres has neither, and the alternative is a syntax error inside `ROLE=migrate` |
116
+ | recorded | only when declared, so no existing sidecar is rewritten |
117
+ | a method that moved | drop and recreate — Postgres cannot alter one in place |
118
+ | drift | a declared GIN against a live btree is `changed-index`, and the live method is reported by the name the catalog gave it |
119
+
120
+ ## Read replicas
121
+
122
+ Opt in twice, and the second one is why it is safe.
123
+
124
+ ```ts
125
+ // 1. the pool — an environment variable, read once by baseClient()
126
+ // DATABASE_REPLICA_URL=postgres://reader@replica.internal:5432/app
127
+
128
+ // 2. the scope — reads inside it may be served by the replica
129
+ import { db, sql, withReplicaReads, withTransaction } from '@ultimat3/db';
130
+
131
+ declare const id: string;
132
+
133
+ await withReplicaReads(async () => {
134
+ await db().query(sql`select id from posts limit 20`); // -> replica
135
+ await db().execute(sql`insert into posts (id) values (${id})`);
136
+ await db().query(sql`select id from posts limit 20`); // -> primary, for the rest of the scope
137
+ await withTransaction(async (tx) => {
138
+ await tx.query(sql`select 1`); // -> primary, always
139
+ });
140
+ });
141
+ ```
142
+
143
+ | Rule | |
144
+ |---|---|
145
+ | unconfigured | no `DATABASE_REPLICA_URL` → one pool, statement for statement what it always was |
146
+ | no scope | no `withReplicaReads` → nothing routes, whatever is configured |
147
+ | read-your-writes | one write anywhere in the scope, at any depth, across any `await`, and every later read in it is the primary's |
148
+ | a transaction | always the primary — `reserve()` is delegated there, so BEGIN, the body and COMMIT are one connection on one server. A `readOnly: true` transaction leaves the scope clean |
149
+ | eligibility | `select` / `table` / `values` / a read-only `with`, minus locking reads, `select … into` and the functions a standby answers instead of refusing (`pg_advisory_lock`, `set_config`, `nextval`). Everything else is the primary's |
150
+ | a replica that fails | the statement is re-run on the primary — exactly-once, because only plain reads are sent there and a `25006` refusal never executed. Three failures in a row park it for ten seconds |
151
+ | observability | `client.stats` (`replica`, `primary`, `fallbacks`, `parked`), and a `db.replica_fallback` warning per fallback |
152
+
153
+ **The URL must name a read-only standby.** The server's own `25006` refusal is the safety net under
154
+ a classifier that cannot be complete; pointed at a writable node a misroute becomes a write on the
155
+ wrong server, silently.
156
+
157
+ **Nothing opens the scope for you yet.** `withReplicaReads` is tier 1 and ships first; wrapping a
158
+ request in it is the app's call — or one line in the HTTP pipeline — so until that lands no
159
+ production traffic is routed.
160
+
101
161
  ## The drift contract
102
162
 
103
163
  `checkDrift()` compares `introspect()` against the snapshot the newest applied migration carries —
@@ -119,8 +179,17 @@ subset, so a database that simply has not migrated yet is pending, not divergent
119
179
  table in the `x_` namespace — `x_migrations`, the queue's tables, the outbox and every
120
180
  `@ultimat3/auth` table are created by `create table if not exists` at boot and appear in no
121
181
  snapshot, so `appTables()` drops them before the diff. `introspect()` keeps its own narrower
122
- exclusion (the ledger alone), because the admin schema view and the MCP `schema.describe` tool
123
- legitimately show `x_users`.
182
+ exclusion (the ledger alone), reserving `x_users` for a schema view that wants it.
183
+
184
+ **Nor is a relation an extension owns, `As of 2026-08-24`.** `create extension pg_stat_statements`
185
+ in `public` is the CNPG, RDS, Supabase and Neon default, and its view read as `unexpected-table`
186
+ with `x db gen "add pg_stat_statements"` as the fix — so every deploy failed terminally and the fix
187
+ would have written an extension's internal view into the app's migration set. `introspect()` now
188
+ excludes every relation Postgres records as extension-owned (`pg_depend`, `deptype = 'e'`), which is
189
+ ownership rather than a name: a `pg_*` prefix rule covers that view and misses `postgis`'
190
+ `spatial_ref_sys`. Views, materialised views and foreign tables go with them — no snapshot records
191
+ one, so counting them could only ever produce a finding an author has no way to clear. A table
192
+ someone created by hand carries no such dependency and is still `unexpected-table`.
124
193
 
125
194
  Rendered output is pinned byte-for-byte:
126
195
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ultimat3/db",
3
- "version": "11.3.0",
3
+ "version": "12.0.0",
4
4
  "description": "Postgres access, transactions, migrations and drift detection",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -31,7 +31,7 @@
31
31
  "test": "bun test"
32
32
  },
33
33
  "dependencies": {
34
- "@ultimat3/core": "11.3.0"
34
+ "@ultimat3/core": "12.0.0"
35
35
  },
36
36
  "peerDependencies": {
37
37
  "@electric-sql/pglite": ">=0.5.0"
@@ -0,0 +1,57 @@
1
+ // Single responsibility: name the relations in a schema that are NOT app schema, so introspection
2
+ // never puts them in a snapshot and drift can never report one. Two disqualifications, one
3
+ // question: an object an extension owns, and an object that is not a table at all.
4
+
5
+ import type { DbClient } from './client';
6
+ import { sql } from './sql';
7
+
8
+ interface NonAppRelationRow {
9
+ readonly name: string;
10
+ }
11
+
12
+ /**
13
+ * Relation names in `schema` that no migration could legitimately declare.
14
+ *
15
+ * **Extension ownership is the rule, never a name prefix.** Every object `create extension` builds
16
+ * carries a `pg_depend` row with `deptype = 'e'` pointing at `pg_extension` — that is Postgres'
17
+ * own record of "this belongs to an extension", and it is the only thing that generalises. A
18
+ * prefix rule spelled `pg_*` would have covered the `pg_stat_statements` view that made every
19
+ * deploy of the demo app fail terminally (issue #340) and missed `postgis`' `spatial_ref_sys`,
20
+ * `timescaledb`'s catalog and `pg_stat_statements`' own `pg_stat_statements_info` sibling. An
21
+ * extension may install a relation under any name at all, so the name is not evidence.
22
+ *
23
+ * **A view, a materialised view and a foreign table are not tables**, whoever created them. They
24
+ * reach `information_schema.columns` (measured on PGlite: a plain `create view` appears there),
25
+ * while the index query already fences on `relkind = 'r'` — so one came back as a table with
26
+ * columns, no primary key and no indexes, which is a `TableDescription` that cannot be true. No
27
+ * `x db gen` diff emits `create view` and no snapshot records one, so counting them as app schema
28
+ * can only ever produce a finding an author has no way to clear.
29
+ *
30
+ * Excluding by NAME is safe because `pg_class` names are unique within a namespace: a name on this
31
+ * list cannot also be an app table in the same schema.
32
+ */
33
+ export async function nonAppRelations(
34
+ client: DbClient,
35
+ schema: string,
36
+ ): Promise<readonly string[]> {
37
+ const rows = await client.query<NonAppRelationRow>(sql`
38
+ select c.relname as name
39
+ from pg_class c
40
+ join pg_namespace n on n.oid = c.relnamespace
41
+ where n.nspname = ${schema}
42
+ and c.relkind in ('r', 'p', 'v', 'm', 'f')
43
+ and (
44
+ c.relkind in ('v', 'm', 'f')
45
+ or exists (
46
+ select 1
47
+ from pg_depend d
48
+ where d.classid = 'pg_class'::regclass
49
+ and d.objid = c.oid
50
+ and d.refclassid = 'pg_extension'::regclass
51
+ and d.deptype = 'e'
52
+ )
53
+ )
54
+ order by c.relname
55
+ `);
56
+ return rows.map((row) => row.name);
57
+ }
package/src/client.ts CHANGED
@@ -5,6 +5,9 @@
5
5
 
6
6
  import { type Role, renderThrowable, resolveRole } from '@ultimat3/core';
7
7
  import { statementAttribution } from './attribution';
8
+ // Deliberate cycle, the same shape as `client.ts ⇄ transaction.ts`: nothing here is referenced at
9
+ // module evaluation, and both sides are `function` declarations, so hoisting covers the TDZ.
10
+ import { defaultClient } from './default-client';
8
11
  import { DbError, dbUnavailable, driverError, poolAcquireTimeout, poolMaxInvalid } from './errors';
9
12
  import { expectedQueryLoopReason } from './expected-loop';
10
13
  import { declaresLibpqOption, mergeLibpqOptions } from './libpq-options';
@@ -123,7 +126,7 @@ export const POOL_MAX_ENV = 'DATABASE_POOL_MAX';
123
126
  * rather than falling back: a fleet that ignored the number it was given is the failure the
124
127
  * variable exists to prevent, and it would only be found in `pg_stat_activity` at 3am.
125
128
  */
126
- function poolMaxFromEnv(): Partial<PoolProfile> {
129
+ export function poolMaxFromEnv(): Partial<PoolProfile> {
127
130
  const raw = process.env[POOL_MAX_ENV];
128
131
  if (raw === undefined || raw.trim() === '') return {};
129
132
  const max = Number(raw);
@@ -437,10 +440,11 @@ export function setDbClient(client: DbClient | undefined): void {
437
440
  * The role default is layered under `DATABASE_POOL_MAX`, because this is the one place the process
438
441
  * builds its own client and therefore the only place an operator's value can reach one:
439
442
  * `createPostgresClient` has always taken a `profile` override and nothing in a running app passed
440
- * it, so `POOL_PROFILES` was the last word in a deployed image.
443
+ * it, so `POOL_PROFILES` was the last word in a deployed image. `default-client.ts` owns what gets
444
+ * built — one pool, or a primary and a replica when `DATABASE_REPLICA_URL` names one.
441
445
  */
442
446
  export function baseClient(): DbClient {
443
- if (ambient === undefined) ambient = createPostgresClient({ profile: poolMaxFromEnv() });
447
+ if (ambient === undefined) ambient = defaultClient();
444
448
  return ambient;
445
449
  }
446
450
 
@@ -0,0 +1,35 @@
1
+ // Single responsibility: the client `baseClient()` builds when an app installed none — the primary
2
+ // pool, plus a read replica when one is configured. Its own file because it is the one place the
3
+ // framework decides a process's database topology from the environment, and `client.ts` is at the
4
+ // line ceiling.
5
+
6
+ import { createPostgresClient, type DbClient, poolMaxFromEnv } from './client';
7
+ import { replicatedClient } from './replica-client';
8
+
9
+ /**
10
+ * A read replica's connection string. Unset — which is every app that has not asked for one — and
11
+ * this builds exactly the single-pool client it always did, statement for statement.
12
+ *
13
+ * It must name a READ-ONLY standby. A statement misrouted to one is refused with `25006` and
14
+ * re-run on the primary, and that refusal is the safety net a text classifier cannot be; a URL
15
+ * pointing at a writable node turns a misroute into a write on the wrong server, silently.
16
+ */
17
+ export const REPLICA_URL_ENV = 'DATABASE_REPLICA_URL';
18
+
19
+ /**
20
+ * Composed rather than folded into `createPostgresClient`, on purpose: `migrate`, `x db branch` and
21
+ * every test build a client that must be exactly one pool, and a second pool reachable through the
22
+ * same factory would be a second thing `reserve()`, `close()` and `ping()` each have to mean two
23
+ * ways.
24
+ *
25
+ * The replica inherits the role's profile, `DATABASE_POOL_MAX` included: a fleet sized against
26
+ * `max_connections` has two servers to size against, not one, and a replica pool that ignored the
27
+ * operator's number would be the exhaustion `poolMaxFromEnv` exists to prevent, on the other host.
28
+ */
29
+ export function defaultClient(): DbClient {
30
+ const profile = poolMaxFromEnv();
31
+ const primary = createPostgresClient({ profile });
32
+ const replicaUrl = process.env[REPLICA_URL_ENV];
33
+ if (replicaUrl === undefined || replicaUrl.trim() === '') return primary;
34
+ return replicatedClient(primary, createPostgresClient({ url: replicaUrl, profile }));
35
+ }
package/src/drift.ts CHANGED
@@ -6,6 +6,7 @@
6
6
  import { baseClient, type DbClient } from './client';
7
7
  import { DbError } from './errors';
8
8
  import { addForeignKey, dropForeignKey, foreignKeyTarget, onDeleteRule } from './foreign-key';
9
+ import { indexMethodOf } from './index-method';
9
10
  import {
10
11
  type ForeignKeyDescription,
11
12
  findTable,
@@ -219,6 +220,12 @@ function changedForeignKey(
219
220
  * a `desc` index rebuilt ascending by hand serves a feed's newest page off the wrong end. `asc` is
220
221
  * normalised to `null` first — `createIndex` writes `"col" asc`, which Postgres stores as
221
222
  * not-descending, so the raw values differ on every ascending index in a correct database.
223
+ *
224
+ * The **access method** is compared for the same reason and normalised the same way (`indexMethodOf`,
225
+ * absent = `btree`). Without it an emitter that can write `using gin` ships a btree for a declared
226
+ * GIN index and nothing anywhere says so — a declared-and-never-wired key, which is worse than the
227
+ * missing capability, and the reason `using` could not land in `@ultimat3/entity` before it landed
228
+ * here.
222
229
  */
223
230
  function compareIndexes(live: TableDescription, expected: TableDescription): DriftDifference[] {
224
231
  const differences: DriftDifference[] = [];
@@ -229,6 +236,17 @@ function compareIndexes(live: TableDescription, expected: TableDescription): Dri
229
236
  differences.push(missingIndex(live.name, index.name));
230
237
  continue;
231
238
  }
239
+ // The method first, and before the column list: a GIN index and a btree over the same column
240
+ // are not the same index with a detail different, they are two structures the planner uses for
241
+ // different operators — `@>` on a jsonb column is a sequential scan on the wrong one. Both
242
+ // sides go through `indexMethodOf`, so a snapshot that predates the field and an index the
243
+ // catalog reports as `btree` agree instead of reporting drift on a correct database.
244
+ if (indexMethodOf(counterpart) !== indexMethodOf(index)) {
245
+ differences.push(
246
+ changedIndex(live.name, index.name, `is a ${indexMethodOf(counterpart)} index`),
247
+ );
248
+ continue;
249
+ }
232
250
  if (counterpart.columns.join(',') !== index.columns.join(',')) {
233
251
  differences.push(
234
252
  changedIndex(live.name, index.name, `covers (${counterpart.columns.join(', ')})`),
@@ -2,6 +2,8 @@
2
2
  // tier 1 and may never import `entity` (tier 2), so a snapshot arrives as a parameter and every
3
3
  // part of it — a column's `on delete` rule included — crosses the seam by shape or not at all.
4
4
 
5
+ import type { IndexMethod } from './index-method';
6
+
5
7
  /** Structurally assignment-compatible with `@ultimat3/entity`'s `ColumnDescription`. */
6
8
  export interface ColumnDescriptionLike {
7
9
  readonly property: string;
@@ -37,6 +39,13 @@ export interface IndexDescriptionLike {
37
39
  readonly where: string | null;
38
40
  /** `null` is Postgres' own default (`asc`), never written out. */
39
41
  readonly order: 'asc' | 'desc' | null;
42
+ /**
43
+ * The access method. Absent is `btree`, which is Postgres' own default and what every index
44
+ * declared before this field existed is — so a description written without it still satisfies
45
+ * the shape, exactly as `ColumnDescriptionLike.onDelete` does. `@ultimat3/entity` (tier 2) is
46
+ * the declarer; this package cannot import it, so the method crosses the seam structurally.
47
+ */
48
+ readonly using?: IndexMethod | undefined;
40
49
  }
41
50
 
42
51
  /** Structurally assignment-compatible with `@ultimat3/entity`'s `EntityDescription`. */
package/src/errors.ts CHANGED
@@ -426,6 +426,22 @@ export const multipleStatements = (statement: string, count: number): DbError =>
426
426
  meta: { count },
427
427
  });
428
428
 
429
+ /**
430
+ * An index method that is not one of the set. `X_SQL_UNSAFE` for the reason `isolationLevelInvalid`
431
+ * uses it: `create index … using <method>` takes no parameters, so the method is SPLICED into the
432
+ * statement text, and an operand TypeScript never saw is an injection rather than a typo — the
433
+ * identical hole a `columnName` built from an unvalidated `meta.name` carried into `create table`.
434
+ *
435
+ * `describeValue`, never the value, for the reason that one gives: this cause is folded into a
436
+ * problem document and a log line.
437
+ */
438
+ export const indexMethodInvalid = (received: unknown): DbError =>
439
+ new DbError({
440
+ code: 'X_SQL_UNSAFE',
441
+ cause: `an index method must be 'btree' or 'gin'; got ${describeValue(received)}`,
442
+ fix: "indexes: [{ on: ['tags'], using: 'gin' }] # or leave it out for the btree default",
443
+ });
444
+
429
445
  export const branchExists = (branch: string): DbError =>
430
446
  new DbError({
431
447
  code: 'X_BRANCH_EXISTS',
package/src/generate.ts CHANGED
@@ -13,6 +13,7 @@ import type {
13
13
  } from './entity-shape';
14
14
  import { migrationIrreversible } from './errors';
15
15
  import { type ConstraintPlans, foreignKeyPlan, foreignKeysOf, type Plan } from './foreign-key-plan';
16
+ import { declaredMethod, indexMethodOf, indexMethodSql } from './index-method';
16
17
  import {
17
18
  type ColumnDescription,
18
19
  findTable,
@@ -121,6 +122,10 @@ export function snapshotOf(entities: readonly EntityDescriptionLike[]): SchemaDe
121
122
  primary: false,
122
123
  where: index.where,
123
124
  order: index.order,
125
+ // Only when one was declared. Writing `using: 'btree'` out for every index would rewrite
126
+ // every sidecar in every app on the next `x db gen` — a diff on every file for a fact
127
+ // that was already true, which `indexMethodOf` reads out of the absence anyway.
128
+ ...(index.using === undefined ? {} : { using: index.using }),
124
129
  })),
125
130
  foreignKeys: foreignKeysOf(entity),
126
131
  };
@@ -154,11 +159,30 @@ function createIndex(table: string, index: IndexDescriptionLike): string {
154
159
  `index "${index.name}" on "${table}" names no columns`,
155
160
  `indexes: [{ on: ['<column>'] }] # name the columns in the entity(), then x db gen`,
156
161
  );
162
+ const method = index.using ?? 'btree';
163
+ // Two rules Postgres has and a declaration can break, refused here rather than at migrate time:
164
+ // GIN supports neither a unique index nor an ASC/DESC option, and either one reaches the server
165
+ // as a syntax error inside `ROLE=migrate` — a release phase that fails with the server's words
166
+ // and none of the entity's. `X_INVARIANT` for the reason `createIndex` already uses it on an
167
+ // index naming no columns: a declaration this build cannot honour is refused, never reinterpreted.
168
+ assert(
169
+ method === 'btree' || !index.unique,
170
+ `index "${index.name}" on "${table}" is unique and ${method}; Postgres has no unique ${method} index`,
171
+ `indexes: [{ on: ['<column>'], using: '${method}' }] # drop unique, or drop using`,
172
+ );
173
+ assert(
174
+ method === 'btree' || index.order === null,
175
+ `index "${index.name}" on "${table}" is ${method} and ${index.order}; only a btree orders its keys`,
176
+ `indexes: [{ on: ['<column>'], using: '${method}' }] # drop order, or drop using`,
177
+ );
157
178
  const kind = index.unique ? 'create unique index' : 'create index';
158
179
  const direction = index.order === null ? '' : ` ${index.order}`;
159
180
  const columns = index.columns.map((column) => `"${column}"${direction}`).join(', ');
160
181
  const predicate = index.where === null ? '' : ` where (${index.where})`;
161
- return `${kind} "${index.name}" on "${table}" (${columns})${predicate};`;
182
+ // Re-derived from the closed set, never spliced: `indexMethodSql` answers `''` for a btree, so
183
+ // an index that declared no method emits the statement this generator always emitted, byte for
184
+ // byte, and one that declared a method Postgres does not have is refused instead of built.
185
+ return `${kind} "${index.name}" on "${table}"${indexMethodSql(method)} (${columns})${predicate};`;
162
186
  }
163
187
 
164
188
  /**
@@ -185,7 +209,13 @@ function retypeColumn(
185
209
 
186
210
  /** The parts of an index Postgres cannot alter in place — every one of them is a rebuild. */
187
211
  function indexShape(index: IndexDescriptionLike | IndexDescription): string {
188
- return JSON.stringify([[...index.columns], index.unique, index.where, index.order ?? null]);
212
+ return JSON.stringify([
213
+ [...index.columns],
214
+ index.unique,
215
+ index.where,
216
+ index.order ?? null,
217
+ indexMethodOf(index),
218
+ ]);
189
219
  }
190
220
 
191
221
  /**
@@ -216,6 +246,10 @@ function redefineIndex(
216
246
  unique: recorded.unique,
217
247
  where: recorded.where,
218
248
  order: recorded.order,
249
+ // `declaredMethod`, never a cast: `recorded` is a snapshot's, typed open because the catalog
250
+ // shares the shape, and a method this generator cannot emit must refuse rather than be
251
+ // rebuilt as a btree — a `down` that recreates the wrong structure is worse than none.
252
+ ...(recorded.using === undefined ? {} : { using: declaredMethod(recorded.using) }),
219
253
  }),
220
254
  `drop index "${index.name}";`,
221
255
  );
@@ -0,0 +1,75 @@
1
+ // Single responsibility: an index's access method — the closed set an entity may declare, the one
2
+ // normalisation both sides of a comparison pass through, and the DDL fragment. Its own file for the
3
+ // reason `foreign-key.ts` holds `onDeleteRule`: a generator and a detector that disagreed about
4
+ // what "the default" is would report drift on a database that is exactly right.
5
+
6
+ import { indexMethodInvalid } from './errors';
7
+
8
+ /**
9
+ * The methods an entity may declare. Two members, deliberately: `btree` is what every index has
10
+ * always been, and `gin` is the one with a caller — `@>` / `<@` / `&&` / `?` on a `json()` or
11
+ * `arrayOf()` column is a sequential scan without it.
12
+ *
13
+ * `gist`, `brin`, `hash` and `spgist` are legitimate Postgres methods and are **not** here, because
14
+ * nothing declares one and each brings a rule of its own that would have to be enforced with no
15
+ * caller to test it — `hash` and `brin` cannot be unique, `gist` needs `btree_gist` to be, and none
16
+ * of the three accepts `asc`/`desc`. Adding a member later is additive; shipping four that nobody
17
+ * uses is four ways for a first caller to be silently wrong. A method the catalog reports and this
18
+ * set does not carry is still READ and still compared — see `indexMethodOf`.
19
+ */
20
+ export const INDEX_METHODS = ['btree', 'gin'] as const;
21
+
22
+ export type IndexMethod = (typeof INDEX_METHODS)[number];
23
+
24
+ export function isIndexMethod(value: unknown): value is IndexMethod {
25
+ return typeof value === 'string' && (INDEX_METHODS as readonly string[]).includes(value);
26
+ }
27
+
28
+ /**
29
+ * What method this index is on, whichever side it came from. `undefined` is `btree` — Postgres'
30
+ * own default, which nothing writes out, which every index created before this existed is, and
31
+ * which is therefore what a snapshot recorded before it carried the field at all.
32
+ *
33
+ * The CATALOG's answer is passed through verbatim, `gist` and an extension's own access method
34
+ * included: the live side is whatever `pg_am` said, and folding an unknown name into `btree` would
35
+ * hide exactly the difference an operator needs to see.
36
+ */
37
+ export function indexMethodOf(index: { readonly using?: string | undefined }): string {
38
+ return index.using ?? 'btree';
39
+ }
40
+
41
+ /**
42
+ * The closed-set reading of a method that arrived on the OPEN side — a catalog row or a snapshot
43
+ * this generator did not write. `undefined` for absent, and a **refusal** for anything the set does
44
+ * not carry, never a silent fall back to `btree`: the one caller is `redefineIndex`, whose `down`
45
+ * recreates the index a previous migration recorded, and a `gist` quietly rebuilt as a btree is a
46
+ * rollback that leaves the database in a state no migration describes.
47
+ */
48
+ export function declaredMethod(using: string | undefined): IndexMethod | undefined {
49
+ if (using === undefined) return undefined;
50
+ if (!isIndexMethod(using)) throw indexMethodInvalid(using);
51
+ return using;
52
+ }
53
+
54
+ /**
55
+ * The clause, or `''` for a btree — so an index that declared nothing emits the statement it always
56
+ * did, byte for byte.
57
+ *
58
+ * The literal is **re-derived from the set, never spliced from the input**, the same shape
59
+ * `isolationMode` uses for `BEGIN`. The type is not the guard: this value reaches `create index …`
60
+ * as text from an entity declaration, a config or a generator, and `using ${method}` on an operand
61
+ * TypeScript never saw is the identical hole to the one `columnName` carried — a name that closed
62
+ * the parenthesis and opened a second command.
63
+ */
64
+ export function indexMethodSql(method: IndexMethod): string {
65
+ switch (method) {
66
+ case 'btree':
67
+ return '';
68
+ case 'gin':
69
+ return ' using gin';
70
+ default: {
71
+ const unhandled: never = method;
72
+ throw indexMethodInvalid(unhandled);
73
+ }
74
+ }
75
+ }
package/src/index.ts CHANGED
@@ -32,6 +32,7 @@ export {
32
32
  poolProfileFor,
33
33
  setDbClient,
34
34
  } from './client';
35
+ export { defaultClient, REPLICA_URL_ENV } from './default-client';
35
36
  export type { DestructiveKind, DestructiveStatement } from './destructive';
36
37
  export {
37
38
  DESTRUCTIVE_CAUSE,
@@ -86,6 +87,14 @@ export type { RecordedStatement, RecordingClient, StubResponse } from './fake';
86
87
  export { createRecordingClient } from './fake';
87
88
  export type { GeneratedMigration, GenerateOptions } from './generate';
88
89
  export { generateMigration, migrationStamp, slugify, snapshotOf } from './generate';
90
+ export type { IndexMethod } from './index-method';
91
+ export {
92
+ declaredMethod,
93
+ INDEX_METHODS,
94
+ indexMethodOf,
95
+ indexMethodSql,
96
+ isIndexMethod,
97
+ } from './index-method';
89
98
  export type {
90
99
  ColumnDescription,
91
100
  ForeignKeyDescription,
@@ -142,6 +151,15 @@ export type { ReadOnlyQueryOptions, ReadOnlyQueryResult } from './readonly-query
142
151
  export { READONLY_TIMEOUT_MS, readOnlyQuery } from './readonly-query';
143
152
  export type { ReadOnlyRoleOptions } from './readonly-role';
144
153
  export { ensureReadOnlyRole, grantReadOnlySql, READONLY_ROLE } from './readonly-role';
154
+ export type {
155
+ ReplicaStats,
156
+ ReplicatedClient,
157
+ ReplicatedClientOptions,
158
+ } from './replica-client';
159
+ export { BREAKER_COOLDOWN_MS, BREAKER_FAILURES, replicatedClient } from './replica-client';
160
+ export { type DbNode, isPlainRead } from './replica-route';
161
+ export type { ReplicaScope } from './replica-scope';
162
+ export { markScopeWrote, replicaScope, withReplicaReads } from './replica-scope';
145
163
  export { snapshotJson } from './snapshot-json';
146
164
  export { parseSnapshot } from './snapshot-parse';
147
165
  export type { SqlFragment } from './sql';
package/src/introspect.ts CHANGED
@@ -1,8 +1,9 @@
1
1
  // Single responsibility: read the live schema out of `information_schema` / `pg_catalog` into a
2
- // plain, sortable description. Three consumers depend on this exact shape: drift detection, the
3
- // generated admin dashboard's schema view, and the MCP `schema.describe` tool. Keep it JSON-safe
4
- // and deterministically ordered it is diffed and it is serialised.
2
+ // plain, sortable description app tables only, never a relation an extension owns and never a
3
+ // view (`app-relation.ts`). `checkDrift` is its one shipped consumer, so what it omits a deploy
4
+ // cannot refuse over. Keep it JSON-safe and deterministically ordered: it is diffed and serialised.
5
5
 
6
+ import { nonAppRelations } from './app-relation';
6
7
  import { type DbClient, db } from './client';
7
8
  import { sql } from './sql';
8
9
 
@@ -29,6 +30,14 @@ export interface IndexDescription {
29
30
  readonly where: string | null;
30
31
  /** `desc` only when every key column is descending; `null` is Postgres' own default. */
31
32
  readonly order: 'asc' | 'desc' | null;
33
+ /**
34
+ * The access method as `pg_am` names it — `btree`, `gin`, `gist`, or an extension's own. Read
35
+ * OPEN rather than as the closed set an entity may declare: the live side is whatever the
36
+ * catalog said, and a `gist` folded into `btree` is the difference drift exists to report.
37
+ * Absent means `btree` — a snapshot written before this field existed carries no method, and
38
+ * every index it recorded was one. `indexMethodOf()` is the one reader of that rule.
39
+ */
40
+ readonly using?: string | undefined;
32
41
  }
33
42
 
34
43
  export interface ForeignKeyDescription {
@@ -55,7 +64,13 @@ export interface SchemaDescription {
55
64
  export interface IntrospectOptions {
56
65
  readonly client?: DbClient | undefined;
57
66
  readonly schema?: string | undefined;
58
- /** The ledger is framework bookkeeping, not user schema — excluded so it never reads as drift. */
67
+ /**
68
+ * The ledger is framework bookkeeping, not user schema — excluded so it never reads as drift.
69
+ *
70
+ * Replaces the default (`['x_migrations']`) rather than adding to it. It never replaces the set
71
+ * `nonAppRelations()` derives: an extension's relations are not app schema in any deployment,
72
+ * so that exclusion is not a caller's to switch off.
73
+ */
59
74
  readonly exclude?: readonly string[] | undefined;
60
75
  }
61
76
 
@@ -76,6 +91,7 @@ interface IndexRow {
76
91
  readonly columns: readonly string[];
77
92
  readonly predicate: string | null;
78
93
  readonly descending: boolean;
94
+ readonly method?: string | undefined;
79
95
  }
80
96
 
81
97
  interface ForeignKeyRow {
@@ -92,7 +108,13 @@ const byName = (a: { name: string }, b: { name: string }): number => (a.name < b
92
108
  export async function introspect(options: IntrospectOptions = {}): Promise<SchemaDescription> {
93
109
  const client = options.client ?? db();
94
110
  const schema = options.schema ?? 'public';
95
- const excluded = options.exclude ?? ['x_migrations'];
111
+ // Asked first, and unconditionally: everything below reads `information_schema`, which admits a
112
+ // view and an extension's own tables alongside the app's. Merged into `excluded` rather than
113
+ // filtered afterwards so one deny list feeds the whole fold.
114
+ const excluded = [
115
+ ...(options.exclude ?? ['x_migrations']),
116
+ ...(await nonAppRelations(client, schema)),
117
+ ];
96
118
 
97
119
  const columns = await client.query<ColumnRow>(sql`
98
120
  select table_name, column_name, data_type, is_nullable, column_default, ordinal_position
@@ -112,16 +134,18 @@ export async function introspect(options: IntrospectOptions = {}): Promise<Schem
112
134
  ix.indisunique as is_unique,
113
135
  ix.indisprimary as is_primary,
114
136
  pg_get_expr(ix.indpred, ix.indrelid) as predicate,
137
+ am.amname as method,
115
138
  array_agg(a.attname order by k.ord) as columns,
116
139
  bool_and((ix.indoption[k.ord - 1] & 1) = 1) as descending
117
140
  from pg_class t
118
141
  join pg_namespace n on n.oid = t.relnamespace
119
142
  join pg_index ix on ix.indrelid = t.oid
120
143
  join pg_class i on i.oid = ix.indexrelid
144
+ join pg_am am on am.oid = i.relam
121
145
  cross join lateral unnest(ix.indkey::smallint[]) with ordinality as k(attnum, ord)
122
146
  join pg_attribute a on a.attrelid = t.oid and a.attnum = k.attnum
123
147
  where n.nspname = ${schema} and t.relkind = 'r' and k.ord <= ix.indnkeyatts
124
- group by t.relname, i.relname, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid
148
+ group by t.relname, i.relname, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid, am.amname
125
149
  order by t.relname, i.relname
126
150
  `);
127
151
 
@@ -175,6 +199,9 @@ export function buildSchema(
175
199
  primary: row.is_primary,
176
200
  where: row.predicate,
177
201
  order: row.descending ? ('desc' as const) : null,
202
+ // `exactOptionalPropertyTypes`: a row with no method is a stub's, and absent must stay
203
+ // absent rather than becoming an explicit `undefined` a strict comparison can see.
204
+ ...(row.method === undefined ? {} : { using: row.method }),
178
205
  }))
179
206
  .sort(byName);
180
207
  return {
@@ -0,0 +1,137 @@
1
+ // Single responsibility: one `DbClient` over a primary and a read replica. It decides nothing about
2
+ // SQL (`replica-route.ts`) and nothing about scope (`replica-scope.ts`) — what lives here is which
3
+ // handle a statement is sent on, what happens when the replica will not answer, and the counters
4
+ // that make both visible to a test that cannot scrape a metrics endpoint.
5
+
6
+ import { type Clock, logger, renderThrowable, systemClock } from '@ultimat3/core';
7
+ import { type DbClient, type DbConnection, isReservable, type ReservableClient } from './client';
8
+ import { isPlainRead } from './replica-route';
9
+ import { markScopeWrote, replicaScope } from './replica-scope';
10
+ import type { SqlFragment } from './sql';
11
+
12
+ export interface ReplicaStats {
13
+ /** Statements the replica answered. */
14
+ readonly replica: number;
15
+ /** Statements sent to the primary, fallbacks included. */
16
+ readonly primary: number;
17
+ /** Replica attempts that failed and were re-run on the primary. */
18
+ readonly fallbacks: number;
19
+ /** True while the breaker is parked and every read is going to the primary. */
20
+ readonly parked: boolean;
21
+ }
22
+
23
+ export interface ReplicatedClientOptions {
24
+ /** Consecutive replica failures before it is parked. */
25
+ readonly breakerFailures?: number | undefined;
26
+ readonly breakerCooldownMs?: number | undefined;
27
+ /** Injection seam; production passes neither. */
28
+ readonly clock?: Clock | undefined;
29
+ }
30
+
31
+ export interface ReplicatedClient extends DbClient {
32
+ readonly stats: ReplicaStats;
33
+ }
34
+
35
+ /** Three in a row, then a ten-second rest — an outage costs 3 doubled reads, not every read. */
36
+ export const BREAKER_FAILURES = 3;
37
+ export const BREAKER_COOLDOWN_MS = 10_000;
38
+
39
+ /**
40
+ * `primary` answers everything that is not provably a replica-safe read inside an open
41
+ * `withReplicaReads` scope. Reservations are ALWAYS the primary's: `withTransaction` pins a
42
+ * connection through `reserve()`, and a BEGIN that landed on a standby is not a transaction, it is
43
+ * `25006` on the first write inside it.
44
+ *
45
+ * `reserve` is present only when the primary has one, so `isReservable()` keeps answering about the
46
+ * database rather than about this wrapper — a wrapper that always exposed `reserve` would make
47
+ * `runRoot` pin a connection out of a client that cannot pin, and one that never exposed it would
48
+ * make `runRoot` run BEGIN, the statements and COMMIT on three different pooled connections.
49
+ */
50
+ export function replicatedClient(
51
+ primary: DbClient,
52
+ replica: DbClient,
53
+ options: ReplicatedClientOptions = {},
54
+ ): ReplicatedClient {
55
+ const clock = options.clock ?? systemClock;
56
+ const limit = options.breakerFailures ?? BREAKER_FAILURES;
57
+ const cooldown = options.breakerCooldownMs ?? BREAKER_COOLDOWN_MS;
58
+ let replicaCount = 0;
59
+ let primaryCount = 0;
60
+ let fallbackCount = 0;
61
+ let consecutiveFailures = 0;
62
+ let parkedUntil = 0;
63
+
64
+ /** Monotonic, never wall clock: a leap second or an NTP step must not un-park the breaker. */
65
+ function parked(): boolean {
66
+ return clock.monotonic() < parkedUntil;
67
+ }
68
+
69
+ function nodeIsReplica(text: string): boolean {
70
+ const scope = replicaScope();
71
+ // No scope: nobody declared these reads replica-safe, so this is a single-pool client.
72
+ if (scope === undefined) return false;
73
+ if (!isPlainRead(text)) {
74
+ markScopeWrote();
75
+ return false;
76
+ }
77
+ // Read-your-writes, and the reason the flag is on a mutable scope value rather than computed
78
+ // per statement: once this scope has written, every later read in it is the primary's. A
79
+ // replica is behind by an unbounded amount — streaming lag is not a number this tier can know
80
+ // — so "the row I just inserted" is the one question a standby is guaranteed to answer wrong.
81
+ if (scope.wrote) return false;
82
+ return !parked();
83
+ }
84
+
85
+ async function send<T>(fragment: SqlFragment, on: (client: DbClient) => Promise<T>): Promise<T> {
86
+ if (!nodeIsReplica(fragment.text)) {
87
+ primaryCount += 1;
88
+ return on(primary);
89
+ }
90
+ try {
91
+ const answer = await on(replica);
92
+ replicaCount += 1;
93
+ consecutiveFailures = 0;
94
+ return answer;
95
+ } catch (error) {
96
+ // Re-running is exactly-once, not at-least-once: only `isPlainRead` statements reach here,
97
+ // and a statement a standby refused (`25006`) never executed. A replica outage therefore
98
+ // costs latency and never an answer — which is the whole point, since a read replica is a
99
+ // capacity tier and must not become a new way for the app to be down.
100
+ consecutiveFailures += 1;
101
+ if (consecutiveFailures >= limit) parkedUntil = clock.monotonic() + cooldown;
102
+ fallbackCount += 1;
103
+ // `renderThrowable`, never `${error}`: a driver error's `message` getter is app code.
104
+ logger.warn('db.replica_fallback', {
105
+ error: renderThrowable(error),
106
+ consecutiveFailures,
107
+ parked: parked(),
108
+ });
109
+ primaryCount += 1;
110
+ return on(primary);
111
+ }
112
+ }
113
+
114
+ const base: ReplicatedClient = {
115
+ get stats(): ReplicaStats {
116
+ return {
117
+ replica: replicaCount,
118
+ primary: primaryCount,
119
+ fallbacks: fallbackCount,
120
+ parked: parked(),
121
+ };
122
+ },
123
+ query: <T>(fragment: SqlFragment) => send(fragment, (client) => client.query<T>(fragment)),
124
+ one: <T>(fragment: SqlFragment) => send(fragment, (client) => client.one<T>(fragment)),
125
+ execute: (fragment: SqlFragment) => send(fragment, (client) => client.execute(fragment)),
126
+ };
127
+
128
+ if (!isReservable(primary)) return base;
129
+ const reservable: ReplicatedClient & ReservableClient = {
130
+ ...base,
131
+ get stats(): ReplicaStats {
132
+ return base.stats;
133
+ },
134
+ reserve: (): Promise<DbConnection> => primary.reserve(),
135
+ };
136
+ return reservable;
137
+ }
@@ -0,0 +1,60 @@
1
+ // Single responsibility: may THIS statement be served by a replica. An allow-list with a refusal
2
+ // list under it, whose default is the primary — the opposite bias to the `readonly.ts` lexer this
3
+ // package deleted, and the reason that deletion does not forbid this file.
4
+
5
+ import { statementVerb } from './statement-shape';
6
+
7
+ export type DbNode = 'primary' | 'replica';
8
+
9
+ /**
10
+ * The only verbs a replica may be offered. `with` is here because a CTE read is the shape half the
11
+ * framework's paginated queries take, and `WRITE_WORD` below is what tells `with … select` from
12
+ * `with … update … returning` — which `statementKind()` calls a read, and is exactly why that
13
+ * function is not the authority here.
14
+ */
15
+ const READ_VERBS: ReadonlySet<string> = new Set(['select', 'table', 'values']);
16
+ const CTE_VERB = 'with';
17
+
18
+ /**
19
+ * A word that disqualifies the whole statement. Matched with word boundaries against the raw
20
+ * lowercased text — NOT against `stripSqlNoise`'d text, deliberately: a `;`-in-a-literal is data
21
+ * and must not split a statement, but a `'update'` in a literal costing one read its replica is a
22
+ * false positive on the SAFE side, and blanking every statement on the hot path to buy back that
23
+ * one read is a cost axiom 6 refuses. `share` covers `for share` and `for key share`; `update`
24
+ * covers `for update` and `for no key update`; `into` covers `select … into`, which creates a table.
25
+ */
26
+ const WRITE_WORD =
27
+ /\b(?:insert|update|delete|merge|truncate|copy|create|drop|alter|grant|revoke|call|do|lock|share|refresh|reindex|vacuum|analyze|set|reset|begin|commit|rollback|savepoint|into)\b/;
28
+
29
+ /**
30
+ * Function names a word boundary cannot reach — `pg_advisory_lock` has a `_` before `advisory`, so
31
+ * `\badvisory\b` never matches it. Every one of these either writes or mutates session state that
32
+ * belongs to whichever backend ran it, and a standby accepts them silently rather than answering
33
+ * `25006`, so the server's own refusal cannot be the safety net here the way it is for a real write.
34
+ */
35
+ const UNSAFE_CALLS: readonly string[] = [
36
+ 'nextval',
37
+ 'setval',
38
+ 'set_config',
39
+ 'advisory',
40
+ 'dblink',
41
+ 'lo_import',
42
+ 'lo_export',
43
+ 'pg_export_snapshot',
44
+ 'pg_replication',
45
+ 'pg_create',
46
+ ];
47
+
48
+ /**
49
+ * Provably a plain read, or `false`. Never "probably": everything this cannot vouch for is the
50
+ * primary's, so a statement shape nobody anticipated costs a replica opportunity and never a wrong
51
+ * answer. That inversion is the whole difference from `readonly.ts`, whose 22-word deny-list read
52
+ * `select pg_sleep(60)` as safe because the default was permission.
53
+ */
54
+ export function isPlainRead(text: string): boolean {
55
+ const lowered = text.toLowerCase();
56
+ const verb = statementVerb(lowered);
57
+ if (!READ_VERBS.has(verb) && verb !== CTE_VERB) return false;
58
+ if (WRITE_WORD.test(lowered)) return false;
59
+ return !UNSAFE_CALLS.some((call) => lowered.includes(call));
60
+ }
@@ -0,0 +1,54 @@
1
+ // Single responsibility: the scope inside which a read may be served by a replica, and the one bit
2
+ // that closes read-your-writes — has this scope written yet. A mutable value on an async context,
3
+ // the same shape `transaction.ts` uses for `TxState.live`, so a write at any depth and across any
4
+ // `await` is seen by every later read in the same scope.
5
+
6
+ import { asyncContext } from '@ultimat3/core';
7
+
8
+ /**
9
+ * Deliberately mutable, and deliberately not `readonly`. The whole mechanism is that a write ten
10
+ * frames and three `await`s below the scope's opener flips this, and the read after it sees it —
11
+ * a fresh object per statement could not carry that, and threading a parameter would be the same
12
+ * fact written at every call site, with every path an author forgot serving a stale row.
13
+ */
14
+ export interface ReplicaScope {
15
+ wrote: boolean;
16
+ }
17
+
18
+ const scope = asyncContext<ReplicaScope>('the replica read scope');
19
+
20
+ /**
21
+ * Declare that reads inside `fn` may be served by a replica — until `fn` writes, after which every
22
+ * read in it is the primary's for the rest of the scope.
23
+ *
24
+ * **Opt-in, and that is the safety argument, not an ergonomic one.** `packages/db` cannot see a
25
+ * request boundary: `@ultimat3/http`'s pipeline opens the `Ctx` and nothing tells this tier when a
26
+ * request ended, so a write-marker keyed on `Ctx.requestId` would be a `Map` that only grows —
27
+ * ~100 bytes per request, forever — and any eviction policy that forgets a request that WROTE
28
+ * serves it a stale row, which is worse than the capacity problem replicas exist to solve. With no
29
+ * scope open nothing routes and the client is byte-identical to a single-pool one, so the failure
30
+ * mode of "nobody opened one" is today's behaviour rather than a wrong answer.
31
+ *
32
+ * Nesting is one scope, not two: an inner `withReplicaReads` inside a scope that has already
33
+ * written must not un-write it. The innermost call reuses the store it finds.
34
+ */
35
+ export function withReplicaReads<T>(fn: () => T): T {
36
+ const open = scope.get();
37
+ if (open !== undefined) return fn();
38
+ return scope.run({ wrote: false }, fn);
39
+ }
40
+
41
+ /** The scope in flight, or `undefined` — which is every caller that never opened one. */
42
+ export function replicaScope(): ReplicaScope | undefined {
43
+ return scope.get();
44
+ }
45
+
46
+ /**
47
+ * Record that this scope has written. Called for every statement that is not provably a plain read
48
+ * — including `begin`, a `set`, and anything the router could not classify — because the direction
49
+ * that is safe to be wrong in is "assume it wrote". A no-op outside a scope, where nothing routes.
50
+ */
51
+ export function markScopeWrote(): void {
52
+ const open = scope.get();
53
+ if (open !== undefined) open.wrote = true;
54
+ }
@@ -36,13 +36,19 @@ function column(value: unknown): ColumnDescription | undefined {
36
36
 
37
37
  function index(value: unknown): IndexDescription | undefined {
38
38
  if (!isRow(value)) return undefined;
39
- const { name, columns, unique, primary, where, order: direction } = value;
39
+ const { name, columns, unique, primary, where, order: direction, using } = value;
40
40
  if (!str(name) || !strings(columns) || !bool(unique) || !bool(primary)) return undefined;
41
41
  // Written by 1.2.0 onwards. A sidecar from before it carries neither, and the total, ascending
42
42
  // reading is what that generation actually emitted — so an older file stays readable rather
43
43
  // than being discarded whole, which would refuse to generate against every existing app.
44
44
  if (!(where === undefined || nullableStr(where))) return undefined;
45
45
  if (!(direction === undefined || order(direction))) return undefined;
46
+ // Any string, not the closed set. The live side of this type is the CATALOG's, which answers
47
+ // `gist` and an extension's own access method, and a sidecar recording one must round-trip so
48
+ // drift can report it — the refusal belongs at generation, where `declaredMethod` names the
49
+ // method and the fix, not here, where it would discard the whole snapshot without saying which
50
+ // field was wrong. Absent stays absent: `indexMethodOf` reads it as the btree it always was.
51
+ if (!(using === undefined || str(using))) return undefined;
46
52
  return {
47
53
  name,
48
54
  columns,
@@ -50,6 +56,7 @@ function index(value: unknown): IndexDescription | undefined {
50
56
  primary,
51
57
  where: where === undefined ? null : where,
52
58
  order: direction === undefined ? null : direction,
59
+ ...(using === undefined ? {} : { using }),
53
60
  };
54
61
  }
55
62
 
@@ -7,6 +7,7 @@ import type { Random } from '@ultimat3/core';
7
7
  import { assert, asyncContext, nanoid } from '@ultimat3/core';
8
8
  import { baseClient, type DbClient, type DbConnection, isReservable } from './client';
9
9
  import { isolationLevelInvalid, serializationExhausted } from './errors';
10
+ import { markScopeWrote } from './replica-scope';
10
11
  import { raw, type SqlFragment } from './sql';
11
12
  import { isRetryableState } from './sqlstate';
12
13
  import { serializationRetryDelayMs } from './transaction-backoff';
@@ -206,6 +207,11 @@ async function runNested<T>(outer: TxState, fn: (tx: DbTx) => Promise<T>): Promi
206
207
  */
207
208
  async function runRoot<T>(fn: (tx: DbTx) => Promise<T>, options: TransactionOptions): Promise<T> {
208
209
  const client = options.client ?? baseClient();
210
+ // A transaction is assumed to write unless it said otherwise, so every read AFTER it in the same
211
+ // `withReplicaReads` scope is the primary's. The pin below already keeps the transaction's own
212
+ // statements off any replica — this is about the rest of the request, which `replica-client.ts`
213
+ // could not otherwise see: `runRoot` sends through a reserved connection, not through the router.
214
+ if (options.readOnly !== true) markScopeWrote();
209
215
  // A pooled BEGIN that lands on a different physical connection than the statements after it is
210
216
  // not a transaction at all, so a reservable client pins one connection for the whole scope.
211
217
  // Held by a `using` declaration rather than a `finally`, because a `finally` only covers what