@ultimat3/db 11.2.0 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +140 -2
- package/README.md +74 -5
- package/package.json +2 -2
- package/src/app-relation.ts +57 -0
- package/src/client.ts +7 -3
- package/src/default-client.ts +35 -0
- package/src/drift.ts +18 -0
- package/src/entity-shape.ts +9 -0
- package/src/errors.ts +16 -0
- package/src/generate.ts +36 -2
- package/src/index-method.ts +75 -0
- package/src/index.ts +18 -0
- package/src/introspect.ts +33 -6
- package/src/replica-client.ts +137 -0
- package/src/replica-route.ts +60 -0
- package/src/replica-scope.ts +54 -0
- package/src/snapshot-parse.ts +8 -1
- package/src/transaction.ts +6 -0
package/CLAUDE.md
CHANGED
|
@@ -369,6 +369,56 @@ anonymous SQL, never as a `job` statement, and will keep seeing it that way unti
|
|
|
369
369
|
own pair through `driver-pg.ts` the way `postgresRepo` now threads entity's — that is still future
|
|
370
370
|
work, not something this change reaches.
|
|
371
371
|
|
|
372
|
+
**An index's ACCESS METHOD is carried end to end, `As of 2026-08-24`, and it had to land here
|
|
373
|
+
before `@ultimat3/entity` could declare it.** `@>` / `<@` / `&&` / `?` on a `json()` or `arrayOf()`
|
|
374
|
+
column is a sequential scan without a GIN index. `IndexInit.using` on the entity side while this
|
|
375
|
+
package ignored it would emit a **btree for a declared GIN index** — a declared-and-never-wired key,
|
|
376
|
+
which is the defect class this release exists to eliminate and strictly worse than the missing
|
|
377
|
+
capability. So the method reaches all four places or none: `createIndex` emits it, `snapshotOf`
|
|
378
|
+
records it, `indexShape` rebuilds on it, and `compareIndexes` reports it.
|
|
379
|
+
|
|
380
|
+
`index-method.ts` is the vocabulary, its own file for the reason `foreign-key.ts` holds
|
|
381
|
+
`onDeleteRule`: a generator and a detector that disagreed about what "the default" is would report
|
|
382
|
+
drift on a database that is exactly right. Four rules.
|
|
383
|
+
|
|
384
|
+
**The set is closed at two — `btree` and `gin`.** `gist`, `brin`, `hash` and `spgist` are legitimate
|
|
385
|
+
and are deliberately absent: nothing declares one, and each brings a rule that would have to be
|
|
386
|
+
enforced with no caller to test it (`hash` and `brin` cannot be unique, `gist` needs `btree_gist` to
|
|
387
|
+
be, none of the three accepts `asc`/`desc`). Adding a member later is additive; shipping four nobody
|
|
388
|
+
uses is four ways for a first caller to be silently wrong.
|
|
389
|
+
|
|
390
|
+
**Declared is CLOSED, live is OPEN.** `IndexDescriptionLike.using` is `IndexMethod | undefined` —
|
|
391
|
+
what an entity may ask for. `IndexDescription.using` is `string | undefined` — whatever `pg_am`
|
|
392
|
+
answered, `gist` and an extension's own access method included. Folding an unknown catalog name into
|
|
393
|
+
`btree` would hide exactly the difference drift exists to report, so `indexMethodOf` passes the live
|
|
394
|
+
side through verbatim and `declaredMethod` is the one place the open reading is narrowed back — a
|
|
395
|
+
**refusal**, never a silent fall back, because its one caller is `redefineIndex`'s `down` and a
|
|
396
|
+
`gist` quietly rebuilt as a btree is a rollback leaving a state no migration describes.
|
|
397
|
+
|
|
398
|
+
**Absent is `btree`, on both sides, through one function.** `indexMethodOf` is that function.
|
|
399
|
+
Postgres' default is written out by nobody, every index created before this existed is one, and
|
|
400
|
+
every sidecar written before the field is silent about it — so `snapshotOf` records `using` only
|
|
401
|
+
when one was declared. Writing `'btree'` out for every index would rewrite every sidecar in every
|
|
402
|
+
app on the next `x db gen`, a diff on every file for a fact that was already true.
|
|
403
|
+
|
|
404
|
+
**The literal is re-derived from the set, never spliced from the input** — `indexMethodSql` is a
|
|
405
|
+
`switch` whose `default` arm is `never` and throws `indexMethodInvalid` (`X_SQL_UNSAFE`, the code
|
|
406
|
+
`isolationLevelInvalid` and `branchNameInvalid` already use for a value spliced into a statement).
|
|
407
|
+
The type is not the guard: this value arrives from an entity declaration, a config or a hand-edited
|
|
408
|
+
snapshot, and `using ${method}` on an operand TypeScript never saw is the **identical hole** to the
|
|
409
|
+
one `columnName` carried when it was `meta.name ?? snake(property)` with only the second branch
|
|
410
|
+
validated — a name that closed the parenthesis and opened a second command, measured through
|
|
411
|
+
`generateMigration`. `create index "x" on "t" using gin ("c") where (...)` also refuses a unique or
|
|
412
|
+
an ordered GIN through core's `assert` (`X_INVARIANT`), the discipline `createIndex` already applies
|
|
413
|
+
to an index naming no columns: Postgres has neither, and a syntax error inside `ROLE=migrate` fails
|
|
414
|
+
the release phase with the server's words and none of the entity's.
|
|
415
|
+
|
|
416
|
+
`introspect()` reads the method from `pg_am` joined through `pg_class.relam`, and
|
|
417
|
+
`introspect-embedded.test.ts` is where that is pinned — a recording client can pin the SQL text and
|
|
418
|
+
nothing more, and a query that silently returned no method would read as `btree` everywhere and make
|
|
419
|
+
drift blind to the one case `using` exists for. Measured on PGlite: a real `using gin` index reads
|
|
420
|
+
back `gin`, the btree beside it and the primary key's own index read back `btree`.
|
|
421
|
+
|
|
372
422
|
**`generate.ts` reads an index, it never re-derives one.** `EntityDescriptionLike.indexes` carries
|
|
373
423
|
`columns`, `unique`, `where` and `order`, and `createIndex` writes every one of them out. It used to
|
|
374
424
|
carry names alone and `parseIndexName` recovered the column list from the `<table>_<a>_<b>_idx`
|
|
@@ -502,6 +552,62 @@ override — `x_migrations.app_version` and `@ultimat3/jobs`' `x_backfills.app_v
|
|
|
502
552
|
durable columns an operator reads side by side, and `jobs` cannot import this package for the
|
|
503
553
|
answer, so the key has one reader at tier 0 rather than one per writer.
|
|
504
554
|
|
|
555
|
+
**Read replicas are opt-in twice, and the second opt-in is the correctness argument, `As of
|
|
556
|
+
2026-08-24`.** A replica pool exists when `DATABASE_REPLICA_URL` names one (`default-client.ts`);
|
|
557
|
+
a read is *offered* to it only inside `withReplicaReads(fn)` (`replica-scope.ts`). With no scope
|
|
558
|
+
open nothing routes and the client is byte-identical to the single-pool one it has always been —
|
|
559
|
+
which is what makes "nobody adopted it yet" today's behaviour rather than a wrong answer.
|
|
560
|
+
|
|
561
|
+
The scope is what closes **read-your-writes**, and the reason it is a scope and not a request id is
|
|
562
|
+
worth writing down because the request id is the obvious answer and it does not work. `Ctx.requestId`
|
|
563
|
+
IS reachable from here — `@ultimat3/http`'s pipeline opens `runWithContext` around every request
|
|
564
|
+
(`packages/http/src/pipeline.ts`), `withChildContext` may not change the id, and `tryUseContext()` is
|
|
565
|
+
tier 0 — but nothing tells tier 1 when a request ENDED. A `Map<requestId, wrote>` therefore only
|
|
566
|
+
grows, ~100 bytes a request forever, and every eviction policy that forgets a request which WROTE
|
|
567
|
+
serves it a stale row on its next read. That is a data-correctness bug strictly worse than the
|
|
568
|
+
capacity problem replicas exist to solve, so the marker lives on a mutable value on an async context
|
|
569
|
+
(`ReplicaScope.wrote`, the same shape as `TxState.live`) whose lifetime somebody else already owns.
|
|
570
|
+
|
|
571
|
+
**`withTransaction` is on the primary structurally, not by rule.** `runRoot` pins a connection
|
|
572
|
+
through `reserve()`, and `replicatedClient` delegates `reserve()` to the primary and exposes it only
|
|
573
|
+
when the primary has one — so BEGIN, every statement and COMMIT are one connection on one server.
|
|
574
|
+
`isReservable` therefore has to keep answering about the DATABASE and not about the wrapper: a
|
|
575
|
+
wrapper that always exposed `reserve` makes `runRoot` pin a client that cannot pin, and one that
|
|
576
|
+
never exposed it makes `runRoot` run BEGIN, the body and COMMIT on three different pooled
|
|
577
|
+
connections. What `runRoot` adds is one line — `markScopeWrote()` unless `readOnly: true` — because
|
|
578
|
+
its statements go through a reservation and never through the router, so the scope could not
|
|
579
|
+
otherwise see that the request has written.
|
|
580
|
+
|
|
581
|
+
**`isPlainRead` is an allow-list, and that inversion is why it is not the lexer this file forbids.**
|
|
582
|
+
`readonly.ts` was deleted for defaulting to PERMISSION: a 22-word deny-list that read
|
|
583
|
+
`select pg_sleep(60)` as safe. This one defaults to the primary — a statement shape nobody
|
|
584
|
+
anticipated costs a replica opportunity and never an answer. **`statementKind()` is not the
|
|
585
|
+
authority and must not become it**: it calls `with … update … returning` a read, which is right for
|
|
586
|
+
an N+1 report and catastrophic for a routing decision, and `replica-route.test.ts` asserts the
|
|
587
|
+
disagreement so the two can never be collapsed. Three refusals earn their line — a locking read
|
|
588
|
+
(`for update`/`for share`; a standby cannot take the row lock), `select … into` (it creates a
|
|
589
|
+
table), and the functions a word boundary cannot reach (`pg_advisory_lock`, `set_config`,
|
|
590
|
+
`nextval`), which a standby ANSWERS rather than refusing, so the server cannot be the safety net for
|
|
591
|
+
those the way it is for a real write.
|
|
592
|
+
|
|
593
|
+
**A misroute fails loudly and repairs itself; a replica outage costs latency and never an answer.**
|
|
594
|
+
A statement a standby refuses (`25006`) never executed, and only `isPlainRead` statements are ever
|
|
595
|
+
sent there, so re-running one on the primary is exactly-once rather than at-least-once — which is
|
|
596
|
+
what makes the blanket fallback in `replica-client.ts` safe. The breaker is what stops that from
|
|
597
|
+
doubling every read during an outage: three consecutive failures park the replica for ten seconds,
|
|
598
|
+
counted on `Clock.monotonic()` so an NTP step cannot un-park it. `ReplicaStats` is exposed on the
|
|
599
|
+
client for a test that cannot scrape, the same reason `@ultimat3/realtime` exposes
|
|
600
|
+
`droppedChannelFrames`, and each fallback logs `db.replica_fallback` with `renderThrowable(error)`.
|
|
601
|
+
|
|
602
|
+
**The URL must name a read-only standby**, and nothing here can check it. The `25006` refusal is the
|
|
603
|
+
whole safety net under a text classifier that cannot be complete; pointed at a writable node, a
|
|
604
|
+
misroute becomes a write on the wrong server with nothing anywhere to report it.
|
|
605
|
+
|
|
606
|
+
**Nothing opens `withReplicaReads` per request yet.** The scope, the client and the wiring are tier
|
|
607
|
+
1 and land here first; the adopter is one call in `@ultimat3/http`'s pipeline (or an app's own
|
|
608
|
+
handler), and until it exists no production traffic is routed. That is the tier rule working —
|
|
609
|
+
lowest tier first, consumers after — not an omission.
|
|
610
|
+
|
|
505
611
|
`checkDrift()` is the **post-migrate verification** and the only drift question that needs a
|
|
506
612
|
database: the live catalog against the ledger the run just wrote. It is asked where a connection is
|
|
507
613
|
open — `@ultimat3/cli`'s `runMigrations`, which is `x db migrate`, `x db reset` and `ROLE=migrate`
|
|
@@ -548,7 +654,7 @@ The `fix:` is the `alter table … set not null` itself and deliberately not `x
|
|
|
548
654
|
never emitted one and would answer with an empty migration.
|
|
549
655
|
|
|
550
656
|
`compareTable` judges **declared** indexes: one the migrations name and the catalog does not hold is
|
|
551
|
-
`missing-index`, and one whose column list or uniqueness moved is `changed-index` — which is what
|
|
657
|
+
`missing-index`, and one whose access method, column list or uniqueness moved is `changed-index` — which is what
|
|
552
658
|
catches a composite index rebuilt with its columns the other way round while the column diff said
|
|
553
659
|
`ok: true`. A live index no snapshot names is deliberately **not** reported: Postgres creates one for
|
|
554
660
|
every primary key and every unique constraint, so counting those is eight findings against a correct
|
|
@@ -681,7 +787,39 @@ exists` at boot, declared by no migration and carried in no snapshot, so counted
|
|
|
681
787
|
are eight `unexpected-table` findings against a correct database. The prefix is the rule, not a
|
|
682
788
|
list, so a table a future package adds needs no second declaration here. `introspect()` keeps its
|
|
683
789
|
narrower default (`x_migrations` alone) because the admin schema view and the MCP `schema.describe`
|
|
684
|
-
tool legitimately show `x_users` — only drift wants the whole namespace gone.
|
|
790
|
+
tool legitimately show `x_users` — only drift wants the whole namespace gone. That last sentence is
|
|
791
|
+
a *reservation*, not a description, `As of 2026-08-24`: nothing outside this package imports
|
|
792
|
+
`introspect()` today, and `schema.describe` (`@ultimat3/mcp`'s `dev-server.ts`) answers from the
|
|
793
|
+
entity registry.
|
|
794
|
+
|
|
795
|
+
**`app-relation.ts` is the other half, and it is ownership, never a name — issue #340,
|
|
796
|
+
`As of 2026-08-24`.** `pg_stat_statements` is a view an extension owns, the CNPG/RDS/Supabase
|
|
797
|
+
default puts it in `public` of every database, and the drift audit after `ROLE=migrate` reported it
|
|
798
|
+
as `unexpected-table` with `x db gen "add pg_stat_statements"` as the fix — so every deploy of the
|
|
799
|
+
demo app failed terminally for 16 hours, and following the fix would have written an extension's
|
|
800
|
+
internal view into the app's migration set. `nonAppRelations(client, schema)` names what
|
|
801
|
+
`introspect()` must not see, and `introspect()` merges it into `excluded` **unconditionally**: an
|
|
802
|
+
explicit `exclude` replaces the `x_migrations` default, never this set, because an extension's
|
|
803
|
+
relations are not app schema in any deployment and that is not a caller's to switch off.
|
|
804
|
+
|
|
805
|
+
Two disqualifications, one question. **Extension ownership is read out of `pg_depend`**
|
|
806
|
+
(`deptype = 'e'`, `refclassid = 'pg_extension'`) — Postgres' own record, and the only rule that
|
|
807
|
+
generalises: a `pg_*` prefix would have covered the reported view and missed `postgis`'
|
|
808
|
+
`spatial_ref_sys`, `timescaledb`'s catalog, and `pg_stat_statements`' own `pg_stat_statements_info`
|
|
809
|
+
sibling, which is a real `relkind = 'r'` table. **A view, a materialised view and a foreign table
|
|
810
|
+
are not tables**, whoever made them: measured on PGlite, a plain `create view` reaches
|
|
811
|
+
`information_schema.columns` while the index query already fences on `relkind = 'r'`, so one arrived
|
|
812
|
+
as a table with columns, no primary key and no indexes — a `TableDescription` that cannot be true,
|
|
813
|
+
and a finding no author could clear because no snapshot records a view. Excluding by NAME is safe
|
|
814
|
+
because `pg_class` names are unique within a namespace.
|
|
815
|
+
|
|
816
|
+
Nothing else in the audit had the same hole. An extension cannot own a **column** of a table it does
|
|
817
|
+
not own — `alter extension … add` has no `COLUMN` form — so `unexpected-column` is unreachable that
|
|
818
|
+
way. **Types and enums** are never compared (`compareTable` reads nullability and existence, never
|
|
819
|
+
the type). **Indexes and foreign keys** are judged on the declared side only, so an extension's
|
|
820
|
+
index on an app table was already silent. `introspect-embedded.test.ts` proves the predicate against
|
|
821
|
+
a real catalog by writing the exact `pg_depend` row `create extension` writes; a recording client
|
|
822
|
+
can only pin the SQL text, which is what `app-relation.test.ts` does.
|
|
685
823
|
|
|
686
824
|
The `X_DB_DRIFT` rendering in `drift.ts` and the title in `DB_ERROR_TITLES` are pinned by the
|
|
687
825
|
framework contract and duplicated in `@ultimat3/entity`. Change them together or not at all.
|
package/README.md
CHANGED
|
@@ -13,7 +13,7 @@ boundary stays thin.
|
|
|
13
13
|
## Public API
|
|
14
14
|
|
|
15
15
|
```ts
|
|
16
|
-
import { db, sql, raw, withTransaction, currentTx,
|
|
16
|
+
import { db, sql, raw, withTransaction, currentTx, setDbClient } from '@ultimat3/db';
|
|
17
17
|
|
|
18
18
|
const rows = await db().query<Post>(sql`select * from posts where org_id = ${orgId}`);
|
|
19
19
|
|
|
@@ -32,7 +32,11 @@ await withTransaction(async (tx) => {
|
|
|
32
32
|
| `sqlState()` / `sqlStateCode()` / `isRetryableState()` / `SQLSTATE` | `As of 2026-08`: the SQLSTATE a driver error carries, and the closed table from it to a code. `Bun.SQL` puts it on `errno`; PGlite puts it on `code`; **one** reader answers for both |
|
|
33
33
|
| `migrate()` / `rollback()` / `readLedger()` | the `x_migrations` ledger |
|
|
34
34
|
| `statementsOf()` | `As of 2026-08`: a SQL script → the statements a driver sends one at a time. One send is one statement, so `migrate()` splits with this — a `;` inside a literal, an identifier, a dollar-quoted body or a comment is data |
|
|
35
|
-
| `
|
|
35
|
+
| `withReplicaReads()` / `replicaScope()` | `As of 2026-08-24`: the scope inside which a plain read may be served by a replica — until it writes, after which every read in it is the primary's. No scope open, nothing routes |
|
|
36
|
+
| `replicatedClient()` / `ReplicaStats` / `REPLICA_URL_ENV` | `As of 2026-08-24`: one `DbClient` over a primary and a standby. `baseClient()` builds one when `DATABASE_REPLICA_URL` is set and the single-pool client when it is not |
|
|
37
|
+
| `INDEX_METHODS` / `IndexMethod` / `indexMethodOf()` / `indexMethodSql()` / `declaredMethod()` / `isIndexMethod()` | `As of 2026-08-24`: an index's access method — `btree` or `gin`, closed. Absent is `btree`, the live side is read open (whatever `pg_am` said), and the DDL literal is re-derived from the set rather than spliced from the input |
|
|
38
|
+
| `isPlainRead()` | `As of 2026-08-24`: whether a statement may leave the primary. An allow-list — everything it cannot vouch for is the primary's |
|
|
39
|
+
| `checkDrift()` / `diffSchema()` / `assertNoDrift()` | drift, with a `--json` report. `checkDrift()` is the **post-migrate verification** — the live database against the ledger: columns, declared indexes (access method `As of 2026-08-24`, columns, uniqueness, direction, and whether a predicate is there at all — never its text) and declared foreign keys, matched on where the key points and not on its constraint name, with the `on delete` rule compared through one normalisation `As of 2026-08-19` |
|
|
36
40
|
| `declaredSchema()` / `expectedSchema()` | `As of 2026-08`: the schema the migrations write down, or `undefined` when the newest one carries no snapshot — never an older snapshot standing in for it |
|
|
37
41
|
| `parseSnapshot()` | `As of 2026-08`: a `<id>.snapshot.json` sidecar validated to the last nested field, or `undefined`. `{"tables":[null]}` is valid JSON and is not a schema |
|
|
38
42
|
| `snapshotJson()` | `As of 2026-08`: the sidecar's **bytes** — the JSON Biome would have printed, trailing newline included. The one writer of a `<id>.snapshot.json`, because `JSON.stringify(…, null, 2)` is not formatter-clean and an app's `lint` step rejected the file `x db gen` had just written |
|
|
@@ -41,7 +45,7 @@ await withTransaction(async (tx) => {
|
|
|
41
45
|
| `generateMigration()` | `x db gen "<name>"` — reversible up/down SQL, and `destructive` for the marker the file must carry. `As of 2026-08` a foreign key is its own `alter table … add constraint`, emitted after every table statement: inline, a `references()` had to point at a table entity registration order happened to create first, and `down` had to drop them in an order it did not control. `As of 2026-08-19` a **removed** `references()` emits its `drop constraint` (it emitted nothing, and the snapshot then denied a constraint the database still held), a changed `onDelete` is a drop-and-add rebuild, and a declared `on delete` rule reaches the clause at all |
|
|
42
46
|
| `destructiveStatements()` / `hasDestructiveMarker()` / `isDestructive()` / `DESTRUCTIVE_MARKER` | `As of 2026-08`: the destructive-SQL rail — does this `up` drop, truncate or retype, and does the file declare it with `-- destructive: true`? One classifier, read by `x db gen` when it writes the marker and by `x verify` when it demands one |
|
|
43
47
|
| `stripSqlNoise()` | comments, literals, dollar-quoted bodies and quoted identifiers blanked **in source order**, so a reader sees the operation and not the prose. Shared by `readOnlyQuery()` and the destructive rail |
|
|
44
|
-
| `introspect()` | live schema → `SchemaDescription` |
|
|
48
|
+
| `introspect()` | live schema → `SchemaDescription`. **App tables only**, `As of 2026-08-24`: a relation an extension owns (`pg_depend`, `deptype = 'e'`) and anything that is not an ordinary or partitioned table are excluded before the fold, and an explicit `exclude` cannot bring them back |
|
|
45
49
|
| `createBranch()` / `dropBranch()` / `reapBranches()` | copy-on-write branch databases. `As of 2026-08-19` the marker comment records the **base** as well as the instant (`ultimate:branch:<base>:<iso>`, on `BranchInfo.base`), and `reapBranches()` sweeps only branches of the database it is connected to — one Postgres hosting two Ultimate apps used to mean one app's nightly reap dropped the other's branches. A pre-3.x marker records no base and is skipped, never dropped |
|
|
46
50
|
| `createPgliteClient()` / `branchPglite()` | the embedded database — Postgres in this process |
|
|
47
51
|
| `ensureReadOnlyRole()` / `grantReadOnlySql()` / `READONLY_ROLE` | a `NOLOGIN`, SELECT-only Postgres role — layer 1 of `db.query`'s defence |
|
|
@@ -98,6 +102,62 @@ statement's own error. Every caller handles that failure; nothing here swallows
|
|
|
98
102
|
`ensureReadOnlyRole()` — otherwise a table created later is not selectable by the role, and
|
|
99
103
|
layer 1 covers only what existed at grant time.
|
|
100
104
|
|
|
105
|
+
## Index access methods
|
|
106
|
+
|
|
107
|
+
`btree` is the default and is never written out; `gin` is what makes `@>` / `<@` / `&&` / `?` on a
|
|
108
|
+
`json()` or `arrayOf()` column an index lookup rather than a sequential scan.
|
|
109
|
+
|
|
110
|
+
| | |
|
|
111
|
+
|---|---|
|
|
112
|
+
| declared | `IndexDescriptionLike.using` — `'btree' \| 'gin'`, closed. Anything else is `X_SQL_UNSAFE` before it reaches DDL |
|
|
113
|
+
| absent | `btree`, on every side, through `indexMethodOf()` — so a snapshot written before the field existed reads as what it always was |
|
|
114
|
+
| emitted | `create index "posts_tags_idx" on "posts" using gin ("tags");` — and `create index "posts_tags_idx" on "posts" ("tags");` when no method was declared, byte for byte what shipped before |
|
|
115
|
+
| refused | a unique GIN and an ordered GIN (`X_INVARIANT`) — Postgres has neither, and the alternative is a syntax error inside `ROLE=migrate` |
|
|
116
|
+
| recorded | only when declared, so no existing sidecar is rewritten |
|
|
117
|
+
| a method that moved | drop and recreate — Postgres cannot alter one in place |
|
|
118
|
+
| drift | a declared GIN against a live btree is `changed-index`, and the live method is reported by the name the catalog gave it |
|
|
119
|
+
|
|
120
|
+
## Read replicas
|
|
121
|
+
|
|
122
|
+
Opt in twice, and the second one is why it is safe.
|
|
123
|
+
|
|
124
|
+
```ts
|
|
125
|
+
// 1. the pool — an environment variable, read once by baseClient()
|
|
126
|
+
// DATABASE_REPLICA_URL=postgres://reader@replica.internal:5432/app
|
|
127
|
+
|
|
128
|
+
// 2. the scope — reads inside it may be served by the replica
|
|
129
|
+
import { db, sql, withReplicaReads, withTransaction } from '@ultimat3/db';
|
|
130
|
+
|
|
131
|
+
declare const id: string;
|
|
132
|
+
|
|
133
|
+
await withReplicaReads(async () => {
|
|
134
|
+
await db().query(sql`select id from posts limit 20`); // -> replica
|
|
135
|
+
await db().execute(sql`insert into posts (id) values (${id})`);
|
|
136
|
+
await db().query(sql`select id from posts limit 20`); // -> primary, for the rest of the scope
|
|
137
|
+
await withTransaction(async (tx) => {
|
|
138
|
+
await tx.query(sql`select 1`); // -> primary, always
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
| Rule | |
|
|
144
|
+
|---|---|
|
|
145
|
+
| unconfigured | no `DATABASE_REPLICA_URL` → one pool, statement for statement what it always was |
|
|
146
|
+
| no scope | no `withReplicaReads` → nothing routes, whatever is configured |
|
|
147
|
+
| read-your-writes | one write anywhere in the scope, at any depth, across any `await`, and every later read in it is the primary's |
|
|
148
|
+
| a transaction | always the primary — `reserve()` is delegated there, so BEGIN, the body and COMMIT are one connection on one server. A `readOnly: true` transaction leaves the scope clean |
|
|
149
|
+
| eligibility | `select` / `table` / `values` / a read-only `with`, minus locking reads, `select … into` and the functions a standby answers instead of refusing (`pg_advisory_lock`, `set_config`, `nextval`). Everything else is the primary's |
|
|
150
|
+
| a replica that fails | the statement is re-run on the primary — exactly-once, because only plain reads are sent there and a `25006` refusal never executed. Three failures in a row park it for ten seconds |
|
|
151
|
+
| observability | `client.stats` (`replica`, `primary`, `fallbacks`, `parked`), and a `db.replica_fallback` warning per fallback |
|
|
152
|
+
|
|
153
|
+
**The URL must name a read-only standby.** The server's own `25006` refusal is the safety net under
|
|
154
|
+
a classifier that cannot be complete; pointed at a writable node a misroute becomes a write on the
|
|
155
|
+
wrong server, silently.
|
|
156
|
+
|
|
157
|
+
**Nothing opens the scope for you yet.** `withReplicaReads` is tier 1 and ships first; wrapping a
|
|
158
|
+
request in it is the app's call — or one line in the HTTP pipeline — so until that lands no
|
|
159
|
+
production traffic is routed.
|
|
160
|
+
|
|
101
161
|
## The drift contract
|
|
102
162
|
|
|
103
163
|
`checkDrift()` compares `introspect()` against the snapshot the newest applied migration carries —
|
|
@@ -119,8 +179,17 @@ subset, so a database that simply has not migrated yet is pending, not divergent
|
|
|
119
179
|
table in the `x_` namespace — `x_migrations`, the queue's tables, the outbox and every
|
|
120
180
|
`@ultimat3/auth` table are created by `create table if not exists` at boot and appear in no
|
|
121
181
|
snapshot, so `appTables()` drops them before the diff. `introspect()` keeps its own narrower
|
|
122
|
-
exclusion (the ledger alone),
|
|
123
|
-
|
|
182
|
+
exclusion (the ledger alone), reserving `x_users` for a schema view that wants it.
|
|
183
|
+
|
|
184
|
+
**Nor is a relation an extension owns, `As of 2026-08-24`.** `create extension pg_stat_statements`
|
|
185
|
+
in `public` is the CNPG, RDS, Supabase and Neon default, and its view read as `unexpected-table`
|
|
186
|
+
with `x db gen "add pg_stat_statements"` as the fix — so every deploy failed terminally and the fix
|
|
187
|
+
would have written an extension's internal view into the app's migration set. `introspect()` now
|
|
188
|
+
excludes every relation Postgres records as extension-owned (`pg_depend`, `deptype = 'e'`), which is
|
|
189
|
+
ownership rather than a name: a `pg_*` prefix rule covers that view and misses `postgis`'
|
|
190
|
+
`spatial_ref_sys`. Views, materialised views and foreign tables go with them — no snapshot records
|
|
191
|
+
one, so counting them could only ever produce a finding an author has no way to clear. A table
|
|
192
|
+
someone created by hand carries no such dependency and is still `unexpected-table`.
|
|
124
193
|
|
|
125
194
|
Rendered output is pinned byte-for-byte:
|
|
126
195
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/db",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "12.0.0",
|
|
4
4
|
"description": "Postgres access, transactions, migrations and drift detection",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"test": "bun test"
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@ultimat3/core": "
|
|
34
|
+
"@ultimat3/core": "12.0.0"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
37
|
"@electric-sql/pglite": ">=0.5.0"
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Single responsibility: name the relations in a schema that are NOT app schema, so introspection
|
|
2
|
+
// never puts them in a snapshot and drift can never report one. Two disqualifications, one
|
|
3
|
+
// question: an object an extension owns, and an object that is not a table at all.
|
|
4
|
+
|
|
5
|
+
import type { DbClient } from './client';
|
|
6
|
+
import { sql } from './sql';
|
|
7
|
+
|
|
8
|
+
interface NonAppRelationRow {
|
|
9
|
+
readonly name: string;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Relation names in `schema` that no migration could legitimately declare.
|
|
14
|
+
*
|
|
15
|
+
* **Extension ownership is the rule, never a name prefix.** Every object `create extension` builds
|
|
16
|
+
* carries a `pg_depend` row with `deptype = 'e'` pointing at `pg_extension` — that is Postgres'
|
|
17
|
+
* own record of "this belongs to an extension", and it is the only thing that generalises. A
|
|
18
|
+
* prefix rule spelled `pg_*` would have covered the `pg_stat_statements` view that made every
|
|
19
|
+
* deploy of the demo app fail terminally (issue #340) and missed `postgis`' `spatial_ref_sys`,
|
|
20
|
+
* `timescaledb`'s catalog and `pg_stat_statements`' own `pg_stat_statements_info` sibling. An
|
|
21
|
+
* extension may install a relation under any name at all, so the name is not evidence.
|
|
22
|
+
*
|
|
23
|
+
* **A view, a materialised view and a foreign table are not tables**, whoever created them. They
|
|
24
|
+
* reach `information_schema.columns` (measured on PGlite: a plain `create view` appears there),
|
|
25
|
+
* while the index query already fences on `relkind = 'r'` — so one came back as a table with
|
|
26
|
+
* columns, no primary key and no indexes, which is a `TableDescription` that cannot be true. No
|
|
27
|
+
* `x db gen` diff emits `create view` and no snapshot records one, so counting them as app schema
|
|
28
|
+
* can only ever produce a finding an author has no way to clear.
|
|
29
|
+
*
|
|
30
|
+
* Excluding by NAME is safe because `pg_class` names are unique within a namespace: a name on this
|
|
31
|
+
* list cannot also be an app table in the same schema.
|
|
32
|
+
*/
|
|
33
|
+
export async function nonAppRelations(
|
|
34
|
+
client: DbClient,
|
|
35
|
+
schema: string,
|
|
36
|
+
): Promise<readonly string[]> {
|
|
37
|
+
const rows = await client.query<NonAppRelationRow>(sql`
|
|
38
|
+
select c.relname as name
|
|
39
|
+
from pg_class c
|
|
40
|
+
join pg_namespace n on n.oid = c.relnamespace
|
|
41
|
+
where n.nspname = ${schema}
|
|
42
|
+
and c.relkind in ('r', 'p', 'v', 'm', 'f')
|
|
43
|
+
and (
|
|
44
|
+
c.relkind in ('v', 'm', 'f')
|
|
45
|
+
or exists (
|
|
46
|
+
select 1
|
|
47
|
+
from pg_depend d
|
|
48
|
+
where d.classid = 'pg_class'::regclass
|
|
49
|
+
and d.objid = c.oid
|
|
50
|
+
and d.refclassid = 'pg_extension'::regclass
|
|
51
|
+
and d.deptype = 'e'
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
order by c.relname
|
|
55
|
+
`);
|
|
56
|
+
return rows.map((row) => row.name);
|
|
57
|
+
}
|
package/src/client.ts
CHANGED
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
|
|
6
6
|
import { type Role, renderThrowable, resolveRole } from '@ultimat3/core';
|
|
7
7
|
import { statementAttribution } from './attribution';
|
|
8
|
+
// Deliberate cycle, the same shape as `client.ts ⇄ transaction.ts`: nothing here is referenced at
|
|
9
|
+
// module evaluation, and both sides are `function` declarations, so hoisting covers the TDZ.
|
|
10
|
+
import { defaultClient } from './default-client';
|
|
8
11
|
import { DbError, dbUnavailable, driverError, poolAcquireTimeout, poolMaxInvalid } from './errors';
|
|
9
12
|
import { expectedQueryLoopReason } from './expected-loop';
|
|
10
13
|
import { declaresLibpqOption, mergeLibpqOptions } from './libpq-options';
|
|
@@ -123,7 +126,7 @@ export const POOL_MAX_ENV = 'DATABASE_POOL_MAX';
|
|
|
123
126
|
* rather than falling back: a fleet that ignored the number it was given is the failure the
|
|
124
127
|
* variable exists to prevent, and it would only be found in `pg_stat_activity` at 3am.
|
|
125
128
|
*/
|
|
126
|
-
function poolMaxFromEnv(): Partial<PoolProfile> {
|
|
129
|
+
export function poolMaxFromEnv(): Partial<PoolProfile> {
|
|
127
130
|
const raw = process.env[POOL_MAX_ENV];
|
|
128
131
|
if (raw === undefined || raw.trim() === '') return {};
|
|
129
132
|
const max = Number(raw);
|
|
@@ -437,10 +440,11 @@ export function setDbClient(client: DbClient | undefined): void {
|
|
|
437
440
|
* The role default is layered under `DATABASE_POOL_MAX`, because this is the one place the process
|
|
438
441
|
* builds its own client and therefore the only place an operator's value can reach one:
|
|
439
442
|
* `createPostgresClient` has always taken a `profile` override and nothing in a running app passed
|
|
440
|
-
* it, so `POOL_PROFILES` was the last word in a deployed image.
|
|
443
|
+
* it, so `POOL_PROFILES` was the last word in a deployed image. `default-client.ts` owns what gets
|
|
444
|
+
* built — one pool, or a primary and a replica when `DATABASE_REPLICA_URL` names one.
|
|
441
445
|
*/
|
|
442
446
|
export function baseClient(): DbClient {
|
|
443
|
-
if (ambient === undefined) ambient =
|
|
447
|
+
if (ambient === undefined) ambient = defaultClient();
|
|
444
448
|
return ambient;
|
|
445
449
|
}
|
|
446
450
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Single responsibility: the client `baseClient()` builds when an app installed none — the primary
|
|
2
|
+
// pool, plus a read replica when one is configured. Its own file because it is the one place the
|
|
3
|
+
// framework decides a process's database topology from the environment, and `client.ts` is at the
|
|
4
|
+
// line ceiling.
|
|
5
|
+
|
|
6
|
+
import { createPostgresClient, type DbClient, poolMaxFromEnv } from './client';
|
|
7
|
+
import { replicatedClient } from './replica-client';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A read replica's connection string. Unset — which is every app that has not asked for one — and
|
|
11
|
+
* this builds exactly the single-pool client it always did, statement for statement.
|
|
12
|
+
*
|
|
13
|
+
* It must name a READ-ONLY standby. A statement misrouted to one is refused with `25006` and
|
|
14
|
+
* re-run on the primary, and that refusal is the safety net a text classifier cannot be; a URL
|
|
15
|
+
* pointing at a writable node turns a misroute into a write on the wrong server, silently.
|
|
16
|
+
*/
|
|
17
|
+
export const REPLICA_URL_ENV = 'DATABASE_REPLICA_URL';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Composed rather than folded into `createPostgresClient`, on purpose: `migrate`, `x db branch` and
|
|
21
|
+
* every test build a client that must be exactly one pool, and a second pool reachable through the
|
|
22
|
+
* same factory would be a second thing `reserve()`, `close()` and `ping()` each have to mean two
|
|
23
|
+
* ways.
|
|
24
|
+
*
|
|
25
|
+
* The replica inherits the role's profile, `DATABASE_POOL_MAX` included: a fleet sized against
|
|
26
|
+
* `max_connections` has two servers to size against, not one, and a replica pool that ignored the
|
|
27
|
+
* operator's number would be the exhaustion `poolMaxFromEnv` exists to prevent, on the other host.
|
|
28
|
+
*/
|
|
29
|
+
export function defaultClient(): DbClient {
|
|
30
|
+
const profile = poolMaxFromEnv();
|
|
31
|
+
const primary = createPostgresClient({ profile });
|
|
32
|
+
const replicaUrl = process.env[REPLICA_URL_ENV];
|
|
33
|
+
if (replicaUrl === undefined || replicaUrl.trim() === '') return primary;
|
|
34
|
+
return replicatedClient(primary, createPostgresClient({ url: replicaUrl, profile }));
|
|
35
|
+
}
|
package/src/drift.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { baseClient, type DbClient } from './client';
|
|
7
7
|
import { DbError } from './errors';
|
|
8
8
|
import { addForeignKey, dropForeignKey, foreignKeyTarget, onDeleteRule } from './foreign-key';
|
|
9
|
+
import { indexMethodOf } from './index-method';
|
|
9
10
|
import {
|
|
10
11
|
type ForeignKeyDescription,
|
|
11
12
|
findTable,
|
|
@@ -219,6 +220,12 @@ function changedForeignKey(
|
|
|
219
220
|
* a `desc` index rebuilt ascending by hand serves a feed's newest page off the wrong end. `asc` is
|
|
220
221
|
* normalised to `null` first — `createIndex` writes `"col" asc`, which Postgres stores as
|
|
221
222
|
* not-descending, so the raw values differ on every ascending index in a correct database.
|
|
223
|
+
*
|
|
224
|
+
* The **access method** is compared for the same reason and normalised the same way (`indexMethodOf`,
|
|
225
|
+
* absent = `btree`). Without it an emitter that can write `using gin` ships a btree for a declared
|
|
226
|
+
* GIN index and nothing anywhere says so — a declared-and-never-wired key, which is worse than the
|
|
227
|
+
* missing capability, and the reason `using` could not land in `@ultimat3/entity` before it landed
|
|
228
|
+
* here.
|
|
222
229
|
*/
|
|
223
230
|
function compareIndexes(live: TableDescription, expected: TableDescription): DriftDifference[] {
|
|
224
231
|
const differences: DriftDifference[] = [];
|
|
@@ -229,6 +236,17 @@ function compareIndexes(live: TableDescription, expected: TableDescription): Dri
|
|
|
229
236
|
differences.push(missingIndex(live.name, index.name));
|
|
230
237
|
continue;
|
|
231
238
|
}
|
|
239
|
+
// The method first, and before the column list: a GIN index and a btree over the same column
|
|
240
|
+
// are not the same index with a detail different, they are two structures the planner uses for
|
|
241
|
+
// different operators — `@>` on a jsonb column is a sequential scan on the wrong one. Both
|
|
242
|
+
// sides go through `indexMethodOf`, so a snapshot that predates the field and an index the
|
|
243
|
+
// catalog reports as `btree` agree instead of reporting drift on a correct database.
|
|
244
|
+
if (indexMethodOf(counterpart) !== indexMethodOf(index)) {
|
|
245
|
+
differences.push(
|
|
246
|
+
changedIndex(live.name, index.name, `is a ${indexMethodOf(counterpart)} index`),
|
|
247
|
+
);
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
232
250
|
if (counterpart.columns.join(',') !== index.columns.join(',')) {
|
|
233
251
|
differences.push(
|
|
234
252
|
changedIndex(live.name, index.name, `covers (${counterpart.columns.join(', ')})`),
|
package/src/entity-shape.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
// tier 1 and may never import `entity` (tier 2), so a snapshot arrives as a parameter and every
|
|
3
3
|
// part of it — a column's `on delete` rule included — crosses the seam by shape or not at all.
|
|
4
4
|
|
|
5
|
+
import type { IndexMethod } from './index-method';
|
|
6
|
+
|
|
5
7
|
/** Structurally assignment-compatible with `@ultimat3/entity`'s `ColumnDescription`. */
|
|
6
8
|
export interface ColumnDescriptionLike {
|
|
7
9
|
readonly property: string;
|
|
@@ -37,6 +39,13 @@ export interface IndexDescriptionLike {
|
|
|
37
39
|
readonly where: string | null;
|
|
38
40
|
/** `null` is Postgres' own default (`asc`), never written out. */
|
|
39
41
|
readonly order: 'asc' | 'desc' | null;
|
|
42
|
+
/**
|
|
43
|
+
* The access method. Absent is `btree`, which is Postgres' own default and what every index
|
|
44
|
+
* declared before this field existed is — so a description written without it still satisfies
|
|
45
|
+
* the shape, exactly as `ColumnDescriptionLike.onDelete` does. `@ultimat3/entity` (tier 2) is
|
|
46
|
+
* the declarer; this package cannot import it, so the method crosses the seam structurally.
|
|
47
|
+
*/
|
|
48
|
+
readonly using?: IndexMethod | undefined;
|
|
40
49
|
}
|
|
41
50
|
|
|
42
51
|
/** Structurally assignment-compatible with `@ultimat3/entity`'s `EntityDescription`. */
|
package/src/errors.ts
CHANGED
|
@@ -426,6 +426,22 @@ export const multipleStatements = (statement: string, count: number): DbError =>
|
|
|
426
426
|
meta: { count },
|
|
427
427
|
});
|
|
428
428
|
|
|
429
|
+
/**
|
|
430
|
+
* An index method that is not one of the set. `X_SQL_UNSAFE` for the reason `isolationLevelInvalid`
|
|
431
|
+
* uses it: `create index … using <method>` takes no parameters, so the method is SPLICED into the
|
|
432
|
+
* statement text, and an operand TypeScript never saw is an injection rather than a typo — the
|
|
433
|
+
* identical hole a `columnName` built from an unvalidated `meta.name` carried into `create table`.
|
|
434
|
+
*
|
|
435
|
+
* `describeValue`, never the value, for the reason that one gives: this cause is folded into a
|
|
436
|
+
* problem document and a log line.
|
|
437
|
+
*/
|
|
438
|
+
export const indexMethodInvalid = (received: unknown): DbError =>
|
|
439
|
+
new DbError({
|
|
440
|
+
code: 'X_SQL_UNSAFE',
|
|
441
|
+
cause: `an index method must be 'btree' or 'gin'; got ${describeValue(received)}`,
|
|
442
|
+
fix: "indexes: [{ on: ['tags'], using: 'gin' }] # or leave it out for the btree default",
|
|
443
|
+
});
|
|
444
|
+
|
|
429
445
|
export const branchExists = (branch: string): DbError =>
|
|
430
446
|
new DbError({
|
|
431
447
|
code: 'X_BRANCH_EXISTS',
|
package/src/generate.ts
CHANGED
|
@@ -13,6 +13,7 @@ import type {
|
|
|
13
13
|
} from './entity-shape';
|
|
14
14
|
import { migrationIrreversible } from './errors';
|
|
15
15
|
import { type ConstraintPlans, foreignKeyPlan, foreignKeysOf, type Plan } from './foreign-key-plan';
|
|
16
|
+
import { declaredMethod, indexMethodOf, indexMethodSql } from './index-method';
|
|
16
17
|
import {
|
|
17
18
|
type ColumnDescription,
|
|
18
19
|
findTable,
|
|
@@ -121,6 +122,10 @@ export function snapshotOf(entities: readonly EntityDescriptionLike[]): SchemaDe
|
|
|
121
122
|
primary: false,
|
|
122
123
|
where: index.where,
|
|
123
124
|
order: index.order,
|
|
125
|
+
// Only when one was declared. Writing `using: 'btree'` out for every index would rewrite
|
|
126
|
+
// every sidecar in every app on the next `x db gen` — a diff on every file for a fact
|
|
127
|
+
// that was already true, which `indexMethodOf` reads out of the absence anyway.
|
|
128
|
+
...(index.using === undefined ? {} : { using: index.using }),
|
|
124
129
|
})),
|
|
125
130
|
foreignKeys: foreignKeysOf(entity),
|
|
126
131
|
};
|
|
@@ -154,11 +159,30 @@ function createIndex(table: string, index: IndexDescriptionLike): string {
|
|
|
154
159
|
`index "${index.name}" on "${table}" names no columns`,
|
|
155
160
|
`indexes: [{ on: ['<column>'] }] # name the columns in the entity(), then x db gen`,
|
|
156
161
|
);
|
|
162
|
+
const method = index.using ?? 'btree';
|
|
163
|
+
// Two rules Postgres has and a declaration can break, refused here rather than at migrate time:
|
|
164
|
+
// GIN supports neither a unique index nor an ASC/DESC option, and either one reaches the server
|
|
165
|
+
// as a syntax error inside `ROLE=migrate` — a release phase that fails with the server's words
|
|
166
|
+
// and none of the entity's. `X_INVARIANT` for the reason `createIndex` already uses it on an
|
|
167
|
+
// index naming no columns: a declaration this build cannot honour is refused, never reinterpreted.
|
|
168
|
+
assert(
|
|
169
|
+
method === 'btree' || !index.unique,
|
|
170
|
+
`index "${index.name}" on "${table}" is unique and ${method}; Postgres has no unique ${method} index`,
|
|
171
|
+
`indexes: [{ on: ['<column>'], using: '${method}' }] # drop unique, or drop using`,
|
|
172
|
+
);
|
|
173
|
+
assert(
|
|
174
|
+
method === 'btree' || index.order === null,
|
|
175
|
+
`index "${index.name}" on "${table}" is ${method} and ${index.order}; only a btree orders its keys`,
|
|
176
|
+
`indexes: [{ on: ['<column>'], using: '${method}' }] # drop order, or drop using`,
|
|
177
|
+
);
|
|
157
178
|
const kind = index.unique ? 'create unique index' : 'create index';
|
|
158
179
|
const direction = index.order === null ? '' : ` ${index.order}`;
|
|
159
180
|
const columns = index.columns.map((column) => `"${column}"${direction}`).join(', ');
|
|
160
181
|
const predicate = index.where === null ? '' : ` where (${index.where})`;
|
|
161
|
-
|
|
182
|
+
// Re-derived from the closed set, never spliced: `indexMethodSql` answers `''` for a btree, so
|
|
183
|
+
// an index that declared no method emits the statement this generator always emitted, byte for
|
|
184
|
+
// byte, and one that declared a method Postgres does not have is refused instead of built.
|
|
185
|
+
return `${kind} "${index.name}" on "${table}"${indexMethodSql(method)} (${columns})${predicate};`;
|
|
162
186
|
}
|
|
163
187
|
|
|
164
188
|
/**
|
|
@@ -185,7 +209,13 @@ function retypeColumn(
|
|
|
185
209
|
|
|
186
210
|
/** The parts of an index Postgres cannot alter in place — every one of them is a rebuild. */
|
|
187
211
|
function indexShape(index: IndexDescriptionLike | IndexDescription): string {
|
|
188
|
-
return JSON.stringify([
|
|
212
|
+
return JSON.stringify([
|
|
213
|
+
[...index.columns],
|
|
214
|
+
index.unique,
|
|
215
|
+
index.where,
|
|
216
|
+
index.order ?? null,
|
|
217
|
+
indexMethodOf(index),
|
|
218
|
+
]);
|
|
189
219
|
}
|
|
190
220
|
|
|
191
221
|
/**
|
|
@@ -216,6 +246,10 @@ function redefineIndex(
|
|
|
216
246
|
unique: recorded.unique,
|
|
217
247
|
where: recorded.where,
|
|
218
248
|
order: recorded.order,
|
|
249
|
+
// `declaredMethod`, never a cast: `recorded` is a snapshot's, typed open because the catalog
|
|
250
|
+
// shares the shape, and a method this generator cannot emit must refuse rather than be
|
|
251
|
+
// rebuilt as a btree — a `down` that recreates the wrong structure is worse than none.
|
|
252
|
+
...(recorded.using === undefined ? {} : { using: declaredMethod(recorded.using) }),
|
|
219
253
|
}),
|
|
220
254
|
`drop index "${index.name}";`,
|
|
221
255
|
);
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Single responsibility: an index's access method — the closed set an entity may declare, the one
|
|
2
|
+
// normalisation both sides of a comparison pass through, and the DDL fragment. Its own file for the
|
|
3
|
+
// reason `foreign-key.ts` holds `onDeleteRule`: a generator and a detector that disagreed about
|
|
4
|
+
// what "the default" is would report drift on a database that is exactly right.
|
|
5
|
+
|
|
6
|
+
import { indexMethodInvalid } from './errors';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The methods an entity may declare. Two members, deliberately: `btree` is what every index has
|
|
10
|
+
* always been, and `gin` is the one with a caller — `@>` / `<@` / `&&` / `?` on a `json()` or
|
|
11
|
+
* `arrayOf()` column is a sequential scan without it.
|
|
12
|
+
*
|
|
13
|
+
* `gist`, `brin`, `hash` and `spgist` are legitimate Postgres methods and are **not** here, because
|
|
14
|
+
* nothing declares one and each brings a rule of its own that would have to be enforced with no
|
|
15
|
+
* caller to test it — `hash` and `brin` cannot be unique, `gist` needs `btree_gist` to be, and none
|
|
16
|
+
* of the three accepts `asc`/`desc`. Adding a member later is additive; shipping four that nobody
|
|
17
|
+
* uses is four ways for a first caller to be silently wrong. A method the catalog reports and this
|
|
18
|
+
* set does not carry is still READ and still compared — see `indexMethodOf`.
|
|
19
|
+
*/
|
|
20
|
+
export const INDEX_METHODS = ['btree', 'gin'] as const;
|
|
21
|
+
|
|
22
|
+
export type IndexMethod = (typeof INDEX_METHODS)[number];
|
|
23
|
+
|
|
24
|
+
export function isIndexMethod(value: unknown): value is IndexMethod {
|
|
25
|
+
return typeof value === 'string' && (INDEX_METHODS as readonly string[]).includes(value);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* What method this index is on, whichever side it came from. `undefined` is `btree` — Postgres'
|
|
30
|
+
* own default, which nothing writes out, which every index created before this existed is, and
|
|
31
|
+
* which is therefore what a snapshot recorded before it carried the field at all.
|
|
32
|
+
*
|
|
33
|
+
* The CATALOG's answer is passed through verbatim, `gist` and an extension's own access method
|
|
34
|
+
* included: the live side is whatever `pg_am` said, and folding an unknown name into `btree` would
|
|
35
|
+
* hide exactly the difference an operator needs to see.
|
|
36
|
+
*/
|
|
37
|
+
export function indexMethodOf(index: { readonly using?: string | undefined }): string {
|
|
38
|
+
return index.using ?? 'btree';
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The closed-set reading of a method that arrived on the OPEN side — a catalog row or a snapshot
|
|
43
|
+
* this generator did not write. `undefined` for absent, and a **refusal** for anything the set does
|
|
44
|
+
* not carry, never a silent fall back to `btree`: the one caller is `redefineIndex`, whose `down`
|
|
45
|
+
* recreates the index a previous migration recorded, and a `gist` quietly rebuilt as a btree is a
|
|
46
|
+
* rollback that leaves the database in a state no migration describes.
|
|
47
|
+
*/
|
|
48
|
+
export function declaredMethod(using: string | undefined): IndexMethod | undefined {
|
|
49
|
+
if (using === undefined) return undefined;
|
|
50
|
+
if (!isIndexMethod(using)) throw indexMethodInvalid(using);
|
|
51
|
+
return using;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The clause, or `''` for a btree — so an index that declared nothing emits the statement it always
|
|
56
|
+
* did, byte for byte.
|
|
57
|
+
*
|
|
58
|
+
* The literal is **re-derived from the set, never spliced from the input**, the same shape
|
|
59
|
+
* `isolationMode` uses for `BEGIN`. The type is not the guard: this value reaches `create index …`
|
|
60
|
+
* as text from an entity declaration, a config or a generator, and `using ${method}` on an operand
|
|
61
|
+
* TypeScript never saw is the identical hole to the one `columnName` carried — a name that closed
|
|
62
|
+
* the parenthesis and opened a second command.
|
|
63
|
+
*/
|
|
64
|
+
export function indexMethodSql(method: IndexMethod): string {
|
|
65
|
+
switch (method) {
|
|
66
|
+
case 'btree':
|
|
67
|
+
return '';
|
|
68
|
+
case 'gin':
|
|
69
|
+
return ' using gin';
|
|
70
|
+
default: {
|
|
71
|
+
const unhandled: never = method;
|
|
72
|
+
throw indexMethodInvalid(unhandled);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -32,6 +32,7 @@ export {
|
|
|
32
32
|
poolProfileFor,
|
|
33
33
|
setDbClient,
|
|
34
34
|
} from './client';
|
|
35
|
+
export { defaultClient, REPLICA_URL_ENV } from './default-client';
|
|
35
36
|
export type { DestructiveKind, DestructiveStatement } from './destructive';
|
|
36
37
|
export {
|
|
37
38
|
DESTRUCTIVE_CAUSE,
|
|
@@ -86,6 +87,14 @@ export type { RecordedStatement, RecordingClient, StubResponse } from './fake';
|
|
|
86
87
|
export { createRecordingClient } from './fake';
|
|
87
88
|
export type { GeneratedMigration, GenerateOptions } from './generate';
|
|
88
89
|
export { generateMigration, migrationStamp, slugify, snapshotOf } from './generate';
|
|
90
|
+
export type { IndexMethod } from './index-method';
|
|
91
|
+
export {
|
|
92
|
+
declaredMethod,
|
|
93
|
+
INDEX_METHODS,
|
|
94
|
+
indexMethodOf,
|
|
95
|
+
indexMethodSql,
|
|
96
|
+
isIndexMethod,
|
|
97
|
+
} from './index-method';
|
|
89
98
|
export type {
|
|
90
99
|
ColumnDescription,
|
|
91
100
|
ForeignKeyDescription,
|
|
@@ -142,6 +151,15 @@ export type { ReadOnlyQueryOptions, ReadOnlyQueryResult } from './readonly-query
|
|
|
142
151
|
export { READONLY_TIMEOUT_MS, readOnlyQuery } from './readonly-query';
|
|
143
152
|
export type { ReadOnlyRoleOptions } from './readonly-role';
|
|
144
153
|
export { ensureReadOnlyRole, grantReadOnlySql, READONLY_ROLE } from './readonly-role';
|
|
154
|
+
export type {
|
|
155
|
+
ReplicaStats,
|
|
156
|
+
ReplicatedClient,
|
|
157
|
+
ReplicatedClientOptions,
|
|
158
|
+
} from './replica-client';
|
|
159
|
+
export { BREAKER_COOLDOWN_MS, BREAKER_FAILURES, replicatedClient } from './replica-client';
|
|
160
|
+
export { type DbNode, isPlainRead } from './replica-route';
|
|
161
|
+
export type { ReplicaScope } from './replica-scope';
|
|
162
|
+
export { markScopeWrote, replicaScope, withReplicaReads } from './replica-scope';
|
|
145
163
|
export { snapshotJson } from './snapshot-json';
|
|
146
164
|
export { parseSnapshot } from './snapshot-parse';
|
|
147
165
|
export type { SqlFragment } from './sql';
|
package/src/introspect.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// Single responsibility: read the live schema out of `information_schema` / `pg_catalog` into a
|
|
2
|
-
// plain, sortable description
|
|
3
|
-
//
|
|
4
|
-
// and deterministically ordered
|
|
2
|
+
// plain, sortable description — app tables only, never a relation an extension owns and never a
|
|
3
|
+
// view (`app-relation.ts`). `checkDrift` is its one shipped consumer, so what it omits a deploy
|
|
4
|
+
// cannot refuse over. Keep it JSON-safe and deterministically ordered: it is diffed and serialised.
|
|
5
5
|
|
|
6
|
+
import { nonAppRelations } from './app-relation';
|
|
6
7
|
import { type DbClient, db } from './client';
|
|
7
8
|
import { sql } from './sql';
|
|
8
9
|
|
|
@@ -29,6 +30,14 @@ export interface IndexDescription {
|
|
|
29
30
|
readonly where: string | null;
|
|
30
31
|
/** `desc` only when every key column is descending; `null` is Postgres' own default. */
|
|
31
32
|
readonly order: 'asc' | 'desc' | null;
|
|
33
|
+
/**
|
|
34
|
+
* The access method as `pg_am` names it — `btree`, `gin`, `gist`, or an extension's own. Read
|
|
35
|
+
* OPEN rather than as the closed set an entity may declare: the live side is whatever the
|
|
36
|
+
* catalog said, and a `gist` folded into `btree` is the difference drift exists to report.
|
|
37
|
+
* Absent means `btree` — a snapshot written before this field existed carries no method, and
|
|
38
|
+
* every index it recorded was one. `indexMethodOf()` is the one reader of that rule.
|
|
39
|
+
*/
|
|
40
|
+
readonly using?: string | undefined;
|
|
32
41
|
}
|
|
33
42
|
|
|
34
43
|
export interface ForeignKeyDescription {
|
|
@@ -55,7 +64,13 @@ export interface SchemaDescription {
|
|
|
55
64
|
export interface IntrospectOptions {
|
|
56
65
|
readonly client?: DbClient | undefined;
|
|
57
66
|
readonly schema?: string | undefined;
|
|
58
|
-
/**
|
|
67
|
+
/**
|
|
68
|
+
* The ledger is framework bookkeeping, not user schema — excluded so it never reads as drift.
|
|
69
|
+
*
|
|
70
|
+
* Replaces the default (`['x_migrations']`) rather than adding to it. It never replaces the set
|
|
71
|
+
* `nonAppRelations()` derives: an extension's relations are not app schema in any deployment,
|
|
72
|
+
* so that exclusion is not a caller's to switch off.
|
|
73
|
+
*/
|
|
59
74
|
readonly exclude?: readonly string[] | undefined;
|
|
60
75
|
}
|
|
61
76
|
|
|
@@ -76,6 +91,7 @@ interface IndexRow {
|
|
|
76
91
|
readonly columns: readonly string[];
|
|
77
92
|
readonly predicate: string | null;
|
|
78
93
|
readonly descending: boolean;
|
|
94
|
+
readonly method?: string | undefined;
|
|
79
95
|
}
|
|
80
96
|
|
|
81
97
|
interface ForeignKeyRow {
|
|
@@ -92,7 +108,13 @@ const byName = (a: { name: string }, b: { name: string }): number => (a.name < b
|
|
|
92
108
|
export async function introspect(options: IntrospectOptions = {}): Promise<SchemaDescription> {
|
|
93
109
|
const client = options.client ?? db();
|
|
94
110
|
const schema = options.schema ?? 'public';
|
|
95
|
-
|
|
111
|
+
// Asked first, and unconditionally: everything below reads `information_schema`, which admits a
|
|
112
|
+
// view and an extension's own tables alongside the app's. Merged into `excluded` rather than
|
|
113
|
+
// filtered afterwards so one deny list feeds the whole fold.
|
|
114
|
+
const excluded = [
|
|
115
|
+
...(options.exclude ?? ['x_migrations']),
|
|
116
|
+
...(await nonAppRelations(client, schema)),
|
|
117
|
+
];
|
|
96
118
|
|
|
97
119
|
const columns = await client.query<ColumnRow>(sql`
|
|
98
120
|
select table_name, column_name, data_type, is_nullable, column_default, ordinal_position
|
|
@@ -112,16 +134,18 @@ export async function introspect(options: IntrospectOptions = {}): Promise<Schem
|
|
|
112
134
|
ix.indisunique as is_unique,
|
|
113
135
|
ix.indisprimary as is_primary,
|
|
114
136
|
pg_get_expr(ix.indpred, ix.indrelid) as predicate,
|
|
137
|
+
am.amname as method,
|
|
115
138
|
array_agg(a.attname order by k.ord) as columns,
|
|
116
139
|
bool_and((ix.indoption[k.ord - 1] & 1) = 1) as descending
|
|
117
140
|
from pg_class t
|
|
118
141
|
join pg_namespace n on n.oid = t.relnamespace
|
|
119
142
|
join pg_index ix on ix.indrelid = t.oid
|
|
120
143
|
join pg_class i on i.oid = ix.indexrelid
|
|
144
|
+
join pg_am am on am.oid = i.relam
|
|
121
145
|
cross join lateral unnest(ix.indkey::smallint[]) with ordinality as k(attnum, ord)
|
|
122
146
|
join pg_attribute a on a.attrelid = t.oid and a.attnum = k.attnum
|
|
123
147
|
where n.nspname = ${schema} and t.relkind = 'r' and k.ord <= ix.indnkeyatts
|
|
124
|
-
group by t.relname, i.relname, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid
|
|
148
|
+
group by t.relname, i.relname, ix.indisunique, ix.indisprimary, ix.indpred, ix.indrelid, am.amname
|
|
125
149
|
order by t.relname, i.relname
|
|
126
150
|
`);
|
|
127
151
|
|
|
@@ -175,6 +199,9 @@ export function buildSchema(
|
|
|
175
199
|
primary: row.is_primary,
|
|
176
200
|
where: row.predicate,
|
|
177
201
|
order: row.descending ? ('desc' as const) : null,
|
|
202
|
+
// `exactOptionalPropertyTypes`: a row with no method is a stub's, and absent must stay
|
|
203
|
+
// absent rather than becoming an explicit `undefined` a strict comparison can see.
|
|
204
|
+
...(row.method === undefined ? {} : { using: row.method }),
|
|
178
205
|
}))
|
|
179
206
|
.sort(byName);
|
|
180
207
|
return {
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// Single responsibility: one `DbClient` over a primary and a read replica. It decides nothing about
|
|
2
|
+
// SQL (`replica-route.ts`) and nothing about scope (`replica-scope.ts`) — what lives here is which
|
|
3
|
+
// handle a statement is sent on, what happens when the replica will not answer, and the counters
|
|
4
|
+
// that make both visible to a test that cannot scrape a metrics endpoint.
|
|
5
|
+
|
|
6
|
+
import { type Clock, logger, renderThrowable, systemClock } from '@ultimat3/core';
|
|
7
|
+
import { type DbClient, type DbConnection, isReservable, type ReservableClient } from './client';
|
|
8
|
+
import { isPlainRead } from './replica-route';
|
|
9
|
+
import { markScopeWrote, replicaScope } from './replica-scope';
|
|
10
|
+
import type { SqlFragment } from './sql';
|
|
11
|
+
|
|
12
|
+
export interface ReplicaStats {
|
|
13
|
+
/** Statements the replica answered. */
|
|
14
|
+
readonly replica: number;
|
|
15
|
+
/** Statements sent to the primary, fallbacks included. */
|
|
16
|
+
readonly primary: number;
|
|
17
|
+
/** Replica attempts that failed and were re-run on the primary. */
|
|
18
|
+
readonly fallbacks: number;
|
|
19
|
+
/** True while the breaker is parked and every read is going to the primary. */
|
|
20
|
+
readonly parked: boolean;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface ReplicatedClientOptions {
|
|
24
|
+
/** Consecutive replica failures before it is parked. */
|
|
25
|
+
readonly breakerFailures?: number | undefined;
|
|
26
|
+
readonly breakerCooldownMs?: number | undefined;
|
|
27
|
+
/** Injection seam; production passes neither. */
|
|
28
|
+
readonly clock?: Clock | undefined;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface ReplicatedClient extends DbClient {
|
|
32
|
+
readonly stats: ReplicaStats;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Three in a row, then a ten-second rest — an outage costs 3 doubled reads, not every read. */
|
|
36
|
+
export const BREAKER_FAILURES = 3;
|
|
37
|
+
export const BREAKER_COOLDOWN_MS = 10_000;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* `primary` answers everything that is not provably a replica-safe read inside an open
|
|
41
|
+
* `withReplicaReads` scope. Reservations are ALWAYS the primary's: `withTransaction` pins a
|
|
42
|
+
* connection through `reserve()`, and a BEGIN that landed on a standby is not a transaction, it is
|
|
43
|
+
* `25006` on the first write inside it.
|
|
44
|
+
*
|
|
45
|
+
* `reserve` is present only when the primary has one, so `isReservable()` keeps answering about the
|
|
46
|
+
* database rather than about this wrapper — a wrapper that always exposed `reserve` would make
|
|
47
|
+
* `runRoot` pin a connection out of a client that cannot pin, and one that never exposed it would
|
|
48
|
+
* make `runRoot` run BEGIN, the statements and COMMIT on three different pooled connections.
|
|
49
|
+
*/
|
|
50
|
+
export function replicatedClient(
|
|
51
|
+
primary: DbClient,
|
|
52
|
+
replica: DbClient,
|
|
53
|
+
options: ReplicatedClientOptions = {},
|
|
54
|
+
): ReplicatedClient {
|
|
55
|
+
const clock = options.clock ?? systemClock;
|
|
56
|
+
const limit = options.breakerFailures ?? BREAKER_FAILURES;
|
|
57
|
+
const cooldown = options.breakerCooldownMs ?? BREAKER_COOLDOWN_MS;
|
|
58
|
+
let replicaCount = 0;
|
|
59
|
+
let primaryCount = 0;
|
|
60
|
+
let fallbackCount = 0;
|
|
61
|
+
let consecutiveFailures = 0;
|
|
62
|
+
let parkedUntil = 0;
|
|
63
|
+
|
|
64
|
+
/** Monotonic, never wall clock: a leap second or an NTP step must not un-park the breaker. */
|
|
65
|
+
function parked(): boolean {
|
|
66
|
+
return clock.monotonic() < parkedUntil;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function nodeIsReplica(text: string): boolean {
|
|
70
|
+
const scope = replicaScope();
|
|
71
|
+
// No scope: nobody declared these reads replica-safe, so this is a single-pool client.
|
|
72
|
+
if (scope === undefined) return false;
|
|
73
|
+
if (!isPlainRead(text)) {
|
|
74
|
+
markScopeWrote();
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
// Read-your-writes, and the reason the flag is on a mutable scope value rather than computed
|
|
78
|
+
// per statement: once this scope has written, every later read in it is the primary's. A
|
|
79
|
+
// replica is behind by an unbounded amount — streaming lag is not a number this tier can know
|
|
80
|
+
// — so "the row I just inserted" is the one question a standby is guaranteed to answer wrong.
|
|
81
|
+
if (scope.wrote) return false;
|
|
82
|
+
return !parked();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function send<T>(fragment: SqlFragment, on: (client: DbClient) => Promise<T>): Promise<T> {
|
|
86
|
+
if (!nodeIsReplica(fragment.text)) {
|
|
87
|
+
primaryCount += 1;
|
|
88
|
+
return on(primary);
|
|
89
|
+
}
|
|
90
|
+
try {
|
|
91
|
+
const answer = await on(replica);
|
|
92
|
+
replicaCount += 1;
|
|
93
|
+
consecutiveFailures = 0;
|
|
94
|
+
return answer;
|
|
95
|
+
} catch (error) {
|
|
96
|
+
// Re-running is exactly-once, not at-least-once: only `isPlainRead` statements reach here,
|
|
97
|
+
// and a statement a standby refused (`25006`) never executed. A replica outage therefore
|
|
98
|
+
// costs latency and never an answer — which is the whole point, since a read replica is a
|
|
99
|
+
// capacity tier and must not become a new way for the app to be down.
|
|
100
|
+
consecutiveFailures += 1;
|
|
101
|
+
if (consecutiveFailures >= limit) parkedUntil = clock.monotonic() + cooldown;
|
|
102
|
+
fallbackCount += 1;
|
|
103
|
+
// `renderThrowable`, never `${error}`: a driver error's `message` getter is app code.
|
|
104
|
+
logger.warn('db.replica_fallback', {
|
|
105
|
+
error: renderThrowable(error),
|
|
106
|
+
consecutiveFailures,
|
|
107
|
+
parked: parked(),
|
|
108
|
+
});
|
|
109
|
+
primaryCount += 1;
|
|
110
|
+
return on(primary);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const base: ReplicatedClient = {
|
|
115
|
+
get stats(): ReplicaStats {
|
|
116
|
+
return {
|
|
117
|
+
replica: replicaCount,
|
|
118
|
+
primary: primaryCount,
|
|
119
|
+
fallbacks: fallbackCount,
|
|
120
|
+
parked: parked(),
|
|
121
|
+
};
|
|
122
|
+
},
|
|
123
|
+
query: <T>(fragment: SqlFragment) => send(fragment, (client) => client.query<T>(fragment)),
|
|
124
|
+
one: <T>(fragment: SqlFragment) => send(fragment, (client) => client.one<T>(fragment)),
|
|
125
|
+
execute: (fragment: SqlFragment) => send(fragment, (client) => client.execute(fragment)),
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
if (!isReservable(primary)) return base;
|
|
129
|
+
const reservable: ReplicatedClient & ReservableClient = {
|
|
130
|
+
...base,
|
|
131
|
+
get stats(): ReplicaStats {
|
|
132
|
+
return base.stats;
|
|
133
|
+
},
|
|
134
|
+
reserve: (): Promise<DbConnection> => primary.reserve(),
|
|
135
|
+
};
|
|
136
|
+
return reservable;
|
|
137
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Single responsibility: may THIS statement be served by a replica. An allow-list with a refusal
|
|
2
|
+
// list under it, whose default is the primary — the opposite bias to the `readonly.ts` lexer this
|
|
3
|
+
// package deleted, and the reason that deletion does not forbid this file.
|
|
4
|
+
|
|
5
|
+
import { statementVerb } from './statement-shape';
|
|
6
|
+
|
|
7
|
+
export type DbNode = 'primary' | 'replica';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The only verbs a replica may be offered. `with` is here because a CTE read is the shape half the
|
|
11
|
+
* framework's paginated queries take, and `WRITE_WORD` below is what tells `with … select` from
|
|
12
|
+
* `with … update … returning` — which `statementKind()` calls a read, and is exactly why that
|
|
13
|
+
* function is not the authority here.
|
|
14
|
+
*/
|
|
15
|
+
const READ_VERBS: ReadonlySet<string> = new Set(['select', 'table', 'values']);
|
|
16
|
+
const CTE_VERB = 'with';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A word that disqualifies the whole statement. Matched with word boundaries against the raw
|
|
20
|
+
* lowercased text — NOT against `stripSqlNoise`'d text, deliberately: a `;`-in-a-literal is data
|
|
21
|
+
* and must not split a statement, but a `'update'` in a literal costing one read its replica is a
|
|
22
|
+
* false positive on the SAFE side, and blanking every statement on the hot path to buy back that
|
|
23
|
+
* one read is a cost axiom 6 refuses. `share` covers `for share` and `for key share`; `update`
|
|
24
|
+
* covers `for update` and `for no key update`; `into` covers `select … into`, which creates a table.
|
|
25
|
+
*/
|
|
26
|
+
const WRITE_WORD =
|
|
27
|
+
/\b(?:insert|update|delete|merge|truncate|copy|create|drop|alter|grant|revoke|call|do|lock|share|refresh|reindex|vacuum|analyze|set|reset|begin|commit|rollback|savepoint|into)\b/;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Function names a word boundary cannot reach — `pg_advisory_lock` has a `_` before `advisory`, so
|
|
31
|
+
* `\badvisory\b` never matches it. Every one of these either writes or mutates session state that
|
|
32
|
+
* belongs to whichever backend ran it, and a standby accepts them silently rather than answering
|
|
33
|
+
* `25006`, so the server's own refusal cannot be the safety net here the way it is for a real write.
|
|
34
|
+
*/
|
|
35
|
+
const UNSAFE_CALLS: readonly string[] = [
|
|
36
|
+
'nextval',
|
|
37
|
+
'setval',
|
|
38
|
+
'set_config',
|
|
39
|
+
'advisory',
|
|
40
|
+
'dblink',
|
|
41
|
+
'lo_import',
|
|
42
|
+
'lo_export',
|
|
43
|
+
'pg_export_snapshot',
|
|
44
|
+
'pg_replication',
|
|
45
|
+
'pg_create',
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Provably a plain read, or `false`. Never "probably": everything this cannot vouch for is the
|
|
50
|
+
* primary's, so a statement shape nobody anticipated costs a replica opportunity and never a wrong
|
|
51
|
+
* answer. That inversion is the whole difference from `readonly.ts`, whose 22-word deny-list read
|
|
52
|
+
* `select pg_sleep(60)` as safe because the default was permission.
|
|
53
|
+
*/
|
|
54
|
+
export function isPlainRead(text: string): boolean {
|
|
55
|
+
const lowered = text.toLowerCase();
|
|
56
|
+
const verb = statementVerb(lowered);
|
|
57
|
+
if (!READ_VERBS.has(verb) && verb !== CTE_VERB) return false;
|
|
58
|
+
if (WRITE_WORD.test(lowered)) return false;
|
|
59
|
+
return !UNSAFE_CALLS.some((call) => lowered.includes(call));
|
|
60
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Single responsibility: the scope inside which a read may be served by a replica, and the one bit
|
|
2
|
+
// that closes read-your-writes — has this scope written yet. A mutable value on an async context,
|
|
3
|
+
// the same shape `transaction.ts` uses for `TxState.live`, so a write at any depth and across any
|
|
4
|
+
// `await` is seen by every later read in the same scope.
|
|
5
|
+
|
|
6
|
+
import { asyncContext } from '@ultimat3/core';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Deliberately mutable, and deliberately not `readonly`. The whole mechanism is that a write ten
|
|
10
|
+
* frames and three `await`s below the scope's opener flips this, and the read after it sees it —
|
|
11
|
+
* a fresh object per statement could not carry that, and threading a parameter would be the same
|
|
12
|
+
* fact written at every call site, with every path an author forgot serving a stale row.
|
|
13
|
+
*/
|
|
14
|
+
export interface ReplicaScope {
|
|
15
|
+
wrote: boolean;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const scope = asyncContext<ReplicaScope>('the replica read scope');
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Declare that reads inside `fn` may be served by a replica — until `fn` writes, after which every
|
|
22
|
+
* read in it is the primary's for the rest of the scope.
|
|
23
|
+
*
|
|
24
|
+
* **Opt-in, and that is the safety argument, not an ergonomic one.** `packages/db` cannot see a
|
|
25
|
+
* request boundary: `@ultimat3/http`'s pipeline opens the `Ctx` and nothing tells this tier when a
|
|
26
|
+
* request ended, so a write-marker keyed on `Ctx.requestId` would be a `Map` that only grows —
|
|
27
|
+
* ~100 bytes per request, forever — and any eviction policy that forgets a request that WROTE
|
|
28
|
+
* serves it a stale row, which is worse than the capacity problem replicas exist to solve. With no
|
|
29
|
+
* scope open nothing routes and the client is byte-identical to a single-pool one, so the failure
|
|
30
|
+
* mode of "nobody opened one" is today's behaviour rather than a wrong answer.
|
|
31
|
+
*
|
|
32
|
+
* Nesting is one scope, not two: an inner `withReplicaReads` inside a scope that has already
|
|
33
|
+
* written must not un-write it. The innermost call reuses the store it finds.
|
|
34
|
+
*/
|
|
35
|
+
export function withReplicaReads<T>(fn: () => T): T {
|
|
36
|
+
const open = scope.get();
|
|
37
|
+
if (open !== undefined) return fn();
|
|
38
|
+
return scope.run({ wrote: false }, fn);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** The scope in flight, or `undefined` — which is every caller that never opened one. */
|
|
42
|
+
export function replicaScope(): ReplicaScope | undefined {
|
|
43
|
+
return scope.get();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Record that this scope has written. Called for every statement that is not provably a plain read
|
|
48
|
+
* — including `begin`, a `set`, and anything the router could not classify — because the direction
|
|
49
|
+
* that is safe to be wrong in is "assume it wrote". A no-op outside a scope, where nothing routes.
|
|
50
|
+
*/
|
|
51
|
+
export function markScopeWrote(): void {
|
|
52
|
+
const open = scope.get();
|
|
53
|
+
if (open !== undefined) open.wrote = true;
|
|
54
|
+
}
|
package/src/snapshot-parse.ts
CHANGED
|
@@ -36,13 +36,19 @@ function column(value: unknown): ColumnDescription | undefined {
|
|
|
36
36
|
|
|
37
37
|
function index(value: unknown): IndexDescription | undefined {
|
|
38
38
|
if (!isRow(value)) return undefined;
|
|
39
|
-
const { name, columns, unique, primary, where, order: direction } = value;
|
|
39
|
+
const { name, columns, unique, primary, where, order: direction, using } = value;
|
|
40
40
|
if (!str(name) || !strings(columns) || !bool(unique) || !bool(primary)) return undefined;
|
|
41
41
|
// Written by 1.2.0 onwards. A sidecar from before it carries neither, and the total, ascending
|
|
42
42
|
// reading is what that generation actually emitted — so an older file stays readable rather
|
|
43
43
|
// than being discarded whole, which would refuse to generate against every existing app.
|
|
44
44
|
if (!(where === undefined || nullableStr(where))) return undefined;
|
|
45
45
|
if (!(direction === undefined || order(direction))) return undefined;
|
|
46
|
+
// Any string, not the closed set. The live side of this type is the CATALOG's, which answers
|
|
47
|
+
// `gist` and an extension's own access method, and a sidecar recording one must round-trip so
|
|
48
|
+
// drift can report it — the refusal belongs at generation, where `declaredMethod` names the
|
|
49
|
+
// method and the fix, not here, where it would discard the whole snapshot without saying which
|
|
50
|
+
// field was wrong. Absent stays absent: `indexMethodOf` reads it as the btree it always was.
|
|
51
|
+
if (!(using === undefined || str(using))) return undefined;
|
|
46
52
|
return {
|
|
47
53
|
name,
|
|
48
54
|
columns,
|
|
@@ -50,6 +56,7 @@ function index(value: unknown): IndexDescription | undefined {
|
|
|
50
56
|
primary,
|
|
51
57
|
where: where === undefined ? null : where,
|
|
52
58
|
order: direction === undefined ? null : direction,
|
|
59
|
+
...(using === undefined ? {} : { using }),
|
|
53
60
|
};
|
|
54
61
|
}
|
|
55
62
|
|
package/src/transaction.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type { Random } from '@ultimat3/core';
|
|
|
7
7
|
import { assert, asyncContext, nanoid } from '@ultimat3/core';
|
|
8
8
|
import { baseClient, type DbClient, type DbConnection, isReservable } from './client';
|
|
9
9
|
import { isolationLevelInvalid, serializationExhausted } from './errors';
|
|
10
|
+
import { markScopeWrote } from './replica-scope';
|
|
10
11
|
import { raw, type SqlFragment } from './sql';
|
|
11
12
|
import { isRetryableState } from './sqlstate';
|
|
12
13
|
import { serializationRetryDelayMs } from './transaction-backoff';
|
|
@@ -206,6 +207,11 @@ async function runNested<T>(outer: TxState, fn: (tx: DbTx) => Promise<T>): Promi
|
|
|
206
207
|
*/
|
|
207
208
|
async function runRoot<T>(fn: (tx: DbTx) => Promise<T>, options: TransactionOptions): Promise<T> {
|
|
208
209
|
const client = options.client ?? baseClient();
|
|
210
|
+
// A transaction is assumed to write unless it said otherwise, so every read AFTER it in the same
|
|
211
|
+
// `withReplicaReads` scope is the primary's. The pin below already keeps the transaction's own
|
|
212
|
+
// statements off any replica — this is about the rest of the request, which `replica-client.ts`
|
|
213
|
+
// could not otherwise see: `runRoot` sends through a reserved connection, not through the router.
|
|
214
|
+
if (options.readOnly !== true) markScopeWrote();
|
|
209
215
|
// A pooled BEGIN that lands on a different physical connection than the statements after it is
|
|
210
216
|
// not a transaction at all, so a reservable client pins one connection for the whole scope.
|
|
211
217
|
// Held by a `using` declaration rather than a `finally`, because a `finally` only covers what
|