@ultimat3/db 29.4.0 → 30.1.0

package/src/pooler.ts

@@ -0,0 +1,161 @@
+// Single responsibility: a connection pooler in the path — the mode the environment names, how long
+// a statement may queue in it, the two bounds as the session reads them, and the two errors a pooler
+// in the path is reported with. The fix text is `pooler-fix.ts`', which `errors.ts` reads too.
+
+import { describeValue, finiteCount } from '@ultimat3/core';
+import type { DbClient } from './client';
+import { DbError } from './errors';
+import type { PoolProfile } from './pool-profile';
+import { POOLER_SESSION_REQUIRED_FIX, STATEMENT_BOUND_LOST_FIX } from './pooler-fix';
+import { sql } from './sql';
+
+/** The variable naming a connection pooler in this process's path. */
+export const POOLER_ENV = 'DATABASE_POOLER';
+
+/** What the pooler in the path multiplexes on: a whole client session, or one transaction. */
+export type PoolerMode = 'session' | 'transaction';
+
+/**
+ * How long a statement may queue in the pooler for a backend, on top of the server's own bound.
+ * `statement_timeout` starts only once the pooler assigns a backend, so a saturated pooler is not
+ * a dead path; PgBouncer's default `query_wait_timeout` is 120 s, and past it the pooler answers
+ * itself. Reproduced 2026-11-03: 20 busy server connections, `select 1` from `web` was retired at
+ * 7005 ms on a healthy server without it.
+ */
+export const POOLER_WAIT_MS = 120_000;
+
+const poolerInvalid = (received: string): DbError =>
+  new DbError({
+    code: 'X_ENV_MISSING',
+    // `describeValue`, never the value, in the cause AND meta: a pooler URL pasted here carries its
+    // password into the boot log and the `--json` payload, where no key is left to redact it by.
+    cause: `DATABASE_POOLER is ${describeValue(received)}, which is not session or transaction`,
+    fix: 'DATABASE_POOLER=session   # or transaction, or unset it',
+    meta: { received: describeValue(received) },
+  });
+
+/**
+ * `DATABASE_POOLER`, or nothing — and nothing keeps today's bytes. A switch and never a detection:
+ * one pooler accepts the libpq `options` and drops them, so there is no failure to retry on. Env,
+ * for `POOL_MAX_ENV`'s reason: one image, and the URL changes per environment. An unknown value
+ * **refuses**, for that one's reason too: a typo here silently re-sends what the pooler refuses,
+ * or strips the bounds off a connection that has no pooler at all. `postgresClient({ env })` is its
+ * one reader in a process; everything else asks `client.pooler`.
+ */
+export function poolerFromEnv(
+  env: Readonly<Record<string, string | undefined>> = process.env,
+): PoolerMode | undefined {
+  const raw = env[POOLER_ENV];
+  if (raw === undefined || raw.trim() === '') return undefined;
+  if (raw === 'session' || raw === 'transaction') return raw;
+  throw poolerInvalid(raw);
+}
+
+/** The deadline extension a pool gets: `0` with no pooler, the budget with one. */
+export function poolerWaitFor(pooler: PoolerMode | undefined, waitMs: number | undefined): number {
+  if (pooler === undefined) return 0;
+  return waitMs === undefined
+    ? POOLER_WAIT_MS
+    : finiteCount('postgresClient', 'poolerWaitMs', waitMs);
+}
+
+/** `subject` is who was refused: `ROLE=migrate`, or the step that takes the migration lock. */
+export const poolerSessionRequired = (subject: string): DbError =>
+  new DbError({
+    code: 'X_DB_POOLER_SESSION_REQUIRED',
+    cause: `${subject} under DATABASE_POOLER=transaction: session advisory locks and replication need one backend, and through this pooler two migrators would both read true from pg_try_advisory_lock`,
+    fix: POOLER_SESSION_REQUIRED_FIX,
+    meta: { subject },
+  });
+
+/** One setting as the session reads it, beside what the role was sized around. Milliseconds. */
+interface LostBound {
+  readonly setting: 'statement_timeout' | 'idle_in_transaction_session_timeout';
+  readonly appliedMs: number;
+  readonly expectedMs: number;
+}
+
+/**
+ * A report, logged and never thrown: a fleet already running this must not stop on upgrade. The
+ * cause states the second cost, because it is not visible as a bound: the unanswered-statement
+ * deadline still runs on the profile's number, so a long healthy statement reads as a dead path.
+ */
+export const statementBoundLost = (role: string, lost: readonly LostBound[]): DbError =>
+  new DbError({
+    code: 'X_DB_STATEMENT_BOUND_LOST',
+    cause:
+      lost
+        .map(
+          (bound) =>
+            `${bound.setting} is ${String(bound.appliedMs)}ms, the ${role} role expects ${String(bound.expectedMs)}ms`,
+        )
+        .join('; ') +
+      '. Nothing on the server ends a long statement now, and the client still judges one by the role bound: past statementTimeoutMs plus 5 s (plus poolerWaitMs behind a pooler) it answers X_DB_UNAVAILABLE and the pool is replaced, failing its siblings with it',
+    fix: STATEMENT_BOUND_LOST_FIX,
+    meta: { role, lost },
+  });
+
+/** One setting: what the session reads, and what the role's profile asks for. Milliseconds both. */
+interface BoundReading {
+  readonly applied: number;
+  readonly expected: number;
+}
+
+export interface StatementBound {
+  readonly statementTimeout: BoundReading;
+  readonly idleInTransaction: BoundReading;
+}
+
+/**
+ * The two bounds as the SESSION reads them. A pooler can accept the libpq `options` and drop them,
+ * so what was sent says nothing: this is the only reading of what is in force. `pg_settings`, not
+ * `show`: both are `ms`-unit settings, so `setting` is a number where `show` answers `10s`.
+ */
+export async function statementBound(
+  client: DbClient & { readonly profile: PoolProfile },
+): Promise<StatementBound> {
+  const rows = await client.query<{ readonly name: string; readonly setting: string }>(
+    sql`select name, setting from pg_settings where name in ('statement_timeout', 'idle_in_transaction_session_timeout')`,
+  );
+  const applied = (name: string): number =>
+    Number(rows.find((row) => row.name === name)?.setting ?? 0);
+  return {
+    statementTimeout: {
+      applied: applied('statement_timeout'),
+      expected: client.profile.statementTimeoutMs,
+    },
+    idleInTransaction: {
+      applied: applied('idle_in_transaction_session_timeout'),
+      expected: client.profile.idleInTransactionMs,
+    },
+  };
+}
+
+/**
+ * The verdict on a reading, or nothing. With no pooler named the framework sent the value, so any
+ * difference is something in the path. With one named it did not, and one database user may serve
+ * two roles, so only `0` — no bound at all, where the role has one — is a lost bound.
+ */
+export function lostStatementBound(
+  bound: StatementBound,
+  role: string,
+  pooler: PoolerMode | undefined,
+): DbError | undefined {
+  const judged: readonly LostBound[] = (
+    [
+      ['statement_timeout', bound.statementTimeout],
+      ['idle_in_transaction_session_timeout', bound.idleInTransaction],
+    ] as const
+  )
+    .filter(([, reading]) =>
+      pooler === undefined
+        ? reading.applied !== reading.expected
+        : reading.applied === 0 && reading.expected !== 0,
+    )
+    .map(([setting, reading]) => ({
+      setting,
+      appliedMs: reading.applied,
+      expectedMs: reading.expected,
+    }));
+  return judged.length === 0 ? undefined : statementBoundLost(role, judged);
+}

package/src/read-transaction.ts

@@ -0,0 +1,59 @@
+// Single responsibility: a read inside its own read-only transaction, with transaction-local
+// settings applied first. Row security's tenant and a read's statement bound both ride this one
+// helper, so a setting meant for one read can never outlive it on a pooled connection.
+
+import { assert } from '@ultimat3/core';
+import type { DbClient } from './client';
+import { type LocalSettings, readLocal, setLocal } from './local-settings';
+import { currentTx, withTransaction } from './transaction';
+import type { DbTx } from './transaction-options';
+
+export interface ReadTransactionOptions {
+  /** Applied through `setLocal`, before `fn` runs. */
+  readonly settings?: LocalSettings | undefined;
+  /** The client to open on — row security's standby read passes the replica. Default the pool. */
+  readonly client?: DbClient | undefined;
+}
+
+/**
+ * `withTransaction(fn, { readOnly: true, client })`, then `setLocal`. The server refuses a write
+ * inside it (`25006`), and `readOnly` keeps a `withReplicaReads` scope from being marked as written.
+ *
+ * Inside an open transaction the access mode is the outer `BEGIN`'s and cannot change, so this
+ * joins it: the settings are applied, `fn` runs, and the values read before are put back — the
+ * rest of the outer transaction sees what it saw. No savepoint: nothing here needs rolling back.
+ */
+export async function withReadTransaction<T>(
+  fn: (tx: DbTx) => Promise<T>,
+  options: ReadTransactionOptions = {},
+): Promise<T> {
+  const settings = options.settings ?? {};
+  const open = currentTx();
+  if (open === undefined) {
+    return withTransaction(
+      async (tx) => {
+        await setLocal(tx, settings);
+        return fn(tx);
+      },
+      { readOnly: true, client: options.client },
+    );
+  }
+  assert(
+    options.client === undefined || options.client === open.origin,
+    'withReadTransaction({ client }) inside a transaction opened on a different client: it joins the open transaction, so nothing would run on the client named here',
+    'run the read outside the open transaction — await withReadTransaction(fn, { client }) after it returns — or drop { client } to read inside it',
+  );
+  const prior = await readLocal(open, Object.keys(settings));
+  await setLocal(open, settings);
+  let result: T;
+  try {
+    result = await fn(open);
+  } catch (error) {
+    // The outer transaction may be aborted by `fn`'s failure; a restore would then fail too, and
+    // the caller needs `fn`'s error. The abort discards the local values with everything else.
+    await setLocal(open, prior).catch(() => undefined);
+    throw error;
+  }
+  await setLocal(open, prior);
+  return result;
+}

package/src/readonly-query.ts

@@ -6,6 +6,7 @@
 import { finiteCount } from '@ultimat3/core';
 import { baseClient, type DbClient, type DbConnection, isReservable } from './client';
 import { multipleStatements } from './errors';
+import { setLocal } from './local-settings';
 import { identifier, raw, sql } from './sql';
 import { stripSqlNoise } from './sql-noise';
 import { statementsOf } from './statement-split';
@@ -96,9 +97,8 @@ export async function readOnlyQuery<T>(
   // against an exhausted pool answered with the pool's error — or waited for a connection it was
   // never going to use — in place of the `X_INVARIANT` naming the option that was wrong.
   //
-  // Clamped and truncated to an integer: the result is a JS number the caller never touches
-  // as text, so there is nothing here for `raw()` to inject — `SET LOCAL` can't bind `$n`
-  // parameters, which is why this can't go through `sql` the normal, parameterised way.
+  // Clamped to a whole number, then sent through `setLocal`, the one parameter-bound writer of a
+  // transaction-local setting — never as spliced `SET LOCAL` text, a second path to the same bound.
   // REFUSED rather than normalised: `NaN` used to take the default silently, so a config typo
   // ran under a timeout nobody wrote. Only an explicit 0 disables the layer, which is why the
   // floor is 0 and not 1; the hour ceiling is a clamp on a number that IS one.
@@ -130,7 +130,7 @@ export async function readOnlyQuery<T>(
       // every request the pool serves afterwards. Caveat: embedded PGlite applies the setting
       // but is single-threaded WASM, so it cannot interrupt a running scan; on a real server
       // the timeout fires. Layers 1 and 3 do not depend on it.
-      await connection.execute(raw(`SET LOCAL statement_timeout = ${ms}`));
+      await setLocal(connection, { statement_timeout: String(ms) });
       guards.push(`timeout:${ms}ms`);
     }
 

package/src/replica-client.ts

@@ -3,8 +3,17 @@
 // handle a statement is sent on, what happens when the replica will not answer, and the counters
 // that make both visible to a test that cannot scrape a metrics endpoint.
 
-import { type Clock, finiteCount, logger, renderThrowable, systemClock } from '@ultimat3/core';
+import {
+  type BreakerTicket,
+  breaker,
+  type Clock,
+  finiteCount,
+  logger,
+  renderThrowable,
+  systemClock,
+} from '@ultimat3/core';
 import { type DbClient, type DbConnection, isReservable, type ReservableClient } from './client';
+import { POSITION_CACHE_MS, primaryPosition, replayReader } from './replica-position';
 import { isPlainRead } from './replica-route';
 import { markScopeWrote, replicaScope } from './replica-scope';
 import type { SqlFragment } from './sql';
@@ -25,12 +34,19 @@ export interface ReplicatedClientOptions {
   /** Consecutive replica failures before it is parked. */
   readonly breakerFailures?: number | undefined;
   readonly breakerCooldownMs?: number | undefined;
+  /** How long one reading of the replica's replay position is reused. Default 50 ms. */
+  readonly positionCacheMs?: number | undefined;
   /** Injection seam; production passes neither. */
   readonly clock?: Clock | undefined;
 }
 
 export interface ReplicatedClient extends DbClient {
   readonly stats: ReplicaStats;
+  /**
+   * The primary's WAL position NOW, read on the primary and never routed. Asked once, after a
+   * request that wrote has committed, so the caller can carry it to the next request.
+   */
+  primaryPosition(): Promise<bigint | undefined>;
 }
 
 /**
@@ -75,9 +91,8 @@ export function replicatedClient(
   options: ReplicatedClientOptions = {},
 ): ReplicatedClient {
   const clock = options.clock ?? systemClock;
-  // Both are comparisons and nothing else — `consecutiveFailures >= limit` opens the breaker,
-  // `monotonic() < parkedUntil` holds it open — so a `NaN` in either is a breaker that never trips
-  // and never parks, with every read still going to the replica that is failing.
+  // Screened HERE as well as by `breaker()`, so a refusal names this function's own option: a
+  // `NaN` in either is a breaker that never trips and never parks.
   const limit = finiteCount(
     'replicatedClient',
     'breakerFailures',
@@ -93,58 +108,73 @@ export function replicatedClient(
   let replicaCount = 0;
   let primaryCount = 0;
   let fallbackCount = 0;
-  let consecutiveFailures = 0;
-  let parkedUntil = 0;
-
-  /** Monotonic, never wall clock: a leap second or an NTP step must not un-park the breaker. */
-  function parked(): boolean {
-    return clock.monotonic() < parkedUntil;
-  }
+  // Core's one breaker machine: monotonic time, so an NTP step never un-parks it. After the
+  // cooldown ONE read probes the replica and the rest take the primary until it answers.
+  const gate = breaker({ name: 'db.replica', failures: limit, cooldownMs: cooldown, clock });
+  const parked = (): boolean => gate.state() === 'open';
+  const replay = replayReader(
+    replica,
+    clock,
+    finiteCount(
+      'replicatedClient',
+      'positionCacheMs',
+      options.positionCacheMs ?? POSITION_CACHE_MS,
+      0,
+    ),
+  );
 
-  function nodeIsReplica(text: string): boolean {
+  /** The breaker's ticket when the statement goes to the replica, else `undefined`. */
+  async function nodeIsReplica(text: string): Promise<BreakerTicket | undefined> {
     const scope = replicaScope();
     // No scope: nobody declared these reads replica-safe, so this is a single-pool client.
-    if (scope === undefined) return false;
+    if (scope === undefined) return undefined;
     if (!isPlainRead(text)) {
       markScopeWrote();
-      return false;
+      return undefined;
     }
     // Read-your-writes, and the reason the flag is on a mutable scope value rather than computed
     // per statement: once this scope has written, every later read in it is the primary's. A
     // replica is behind by an unbounded amount — streaming lag is not a number this tier can know
     // — so "the row I just inserted" is the one question a standby is guaranteed to answer wrong.
-    if (scope.wrote) return false;
-    return !parked();
+    if (scope.wrote) return undefined;
+    // Read-your-writes ACROSS requests: an earlier request's write the replica has not replayed yet.
+    // Before `admit()`, never after: `admit()` spends the half-open probe, and a read then sent to
+    // the primary for being behind would strand the probe until it lapsed.
+    if (scope.minPosition !== undefined && !(await replay.caughtUp(scope.minPosition))) {
+      return undefined;
+    }
+    // Last, so only a statement about to reach the replica consumes the half-open probe.
+    return gate.admit();
   }
 
   async function send<T>(fragment: SqlFragment, on: (client: DbClient) => Promise<T>): Promise<T> {
-    if (!nodeIsReplica(fragment.text)) {
+    const ticket = await nodeIsReplica(fragment.text);
+    if (ticket === undefined) {
       primaryCount += 1;
       return on(primary);
     }
     try {
       const answer = await on(replica);
       replicaCount += 1;
-      consecutiveFailures = 0;
+      gate.succeeded(ticket);
       return answer;
     } catch (error) {
       if (!replicaUnavailable(error)) {
         // The replica answered — with a refusal of the statement — so it is up, and the run of
         // failures the breaker is counting is over.
-        consecutiveFailures = 0;
+        gate.succeeded(ticket);
         throw error;
       }
       // Re-running is exactly-once, not at-least-once: only `isPlainRead` statements reach here,
       // and a statement a standby refused (`25006`) never executed. A replica outage therefore
       // costs latency and never an answer — which is the whole point, since a read replica is a
       // capacity tier and must not become a new way for the app to be down.
-      consecutiveFailures += 1;
-      if (consecutiveFailures >= limit) parkedUntil = clock.monotonic() + cooldown;
+      gate.failed(ticket);
       fallbackCount += 1;
       // `renderThrowable`, never `${error}`: a driver error's `message` getter is app code.
       logger.warn('db.replica_fallback', {
         error: renderThrowable(error),
-        consecutiveFailures,
+        consecutiveFailures: gate.failures(),
         parked: parked(),
       });
       primaryCount += 1;
@@ -161,6 +191,7 @@ export function replicatedClient(
         parked: parked(),
       };
     },
+    primaryPosition: () => primaryPosition(primary),
     query: <T>(fragment: SqlFragment) => send(fragment, (client) => client.query<T>(fragment)),
     one: <T>(fragment: SqlFragment) => send(fragment, (client) => client.one<T>(fragment)),
     execute: (fragment: SqlFragment) => send(fragment, (client) => client.execute(fragment)),
@@ -172,6 +203,7 @@ export function replicatedClient(
     get stats(): ReplicaStats {
       return base.stats;
     },
+    primaryPosition: base.primaryPosition,
     reserve: (): Promise<DbConnection> => primary.reserve(),
   };
   return reservable;

package/src/replica-position.ts

@@ -0,0 +1,89 @@
+// Single responsibility: a WAL position — parsed from the wire, read off the primary after a write,
+// and compared against how far a standby has replayed. Read-your-writes across requests is this
+// one comparison: a request that carries the position its own write produced is served by a
+// replica only once the replica has replayed past it.
+
+import type { Clock } from '@ultimat3/core';
+import type { DbClient } from './client';
+import { sql } from './sql';
+
+/** A `pg_lsn` is two hex words, `X/Y`, each at most 32 bits. Anything else is not one. */
+const LSN = /^([0-9A-Fa-f]{1,8})\/([0-9A-Fa-f]{1,8})$/;
+
+/** How long one answer to "how far has the replica replayed" is reused, per process. */
+export const POSITION_CACHE_MS = 50;
+
+/**
+ * The position as a 64-bit number, or `undefined` for anything that is not a `pg_lsn`. A value off
+ * a cookie or a header is the sender's to choose, so it is parsed and never echoed: a forged one
+ * costs its sender a primary read, and nothing it writes reaches SQL or a response.
+ */
+export function parsePosition(value: string | null | undefined): bigint | undefined {
+  if (value === null || value === undefined) return undefined;
+  const match = LSN.exec(value.trim());
+  if (match === null) return undefined;
+  const [, high = '', low = ''] = match;
+  return (BigInt(`0x${high}`) << 32n) | BigInt(`0x${low}`);
+}
+
+/** The canonical `X/Y` spelling, the one Postgres prints. Only ever applied to a parsed value. */
+export function formatPosition(position: bigint): string {
+  const high = (position >> 32n).toString(16).toUpperCase();
+  const low = (position & 0xffffffffn).toString(16).toUpperCase();
+  return `${high}/${low}`;
+}
+
+interface PositionRow {
+  readonly lsn: string | null;
+}
+
+/**
+ * The primary's WAL INSERT position, read once after a request that wrote has committed. Not
+ * `pg_current_wal_lsn()`, the WRITE position: under `synchronous_commit = off` it trails the
+ * commit just made, so a standby replayed exactly that far looked caught up without the row
+ * (measured on the test services' standby: write 0/3029ED0, insert 0/302AE80).
+ */
+export async function primaryPosition(primary: DbClient): Promise<bigint | undefined> {
+  const row = await primary.one<PositionRow>(sql`select pg_current_wal_insert_lsn()::text as lsn`);
+  return parsePosition(row?.lsn);
+}
+
+export interface ReplayReader {
+  /** Whether the replica has replayed at least `required`. Unknown answers `false`. */
+  caughtUp(required: bigint): Promise<boolean>;
+}
+
+interface Sample {
+  readonly at: number;
+  readonly replayed: Promise<bigint | undefined>;
+}
+
+/**
+ * One reader per replicated client, so the cache is per process and per standby. A sample is the
+ * PROMISE, published before it settles: twenty reads arriving in one tick ask the standby once.
+ * `pg_last_wal_replay_lsn()` is `null` on a server that is not in recovery and the query fails on a
+ * standby that is down; both are "unknown", which is the primary's — never a guess that it caught up.
+ */
+export function replayReader(replica: DbClient, clock: Clock, cacheMs: number): ReplayReader {
+  let sample: Sample | undefined;
+  const read = async (): Promise<bigint | undefined> => {
+    try {
+      const row = await replica.one<PositionRow>(sql`select pg_last_wal_replay_lsn()::text as lsn`);
+      return parsePosition(row?.lsn);
+    } catch {
+      // Deliberate: a standby that cannot say how far it is serves no read that needs to know. The
+      // read goes to the primary, and the breaker hears about the outage from the next plain read.
+      return undefined;
+    }
+  };
+  return {
+    async caughtUp(required: bigint): Promise<boolean> {
+      const now = clock.monotonic();
+      if (sample === undefined || now - sample.at >= cacheMs) {
+        sample = { at: now, replayed: read() };
+      }
+      const replayed = await sample.replayed;
+      return replayed !== undefined && replayed >= required;
+    },
+  };
+}

package/src/replica-scope.ts

@@ -13,6 +13,17 @@ import { asyncContext } from '@ultimat3/core';
  */
 export interface ReplicaScope {
   wrote: boolean;
+  /**
+   * The WAL position an EARLIER request's write produced, carried back by its caller. A replica
+   * serves a read in this scope only once it has replayed past it (`replica-position.ts`). Absent:
+   * no earlier write this caller knows of, and a caught-up replica is not asked about.
+   */
+  readonly minPosition?: bigint | undefined;
+}
+
+export interface ReplicaReadsOptions {
+  /** Parsed by `parsePosition` from the request — never a raw cookie value. */
+  readonly minPosition?: bigint | undefined;
 }
 
 const scope = asyncContext<ReplicaScope>('the replica read scope');
@@ -30,12 +41,22 @@ const scope = asyncContext<ReplicaScope>('the replica read scope');
  * mode of "nobody opened one" is today's behaviour rather than a wrong answer.
  *
  * Nesting is one scope, not two: an inner `withReplicaReads` inside a scope that has already
- * written must not un-write it. The innermost call reuses the store it finds.
+ * written must not un-write it. The innermost call reuses the store it finds, `minPosition` too.
+ *
+ * `fn` receives the scope so the opener can ask, once `fn` settles, whether it wrote — the boot
+ * reads it to hand the caller the position its write produced.
  */
-export function withReplicaReads<T>(fn: () => T): T {
+export function withReplicaReads<T>(
+  fn: (scope: ReplicaScope) => T,
+  options: ReplicaReadsOptions = {},
+): T {
   const open = scope.get();
-  if (open !== undefined) return fn();
-  return scope.run({ wrote: false }, fn);
+  if (open !== undefined) return fn(open);
+  const fresh: ReplicaScope =
+    options.minPosition === undefined
+      ? { wrote: false }
+      : { wrote: false, minPosition: options.minPosition };
+  return scope.run(fresh, () => fn(fresh));
 }
 
 /** The scope in flight, or `undefined` — which is every caller that never opened one. */