@mistriwork/engine 0.0.0-stage → 1.0.0-rc.10

package/dist/adapters/postgresPool.js

@@ -0,0 +1,838 @@
+/**
+ * The PostgreSQL connections of one database: a bounded reader pool, a
+ * writer pool, and one background writer, each opened on first use, every
+ * lease given back in `finally`.
+ *
+ * Three things vary by deployment, per manager or, for the whole process,
+ * from the command line:
+ *
+ * - How many readers the pool may hold: for an App as many as its writers
+ *   (at least balancedPoolSize), four for the database browser, unless told
+ *   otherwise; one to sixty-four.
+ * - How many writers: balancedPoolSize for a process serving Apps unless told otherwise
+ *   (`pool.write`, `--write-pool`; writeLanes.ts), one to sixty-four, each running one
+ *   transaction beside the others (writeLanes.ts orders writes of one
+ *   record). The database browser keeps one: its writes run one at a time,
+ *   the order its Undo depends on (docs/postgresql.md).
+ * - Background work — a job's command — writes through a connection of its
+ *   own, so it never takes the writer a person's save is waiting for.
+ * - What sits between this process and the server. Directly, or through a
+ *   pooler that gives each client a whole server session, the read-only
+ *   default and the statement cap are set once per physical connection.
+ *   Through a transaction-mode pooler they are set per transaction instead
+ *   (postgresPooler.ts): the next transaction may run on another server
+ *   connection, and anything set on this one stays for the pooler's next
+ *   client.
+ *
+ * Every checkout is metered (poolMetrics.ts) — the wait for a connection,
+ * how long it was held, how many are queued now — for the operator surface.
+ *
+ * A connection that names replicas gets a manager per replica, sized like
+ * this one's readers, and the reader handle's statements go to one of them
+ * when the work they run in tolerates lag (replicas.ts). Everything that
+ * leases a whole client — a snapshot, a raw SQL read, a store's read, every
+ * write — stays here, on the primary.
+ */
+import { cancellation, onCancel, STATEMENT_TIMEOUT_MS, StatementStopped, throwIfCancelled } from './statementControl.js';
+import { PoolBusy, PoolMeter, POOL_LIMIT_HOLD_MS, POOL_QUEUE_PER_CONNECTION, POOL_QUEUE_WAIT_MS, } from './poolMetrics.js';
+import { PoolAdmission } from './poolAdmission.js';
+import { holdConnections, managerConnections } from './connectionBudget.js';
+import { pgConnectionOptions } from './postgresValues.js';
+import { sealedPgOptions } from './net/untrustedDatabase.js';
+import { transactionScoped } from './postgresPooler.js';
+import { balancedPoolSize, inBackgroundWork, isWriterCount, lockTimeoutMs, MAX_WRITERS, writeOrder, writerPoolSize } from './writeLanes.js';
+import { startSpan } from './telemetry/trace.js';
+import { delayFlights } from './dbLatency.js';
+import { configureServerSettings, serverSettings } from './serverSettings.js';
+import { MAX_REPLICAS, noteWrite, primaryReads, ReplicaSet } from './replicas.js';
+/* The leases whose client sends each query without waiting for the one before
+   (a writer's, on node-postgres): what pipelined() may send together. */
+const pipelining = new WeakSet();
+/** This lease's client pipelines: several queries asked at once leave together. */
+export const pipelines = (client) => pipelining.has(client);
+/** `wrapper` sends through `client` and nothing else: it pipelines where `client` does. */
+export const pipelinesAs = (wrapper, client) => {
+    if (pipelining.has(client))
+        pipelining.add(wrapper);
+};
+const loadPg = async () => {
+    const specifier = 'pg';
+    return import(specifier).catch(() => {
+        throw new Error('Postgres support needs the "pg" package. Install it with: npm install pg');
+    });
+};
+/** The database browser's reader pool size unless a connection or the command line says otherwise.
+ * An App's is balanced against its writers (appReaders). */
+export const DEFAULT_READERS = 4;
+/** Readers kept connected through an idle spell. Without them the first read
+ * after thirty quiet seconds paid a new connection and its two SETs — and a
+ * dashboard's fan-out paid it several times at once. Two, not the whole pool:
+ * a warm connection is a server backend held for nothing most of the day. */
+const WARM_READERS = 2;
+/** The most readers one database's pool may hold. Past this, a pooler in front of the server is the tool. */
+export const MAX_READERS = 64;
+/** What pg-pool says when no connection came free within the acquisition timeout. */
+const ACQUISITION_TIMEOUT = /timeout exceeded when trying to connect/i;
+/** The server refused this process another connection: too many for its role, or for the server (SQLSTATE 53300). */
+const atConnectionLimit = (error) => error !== null && typeof error === 'object' && error.code === '53300';
+const LIMIT_HOLD_MS = POOL_LIMIT_HOLD_MS;
+/** How many times one checkout asks again after a refusal, a held connection having gone meanwhile. */
+const LIMIT_ATTEMPTS = 3;
+/** Whether `value` is a reader pool size a manager accepts: a whole number from 1 to 64. */
+export function isReaderCount(value) {
+    return typeof value === 'number' && Number.isInteger(value) && value >= 1 && value <= MAX_READERS;
+}
+/* Server timing (telemetry/serverTiming.ts) is loaded with the first pool, not with this module: a print-and-exit
+   flag reaches this file through the settings it reads and never opens a pool (test/cli-startup.test.ts counts what
+   it loads). Every lease comes from a pool, so a lease always finds it loaded. */
+let timing;
+/** What no one chose: an App (keyed writes) reads as widely as it writes, never fewer, because writes without
+ * readers to match only moved the queue to the reads; the database browser keeps four. */
+function defaultReaders(writers) {
+    return writeOrder() === 'keyed' ? appReaders(writers) : DEFAULT_READERS;
+}
+/** An App's reader pool where none was chosen: as wide as its writers, at least the balanced size for `cores`
+ * (this machine's, unless a deployment check asks for another's). */
+export function appReaders(writers, cores) {
+    return Math.min(MAX_READERS, Math.max(balancedPoolSize(cores), writers));
+}
+/**
+ * The defaults every manager in this process takes when it names none: the
+ * command line's `--read-pool` and `--pooler`. Server-wide on purpose
+ * (serverSettings.ts): besides the registry's adapters, tenant sources, the
+ * job host and startup checks open pools of their own on the same databases,
+ * and a transaction-mode pooler in front of one of them has to be honored by
+ * every one; a per-connection switch would reach only some. Called once, at
+ * startup; whatever it leaves unsaid returns to the built-in default. The
+ * reader count stays unset until a manager is made: the write order, which
+ * says whether this is an App, is configured after the flags are read.
+ */
+export function configurePostgresPools(options) {
+    const readers = options.readers;
+    if (readers !== undefined && !isReaderCount(readers))
+        throw new RangeError(`A PostgreSQL read pool holds 1 to ${MAX_READERS} connections; ${String(readers)} is not one of them.`);
+    const pooler = options.pooler ?? 'session';
+    if (pooler !== 'session' && pooler !== 'transaction')
+        throw new RangeError('The pooler is "session" or "transaction".');
+    configureServerSettings({ readers, pooler });
+}
+/** The defaults, as this server's settings leave them. */
+export function postgresPoolDefaults() {
+    const settings = serverSettings();
+    return {
+        readers: settings.readers ?? defaultReaders(writerPoolSize()),
+        pooler: settings.pooler,
+    };
+}
+/** Separate bounded pools keep browsing read-only and transactions on one lease. */
+export class PgConnectionManager {
+    connectionString;
+    onLoad;
+    load;
+    reader;
+    writer;
+    background;
+    /** The pools once they exist, for the meters' physical counts. */
+    opened = {};
+    closed = false;
+    /* Server timing's gauges on this manager's pools, taken out at its close: each holds its pool alive. */
+    gauges = [];
+    closing;
+    /** What sits between this process and the server, for every lease of this manager. */
+    pooler;
+    capMs;
+    lockMs;
+    queueWaitMs;
+    queuePerConnection;
+    meters;
+    /** The line each person-facing role's checkouts wait in, turn by turn across accounts (poolAdmission.ts). */
+    admissions;
+    /* One object, so every caller of read() holds the same handle, as when the handle was the pool itself. */
+    reads;
+    /** This database's replicas, each a manager of its own whose writers are never opened. */
+    replicas;
+    network;
+    pipelineReads;
+    timingLabel;
+    /** A pool the server refused another connection, held at what it had until the timer lets it ask again. */
+    limitHolds = {};
+    /** When each pool last said it reached the connection limit: once a spell, not once a checkout. */
+    limitSaid = {};
+    /** Gives back what this manager told the connection budget it may hold (connectionBudget.ts). */
+    releaseBudget;
+    constructor(connectionString, onLoad = () => { }, load = loadPg, options = {}) {
+        this.connectionString = connectionString;
+        this.onLoad = onLoad;
+        this.load = load;
+        /* Where writes run one at a time a second writer would sit idle: the
+           pool is one, whatever the connection asked for. */
+        const writers = writeOrder() === 'serial' ? 1 : (options.writers ?? writerPoolSize());
+        const readers = options.readers ?? serverSettings().readers ?? defaultReaders(writers);
+        if (!isReaderCount(readers))
+            throw new RangeError(`A PostgreSQL read pool holds 1 to ${MAX_READERS} connections; ${String(readers)} is not one of them.`);
+        if (!isWriterCount(writers))
+            throw new RangeError(`A PostgreSQL write pool holds 1 to ${MAX_WRITERS} connections; ${String(writers)} is not one of them.`);
+        this.pooler = options.pooler ?? serverSettings().pooler;
+        this.network = options.network;
+        this.pipelineReads = options.pipelineReads === true;
+        this.timingLabel = options.timingLabel ?? 'app';
+        this.capMs = options.capMs ?? STATEMENT_TIMEOUT_MS;
+        this.queueWaitMs = options.queueWaitMs ?? POOL_QUEUE_WAIT_MS;
+        this.queuePerConnection = options.queuePerConnection ?? POOL_QUEUE_PER_CONNECTION;
+        this.lockMs = Math.min(options.lockTimeoutMs ?? lockTimeoutMs(), this.capMs);
+        this.meters = Object.freeze({
+            read: new PoolMeter('read', readers),
+            write: new PoolMeter('write', writers),
+            background: new PoolMeter('background', 1),
+        });
+        this.admissions = Object.freeze({
+            read: new PoolAdmission('read', readers, readers * this.queuePerConnection),
+            write: new PoolAdmission('write', writers, writers * this.queuePerConnection),
+        });
+        const replicas = options.replicas ?? [];
+        if (replicas.length > MAX_REPLICAS)
+            throw new RangeError(`A connection names at most ${MAX_REPLICAS} replicas; this one names ${replicas.length}.`);
+        /* A manager may open readers, writers and the background writer; a replica's own are readers only. */
+        const releases = options.counted === false
+            ? []
+            : [
+                holdConnections(connectionString, options.readersOnly ? readers : managerConnections(readers, writers), () => this.read()),
+                ...replicas.map((url) => holdConnections(url, readers)),
+            ];
+        this.releaseBudget = () => releases.forEach((release) => release());
+        /* Sized as this manager's readers are, the same pooler, cap and queue
+           bounds: a replica is where this pool's reads go, so it must hold them. */
+        if (replicas.length)
+            this.replicas = new ReplicaSet(replicas.map((url) => new PgConnectionManager(url, onLoad, load, {
+                ...options,
+                readers,
+                writers,
+                replicas: undefined,
+                counted: false,
+                timingLabel: 'app-replica',
+            })));
+        const run = (client, text, params) => client.query(text, params);
+        const statement = (text, params) => {
+            const primary = () => this.withClient(false, (client) => run(client, text, params), false);
+            return this.replicas
+                ? this.replicas.read((replica) => replica.withReadClient((client) => run(client, text, params)), primary)
+                : primary();
+        };
+        this.reads = { query: statement };
+    }
+    /**
+     * A handle for single statements on the reader pool. Each statement leases
+     * a client of its own for exactly its length, which is what `pool.query`
+     * did — so the checkout is metered, a cancel knows which client to stop,
+     * and behind a transaction-mode pooler the statement runs inside a
+     * read-only transaction of its own.
+     */
+    async read() {
+        await this.pool('read');
+        return this.reads;
+    }
+    /** Native snapshot owners opt into callback drain after cancellation. A
+     * callback must settle after its driver operations are destroyed; arbitrary
+     * never-settling application work cannot offer a bounded drain guarantee. */
+    async withReadClient(run, destroyAfter = false, drainCancelled = false) {
+        return this.withClient(false, run, destroyAfter, drainCancelled);
+    }
+    /** Private native snapshot seam: guard observes this exact checkout, not
+     * instantaneous remote-server health. Ordinary read/write behavior is unchanged. */
+    async withObservedReadClient(run) {
+        return this.withClient(false, (client, current) => {
+            if (!current)
+                throw new Error('Observed PostgreSQL checkout is unavailable.');
+            return run(client, current);
+        }, false, true, true);
+    }
+    /** A writer for one transaction: from the writer pool, or inside background work (asBackgroundWork) the background writer. */
+    async withWriteClient(run, destroyAfter = false) {
+        return this.withClient(true, run, destroyAfter);
+    }
+    /** The meters of the pools opened so far, readers first, then each replica's readers. Readable after close. */
+    stats() {
+        const own = ['read', 'write', 'background'].flatMap((role) => {
+            const pool = this.opened[role];
+            if (!pool)
+                return [];
+            const total = pool.totalCount;
+            return [this.meters[role].stats(this.pooler, typeof total === 'number' ? total : null)];
+        });
+        const replicas = (this.replicas?.pools ?? []).flatMap((replica, index) => replica.stats().map((stats) => ({
+            ...stats,
+            role: 'replica',
+            replica: index + 1,
+        })));
+        return [...own, ...replicas];
+    }
+    /** Whether this database names any replica. */
+    hasReplicas() {
+        return this.replicas !== undefined;
+    }
+    /**
+     * `work` on each replica in use now, its failures left out: what live sync
+     * asks of every healthy replica before it tells readers of a change, so a
+     * refetch it sets off reads a replica that has the change (server/live).
+     */
+    async onHealthyReplicas(work) {
+        const set = this.replicas;
+        if (!set)
+            return [];
+        const answers = await Promise.allSettled(set.pools.map((replica, index) => (set.healthy(index) ? replica.withReadClient(work) : Promise.reject(new Error('passed over')))));
+        return answers.flatMap((answer) => (answer.status === 'fulfilled' ? [answer.value] : []));
+    }
+    /** Each replica: whether it is in use now, and how many reads it answered and handed back to the primary. */
+    replicaHealth() {
+        const set = this.replicas;
+        return set
+            ? set.pools.map((_, index) => ({
+                replica: index + 1,
+                healthy: set.healthy(index),
+                served: set.served[index],
+                fellBack: set.fellBack[index],
+            }))
+            : [];
+    }
+    close() {
+        this.closed = true;
+        this.releaseBudget();
+        for (const off of this.gauges.splice(0))
+            off();
+        for (const timer of Object.values(this.limitHolds))
+            clearTimeout(timer);
+        this.closing ??= Promise.all([this.endPools(), ...(this.replicas?.pools ?? []).map((replica) => replica.close())]).then(() => { });
+        return this.closing;
+    }
+    /* What pg is told about where to connect: the URL, or for an untrusted
+       address every part spelled out so nothing comes from this process. */
+    dialOptions() {
+        return this.network
+            ? sealedPgOptions(this.connectionString, this.network)
+            : pgConnectionOptions(this.connectionString);
+    }
+    assertOpen() {
+        if (this.closed)
+            throw new Error('Postgres connection manager is closed.');
+    }
+    async pool(role) {
+        this.assertOpen();
+        const pending = role === 'write'
+            ? (this.writer ??= this.createPool('write'))
+            : role === 'background'
+                ? (this.background ??= this.createPool('background'))
+                : (this.reader ??= this.createPool('read'));
+        const pool = await pending;
+        this.assertOpen();
+        return pool;
+    }
+    async createPool(role) {
+        const pg = await this.load();
+        timing ??= await import('./telemetry/serverTiming.js');
+        this.assertOpen();
+        this.onLoad(pg);
+        const module = pg;
+        const Pool = module.Pool ?? module.default?.Pool;
+        if (!Pool)
+            throw new Error('Could not find a Pool export in the "pg" package.');
+        const write = role !== 'read';
+        const session = this.pooler === 'session';
+        const pool = new Pool({
+            ...this.dialOptions(),
+            max: this.meters[role].size,
+            connectionTimeoutMillis: 10_000,
+            idleTimeoutMillis: 30_000,
+            /* Readers only: a writer is opened for a write and may go. The idle
+               ones never keep the process alive by themselves, so a one-shot mode
+               that forgets to close still exits. */
+            ...(write
+                ? {}
+                : {
+                    min: Math.min(WARM_READERS, this.meters[role].size),
+                    allowExitOnIdle: true,
+                }),
+            /* A writer sends each statement as it is asked, not after the one
+               before it is answered: a transaction that asks several at once
+               (pipelined, postgresTransactionHandle.ts) pays one round trip for
+               them. One asked at a time is sent and answered exactly as before.
+               A reader only where its owner asked (pipelineReads). */
+            ...(write || this.pipelineReads ? { pipeline: true } : {}),
+            /* pg awaits this hook for every physical client before its first
+               checkout. Behind a transaction-mode pooler a session setting would
+               land on whichever server connection the client holds at that moment
+               and stay there for the pooler's next client, so nothing is set; each
+               lease sets its own, per transaction. */
+            onConnect: async (client) => {
+                /* A leased client's 'error' is its own to hear: the pool listens only
+                   while it is idle, and a backend ended under a lease (a failover, a
+                   pg_terminate_backend) would otherwise end the process. The query it
+                   was running rejects with the same failure, which is where it is
+                   answered. */
+                client.on?.('error', () => { });
+                if (!session)
+                    return;
+                if (!write)
+                    await client.query('SET default_transaction_read_only = on');
+                /* A writer also stops waiting for a lock after lockMs: a row held by
+                   another transaction (another pod, a migration's DDL) fails the
+                   statement fast, with 55P03, rather than holding this connection —
+                   and the save behind it — for the whole statement cap. One message:
+                   the settings are two statements and one round trip. */
+                await client.query(write
+                    ? `SET statement_timeout = '${this.capMs / 1000}s'; SET lock_timeout = '${this.lockMs}ms'`
+                    : `SET statement_timeout = '${this.capMs / 1000}s'`);
+            },
+        });
+        // pg removes failed idle clients itself; an unhandled pool error would crash Node.
+        pool.on('error', () => { });
+        this.opened[role] = pool;
+        /* In use and waiting, for the process entry of server timing (a no-op when it is off). pg-pool's own counters. */
+        const counts = pool;
+        if (timing.serverTimingOn() && !this.closed)
+            this.gauges.push(timing.registerGauge(`pool-${role}-use`, () => (counts.totalCount ?? 0) - (counts.idleCount ?? 0)), timing.registerGauge(`pool-${role}-wait`, () => counts.waitingCount ?? 0));
+        /* With server timing on, the round trip from here to this database, once, on a connection of its own: a
+           driver without a Client (a test's) is not measured. */
+        if (!write)
+            void timing.probeRoundTrips(this.timingLabel, () => {
+                const found = pg;
+                const Client = found.Client ?? found.default?.Client;
+                return Client && new Client({ ...this.dialOptions(), connectionTimeoutMillis: 10_000 });
+            });
+        return pool;
+    }
+    /* `write` stays a boolean: test/support/tenantDialects.ts intercepts every
+       lease here, by that flag, to fault a writer's COMMIT and nothing else. */
+    async withClient(write, run, destroyAfter, drainCancelled = false, observeCheckout = false) {
+        const role = !write ? 'read' : inBackgroundWork() ? 'background' : 'write';
+        /* Only reads are stopped on cancel: a write that has started runs to its
+           end, so its outcome is known rather than guessed. */
+        const signal = write ? undefined : cancellation();
+        throwIfCancelled(signal);
+        const pool = await this.pool(role);
+        /* A write that finds every writer busy says so on the request's trace
+           (the dev observer): the wait for a connection, beside the wait in its
+           lane (writeLanes.ts) and the statements that follow. */
+        const waiting = write && this.meters[role].full()
+            ? startSpan('writer wait', {
+                childOnly: true,
+                attributes: { 'db.client.connection.pool.name': role },
+            })
+            : undefined;
+        const meter = this.meters[role];
+        /* The background writer is one connection that background work waits for by design, for as long as the driver's
+           connect timeout rather than the queue-wait budget a person's read or write has. Its line is bounded all the
+           same: past the queue's length, or past that timeout, the work is answered busy before anything was leased, so
+           a job's command, a delivery or a workflow step is asked again (temporarily_unavailable, a retry) instead of
+           a burst of them failing together as an unavailable store. */
+        const bounded = role !== 'background';
+        if (!bounded && meter.waiting() >= meter.size * this.queuePerConnection) {
+            meter.refuse();
+            throw new PoolBusy('write', 'queue_full');
+        }
+        const checkout = meter.begin();
+        const had = timing?.poolWait(`${this.timingLabel}-${role}`);
+        let client;
+        /* A person's checkout waits its turn in the manager's own line, and only
+           then asks the driver, for a connection that is its: the line and the
+           connection share the one queue-wait budget. */
+        let place;
+        try {
+            if (bounded) {
+                const deadline = performance.now() + this.queueWaitMs;
+                place = await this.admissions[role].enter(this.queueWaitMs);
+                client = await this.connectWithin(pool, role, Math.max(1, deadline - performance.now()));
+            }
+            else
+                client = await this.connectHeld(pool, role);
+            had?.();
+        }
+        catch (error) {
+            had?.();
+            place?.();
+            if (error instanceof PoolBusy) {
+                checkout.failed(true);
+                waiting?.fail('pool_busy');
+                waiting?.end();
+                throw error;
+            }
+            const timedOut = ACQUISITION_TIMEOUT.test(error instanceof Error ? error.message : '');
+            checkout.failed(timedOut);
+            waiting?.fail('pool_timeout');
+            waiting?.end();
+            if (!bounded && timedOut)
+                throw new PoolBusy('write', 'queue_timeout', error);
+            throw error;
+        }
+        waiting?.end();
+        timing?.timeClient(client, this.timingLabel);
+        delayFlights(client);
+        const giveBack = checkout.acquired();
+        const scoped = this.pooler === 'transaction'
+            ? transactionScoped(client, {
+                role: write ? 'write' : 'read',
+                capMs: this.capMs,
+                untrusted: destroyAfter,
+                ...(write ? { lockTimeoutMs: this.lockMs } : {}),
+                /* A writer's client pipelines (below): a statement it sends outside
+                   a transaction leaves with its BEGIN and COMMIT, in one round trip. */
+                pipelines: client.pipeline === true,
+            })
+            : undefined;
+        let destroy = destroyAfter;
+        /* A cancelled client never goes back to the pool: the cancel travels on
+           another connection and can land late, and it must not land on a
+           statement a later lease of the same backend has started. */
+        let cancelled = false;
+        let released = false;
+        let invalidateHealth = () => { }, removeHealthListeners = () => { };
+        const release = (destroyed) => {
+            if (released)
+                return;
+            released = true;
+            invalidateHealth();
+            client.release(destroyed);
+        };
+        /* Behind a transaction-mode pooler the backend id a client was told is
+           the pooler's, not the server's, so a pg_cancel_backend by it would
+           stop whatever unrelated statement holds that id on the server. The
+           ordinary caller stops waiting at once instead; snapshot owners destroy
+           the client first and then drain their callback. Server work remains
+           bounded by the transaction's cap, not by local callback completion. */
+        let stop;
+        const stopped = scoped && signal && !drainCancelled
+            ? new Promise((_, reject) => {
+                stop = reject;
+            })
+            : undefined;
+        void stopped?.catch(() => { });
+        const disarm = onCancel(signal, () => {
+            cancelled = true;
+            invalidateHealth();
+            if (scoped) {
+                const reason = new StatementStopped('cancelled');
+                if (drainCancelled) {
+                    // Invalidate continuations first, then break active driver I/O before
+                    // awaiting callback/finally drain. This drains local promises, not
+                    // server locks: server work may persist until its configured cap.
+                    // Never cancel a pooler's backend ID.
+                    scoped.close(reason);
+                    // A release exception must not escape the AbortSignal listener. The
+                    // operation remains cancelled, and still waits for its callback.
+                    try {
+                        release(true);
+                    }
+                    catch {
+                        /* No retry of a possibly released client. */
+                    }
+                }
+                else
+                    stop?.(reason);
+            }
+            else
+                void this.cancel(client.processID);
+        });
+        try {
+            this.assertOpen();
+            throwIfCancelled(signal);
+            let borrowed = scoped ?? client;
+            let health;
+            if (observeCheckout) {
+                if (typeof client.on !== 'function' || typeof client.off !== 'function')
+                    throw new Error('Observed PostgreSQL checkout is unavailable.');
+                let healthy = true;
+                const on = client.on.bind(client), off = client.off.bind(client);
+                invalidateHealth = () => {
+                    healthy = false;
+                };
+                removeHealthListeners = () => {
+                    off('error', invalidateHealth);
+                    off('end', invalidateHealth);
+                };
+                on('error', invalidateHealth);
+                on('end', invalidateHealth);
+                health = () => {
+                    this.assertOpen();
+                    throwIfCancelled(signal);
+                    if (!healthy || released)
+                        throw new Error('Observed PostgreSQL checkout is unavailable.');
+                };
+                const current = health, target = borrowed;
+                async function query(text, params) {
+                    try {
+                        current();
+                        const result = typeof text === 'string' ? await target.query(text, params) : await target.query(text);
+                        current();
+                        return result;
+                    }
+                    catch (error) {
+                        invalidateHealth();
+                        throw error;
+                    }
+                }
+                borrowed = Object.freeze({ query });
+            }
+            /* A write's own reads — the pre-flight count, a reread — are of the
+               primary it writes. */
+            const lent = borrowed;
+            /* A lease that runs statements it does not control (destroyAfter: the
+               SQL tool's) brackets each in a savepoint it awaits first
+               (postgresPooler.ts), so one sent behind another would overtake it:
+               that lease asks one at a time. */
+            if (client.pipeline === true && !destroyAfter)
+                pipelining.add(lent);
+            const running = write ? primaryReads(() => run(lent, health)) : run(borrowed, health);
+            if (stopped)
+                void running.catch(() => { });
+            const out = await (stopped ? Promise.race([running, stopped]) : running);
+            health?.();
+            await scoped?.settle();
+            health?.();
+            if (drainCancelled)
+                throwIfCancelled(signal);
+            return out;
+        }
+        catch (error) {
+            destroy = true;
+            if (scoped && !cancelled)
+                await scoped.settle().catch(() => { });
+            if (cancelled)
+                throw error instanceof StatementStopped ? error : new StatementStopped('cancelled', undefined, { cause: error });
+            throw error;
+        }
+        finally {
+            /* The person who wrote reads the primary for a while after, so the
+               list they go back to shows what they saved (replicas.ts). Noted
+               however the write ended: a failed one may still have committed. */
+            if (write)
+                noteWrite();
+            disarm();
+            try {
+                release(destroy || cancelled);
+            }
+            finally {
+                try {
+                    removeHealthListeners();
+                }
+                finally {
+                    try {
+                        giveBack();
+                    }
+                    finally {
+                        place?.();
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * A connection, or PoolBusy once what is left of the queue-wait budget, `waitMs`, is spent. The driver
+     * cannot withdraw a request it has queued, so one that arrives late is given
+     * straight back to the pool rather than leaked.
+     */
+    connectWithin(pool, role, waitMs) {
+        const connecting = this.connectHeld(pool, role);
+        return new Promise((resolve, reject) => {
+            let over = false;
+            const timer = setTimeout(() => {
+                over = true;
+                reject(new PoolBusy(role, 'queue_timeout'));
+            }, waitMs);
+            connecting.then((client) => {
+                clearTimeout(timer);
+                if (over)
+                    client.release();
+                else
+                    resolve(client);
+            }, (error) => {
+                clearTimeout(timer);
+                if (!over)
+                    reject(error);
+            });
+        });
+    }
+    /**
+     * `pool.connect()`, within the connections the server allows. Refused
+     * another one (53300), the pool is held at the connections it has, so this
+     * checkout and the next wait for one of them as they would behind a full
+     * pool, and it asks for more again after LIMIT_HOLD_MS; holding none, the
+     * checkout is refused as busy. A refusal at open leased nothing, so nothing
+     * on any connection moved: a tenant writer stays open on it
+     * (tenantWriteSource.ts). It used to rise as a database fault, and a
+     * tenant writer closed for it and reopened into the same refusal, again and
+     * again, with two commands in three failing at three users while its one
+     * connection sat idle (a field report: pools sized past the role's limit).
+     */
+    async connectHeld(pool, role) {
+        for (let attempt = 1;; attempt += 1) {
+            try {
+                return await pool.connect();
+            }
+            catch (error) {
+                if (!atConnectionLimit(error))
+                    throw error;
+                const held = this.holdAtLimit(pool, role, error);
+                if (!held || this.closed || attempt >= LIMIT_ATTEMPTS)
+                    throw new PoolBusy(role === 'read' ? 'read' : 'write', 'connection_limit', error);
+            }
+        }
+    }
+    /** Holds `pool` at the connections it has (pg-pool reads `options.max` at each checkout), says so once a spell, and returns how many. */
+    holdAtLimit(pool, role, refusal) {
+        const found = pool;
+        const held = typeof found.totalCount === 'number' ? found.totalCount : 0;
+        const size = this.meters[role].size;
+        const options = found.options;
+        if (held > 0 && options && typeof options.max === 'number') {
+            const most = Math.min(options.max, held);
+            options.max = most;
+            const line = role === 'background' ? undefined : this.admissions[role];
+            line?.resize(most);
+            clearTimeout(this.limitHolds[role]);
+            const timer = setTimeout(() => {
+                delete this.limitHolds[role];
+                options.max = size;
+                line?.resize(size);
+            }, LIMIT_HOLD_MS);
+            timer.unref?.();
+            this.limitHolds[role] = timer;
+        }
+        const now = Date.now();
+        if (now - (this.limitSaid[role] ?? -Infinity) >= LIMIT_HOLD_MS) {
+            this.limitSaid[role] = now;
+            const why = refusal instanceof Error ? refusal.message : 'refused';
+            console.error(`mistri: connection limit reached: the ${this.timingLabel} ${role} pool holds ${held} of ${size} connections; the database refused another (${why}). ` +
+                (held
+                    ? `It goes on with ${held} and asks for more in ${LIMIT_HOLD_MS / 1000} s. `
+                    : 'Its checkouts are answered busy (503, try again) until one opens. ') +
+                "This process's pools ask for more connections than the database allows: lower the pools (--read-pool, --write-pool, the auth pool) or raise the limit.");
+        }
+        return held;
+    }
+    /**
+     * Stop what backend `pid` is running, with pg_cancel_backend from a
+     * connection of its own: the pool's may all be busy, one of them with the
+     * statement being stopped. Best effort — a cancel that cannot connect
+     * leaves the statement to its cap.
+     */
+    async cancel(pid) {
+        if (!pid)
+            return;
+        try {
+            const pg = (await this.load());
+            const Client = pg.Client ?? pg.default?.Client;
+            if (!Client)
+                return;
+            const client = new Client({
+                ...this.dialOptions(),
+                connectionTimeoutMillis: 10_000,
+            });
+            client.on('error', () => { });
+            await client.connect();
+            try {
+                await client.query('SELECT pg_cancel_backend($1)', [pid]);
+            }
+            finally {
+                await client.end().catch(() => { });
+            }
+        }
+        catch {
+            /* The statement's cap still stops it. */
+        }
+    }
+    async endPools() {
+        const opened = await Promise.allSettled([this.reader, this.writer, this.background]);
+        const ended = await Promise.allSettled(opened.map((result) => (result.status === 'fulfilled' ? result.value?.end() : undefined)));
+        const failed = ended.find((result) => result.status === 'rejected');
+        if (failed?.status === 'rejected')
+            throw failed.reason;
+    }
+}
+/** The reader and writer counts a manager takes for `shape`: its own, else the process's (`--read-pool`, `--write-pool`) or the defaults. */
+export function poolSizes(shape = {}) {
+    const writers = writeOrder() === 'serial' ? 1 : (shape.writers ?? writerPoolSize());
+    return { writers, readers: shape.readers ?? serverSettings().readers ?? defaultReaders(writers) };
+}
+/* The managers holders share, by database URL and the settings that make two managers alike (sharedPgConnections). */
+const sharedManagers = new Map();
+/* A load hook told apart by identity in the key: two holders share only where they load pg alike. */
+const loadHooks = new WeakMap();
+let hooks = 0;
+const hookId = (hook) => {
+    let id = loadHooks.get(hook);
+    if (id === undefined)
+        loadHooks.set(hook, (id = ++hooks));
+    return id;
+};
+/**
+ * The manager every holder of `url` with this `shape` shares, instead of one
+ * each. Every pool sized for its process and blind to its neighbors was how a
+ * tenant App came to hold `3R + 6W + 5` connections on one database — its
+ * reader, discovery, writer, administration and support each with a manager of
+ * their own (docs/postgresql.md) — while each sat mostly idle: a field
+ * deployment whose role allowed eighteen ran fixed four-and-four pools and
+ * still could not fit. Shared, the readers and writers are one pool for all of
+ * them, sized once, and counted once (connectionBudget.ts).
+ *
+ * What sharing leaves alone: each holder keeps its own write lanes
+ * (writeLanes.ts), so per-record order is what it was; and a manager is shared
+ * only with holders that would have built the same one — the URL, the sizes
+ * after the process defaults, the pooler, the lock wait and the load hook all
+ * in the key — so a handler mounted with settings of its own never borrows
+ * another's pools.
+ *
+ * Each call is one holder. The handle is the manager behind a proxy whose
+ * `close` lets that holder go — once, however often it is called — and the
+ * pools close with the last holder; everything else is the manager's own, so
+ * the handle passes wherever a manager does (`instanceof` too).
+ */
+export function sharedPgConnections(url, shape = {}, onLoad) {
+    const { readers, writers } = poolSizes(shape);
+    const pipelineReads = shape.pipelineReads === true;
+    const readersOnly = shape.readersOnly === true;
+    const key = JSON.stringify([
+        url,
+        readers,
+        writers,
+        pipelineReads,
+        readersOnly,
+        serverSettings().pooler,
+        lockTimeoutMs(),
+        onLoad ? hookId(onLoad) : 0,
+    ]);
+    let entry = sharedManagers.get(key);
+    if (!entry) {
+        const manager = new PgConnectionManager(url, onLoad, undefined, { readers, writers, pipelineReads, readersOnly });
+        entry = { manager, holders: 0 };
+        sharedManagers.set(key, entry);
+    }
+    const held = entry;
+    held.holders += 1;
+    let released = false;
+    const close = () => {
+        if (released)
+            return Promise.resolve();
+        released = true;
+        held.holders -= 1;
+        if (held.holders > 0)
+            return Promise.resolve();
+        if (sharedManagers.get(key) === held)
+            sharedManagers.delete(key);
+        return held.manager.close();
+    };
+    /* Methods are bound once per handle: a store asks for withWriteClient on every statement. Kept by the function
+       found, not its name, so a method replaced on the class (a test's mock) is the one called. */
+    const bound = new WeakMap();
+    return new Proxy(held.manager, {
+        get(target, property) {
+            if (property === 'close')
+                return close;
+            const value = Reflect.get(target, property, target);
+            if (typeof value !== 'function')
+                return value;
+            let method = bound.get(value);
+            if (method === undefined)
+                bound.set(value, (method = value.bind(target)));
+            return method;
+        },
+    });
+}