@steve31415/baselib 3.41.0 → 3.43.0

package/dist/db.d.ts

@@ -49,6 +49,9 @@ export interface PoolOptions {
     /** Receives idle-client errors (see logPoolErrors). Without one they go to
      *  stderr — on the Cloud Logging backstop, but not in Axiom. */
     logger?: Logger;
+    /** Tuning for `probeIdleCheckouts` (defaults IDLE_PROBE_AFTER_MS and
+     *  IDLE_PROBE_TIMEOUT_MS); tests shorten them. */
+    idleProbe?: IdleProbeOptions;
 }
 /** Why a fleet-wide statement timeout (mirror, 2026-09-08): a transaction an
  * app deliberately holds open across an application await (mirror's job
@@ -78,12 +81,34 @@ export interface PoolLoggingOptions {
  *  instead of hanging, and TCP keepalive so a half-open socket is noticed
  *  while the process has CPU. No max lifetime: reconnecting is the
  *  operation that fails on a throttled instance, so forcing more of it
- *  would be backwards. */
+ *  would be backwards.
+ *
+ *  One warm connection (2026-10-30, `~/migration/prompts/baselib-pool-idle-timeout.md`):
+ *  pg-pool's default 10 s idle timeout emptied every pool between a single
+ *  user's requests, so nearly every request paid a ~77 ms Cloud SQL connect.
+ *  pg-pool 3.14's `min` does not open connections; it exempts the first
+ *  `min` connections from the idle timeout, so the pool keeps one once it
+ *  has made one. Connections above it close after `idleTimeoutMillis` (a
+ *  bound, never unlimited). A held connection can die while idle (the
+ *  weekly Thursday bursts), unnoticed until a checkout, so `createPool`
+ *  also probes a client that sat idle (`probeIdleCheckouts`). Budget: one
+ *  connection per pool per instance at rest, about +12 fleet-wide. A
+ *  process that forgets `pool.end()` no longer exits by itself: end pools
+ *  in scripts and jobs. */
 export declare const FLEET_POOL_DEFAULTS: {
     readonly connectionTimeoutMillis: 15000;
     readonly keepAlive: true;
     readonly keepAliveInitialDelayMillis: 10000;
+    readonly min: 1;
+    readonly idleTimeoutMillis: 60000;
 };
+/** A client idle at least this long is probed before it is handed out.
+ *  Below it the pool reused connections without a probe before 2026-10-30
+ *  too (they lived 10 s), so nothing is less safe than it was. */
+export declare const IDLE_PROBE_AFTER_MS = 10000;
+/** How long the probe waits. A healthy `select 1` takes a millisecond or
+ *  two; a half-open socket would otherwise wait for TCP to give up. */
+export declare const IDLE_PROBE_TIMEOUT_MS = 3000;
 /** Keep a pg connection's death from killing the process, and log every
  *  way a pool loses or fails to get a connection as one WARN family:
  *  `kind=db-connection-loss`, `meta.phase`:
@@ -93,7 +118,9 @@ export declare const FLEET_POOL_DEFAULTS: {
  *  - `wait`: the checkout timed out waiting for a free client. Pool
  *    starvation, not loss: the pool's `max` or a client held too long.
  *  - `idle`: a client idle in the pool died; pg-pool discards it and the
- *    next checkout opens a fresh one. Harmless by construction.
+ *    next checkout opens a fresh one. Harmless by construction. With
+ *    `meta.detectedAt: 'checkout'` the death went unnoticed until a
+ *    checkout's probe found it (`probeIdleCheckouts`); also harmless.
  *  - `checked-out`: a borrowed client died. Its next query fails (pg has
  *    already failed its queued queries and marked it unqueryable, so the
  *    borrower cannot mistake the dead transaction for a live one), and
@@ -117,6 +144,24 @@ export declare const FLEET_POOL_DEFAULTS: {
  *  `createPool` attaches all of this; exported for a pool an app built
  *  itself. Idempotent per pool. */
 export declare function logPoolErrors(pool: pg.Pool, logger?: Logger, options?: PoolLoggingOptions): void;
+export interface IdleProbeOptions {
+    afterIdleMs?: number;
+    timeoutMs?: number;
+}
+/** Hand out no pooled client that died while idle. A client idle for
+ *  `afterIdleMs` or more runs `select 1` (bounded by `timeoutMs`) before
+ *  the borrower gets it; if that fails the client is discarded, one WARN
+ *  (`kind=db-connection-loss`, `phase=idle`, `detectedAt=checkout`) is
+ *  logged, and the checkout is retried, ending at a fresh connection at
+ *  worst. Retrying is safe because the probe is the only statement that
+ *  ran on the dead client. What stays unretried, as before: a connection
+ *  that dies after the probe, mid-statement or mid-transaction (the
+ *  borrower sees the error; whether a lone statement took effect is
+ *  unknowable), and a failed new connection (`phase=connect`). New clients
+ *  and recently used ones are not probed. Both connect forms are covered
+ *  (`pool.query` checks out in callback form). `createPool` attaches this;
+ *  exported for a pool an app built itself. Idempotent per pool. */
+export declare function probeIdleCheckouts(pool: pg.Pool, logger?: Logger, options?: IdleProbeOptions): void;
 export declare function createPool(opts?: PoolOptions): Promise<pg.Pool>;
 /** End a pool and resolve only once every client's connection has actually
  * closed. pg-pool's own `end()` resolves as soon as it has *asked* its idle

package/dist/db.js

@@ -78,18 +78,69 @@ export const DEFAULT_STATEMENT_TIMEOUT_MS = 10 * 60_000;
  *  instead of hanging, and TCP keepalive so a half-open socket is noticed
  *  while the process has CPU. No max lifetime: reconnecting is the
  *  operation that fails on a throttled instance, so forcing more of it
- *  would be backwards. */
+ *  would be backwards.
+ *
+ *  One warm connection (2026-10-30, `~/migration/prompts/baselib-pool-idle-timeout.md`):
+ *  pg-pool's default 10 s idle timeout emptied every pool between a single
+ *  user's requests, so nearly every request paid a ~77 ms Cloud SQL connect.
+ *  pg-pool 3.14's `min` does not open connections; it exempts the first
+ *  `min` connections from the idle timeout, so the pool keeps one once it
+ *  has made one. Connections above it close after `idleTimeoutMillis` (a
+ *  bound, never unlimited). A held connection can die while idle (the
+ *  weekly Thursday bursts), unnoticed until a checkout, so `createPool`
+ *  also probes a client that sat idle (`probeIdleCheckouts`). Budget: one
+ *  connection per pool per instance at rest, about +12 fleet-wide. A
+ *  process that forgets `pool.end()` no longer exits by itself: end pools
+ *  in scripts and jobs. */
 export const FLEET_POOL_DEFAULTS = {
     connectionTimeoutMillis: 15_000,
     keepAlive: true,
     keepAliveInitialDelayMillis: 10_000,
+    min: 1,
+    idleTimeoutMillis: 60_000,
 };
+/** A client idle at least this long is probed before it is handed out.
+ *  Below it the pool reused connections without a probe before 2026-10-30
+ *  too (they lived 10 s), so nothing is less safe than it was. */
+export const IDLE_PROBE_AFTER_MS = 10_000;
+/** How long the probe waits. A healthy `select 1` takes a millisecond or
+ *  two; a half-open socket would otherwise wait for TCP to give up. */
+export const IDLE_PROBE_TIMEOUT_MS = 3_000;
+const IDLE_PROBE_SQL = 'select 1 /* baselib idle probe */';
 const CONNECTION_LOSS_KIND = 'db-connection-loss';
 const ACQUIRE_FAILED_MESSAGE = {
     connect: 'database pool: new connection failed',
     wait: 'database pool: checkout timed out waiting for a free client',
 };
 const instrumented = new WeakSet();
+// Clients under an idle probe: their death is reported once, by the probe,
+// as an idle loss rather than a checked-out one.
+const probing = new WeakSet();
+/** One `kind=db-connection-loss` WARN, the shape every pool loss shares. */
+function connectionLossWarner(pool, logger, credentialExpiry) {
+    const log = logger?.with({ kind: CONNECTION_LOSS_KIND });
+    return (phase, message, err, since, extra = {}) => {
+        const expiry = credentialExpiry?.();
+        const meta = {
+            phase,
+            ...extra,
+            ...(since === undefined ? {} : { elapsedMs: Date.now() - since }),
+            code: err.code,
+            pool: { total: pool.totalCount, idle: pool.idleCount, waiting: pool.waitingCount },
+            ...(credentialExpiry
+                ? expiry === undefined
+                    ? { credentialUnknown: true }
+                    : { credentialRemainingMs: expiry - Date.now() }
+                : {}),
+            error: err,
+        };
+        if (log)
+            log.warn(message, meta);
+        // No logger to add the activity fields, so add them here.
+        else
+            console.warn(message, { kind: CONNECTION_LOSS_KIND, ...processMeta(), ...meta, error: serializeError(err) });
+    };
+}
 /** Keep a pg connection's death from killing the process, and log every
  *  way a pool loses or fails to get a connection as one WARN family:
  *  `kind=db-connection-loss`, `meta.phase`:
@@ -99,7 +150,9 @@ const instrumented = new WeakSet();
  *  - `wait`: the checkout timed out waiting for a free client. Pool
  *    starvation, not loss: the pool's `max` or a client held too long.
  *  - `idle`: a client idle in the pool died; pg-pool discards it and the
- *    next checkout opens a fresh one. Harmless by construction.
+ *    next checkout opens a fresh one. Harmless by construction. With
+ *    `meta.detectedAt: 'checkout'` the death went unnoticed until a
+ *    checkout's probe found it (`probeIdleCheckouts`); also harmless.
  *  - `checked-out`: a borrowed client died. Its next query fails (pg has
  *    already failed its queued queries and marked it unqueryable, so the
  *    borrower cannot mistake the dead transaction for a live one), and
@@ -126,27 +179,7 @@ export function logPoolErrors(pool, logger, options = {}) {
     if (instrumented.has(pool))
         return;
     instrumented.add(pool);
-    const log = logger?.with({ kind: CONNECTION_LOSS_KIND });
-    const warn = (phase, message, err, since) => {
-        const expiry = options.credentialExpiry?.();
-        const meta = {
-            phase,
-            ...(since === undefined ? {} : { elapsedMs: Date.now() - since }),
-            code: err.code,
-            pool: { total: pool.totalCount, idle: pool.idleCount, waiting: pool.waitingCount },
-            ...(options.credentialExpiry
-                ? expiry === undefined
-                    ? { credentialUnknown: true }
-                    : { credentialRemainingMs: expiry - Date.now() }
-                : {}),
-            error: err,
-        };
-        if (log)
-            log.warn(message, meta);
-        // No logger to add the activity fields, so add them here.
-        else
-            console.warn(message, { kind: CONNECTION_LOSS_KIND, ...processMeta(), ...meta, error: serializeError(err) });
-    };
+    const warn = connectionLossWarner(pool, logger, options.credentialExpiry);
     // Per-client bookkeeping across one checkout: the borrower-side error
     // listener (pg-pool drops its own at checkout) and when the client was
     // taken and given back.
@@ -167,7 +200,7 @@ export function logPoolErrors(pool, logger, options = {}) {
         // close); one line per checkout is enough.
         let logged = false;
         entry.listener = (err) => {
-            if (logged)
+            if (logged || probing.has(client))
                 return;
             logged = true;
             warn('checked-out', 'database pool: checked-out client lost its connection (any transaction or advisory lock it held is gone; its next query fails)', err, entry.acquiredAt);
@@ -208,6 +241,76 @@ export function logPoolErrors(pool, logger, options = {}) {
         });
     });
 }
+const probed = new WeakSet();
+/** Hand out no pooled client that died while idle. A client idle for
+ *  `afterIdleMs` or more runs `select 1` (bounded by `timeoutMs`) before
+ *  the borrower gets it; if that fails the client is discarded, one WARN
+ *  (`kind=db-connection-loss`, `phase=idle`, `detectedAt=checkout`) is
+ *  logged, and the checkout is retried, ending at a fresh connection at
+ *  worst. Retrying is safe because the probe is the only statement that
+ *  ran on the dead client. What stays unretried, as before: a connection
+ *  that dies after the probe, mid-statement or mid-transaction (the
+ *  borrower sees the error; whether a lone statement took effect is
+ *  unknowable), and a failed new connection (`phase=connect`). New clients
+ *  and recently used ones are not probed. Both connect forms are covered
+ *  (`pool.query` checks out in callback form). `createPool` attaches this;
+ *  exported for a pool an app built itself. Idempotent per pool. */
+export function probeIdleCheckouts(pool, logger, options = {}) {
+    if (probed.has(pool))
+        return;
+    probed.add(pool);
+    const afterIdleMs = options.afterIdleMs ?? IDLE_PROBE_AFTER_MS;
+    const timeoutMs = options.timeoutMs ?? IDLE_PROBE_TIMEOUT_MS;
+    const releasedAt = new WeakMap();
+    pool.on('release', (_err, client) => releasedAt.set(client, Date.now()));
+    const warn = connectionLossWarner(pool, logger);
+    const connect = pool.connect.bind(pool);
+    const checkout = async () => {
+        // Each failed probe removes a client, so this ends at a client that
+        // passes or a fresh one (or a connect error, which propagates).
+        for (;;) {
+            const client = await connect();
+            const idleSince = releasedAt.get(client);
+            if (idleSince === undefined || Date.now() - idleSince < afterIdleMs)
+                return client;
+            probing.add(client);
+            // The death also arrives as a client 'error', possibly after the
+            // discard below; the probe reports it, and a discarded client keeps
+            // this listener so a late socket error cannot go unhandled.
+            const ignore = () => { };
+            client.on('error', ignore);
+            let timer;
+            try {
+                const probe = client.query(IDLE_PROBE_SQL);
+                // On a timeout the release below destroys the socket (pg's end()
+                // with a query active), which rejects the probe; nothing awaits it.
+                probe.catch(() => { });
+                await Promise.race([
+                    probe,
+                    new Promise((_, reject) => {
+                        timer = setTimeout(() => reject(new Error(`idle probe timed out after ${timeoutMs} ms`)), timeoutMs);
+                    }),
+                ]);
+                client.removeListener('error', ignore);
+                return client;
+            }
+            catch (err) {
+                const error = err instanceof Error ? err : new Error(String(err));
+                warn('idle', 'database pool: idle client found dead at checkout; replaced', error, idleSince, { detectedAt: 'checkout' });
+                client.release(error);
+            }
+            finally {
+                clearTimeout(timer);
+                probing.delete(client);
+            }
+        }
+    };
+    pool.connect = ((cb) => {
+        if (typeof cb !== 'function')
+            return checkout();
+        checkout().then((client) => cb(undefined, client, (release) => client.release(release)), (err) => cb(err));
+    });
+}
 export async function createPool(opts = {}) {
     // pg sends statement_timeout in the startup packet of every connection —
     // server-enforced, no per-connection SET. A 0 is falsy, so pg omits it and
@@ -230,6 +333,7 @@ export async function createPool(opts = {}) {
             max: opts.max ?? 3, // shared instance: default max_connections is low
         });
         logPoolErrors(pool, opts.logger, { credentialExpiry });
+        probeIdleCheckouts(pool, opts.logger, opts.idleProbe);
         return pool;
     }
     const url = new URL(requireEnv('DATABASE_URL'));
@@ -239,6 +343,7 @@ export async function createPool(opts = {}) {
         url.username = opts.user;
     const pool = new pg.Pool({ connectionString: url.toString(), ...shared, max: opts.max ?? 5 });
     logPoolErrors(pool, opts.logger);
+    probeIdleCheckouts(pool, opts.logger, opts.idleProbe);
     return pool;
 }
 /** End a pool and resolve only once every client's connection has actually

package/dist/log-core.d.ts

@@ -65,7 +65,13 @@ export interface ShipperOptions {
      *  (uncounted) attempt (default 40 s). */
     offlineRetryMs?: number;
     /** Per-attempt fetch timeout (default 10s; 3s aborted most of the fleet's
-     *  attempts in the two weeks to 2026-08-28). */
+     *  attempts in the two weeks to 2026-08-28). Headroom judgment 2026-10-30
+     *  (watchdog task f4ec3337): steady-state ingest takes ~0.2–2 s (≤ 0.2 of
+     *  the limit). What still times out is a fresh instance's startup line
+     *  shipped while Cloud Run throttles its CPU (no request in flight), which
+     *  a longer timeout would not cure — it waits for CPU, not for Axiom — and
+     *  flush() callers wait up to this + 500 ms, so raising it lengthens the
+     *  worst-case request. Kept at 10 s; the retry delivers those batches. */
     attemptTimeoutMs?: number;
     /** Caps across ALL undelivered batches; overflow drops oldest with an
      *  onGiveUp('buffer-overflow'). Defaults 5000 events / 5 MB. */

package/dist/log.js

@@ -88,6 +88,14 @@ export function stdoutShipMarkers(app, dataset) {
             attempt: batch.attempt,
             max_attempts: batch.maxAttempts,
             error: String(error),
+            // Whether Cloud Run had CPU allocated: request-based billing throttles
+            // an idle instance to almost nothing, and a fresh instance's startup
+            // line ships just after its startup probe passes — with no request in
+            // flight when a deploy or replacement starts it ahead of traffic. Its
+            // 10 s attempt then times out (watchdog task f4ec3337, 2026-10-30:
+            // all 12 batches that failed on 10-29 13:33–16:25Z were such startup lines).
+            uptime_ms: Math.round(process.uptime() * 1000),
+            ...processMeta(),
             message: `axiom ship attempt ${batch.attempt}/${batch.maxAttempts} failed: ${String(error)} ` +
                 `(${batch.events.length} events, ${batch.bytes} bytes; will retry)`,
         }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.41.0",
+  "version": "3.43.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",