@steve31415/baselib 3.11.0 → 3.13.0

package/dist/activity.d.ts

@@ -0,0 +1,14 @@
+export declare function requestStarted(): void;
+export declare function requestEnded(): void;
+/** The fields a diagnostic event carries: requests in flight right now and
+ *  how long ago the last one ended (undefined before the first request). */
+export declare function activitySnapshot(): {
+    inFlightRequests: number;
+    sinceLastRequestMs?: number;
+};
+/** Wrap a fetch-style handler so its requests count as activity. The
+ *  request ends when the handler returns its Response; a streaming body
+ *  that outlives that is not tracked. */
+export declare function trackActivity<A extends unknown[]>(fetch: (...args: A) => Response | Promise<Response>): (...args: A) => Promise<Response>;
+/** Test hook: forget everything. */
+export declare function resetActivity(): void;

package/dist/activity.js

@@ -0,0 +1,49 @@
+// Request activity of this process, for log events that need to know
+// whether Cloud Run had CPU allocated at the time: with request-based
+// billing (the fleet default) an instance's CPU is throttled to almost
+// nothing between requests, and timers, connection handshakes and
+// credential refreshes crawl. A connection failure logged with
+// `inFlightRequests: 0, sinceLastRequestMs: 31000` happened in that
+// throttled tail; one with `inFlightRequests: 1` did not.
+//
+// `startServer` tracks every HTTP request automatically. A custom server
+// wraps its fetch handler with `trackActivity`. WebSocket sessions are not
+// counted (they hold CPU on Cloud Run, but the upgrade bypasses fetch), so
+// on a WebSocket-heavy app zero in-flight requests does not prove the
+// instance was idle.
+let inFlight = 0;
+let lastRequestEndedAt = 0;
+export function requestStarted() {
+    inFlight += 1;
+}
+export function requestEnded() {
+    inFlight = Math.max(0, inFlight - 1);
+    lastRequestEndedAt = Date.now();
+}
+/** The fields a diagnostic event carries: requests in flight right now and
+ *  how long ago the last one ended (undefined before the first request). */
+export function activitySnapshot() {
+    return {
+        inFlightRequests: inFlight,
+        ...(lastRequestEndedAt ? { sinceLastRequestMs: Date.now() - lastRequestEndedAt } : {}),
+    };
+}
+/** Wrap a fetch-style handler so its requests count as activity. The
+ *  request ends when the handler returns its Response; a streaming body
+ *  that outlives that is not tracked. */
+export function trackActivity(fetch) {
+    return async (...args) => {
+        requestStarted();
+        try {
+            return await fetch(...args);
+        }
+        finally {
+            requestEnded();
+        }
+    };
+}
+/** Test hook: forget everything. */
+export function resetActivity() {
+    inFlight = 0;
+    lastRequestEndedAt = 0;
+}

package/dist/db.d.ts

@@ -2,23 +2,46 @@ import type { ConnectorOptions } from '@google-cloud/cloud-sql-connector';
 import pg from 'pg';
 import { type Logger } from './log-core.js';
 export declare const dateAsText: (v: string) => string;
+type Connector = import('@google-cloud/cloud-sql-connector').Connector;
+type InstanceOptions = Parameters<Connector['getOptions']>[0];
 /** Connection options for a Cloud SQL instance the fleet way: the Node
  * connector, IAM authentication, public IP. `auth` replaces the connector's
  * default credential resolution (the agent's tooling needs that; deployed
  * code uses the default). The connector stays open for the connections'
- * lifetime; close it when they are done. */
+ * lifetime; close it when they are done. `credentialExpiry` reads the
+ * expiry of the certificate it currently holds (see cloudSqlCredentialExpiry). */
 export declare function cloudSqlClientOptions(instance: string, auth?: ConnectorOptions['auth']): Promise<{
     options: Awaited<ReturnType<Connector['getOptions']>>;
     connector: Connector;
+    credentialExpiry: () => number | undefined;
 }>;
-type Connector = import('@google-cloud/cloud-sql-connector').Connector;
+/** Reads the expiry (epoch ms) of the ephemeral certificate the connector
+ * currently holds for an instance, or undefined when it cannot. The
+ * connector refreshes that certificate on a timer and never checks it on
+ * the connect path (1.11.3 `connector.js` `stream()`), so a connect that
+ * fails with the certificate already expired, or seconds from it, points
+ * at a refresh timer that fired late. This reads the connector's internal
+ * instance map (`instances.getInstance(opts).ephemeralCert.expirationTime`,
+ * an ISO string about an hour out; verified live against plasticine-pg on
+ * 2026-09-16). A connector version that moves it yields undefined, never a
+ * throw; the event then carries `credentialUnknown: true` so the gap is
+ * visible rather than read as "no credential involved". */
+export declare function cloudSqlCredentialExpiry(connector: Connector, instanceOptions: InstanceOptions): () => number | undefined;
 export interface PoolOptions {
     /** Override the database name (default: env DB_NAME / DATABASE_URL path). */
     database?: string;
+    /** Override the role to connect as (default: env DB_IAM_USER on Cloud
+     *  SQL, the DATABASE_URL user locally). */
+    user?: string;
+    /** Postgres command-line options sent in the startup packet, e.g.
+     *  `-c role=app_runtime -c search_path=app` (pg's `options`): session
+     *  settings that must hold before the first statement. */
+    sessionOptions?: string;
     max?: number;
-    /** How long a checkout waits for a free connection before failing (pg's
-     *  default is to wait forever, which turns pool starvation into a silent
-     *  hang instead of an error). */
+    /** How long a checkout waits, for a free connection or for a new one to
+     *  connect, before failing (default POOL_HYGIENE.connectionTimeoutMillis;
+     *  pg's own default is to wait forever, which turns pool starvation into
+     *  a silent hang instead of an error). */
     connectionTimeoutMillis?: number;
     /** Server-side cap on any single SQL statement, sent in each connection's
      *  startup packet (default DEFAULT_STATEMENT_TIMEOUT_MS; 0 disables). */
@@ -41,43 +64,59 @@ export interface PoolOptions {
  * `set local statement_timeout = 0` — migrate() runs each file in its own
  * transaction, so the override scopes to that migration. */
 export declare const DEFAULT_STATEMENT_TIMEOUT_MS: number;
-/** Keep a pg connection's death from killing the process, on both paths a
- * client can die:
- *
- * - **Idle in the pool** — pg-pool re-emits the socket error as its own
- *   'error' event (in production "Connection terminated unexpectedly" when
- *   Cloud SQL closes an idle connection). Unhandled, that event throws and
- *   takes the process down (todo 2026-08-20, notes 2026-08-27 ×2). The
- *   failed client has already been removed from the pool and the next
- *   checkout opens a fresh connection, so logging it is the whole response.
- * - **Checked out** (`pool.connect()`) — pg-pool removes its listener at
- *   checkout, so the client's own 'error' event is the borrower's problem;
- *   with no listener it throws inside the socket callback and the process
- *   dies (reader2's LISTEN bridge, 2026-09-10). Every checkout gets a WARN
- *   listener here, attached on pg-pool's 'acquire' and removed on 'release'
- *   (both fire while pg-pool's own listener is still attached, so there is
- *   no listenerless instant). pg fails the in-flight and queued queries and
- *   marks the client unqueryable *before* emitting, so a transaction on the
- *   dead client still fails at its next await and rolls back server-side:
- *   nothing is silently lost on the connection itself.
+/** Options for the connection-outcome events `logPoolErrors` emits. */
+export interface PoolLoggingOptions {
+    /** Epoch ms when the connector's cached Cloud SQL credential expires
+     *  (`cloudSqlCredentialExpiry`); logged as `credentialRemainingMs`. */
+    credentialExpiry?: () => number | undefined;
+}
+/** Pool settings every fleet pool carries; `createPool` applies them and
+ *  is the way to get them (a hand-built pool spreads this, but prefer
+ *  `createPool` with `user` / `sessionOptions` over building one). Decided
+ *  2026-09-16 (`~/migration/research/db-connection-lifecycle-2026-09-16.md`):
+ *  a bounded checkout wait so starvation and a stuck handshake fail loudly
+ *  instead of hanging, and TCP keepalive so a half-open socket is noticed
+ *  while the process has CPU. No max lifetime: reconnecting is the
+ *  operation that fails on a throttled instance, so forcing more of it
+ *  would be backwards. */
+export declare const FLEET_POOL_DEFAULTS: {
+    readonly connectionTimeoutMillis: 15000;
+    readonly keepAlive: true;
+    readonly keepAliveInitialDelayMillis: 10000;
+};
+/** Keep a pg connection's death from killing the process, and log every
+ *  way a pool loses or fails to get a connection as one WARN family:
+ *  `kind=db-connection-loss`, `meta.phase`:
+ *  - `connect`: a new connection failed to establish (handshake dropped,
+ *    credential rejected, `connectionTimeoutMillis` hit mid-handshake).
+ *    No statement ran, so the operation is safe to retry.
+ *  - `wait`: the checkout timed out waiting for a free client. Pool
+ *    starvation, not loss: the pool's `max` or a client held too long.
+ *  - `idle`: a client idle in the pool died; pg-pool discards it and the
+ *    next checkout opens a fresh one. Harmless by construction.
+ *  - `checked-out`: a borrowed client died. Its next query fails (pg has
+ *    already failed its queued queries and marked it unqueryable, so the
+ *    borrower cannot mistake the dead transaction for a live one), and
+ *    any transaction or advisory lock it held is gone.
+ *  Each event carries `elapsedMs` (time in that phase: the connect
+ *  attempt, the wait, the idle spell, the checkout), the pg `code`, the
+ *  pool's counts, `inFlightRequests` + `sinceLastRequestMs` (`activity.ts`:
+ *  was Cloud Run CPU allocated?) and, given a Cloud SQL credential reader,
+ *  `credentialRemainingMs`. Query:
+ *  `['apps'] | where kind == 'db-connection-loss' | summarize count() by app, tostring(meta.phase)`.
  *
- * **What this does NOT do — read before holding an advisory lock.** A
- * session-level advisory lock, or a transaction-scoped one on a client kept
- * idle in `begin` while the work runs elsewhere, evaporates the moment the
- * connection dies. The listener only logs; the work that believed itself
- * exclusive keeps running, unaware, until it next touches the dead client.
- * Another instance can take the lock and run the same work concurrently in
- * the meantime. Fleet policy (Steve, 2026-09-10; Plasticine Way
- * `docs/DATABASE.md`): our advisory locks limit *duplicated effort*, they
- * do not guarantee mutual exclusion, and code under one must be safe to
- * overlap. Long-running or expensive work under a lock should check the
- * lock connection periodically (a `select 1` on it) and stop on failure —
- * mirror2's ingestion loop is the reference. Do not rely on the crash that
- * used to happen here as a safety mechanism.
+ *  The checked-out path is the one that used to crash the process: pg-pool
+ *  listens for a client's `error` only while it is idle in the pool
+ *  (`docs/IMPL.md` has the incident history). The survival comes with a
+ *  rule: an advisory lock is not mutual exclusion. Our locks limit
+ *  duplicated effort; code under one must be safe to overlap, and long
+ *  work should probe the lock connection periodically (`select 1`) and
+ *  stop on failure (Plasticine Way `docs/DATABASE.md`; mirror2's ingestion
+ *  loop is the reference).
  *
- * createPool attaches all of this; exported for apps that build their own
- * pg.Pool. */
-export declare function logPoolErrors(pool: pg.Pool, logger?: Logger): void;
+ *  `createPool` attaches all of this; exported for a pool an app built
+ *  itself. Idempotent per pool. */
+export declare function logPoolErrors(pool: pg.Pool, logger?: Logger, options?: PoolLoggingOptions): void;
 export declare function createPool(opts?: PoolOptions): Promise<pg.Pool>;
 /** End a pool and resolve only once every client's connection has actually
  * closed. pg-pool's own `end()` resolves as soon as it has *asked* its idle

package/dist/db.js

@@ -5,6 +5,7 @@
 import { readdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import pg from 'pg';
+import { activitySnapshot } from './activity.js';
 import { requireEnv } from './config.js';
 import { serializeError } from './log-core.js';
 // DATE columns come back as 'YYYY-MM-DD' strings, not JS Dates: calendar
@@ -19,16 +20,41 @@ pg.types.setTypeParser(pg.types.builtins.DATE, dateAsText);
  * connector, IAM authentication, public IP. `auth` replaces the connector's
  * default credential resolution (the agent's tooling needs that; deployed
  * code uses the default). The connector stays open for the connections'
- * lifetime; close it when they are done. */
+ * lifetime; close it when they are done. `credentialExpiry` reads the
+ * expiry of the certificate it currently holds (see cloudSqlCredentialExpiry). */
 export async function cloudSqlClientOptions(instance, auth) {
     const { Connector, AuthTypes, IpAddressTypes } = await import('@google-cloud/cloud-sql-connector');
     const connector = new Connector(auth ? { auth } : {});
-    const options = await connector.getOptions({
+    const instanceOptions = {
         instanceConnectionName: instance,
         ipType: IpAddressTypes.PUBLIC,
         authType: AuthTypes.IAM,
-    });
-    return { options, connector };
+    };
+    const options = await connector.getOptions(instanceOptions);
+    return { options, connector, credentialExpiry: cloudSqlCredentialExpiry(connector, instanceOptions) };
+}
+/** Reads the expiry (epoch ms) of the ephemeral certificate the connector
+ * currently holds for an instance, or undefined when it cannot. The
+ * connector refreshes that certificate on a timer and never checks it on
+ * the connect path (1.11.3 `connector.js` `stream()`), so a connect that
+ * fails with the certificate already expired, or seconds from it, points
+ * at a refresh timer that fired late. This reads the connector's internal
+ * instance map (`instances.getInstance(opts).ephemeralCert.expirationTime`,
+ * an ISO string about an hour out; verified live against plasticine-pg on
+ * 2026-09-16). A connector version that moves it yields undefined, never a
+ * throw; the event then carries `credentialUnknown: true` so the gap is
+ * visible rather than read as "no credential involved". */
+export function cloudSqlCredentialExpiry(connector, instanceOptions) {
+    return () => {
+        try {
+            const internals = connector;
+            const ms = new Date(internals.instances.getInstance(instanceOptions).ephemeralCert.expirationTime).getTime();
+            return Number.isFinite(ms) ? ms : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    };
 }
 /** Why a fleet-wide statement timeout (mirror, 2026-09-08): a transaction an
  * app deliberately holds open across an application await (mirror's job
@@ -44,103 +70,176 @@ export async function cloudSqlClientOptions(instance, auth) {
  * `set local statement_timeout = 0` — migrate() runs each file in its own
  * transaction, so the override scopes to that migration. */
 export const DEFAULT_STATEMENT_TIMEOUT_MS = 10 * 60_000;
-/** Keep a pg connection's death from killing the process, on both paths a
- * client can die:
+/** Pool settings every fleet pool carries; `createPool` applies them and
+ *  is the way to get them (a hand-built pool spreads this, but prefer
+ *  `createPool` with `user` / `sessionOptions` over building one). Decided
+ *  2026-09-16 (`~/migration/research/db-connection-lifecycle-2026-09-16.md`):
+ *  a bounded checkout wait so starvation and a stuck handshake fail loudly
+ *  instead of hanging, and TCP keepalive so a half-open socket is noticed
+ *  while the process has CPU. No max lifetime: reconnecting is the
+ *  operation that fails on a throttled instance, so forcing more of it
+ *  would be backwards. */
+export const FLEET_POOL_DEFAULTS = {
+    connectionTimeoutMillis: 15_000,
+    keepAlive: true,
+    keepAliveInitialDelayMillis: 10_000,
+};
+const CONNECTION_LOSS_KIND = 'db-connection-loss';
+const ACQUIRE_FAILED_MESSAGE = {
+    connect: 'database pool: new connection failed',
+    wait: 'database pool: checkout timed out waiting for a free client',
+};
+const instrumented = new WeakSet();
+/** Keep a pg connection's death from killing the process, and log every
+ *  way a pool loses or fails to get a connection as one WARN family:
+ *  `kind=db-connection-loss`, `meta.phase`:
+ *  - `connect`: a new connection failed to establish (handshake dropped,
+ *    credential rejected, `connectionTimeoutMillis` hit mid-handshake).
+ *    No statement ran, so the operation is safe to retry.
+ *  - `wait`: the checkout timed out waiting for a free client. Pool
+ *    starvation, not loss: the pool's `max` or a client held too long.
+ *  - `idle`: a client idle in the pool died; pg-pool discards it and the
+ *    next checkout opens a fresh one. Harmless by construction.
+ *  - `checked-out`: a borrowed client died. Its next query fails (pg has
+ *    already failed its queued queries and marked it unqueryable, so the
+ *    borrower cannot mistake the dead transaction for a live one), and
+ *    any transaction or advisory lock it held is gone.
+ *  Each event carries `elapsedMs` (time in that phase: the connect
+ *  attempt, the wait, the idle spell, the checkout), the pg `code`, the
+ *  pool's counts, `inFlightRequests` + `sinceLastRequestMs` (`activity.ts`:
+ *  was Cloud Run CPU allocated?) and, given a Cloud SQL credential reader,
+ *  `credentialRemainingMs`. Query:
+ *  `['apps'] | where kind == 'db-connection-loss' | summarize count() by app, tostring(meta.phase)`.
  *
- * - **Idle in the pool** — pg-pool re-emits the socket error as its own
- *   'error' event (in production "Connection terminated unexpectedly" when
- *   Cloud SQL closes an idle connection). Unhandled, that event throws and
- *   takes the process down (todo 2026-08-20, notes 2026-08-27 ×2). The
- *   failed client has already been removed from the pool and the next
- *   checkout opens a fresh connection, so logging it is the whole response.
- * - **Checked out** (`pool.connect()`) — pg-pool removes its listener at
- *   checkout, so the client's own 'error' event is the borrower's problem;
- *   with no listener it throws inside the socket callback and the process
- *   dies (reader2's LISTEN bridge, 2026-09-10). Every checkout gets a WARN
- *   listener here, attached on pg-pool's 'acquire' and removed on 'release'
- *   (both fire while pg-pool's own listener is still attached, so there is
- *   no listenerless instant). pg fails the in-flight and queued queries and
- *   marks the client unqueryable *before* emitting, so a transaction on the
- *   dead client still fails at its next await and rolls back server-side:
- *   nothing is silently lost on the connection itself.
+ *  The checked-out path is the one that used to crash the process: pg-pool
+ *  listens for a client's `error` only while it is idle in the pool
+ *  (`docs/IMPL.md` has the incident history). The survival comes with a
+ *  rule: an advisory lock is not mutual exclusion. Our locks limit
+ *  duplicated effort; code under one must be safe to overlap, and long
+ *  work should probe the lock connection periodically (`select 1`) and
+ *  stop on failure (Plasticine Way `docs/DATABASE.md`; mirror2's ingestion
+ *  loop is the reference).
  *
- * **What this does NOT do — read before holding an advisory lock.** A
- * session-level advisory lock, or a transaction-scoped one on a client kept
- * idle in `begin` while the work runs elsewhere, evaporates the moment the
- * connection dies. The listener only logs; the work that believed itself
- * exclusive keeps running, unaware, until it next touches the dead client.
- * Another instance can take the lock and run the same work concurrently in
- * the meantime. Fleet policy (Steve, 2026-09-10; Plasticine Way
- * `docs/DATABASE.md`): our advisory locks limit *duplicated effort*, they
- * do not guarantee mutual exclusion, and code under one must be safe to
- * overlap. Long-running or expensive work under a lock should check the
- * lock connection periodically (a `select 1` on it) and stop on failure —
- * mirror2's ingestion loop is the reference. Do not rely on the crash that
- * used to happen here as a safety mechanism.
- *
- * createPool attaches all of this; exported for apps that build their own
- * pg.Pool. */
-export function logPoolErrors(pool, logger) {
-    const warn = (message, err) => {
-        if (logger)
-            logger.warn(message, { error: err });
+ *  `createPool` attaches all of this; exported for a pool an app built
+ *  itself. Idempotent per pool. */
+export function logPoolErrors(pool, logger, options = {}) {
+    if (instrumented.has(pool))
+        return;
+    instrumented.add(pool);
+    const log = logger?.with({ kind: CONNECTION_LOSS_KIND });
+    const warn = (phase, message, err, since) => {
+        const expiry = options.credentialExpiry?.();
+        const meta = {
+            phase,
+            ...(since === undefined ? {} : { elapsedMs: Date.now() - since }),
+            code: err.code,
+            pool: { total: pool.totalCount, idle: pool.idleCount, waiting: pool.waitingCount },
+            ...activitySnapshot(),
+            ...(options.credentialExpiry
+                ? expiry === undefined
+                    ? { credentialUnknown: true }
+                    : { credentialRemainingMs: expiry - Date.now() }
+                : {}),
+            error: err,
+        };
+        if (log)
+            log.warn(message, meta);
         else
-            console.warn(message, serializeError(err));
+            console.warn(message, { kind: CONNECTION_LOSS_KIND, ...meta, error: serializeError(err) });
+    };
+    // Per-client bookkeeping across one checkout: the borrower-side error
+    // listener (pg-pool drops its own at checkout) and when the client was
+    // taken and given back.
+    const clients = new WeakMap();
+    const record = (client) => {
+        let entry = clients.get(client);
+        if (!entry)
+            clients.set(client, (entry = {}));
+        return entry;
     };
-    pool.on('error', (err) => warn('database pool: idle client error', err));
-    const listeners = new WeakMap();
+    pool.on('error', (err, client) => {
+        warn('idle', 'database pool: idle client error', err, client ? clients.get(client)?.releasedAt : undefined);
+    });
     pool.on('acquire', (client) => {
+        const entry = record(client);
+        entry.acquiredAt = Date.now();
         // pg emits twice for one death (the server's FATAL, then the socket
         // close); one line per checkout is enough.
         let logged = false;
-        const listener = (err) => {
+        entry.listener = (err) => {
             if (logged)
                 return;
             logged = true;
-            warn('database pool: checked-out client lost its connection (any transaction or advisory lock it held is gone; its next query fails)', err);
+            warn('checked-out', 'database pool: checked-out client lost its connection (any transaction or advisory lock it held is gone; its next query fails)', err, entry.acquiredAt);
         };
-        listeners.set(client, listener);
-        client.on('error', listener);
+        client.on('error', entry.listener);
     });
     pool.on('release', (_err, client) => {
-        const listener = listeners.get(client);
-        if (!listener)
+        const entry = record(client);
+        entry.releasedAt = Date.now();
+        if (entry.listener)
+            client.removeListener('error', entry.listener);
+        entry.listener = undefined;
+    });
+    // pg-pool has no event for a failed checkout: `pool.connect()` just
+    // rejects, and `pool.query()` calls `this.connect(cb)` in callback form.
+    // Wrap the instance method so both forms log before propagating.
+    const connect = pool.connect.bind(pool);
+    const onFailure = (err, started) => {
+        const message = String(err?.message ?? '');
+        // A checkout after end() is the caller's shutdown bug, not a lost connection.
+        if (message.startsWith('Cannot use a pool after calling end'))
             return;
-        listeners.delete(client);
-        client.removeListener('error', listener);
+        const phase = message === 'timeout exceeded when trying to connect' ? 'wait' : 'connect';
+        warn(phase, ACQUIRE_FAILED_MESSAGE[phase], err, started);
+    };
+    pool.connect = ((cb) => {
+        const started = Date.now();
+        if (typeof cb === 'function') {
+            return connect((err, ...rest) => {
+                if (err)
+                    onFailure(err, started);
+                cb(err, ...rest);
+            });
+        }
+        return connect().catch((err) => {
+            onFailure(err, started);
+            throw err;
+        });
     });
 }
 export async function createPool(opts = {}) {
-    const pool = await newPool(opts);
-    logPoolErrors(pool, opts.logger);
-    return pool;
-}
-async function newPool(opts) {
     // pg sends statement_timeout in the startup packet of every connection —
     // server-enforced, no per-connection SET. A 0 is falsy, so pg omits it and
     // the server default (no timeout) applies.
     const statement_timeout = opts.statementTimeoutMillis ?? DEFAULT_STATEMENT_TIMEOUT_MS;
+    const shared = {
+        ...FLEET_POOL_DEFAULTS,
+        connectionTimeoutMillis: opts.connectionTimeoutMillis ?? FLEET_POOL_DEFAULTS.connectionTimeoutMillis,
+        statement_timeout,
+        ...(opts.sessionOptions ? { options: opts.sessionOptions } : {}),
+    };
     const instance = process.env.CLOUD_SQL_INSTANCE;
     if (instance) {
-        const { options: clientOpts } = await cloudSqlClientOptions(instance);
-        return new pg.Pool({
+        const { options: clientOpts, credentialExpiry } = await cloudSqlClientOptions(instance);
+        const pool = new pg.Pool({
             ...clientOpts,
-            user: requireEnv('DB_IAM_USER'),
+            ...shared,
+            user: opts.user ?? requireEnv('DB_IAM_USER'),
             database: opts.database ?? requireEnv('DB_NAME'),
             max: opts.max ?? 3, // shared instance: default max_connections is low
-            connectionTimeoutMillis: opts.connectionTimeoutMillis,
-            statement_timeout,
         });
+        logPoolErrors(pool, opts.logger, { credentialExpiry });
+        return pool;
     }
     const url = new URL(requireEnv('DATABASE_URL'));
     if (opts.database)
         url.pathname = `/${opts.database}`;
-    return new pg.Pool({
-        connectionString: url.toString(),
-        max: opts.max ?? 5,
-        connectionTimeoutMillis: opts.connectionTimeoutMillis,
-        statement_timeout,
-    });
+    if (opts.user)
+        url.username = opts.user;
+    const pool = new pg.Pool({ connectionString: url.toString(), ...shared, max: opts.max ?? 5 });
+    logPoolErrors(pool, opts.logger);
+    return pool;
 }
 /** End a pool and resolve only once every client's connection has actually
  * closed. pg-pool's own `end()` resolves as soon as it has *asked* its idle

package/dist/http.js

@@ -2,6 +2,7 @@
 // and the standard server entry point with graceful shutdown.
 import { createMiddleware } from 'hono/factory';
 import { serve } from '@hono/node-server';
+import { trackActivity } from './activity.js';
 import { assertBootSafety } from './config.js';
 import { flushAllLoggers, logUncaughtErrors } from './log.js';
 import { truncate } from './log-core.js';
@@ -54,7 +55,9 @@ export function startServer(opts) {
     assertBootSafety();
     logUncaughtErrors(opts.logger);
     const port = opts.port ?? Number(process.env.PORT ?? 8080);
-    const server = serve({ fetch: opts.app.fetch, port }, (info) => {
+    // Every request counts as activity (activity.ts) so connection-loss
+    // events can say whether Cloud Run had CPU allocated at the time.
+    const server = serve({ fetch: trackActivity(opts.app.fetch), port }, (info) => {
         opts.logger.info('server started', { port: info.port });
     });
     process.once('SIGTERM', () => {

package/dist/index.d.ts

@@ -2,5 +2,6 @@ export * from './config.js';
 export * from './log.js';
 export * from './s2s.js';
 export * from './auth.js';
+export { trackActivity } from './activity.js';
 export * from './db.js';
 export * from './http.js';

package/dist/index.js

@@ -2,5 +2,6 @@ export * from './config.js';
 export * from './log.js';
 export * from './s2s.js';
 export * from './auth.js';
+export { trackActivity } from './activity.js';
 export * from './db.js';
 export * from './http.js';

package/dist/sync-browser/client.d.ts

@@ -125,6 +125,11 @@ export declare class SyncClient<B, C extends EventBody, S extends EventBody, Sum
     private refetchInFlight;
     private probeInFlight;
     private resyncTimes;
+    /** Forward-gap events delivered while a resync is reconciling; drained
+     * after the bootstrap installs. Non-null exactly while reconciling. */
+    private gapBuffer;
+    private resyncBackoffTimer;
+    private loopEscalated;
     private reconciliationTail;
     private initialization;
     private disposed;
@@ -169,6 +174,18 @@ export declare class SyncClient<B, C extends EventBody, S extends EventBody, Sum
     private onSyncEstablished;
     private requestReload;
     private resyncFromServer;
+    /** Applies the events delivered while the bootstrap was in flight. On a
+     * single socket they are contiguous with the install point, so this is
+     * where a resync under a live stream converges; a hole (socket replaced
+     * mid-fetch, buffer overflow) schedules a fresh resync via the normal gap
+     * path instead. */
+    private drainAfterResync;
+    /** A stall episode ends only on proven convergence: a resync whose drain
+     * scheduled nothing further, or a complete socket catch-up. A successful
+     * reconcile alone is not convergence — in a chronic diverge-reconverge
+     * cycle every reconcile succeeds. */
+    private endResyncEpisode;
+    private scheduleResyncRetry;
     private sendConnect;
     private sendRaw;
     private trySend;

package/dist/sync-browser/client.js

@@ -8,6 +8,11 @@ const FAILURE_TIMEOUT_MS = 60_000;
 // blips (Wi-Fi roam, VPN re-auth — observed 30-40 s) must not page as ERROR
 // (Steve, 2026-09-11; raised from 30 s, which flagged those blips).
 const OUTAGE_ESCALATE_MS = 120_000;
+const RESYNC_LOOP_WINDOW_MS = 60_000;
+const RESYNC_LOOP_MAX = 3;
+const RESYNC_RETRY_MS = 60_000;
+const GAP_BUFFER_MAX = 1_000;
+const RECONCILE_MOVED_MAX_REFETCHES = 3;
 const CLOSE_NO_USER = 4403;
 const PROBE_AFTER_ATTEMPTS = 2;
 const SOCKET_OPEN = 1;
@@ -49,6 +54,11 @@ export class SyncClient {
     refetchInFlight = false;
     probeInFlight = false;
     resyncTimes = [];
+    /** Forward-gap events delivered while a resync is reconciling; drained
+     * after the bootstrap installs. Non-null exactly while reconciling. */
+    gapBuffer = null;
+    resyncBackoffTimer = null;
+    loopEscalated = false;
     reconciliationTail = Promise.resolve();
     initialization = null;
     disposed = false;
@@ -423,7 +433,22 @@ export class SyncClient {
             return true;
         }
         if (event.seq > currentSeq + 1) {
-            this.requestReload('sequence-gap');
+            // An own-write echo proves the commit even when the gap prevents
+            // applying it: settle the pending (without a settlement reconcile —
+            // the resync below installs fresh state anyway) so the 60 s failure
+            // timer cannot report a committed write as failed.
+            if (this.pending.has(event.eventGuid))
+                this.confirmPending(event.eventGuid, false);
+            if (this.gapBuffer !== null) {
+                if (this.gapBuffer.length < GAP_BUFFER_MAX)
+                    this.gapBuffer.push(event);
+                // On overflow, drop the newest: the kept prefix stays contiguous
+                // with the install point; the drain then ends at a hole and
+                // schedules a fresh resync.
+            }
+            else {
+                this.requestReload('sequence-gap');
+            }
             return false;
         }
         this.options.store.applyServerEvent(event);
@@ -579,15 +604,24 @@ export class SyncClient {
         });
         await preceding;
         try {
-            while (true) {
+            for (let attempt = 0;; attempt++) {
                 if (this.stopBeforeStoreMutation())
                     return false;
                 const startingSeq = this.options.store.currentSeq();
                 const bootstrap = await this.abortable((signal) => this.options.fetchBootstrap(signal));
                 if (this.stopBeforeStoreMutation())
                     return false;
-                if (this.options.store.currentSeq() !== startingSeq)
-                    continue;
+                if (this.options.store.currentSeq() !== startingSeq) {
+                    // The store advanced mid-fetch: live contiguous events are
+                    // applying, so the client is converged-and-advancing. Refetch a
+                    // bounded number of times for a quiescent snapshot; past that,
+                    // keep the live state — installing the older bootstrap would
+                    // rewind it. Any residual divergence surfaces as a normal gap
+                    // and a fresh, detector-counted resync.
+                    if (attempt < RECONCILE_MOVED_MAX_REFETCHES)
+                        continue;
+                    return true;
+                }
                 this.options.store.installBootstrap(bootstrap);
                 for (const [eventGuid, entry] of this.pending) {
                     if (this.settledPending.has(eventGuid))
@@ -641,9 +675,18 @@ export class SyncClient {
                 this.trySend(entry);
             }
         }
+        // A complete catch-up IS convergence: end any stall episode and cancel
+        // its pending backoff retry.
+        this.endResyncEpisode();
         this.updateConnectedState();
     }
     requestReload(reason) {
+        // While a resync is reconciling or backing off, further triggers are
+        // absorbed: the in-flight/scheduled attempt already covers them, and
+        // per-event callbacks were the WARN/ERROR spam of the 2026-09-10
+        // incident. The callback fires once per resync attempt.
+        if (this.refetchInFlight || this.resyncBackoffTimer !== null)
+            return;
         this.options.onReloadRequired(reason);
         void this.resyncFromServer(reason === 'server' ? 'server requested reload' : 'sequence gap');
     }
@@ -651,23 +694,35 @@ export class SyncClient {
         if (this.refetchInFlight || this.halted || this.disposed)
             return;
         const now = this.options.runtime.now();
-        this.resyncTimes = [...this.resyncTimes.filter((time) => now - time < 60_000), now];
-        if (this.resyncTimes.length > 3) {
-            this.options.logger.error('resync loop detected', { reason });
+        // `time <= now` drops entries stranded in the future by a backward
+        // clock jump, which would otherwise pin the detector tripped.
+        this.resyncTimes = [
+            ...this.resyncTimes.filter((time) => time <= now && now - time < RESYNC_LOOP_WINDOW_MS),
+            now,
+        ];
+        if (this.resyncTimes.length > RESYNC_LOOP_MAX) {
+            if (!this.loopEscalated) {
+                this.loopEscalated = true;
+                this.options.logger.error('resync loop detected', { reason });
+            }
             this.publish({
                 kind: 'error',
                 pending: this.pendingCount(),
-                message: 'Repeated resyncs require a page reload',
+                message: 'Repeated resyncs; retrying automatically',
             });
+            this.scheduleResyncRetry();
             return;
         }
         this.refetchInFlight = true;
+        this.gapBuffer = [];
+        let reconciled = false;
         try {
             this.options.logger.info('resyncing from server', { reason });
-            if (!(await this.reconcileStore()))
-                return;
-            for (const entry of this.pending.values())
-                this.trySend(entry);
+            reconciled = await this.reconcileStore();
+            if (reconciled) {
+                for (const entry of this.pending.values())
+                    this.trySend(entry);
+            }
         }
         catch (error) {
             if (!isLifecycleAbort(error) && !this.disposed && !this.halted) {
@@ -675,9 +730,54 @@ export class SyncClient {
             }
         }
         finally {
+            // Ordering matters: take the buffer and release the flag before the
+            // drain, so a drain-triggered nested resync owns `refetchInFlight`
+            // (and a fresh buffer) alone — this finally has nothing left to clear.
+            const buffered = this.gapBuffer ?? [];
+            this.gapBuffer = null;
             this.refetchInFlight = false;
+            if (reconciled)
+                this.drainAfterResync(buffered);
         }
     }
+    /** Applies the events delivered while the bootstrap was in flight. On a
+     * single socket they are contiguous with the install point, so this is
+     * where a resync under a live stream converges; a hole (socket replaced
+     * mid-fetch, buffer overflow) schedules a fresh resync via the normal gap
+     * path instead. */
+    drainAfterResync(buffered) {
+        for (const event of buffered) {
+            if (this.halted || this.disposed)
+                return;
+            if (!this.handleServerEvent(event))
+                return;
+        }
+        this.endResyncEpisode();
+    }
+    /** A stall episode ends only on proven convergence: a resync whose drain
+     * scheduled nothing further, or a complete socket catch-up. A successful
+     * reconcile alone is not convergence — in a chronic diverge-reconverge
+     * cycle every reconcile succeeds. */
+    endResyncEpisode() {
+        if (this.resyncBackoffTimer !== null) {
+            this.options.runtime.clearTimeout(this.resyncBackoffTimer);
+            this.resyncBackoffTimer = null;
+        }
+        if (!this.loopEscalated)
+            return;
+        this.loopEscalated = false;
+        this.updateConnectedState();
+    }
+    scheduleResyncRetry() {
+        if (this.resyncBackoffTimer !== null || this.halted || this.disposed)
+            return;
+        // A hidden tab's timer throttling may stretch this well past the nominal
+        // delay; that only postpones recovery, it never breaks it.
+        this.resyncBackoffTimer = this.options.runtime.setTimeout(() => {
+            this.resyncBackoffTimer = null;
+            void this.resyncFromServer('retry after resync backoff');
+        }, RESYNC_RETRY_MS);
+    }
     sendConnect() {
         this.sendRaw({
             type: 'connect',
@@ -800,11 +900,14 @@ export class SyncClient {
             this.options.runtime.clearInterval(this.countdownTimer);
         if (this.escalateTimer !== null)
             this.options.runtime.clearTimeout(this.escalateTimer);
+        if (this.resyncBackoffTimer !== null)
+            this.options.runtime.clearTimeout(this.resyncBackoffTimer);
         for (const timer of this.failureTimers.values())
             this.options.runtime.clearTimeout(timer);
         this.reconnectTimer = null;
         this.countdownTimer = null;
         this.escalateTimer = null;
+        this.resyncBackoffTimer = null;
         this.failureTimers.clear();
     }
     detachSocket(socket) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.11.0",
+  "version": "3.13.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",