@steve31415/baselib 3.9.0 → 3.11.0

package/dist/db.d.ts

@@ -41,13 +41,42 @@ export interface PoolOptions {
  * `set local statement_timeout = 0` — migrate() runs each file in its own
  * transaction, so the override scopes to that migration. */
 export declare const DEFAULT_STATEMENT_TIMEOUT_MS: number;
-/** pg-pool re-emits an idle client's socket error as its own 'error' event —
- * in production "Connection terminated unexpectedly" when Cloud SQL closes an
- * idle connection. Unhandled, that event throws and takes the process down
- * (todo 2026-08-20, notes 2026-08-27 ×2). The failed client has already been
- * removed from the pool and the next checkout opens a fresh connection, so
- * the right response is to log it and carry on. createPool attaches this;
- * exported for apps that build their own pg.Pool. */
+/** Keep a pg connection's death from killing the process, on both paths a
+ * client can die:
+ *
+ * - **Idle in the pool** — pg-pool re-emits the socket error as its own
+ *   'error' event (in production "Connection terminated unexpectedly" when
+ *   Cloud SQL closes an idle connection). Unhandled, that event throws and
+ *   takes the process down (todo 2026-08-20, notes 2026-08-27 ×2). The
+ *   failed client has already been removed from the pool and the next
+ *   checkout opens a fresh connection, so logging it is the whole response.
+ * - **Checked out** (`pool.connect()`) — pg-pool removes its listener at
+ *   checkout, so the client's own 'error' event is the borrower's problem;
+ *   with no listener it throws inside the socket callback and the process
+ *   dies (reader2's LISTEN bridge, 2026-09-10). Every checkout gets a WARN
+ *   listener here, attached on pg-pool's 'acquire' and removed on 'release'
+ *   (both fire while pg-pool's own listener is still attached, so there is
+ *   no listenerless instant). pg fails the in-flight and queued queries and
+ *   marks the client unqueryable *before* emitting, so a transaction on the
+ *   dead client still fails at its next await and rolls back server-side:
+ *   nothing is silently lost on the connection itself.
+ *
+ * **What this does NOT do — read before holding an advisory lock.** A
+ * session-level advisory lock, or a transaction-scoped one on a client kept
+ * idle in `begin` while the work runs elsewhere, evaporates the moment the
+ * connection dies. The listener only logs; the work that believed itself
+ * exclusive keeps running, unaware, until it next touches the dead client.
+ * Another instance can take the lock and run the same work concurrently in
+ * the meantime. Fleet policy (Steve, 2026-09-10; Plasticine Way
+ * `docs/DATABASE.md`): our advisory locks limit *duplicated effort*, they
+ * do not guarantee mutual exclusion, and code under one must be safe to
+ * overlap. Long-running or expensive work under a lock should check the
+ * lock connection periodically (a `select 1` on it) and stop on failure —
+ * mirror2's ingestion loop is the reference. Do not rely on the crash that
+ * used to happen here as a safety mechanism.
+ *
+ * createPool attaches all of this; exported for apps that build their own
+ * pg.Pool. */
 export declare function logPoolErrors(pool: pg.Pool, logger?: Logger): void;
 export declare function createPool(opts?: PoolOptions): Promise<pg.Pool>;
 /** End a pool and resolve only once every client's connection has actually

package/dist/db.js

@@ -44,19 +44,70 @@ export async function cloudSqlClientOptions(instance, auth) {
  * `set local statement_timeout = 0` — migrate() runs each file in its own
  * transaction, so the override scopes to that migration. */
 export const DEFAULT_STATEMENT_TIMEOUT_MS = 10 * 60_000;
-/** pg-pool re-emits an idle client's socket error as its own 'error' event —
- * in production "Connection terminated unexpectedly" when Cloud SQL closes an
- * idle connection. Unhandled, that event throws and takes the process down
- * (todo 2026-08-20, notes 2026-08-27 ×2). The failed client has already been
- * removed from the pool and the next checkout opens a fresh connection, so
- * the right response is to log it and carry on. createPool attaches this;
- * exported for apps that build their own pg.Pool. */
+/** Keep a pg connection's death from killing the process, on both paths a
+ * client can die:
+ *
+ * - **Idle in the pool** — pg-pool re-emits the socket error as its own
+ *   'error' event (in production "Connection terminated unexpectedly" when
+ *   Cloud SQL closes an idle connection). Unhandled, that event throws and
+ *   takes the process down (todo 2026-08-20, notes 2026-08-27 ×2). The
+ *   failed client has already been removed from the pool and the next
+ *   checkout opens a fresh connection, so logging it is the whole response.
+ * - **Checked out** (`pool.connect()`) — pg-pool removes its listener at
+ *   checkout, so the client's own 'error' event is the borrower's problem;
+ *   with no listener it throws inside the socket callback and the process
+ *   dies (reader2's LISTEN bridge, 2026-09-10). Every checkout gets a WARN
+ *   listener here, attached on pg-pool's 'acquire' and removed on 'release'
+ *   (both fire while pg-pool's own listener is still attached, so there is
+ *   no listenerless instant). pg fails the in-flight and queued queries and
+ *   marks the client unqueryable *before* emitting, so a transaction on the
+ *   dead client still fails at its next await and rolls back server-side:
+ *   nothing is silently lost on the connection itself.
+ *
+ * **What this does NOT do — read before holding an advisory lock.** A
+ * session-level advisory lock, or a transaction-scoped one on a client kept
+ * idle in `begin` while the work runs elsewhere, evaporates the moment the
+ * connection dies. The listener only logs; the work that believed itself
+ * exclusive keeps running, unaware, until it next touches the dead client.
+ * Another instance can take the lock and run the same work concurrently in
+ * the meantime. Fleet policy (Steve, 2026-09-10; Plasticine Way
+ * `docs/DATABASE.md`): our advisory locks limit *duplicated effort*, they
+ * do not guarantee mutual exclusion, and code under one must be safe to
+ * overlap. Long-running or expensive work under a lock should check the
+ * lock connection periodically (a `select 1` on it) and stop on failure —
+ * mirror2's ingestion loop is the reference. Do not rely on the crash that
+ * used to happen here as a safety mechanism.
+ *
+ * createPool attaches all of this; exported for apps that build their own
+ * pg.Pool. */
 export function logPoolErrors(pool, logger) {
-    pool.on('error', (err) => {
+    const warn = (message, err) => {
         if (logger)
-            logger.warn('database pool: idle client error', { error: err });
+            logger.warn(message, { error: err });
         else
-            console.warn('database pool: idle client error', serializeError(err));
+            console.warn(message, serializeError(err));
+    };
+    pool.on('error', (err) => warn('database pool: idle client error', err));
+    const listeners = new WeakMap();
+    pool.on('acquire', (client) => {
+        // pg emits twice for one death (the server's FATAL, then the socket
+        // close); one line per checkout is enough.
+        let logged = false;
+        const listener = (err) => {
+            if (logged)
+                return;
+            logged = true;
+            warn('database pool: checked-out client lost its connection (any transaction or advisory lock it held is gone; its next query fails)', err);
+        };
+        listeners.set(client, listener);
+        client.on('error', listener);
+    });
+    pool.on('release', (_err, client) => {
+        const listener = listeners.get(client);
+        if (!listener)
+            return;
+        listeners.delete(client);
+        client.removeListener('error', listener);
     });
 }
 export async function createPool(opts = {}) {
@@ -114,6 +165,9 @@ export async function endPool(pool) {
 // Plain-SQL migrations: numbered files (001-*.sql, 002-*.sql, …) applied in
 // order, recorded in schema_migrations, each inside a transaction, guarded by
 // an advisory lock so concurrent boots can't race. Applied at service boot.
+// Lock loss (see logPoolErrors): all the work runs on the lock's own client,
+// so a dead connection fails the next statement and the boot aborts — no
+// window in which a migration runs unlocked.
 export async function migrate(pool, migrationsDir, logger) {
     const files = (await readdir(migrationsDir)).filter((f) => f.endsWith('.sql')).sort();
     const client = await pool.connect();

package/dist/http.d.ts

@@ -16,7 +16,8 @@ export interface StartServerOptions {
     logger: Logger;
     port?: number;
 }
-/** Standard entry point: safety check, serve, graceful SIGTERM shutdown
+/** Standard entry point: safety check, crash logging (logUncaughtErrors),
+ *  serve, graceful SIGTERM shutdown
  *  (stop accepting, then flush all loggers, then exit). Returns the Node
  *  server so callers can attach protocol upgrades (e.g. @hono/node-ws's
  *  injectWebSocket). */

package/dist/http.js

@@ -3,7 +3,7 @@
 import { createMiddleware } from 'hono/factory';
 import { serve } from '@hono/node-server';
 import { assertBootSafety } from './config.js';
-import { flushAllLoggers } from './log.js';
+import { flushAllLoggers, logUncaughtErrors } from './log.js';
 import { truncate } from './log-core.js';
 /** One INFO line per request on exit (method, path, status, elapsed,
  *  identity); /health excluded. Entry-point detail beyond this (key
@@ -45,12 +45,14 @@ export function errorHandler(logger) {
             : c.text(`Internal error: ${String(err.message ?? err)}`, 500);
     };
 }
-/** Standard entry point: safety check, serve, graceful SIGTERM shutdown
+/** Standard entry point: safety check, crash logging (logUncaughtErrors),
+ *  serve, graceful SIGTERM shutdown
  *  (stop accepting, then flush all loggers, then exit). Returns the Node
  *  server so callers can attach protocol upgrades (e.g. @hono/node-ws's
  *  injectWebSocket). */
 export function startServer(opts) {
     assertBootSafety();
+    logUncaughtErrors(opts.logger);
     const port = opts.port ?? Number(process.env.PORT ?? 8080);
     const server = serve({ fetch: opts.app.fetch, port }, (info) => {
         opts.logger.info('server started', { port: info.port });

package/dist/log.d.ts

@@ -24,6 +24,21 @@ export declare function registerShipper(shipper: LogShipper): void;
 export declare function flushAllLoggers(opts?: {
     final?: boolean;
 }): Promise<void>;
+/** Make a crash visible where triage looks. Node's default for an uncaught
+ *  exception (and, under the default `--unhandled-rejections=throw`, an
+ *  unhandled rejection) is a stack on stderr and exit: on the Cloud Logging
+ *  backstop, but never an Axiom ERROR — an app could crash-loop with nothing
+ *  in the dataset the monitors watch (the fleet's pg connection-loss crashes
+ *  were found by hand, 2026-09-10). This logs the crash at ERROR, flushes
+ *  every registered shipper (time-boxed), then exits 1. It never keeps the
+ *  process alive: a failure while flushing exits at once. startServer
+ *  installs it; jobs and scripts call it themselves after createLogger. */
+export declare function logUncaughtErrors(logger: Logger, opts?: {
+    exit?: (code: number) => void;
+}): void;
+/** The listener logUncaughtErrors installs, separate for testing without
+ *  touching the real process. */
+export declare function uncaughtErrorHandler(logger: Logger, exit: (code: number) => void): (err: unknown, origin: string) => void;
 /** Ship-failure marker callbacks, writing one JSON line per event to stdout.
  *  writeSync, not process.stdout.write: the markers must survive an
  *  immediately following process.exit(), which can truncate async pipe

package/dist/log.js

@@ -26,6 +26,34 @@ export function registerShipper(shipper) {
 export async function flushAllLoggers(opts = {}) {
     await Promise.all([...allShippers].map((s) => (opts.final ? s.finalFlush() : s.flush())));
 }
+/** Make a crash visible where triage looks. Node's default for an uncaught
+ *  exception (and, under the default `--unhandled-rejections=throw`, an
+ *  unhandled rejection) is a stack on stderr and exit: on the Cloud Logging
+ *  backstop, but never an Axiom ERROR — an app could crash-loop with nothing
+ *  in the dataset the monitors watch (the fleet's pg connection-loss crashes
+ *  were found by hand, 2026-09-10). This logs the crash at ERROR, flushes
+ *  every registered shipper (time-boxed), then exits 1. It never keeps the
+ *  process alive: a failure while flushing exits at once. startServer
+ *  installs it; jobs and scripts call it themselves after createLogger. */
+export function logUncaughtErrors(logger, opts = {}) {
+    process.on('uncaughtException', uncaughtErrorHandler(logger, opts.exit ?? ((code) => process.exit(code))));
+}
+const CRASH_FLUSH_DEADLINE_MS = 3_000;
+/** The listener logUncaughtErrors installs, separate for testing without
+ *  touching the real process. */
+export function uncaughtErrorHandler(logger, exit) {
+    let crashing = false;
+    return (err, origin) => {
+        if (crashing)
+            return exit(1);
+        crashing = true;
+        console.error(err); // the stack on stderr, as Node would have printed it
+        logger.error('uncaught exception, exiting', { error: err, origin });
+        const done = () => exit(1);
+        setTimeout(done, CRASH_FLUSH_DEADLINE_MS);
+        flushAllLoggers({ final: true }).then(done, done);
+    };
+}
 /** Ship-failure marker callbacks, writing one JSON line per event to stdout.
  *  writeSync, not process.stdout.write: the markers must survive an
  *  immediately following process.exit(), which can truncate async pipe

package/dist/sync-browser/client.js

@@ -3,7 +3,11 @@ const PING_INTERVAL_MS = 30_000;
 const CONNECT_WATCHDOG_MS = 10_000;
 const DEFAULT_RECONNECT_BEFORE_MS = 55 * 60 * 1000;
 const FAILURE_TIMEOUT_MS = 60_000;
-const OUTAGE_ESCALATE_MS = 30_000;
+// ERROR only after a sustained foreground outage. The fleet is single-user
+// and triage runs daily, so detection latency is cheap; short client-network
+// blips (Wi-Fi roam, VPN re-auth — observed 30-40 s) must not page as ERROR
+// (Steve, 2026-09-11; raised from 30 s, which flagged those blips).
+const OUTAGE_ESCALATE_MS = 120_000;
 const CLOSE_NO_USER = 4403;
 const PROBE_AFTER_ATTEMPTS = 2;
 const SOCKET_OPEN = 1;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.9.0",
+  "version": "3.11.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",