@steve31415/baselib 3.43.0 → 3.45.0

package/dist/auth.d.ts

@@ -1,5 +1,5 @@
 import type { Context } from 'hono';
-import type { Logger } from './log-core.js';
+import { type Logger } from './log-core.js';
 import { ServiceCaller, S2sOptions } from './s2s.js';
 export declare const DEFAULT_AUTH_URL = "https://auth.apps.snewman.net";
 export declare const SESSION_COOKIE = "pw_session";

package/dist/auth.js

@@ -9,7 +9,11 @@
 //      policy, app-level checks) still runs.
 //   2. Authorization: Bearer → service-to-service OIDC (s2s.ts).
 //   3. pw_session cookie → whoami call to the auth service, cached
-//      in-instance (10 min hits / 30 s misses — the old-world numbers).
+//      in-instance (10 min hits / 30 s misses — the old-world numbers),
+//      one call per cookie at a time (concurrent misses share it).
+//      Every call (never a cache hit) logs one INFO 'session check' event
+//      saying why the cache could not answer and what the call cost the
+//      waiting request (~/migration/research/whoami-latency-2026-10-30.md).
 //
 // Then the user-access policy: by default an app admits only users whose
 // auth role is 'owner'; `userAccess: 'invited'` admits any active fleet
@@ -25,6 +29,7 @@
 import { createHash } from 'node:crypto';
 import { createMiddleware } from 'hono/factory';
 import { authMode } from './config.js';
+import { truncate } from './log-core.js';
 import { RateLimiter, resolveS2sCaller } from './s2s.js';
 export const DEFAULT_AUTH_URL = 'https://auth.apps.snewman.net';
 export const SESSION_COOKIE = 'pw_session';
@@ -46,6 +51,13 @@ export function requireIdentity(opts) {
     const policy = opts.userAccess ?? 'owner';
     const limiter = new RateLimiter(opts.rateLimitPerMin ?? 600);
     const cache = new Map();
+    // The whoami call in flight per cache key (single-flight).
+    const inFlight = new Map();
+    // Session-check log state (per middleware, like the cache): cache hits
+    // since the last whoami call, and when the last call ended.
+    let hitsSinceLastCall = 0;
+    let callCount = 0;
+    let lastCallEndedAt;
     const isApi = opts.isApiRequest ??
         ((c) => c.req.path.startsWith('/api/') || (c.req.header('accept') ?? '').includes('application/json'));
     const unauthenticated = (c) => {
@@ -108,18 +120,77 @@ export function requireIdentity(opts) {
         const key = createHash('sha256').update(cookieHeader).digest('hex');
         const cached = cache.get(key);
         if (cached && cached.expires > Date.now()) {
+            hitsSinceLastCall++;
             if (!cached.user)
                 return unauthenticated(c);
             return admit(cached.user);
         }
+        // Single-flight: a request that misses while a call for the same cookie
+        // is already in flight shares that call's answer instead of making its
+        // own (30% of calls were such duplicates). Every request still gets an
+        // answer fetched during its own lifetime, so what is admitted does not
+        // change; the answer is applied per request below.
+        const flight = inFlight.get(key);
+        if (flight) {
+            flight.joiners.count++;
+            return settle(await flight.outcome);
+        }
+        const joiners = { count: 0 };
+        const outcome = callWhoami(c, key, cached, cookieHeader, joiners).finally(() => inFlight.delete(key));
+        inFlight.set(key, { outcome, joiners });
+        return settle(await outcome);
+        function settle(outcome) {
+            if (outcome.kind === 'user')
+                return admit(outcome.user);
+            if (outcome.kind === 'unauthenticated')
+                return unauthenticated(c);
+            return c.json({ error: 'auth service unavailable' }, 503);
+        }
+    });
+    // One whoami call, whose answer every request that joined it shares. The
+    // waiting request pays its full latency. One INFO event per call (a few
+    // hundred a day fleet-wide; never per hit or per joined request), with a
+    // fixed vocabulary of keys and nothing from the cookie but how many
+    // cookies the header carries (a changing header defeats the cache).
+    async function callWhoami(c, key, cached, cookieHeader, joiners) {
+        const started = Date.now();
+        const cacheState = !cached ? 'absent' : cached.user ? 'expired-user' : 'expired-unauthenticated';
+        const callFields = {
+            cacheState,
+            ...(cached ? { entryAgeMs: started - cached.filled } : {}),
+            hitsSinceLastCall,
+            ...(lastCallEndedAt !== undefined ? { sinceLastCallMs: started - lastCallEndedAt } : {}),
+            ...(callCount === 0 ? { firstCall: true } : {}),
+            uptimeMs: Math.round(process.uptime() * 1000),
+            method: c.req.method,
+            path: truncate(c.req.path, 200),
+            upgrade: (c.req.header('upgrade') ?? '').toLowerCase() === 'websocket',
+            cookieCount: cookieHeader.split(';').filter((part) => part.includes('=')).length,
+        };
+        hitsSinceLastCall = 0;
+        callCount++;
+        // `joinedRequests`: requests that shared this call (single-flight),
+        // counted when the call ends, so it includes every joiner.
+        const callEnded = (status) => {
+            const ended = Date.now();
+            lastCallEndedAt = ended;
+            opts.logger?.info('session check', {
+                ...callFields,
+                joinedRequests: joiners.count,
+                status,
+                elapsedMs: ended - started,
+            });
+        };
         let res;
         try {
             res = await whoamiFetch(`${authUrl}/api/whoami`, { headers: { cookie: cookieHeader } });
         }
         catch (err) {
+            callEnded(0);
             opts.logger?.error('auth service unreachable', { error: err });
-            return c.json({ error: 'auth service unavailable' }, 503);
+            return { kind: 'unavailable' };
         }
+        callEnded(res.status);
         if (res.ok) {
             const data = (await res.json());
             const role = parseRole(data.role);
@@ -128,19 +199,21 @@ export function requireIdentity(opts) {
                 // protocol change: a fault, not an "invited" answer. Fail closed,
                 // loudly, and without caching so recovery is instant.
                 opts.logger?.error('whoami answer has no role', { email: data.email, role: data.role });
-                return c.json({ error: 'auth service unavailable' }, 503);
+                return { kind: 'unavailable' };
             }
             const user = { type: 'user', email: data.email, name: data.name, userId: data.user_id, role };
-            cache.set(key, { expires: Date.now() + hitMs, user });
-            return admit(user);
+            const now = Date.now();
+            cache.set(key, { expires: now + hitMs, filled: now, user });
+            return { kind: 'user', user };
         }
         if (res.status === 401) {
-            cache.set(key, { expires: Date.now() + missMs });
-            return unauthenticated(c);
+            const now = Date.now();
+            cache.set(key, { expires: now + missMs, filled: now });
+            return { kind: 'unauthenticated' };
         }
         opts.logger?.error('auth service error', { meta_status: res.status });
-        return c.json({ error: 'auth service unavailable' }, 503);
-    });
+        return { kind: 'unavailable' };
+    }
 }
 // The fleet-wide "not enabled" page, in the auth service's plain style so
 // every app shows the same thing. Carries nothing but the request's host,

package/dist/sync/db-bridge.d.ts

@@ -0,0 +1,89 @@
+import type pg from 'pg';
+import type { Logger } from '../log.js';
+/** How often the bridge proves its LISTEN connection alive with a `select
+ *  1`. Cloud SQL drops idle connections silently, the error surfaces only on
+ *  the next query, and a pure listener never issues one, so without this the
+ *  bridge would go permanently deaf while its logs claim health. */
+export declare const DB_BRIDGE_HEARTBEAT_MS = 60000;
+/** How long a heartbeat (or the `unlisten` in stop()) may take before the
+ *  connection is declared lost: a half-open socket never answers and never
+ *  errors. Generous, because a request-billed Cloud Run instance runs this
+ *  timer with almost no CPU. */
+export declare const DB_BRIDGE_HEARTBEAT_TIMEOUT_MS = 10000;
+export declare const DB_BRIDGE_RECONNECT_DELAYS_MS: readonly number[];
+export interface DbBroadcastBridgeOptions {
+    /** The app's pool. The bridge holds one of its clients for as long as it
+     *  runs, so size `max` as working connections + 1, and stop() the bridge
+     *  before the pool ends. */
+    pool: pg.Pool;
+    /** The NOTIFY channel, per app (`<app>_updates`): a lowercase identifier. */
+    channel: string;
+    logger: Logger;
+    /** Each notification's parsed JSON payload. An invalid payload is logged
+     *  and skipped; a throw is logged and does not reach pg. */
+    onMessage: (message: unknown) => void;
+    /** Called after the listener reconnects (not on the first connect).
+     *  Notifications sent while it was down are lost and unknowable, so the
+     *  app re-announces availability to every local scope. */
+    onRecovered: () => void;
+    /** Skip messages whose top-level `origin` is this bridge's `originId`,
+     *  for an app that delivers its own writes locally and publishes them via
+     *  notify() for the other copies. Default false. */
+    ignoreOwn?: boolean;
+    heartbeatMs?: number;
+    heartbeatTimeoutMs?: number;
+    reconnectDelaysMs?: readonly number[];
+}
+type Queryable = Pick<pg.Pool | pg.PoolClient, 'query'>;
+export declare class DbBroadcastBridge {
+    /** Tags this process's notify() payloads when `ignoreOwn` is on. */
+    readonly originId: `${string}-${string}-${string}-${string}-${string}`;
+    private readonly pool;
+    private readonly channel;
+    private readonly logger;
+    private readonly onMessage;
+    private readonly onRecovered;
+    private readonly ignoreOwn;
+    private readonly heartbeatMs;
+    private readonly heartbeatTimeoutMs;
+    private readonly reconnectDelaysMs;
+    private listener;
+    private stopped;
+    private connecting;
+    private connectedOnce;
+    private reconnectAttempt;
+    private reconnectTimer;
+    private heartbeatTimer;
+    constructor(options: DbBroadcastBridgeOptions);
+    /** Connects the listener; rejects if the first connect fails. */
+    start(): Promise<void>;
+    /** Idempotent. Releases the listener client back to the pool; call it
+     *  before the pool ends. Nothing is delivered after it resolves. */
+    stop(): Promise<void>;
+    /** `pg_notify` on this bridge's channel. Pass the transaction's client to
+     *  announce only on commit. With `ignoreOwn`, tags the payload with
+     *  `originId`. */
+    notify(message: Record<string, unknown>, queryable?: Queryable): Promise<void>;
+    private connectListener;
+    /** pg-pool exempts its first `min` clients, counted over all of them,
+     *  checked out or idle, from the idle timeout, and reads `options.min`
+     *  each time it decides. The listener is one of those clients, so without
+     *  this a pool with `min: 1` (FLEET_POOL_DEFAULTS) would reap its last
+     *  working connection after the idle timeout: the warm connection min
+     *  exists to keep. Raise min while the listener is held, lower it on
+     *  release. */
+    private adjustPoolMin;
+    /** Each heartbeat is scheduled when the previous one ends, so a slow one
+     *  never overlaps the next. */
+    private scheduleHeartbeat;
+    /** Clears the current listener, its timers and handlers, and the raised
+     *  pool min; returns the client for the caller to release. */
+    private detachListener;
+    private onListenerLost;
+    private scheduleReconnect;
+    private onNotification;
+    /** A throw from the app's callback would otherwise escape through pg's
+     *  socket handler and crash the process. */
+    private callApp;
+}
+export {};

package/dist/sync/db-bridge.js

@@ -0,0 +1,256 @@
+// The cross-copy half of live delivery: one dedicated LISTEN connection per
+// process relays a Postgres NOTIFY channel to the app's local fan-out. Every
+// copy of a service shares the database, so a hint published by any writer
+// (another copy, the old revision during a deploy, a job) reaches every
+// copy's sessions. Rules: ~/plasticine-way/docs/DATABASE.md "Cross-instance
+// live delivery: LISTEN/NOTIFY". Moved here from todo's src/db-broadcast.ts.
+import { randomUUID } from 'node:crypto';
+/** How often the bridge proves its LISTEN connection alive with a `select
+ *  1`. Cloud SQL drops idle connections silently, the error surfaces only on
+ *  the next query, and a pure listener never issues one, so without this the
+ *  bridge would go permanently deaf while its logs claim health. */
+export const DB_BRIDGE_HEARTBEAT_MS = 60_000;
+/** How long a heartbeat (or the `unlisten` in stop()) may take before the
+ *  connection is declared lost: a half-open socket never answers and never
+ *  errors. Generous, because a request-billed Cloud Run instance runs this
+ *  timer with almost no CPU. */
+export const DB_BRIDGE_HEARTBEAT_TIMEOUT_MS = 10_000;
+export const DB_BRIDGE_RECONNECT_DELAYS_MS = [250, 1_000, 4_000, 15_000, 30_000];
+export class DbBroadcastBridge {
+    /** Tags this process's notify() payloads when `ignoreOwn` is on. */
+    originId = randomUUID();
+    pool;
+    channel;
+    logger;
+    onMessage;
+    onRecovered;
+    ignoreOwn;
+    heartbeatMs;
+    heartbeatTimeoutMs;
+    reconnectDelaysMs;
+    listener = null;
+    stopped = true;
+    connecting = false;
+    connectedOnce = false;
+    reconnectAttempt = 0;
+    reconnectTimer = null;
+    heartbeatTimer = null;
+    constructor(options) {
+        if (!/^[a-z_][a-z0-9_]*$/.test(options.channel)) {
+            throw new Error(`DbBroadcastBridge: channel must be a lowercase identifier, got ${JSON.stringify(options.channel)}`);
+        }
+        this.pool = options.pool;
+        this.channel = options.channel;
+        this.logger = options.logger;
+        this.onMessage = options.onMessage;
+        this.onRecovered = options.onRecovered;
+        this.ignoreOwn = options.ignoreOwn ?? false;
+        this.heartbeatMs = options.heartbeatMs ?? DB_BRIDGE_HEARTBEAT_MS;
+        this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? DB_BRIDGE_HEARTBEAT_TIMEOUT_MS;
+        this.reconnectDelaysMs = options.reconnectDelaysMs ?? DB_BRIDGE_RECONNECT_DELAYS_MS;
+    }
+    /** Connects the listener; rejects if the first connect fails. */
+    async start() {
+        if (!this.stopped)
+            return;
+        this.stopped = false;
+        await this.connectListener(true);
+    }
+    /** Idempotent. Releases the listener client back to the pool; call it
+     *  before the pool ends. Nothing is delivered after it resolves. */
+    async stop() {
+        this.stopped = true;
+        if (this.reconnectTimer)
+            clearTimeout(this.reconnectTimer);
+        this.reconnectTimer = null;
+        const listener = this.detachListener();
+        if (!listener)
+            return;
+        // pg emits 'error' on a client whose socket dies even mid-query, and a
+        // checked-out client has no pool listener: keep one until it is back.
+        const ignore = () => { };
+        listener.on('error', ignore);
+        try {
+            await queryWithin(listener, 'unlisten *', this.heartbeatTimeoutMs);
+            listener.removeListener('error', ignore);
+            listener.release();
+        }
+        catch (error) {
+            listener.release(toError(error));
+        }
+    }
+    /** `pg_notify` on this bridge's channel. Pass the transaction's client to
+     *  announce only on commit. With `ignoreOwn`, tags the payload with
+     *  `originId`. */
+    async notify(message, queryable = this.pool) {
+        const payload = this.ignoreOwn ? { ...message, origin: this.originId } : message;
+        await queryable.query('select pg_notify($1, $2)', [this.channel, JSON.stringify(payload)]);
+    }
+    async connectListener(initial) {
+        if (this.stopped || this.connecting || this.listener)
+            return;
+        this.connecting = true;
+        try {
+            const client = await this.pool.connect();
+            // A freshly checked-out pool client carries no 'error' listener — pg-pool
+            // strips the idle one on checkout. If Cloud SQL has silently reset this
+            // (reused) connection, the very next query makes pg emit an 'error' event
+            // on the client even while that query is in flight; with no listener Node
+            // escalates it to an uncaught exception and the process exits (the Reader
+            // 2026-09-10 crash, which hit here during a reconnect). Attach the
+            // handlers BEFORE issuing `listen`. In this pre-`listen` window
+            // this.listener is still null, so onListenerLost no-ops and the failing
+            // `listen` query below drives the cleanup and retry instead.
+            client.on('error', (error) => this.onListenerLost(client, error));
+            client.on('end', () => this.onListenerLost(client, new Error('database listener ended')));
+            try {
+                await client.query(`listen ${this.channel}`);
+            }
+            catch (error) {
+                client.removeAllListeners('error');
+                client.removeAllListeners('end');
+                client.release(toError(error));
+                throw error;
+            }
+            if (this.stopped) {
+                client.removeAllListeners('error');
+                client.removeAllListeners('end');
+                client.release();
+                return;
+            }
+            this.listener = client;
+            this.adjustPoolMin(1);
+            client.on('notification', (notification) => this.onNotification(notification.payload));
+            this.scheduleHeartbeat(client);
+            const recovered = this.connectedOnce;
+            this.connectedOnce = true;
+            this.reconnectAttempt = 0;
+            if (recovered) {
+                this.logger.info('database broadcast listener recovered', { channel: this.channel });
+                this.callApp('onRecovered', () => this.onRecovered());
+            }
+            else if (!initial) {
+                this.logger.info('database broadcast listener connected', { channel: this.channel });
+            }
+        }
+        finally {
+            this.connecting = false;
+        }
+    }
+    /** pg-pool exempts its first `min` clients, counted over all of them,
+     *  checked out or idle, from the idle timeout, and reads `options.min`
+     *  each time it decides. The listener is one of those clients, so without
+     *  this a pool with `min: 1` (FLEET_POOL_DEFAULTS) would reap its last
+     *  working connection after the idle timeout: the warm connection min
+     *  exists to keep. Raise min while the listener is held, lower it on
+     *  release. */
+    adjustPoolMin(delta) {
+        this.pool.options.min = (this.pool.options.min ?? 0) + delta;
+    }
+    /** Each heartbeat is scheduled when the previous one ends, so a slow one
+     *  never overlaps the next. */
+    scheduleHeartbeat(client) {
+        this.heartbeatTimer = setTimeout(() => {
+            this.heartbeatTimer = null;
+            if (this.listener !== client)
+                return;
+            queryWithin(client, 'select 1', this.heartbeatTimeoutMs).then(() => {
+                if (this.listener === client)
+                    this.scheduleHeartbeat(client);
+            }, (error) => this.onListenerLost(client, toError(error)));
+        }, this.heartbeatMs);
+    }
+    /** Clears the current listener, its timers and handlers, and the raised
+     *  pool min; returns the client for the caller to release. */
+    detachListener() {
+        const listener = this.listener;
+        if (!listener)
+            return null;
+        this.listener = null;
+        if (this.heartbeatTimer)
+            clearTimeout(this.heartbeatTimer);
+        this.heartbeatTimer = null;
+        listener.removeAllListeners('notification');
+        listener.removeAllListeners('error');
+        listener.removeAllListeners('end');
+        this.adjustPoolMin(-1);
+        return listener;
+    }
+    onListenerLost(client, error) {
+        if (this.stopped || this.listener !== client)
+            return;
+        this.detachListener();
+        // The pool re-adds its idle error listener on release, so a late socket
+        // error on the discarded client stays handled.
+        client.release(error);
+        this.logger.warn('database broadcast listener lost', { channel: this.channel, error: String(error) });
+        this.scheduleReconnect();
+    }
+    scheduleReconnect() {
+        if (this.stopped || this.reconnectTimer)
+            return;
+        const delays = this.reconnectDelaysMs;
+        const delay = delays[Math.min(this.reconnectAttempt++, delays.length - 1)];
+        this.reconnectTimer = setTimeout(() => {
+            this.reconnectTimer = null;
+            void this.connectListener(false).catch((error) => {
+                this.logger.warn('database broadcast reconnect failed', { channel: this.channel, error: String(error) });
+                this.scheduleReconnect();
+            });
+        }, delay);
+    }
+    onNotification(payload) {
+        if (this.stopped || !payload)
+            return;
+        let message;
+        try {
+            message = JSON.parse(payload);
+        }
+        catch (error) {
+            this.logger.warn('invalid database broadcast', {
+                channel: this.channel,
+                error: String(error),
+                payload: payload.slice(0, 200),
+            });
+            return;
+        }
+        if (this.ignoreOwn && message?.origin === this.originId)
+            return;
+        this.callApp('onMessage', () => this.onMessage(message));
+    }
+    /** A throw from the app's callback would otherwise escape through pg's
+     *  socket handler and crash the process. */
+    callApp(callback, fn) {
+        try {
+            fn();
+        }
+        catch (error) {
+            this.logger.warn('database broadcast handler failed', { channel: this.channel, callback, error });
+        }
+    }
+}
+/** Runs `sql`, failing if it has not answered within `timeoutMs`. The verdict
+ *  waits one more event-loop turn (setImmediate runs after the poll phase),
+ *  so a reply already buffered when a throttled instance gets CPU back is
+ *  read before the timer, which runs first, can convict it. */
+async function queryWithin(client, sql, timeoutMs) {
+    const query = client.query(sql);
+    // On a timeout the caller's release(error) destroys the socket, which
+    // rejects this query; nothing else awaits it.
+    query.catch(() => { });
+    let timer;
+    try {
+        await Promise.race([
+            query,
+            new Promise((_, reject) => {
+                timer = setTimeout(() => setImmediate(() => reject(new Error(`database listener: ${sql} timed out after ${timeoutMs} ms`))), timeoutMs);
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+function toError(error) {
+    return error instanceof Error ? error : new Error(String(error));
+}

package/dist/sync/index.d.ts

@@ -3,3 +3,4 @@ export * from './schema.js';
 export * from './engine.js';
 export * from './broadcaster.js';
 export * from './session.js';
+export * from './db-bridge.js';

package/dist/sync/index.js

@@ -3,3 +3,4 @@ export * from './schema.js';
 export * from './engine.js';
 export * from './broadcaster.js';
 export * from './session.js';
+export * from './db-bridge.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.43.0",
+  "version": "3.45.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",