@steve31415/baselib 3.44.0 → 3.46.0

package/dist/auth.d.ts

@@ -1,5 +1,5 @@
 import type { Context } from 'hono';
-import type { Logger } from './log-core.js';
+import { type Logger } from './log-core.js';
 import { ServiceCaller, S2sOptions } from './s2s.js';
 export declare const DEFAULT_AUTH_URL = "https://auth.apps.snewman.net";
 export declare const SESSION_COOKIE = "pw_session";

package/dist/auth.js

@@ -9,7 +9,11 @@
 //      policy, app-level checks) still runs.
 //   2. Authorization: Bearer → service-to-service OIDC (s2s.ts).
 //   3. pw_session cookie → whoami call to the auth service, cached
-//      in-instance (10 min hits / 30 s misses — the old-world numbers).
+//      in-instance (10 min hits / 30 s misses — the old-world numbers),
+//      one call per cookie at a time (concurrent misses share it).
+//      Every call (never a cache hit) logs one INFO 'session check' event
+//      saying why the cache could not answer and what the call cost the
+//      waiting request (~/migration/research/whoami-latency-2026-10-30.md).
 //
 // Then the user-access policy: by default an app admits only users whose
 // auth role is 'owner'; `userAccess: 'invited'` admits any active fleet
@@ -25,6 +29,7 @@
 import { createHash } from 'node:crypto';
 import { createMiddleware } from 'hono/factory';
 import { authMode } from './config.js';
+import { truncate } from './log-core.js';
 import { RateLimiter, resolveS2sCaller } from './s2s.js';
 export const DEFAULT_AUTH_URL = 'https://auth.apps.snewman.net';
 export const SESSION_COOKIE = 'pw_session';
@@ -46,6 +51,13 @@ export function requireIdentity(opts) {
     const policy = opts.userAccess ?? 'owner';
     const limiter = new RateLimiter(opts.rateLimitPerMin ?? 600);
     const cache = new Map();
+    // The whoami call in flight per cache key (single-flight).
+    const inFlight = new Map();
+    // Session-check log state (per middleware, like the cache): cache hits
+    // since the last whoami call, and when the last call ended.
+    let hitsSinceLastCall = 0;
+    let callCount = 0;
+    let lastCallEndedAt;
     const isApi = opts.isApiRequest ??
         ((c) => c.req.path.startsWith('/api/') || (c.req.header('accept') ?? '').includes('application/json'));
     const unauthenticated = (c) => {
@@ -108,18 +120,77 @@ export function requireIdentity(opts) {
         const key = createHash('sha256').update(cookieHeader).digest('hex');
         const cached = cache.get(key);
         if (cached && cached.expires > Date.now()) {
+            hitsSinceLastCall++;
             if (!cached.user)
                 return unauthenticated(c);
             return admit(cached.user);
         }
+        // Single-flight: a request that misses while a call for the same cookie
+        // is already in flight shares that call's answer instead of making its
+        // own (30% of calls were such duplicates). Every request still gets an
+        // answer fetched during its own lifetime, so what is admitted does not
+        // change; the answer is applied per request below.
+        const flight = inFlight.get(key);
+        if (flight) {
+            flight.joiners.count++;
+            return settle(await flight.outcome);
+        }
+        const joiners = { count: 0 };
+        const outcome = callWhoami(c, key, cached, cookieHeader, joiners).finally(() => inFlight.delete(key));
+        inFlight.set(key, { outcome, joiners });
+        return settle(await outcome);
+        function settle(outcome) {
+            if (outcome.kind === 'user')
+                return admit(outcome.user);
+            if (outcome.kind === 'unauthenticated')
+                return unauthenticated(c);
+            return c.json({ error: 'auth service unavailable' }, 503);
+        }
+    });
+    // One whoami call, whose answer every request that joined it shares. The
+    // waiting request pays its full latency. One INFO event per call (a few
+    // hundred a day fleet-wide; never per hit or per joined request), with a
+    // fixed vocabulary of keys and nothing from the cookie but how many
+    // cookies the header carries (a changing header defeats the cache).
+    async function callWhoami(c, key, cached, cookieHeader, joiners) {
+        const started = Date.now();
+        const cacheState = !cached ? 'absent' : cached.user ? 'expired-user' : 'expired-unauthenticated';
+        const callFields = {
+            cacheState,
+            ...(cached ? { entryAgeMs: started - cached.filled } : {}),
+            hitsSinceLastCall,
+            ...(lastCallEndedAt !== undefined ? { sinceLastCallMs: started - lastCallEndedAt } : {}),
+            ...(callCount === 0 ? { firstCall: true } : {}),
+            uptimeMs: Math.round(process.uptime() * 1000),
+            method: c.req.method,
+            path: truncate(c.req.path, 200),
+            upgrade: (c.req.header('upgrade') ?? '').toLowerCase() === 'websocket',
+            cookieCount: cookieHeader.split(';').filter((part) => part.includes('=')).length,
+        };
+        hitsSinceLastCall = 0;
+        callCount++;
+        // `joinedRequests`: requests that shared this call (single-flight),
+        // counted when the call ends, so it includes every joiner.
+        const callEnded = (status) => {
+            const ended = Date.now();
+            lastCallEndedAt = ended;
+            opts.logger?.info('session check', {
+                ...callFields,
+                joinedRequests: joiners.count,
+                status,
+                elapsedMs: ended - started,
+            });
+        };
         let res;
         try {
             res = await whoamiFetch(`${authUrl}/api/whoami`, { headers: { cookie: cookieHeader } });
         }
         catch (err) {
+            callEnded(0);
             opts.logger?.error('auth service unreachable', { error: err });
-            return c.json({ error: 'auth service unavailable' }, 503);
+            return { kind: 'unavailable' };
         }
+        callEnded(res.status);
         if (res.ok) {
             const data = (await res.json());
             const role = parseRole(data.role);
@@ -128,19 +199,21 @@ export function requireIdentity(opts) {
                 // protocol change: a fault, not an "invited" answer. Fail closed,
                 // loudly, and without caching so recovery is instant.
                 opts.logger?.error('whoami answer has no role', { email: data.email, role: data.role });
-                return c.json({ error: 'auth service unavailable' }, 503);
+                return { kind: 'unavailable' };
             }
             const user = { type: 'user', email: data.email, name: data.name, userId: data.user_id, role };
-            cache.set(key, { expires: Date.now() + hitMs, user });
-            return admit(user);
+            const now = Date.now();
+            cache.set(key, { expires: now + hitMs, filled: now, user });
+            return { kind: 'user', user };
         }
         if (res.status === 401) {
-            cache.set(key, { expires: Date.now() + missMs });
-            return unauthenticated(c);
+            const now = Date.now();
+            cache.set(key, { expires: now + missMs, filled: now });
+            return { kind: 'unauthenticated' };
         }
         opts.logger?.error('auth service error', { meta_status: res.status });
-        return c.json({ error: 'auth service unavailable' }, 503);
-    });
+        return { kind: 'unavailable' };
+    }
 }
 // The fleet-wide "not enabled" page, in the auth service's plain style so
 // every app shows the same thing. Carries nothing but the request's host,

package/dist/s2s.d.ts

@@ -67,9 +67,20 @@ export declare function resolveS2sCaller(opts: S2sOptions & {
 export declare function s2sAuth(opts: S2sOptions): import("hono").MiddlewareHandler<any, string, {}, Response | (Response & import("hono").TypedResponse<{
     error: string;
 }, 429 | 401 | 403, "json">)>;
+/** A cached ID token is replaced this long before its `exp`, as
+ *  google-auth-library's own IdTokenClient does (its eager-refresh default).
+ *  Tokens live an hour. */
+export declare const ID_TOKEN_REFRESH_MARGIN_MS: number;
+/** The longest one mint may take. On Cloud Run gcp-metadata sends the
+ *  metadata-server request with no timeout at all; a healthy mint takes
+ *  milliseconds. */
+export declare const ID_TOKEN_MINT_TIMEOUT_MS = 5000;
 /** fetch() with an OIDC identity token for the target's canonical audience.
  *  In test mode, sends the synthetic X-Test-S2S-Caller header instead
- *  (identity from TEST_S2S_IDENTITY, default 'test-service@test'). */
+ *  (identity from TEST_S2S_IDENTITY, default 'test-service@test'). The token
+ *  is cached per audience until ID_TOKEN_REFRESH_MARGIN_MS before it
+ *  expires; concurrent calls share one mint, which is bounded by
+ *  ID_TOKEN_MINT_TIMEOUT_MS, and `init.signal` also ends the wait for it. */
 export declare function serviceFetch(url: string, opts: {
     audience: string;
 } & RequestInit): Promise<Response>;

package/dist/s2s.js

@@ -182,10 +182,26 @@ export function s2sAuth(opts) {
 }
 // ---------------------------------------------------------------------------
 // Caller side
-const idTokenClients = new Map();
+/** A cached ID token is replaced this long before its `exp`, as
+ *  google-auth-library's own IdTokenClient does (its eager-refresh default).
+ *  Tokens live an hour. */
+export const ID_TOKEN_REFRESH_MARGIN_MS = 5 * 60_000;
+/** The longest one mint may take. On Cloud Run gcp-metadata sends the
+ *  metadata-server request with no timeout at all; a healthy mint takes
+ *  milliseconds. */
+export const ID_TOKEN_MINT_TIMEOUT_MS = 5_000;
+// serviceFetch calls the provider directly, which bypasses IdTokenClient's
+// token cache (it lives in getRequestMetadataAsync), so the cache, the
+// single flight and the bound are kept here. Failures are never cached.
+let idTokenProvider;
+const idTokens = new Map();
+const idTokenMints = new Map();
 /** fetch() with an OIDC identity token for the target's canonical audience.
  *  In test mode, sends the synthetic X-Test-S2S-Caller header instead
- *  (identity from TEST_S2S_IDENTITY, default 'test-service@test'). */
+ *  (identity from TEST_S2S_IDENTITY, default 'test-service@test'). The token
+ *  is cached per audience until ID_TOKEN_REFRESH_MARGIN_MS before it
+ *  expires; concurrent calls share one mint, which is bounded by
+ *  ID_TOKEN_MINT_TIMEOUT_MS, and `init.signal` also ends the wait for it. */
 export async function serviceFetch(url, opts) {
     const { audience, ...init } = opts;
     const headers = new Headers(init.headers);
@@ -193,17 +209,75 @@ export async function serviceFetch(url, opts) {
         headers.set('x-test-s2s-caller', process.env.TEST_S2S_IDENTITY ?? 'test-service@test');
     }
     else {
-        let clientPromise = idTokenClients.get(audience);
-        if (!clientPromise) {
-            clientPromise = (async () => {
-                const { GoogleAuth } = await import('google-auth-library');
-                return new GoogleAuth().getIdTokenClient(audience);
-            })();
-            idTokenClients.set(audience, clientPromise);
-        }
-        const client = await clientPromise;
-        const token = await client.idTokenProvider.fetchIdToken(audience);
+        const token = await abortable(idTokenFor(audience), init.signal);
         headers.set('authorization', `Bearer ${token}`);
     }
     return fetch(url, { ...init, headers });
 }
+function idTokenFor(audience) {
+    const cached = idTokens.get(audience);
+    if (cached && Date.now() < cached.refreshAt)
+        return Promise.resolve(cached.token);
+    let mint = idTokenMints.get(audience);
+    if (!mint) {
+        mint = withinMs(mintIdToken(audience), ID_TOKEN_MINT_TIMEOUT_MS, `ID token mint for ${audience}`)
+            .then((token) => {
+            const expiresAt = idTokenExpiry(token);
+            if (expiresAt !== undefined) {
+                idTokens.set(audience, { token, refreshAt: expiresAt - ID_TOKEN_REFRESH_MARGIN_MS });
+            }
+            return token;
+        })
+            .finally(() => idTokenMints.delete(audience));
+        idTokenMints.set(audience, mint);
+    }
+    return mint;
+}
+async function mintIdToken(audience) {
+    idTokenProvider ??= (async () => {
+        const { GoogleAuth } = await import('google-auth-library');
+        return (await new GoogleAuth().getIdTokenClient(audience)).idTokenProvider;
+    })();
+    const provider = idTokenProvider;
+    // A failed credential lookup is retried by the next call.
+    provider.catch(() => {
+        if (idTokenProvider === provider)
+            idTokenProvider = undefined;
+    });
+    return (await provider).fetchIdToken(audience);
+}
+/** Our own token's `exp` in ms, unverified; undefined if unreadable. */
+function idTokenExpiry(token) {
+    try {
+        const exp = JSON.parse(Buffer.from(token.split('.')[1] ?? '', 'base64url').toString()).exp;
+        return typeof exp === 'number' ? exp * 1000 : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** `promise`, or a rejection once `ms` pass. The work itself cannot be
+ *  cancelled (fetchIdToken takes no signal); it is only no longer awaited. */
+function withinMs(promise, ms, what) {
+    let timer;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(`${what} timed out after ${ms} ms`)), ms);
+    });
+    return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+/** `promise`, or a rejection with the signal's reason when it aborts (as
+ *  fetch rejects), leaving the promise itself to its other waiters. Always
+ *  subscribes to `promise`, so a mint no one waits for any more never
+ *  rejects unhandled. */
+function abortable(promise, signal) {
+    if (!signal)
+        return promise;
+    return new Promise((resolve, reject) => {
+        const onAbort = () => reject(signal.reason);
+        if (signal.aborted)
+            onAbort();
+        else
+            signal.addEventListener('abort', onAbort, { once: true });
+        promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort));
+    });
+}

package/dist/sync/broadcaster.d.ts

@@ -6,4 +6,8 @@ export declare class ScopeBroadcaster<T> {
      * after the remaining listeners have received the message. */
     publish(scopeKey: string, message: T, except?: (message: T) => void): void;
     count(scopeKey: string): number;
+    /** The scopes with at least one listener: for an app that must
+     * re-announce to every local scope, e.g. after its database bridge may
+     * have missed notifications. */
+    scopes(): string[];
 }

package/dist/sync/broadcaster.js

@@ -44,4 +44,10 @@ export class ScopeBroadcaster {
     count(scopeKey) {
         return this.listeners.get(scopeKey)?.size ?? 0;
     }
+    /** The scopes with at least one listener: for an app that must
+     * re-announce to every local scope, e.g. after its database bridge may
+     * have missed notifications. */
+    scopes() {
+        return [...this.listeners.keys()];
+    }
 }

package/dist/sync/db-bridge.d.ts

@@ -22,9 +22,11 @@ export interface DbBroadcastBridgeOptions {
     /** Each notification's parsed JSON payload. An invalid payload is logged
      *  and skipped; a throw is logged and does not reach pg. */
     onMessage: (message: unknown) => void;
-    /** Called after the listener reconnects (not on the first connect).
-     *  Notifications sent while it was down are lost and unknowable, so the
-     *  app re-announces availability to every local scope. */
+    /** Called on every connect after start() has returned: a reconnect after
+     *  a loss, or the first success after a failed first attempt (not on a
+     *  first attempt that succeeds). Notifications sent while nothing was
+     *  listening are lost and unknowable, so the app re-announces availability
+     *  to every local scope. */
     onRecovered: () => void;
     /** Skip messages whose top-level `origin` is this bridge's `originId`,
      *  for an app that delivers its own writes locally and publishes them via
@@ -50,12 +52,16 @@ export declare class DbBroadcastBridge {
     private listener;
     private stopped;
     private connecting;
-    private connectedOnce;
+    /** Set once start() has returned. From then on the app may have sessions
+     *  that a missed notification leaves stale, so every later connect is a
+     *  recovery. */
+    private startReturned;
     private reconnectAttempt;
     private reconnectTimer;
     private heartbeatTimer;
     constructor(options: DbBroadcastBridgeOptions);
-    /** Connects the listener; rejects if the first connect fails. */
+    /** Makes the first connect attempt; never rejects. A failure is logged
+     *  and retried on the reconnect ladder (see the header). */
     start(): Promise<void>;
     /** Idempotent. Releases the listener client back to the pool; call it
      *  before the pool ends. Nothing is delivered after it resolves. */

package/dist/sync/db-bridge.js

@@ -4,6 +4,16 @@
 // (another copy, the old revision during a deploy, a job) reaches every
 // copy's sessions. Rules: ~/plasticine-way/docs/DATABASE.md "Cross-instance
 // live delivery: LISTEN/NOTIFY". Moved here from todo's src/db-broadcast.ts.
+//
+// Lifecycle contract. start() makes the first connect attempt and resolves
+// when it ends, whether or not it connected; it never rejects. A failed first
+// attempt is logged as a loss (WARN 'database broadcast listener lost',
+// `atStart: true`) and enters the same reconnect ladder as a later loss, so a
+// process booting while the database is unreachable neither crash-loops on a
+// top-level `await start()` nor stays deaf for life. Every connect after
+// start() has returned (after a loss, or after a failed first attempt) logs
+// INFO 'database broadcast listener recovered' and calls onRecovered: the app
+// may have been serving sessions while nothing was listening.
 import { randomUUID } from 'node:crypto';
 /** How often the bridge proves its LISTEN connection alive with a `select
  *  1`. Cloud SQL drops idle connections silently, the error surfaces only on
@@ -31,7 +41,10 @@ export class DbBroadcastBridge {
     listener = null;
     stopped = true;
     connecting = false;
-    connectedOnce = false;
+    /** Set once start() has returned. From then on the app may have sessions
+     *  that a missed notification leaves stale, so every later connect is a
+     *  recovery. */
+    startReturned = false;
     reconnectAttempt = 0;
     reconnectTimer = null;
     heartbeatTimer = null;
@@ -49,12 +62,28 @@ export class DbBroadcastBridge {
         this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? DB_BRIDGE_HEARTBEAT_TIMEOUT_MS;
         this.reconnectDelaysMs = options.reconnectDelaysMs ?? DB_BRIDGE_RECONNECT_DELAYS_MS;
     }
-    /** Connects the listener; rejects if the first connect fails. */
+    /** Makes the first connect attempt; never rejects. A failure is logged
+     *  and retried on the reconnect ladder (see the header). */
     async start() {
         if (!this.stopped)
             return;
         this.stopped = false;
-        await this.connectListener(true);
+        try {
+            await this.connectListener();
+        }
+        catch (error) {
+            if (!this.stopped) {
+                this.logger.warn('database broadcast listener lost', {
+                    channel: this.channel,
+                    error: String(error),
+                    atStart: true,
+                });
+                this.scheduleReconnect();
+            }
+        }
+        finally {
+            this.startReturned = true;
+        }
     }
     /** Idempotent. Releases the listener client back to the pool; call it
      *  before the pool ends. Nothing is delivered after it resolves. */
@@ -86,7 +115,7 @@ export class DbBroadcastBridge {
         const payload = this.ignoreOwn ? { ...message, origin: this.originId } : message;
         await queryable.query('select pg_notify($1, $2)', [this.channel, JSON.stringify(payload)]);
     }
-    async connectListener(initial) {
+    async connectListener() {
         if (this.stopped || this.connecting || this.listener)
             return;
         this.connecting = true;
@@ -122,16 +151,11 @@ export class DbBroadcastBridge {
             this.adjustPoolMin(1);
             client.on('notification', (notification) => this.onNotification(notification.payload));
             this.scheduleHeartbeat(client);
-            const recovered = this.connectedOnce;
-            this.connectedOnce = true;
             this.reconnectAttempt = 0;
-            if (recovered) {
+            if (this.startReturned) {
                 this.logger.info('database broadcast listener recovered', { channel: this.channel });
                 this.callApp('onRecovered', () => this.onRecovered());
             }
-            else if (!initial) {
-                this.logger.info('database broadcast listener connected', { channel: this.channel });
-            }
         }
         finally {
             this.connecting = false;
@@ -193,7 +217,7 @@ export class DbBroadcastBridge {
         const delay = delays[Math.min(this.reconnectAttempt++, delays.length - 1)];
         this.reconnectTimer = setTimeout(() => {
             this.reconnectTimer = null;
-            void this.connectListener(false).catch((error) => {
+            void this.connectListener().catch((error) => {
                 this.logger.warn('database broadcast reconnect failed', { channel: this.channel, error: String(error) });
                 this.scheduleReconnect();
             });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.44.0",
+  "version": "3.46.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",