@steve31415/baselib 3.52.0 → 3.53.0

package/dist/hub/server.d.ts

@@ -3,6 +3,7 @@ import type { UpgradeWebSocket } from 'hono/ws';
 import type pg from 'pg';
 import { type RequireIdentityOptions } from '../auth.js';
 import { type Logger } from '../log.js';
+import { type SocketAgeCapOptions } from '../socket-age.js';
 import type { Grants } from '../s2s.js';
 import { type DbBroadcastBridgeOptions } from '../sync/db-bridge.js';
 /** The hub database's one table (the hub repo's first migration carries the
@@ -28,7 +29,7 @@ export declare const HUB_PUBLISHES_IN_FLIGHT_MAX = 16;
 /** The largest frame a browser may send (its frames are `connect` and `ping`). */
 export declare const HUB_SOCKET_MAX_PAYLOAD = 4096;
 /** A socket that has sent nothing (clients ping every 30 s) for this long is
- *  closed: a half-dead socket would otherwise hold a slot for an hour. */
+ *  closed: a half-dead socket would otherwise hold a slot until the age cap. */
 export declare const HUB_SOCKET_IDLE_MS = 180000;
 export interface HubOptions {
     pool: pg.Pool;
@@ -68,6 +69,8 @@ export interface HubOptions {
         idleMs: number;
         sweepMs: number;
     };
+    /** Tests: the socket age cap (default 300 s, then 5 s to complete the close). */
+    socketAge?: Pick<SocketAgeCapOptions, 'maxAgeMs' | 'graceMs'>;
 }
 export interface Hub {
     /** Start the cross-copy bridge. Call before the server accepts

package/dist/hub/server.js

@@ -24,12 +24,14 @@
 //                               caller, for the dark-run curl); answers on a
 //                               `work` hint, else 204 after `wait` seconds
 //   GET  /api/ws?app=<app>      browser WebSocket, fleet cookie, Origin
-//                               must be that app's page
+//                               must be that app's page; closed (1012) at
+//                               300 s of age, clients roll at ~270 s
 import { createHash } from 'node:crypto';
 import { createMiddleware } from 'hono/factory';
 import { requireIdentity } from '../auth.js';
 import { forwardedClientIp } from '../http.js';
 import { serializeError } from '../log.js';
+import { capSocketAge } from '../socket-age.js';
 import { DbBroadcastBridge } from '../sync/db-bridge.js';
 import { checkHint, isAppName } from './types.js';
 /** The hub database's one table (the hub repo's first migration carries the
@@ -65,7 +67,7 @@ export const HUB_PUBLISHES_IN_FLIGHT_MAX = 16;
 /** The largest frame a browser may send (its frames are `connect` and `ping`). */
 export const HUB_SOCKET_MAX_PAYLOAD = 4096;
 /** A socket that has sent nothing (clients ping every 30 s) for this long is
- *  closed: a half-dead socket would otherwise hold a slot for an hour. */
+ *  closed: a half-dead socket would otherwise hold a slot until the age cap. */
 export const HUB_SOCKET_IDLE_MS = 180_000;
 const HUB_SOCKET_SWEEP_MS = 30_000;
 const TOKEN_FORMAT = /^hubt_[A-Za-z0-9_-]{43}$/;
@@ -381,7 +383,12 @@ export function createHub(app, opts) {
             return c.body(null, 400); // no one reads it
         await next();
     });
-    app.get('/api/ws', originCheck, socketIdentity, opts.upgradeWebSocket((c) => {
+    app.get('/api/ws', originCheck, socketIdentity, 
+    // Every socket closes (1012) at 300 s of age, like the apps' own sockets:
+    // a socket on a retired hub copy holds that copy, and its slot under
+    // --max-instances, until it ends (PW docs/OPERATIONS.md, "Socket
+    // lifetime"). SIGTERM comes only after the last socket ends.
+    capSocketAge(opts.upgradeWebSocket, { logger, ...opts.socketAge })((c) => {
         const appName = c.req.query('app');
         const refusal = refusals.get(c);
         const id = c.var.identity;

package/dist/socket-age.d.ts

@@ -0,0 +1,22 @@
+import type { UpgradeWebSocket } from 'hono/ws';
+import type { Logger } from './log-core.js';
+/** Sockets older than this are closed by the server. */
+export declare const SOCKET_MAX_AGE_MS = 300000;
+/** After the close frame, how long the client gets to answer it before the
+ *  connection is terminated. A frozen tab answers nothing, and the open TCP
+ *  connection is what holds the instance. */
+export declare const SOCKET_CLOSE_GRACE_MS = 5000;
+/** "Service restart": clients treat it as a planned roll and reconnect at once. */
+export declare const CLOSE_SERVICE_RESTART = 1012;
+export interface SocketAgeCapOptions {
+    logger: Pick<Logger, 'info'>;
+    /** Tests. */
+    maxAgeMs?: number;
+    graceMs?: number;
+}
+/** `upgradeWebSocket`, with every socket it opens closed at `maxAgeMs` of age
+ *  (1012), and terminated `graceMs` later if the client hasn't completed the
+ *  close. One INFO line per cut: `socket age cap`, with the socket's path, its
+ *  age and whether it had to be terminated. A socket the route's own onOpen
+ *  closed (a refused identity) is left alone. */
+export declare function capSocketAge(upgradeWebSocket: UpgradeWebSocket, options: SocketAgeCapOptions): UpgradeWebSocket;

package/dist/socket-age.js

@@ -0,0 +1,92 @@
+// A cap on WebSocket age, applied by wrapping a Hono `upgradeWebSocket`.
+//
+// An open socket is an open request, and Cloud Run keeps a retired instance
+// alive (and counted against --max-instances) until its last request ends.
+// On 2026-11-02 two such instances held radar's only slot and every request
+// got 429 for 14 minutes (PW docs/LESSONS.md). Clients replace their sockets
+// at about 270 s; this cut closes whatever didn't (a frozen tab, a sleeping
+// laptop, an old client) at 300 s of age, in code rather than through the
+// service's --timeout, which must stay long enough for long background
+// routes (PW docs/OPERATIONS.md, "Socket lifetime").
+/** Sockets older than this are closed by the server. */
+export const SOCKET_MAX_AGE_MS = 300_000;
+/** After the close frame, how long the client gets to answer it before the
+ *  connection is terminated. A frozen tab answers nothing, and the open TCP
+ *  connection is what holds the instance. */
+export const SOCKET_CLOSE_GRACE_MS = 5_000;
+/** "Service restart": clients treat it as a planned roll and reconnect at once. */
+export const CLOSE_SERVICE_RESTART = 1012;
+const WS_OPEN = 1;
+/** `upgradeWebSocket`, with every socket it opens closed at `maxAgeMs` of age
+ *  (1012), and terminated `graceMs` later if the client hasn't completed the
+ *  close. One INFO line per cut: `socket age cap`, with the socket's path, its
+ *  age and whether it had to be terminated. A socket the route's own onOpen
+ *  closed (a refused identity) is left alone. */
+export function capSocketAge(upgradeWebSocket, options) {
+    const maxAgeMs = options.maxAgeMs ?? SOCKET_MAX_AGE_MS;
+    const graceMs = options.graceMs ?? SOCKET_CLOSE_GRACE_MS;
+    const cap = (events, c) => withAgeCap(events, c.req.path, maxAgeMs, graceMs, options.logger);
+    // Both of UpgradeWebSocket's call forms: (createEvents, options) returning
+    // middleware, and (c, events, options) upgrading at once.
+    function wrapped(first, second, third) {
+        if (typeof first === 'function') {
+            const createEvents = first;
+            return upgradeWebSocket(async (c) => cap(await createEvents(c), c), second);
+        }
+        return upgradeWebSocket(first, cap(second, first), third);
+    }
+    return wrapped;
+}
+function withAgeCap(events, path, maxAgeMs, graceMs, logger) {
+    let openedAt = 0;
+    let ageTimer = null;
+    let graceTimer = null;
+    let cutAt = null;
+    let logged = false;
+    const clearTimers = () => {
+        if (ageTimer !== null)
+            clearTimeout(ageTimer);
+        if (graceTimer !== null)
+            clearTimeout(graceTimer);
+        ageTimer = graceTimer = null;
+    };
+    const logCut = (terminated) => {
+        if (cutAt === null || logged)
+            return;
+        logged = true;
+        logger.info('socket age cap', { path, ageMs: cutAt - openedAt, terminated });
+    };
+    const cut = (ws) => {
+        ageTimer = null;
+        cutAt = Date.now();
+        try {
+            ws.close(CLOSE_SERVICE_RESTART, 'socket age limit');
+        }
+        catch {
+            // Already closing: its close handler logs the cut.
+        }
+        graceTimer = setTimeout(() => {
+            graceTimer = null;
+            const raw = ws.raw;
+            raw?.terminate?.();
+            logCut(true);
+        }, graceMs);
+        graceTimer.unref?.();
+    };
+    return {
+        ...events,
+        onOpen: (evt, ws) => {
+            events.onOpen?.(evt, ws);
+            if (ws.readyState !== WS_OPEN)
+                return;
+            openedAt = Date.now();
+            ageTimer = setTimeout(() => cut(ws), maxAgeMs);
+            ageTimer.unref?.();
+        },
+        onClose: (evt, ws) => {
+            clearTimers();
+            logCut(false);
+            events.onClose?.(evt, ws);
+        },
+    };
+}

package/dist/sync-browser/client.d.ts

@@ -101,6 +101,10 @@ export interface SyncClientOptions<B, C extends EventBody, S extends EventBody,
     onBuildObserved?(buildId: string): void;
     onBuildChanged?(): void;
     markPerformance?(name: 'outbox-start' | 'outbox-ready' | 'bootstrap-start' | 'bootstrap-response' | 'store-ready'): void;
+    /** The socket's planned life: it is replaced (make-before-break) this long
+     * after it opens. Default 270 s ±10%, jittered per socket so tabs restored
+     * together don't roll together; an explicit value is used exactly. Servers
+     * cut sockets at 300 s (PW docs/OPERATIONS.md, "Socket lifetime"). */
     reconnectBeforeMs?: number;
     /** While hidden and disconnected, one HTTP catch-up per this interval
      * (default 5 min). */
@@ -173,7 +177,7 @@ export declare class SyncClient<B, C extends EventBody, S extends EventBody, Sum
     private connectReplyTimer;
     private socketCreatedHidden;
     private socketOpenedAt;
-    private plannedClose;
+    private candidate;
     private outageCloseCode;
     private outageInfo;
     private lastContactAt;
@@ -242,6 +246,23 @@ export declare class SyncClient<B, C extends EventBody, S extends EventBody, Sum
      * connects at once. */
     private replaceSocket;
     private connect;
+    /** The live socket's message and close handling; a rollover's replacement
+     * gets it when it takes over. */
+    private attachLiveHandlers;
+    private startLiveTimers;
+    private armRollover;
+    private rolloverDelayMs;
+    /** Make-before-break, as notes' document sockets do: a replacement connects
+     * while the live socket keeps serving (sends, echoes, state), and takes over
+     * when the server answers its connect. Nothing the user sees changes, even
+     * with an edit unsent. The replacement's handshake re-runs the server's
+     * identity check; a 4403 halts as on any socket. If it fails, the live socket
+     * keeps serving until the server's age cap closes it (1012). */
+    private startRollover;
+    private promoteCandidate;
+    /** Abandons a rollover's replacement; `failure` logs why and tries again
+     * after another socket life (the server's age cap usually comes first). */
+    private dropCandidate;
     /** Hidden and disconnected: no WebSocket retries; HTTP catch-ups on a slow
      * cadence instead (docs/SYNC-BACKGROUND-PLAN.md). */
     private enterDormant;
@@ -308,6 +329,7 @@ export declare class SyncClient<B, C extends EventBody, S extends EventBody, Sum
     private scheduleResyncFailureRetry;
     private cancelResyncFailureRetry;
     private sendConnect;
+    private connectMessage;
     private sendRaw;
     private trySend;
     private flushViaBeacon;

package/dist/sync-browser/client.js

@@ -5,7 +5,16 @@ const PING_INTERVAL_MS = 30_000;
 // times in 25 minutes on 10-19, but none of those handshakes ever reached
 // the load balancer, so a longer limit would only have delayed the retry.
 const CONNECT_WATCHDOG_MS = 10_000;
-const DEFAULT_RECONNECT_BEFORE_MS = 55 * 60 * 1000;
+/** A socket that holds a retired Cloud Run instance keeps that instance
+ * alive and counted against --max-instances (radar, 2026-11-02: 14 minutes of
+ * 429s). Rolling at 270 s, under the servers' 300 s cut, bounds that hold. */
+const DEFAULT_RECONNECT_BEFORE_MS = 270_000;
+/** ± this fraction of the default rollover, per socket: every fleet app shares
+ * the load balancer's IP, and Chrome queues handshakes per IP. */
+const ROLLOVER_JITTER = 0.1;
+/** "Service restart": the server's planned close (its socket age cap, or a
+ * hub copy stopping). Handled like our own rollover. */
+const CLOSE_SERVICE_RESTART = 1012;
 const FAILURE_TIMEOUT_MS = 60_000;
 // ERROR only after a sustained foreground outage. The fleet is single-user
 // and triage runs daily, so detection latency is cheap; short client-network
@@ -44,6 +53,14 @@ const SLOW_HANDSHAKE_MS = 3_000;
 const STALL_MIN_MS = 1_000;
 const SOCKET_CONNECTING = 0;
 const SOCKET_OPEN = 1;
+function messageType(raw) {
+    try {
+        return JSON.parse(raw)?.type;
+    }
+    catch {
+        return undefined;
+    }
+}
 class LifecycleAbort {
     reason;
     constructor(reason) {
@@ -112,7 +129,7 @@ export class SyncClient {
     connectReplyTimer = null;
     socketCreatedHidden = false;
     socketOpenedAt = null;
-    plannedClose = null;
+    candidate = null;
     outageCloseCode = 1006;
     outageInfo = null;
     lastContactAt = null;
@@ -280,6 +297,7 @@ export class SyncClient {
         for (const remove of this.removeListeners.splice(0))
             remove();
         this.clearAllTimers();
+        this.dropCandidate();
         const socket = this.socket;
         this.socket = null;
         if (socket) {
@@ -555,6 +573,7 @@ export class SyncClient {
         const previous = this.socket;
         this.socket = null;
         this.clearConnectionTimers();
+        this.dropCandidate();
         if (previous) {
             previous.close();
             this.detachSocket(previous);
@@ -614,13 +633,13 @@ export class SyncClient {
             }
             this.cancelBackgroundTick();
             this.sendConnect();
-            this.pingTimer = this.setRepeating(() => this.sendRaw({ type: 'ping' }), PING_INTERVAL_MS);
-            this.rolloverTimer = this.setTimer(() => {
-                this.options.logger.info('ws rollover', {});
-                this.plannedClose = socket;
-                socket.close();
-            }, this.options.reconnectBeforeMs ?? DEFAULT_RECONNECT_BEFORE_MS);
+            this.startLiveTimers();
         };
+        this.attachLiveHandlers(socket, () => killedByWatchdog);
+    }
+    /** The live socket's message and close handling; a rollover's replacement
+     * gets it when it takes over. */
+    attachLiveHandlers(socket, killedByWatchdog) {
         socket.onmessage = (event) => {
             if (this.socket !== socket)
                 return;
@@ -638,7 +657,7 @@ export class SyncClient {
             if (this.halted || this.disposed)
                 return;
             const socketAgeMs = this.now() - this.socketCreatedAt;
-            if (this.socketOpenedAt === null && !killedByWatchdog && socketAgeMs >= SLOW_HANDSHAKE_MS) {
+            if (this.socketOpenedAt === null && !killedByWatchdog() && socketAgeMs >= SLOW_HANDSHAKE_MS) {
                 this.options.logger.info('sync connect failed', {
                     socketAgeMs,
                     closeCode: event.code,
@@ -651,20 +670,26 @@ export class SyncClient {
                 this.halt();
                 return;
             }
-            const planned = this.plannedClose === socket;
-            this.plannedClose = null;
             const openForMs = this.socketOpenedAt === null ? null : Math.max(0, this.now() - this.socketOpenedAt);
+            // Ended before its replacement took over: start afresh rather than wait
+            // on a handshake begun for a handover.
+            const rolling = this.candidate !== null;
+            this.dropCandidate();
+            if (event.code === CLOSE_SERVICE_RESTART) {
+                // The server's planned close (its age cap: a tab whose own rollover
+                // didn't run, such as a frozen one): reconnect at once, as after our
+                // own rollover, with no outage and no ladder step. A failure of that
+                // reconnect is an ordinary outage.
+                this.options.logger.info('ws closed by server', { closeCode: event.code, openForMs, rolling });
+                this.connectNow();
+                return;
+            }
             this.startOutage(event.code, event.wasClean === true, openForMs);
             if (!this.isHidden()) {
                 this.noteReturnFailure(event.code);
                 this.scheduleReconnect();
                 return;
             }
-            if (planned) {
-                // Our own rollover: this spell's sockets have lasted; reconnect once.
-                this.connectNow();
-                return;
-            }
             const openedAt = this.socketOpenedAt;
             if (this.socketCreatedHidden && (openedAt === null || this.now() - openedAt < HIDDEN_SOCKET_MIN_LIFE_MS)) {
                 this.hiddenSocketFailed = true;
@@ -672,6 +697,145 @@ export class SyncClient {
             this.enterDormant();
         };
     }
+    startLiveTimers() {
+        this.pingTimer = this.setRepeating(() => this.sendRaw({ type: 'ping' }), PING_INTERVAL_MS);
+        this.armRollover();
+    }
+    armRollover() {
+        this.rolloverTimer = this.setTimer(() => {
+            this.rolloverTimer = null;
+            this.startRollover();
+        }, this.rolloverDelayMs());
+    }
+    rolloverDelayMs() {
+        if (this.options.reconnectBeforeMs !== undefined)
+            return this.options.reconnectBeforeMs;
+        const factor = 1 - ROLLOVER_JITTER + 2 * ROLLOVER_JITTER * Math.random();
+        return Math.round(DEFAULT_RECONNECT_BEFORE_MS * factor);
+    }
+    /** Make-before-break, as notes' document sockets do: a replacement connects
+     * while the live socket keeps serving (sends, echoes, state), and takes over
+     * when the server answers its connect. Nothing the user sees changes, even
+     * with an edit unsent. The replacement's handshake re-runs the server's
+     * identity check; a 4403 halts as on any socket. If it fails, the live socket
+     * keeps serving until the server's age cap closes it (1012). */
+    startRollover() {
+        if (this.halted || this.disposed || this.candidate !== null || this.socket?.readyState !== SOCKET_OPEN)
+            return;
+        let socket;
+        try {
+            socket = this.options.runtime.createSocket(this.options.socketUrl());
+        }
+        catch (error) {
+            this.options.logger.warn('socket creation failed', { error });
+            return;
+        }
+        const candidate = {
+            socket,
+            createdAt: this.now(),
+            createdHidden: this.isHidden(),
+            openedAt: null,
+            watchdog: null,
+            early: [],
+        };
+        this.candidate = candidate;
+        candidate.watchdog = this.setTimer(() => {
+            candidate.watchdog = null;
+            if (this.candidate === candidate)
+                this.dropCandidate({ reason: 'timeout' });
+        }, CONNECT_WATCHDOG_MS);
+        socket.onopen = () => {
+            if (this.candidate !== candidate) {
+                socket.close();
+                return;
+            }
+            if (candidate.watchdog !== null)
+                this.options.runtime.clearTimeout(candidate.watchdog);
+            candidate.watchdog = null;
+            candidate.openedAt = this.now();
+            try {
+                socket.send(JSON.stringify(this.connectMessage()));
+            }
+            catch {
+                // Its close handler abandons it.
+            }
+        };
+        socket.onmessage = (event) => {
+            if (this.candidate !== candidate)
+                return;
+            const raw = String(event.data);
+            const type = messageType(raw);
+            if (type !== 'catchup' && type !== 'reload') {
+                candidate.early.push(raw);
+                return;
+            }
+            this.promoteCandidate(candidate);
+            this.lastContactAt = this.now();
+            this.handleMessage(raw);
+            for (const early of candidate.early) {
+                if (this.socket !== socket)
+                    return;
+                this.handleMessage(early);
+            }
+        };
+        socket.onerror = () => { };
+        socket.onclose = (event) => {
+            if (this.candidate !== candidate)
+                return;
+            if (event.code === CLOSE_NO_USER) {
+                this.dropCandidate();
+                this.options.authState.markBlocked();
+                this.halt();
+                return;
+            }
+            this.dropCandidate({ closeCode: event.code });
+        };
+    }
+    promoteCandidate(candidate) {
+        this.candidate = null;
+        if (candidate.watchdog !== null)
+            this.options.runtime.clearTimeout(candidate.watchdog);
+        const now = this.now();
+        const previous = this.socket;
+        const previousOpenedAt = this.socketOpenedAt;
+        this.clearConnectionTimers();
+        if (previous) {
+            this.detachSocket(previous);
+            previous.close();
+        }
+        this.socket = candidate.socket;
+        this.socketCreatedAt = candidate.createdAt;
+        this.socketCreatedHidden = candidate.createdHidden;
+        this.socketOpenedAt = candidate.openedAt;
+        candidate.socket.onopen = null;
+        this.attachLiveHandlers(candidate.socket, () => false);
+        this.startLiveTimers();
+        this.options.logger.info('ws rollover', {
+            openForMs: previousOpenedAt === null ? null : Math.max(0, now - previousOpenedAt),
+            handoverMs: Math.max(0, now - candidate.createdAt),
+        });
+    }
+    /** Abandons a rollover's replacement; `failure` logs why and tries again
+     * after another socket life (the server's age cap usually comes first). */
+    dropCandidate(failure) {
+        const candidate = this.candidate;
+        if (candidate === null)
+            return;
+        this.candidate = null;
+        if (candidate.watchdog !== null)
+            this.options.runtime.clearTimeout(candidate.watchdog);
+        this.detachSocket(candidate.socket);
+        candidate.socket.close();
+        if (failure) {
+            this.options.logger.info('ws rollover failed', {
+                ...failure,
+                opened: candidate.openedAt !== null,
+                socketAgeMs: Math.max(0, this.now() - candidate.createdAt),
+            });
+            if (this.socket?.readyState === SOCKET_OPEN && this.rolloverTimer === null)
+                this.armRollover();
+        }
+    }
     /** Hidden and disconnected: no WebSocket retries; HTTP catch-ups on a slow
      * cadence instead (docs/SYNC-BACKGROUND-PLAN.md). */
     enterDormant() {
@@ -1510,11 +1674,10 @@ export class SyncClient {
         this.resyncFailureTimer = null;
     }
     sendConnect() {
-        this.sendRaw({
-            type: 'connect',
-            clientGuid: this.options.clientGuid,
-            lastSeq: this.options.store.currentSeq(),
-        });
+        this.sendRaw(this.connectMessage());
+    }
+    connectMessage() {
+        return { type: 'connect', clientGuid: this.options.clientGuid, lastSeq: this.options.store.currentSeq() };
     }
     sendRaw(message) {
         if (this.socket?.readyState !== SOCKET_OPEN)
@@ -1605,6 +1768,7 @@ export class SyncClient {
         this.abortFetches();
         this.firstLoadRetry?.stop();
         this.clearAllTimers();
+        this.dropCandidate();
         const socket = this.socket;
         this.socket = null;
         if (socket) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.52.0",
+  "version": "3.53.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",
@@ -69,6 +69,10 @@
       "types": "./dist/hub/index.d.ts",
       "default": "./dist/hub/index.js"
     },
+    "./socket-age": {
+      "types": "./dist/socket-age.d.ts",
+      "default": "./dist/socket-age.js"
+    },
     "./app-update": {
       "types": "./dist/app-update/index.d.ts",
       "default": "./dist/app-update/index.js"