@inkbox/sdk 0.5.13 → 0.5.15

package/dist/tunnels/client/_runtime.js

@@ -17,6 +17,7 @@
  * stack against a real flow-control sequence.
  */
 import * as http2 from "node:http2";
+import { AsyncLocalStorage } from "node:async_hooks";
 import { setTimeout as setTimeoutPromise } from "node:timers/promises";
 import { BRIDGE_CLEANUP_SEND_TIMEOUT_MS, BRIDGE_CLOSE_CODE, BRIDGE_HALF_CLOSE_GRACE_MS, BRIDGE_STATUS_TIMEOUT_MS, BridgeProtocolError, BridgeStreamReset, makeBridgeStats, } from "./_bridge.js";
 import { filterResponseHeaders, parseEnvelope, } from "./_envelope.js";
@@ -25,7 +26,7 @@ import { ControlHeaders, ControlPaths, HOP_BY_HOP_RESPONSE, INKBOX_FORWARDED_HEA
 import { validateEnvelopePath } from "./_validation.js";
 import { createUndiciAgentCache, forwardEnvelopeToUrl, } from "./_url_forward.js";
 import { WS_OPCODE_BINARY, WS_OPCODE_CLOSE, WS_OPCODE_CONTINUATION, WS_OPCODE_PING, WS_OPCODE_PONG, WS_OPCODE_TEXT, WsFrameDecoder, encodeWsEnvelope, encodeWsFrame, } from "./_wsframe.js";
-import { dispatchWsUpgradeInProcess, WsServerDraining, } from "./_ws.js";
+import { dispatchWsUpgradeInProcess, WsConnectionLost, WsServerDraining, } from "./_ws.js";
 const HTTP2_HEADER_METHOD = http2.constants.HTTP2_HEADER_METHOD;
 const HTTP2_HEADER_PATH = http2.constants.HTTP2_HEADER_PATH;
 const HTTP2_HEADER_SCHEME = http2.constants.HTTP2_HEADER_SCHEME;
@@ -38,6 +39,9 @@ export const PING_INTERVAL_MS = 20_000;
  * a dead TCP doesn't strand the runtime past the next intake.
  */
 export const PING_ACK_TIMEOUT_MS = 10_000;
+export const CONNECT_TIMEOUT_MS = 10_000;
+export const HELLO_TIMEOUT_MS = 15_000;
+export const INITIAL_BACKOFF_MS = 1_000;
 export const BACKOFF_CAP_SEC = 30.0;
 export const BACKOFF_JITTER = 0.25;
 // On drain, keep a post-GOAWAY connection alive for its in-flight bridges
@@ -77,6 +81,14 @@ export class TunnelSupersededError extends Error {
         this.name = "TunnelSupersededError";
     }
 }
+class TunnelPhaseTimeoutError extends Error {
+    phase;
+    constructor(phase, timeoutMs) {
+        super(`tunnel ${phase} timed out after ${timeoutMs}ms`);
+        this.phase = phase;
+        this.name = "TunnelPhaseTimeoutError";
+    }
+}
 class OwnerTokenInvalidError extends Error {
     constructor(message) {
         super(message);
@@ -145,9 +157,17 @@ class Connection {
     streams = new Map();
     bridgeStreamIds = new Set();
     pingHandle = null;
-    pingAbort = null;
+    pingAckHandles = new Set();
+    closed;
+    resolveClosed;
+    closePublished = false;
+    detachTransportListeners = null;
+    cancelConnectWait = null;
     constructor(id) {
         this.id = id;
+        this.closed = new Promise((resolve) => {
+            this.resolveClosed = resolve;
+        });
     }
     /** Live WS/TCP bridge count — drives the drain-quiescent check. */
     get liveBridges() {
@@ -174,6 +194,12 @@ export class TunnelRuntime {
     onStatus;
     rng;
     http2Connect;
+    connectTimeoutMs;
+    helloTimeoutMs;
+    pingIntervalMs;
+    pingAckTimeoutMs;
+    handoffRedialBudgetMs;
+    sleep;
     // The pool that parks new intakes. Swapped atomically on handoff.
     active = null;
     // Post-GOAWAY connections finishing in-flight work before close.
@@ -203,6 +229,14 @@ export class TunnelRuntime {
     // in aclose().
     undiciAgentCache = createUndiciAgentCache();
     shutdownAbort = new AbortController();
+    runtimeStatus = "idle";
+    lastConnectedAtMs = null;
+    hasConnected = false;
+    coldAttempt = 0;
+    terminalError = null;
+    pendingConnection = null;
+    coldGeneration = 0;
+    dispatchGeneration = new AsyncLocalStorage();
     constructor(opts) {
         this.tunnelId = opts.tunnelId;
         this.apiKey = opts.apiKey;
@@ -218,21 +252,44 @@ export class TunnelRuntime {
         this.onStatus = opts.onStatus;
         this.rng = opts.rng ?? Math.random;
         this.http2Connect = opts.http2Connect ?? http2.connect.bind(http2);
+        this.connectTimeoutMs = opts.connectTimeoutMs ?? CONNECT_TIMEOUT_MS;
+        this.helloTimeoutMs = opts.helloTimeoutMs ?? HELLO_TIMEOUT_MS;
+        this.pingIntervalMs = opts.pingIntervalMs ?? PING_INTERVAL_MS;
+        this.pingAckTimeoutMs = opts.pingAckTimeoutMs ?? PING_ACK_TIMEOUT_MS;
+        this.handoffRedialBudgetMs =
+            opts.handoffRedialBudgetMs ?? HANDOFF_REDIAL_BUDGET_MS;
+        this.sleep = opts.sleep ?? ((delayMs, signal) => setTimeoutPromise(delayMs, undefined, { signal }));
     }
     // --- public lifecycle ---------------------------------------------------
+    get status() {
+        return this.runtimeStatus;
+    }
+    get isConnected() {
+        return this.runtimeStatus === "connected";
+    }
+    get lastConnectedAt() {
+        return this.lastConnectedAtMs === null
+            ? null
+            : new Date(this.lastConnectedAtMs);
+    }
     /**
      * Drive the runtime forever. Reconnects with jittered exponential
      * backoff; rejects only on permanent auth failure (rotate the
      * secret) or after `aclose()`.
      */
     async serveForever() {
-        let backoff = 1.0;
+        let backoff = INITIAL_BACKOFF_MS / 1000;
         let consecutiveFailures = 0;
-        this.notifyStatus("connecting");
         while (!this.stop) {
+            this.coldAttempt += 1;
+            const attempt = this.coldAttempt;
+            if (attempt === 1)
+                this.notifyStatus("connecting");
+            // eslint-disable-next-line no-console
+            console.info(`tunnel runtime: connection attempt #${attempt}`);
             try {
                 await this.runOnce();
-                backoff = 1.0;
+                backoff = INITIAL_BACKOFF_MS / 1000;
                 consecutiveFailures = 0;
             }
             catch (err) {
@@ -249,9 +306,21 @@ export class TunnelRuntime {
                 if (this.superseded) {
                     this.stopSuperseded();
                 }
+                if (this.stop) {
+                    if (this.runtimeStatus !== "superseded")
+                        this.notifyStatus("closed");
+                    return;
+                }
                 consecutiveFailures += 1;
-                // eslint-disable-next-line no-console
-                console.warn(`tunnel runtime: connection error (#${consecutiveFailures}); reconnecting`, err);
+                if (err instanceof TunnelPhaseTimeoutError) {
+                    // eslint-disable-next-line no-console
+                    console.warn(`tunnel runtime: ${err.phase} timeout on attempt #${attempt}; reconnecting`);
+                }
+                else {
+                    // eslint-disable-next-line no-console
+                    console.warn(`tunnel runtime: connection error on attempt #${attempt} ` +
+                        `(consecutive failure #${consecutiveFailures}); reconnecting`, err);
+                }
                 this.notifyStatus("reconnecting");
             }
             if (this.stop) {
@@ -264,10 +333,11 @@ export class TunnelRuntime {
             //   backoff = min(backoff * 2, 30.0)
             const jitter = backoff * BACKOFF_JITTER * (2 * this.rng() - 1);
             const sleepFor = Math.max(0.1, backoff + jitter);
+            const delayMs = sleepFor * 1000;
+            // eslint-disable-next-line no-console
+            console.info(`tunnel runtime: retrying with attempt #${attempt + 1} in ${delayMs}ms`);
             try {
-                await setTimeoutPromise(sleepFor * 1000, undefined, {
-                    signal: this.shutdownAbort.signal,
-                });
+                await this.sleep(delayMs, this.shutdownAbort.signal);
             }
             catch {
                 // aborted by aclose()
@@ -281,6 +351,9 @@ export class TunnelRuntime {
     /** Graceful shutdown. Signals all loops to exit; closes every conn. */
     async aclose() {
         this.stop = true;
+        this.coldGeneration += 1;
+        if (this.runtimeStatus !== "superseded")
+            this.notifyStatus("closed");
         this.shutdownAbort.abort();
         if (this.passthroughDispatch !== null) {
             try {
@@ -299,10 +372,12 @@ export class TunnelRuntime {
         }
         // Close active + every draining conn; stop each one's ping loop so no
         // ping loop leaks across the handoff set.
-        const conns = [this.active, ...this.draining].filter((c) => c !== null);
-        for (const conn of conns) {
+        const conns = [this.active, this.pendingConnection, ...this.draining].filter((c) => c !== null);
+        for (const conn of new Set(conns)) {
             this.stopPingLoop(conn);
             await this.closeConnection(conn);
+            this.publishConnectionClosed(conn);
+            conn.session = null;
         }
     }
     /**
@@ -310,55 +385,118 @@ export class TunnelRuntime {
      * short grace. The intake pool parks streams indefinitely, so a plain
      * `close()` would never resolve; we GOAWAY then destroy after 250ms.
      */
-    async closeConnection(conn) {
+    async closeConnection(conn, graceMs = 250) {
         const session = conn.session;
-        conn.session = null;
-        if (session === null || session.closed)
+        if (session === null || session.closed || session.destroyed) {
+            this.publishConnectionClosed(conn);
             return;
+        }
         try {
             session.goaway();
         }
         catch {
             /* swallow */
         }
+        if (graceMs <= 0) {
+            this.destroyConnection(conn);
+            return;
+        }
         await new Promise((resolve) => {
             const t = setTimeout(() => {
                 try {
                     session.destroy();
                 }
-                catch {
-                    /* swallow */
-                }
-                resolve();
-            }, 250);
-            session.once("close", () => {
+                catch { /* swallow */ }
+                setImmediate(() => {
+                    this.publishConnectionClosed(conn);
+                });
+            }, graceMs);
+            void conn.closed.then(() => {
                 clearTimeout(t);
-                resolve();
+                setTimeout(resolve, 10);
             });
             try {
                 session.close();
             }
             catch {
                 clearTimeout(t);
-                try {
-                    session.destroy();
-                }
-                catch {
-                    /* swallow */
-                }
-                resolve();
+                this.destroyConnection(conn);
             }
         });
     }
+    publishConnectionClosed(conn) {
+        if (!conn.closePublished) {
+            conn.closePublished = true;
+            for (const [, bus] of conn.streams) {
+                if (!bus.ended) {
+                    bus.events.push({ kind: "reset", code: 0 });
+                    bus.ended = true;
+                    this.wake(bus);
+                }
+            }
+            conn.resolveClosed();
+        }
+        conn.detachTransportListeners?.();
+        conn.detachTransportListeners = null;
+        conn.cancelConnectWait?.();
+        conn.cancelConnectWait = null;
+    }
+    destroyConnection(conn) {
+        const session = conn.session;
+        this.publishConnectionClosed(conn);
+        try {
+            session?.destroy();
+        }
+        catch { /* swallow */ }
+    }
     // --- per-connection lifecycle -----------------------------------------
+    async withPhaseTimeout(conn, phase, timeoutMs, operation) {
+        let timer = null;
+        let timeoutError = null;
+        const timeout = new Promise((_, reject) => {
+            timer = setTimeout(() => {
+                timeoutError = new TunnelPhaseTimeoutError(phase, timeoutMs);
+                this.destroyConnection(conn);
+                reject(timeoutError);
+            }, Math.max(1, timeoutMs));
+        });
+        void operation.catch(() => undefined);
+        try {
+            return await Promise.race([operation, timeout]);
+        }
+        catch (err) {
+            if (timeoutError !== null &&
+                !(err instanceof TunnelAuthError) &&
+                !(err instanceof TunnelSupersededError)) {
+                throw timeoutError;
+            }
+            throw err;
+        }
+        finally {
+            if (timer !== null)
+                clearTimeout(timer);
+        }
+    }
+    markConnected(emitStatus) {
+        const recovered = this.hasConnected;
+        this.hasConnected = true;
+        this.lastConnectedAtMs = Date.now();
+        if (emitStatus) {
+            this.notifyStatus("connected");
+            // eslint-disable-next-line no-console
+            console.info(recovered
+                ? "tunnel runtime: connection recovered"
+                : "tunnel runtime: initial connection established");
+        }
+    }
     async runOnce() {
         const first = new Connection(this.nextConnId++);
         let conn = first;
         this.active = first;
         try {
-            await this.openConnection(first);
-            await this.sendHello(first);
-            this.notifyStatus("connected");
+            await this.withPhaseTimeout(first, "connect", this.connectTimeoutMs, this.openConnection(first));
+            await this.withPhaseTimeout(first, "hello", this.helloTimeoutMs, this.sendHello(first));
+            this.markConnected(true);
             this.startServing(first);
             // Supervise the active connection. A GOAWAY handoff swaps in a new
             // active conn out-of-band; follow it without going through the
@@ -374,6 +512,8 @@ export class TunnelRuntime {
                 if (this.handoffInFlight && this.handoffPromise !== null) {
                     await this.handoffPromise;
                 }
+                if (this.terminalError !== null)
+                    throw this.terminalError;
                 const next = this.active;
                 if (next !== null && next !== conn && !next.draining) {
                     conn = next;
@@ -387,11 +527,20 @@ export class TunnelRuntime {
             if (this.superseded) {
                 throw new TunnelSupersededError("another client connected to this tunnel; not reconnecting");
             }
+            if (!this.stop) {
+                // eslint-disable-next-line no-console
+                console.warn("tunnel runtime: connected session lost; reconnecting");
+                this.notifyStatus("reconnecting");
+            }
         }
         finally {
+            if (!this.stop)
+                this.coldGeneration += 1;
             this.stopPingLoop(conn);
             conn.streams.clear();
             conn.bridgeStreamIds.clear();
+            await this.closeConnection(conn);
+            this.publishConnectionClosed(conn);
             conn.session = null;
             if (this.active === conn)
                 this.active = null;
@@ -409,18 +558,10 @@ export class TunnelRuntime {
     }
     /** Resolve once the supervised conn closes OR a handoff swaps active. */
     waitCloseOrHandoff(conn) {
-        const closed = new Promise((resolve) => {
-            const session = conn.session;
-            if (session === null || session.closed) {
-                resolve();
-                return;
-            }
-            session.once("close", () => resolve());
-        });
         const woken = new Promise((resolve) => {
             this.wakeSupervisor = resolve;
         });
-        return Promise.race([closed, woken]).finally(() => {
+        return Promise.race([conn.closed, woken]).finally(() => {
             this.wakeSupervisor = null;
         });
     }
@@ -507,6 +648,7 @@ export class TunnelRuntime {
         try {
             const newConn = await this.makeReplacementConnection();
             this.active = newConn;
+            this.markConnected(false);
             // Supervisor was watching oldConn; wake it to follow newConn.
             this.signalSupervisor();
         }
@@ -515,10 +657,12 @@ export class TunnelRuntime {
                 // External takeover during the handoff hello: `superseded` is set, so
                 // end the old conn and wake the supervisor to stop terminally (no cold
                 // redial, which would boot the client that replaced us).
-                try {
-                    oldConn.session?.destroy();
-                }
-                catch { /* swallow */ }
+                this.destroyConnection(oldConn);
+                this.signalSupervisor();
+            }
+            else if (err instanceof TunnelAuthError) {
+                this.terminalError = err;
+                this.destroyConnection(oldConn);
                 this.signalSupervisor();
             }
             else {
@@ -527,10 +671,7 @@ export class TunnelRuntime {
                 // the old session closed so the supervisor returns.
                 // eslint-disable-next-line no-console
                 console.warn("tunnel runtime: handoff failed; reconnecting cold", err);
-                try {
-                    oldConn.session?.destroy();
-                }
-                catch { /* swallow */ }
+                this.destroyConnection(oldConn);
                 this.signalSupervisor();
             }
         }
@@ -544,20 +685,33 @@ export class TunnelRuntime {
     /** Dial + hello + park a replacement, retrying transient hello failures. */
     async makeReplacementConnection() {
         let backoff = 0.1;
-        const start = Date.now();
+        const deadline = performance.now() + this.handoffRedialBudgetMs;
         while (!this.stop) {
+            let remaining = deadline - performance.now();
+            if (remaining <= 0)
+                throw new Error("handoff redial budget exhausted");
             const conn = new Connection(this.nextConnId++);
+            this.pendingConnection = conn;
             try {
-                await this.openConnection(conn);
-                await this.sendHello(conn);
+                await this.withPhaseTimeout(conn, "connect", Math.min(this.connectTimeoutMs, remaining), this.openConnection(conn));
+                remaining = deadline - performance.now();
+                if (remaining <= 0)
+                    throw new Error("handoff redial budget exhausted");
+                await this.withPhaseTimeout(conn, "hello", Math.min(this.helloTimeoutMs, remaining), this.sendHello(conn));
                 this.startServing(conn);
+                this.pendingConnection = null;
                 return conn;
             }
             catch (err) {
+                remaining = deadline - performance.now();
                 try {
-                    await this.closeConnection(conn);
+                    await this.closeConnection(conn, Math.min(250, Math.max(0, remaining)));
                 }
                 catch { /* swallow */ }
+                this.publishConnectionClosed(conn);
+                conn.session = null;
+                if (this.pendingConnection === conn)
+                    this.pendingConnection = null;
                 // A rejected key or a takeover is terminal; never retry either
                 // (retrying a takeover would boot the client that replaced us).
                 if (err instanceof TunnelAuthError || err instanceof TunnelSupersededError) {
@@ -569,15 +723,15 @@ export class TunnelRuntime {
                 if (this.superseded) {
                     throw new TunnelSupersededError("another client connected to this tunnel during handoff");
                 }
-                if (Date.now() - start > HANDOFF_REDIAL_BUDGET_MS) {
+                remaining = deadline - performance.now();
+                if (remaining <= 0) {
                     throw new Error("handoff redial budget exhausted");
                 }
                 // A drain 503 on the new hello means the NLB landed us back on the
                 // draining task; back off (jittered) so it re-routes us elsewhere.
                 const jitter = backoff * BACKOFF_JITTER * (2 * this.rng() - 1);
-                await setTimeoutPromise(Math.max(50, (backoff + jitter) * 1000), undefined, {
-                    signal: this.shutdownAbort.signal,
-                }).catch(() => undefined);
+                const delayMs = Math.min(Math.max(50, (backoff + jitter) * 1000), remaining);
+                await this.sleep(delayMs, this.shutdownAbort.signal).catch(() => undefined);
                 backoff = Math.min(backoff * 2, 5.0);
             }
         }
@@ -592,6 +746,8 @@ export class TunnelRuntime {
             await setTimeoutPromise(250).catch(() => undefined);
         }
         await this.closeConnection(oldConn);
+        this.publishConnectionClosed(oldConn);
+        oldConn.session = null;
         oldConn.streams.clear();
         oldConn.bridgeStreamIds.clear();
         this.draining.delete(oldConn);
@@ -607,27 +763,27 @@ export class TunnelRuntime {
             // (Spike 1) — the setting line doesn't translate.
         });
         conn.session = session;
-        session.on("close", () => {
+        const socket = (() => {
+            try {
+                return session.socket ?? null;
+            }
+            catch {
+                return null;
+            }
+        })();
+        const onSessionClose = () => {
             // eslint-disable-next-line no-console
             console.info("tunnel runtime: h2 session closed");
-            // Drain all open streams with a synthetic reset event so any
-            // awaiters wake up.
-            for (const [, bus] of conn.streams) {
-                if (!bus.ended) {
-                    bus.events.push({ kind: "reset", code: 0 });
-                    bus.ended = true;
-                    this.wake(bus);
-                }
-            }
-        });
-        session.on("error", (err) => {
+            this.publishConnectionClosed(conn);
+        };
+        const onSessionError = (err) => {
             // Visibility into session-fatal errors. Stream-level errors
             // surface separately via stream events; this is genuinely
             // session-terminal.
             // eslint-disable-next-line no-console
             console.warn("tunnel runtime: h2 session error", err);
-        });
-        session.on("goaway", (errorCode, lastStreamId, opaqueData) => {
+        };
+        const onGoaway = (errorCode, lastStreamId, opaqueData) => {
             // eslint-disable-next-line no-console
             console.info(`tunnel runtime: GOAWAY received error_code=${errorCode} last_stream_id=${lastStreamId}`);
             // NO_ERROR GOAWAY = drain (make-before-break handoff). A non-zero code
@@ -640,51 +796,69 @@ export class TunnelRuntime {
             else {
                 this.maybeMarkSupersededGoaway(conn, errorCode, opaqueData);
             }
-        });
-        // Watch the underlying TCP/TLS socket directly. Node's h2 client
-        // sometimes loses the connection without emitting ``error`` or
-        // ``close`` on the session itself — the underlying socket reliably
-        // emits them. Force-destroy the session on either so
-        // ``waitForSessionClose`` resolves promptly and ``serveForever``
-        // reconnects without waiting for the ``PING_ACK_TIMEOUT_MS`` window.
-        try {
-            const sock = session.socket;
-            const onSocketDeath = (label, err) => {
+        };
+        const onSocketClose = (hadError) => {
+            onSocketDeath(`closed hadError=${hadError}`);
+        };
+        const onSocketError = (err) => {
+            onSocketDeath("error", err);
+        };
+        const onSocketDeath = (label, err) => {
+            // eslint-disable-next-line no-console
+            console.info(`tunnel runtime: underlying socket ${label}` +
+                (err !== undefined ? ` err=${err.message}` : ""));
+            if (!session.closed && !session.destroyed) {
                 // eslint-disable-next-line no-console
-                console.info(`tunnel runtime: underlying socket ${label}` +
-                    (err !== undefined ? ` err=${err.message}` : ""));
-                if (!session.closed && !session.destroyed) {
-                    // Forensic — log the stack so we can see which path
-                    // triggered this destroy in production.
-                    // eslint-disable-next-line no-console
-                    console.warn("tunnel runtime: forcing session.destroy() from socket-death", new Error("trace").stack);
-                    try {
-                        session.destroy();
-                    }
-                    catch { /* swallow */ }
-                }
-            };
-            sock?.once?.("close", (hadError) => {
-                onSocketDeath(`closed hadError=${hadError}`);
-            });
-            sock?.once?.("error", (err) => {
-                onSocketDeath("error", err);
-            });
-        }
-        catch {
-            /* swallow */
-        }
+                console.warn("tunnel runtime: forcing session.destroy() from socket-death");
+                this.destroyConnection(conn);
+            }
+        };
+        session.on("close", onSessionClose);
+        session.on("error", onSessionError);
+        session.on("goaway", onGoaway);
+        socket?.once("close", onSocketClose);
+        socket?.once("error", onSocketError);
+        conn.detachTransportListeners = () => {
+            session.off("close", onSessionClose);
+            session.off("error", onSessionError);
+            session.off("goaway", onGoaway);
+            try {
+                socket?.off("close", onSocketClose);
+                socket?.off("error", onSocketError);
+            }
+            catch {
+                /* the HTTP/2 socket proxy becomes inaccessible after disconnect */
+            }
+        };
         await new Promise((resolve, reject) => {
-            const onConnect = () => {
+            let settled = false;
+            const cleanup = () => {
+                session.off("connect", onConnect);
                 session.off("error", onError);
+                session.off("close", onClose);
+                conn.cancelConnectWait = null;
+            };
+            const succeed = () => {
+                if (settled)
+                    return;
+                settled = true;
+                cleanup();
                 resolve();
             };
-            const onError = (err) => {
-                session.off("connect", onConnect);
+            const fail = (err) => {
+                if (settled)
+                    return;
+                settled = true;
+                cleanup();
                 reject(err);
             };
+            const onConnect = () => succeed();
+            const onError = (err) => fail(err);
+            const onClose = () => fail(new Error("h2 session closed before connect completed"));
+            conn.cancelConnectWait = () => fail(new Error("h2 session closed before connect completed"));
             session.once("connect", onConnect);
             session.once("error", onError);
+            session.once("close", onClose);
         });
         // OS-level keepalive on the underlying TCP socket so a silently-
         // dropped connection (NAT timeout, NLB idle eviction, peer power-
@@ -693,25 +867,12 @@ export class TunnelRuntime {
         // tracking (see startPingLoop) is the load-bearing detector;
         // this is defense-in-depth.
         try {
-            const sock = session.socket;
-            sock?.setKeepAlive?.(true, 30_000);
+            socket?.setKeepAlive?.(true, 30_000);
         }
         catch {
             /* swallow */
         }
     }
-    waitForSessionClose(conn) {
-        const session = conn.session;
-        if (session === null)
-            return Promise.resolve();
-        return new Promise((resolve) => {
-            if (session.closed) {
-                resolve();
-                return;
-            }
-            session.once("close", () => resolve());
-        });
-    }
     // --- handshake ---------------------------------------------------------
     async sendHello(conn) {
         conn.ownerToken = null;
@@ -731,7 +892,14 @@ export class TunnelRuntime {
             helloHeaders[ControlHeaders.POOL_SIZE] = String(this.poolSize);
         }
         const stream = this.openStream(conn, helloHeaders, { endStream: true });
-        const { status, body } = await this.awaitResponse(conn, stream.streamId);
+        let status;
+        let body;
+        try {
+            ({ status, body } = await this.awaitResponse(conn, stream.streamId));
+        }
+        finally {
+            conn.streams.delete(stream.streamId);
+        }
         if (status === 401 || status === 403) {
             throw new TunnelAuthError(`${ControlPaths.HELLO} returned ${status}; the API key was rejected (check the key matches the tunnel's identity scope, or use an admin-scoped key in the tunnel's org)`);
         }
@@ -881,7 +1049,8 @@ export class TunnelRuntime {
         while (!this.stop &&
             !conn.draining &&
             conn.session !== null &&
-            !conn.session.closed) {
+            !conn.session.closed &&
+            !conn.session.destroyed) {
             let envelope;
             try {
                 envelope = await this.parkOneIntake(conn, slot);
@@ -891,17 +1060,14 @@ export class TunnelRuntime {
                     // Another client took over: force this conn down so the supervisor
                     // observes the terminal flag and stops (no reconnect).
                     this.markSuperseded();
-                    try {
-                        conn.session?.destroy();
-                    }
-                    catch { /* swallow */ }
+                    this.destroyConnection(conn);
                     return;
                 }
                 if (err instanceof OwnerTokenInvalidError) {
                     // eslint-disable-next-line no-console
                     console.warn(`intake slot ${slot}: owner_token rejected; ` +
                         `forcing session.destroy() and reconnecting`);
-                    conn.session?.destroy();
+                    this.destroyConnection(conn);
                     return;
                 }
                 if (isSessionTerminalError(err) || conn.session?.destroyed) {
@@ -916,10 +1082,7 @@ export class TunnelRuntime {
                     console.warn(`intake slot ${slot}: h2 session terminal (` +
                         `${err?.code ?? "no code"}); ` +
                         `exiting slot`, err);
-                    try {
-                        conn.session?.destroy();
-                    }
-                    catch { /* swallow */ }
+                    this.destroyConnection(conn);
                     return;
                 }
                 // eslint-disable-next-line no-console
@@ -932,7 +1095,10 @@ export class TunnelRuntime {
             // Fire-and-forget dispatch; tracked on the runtime (not the conn) so
             // an in-flight handler survives this conn draining and can post its
             // reply on the new active conn during a handoff.
-            const task = this.dispatchEnvelope(conn, envelope).catch((err) => {
+            const generation = this.coldGeneration;
+            const task = this.dispatchGeneration
+                .run(generation, () => this.dispatchEnvelope(conn, envelope))
+                .catch((err) => {
                 // eslint-disable-next-line no-console
                 console.warn(`dispatch failed request_id=${envelope.requestId}`, err);
             });
@@ -1003,10 +1169,9 @@ export class TunnelRuntime {
     }
     // --- ping loop ---------------------------------------------------------
     startPingLoop(conn) {
-        conn.pingAbort = new AbortController();
         conn.pingHandle = setInterval(() => {
             const session = conn.session;
-            if (session === null || session.closed)
+            if (session === null || session.closed || session.destroyed)
                 return;
             let ackTimer = null;
             let acked = false;
@@ -1015,27 +1180,24 @@ export class TunnelRuntime {
                     acked = true;
                     if (ackTimer !== null) {
                         clearTimeout(ackTimer);
+                        conn.pingAckHandles.delete(ackTimer);
                         ackTimer = null;
                     }
                     if (err !== null && err !== undefined) {
                         // eslint-disable-next-line no-console
                         console.warn("tunnel runtime: PING errored; forcing session.destroy()", err);
-                        try {
-                            session.destroy();
-                        }
-                        catch { /* swallow */ }
+                        this.destroyConnection(conn);
                     }
                 });
             }
             catch (err) {
                 // eslint-disable-next-line no-console
                 console.warn("tunnel runtime: session.ping() threw synchronously; forcing destroy", err);
-                try {
-                    session.destroy();
-                }
-                catch { /* swallow */ }
+                this.destroyConnection(conn);
                 return;
             }
+            if (acked)
+                return;
             // Application-level liveness check: if the ack doesn't come
             // back within PING_ACK_TIMEOUT_MS, the underlying TCP is gone
             // (kernel send buffer absorbing writes silently is the typical
@@ -1043,18 +1205,18 @@ export class TunnelRuntime {
             // without our help). Force-destroy the session; serveForever
             // observes the close and reconnects.
             ackTimer = setTimeout(() => {
+                if (ackTimer !== null)
+                    conn.pingAckHandles.delete(ackTimer);
                 if (acked)
                     return;
                 // eslint-disable-next-line no-console
                 console.warn(`tunnel runtime: PING ack not received within ` +
-                    `${PING_ACK_TIMEOUT_MS}ms; assuming dead connection, ` +
+                    `${this.pingAckTimeoutMs}ms; assuming dead connection, ` +
                     `forcing reconnect`);
-                try {
-                    session.destroy();
-                }
-                catch { /* swallow */ }
-            }, PING_ACK_TIMEOUT_MS);
-        }, PING_INTERVAL_MS);
+                this.destroyConnection(conn);
+            }, this.pingAckTimeoutMs);
+            conn.pingAckHandles.add(ackTimer);
+        }, this.pingIntervalMs);
         // Do NOT unref(): explicit cancellation in stopPingLoop().
     }
     stopPingLoop(conn) {
@@ -1062,8 +1224,9 @@ export class TunnelRuntime {
             clearInterval(conn.pingHandle);
             conn.pingHandle = null;
         }
-        conn.pingAbort?.abort();
-        conn.pingAbort = null;
+        for (const handle of conn.pingAckHandles)
+            clearTimeout(handle);
+        conn.pingAckHandles.clear();
     }
     // --- envelope dispatch -------------------------------------------------
     async dispatchEnvelope(conn, envelope) {
@@ -1514,13 +1677,13 @@ export class TunnelRuntime {
                     if (id === null) {
                         if (conn.draining)
                             throw new WsServerDraining();
-                        return;
+                        throw new WsConnectionLost();
                     }
                     const ev = await self.nextEvent(conn, id);
                     if (ev === null) {
                         if (conn.draining)
                             throw new WsServerDraining();
-                        return;
+                        throw new WsConnectionLost();
                     }
                     if (ev.kind === "data") {
                         yield ev.data;
@@ -1528,14 +1691,14 @@ export class TunnelRuntime {
                     else if (ev.kind === "end") {
                         if (conn.draining)
                             throw new WsServerDraining();
-                        return;
+                        throw new WsConnectionLost();
                     }
                     else if (ev.kind === "reset") {
                         // A reset while the conn is draining is the redeploy drain, not
                         // a peer error — surface it typed so the handler can reconnect.
                         if (conn.draining)
                             throw new WsServerDraining();
-                        throw new Error(`bridge stream reset code=${ev.code}`);
+                        throw new WsConnectionLost(`bridge stream reset code=${ev.code}`);
                     }
                 }
             })();
@@ -1958,6 +2121,8 @@ export class TunnelRuntime {
                 setTimeoutPromise(POST_ACTIVE_WAIT_MS),
             ]);
         }
+        if (!this.dispatchIsCurrent())
+            return;
         const target = this.pickReplyConnection(origin);
         if (target === null) {
             // eslint-disable-next-line no-console
@@ -1968,7 +2133,11 @@ export class TunnelRuntime {
     }
     /** The active conn if it can take new streams, else the origin if it can. */
     pickReplyConnection(origin) {
-        const usable = (c) => c !== null && !c.draining && c.session !== null && !c.session.closed;
+        const usable = (c) => c !== null &&
+            !c.draining &&
+            c.session !== null &&
+            !c.session.closed &&
+            !c.session.destroyed;
         if (usable(this.active))
             return this.active;
         if (usable(origin))
@@ -1976,6 +2145,8 @@ export class TunnelRuntime {
         return null;
     }
     async postResponse(conn, requestId, status, userHeaders, body) {
+        if (!this.dispatchIsCurrent())
+            return;
         const reqHeaders = {
             [HTTP2_HEADER_METHOD]: "POST",
             [HTTP2_HEADER_SCHEME]: "https",
@@ -2039,7 +2210,14 @@ export class TunnelRuntime {
         }
     }
     // --- utilities ---------------------------------------------------------
+    dispatchIsCurrent() {
+        const generation = this.dispatchGeneration.getStore();
+        return generation === undefined || generation === this.coldGeneration;
+    }
     notifyStatus(status) {
+        if (this.runtimeStatus === status)
+            return;
+        this.runtimeStatus = status;
         if (this.onStatus !== undefined) {
             try {
                 this.onStatus(status);