@catalyst-cloud/sdk 0.6.0 → 0.8.0

package/dist/live-sync-client.js

@@ -30,8 +30,9 @@
 // silence it sends the pinned `{"type":"ping"}` frame; the mirror answers via `setWebSocketAutoResponse`
 // (which replies WITHOUT waking a hibernated DO — ADR-0009's cost model is preserved). If no frame
 // arrives within `pongTimeoutMs`, the socket is force-reconnected through the existing backoff path.
-// Traffic postpones pings (no keepalive on a busy stream), and a 3-probe feature-detect disables the
-// watchdog against an old server that never pongs — so it degrades to exactly today's behavior.
+// Traffic postpones pings (no keepalive on a busy stream), and a 3-probe feature-detect DEGRADES the
+// watchdog against an old server that never pongs — re-probing at 10x the interval instead of never
+// (CTC-281: detection can be slowed, but never permanently lost).
 //
 // Gap detection (CTL-1402): the server's live push (`broadcastChange`) is at-most-once — a send into a
 // half-open socket is silently swallowed, and the dropped frame used to be sealed over permanently the
@@ -44,7 +45,30 @@
 // paginated and answers `{type:"resync"}` on underflow). Bounded: after `gapRetryLimit` no-PROGRESS
 // windows (the heal deadline re-arms on every delivered frame, so a big-but-advancing heal never
 // escalates) the client escalates to the full re-seed path rather than spinning — a gap is never
-// silently accepted. Every transition emits the `catalyst.replica.gap` log/counter signal, the
+// silently accepted.
+//
+// Wedge-proofing (CTC-281): the Jul 17-23 fleet incident (6 windows of server-side half-opens with no
+// FIN/RST; cursors frozen 28-215 min while clients said "live") exposed four restart-only states this
+// client could reach. The invariant now enforced: EVERY state that is not "stopped" holds either a
+// pending timer or a socket whose events re-enter the machine — there is no state only a process
+// restart clears. Concretely: (1) a connect attempt whose ws impl never fires open/close/error is
+// bounded by `openTimeoutMs`; (2) `onerror` without a follow-up `onclose` (real undici bugs #3697/
+// #3546) arms a one-shot fallback reconnect; (3) the watchdog feature-detect can only DEGRADE itself
+// (a slow re-probe every DEGRADED_PROBE_MULTIPLIER x pingIntervalMs), never disable itself outright —
+// and only while pong capability is UNPROVEN; once ANY pong has ever been observed, silence is always
+// treated as a liveness failure (during the incident, 3 open-then-silent sockets used to disable
+// detection for the client's lifetime ~6 min into a window — and because the pong latch is per-process,
+// a client RESTARTED mid-window would have re-latched the disable, so the degrade-not-disable shape is
+// what actually guarantees convergence); (4) a FAILED reseed re-enters the backoff path instead
+// of hot-reopening — and the reseed await itself is bounded by `reseedTimeoutMs` (the injected
+// callback is a trust boundary like the ws impl: both first-party reseeds self-bound (the node
+// replica's seedFromSnapshot and the browser replica's OPFS seed both abort on an idle body), but an
+// arbitrary consumer-supplied reseed can still hang, and "resyncing" holds no socket and suppresses
+// scheduleReconnect, so without this bound
+// it was the one remaining zero-timer state; a timed-out reseed is ABANDONED, its late settle
+// discarded, and the client re-enters backoff); (5) closeSocket() escalates past `close()` to a duck-typed `terminate()` (Bun /
+// the 'ws' package expose one; undici does not — its close-handshake wait is why teardown must not
+// depend on a graceful close against a half-open peer). Every transition emits the `catalyst.replica.gap` log/counter signal, the
 // detector the per-frame apply telemetry is structurally blind to (an undelivered frame lands in no
 // apply bucket). Gaps are the STEADY-STATE path here — the mirror's reconcile pass appends change_log
 // rows it never broadcasts, so every pass punches a hole that heals via re-request — hence `detected`
@@ -54,6 +78,16 @@
 // beyond-gap change frame (re-request the hole `deliveredSeq+1..head`) but never applies it.
 import { PING_FRAME } from "./types.js";
 import { NOOP_TELEMETRY, createTelemetry, CATALYST_ATTR, REPLICA_LOG, REPLICA_METRIC, REPLICA_SPAN, DEFAULT_SCOPE_NAME, } from "./otel.js";
+/**
+ * How long a CANCELLED reseed gets to unwind before the transport settles anyway.
+ *
+ * Cancelling is asynchronous on the consumer's side — the browser seed aborts a fetch, trips its
+ * supersede guard, posts `seedAbort` and resumes its delta queue — and the transport reconnects the
+ * moment it settles, so it must wait for that unwind or it reconnects into a still-paused consumer.
+ * Bounded because the deadline that triggered this exists precisely for an unresponsive callback: a
+ * cleanup that also hangs must not wedge the transport (CTC-114 review round 12).
+ */
+const DEFAULT_CANCEL_CLEANUP_GRACE_MS = 250;
 /** Resolve the runtime global WebSocket, or fail with an actionable message. */
 function defaultWsFactory(url) {
     const Ctor = globalThis.WebSocket;
@@ -83,15 +117,43 @@ export function buildConnectUrl(opts) {
     const params = new URLSearchParams();
     if (opts.auth.kind === "token")
         params.set("token", opts.auth.token);
-    params.set("account", opts.accountId);
-    return `${origin}${opts.connectPath}?${params.toString()}`;
+    // Only when a tenant was actually named. `?account=` is NOT the same as no account: the server's
+    // consumers are truthiness checks, so empty takes the omitted path anyway — but it would freeze a
+    // contract in which "" is a legal mirror name, and it puts `catalyst.tenant=""` on every span.
+    if (opts.accountId)
+        params.set("account", opts.accountId);
+    const query = params.toString();
+    // Guard the dangling "?" — with cookie auth and no account there are no params at all.
+    return `${origin}${opts.connectPath}${query ? `?${query}` : ""}`;
 }
 /**
- * Consecutive opened-then-never-ponged connections after which the watchdog disables itself for the
- * client's lifetime (feature-detect for a server without auto-pong). Bounds worst-case reconnect
- * churn against an old server to exactly this many attempts, making mirror/SDK deploy order harmless.
+ * Consecutive opened-then-never-ponged connections after which the watchdog DEGRADES itself
+ * (feature-detect for a server without auto-pong). Bounds worst-case reconnect churn against an old
+ * server, making mirror/SDK deploy order harmless.
  */
 const PROBE_FAILURE_LIMIT = 3;
+/**
+ * Degraded-watchdog probe interval, as a multiple of `pingIntervalMs` (CTC-281). After
+ * {@link PROBE_FAILURE_LIMIT} never-ponged connections the watchdog does NOT turn off — it re-probes
+ * at this heavily backed-off cadence (stock: every 15 min instead of 90 s). A hard lifetime disable
+ * was the incident's restart-only residual: the `pongEverObserved` latch is per-PROCESS, so a client
+ * (re)started inside an incident window (supervisors restarted processes mid-window) came up
+ * unproven, burned its 3 probes against open-but-silent sockets, and went permanently blind — the
+ * next half-open then froze it as "live" forever, and only another restart (which repeats the cycle)
+ * cleared it. Degrading instead keeps a probe pending in EVERY non-stopped state: a half-open socket
+ * under a degraded watchdog is still detected within ~this multiple of the interval, and the first
+ * pong after recovery re-arms full-speed detection (and latches capability as proven). Against a
+ * genuinely old server the cost is one bounded reconnect per degraded window — churn, never wedge.
+ */
+const DEGRADED_PROBE_MULTIPLIER = 10;
+/**
+ * How long (ms) after `onerror` to wait for the spec-mandated follow-up `onclose` before forcing the
+ * reconnect ourselves (CTC-281). WHATWG requires close-after-error, but real impls have shipped
+ * violations (undici #3697 "close not emitted on error", #3546 "close not fired if the connection
+ * failed to be established") — and `WebSocketLike` is structural, so an injected impl is trusted
+ * blindly. Pre-open, a missing onclose used to be a ZERO-timer permanent-"error" wedge.
+ */
+const ERROR_CLOSE_GRACE_MS = 5_000;
 export class LiveSyncClient {
     baseUrl;
     accountId;
@@ -106,6 +168,9 @@ export class LiveSyncClient {
     maxBackoffMs;
     pingIntervalMs;
     pongTimeoutMs;
+    openTimeoutMs;
+    reseedTimeoutMs;
+    cancelCleanupGraceMs;
     gapTimeoutMs;
     gapRetryLimit;
     wsFactory;
@@ -113,6 +178,36 @@ export class LiveSyncClient {
     telemetryConfig;
     ws = null;
     stopped = false;
+    /**
+     * Has start() been entered? `stopped` alone cannot answer this — it is false BEFORE the first
+     * start() as well as during a run, so every "am I running?" guard read true on a client that had
+     * never booted. Only the public `requestResync()` can reach that window (CTC-114 review round 5):
+     * it would reseed and open a socket with no lifecycle deferred and no telemetry resolved, and the
+     * later real start() would then openSocket() again — overwriting `this.ws`, so the first socket
+     * kept delivering duplicate frames and could no longer be closed through the stored reference.
+     */
+    started = false;
+    /** The in-flight boot task, or null once it settles — `requestResync()` serializes behind it. */
+    bootTask = null;
+    /** Settles an in-flight `boundedReseed` wrapper on stop(), so an awaited resync cannot hang. */
+    abandonReseed = null;
+    /**
+     * The in-flight resync, so a concurrent caller AWAITS it instead of being handed an
+     * already-resolved promise and acting on a store still being rebuilt (round 10).
+     */
+    activeResync = null;
+    /** Did the boot this request waited on perform a COLD re-seed? Only then may it be absorbed. */
+    bootColdSeeded = false;
+    /**
+     * Did the last boot REJECT? Latched until the next `start()` (CTC-114 review round 9).
+     *
+     * Round 8 read the failure from the awaited `bootTask`, but that handle is nulled once the boot
+     * settles — so a `requestResync()` arriving after that microtask found no record of the failure,
+     * with `started` still true and `stopped` still false, and sailed past the guard into a reseed that
+     * could open a live socket under an application already told startup had failed. The outcome has to
+     * outlive the handle.
+     */
+    bootFailed = false;
     resyncing = false;
     backoff;
     reconnectTimer = null;
@@ -148,12 +243,44 @@ export class LiveSyncClient {
     pingSentAt = 0;
     /** Client-lifetime: consecutive opened-then-never-ponged connections. Reset to 0 by ANY pong. */
     probeFailures = 0;
-    /** Client-lifetime: after PROBE_FAILURE_LIMIT never-ponged connections we stop pinging for good
-     *  (an old server without auto-pong), degrading to close/error-only detection. */
-    watchdogDisabled = false;
+    /** Client-lifetime until a pong: after PROBE_FAILURE_LIMIT never-ponged connections the watchdog
+     *  DEGRADES to a {@link DEGRADED_PROBE_MULTIPLIER}x-slower re-probe (an old server without auto-pong)
+     *  — it never turns off outright, so detection is never a restart-only casualty (CTC-281). Only
+     *  reachable while pong capability is UNPROVEN ({@link pongEverObserved}); the first pong clears it. */
+    watchdogDegraded = false;
+    /** Client-lifetime pong latch (CTC-281): has ANY connection EVER answered a ping? Once true, the
+     *  server's auto-pong capability is PROVEN for good — a later never-ponged connection is a liveness
+     *  failure (the incident's open-but-silent socket), never feature-detect evidence, so the watchdog
+     *  can no longer even degrade itself. PER-PROCESS by design — which is exactly why the degrade must
+     *  be soft (see {@link DEGRADED_PROBE_MULTIPLIER}): a restart mid-incident resets this latch. */
+    pongEverObserved = false;
     pingTimer = null;
     pongDeadline = null;
+    /** Per-CONNECTION connect/open deadline (CTC-281): armed when the socket is constructed, cleared on
+     *  open/close/teardown. The only timer pending between openSocket() and onopen — the guarantee that
+     *  a never-firing ws impl cannot leave the client wedged in "connecting" with nothing scheduled. */
+    connectTimer = null;
+    /** Per-CONNECTION onerror→onclose fallback (CTC-281): armed by onerror, fires forceReconnect once
+     *  if the impl never follows error with close (undici #3697/#3546). Cleared on open/close/teardown. */
+    errorFallbackTimer = null;
+    /** Per-CONNECTION: has THIS socket fired onopen? The connect-deadline's late-timer guard (a
+     *  throttled background tab can fire the deadline after onopen already ran and cleared it). */
+    socketOpened = false;
+    /** The pending reseed deadline (CTC-281) — the timer that makes "resyncing" (no socket, reconnect
+     *  suppressed) a bounded state instead of a restart-only wedge. Cleared when the reseed settles in
+     *  time and by stop() (ask 4: stop() leaves NOTHING pending). At most one reseed is ever in flight:
+     *  `resyncing` guards the resync path, and the boot cold seed SETS that same flag for its duration.
+     *  (It used to rely on "the boot seed runs before any socket exists" — true only while a resync
+     *  needed a server frame. The public `requestResync()` added in 0.8.0 needs no socket.) */
+    reseedTimer = null;
     constructor(opts) {
+        // Fail fast, and fail HERE. A token-authed client has no session to fall back to, so an omitted
+        // account is a misconfiguration, not a default. It is checked in the constructor rather than in
+        // buildConnectUrl because `connectUrl()` is called from `openSocket()` OUTSIDE its try/catch — a
+        // throw down there escapes the reconnect machinery entirely instead of surfacing to the caller.
+        if (opts.auth.kind === "token" && !opts.accountId) {
+            throw new Error("LiveSyncClient: accountId is required with token auth (only cookie auth can fall back to the session's own tenant)");
+        }
         this.baseUrl = stripTrailingSlashes(opts.baseUrl);
         this.accountId = opts.accountId;
         this.connectPath = opts.connectPath ?? "/connect";
@@ -167,6 +294,10 @@ export class LiveSyncClient {
         this.maxBackoffMs = opts.maxBackoffMs ?? 30_000;
         this.pingIntervalMs = opts.pingIntervalMs ?? 90_000;
         this.pongTimeoutMs = opts.pongTimeoutMs ?? 15_000;
+        this.openTimeoutMs = opts.openTimeoutMs ?? 20_000;
+        this.reseedTimeoutMs = opts.reseedTimeoutMs ?? 600_000;
+        this.cancelCleanupGraceMs =
+            opts.cancelCleanupGraceMs ?? DEFAULT_CANCEL_CLEANUP_GRACE_MS;
         this.gapTimeoutMs = opts.gapTimeoutMs ?? 10_000;
         this.gapRetryLimit = opts.gapRetryLimit ?? 3;
         this.wsFactory = opts.wsFactory ?? defaultWsFactory;
@@ -182,32 +313,106 @@ export class LiveSyncClient {
      * forever" contract) — the open WebSocket keeps the process alive between deltas. In a browser the
      * returned Promise is simply never awaited; call stop() on teardown.
      */
-    async start() {
+    start() {
         this.stopped = false;
-        // Resolve the OTel seam ONCE up front (before the first reseed, so the seed span exists on the
-        // cold-start path too). Keep the OFF path FULLY SYNCHRONOUS — no `await`, so a caller that opens
-        // the socket and inspects it in the same tick still sees it; only pay the async resolution (guarded
-        // dynamic import, or a CatalystReplica passing its already-resolved instance) when telemetry is on.
-        this.telemetry =
-            this.telemetryConfig === undefined || this.telemetryConfig === false
-                ? NOOP_TELEMETRY
-                : await createTelemetry(this.telemetryConfig, {
-                    tracerName: DEFAULT_SCOPE_NAME,
-                    meterName: DEFAULT_SCOPE_NAME,
-                });
-        this.gapCounter = this.telemetry.counter(REPLICA_METRIC.gaps, {
-            description: "Change-feed seq-gap lifecycle events (detected/healed/escalated).",
-            unit: "{gap}",
-        });
-        const saved = this.getCursor();
-        if (saved == null) {
-            this.setStatus("resyncing");
-            await this.reseed();
-        }
-        this.openSocket();
-        return new Promise((resolve) => {
+        this.started = true;
+        // RESET per boot. `start()` is restartable after `stop()`, and a stale `true` from a previous
+        // cold boot would make the NEXT boot — warm, and therefore re-seeding nothing — absorb a resync
+        // it should have honoured. Found while re-reading this path rather than reported; the same class
+        // of staleness as the `bootTask` handle being nulled when it settles.
+        this.bootColdSeeded = false;
+        this.bootFailed = false;
+        // The done deferred is created BEFORE the boot body runs (CTC-281 N2): stop() during the cold-seed
+        // await used to find resolveDone still null and leave the returned promise pending forever — a
+        // contract violation for a consumer awaiting start(). The boot body below is deliberately its OWN
+        // async task raced against this deferred, because an `async start()` suspended at `await reseed()`
+        // can never reach a `return done` — stop() must be able to resolve the caller regardless of the
+        // boot phase (openSocket() already no-ops on stopped, so a late-settling seed is harmless).
+        const done = new Promise((resolve) => {
             this.resolveDone = resolve;
         });
+        const boot = (async () => {
+            // The WHOLE boot is an in-flight resync, not just the cold seed (CTC-114 review rounds 4 + 6).
+            //
+            // `requestResync()` — public as of 0.8.0 — is callable the moment start() returns its promise,
+            // which is before ANY of this settles. Without the latch, `handleResync()`'s re-entrancy guard
+            // read false and started a SECOND concurrent reseed: two seeds interleaving writes through a
+            // non-reentrant consumer callback, then each completion calling openSocket() — and since
+            // openSocket() overwrites `this.ws`, the first socket was orphaned, still delivering duplicate
+            // frames and unreachable by stop().
+            //
+            // Round 4 latched only the cold seed. That was not enough: `createTelemetry()` below is awaited
+            // BEFORE the seed, so with telemetry enabled the boot suspends in a window where `started` is
+            // already true and the latch is not yet set. The latch therefore has to cover the entire body.
+            //
+            // Until this release the invariant held for free — a resync could only be driven by a server
+            // frame, and a frame needs a socket, which does not exist until openSocket() below.
+            //
+            // The latch makes the request WAIT; whether it is then absorbed or honoured is decided in
+            // requestResync() from `bootColdSeeded`, once this task has settled.
+            //
+            // Round 6 absorbed it on BOTH arms, arguing that a warm boot's `{type:"sync", after:<cursor>}`
+            // is itself the catch-up. That was wrong (round 7), and wrong against this method's whole
+            // reason for existing: a consumer calls requestResync() when it has discovered ON ITS OWN SIDE
+            // that deltas can no longer catch its store up — the browser replica's dropped overflow buffer
+            // is the motivating case. Replaying from the cursor cannot rebuild rows the consumer already
+            // lost, so silently swallowing the request left it permanently inconsistent. Only a COLD boot
+            // may absorb it, because that boot really is a full re-seed from /snapshot.
+            this.resyncing = true;
+            try {
+                // Resolve the OTel seam ONCE up front (before the first reseed, so the seed span exists on the
+                // cold-start path too). Keep the OFF path FULLY SYNCHRONOUS — no `await`, so a caller that opens
+                // the socket and inspects it in the same tick still sees it (the boot body runs synchronously up
+                // to its first await); only pay the async resolution (guarded dynamic import, or a
+                // CatalystReplica passing its already-resolved instance) when telemetry is on.
+                this.telemetry =
+                    this.telemetryConfig === undefined || this.telemetryConfig === false
+                        ? NOOP_TELEMETRY
+                        : await createTelemetry(this.telemetryConfig, {
+                            tracerName: DEFAULT_SCOPE_NAME,
+                            meterName: DEFAULT_SCOPE_NAME,
+                        });
+                this.gapCounter = this.telemetry.counter(REPLICA_METRIC.gaps, {
+                    description: "Change-feed seq-gap lifecycle events (detected/healed/escalated).",
+                    unit: "{gap}",
+                });
+                const saved = this.getCursor();
+                if (saved == null) {
+                    this.setStatus("resyncing");
+                    // Bounded like the resync-path reseed (CTC-281): a hanging COLD seed surfaces as a start()
+                    // rejection (the boot arm rejects) instead of a silent forever-"resyncing" start().
+                    await this.boundedReseed();
+                    // Only NOW may a request that waited on this boot be absorbed — this really was a full
+                    // re-seed from /snapshot. A warm boot sets nothing, so the waiter is honoured instead.
+                    this.bootColdSeeded = true;
+                }
+            }
+            finally {
+                // Must clear on the FAILURE arm too, or a failed boot latches the client into a state where
+                // every later resync — and scheduleReconnect — is suppressed forever.
+                this.resyncing = false;
+            }
+            this.openSocket();
+        })();
+        this.bootTask = boot;
+        // Clear the handle once boot settles, so a resync arriving LONG after startup is never mistaken
+        // for one that raced it — otherwise `bootColdSeeded` would absorb legitimate later requests
+        // forever. The catch keeps a boot rejection from surfacing as an unhandled one on this arm; the
+        // race below is what actually reports it.
+        void boot
+            .catch(() => {
+            // LATCH the failure before the handle is dropped — `bootTask` is the transient record, this is
+            // the durable one, and requestResync() has to be able to see it afterwards (round 9).
+            this.bootFailed = true;
+        })
+            .then(() => {
+            if (this.bootTask === boot)
+                this.bootTask = null;
+        });
+        // Settles when stop() resolves the deferred, OR rejects if the boot (cold seed) fails — a boot
+        // SUCCESS deliberately keeps waiting on `done` (the "runs forever" contract). Promise.race
+        // attaches handlers to both arms, so a boot rejection after stop() is never an unhandled one.
+        return Promise.race([done, boot.then(() => done)]);
     }
     /** Stop the client: close the socket, cancel any pending reconnect, resolve start(). Idempotent. */
     stop() {
@@ -216,11 +421,21 @@ export class LiveSyncClient {
             clearTimeout(this.reconnectTimer);
             this.reconnectTimer = null;
         }
+        // Ask 4 (bounded teardown): the reseed deadline must not hold process exit for up to
+        // reseedTimeoutMs. With it cleared a still-hanging reseed simply never settles its (now
+        // irrelevant) await — every post-await path in handleResync/boot checks `stopped` first.
+        this.clearReseedTimer();
         this.closeSocket();
         this.setStatus("stopped");
         const done = this.resolveDone;
         this.resolveDone = null;
         done?.();
+        // AFTER resolving start()'s deferred, so the race below settles on `done` and a boot arm that
+        // rejects from this abandon lands on an already-settled race rather than surfacing as the
+        // outcome of start(). Settles a `requestResync()` a consumer is awaiting — see boundedReseed.
+        const abandon = this.abandonReseed;
+        this.abandonReseed = null;
+        abandon?.();
     }
     /** The ws(s):// URL this client opens, for diagnostics/tests. Re-derived from the options. */
     connectUrl() {
@@ -231,6 +446,15 @@ export class LiveSyncClient {
             auth: this.auth,
         });
     }
+    /**
+     * The tenant label for telemetry. Attributes are `Record<string, string>`, and an empty string reads
+     * as a MISSING attribute in Loki/Tempo — which silently merges every session-scoped browser client
+     * into one unlabelled bucket. `"session"` names the case instead: this client is scoped to whatever
+     * tenant the cookie resolves to.
+     */
+    get tenantAttr() {
+        return this.accountId ?? "session";
+    }
     setStatus(status) {
         try {
             this.onStatus?.(status);
@@ -243,10 +467,19 @@ export class LiveSyncClient {
         if (this.stopped)
             return;
         this.setStatus("connecting");
+        // RE-CHECK after the status callback (CTC-114 review round 13). `setStatus` calls into consumer
+        // code, and a consumer may synchronously tear down from it — the browser replica's very first
+        // "reconnecting" notification is a documented place to do so. `stop()` then ran while `this.ws`
+        // was still null, so it had nothing to close; we resumed here, constructed a socket, stored it in
+        // an already-stopped client, and left it open processing frames with the teardown long finished.
+        // Every guarded entry point that calls out and then continues needs this; this is the one that
+        // creates a resource afterwards.
+        if (this.stopped)
+            return;
         // One span per connect attempt: started here, ended OK in onopen, ERROR on construct-fail / a close
         // before open. Manual (not active) because the lifecycle spans onopen…onclose callbacks.
         this.connectSpan = this.telemetry.startSpan(REPLICA_SPAN.reconnect, {
-            [CATALYST_ATTR.tenant]: this.accountId,
+            [CATALYST_ATTR.tenant]: this.tenantAttr,
         });
         const wsUrl = this.connectUrl();
         let ws;
@@ -261,7 +494,28 @@ export class LiveSyncClient {
             return;
         }
         this.ws = ws;
+        this.socketOpened = false;
+        this.clearConnectTimers(); // never stack deadlines across attempts (every teardown clears too)
+        // Connect/open deadline (CTC-281): from here until onopen, THIS timer is the client's only
+        // guaranteed pending work (the reconnectTimer that led here was already nulled). If the impl
+        // never fires open/close/error — a stalled upgrade with no FIN/RST, or a buggy injected ws —
+        // this converts the dead attempt into an ordinary backoff retry instead of a permanent
+        // "connecting" wedge.
+        if (this.openTimeoutMs > 0) {
+            this.connectTimer = setTimeout(() => {
+                this.connectTimer = null;
+                // Late-timer guard (same discipline as onPongDeadline): a throttled tab can fire this after
+                // onopen already ran, or after this socket was already replaced/torn down.
+                if (this.stopped || this.ws !== ws || this.socketOpened)
+                    return;
+                this.log("warn", `ws open timed out after ${this.openTimeoutMs}ms; forcing reconnect`);
+                this.endConnectSpan(new Error("open timeout"));
+                this.forceReconnect();
+            }, this.openTimeoutMs);
+        }
         ws.onopen = () => {
+            this.socketOpened = true;
+            this.clearConnectTimers(); // the attempt succeeded — the open deadline + error fallback die here
             this.backoff = this.backoffMs; // a successful open resets the backoff ramp
             this.setStatus("live");
             this.endConnectSpan();
@@ -302,6 +556,19 @@ export class LiveSyncClient {
             catch {
                 // already closing/closed
             }
+            // Fallback (CTC-281): if the impl violates the spec and never follows error with close (undici
+            // #3697/#3546), force the reconnect ourselves after a short grace. One-shot per socket, guarded
+            // on identity — a spec-conforming onclose lands first, clears this timer, and reconnects
+            // normally (scheduleReconnect's reconnectTimer check also prevents any double-schedule).
+            if (this.errorFallbackTimer == null) {
+                this.errorFallbackTimer = setTimeout(() => {
+                    this.errorFallbackTimer = null;
+                    if (this.stopped || this.ws !== ws)
+                        return; // onclose (or a teardown) already handled it
+                    this.log("warn", "ws error was never followed by close; forcing reconnect (CTC-281)");
+                    this.forceReconnect();
+                }, ERROR_CLOSE_GRACE_MS);
+            }
         };
     }
     /** End the in-flight connect span exactly once (idempotent — nulls the handle). */
@@ -333,6 +600,23 @@ export class LiveSyncClient {
         catch {
             // already closed
         }
+        // Escalate past the graceful close (CTC-281): against a half-open peer the Close frame goes into
+        // a black hole, and undici waits on the never-answered handshake with NO timeout — the ref'd TCP
+        // handle then holds a supervised process's exit hostage for up to the OS retransmission timeout
+        // (~minutes). Bun's WebSocket and the node 'ws' package both expose a non-standard `terminate()`
+        // that destroys the connection immediately; duck-type it (structurally, never `as any`) and call
+        // it when present. Handlers are already detached above, so a hard kill is behaviorally safe;
+        // native/undici sockets simply lack the member and keep today's behavior (documented gap — on
+        // Node, inject a 'ws'-package wsFactory if bounded process exit matters).
+        const t = ws;
+        if (typeof t.terminate === "function") {
+            try {
+                t.terminate();
+            }
+            catch {
+                // best-effort — already destroyed
+            }
+        }
     }
     scheduleReconnect() {
         if (this.stopped || this.resyncing || this.reconnectTimer != null)
@@ -364,7 +648,15 @@ export class LiveSyncClient {
             // capability and reset the feature-detect counter; a pong is NEVER surfaced to onFrame/onChange.
             // (lastFrameAt + the pending-deadline clear already happened synchronously in onInboundFrame.)
             this.pongObserved = true;
+            this.pongEverObserved = true; // CTC-281: capability proven for the client's LIFETIME
             this.probeFailures = 0;
+            if (this.watchdogDegraded) {
+                // The degraded slow re-probe just paid off (the server pongs after all — recovered mid-window
+                // or upgraded): re-arm full-speed detection immediately (CTC-281).
+                this.watchdogDegraded = false;
+                this.log("info", "pong observed on a degraded watchdog; full-speed liveness detection re-armed (CTC-281)");
+                this.armPing();
+            }
             return;
         }
         if (frame.type === "head") {
@@ -516,7 +808,7 @@ export class LiveSyncClient {
      *  KEY ON `escalated` ONLY (logged at ERROR); a gap that heals is routine and boring. */
     recordGap(event, gap) {
         this.gapCounter.add(1, {
-            [CATALYST_ATTR.tenant]: this.accountId,
+            [CATALYST_ATTR.tenant]: this.tenantAttr,
             [CATALYST_ATTR.gapEvent]: event,
         });
         this.log(event === "escalated" ? "error" : "info", REPLICA_LOG.gap, {
@@ -539,6 +831,186 @@ export class LiveSyncClient {
         this.gap = null;
         this.clearGapTimer();
     }
+    clearReseedTimer() {
+        if (this.reseedTimer != null) {
+            clearTimeout(this.reseedTimer);
+            this.reseedTimer = null;
+        }
+    }
+    /**
+     * Run the injected reseed() bounded by {@link LiveSyncClientOptions.reseedTimeoutMs} (CTC-281).
+     * The injected callback is a trust boundary like the ws impl: while it runs there is NO socket and
+     * scheduleReconnect is suppressed, so an unbounded await here was the last zero-timer wedge — the
+     * deadline below is the pending timer that upholds the header invariant for the "resyncing" state.
+     * On timeout the attempt is both CANCELLED and abandoned (CTC-114 review round 10). It used to be
+     * abandoned only, justified as "the callback owns its own I/O bounds" — but the callback owning
+     * bounds is exactly what makes a SECOND, independent deadline here dangerous. The browser seed is
+     * bounded by network idleness and by per-RPC worker deadlines, both of which a legitimately slow
+     * ~100 MB snapshot satisfies indefinitely; this total deadline could therefore fire on a seed that
+     * was making honest progress, whereupon the transport reconnected while the callback kept writing.
+     * Frames past the abandoned seed's cursor were then accepted by the socket and discarded by the
+     * replica's paused queue, and a late seed completion left the socket advanced over a hole that
+     * later deltas sealed for good.
+     *
+     * So the deadline now fires an AbortSignal FIRST and rejects second: whoever is told the attempt is
+     * over is also told to stop. A late settle is still discarded via the `settled` latch, and a late
+     * REJECTION is still swallowed so it can never surface as an unhandled rejection.
+     */
+    boundedReseed() {
+        // Cancellation is scoped to THIS attempt. Aborting it must not disturb a successor.
+        const cancel = new AbortController();
+        const seed = this.reseed(cancel.signal);
+        void seed.catch(() => { }); // an abandoned attempt's late rejection must never go unhandled
+        // ALWAYS wrapped, even with the deadline disabled (CTC-114 review round 8). This used to
+        // early-return the raw seed promise when `reseedTimeoutMs <= 0` — the documented way to turn the
+        // deadline off — which skipped installing `abandonReseed` and so bypassed round 7's stop() fix
+        // entirely on that path. Disabling the DEADLINE must not also disable teardown: the two are
+        // independent, and `stop()` has to be able to settle an awaited `requestResync()` either way.
+        return new Promise((resolve, reject) => {
+            let settled = false;
+            let timer = null;
+            const finish = (fn) => {
+                if (settled)
+                    return; // stale settle: the deadline or stop() already took this attempt
+                settled = true;
+                if (timer !== null) {
+                    clearTimeout(timer);
+                    if (this.reseedTimer === timer)
+                        this.reseedTimer = null;
+                }
+                this.abandonReseed = null;
+                fn();
+            };
+            /**
+             * Give up on this attempt: tell it to stop, WAIT for it to unwind, then settle.
+             *
+             * Signalling alone was not enough (CTC-114 review round 12, P1). `abort()` only *initiates* the
+             * consumer's cleanup — the browser seed still has to abort its fetch, let the supersede guard
+             * trip, post its `seedAbort`, and run the `finally` that resumes its delta queue. Settling
+             * immediately let `runResync()` reconnect after one backoff while that queue was still PAUSED,
+             * so arriving frames were counted as delivered and then discarded: exactly the hole the queue's
+             * discard/rollback pairing exists to prevent, re-opened from the other side.
+             *
+             * Bounded, because the whole point of this deadline is that the callback may be unresponsive: a
+             * cleanup that itself hangs must not wedge the transport, so we settle anyway after a grace.
+             */
+            const giveUp = (err) => {
+                if (settled)
+                    return;
+                cancel.abort();
+                // ALWAYS wait for the unwind, bounded by the grace (CTC-114 review round 14).
+                //
+                // Round 12 skipped the wait when `reseed.length === 0`, reasoning that a zero-arg callback
+                // cannot observe the signal. `Function.length` does not support that inference: it counts only
+                // parameters before the first default or rest, so `reseed: (signal = undefined) => …` reports
+                // 0 while receiving and honouring the signal — and that consumer got no wait at all, which is
+                // exactly the hazard the wait exists for. There is no sound way to ask a function whether it
+                // will act on a signal, so stop trying: wait for everyone, and let the grace bound it. A
+                // consumer that ignores the signal simply hits the grace, which is the pre-round-12 behaviour
+                // delayed by `cancelCleanupGraceMs` — the reason that default is small and this is the knob
+                // the browser replica raises.
+                // 0 DISABLES the wait outright rather than scheduling a zero-delay timer — the same
+                // disable-by-zero convention the other bounds use, and the honest meaning of "this consumer
+                // has nothing to unwind".
+                if (this.cancelCleanupGraceMs <= 0) {
+                    finish(() => reject(err));
+                    return;
+                }
+                const grace = setTimeout(() => finish(() => reject(err)), this.cancelCleanupGraceMs);
+                void seed
+                    .catch(() => undefined)
+                    .then(() => {
+                    clearTimeout(grace);
+                    finish(() => reject(err));
+                });
+            };
+            if (this.reseedTimeoutMs > 0) {
+                timer = setTimeout(() => giveUp(new Error(`reseed did not settle within ${this.reseedTimeoutMs}ms; cancelled (CTC-281)`)), this.reseedTimeoutMs);
+                this.reseedTimer = timer;
+            }
+            // stop() settles this wrapper (CTC-114 review round 7). Clearing the deadline is not enough:
+            // if the injected reseed() never settles, nothing else ever settles THIS promise, and while
+            // that was merely "an irrelevant await nobody holds" when boundedReseed was internal, the
+            // public `requestResync()` is now awaited by consumers — so teardown or recovery code holding
+            // that await hung forever on stop(). Rejecting rather than resolving keeps the outcome honest;
+            // requestResync() catches it and still upholds its never-rejects contract.
+            this.abandonReseed = () => giveUp(new Error("client stopped while re-seeding"));
+            seed.then((cursor) => finish(() => resolve(cursor)), (err) => finish(() => reject(err instanceof Error ? err : new Error(String(err)))));
+        });
+    }
+    /**
+     * Ask the client to drop the socket, re-seed, and reconnect from the fresh cursor — the same path a
+     * server `{type:"resync"}` frame drives, exposed for a consumer that discovers ON ITS OWN SIDE that
+     * its store can no longer be caught up by deltas.
+     *
+     * The browser replica is the motivating caller: when its delta queue overflows (or its applies keep
+     * rejecting) the buffered frames are dropped, so the transport's notion of what has been delivered is
+     * now ahead of what the store actually holds. Re-seeding through here — rather than calling the
+     * `reseed` callback directly — is what makes that safe: this CLOSES THE SOCKET FIRST, so no live
+     * frame interleaves with the snapshot and lands in the window that is in neither the snapshot nor the
+     * store.
+     *
+     * Delegates to the existing resync path in full — same `resyncing` re-entrancy guard, same
+     * `boundedReseed` deadline, same span, same failure→backoff behaviour. NEVER rejects: a failed
+     * re-seed is already handled internally by re-entering the reconnect path, and this is called from
+     * event handlers and `void` contexts where a rejection would surface as an unhandled promise.
+     */
+    async requestResync() {
+        if (this.stopped)
+            return;
+        // BEFORE start() there is nothing to resync (CTC-114 review round 5). `stopped` is false on a
+        // never-started client, so it cannot carry this guard by itself. Reseeding here would run with no
+        // lifecycle deferred and no telemetry resolved, and — worse — open a socket that the later real
+        // start() would orphan: openSocket() overwrites `this.ws`, so the first socket keeps delivering
+        // frames and stop() can no longer reach it. Ignore rather than throw: the contract above is that
+        // this never rejects, and it is called from `void` contexts.
+        if (!this.started) {
+            this.log("warn", "requestResync() before start() — ignored");
+            return;
+        }
+        // SERIALIZE behind an in-flight boot (CTC-114 review rounds 6 + 7). Running concurrently with it
+        // meant two reseeds and two openSocket() calls, the second overwriting `this.ws` and orphaning the
+        // first socket. Waiting — rather than dropping — is what keeps the WARM path correct: a warm boot
+        // performs no re-seed, and this method's whole purpose is a consumer that has discovered its store
+        // can no longer be caught up by deltas, which replaying from the cursor cannot fix.
+        const boot = this.bootTask;
+        if (boot) {
+            await boot.catch(() => undefined); // the outcome is read from `bootFailed`, latched below
+            if (this.stopped)
+                return;
+        }
+        // A FAILED boot must not be recovered from here (CTC-114 review rounds 8 + 9). Round 7 swallowed
+        // the rejection and carried straight on into handleResync — so a cold start whose /snapshot failed
+        // would reject the caller's start(), sending the application into its boot-error path, and then a
+        // later successful reseed here would quietly open a live socket underneath it. Round 8 read the
+        // failure from the awaited task, which round 9 showed is not enough: `bootTask` is nulled when the
+        // boot settles, so a request arriving after that microtask saw no failure at all. Checked OUTSIDE
+        // the `if (boot)` for exactly that reason — the latch outlives the handle, until the next start().
+        if (this.bootFailed) {
+            this.log("warn", "requestResync() ignored — startup failed");
+            return;
+        }
+        // A COLD boot re-seeded from /snapshot while we waited, which IS what was being asked for.
+        if (boot && this.bootColdSeeded) {
+            this.log("info", "requestResync() absorbed by the boot's cold seed");
+            return;
+        }
+        // Unlike the frame path, this entry point can be called MID-BACKOFF: the queue overflowed while the
+        // client was already waiting to reconnect. handleResync would then reopen the socket itself and the
+        // pending timer would open a second one on top of it.
+        if (this.reconnectTimer != null) {
+            clearTimeout(this.reconnectTimer);
+            this.reconnectTimer = null;
+        }
+        try {
+            await this.handleResync();
+        }
+        catch (err) {
+            // handleResync already catches its own reseed failure; this is the belt-and-braces guard for the
+            // contract above (never reject) against a throw from a consumer callback it invokes.
+            this.log("error", "requestResync failed", err);
+        }
+    }
     /**
      * Cursor underflow: the deltas we need were evicted from the service's retained change buffer. Close the socket
      * (so no live frame interleaves with the re-seed), re-seed via the injected callback, then reconnect
@@ -546,6 +1018,45 @@ export class LiveSyncClient {
      * frame and suppresses scheduleReconnect for the duration so we reopen exactly once.
      */
     async handleResync() {
+        // RETURN THE RUNNING ONE, do not resolve immediately (CTC-114 review round 10). The re-entrancy
+        // guard used to `return` bare, so a caller awaiting the public `requestResync()` while a
+        // server-driven resync was already re-seeding got a promise that resolved AT ONCE — and then read
+        // the store while the replacement snapshot was still being written into it. Awaiting recovery has
+        // to mean recovery finished, whoever started it.
+        //
+        // This is the same seam as the boot serialization, which covered only `bootTask`: one handle for
+        // "a re-seed is in flight", awaited by everything that needs it to be done.
+        // `return await`, not `return` (CodeQL, round 11). Behaviour is identical today, but returning a
+        // bare promise from an async function means a later `try`/`finally` added around this line would
+        // settle BEFORE the awaited work — a foot-gun this file has been bitten by often enough to be
+        // worth foreclosing, and it keeps this frame in the stack trace when the resync rejects.
+        if (this.activeResync)
+            return await this.activeResync;
+        const run = this.runResync();
+        this.activeResync = run;
+        try {
+            await run;
+        }
+        finally {
+            if (this.activeResync === run)
+                this.activeResync = null;
+        }
+    }
+    /**
+     * The resync body. Never call directly — `handleResync()` owns the in-flight handle.
+     *
+     * TWO FLAGS, ONE FACT — documented rather than consolidated here, deliberately. `resyncing` and
+     * `activeResync` both mean "a re-seed is in flight", and they can disagree: the boot path sets
+     * `resyncing` directly (so a resync cannot race startup) without ever creating an `activeResync`
+     * handle. In that window a call landing here would hit the guard below and resolve silently, which
+     * is the very shape round 10 fixed for the live path.
+     *
+     * It is NOT reachable today, and both reasons are load-bearing: `requestResync()` serialises behind
+     * `bootTask` before it can get here, and the server-frame path needs a socket, which does not exist
+     * until the boot opens one. Consolidating the two into a single handle is the right fix and belongs
+     * with the wider lifecycle rework — not in a release candidate at round eleven.
+     */
+    async runResync() {
         if (this.resyncing)
             return;
         this.resyncing = true;
@@ -554,13 +1065,15 @@ export class LiveSyncClient {
         this.clearGapState();
         this.setStatus("resyncing");
         this.closeSocket();
+        let reseeded = false;
         try {
             // The reseed runs inside an ACTIVE span so the replica's seed span (the injected reseed IS
             // seedFromSnapshot) auto-parents under this resync span.
-            await this.telemetry.withActiveSpan(REPLICA_SPAN.resync, { [CATALYST_ATTR.tenant]: this.accountId }, async () => {
-                const cursor = await this.reseed();
+            await this.telemetry.withActiveSpan(REPLICA_SPAN.resync, { [CATALYST_ATTR.tenant]: this.tenantAttr }, async () => {
+                const cursor = await this.boundedReseed();
                 this.log("info", `resynced, cursor=${cursor}`);
             });
+            reseeded = true;
         }
         catch (err) {
             this.log("error", "resync reseed failed; will retry on reconnect", err);
@@ -568,8 +1081,19 @@ export class LiveSyncClient {
         finally {
             this.resyncing = false;
         }
-        if (!this.stopped)
+        if (this.stopped)
+            return;
+        if (reseeded) {
+            // A completed re-seed reopens immediately — the store is fresh and the endpoint just served us.
             this.openSocket();
+            return;
+        }
+        // A FAILED reseed re-enters the BACKOFF path (CTC-281): the old unconditional reopen made each
+        // gap-escalate → /snapshot-fail → reopen cycle run hot (~30-40s of upgrade + replays + /snapshot
+        // per client, fleet-wide, backoff reset on every open) against exactly the sick server the ticket
+        // covers. scheduleReconnect converges identically once the endpoint recovers — just politely.
+        this.setStatus("reconnecting");
+        this.scheduleReconnect();
     }
     // ── Liveness watchdog (CTC-135) ──
     /** Epoch ms of the last inbound frame (change, pong, or malformed) — null before the first frame.
@@ -594,19 +1118,24 @@ export class LiveSyncClient {
         this.clearPongDeadline();
         this.armPing();
     }
-    /** (Re)arm the idle-ping timer. No-op when the watchdog is disabled/off or there is no live socket,
-     *  so it is safe to call on every frame. A setTimeout chain (not setInterval): each frame resets it. */
+    /** (Re)arm the idle-ping timer. No-op when the watchdog is off or there is no live socket, so it is
+     *  safe to call on every frame. A setTimeout chain (not setInterval): each frame resets it. A
+     *  DEGRADED watchdog still arms — at {@link DEGRADED_PROBE_MULTIPLIER}x the interval — so detection
+     *  is never permanently off (CTC-281): every live socket always has a probe pending. */
     armPing() {
         this.clearPingTimer();
-        if (this.watchdogDisabled || this.pingIntervalMs <= 0 || this.stopped || !this.ws)
+        if (this.pingIntervalMs <= 0 || this.stopped || !this.ws)
             return;
-        this.pingTimer = setTimeout(() => this.sendPing(), this.pingIntervalMs);
+        const interval = this.watchdogDegraded
+            ? this.pingIntervalMs * DEGRADED_PROBE_MULTIPLIER
+            : this.pingIntervalMs;
+        this.pingTimer = setTimeout(() => this.sendPing(), interval);
     }
     /** The feed has been idle for a full interval: send one liveness ping and start the pong deadline. A
      *  synchronous send throw means the socket is already dead — treat it as an unanswered probe now. */
     sendPing() {
         this.pingTimer = null; // this timer just fired
-        if (this.stopped || this.watchdogDisabled || !this.ws)
+        if (this.stopped || !this.ws)
             return;
         this.pingSentAt = Date.now();
         try {
@@ -633,18 +1162,30 @@ export class LiveSyncClient {
         this.onProbeUnanswered();
     }
     /** A ping went unanswered (deadline elapsed or the send threw). If this connection had already proven
-     *  pong capability it is a genuine liveness timeout; otherwise it counts toward the feature-detect —
-     *  after PROBE_FAILURE_LIMIT never-ponged connections the watchdog disables itself for good (an old
-     *  server without auto-pong). Either way, force-reconnect through the existing backoff path. */
+     *  pong capability it is a genuine liveness timeout. If pong capability was proven EARLIER in this
+     *  client's lifetime (CTC-281), a never-ponged connection is the incident signature — a half-open
+     *  socket against a server we KNOW auto-pongs — so it too is a liveness failure and must NEVER count
+     *  toward the feature-detect (during the Jul 17-23 windows, 3 such sockets permanently disabled
+     *  detection). Only while capability is UNPROVEN does the failure count toward the DEGRADE — after
+     *  PROBE_FAILURE_LIMIT never-ponged connections the watchdog backs its probes off to
+     *  DEGRADED_PROBE_MULTIPLIER x pingIntervalMs (an old server without auto-pong costs one bounded
+     *  reconnect per degraded window; a mid-incident restart — per-process latch reset — still detects
+     *  the next half-open within one degraded window, never restart-only; CTC-281). Every path
+     *  force-reconnects through the existing backoff. */
     onProbeUnanswered() {
         if (this.pongObserved) {
             this.log("warn", "liveness timeout: no frame within the pong deadline; reconnecting");
         }
+        else if (this.pongEverObserved) {
+            // A distinct signal from the plain liveness timeout: a PROVEN-pong server delivered zero frames
+            // on a whole connection — the fleet-incident shape (server accepts upgrades, feed is dead).
+            this.log("warn", "liveness timeout on a never-ponged connection against a proven-pong server (half-open or dead feed); reconnecting — watchdog stays armed (CTC-281)");
+        }
         else {
             this.probeFailures += 1;
-            if (this.probeFailures >= PROBE_FAILURE_LIMIT) {
-                this.watchdogDisabled = true;
-                this.log("warn", `liveness watchdog disabled after ${PROBE_FAILURE_LIMIT} unanswered probes (server lacks auto-pong); relying on close/error detection`);
+            if (this.probeFailures >= PROBE_FAILURE_LIMIT && !this.watchdogDegraded) {
+                this.watchdogDegraded = true;
+                this.log("warn", `liveness watchdog degraded after ${PROBE_FAILURE_LIMIT} unanswered probes (server may lack auto-pong); re-probing every ${DEGRADED_PROBE_MULTIPLIER}x pingIntervalMs (CTC-281)`);
             }
         }
         this.forceReconnect();
@@ -676,6 +1217,20 @@ export class LiveSyncClient {
         // A pending gap re-request dies with its socket: the timer must not fire against the next one
         // (whose onopen re-baselines and re-requests from the durable cursor anyway).
         this.clearGapTimer();
+        // The connect deadline + onerror fallback are per-connection too (CTC-281) — they die with the
+        // socket on both teardown routes (closeSocket and the server-close path), same as the pair above.
+        this.clearConnectTimers();
+    }
+    /** Clear the per-connection connect/open deadline + onerror→onclose fallback (CTC-281). */
+    clearConnectTimers() {
+        if (this.connectTimer != null) {
+            clearTimeout(this.connectTimer);
+            this.connectTimer = null;
+        }
+        if (this.errorFallbackTimer != null) {
+            clearTimeout(this.errorFallbackTimer);
+            this.errorFallbackTimer = null;
+        }
     }
 }
 /** Parse a WS frame (string or ArrayBuffer) into a known server frame, or null for anything malformed. */