@catalyst-cloud/sdk 0.5.0 → 0.7.0

package/dist/live-sync-client.js

@@ -30,10 +30,53 @@
 // silence it sends the pinned `{"type":"ping"}` frame; the mirror answers via `setWebSocketAutoResponse`
 // (which replies WITHOUT waking a hibernated DO — ADR-0009's cost model is preserved). If no frame
 // arrives within `pongTimeoutMs`, the socket is force-reconnected through the existing backoff path.
-// Traffic postpones pings (no keepalive on a busy stream), and a 3-probe feature-detect disables the
-// watchdog against an old server that never pongs — so it degrades to exactly today's behavior.
+// Traffic postpones pings (no keepalive on a busy stream), and a 3-probe feature-detect DEGRADES the
+// watchdog against an old server that never pongs — re-probing at 10x the interval instead of never
+// (CTC-281: detection can be slowed, but never permanently lost).
+//
+// Gap detection (CTL-1402): the server's live push (`broadcastChange`) is at-most-once — a send into a
+// half-open socket is silently swallowed, and the dropped frame used to be sealed over permanently the
+// moment the next delivered frame advanced the cursor. The change_log itself is durable, so every lost
+// frame is recoverable — the client just never asked again. This client now tracks the contiguity of
+// delivered seqs: a frame arriving at `seq > deliveredSeq + 1` is NOT delivered to `onChange`; instead
+// the client re-requests the hole with the SAME `{type:"sync", after:<deliveredSeq>}` control frame it
+// already sends on every (re)connect, and the mirror replays the missing rows as ordinary change
+// frames through this same path (server support has existed since CTC-63 — `replaySince` is keyset-
+// paginated and answers `{type:"resync"}` on underflow). Bounded: after `gapRetryLimit` no-PROGRESS
+// windows (the heal deadline re-arms on every delivered frame, so a big-but-advancing heal never
+// escalates) the client escalates to the full re-seed path rather than spinning — a gap is never
+// silently accepted.
+//
+// Wedge-proofing (CTC-281): the Jul 17-23 fleet incident (6 windows of server-side half-opens with no
+// FIN/RST; cursors frozen 28-215 min while clients said "live") exposed four restart-only states this
+// client could reach. The invariant now enforced: EVERY state that is not "stopped" holds either a
+// pending timer or a socket whose events re-enter the machine — there is no state only a process
+// restart clears. Concretely: (1) a connect attempt whose ws impl never fires open/close/error is
+// bounded by `openTimeoutMs`; (2) `onerror` without a follow-up `onclose` (real undici bugs #3697/
+// #3546) arms a one-shot fallback reconnect; (3) the watchdog feature-detect can only DEGRADE itself
+// (a slow re-probe every DEGRADED_PROBE_MULTIPLIER x pingIntervalMs), never disable itself outright —
+// and only while pong capability is UNPROVEN; once ANY pong has ever been observed, silence is always
+// treated as a liveness failure (during the incident, 3 open-then-silent sockets used to disable
+// detection for the client's lifetime ~6 min into a window — and because the pong latch is per-process,
+// a client RESTARTED mid-window would have re-latched the disable, so the degrade-not-disable shape is
+// what actually guarantees convergence); (4) a FAILED reseed re-enters the backoff path instead
+// of hot-reopening — and the reseed await itself is bounded by `reseedTimeoutMs` (the injected
+// callback is a trust boundary like the ws impl: the replica's seedFromSnapshot self-bounds via its
+// idle abort, but a consumer-supplied reseed — the browser's OPFS seed() over an unbounded fetch —
+// can hang, and "resyncing" holds no socket and suppresses scheduleReconnect, so without this bound
+// it was the one remaining zero-timer state; a timed-out reseed is ABANDONED, its late settle
+// discarded, and the client re-enters backoff); (5) closeSocket() escalates past `close()` to a duck-typed `terminate()` (Bun /
+// the 'ws' package expose one; undici does not — its close-handshake wait is why teardown must not
+// depend on a graceful close against a half-open peer). Every transition emits the `catalyst.replica.gap` log/counter signal, the
+// detector the per-frame apply telemetry is structurally blind to (an undelivered frame lands in no
+// apply bucket). Gaps are the STEADY-STATE path here — the mirror's reconcile pass appends change_log
+// rows it never broadcasts, so every pass punches a hole that heals via re-request — hence `detected`
+// /`healed` log at INFO and only `escalated` alerts. A reconcile pass with NO webhook frame after it
+// would leave nothing to detect the hole FROM, so the mirror also broadcasts one end-of-pass
+// `{type:"head", seq:<feed head>}` nudge; the client treats a head beyond its baseline exactly like a
+// beyond-gap change frame (re-request the hole `deliveredSeq+1..head`) but never applies it.
 import { PING_FRAME } from "./types.js";
-import { NOOP_TELEMETRY, createTelemetry, CATALYST_ATTR, REPLICA_SPAN, DEFAULT_SCOPE_NAME, } from "./otel.js";
+import { NOOP_TELEMETRY, createTelemetry, CATALYST_ATTR, REPLICA_LOG, REPLICA_METRIC, REPLICA_SPAN, DEFAULT_SCOPE_NAME, } from "./otel.js";
 /** Resolve the runtime global WebSocket, or fail with an actionable message. */
 function defaultWsFactory(url) {
     const Ctor = globalThis.WebSocket;
@@ -67,11 +110,33 @@ export function buildConnectUrl(opts) {
     return `${origin}${opts.connectPath}?${params.toString()}`;
 }
 /**
- * Consecutive opened-then-never-ponged connections after which the watchdog disables itself for the
- * client's lifetime (feature-detect for a server without auto-pong). Bounds worst-case reconnect
- * churn against an old server to exactly this many attempts, making mirror/SDK deploy order harmless.
+ * Consecutive opened-then-never-ponged connections after which the watchdog DEGRADES itself
+ * (feature-detect for a server without auto-pong). Bounds worst-case reconnect churn against an old
+ * server, making mirror/SDK deploy order harmless.
  */
 const PROBE_FAILURE_LIMIT = 3;
+/**
+ * Degraded-watchdog probe interval, as a multiple of `pingIntervalMs` (CTC-281). After
+ * {@link PROBE_FAILURE_LIMIT} never-ponged connections the watchdog does NOT turn off — it re-probes
+ * at this heavily backed-off cadence (stock: every 15 min instead of 90 s). A hard lifetime disable
+ * was the incident's restart-only residual: the `pongEverObserved` latch is per-PROCESS, so a client
+ * (re)started inside an incident window (supervisors restarted processes mid-window) came up
+ * unproven, burned its 3 probes against open-but-silent sockets, and went permanently blind — the
+ * next half-open then froze it as "live" forever, and only another restart (which repeats the cycle)
+ * cleared it. Degrading instead keeps a probe pending in EVERY non-stopped state: a half-open socket
+ * under a degraded watchdog is still detected within ~this multiple of the interval, and the first
+ * pong after recovery re-arms full-speed detection (and latches capability as proven). Against a
+ * genuinely old server the cost is one bounded reconnect per degraded window — churn, never wedge.
+ */
+const DEGRADED_PROBE_MULTIPLIER = 10;
+/**
+ * How long (ms) after `onerror` to wait for the spec-mandated follow-up `onclose` before forcing the
+ * reconnect ourselves (CTC-281). WHATWG requires close-after-error, but real impls have shipped
+ * violations (undici #3697 "close not emitted on error", #3546 "close not fired if the connection
+ * failed to be established") — and `WebSocketLike` is structural, so an injected impl is trusted
+ * blindly. Pre-open, a missing onclose used to be a ZERO-timer permanent-"error" wedge.
+ */
+const ERROR_CLOSE_GRACE_MS = 5_000;
 export class LiveSyncClient {
     baseUrl;
     accountId;
@@ -86,6 +151,10 @@ export class LiveSyncClient {
     maxBackoffMs;
     pingIntervalMs;
     pongTimeoutMs;
+    openTimeoutMs;
+    reseedTimeoutMs;
+    gapTimeoutMs;
+    gapRetryLimit;
     wsFactory;
     log;
     telemetryConfig;
@@ -99,6 +168,21 @@ export class LiveSyncClient {
     telemetry = NOOP_TELEMETRY;
     /** The in-flight connect-attempt span (ended OK on open, ERROR on construct-fail / close-before-open). */
     connectSpan = null;
+    // ── Gap-detection state (CTL-1402) ──
+    /** Transport high-water: the highest seq DELIVERED to onChange — the contiguity baseline. Re-seeded
+     *  from getCursor() on every (re)open (the durable cursor never advanced past an undelivered frame,
+     *  so the on-open `{type:"sync"}` replay covers any previously-pending hole). `-1` = no baseline yet
+     *  (fresh cursor / cursorless store) — gap checks are suspended until a baseline exists. */
+    deliveredSeq = -1;
+    /** The gap currently being re-requested, or null when the stream is contiguous. `seqFrom..seqTo` is
+     *  the detected hole (fixed at detection); `retries` counts the sync re-requests spent on it. */
+    gap = null;
+    gapTimer = null;
+    /** Epoch ms of the last inbound CHANGE frame (live or replayed, delivered or gap-dropped). Unlike
+     *  {@link lastFrameAt} this ignores pongs, so it only moves when the feed actually pushes data. */
+    _lastChangeFrameAt = null;
+    /** The `catalyst.replica.gaps` counter (no-op until telemetry resolves in start()). */
+    gapCounter = NOOP_TELEMETRY.counter(REPLICA_METRIC.gaps);
     // ── Liveness watchdog state (CTC-135) ──
     /** Epoch ms of the last inbound frame (any bytes: change, pong, even malformed). Null before the
      *  first frame. Public via {@link lastFrameAt} — the per-frame timestamp the catalyst daemon's
@@ -111,11 +195,34 @@ export class LiveSyncClient {
     pingSentAt = 0;
     /** Client-lifetime: consecutive opened-then-never-ponged connections. Reset to 0 by ANY pong. */
     probeFailures = 0;
-    /** Client-lifetime: after PROBE_FAILURE_LIMIT never-ponged connections we stop pinging for good
-     *  (an old server without auto-pong), degrading to close/error-only detection. */
-    watchdogDisabled = false;
+    /** Client-lifetime until a pong: after PROBE_FAILURE_LIMIT never-ponged connections the watchdog
+     *  DEGRADES to a {@link DEGRADED_PROBE_MULTIPLIER}x-slower re-probe (an old server without auto-pong)
+     *  — it never turns off outright, so detection is never a restart-only casualty (CTC-281). Only
+     *  reachable while pong capability is UNPROVEN ({@link pongEverObserved}); the first pong clears it. */
+    watchdogDegraded = false;
+    /** Client-lifetime pong latch (CTC-281): has ANY connection EVER answered a ping? Once true, the
+     *  server's auto-pong capability is PROVEN for good — a later never-ponged connection is a liveness
+     *  failure (the incident's open-but-silent socket), never feature-detect evidence, so the watchdog
+     *  can no longer even degrade itself. PER-PROCESS by design — which is exactly why the degrade must
+     *  be soft (see {@link DEGRADED_PROBE_MULTIPLIER}): a restart mid-incident resets this latch. */
+    pongEverObserved = false;
     pingTimer = null;
     pongDeadline = null;
+    /** Per-CONNECTION connect/open deadline (CTC-281): armed when the socket is constructed, cleared on
+     *  open/close/teardown. The only timer pending between openSocket() and onopen — the guarantee that
+     *  a never-firing ws impl cannot leave the client wedged in "connecting" with nothing scheduled. */
+    connectTimer = null;
+    /** Per-CONNECTION onerror→onclose fallback (CTC-281): armed by onerror, fires forceReconnect once
+     *  if the impl never follows error with close (undici #3697/#3546). Cleared on open/close/teardown. */
+    errorFallbackTimer = null;
+    /** Per-CONNECTION: has THIS socket fired onopen? The connect-deadline's late-timer guard (a
+     *  throttled background tab can fire the deadline after onopen already ran and cleared it). */
+    socketOpened = false;
+    /** The pending reseed deadline (CTC-281) — the timer that makes "resyncing" (no socket, reconnect
+     *  suppressed) a bounded state instead of a restart-only wedge. Cleared when the reseed settles in
+     *  time and by stop() (ask 4: stop() leaves NOTHING pending). At most one reseed is ever in flight
+     *  (`resyncing` guards the resync path; the boot seed runs before any socket exists). */
+    reseedTimer = null;
     constructor(opts) {
         this.baseUrl = stripTrailingSlashes(opts.baseUrl);
         this.accountId = opts.accountId;
@@ -130,6 +237,10 @@ export class LiveSyncClient {
         this.maxBackoffMs = opts.maxBackoffMs ?? 30_000;
         this.pingIntervalMs = opts.pingIntervalMs ?? 90_000;
         this.pongTimeoutMs = opts.pongTimeoutMs ?? 15_000;
+        this.openTimeoutMs = opts.openTimeoutMs ?? 20_000;
+        this.reseedTimeoutMs = opts.reseedTimeoutMs ?? 600_000;
+        this.gapTimeoutMs = opts.gapTimeoutMs ?? 10_000;
+        this.gapRetryLimit = opts.gapRetryLimit ?? 3;
         this.wsFactory = opts.wsFactory ?? defaultWsFactory;
         this.log =
             opts.log ??
@@ -143,28 +254,47 @@ export class LiveSyncClient {
      * forever" contract) — the open WebSocket keeps the process alive between deltas. In a browser the
      * returned Promise is simply never awaited; call stop() on teardown.
      */
-    async start() {
+    start() {
         this.stopped = false;
-        // Resolve the OTel seam ONCE up front (before the first reseed, so the seed span exists on the
-        // cold-start path too). Keep the OFF path FULLY SYNCHRONOUS — no `await`, so a caller that opens
-        // the socket and inspects it in the same tick still sees it; only pay the async resolution (guarded
-        // dynamic import, or a CatalystReplica passing its already-resolved instance) when telemetry is on.
-        this.telemetry =
-            this.telemetryConfig === undefined || this.telemetryConfig === false
-                ? NOOP_TELEMETRY
-                : await createTelemetry(this.telemetryConfig, {
-                    tracerName: DEFAULT_SCOPE_NAME,
-                    meterName: DEFAULT_SCOPE_NAME,
-                });
-        const saved = this.getCursor();
-        if (saved == null) {
-            this.setStatus("resyncing");
-            await this.reseed();
-        }
-        this.openSocket();
-        return new Promise((resolve) => {
+        // The done deferred is created BEFORE the boot body runs (CTC-281 N2): stop() during the cold-seed
+        // await used to find resolveDone still null and leave the returned promise pending forever — a
+        // contract violation for a consumer awaiting start(). The boot body below is deliberately its OWN
+        // async task raced against this deferred, because an `async start()` suspended at `await reseed()`
+        // can never reach a `return done` — stop() must be able to resolve the caller regardless of the
+        // boot phase (openSocket() already no-ops on stopped, so a late-settling seed is harmless).
+        const done = new Promise((resolve) => {
             this.resolveDone = resolve;
         });
+        const boot = (async () => {
+            // Resolve the OTel seam ONCE up front (before the first reseed, so the seed span exists on the
+            // cold-start path too). Keep the OFF path FULLY SYNCHRONOUS — no `await`, so a caller that opens
+            // the socket and inspects it in the same tick still sees it (the boot body runs synchronously up
+            // to its first await); only pay the async resolution (guarded dynamic import, or a
+            // CatalystReplica passing its already-resolved instance) when telemetry is on.
+            this.telemetry =
+                this.telemetryConfig === undefined || this.telemetryConfig === false
+                    ? NOOP_TELEMETRY
+                    : await createTelemetry(this.telemetryConfig, {
+                        tracerName: DEFAULT_SCOPE_NAME,
+                        meterName: DEFAULT_SCOPE_NAME,
+                    });
+            this.gapCounter = this.telemetry.counter(REPLICA_METRIC.gaps, {
+                description: "Change-feed seq-gap lifecycle events (detected/healed/escalated).",
+                unit: "{gap}",
+            });
+            const saved = this.getCursor();
+            if (saved == null) {
+                this.setStatus("resyncing");
+                // Bounded like the resync-path reseed (CTC-281): a hanging COLD seed surfaces as a start()
+                // rejection (the boot arm rejects) instead of a silent forever-"resyncing" start().
+                await this.boundedReseed();
+            }
+            this.openSocket();
+        })();
+        // Settles when stop() resolves the deferred, OR rejects if the boot (cold seed) fails — a boot
+        // SUCCESS deliberately keeps waiting on `done` (the "runs forever" contract). Promise.race
+        // attaches handlers to both arms, so a boot rejection after stop() is never an unhandled one.
+        return Promise.race([done, boot.then(() => done)]);
     }
     /** Stop the client: close the socket, cancel any pending reconnect, resolve start(). Idempotent. */
     stop() {
@@ -173,6 +303,10 @@ export class LiveSyncClient {
             clearTimeout(this.reconnectTimer);
             this.reconnectTimer = null;
         }
+        // Ask 4 (bounded teardown): the reseed deadline must not hold process exit for up to
+        // reseedTimeoutMs. With it cleared a still-hanging reseed simply never settles its (now
+        // irrelevant) await — every post-await path in handleResync/boot checks `stopped` first.
+        this.clearReseedTimer();
         this.closeSocket();
         this.setStatus("stopped");
         const done = this.resolveDone;
@@ -218,13 +352,38 @@ export class LiveSyncClient {
             return;
         }
         this.ws = ws;
+        this.socketOpened = false;
+        this.clearConnectTimers(); // never stack deadlines across attempts (every teardown clears too)
+        // Connect/open deadline (CTC-281): from here until onopen, THIS timer is the client's only
+        // guaranteed pending work (the reconnectTimer that led here was already nulled). If the impl
+        // never fires open/close/error — a stalled upgrade with no FIN/RST, or a buggy injected ws —
+        // this converts the dead attempt into an ordinary backoff retry instead of a permanent
+        // "connecting" wedge.
+        if (this.openTimeoutMs > 0) {
+            this.connectTimer = setTimeout(() => {
+                this.connectTimer = null;
+                // Late-timer guard (same discipline as onPongDeadline): a throttled tab can fire this after
+                // onopen already ran, or after this socket was already replaced/torn down.
+                if (this.stopped || this.ws !== ws || this.socketOpened)
+                    return;
+                this.log("warn", `ws open timed out after ${this.openTimeoutMs}ms; forcing reconnect`);
+                this.endConnectSpan(new Error("open timeout"));
+                this.forceReconnect();
+            }, this.openTimeoutMs);
+        }
         ws.onopen = () => {
+            this.socketOpened = true;
+            this.clearConnectTimers(); // the attempt succeeded — the open deadline + error fallback die here
             this.backoff = this.backoffMs; // a successful open resets the backoff ramp
             this.setStatus("live");
             this.endConnectSpan();
             // Fresh connection: reset per-connection watchdog state, then start the idle-ping countdown.
             this.pongObserved = false;
             this.pingSentAt = 0;
+            // Re-baseline gap detection from the durable cursor: it never advanced past an undelivered
+            // frame, so the sync we are about to send re-requests any previously-pending hole anyway.
+            this.clearGapState();
+            this.deliveredSeq = this.getCursor() ?? -1;
             this.sendSync();
             this.armPing();
         };
@@ -255,6 +414,19 @@ export class LiveSyncClient {
             catch {
                 // already closing/closed
             }
+            // Fallback (CTC-281): if the impl violates the spec and never follows error with close (undici
+            // #3697/#3546), force the reconnect ourselves after a short grace. One-shot per socket, guarded
+            // on identity — a spec-conforming onclose lands first, clears this timer, and reconnects
+            // normally (scheduleReconnect's reconnectTimer check also prevents any double-schedule).
+            if (this.errorFallbackTimer == null) {
+                this.errorFallbackTimer = setTimeout(() => {
+                    this.errorFallbackTimer = null;
+                    if (this.stopped || this.ws !== ws)
+                        return; // onclose (or a teardown) already handled it
+                    this.log("warn", "ws error was never followed by close; forcing reconnect (CTC-281)");
+                    this.forceReconnect();
+                }, ERROR_CLOSE_GRACE_MS);
+            }
         };
     }
     /** End the in-flight connect span exactly once (idempotent — nulls the handle). */
@@ -286,6 +458,23 @@ export class LiveSyncClient {
         catch {
             // already closed
         }
+        // Escalate past the graceful close (CTC-281): against a half-open peer the Close frame goes into
+        // a black hole, and undici waits on the never-answered handshake with NO timeout — the ref'd TCP
+        // handle then holds a supervised process's exit hostage for up to the OS retransmission timeout
+        // (~minutes). Bun's WebSocket and the node 'ws' package both expose a non-standard `terminate()`
+        // that destroys the connection immediately; duck-type it (structurally, never `as any`) and call
+        // it when present. Handlers are already detached above, so a hard kill is behaviorally safe;
+        // native/undici sockets simply lack the member and keep today's behavior (documented gap — on
+        // Node, inject a 'ws'-package wsFactory if bounded process exit matters).
+        const t = ws;
+        if (typeof t.terminate === "function") {
+            try {
+                t.terminate();
+            }
+            catch {
+                // best-effort — already destroyed
+            }
+        }
     }
     scheduleReconnect() {
         if (this.stopped || this.resyncing || this.reconnectTimer != null)
@@ -317,7 +506,24 @@ export class LiveSyncClient {
             // capability and reset the feature-detect counter; a pong is NEVER surfaced to onFrame/onChange.
             // (lastFrameAt + the pending-deadline clear already happened synchronously in onInboundFrame.)
             this.pongObserved = true;
+            this.pongEverObserved = true; // CTC-281: capability proven for the client's LIFETIME
             this.probeFailures = 0;
+            if (this.watchdogDegraded) {
+                // The degraded slow re-probe just paid off (the server pongs after all — recovered mid-window
+                // or upgraded): re-arm full-speed detection immediately (CTC-281).
+                this.watchdogDegraded = false;
+                this.log("info", "pong observed on a degraded watchdog; full-speed liveness detection re-armed (CTC-281)");
+                this.armPing();
+            }
+            return;
+        }
+        if (frame.type === "head") {
+            // Transport-internal end-of-pass nudge (CTL-1402): a reconcile pass appended change_log rows it
+            // never broadcast individually, so this head seq may sit beyond our contiguous baseline with no
+            // change frame to detect the hole FROM. Treat that as a gap to re-request — but the head frame
+            // is itself NEVER delivered/applied and advances NO cursor. Not surfaced to onFrame/onChange
+            // (same convention as pong); onInboundFrame already stamped it as liveness bytes.
+            this.onHeadFrame(frame);
             return;
         }
         try {
@@ -330,12 +536,201 @@ export class LiveSyncClient {
             await this.handleResync();
             return;
         }
+        // A change frame: live pushes and `{type:"sync"}` replays arrive through this one path by design.
+        this._lastChangeFrameAt = Date.now();
+        // Gap check (CTL-1402): only with a real baseline (deliveredSeq > 0 — a fresh/cursorless store has
+        // nothing to be contiguous WITH). A frame beyond deliveredSeq+1 means the frames in between were
+        // never delivered — re-request them instead of applying it (the replay will redeliver it in order).
+        if (this.deliveredSeq > 0 && frame.seq > this.deliveredSeq + 1) {
+            this.onGapFrame(frame);
+            return;
+        }
+        // Contiguous — or a duplicate/out-of-order oldie (seq <= deliveredSeq), which is passed through
+        // unchanged: the consumer's stale-guard already dedups it and its cursor never moves backward.
         try {
             this.onChange(frame);
         }
         catch (err) {
             this.log("error", `onChange failed for ${frame.entity} seq=${frame.seq}`, err);
         }
+        if (frame.seq > this.deliveredSeq) {
+            this.deliveredSeq = frame.seq;
+            if (this.gap) {
+                if (this.deliveredSeq >= this.gap.seqTo) {
+                    // The replay walked the whole detected hole: contiguity is restored, live frames resume.
+                    this.onGapHealed();
+                }
+                else {
+                    // Progress WITHIN the hole: refund the heal deadline. It measures "no progress for
+                    // gapTimeoutMs", NOT "not fully healed within gapTimeoutMs" — the steady-state first heal
+                    // replays thousands of frames (keyset-paginated server-side), and a slow consumer / heavy
+                    // first heal must not retry (overlapping replays on the same socket) or escalate to a full
+                    // /snapshot WHILE frames are actively landing. The retry budget still bounds a genuinely
+                    // STALLED heal (no frames at all for a full window).
+                    this.armGapDeadline();
+                }
+            }
+        }
+    }
+    // ── Gap detection + self-healing re-request (CTL-1402) ──
+    /**
+     * A change frame arrived BEYOND the contiguous next seq: every frame in `(deliveredSeq, frame.seq)`
+     * was silently dropped by the at-most-once live push. The frame is NOT applied and the baseline is
+     * NOT advanced past the hole — the mirror's change_log is durable, so the client re-requests the
+     * gap with the same `{type:"sync", after}` control frame it sends on every (re)connect, and the
+     * mirror replays the missing rows as ordinary change frames through the same apply path. While a
+     * re-request is in flight, further beyond-the-gap frames (in-flight live pushes the replay will
+     * cover) are dropped the same way WITHOUT sending another sync — one request per gap episode.
+     */
+    onGapFrame(frame) {
+        if (this.gap)
+            return; // a re-request is already in flight; the replay redelivers this frame too
+        this.gap = { seqFrom: this.deliveredSeq + 1, seqTo: frame.seq - 1, retries: 0 };
+        this.recordGap("detected", this.gap);
+        this.sendGapRequest();
+    }
+    /**
+     * An end-of-pass head nudge (CTL-1402): `{type:"head", seq:N}` reports the mirror's current feed
+     * head after a reconcile pass whose change_log rows were never individually broadcast. If N sits
+     * beyond our contiguous baseline and no gap episode is pending, start one exactly as a beyond-gap
+     * change frame would — the hole is `deliveredSeq+1..N` (N itself is a real, un-broadcast row, so —
+     * unlike a change frame's trigger seq — it IS part of the hole) — and re-request it with the same
+     * bounded retry/escalation machinery and telemetry. The head frame is never applied and never
+     * advances a cursor. A head at/below the baseline is a no-op (already caught up); a head arriving
+     * while a gap is already pending is ignored (the in-flight replay already covers up to the head).
+     */
+    onHeadFrame(frame) {
+        if (this.gap)
+            return; // a re-request is pending; its replay already walks up to (past) this head
+        if (this.deliveredSeq <= 0)
+            return; // no baseline yet — nothing to be contiguous with (see gap check)
+        if (frame.seq <= this.deliveredSeq)
+            return; // caught up (or a stale head): no hole to re-request
+        this.gap = { seqFrom: this.deliveredSeq + 1, seqTo: frame.seq, retries: 0 };
+        this.recordGap("detected", this.gap);
+        this.sendGapRequest();
+    }
+    /** Send (or re-send) the gap re-request from the current baseline and arm the heal deadline. */
+    sendGapRequest() {
+        const req = { type: "sync", after: this.deliveredSeq };
+        try {
+            this.ws?.send(JSON.stringify(req));
+        }
+        catch (err) {
+            // A dead socket: leave the deadline armed — it retries/escalates, and a reconnect re-baselines.
+            this.log("error", "gap re-request send failed", err);
+        }
+        this.armGapDeadline();
+    }
+    /** (Re)arm the gap heal deadline. Armed when a re-request is sent AND refreshed on every frame of
+     *  heal progress (the delivery path), so the deadline measures "no PROGRESS for gapTimeoutMs" rather
+     *  than "not fully healed within gapTimeoutMs". `gapTimeoutMs === 0` disables it (heals only via the
+     *  replay itself or the next reconnect). Only ever called with a gap episode pending. */
+    armGapDeadline() {
+        this.clearGapTimer();
+        if (this.gapTimeoutMs > 0) {
+            this.gapTimer = setTimeout(() => this.onGapTimeout(), this.gapTimeoutMs);
+        }
+    }
+    /** The heal deadline elapsed with the hole still open: retry within budget, else escalate to the
+     *  full re-seed path. Escalation — never silent acceptance — is what keeps a drop from becoming a
+     *  permanent hole one level up. */
+    onGapTimeout() {
+        this.gapTimer = null;
+        const gap = this.gap;
+        if (!gap || this.stopped || !this.ws)
+            return;
+        gap.retries += 1;
+        if (gap.retries >= this.gapRetryLimit) {
+            this.recordGap("escalated", gap);
+            this.gap = null;
+            void this.handleResync();
+            return;
+        }
+        this.sendGapRequest();
+    }
+    /** The replay redelivered the whole detected hole (deliveredSeq reached seqTo contiguously). */
+    onGapHealed() {
+        const gap = this.gap;
+        if (!gap)
+            return;
+        this.gap = null;
+        this.clearGapTimer();
+        this.recordGap("healed", gap);
+    }
+    /** Emit the gap lifecycle signal: a structured `catalyst.replica.gap` log line (the fleet's primary,
+     *  Loki-materialized channel — same convention as `catalyst.replica.apply`) + a `result`-style
+     *  low-cardinality bump of the `catalyst.replica.gaps` counter. seq_from/seq_to/size ride the log
+     *  line as VALUES, never labels. Levels: `detected` and `healed` log at INFO — a gap is the
+     *  steady-state path (every reconcile pass punches one that heals via re-request), so ALERTING MUST
+     *  KEY ON `escalated` ONLY (logged at ERROR); a gap that heals is routine and boring. */
+    recordGap(event, gap) {
+        this.gapCounter.add(1, {
+            [CATALYST_ATTR.tenant]: this.accountId,
+            [CATALYST_ATTR.gapEvent]: event,
+        });
+        this.log(event === "escalated" ? "error" : "info", REPLICA_LOG.gap, {
+            event,
+            seq_from: gap.seqFrom,
+            seq_to: gap.seqTo,
+            size: gap.seqTo - gap.seqFrom + 1,
+            retries: gap.retries,
+        });
+    }
+    clearGapTimer() {
+        if (this.gapTimer != null) {
+            clearTimeout(this.gapTimer);
+            this.gapTimer = null;
+        }
+    }
+    /** Drop all gap state (pending hole + timer) — on (re)open and on the resync/re-seed path, both of
+     *  which re-request/rebuild from the durable cursor and so supersede any pending re-request. */
+    clearGapState() {
+        this.gap = null;
+        this.clearGapTimer();
+    }
+    clearReseedTimer() {
+        if (this.reseedTimer != null) {
+            clearTimeout(this.reseedTimer);
+            this.reseedTimer = null;
+        }
+    }
+    /**
+     * Run the injected reseed() bounded by {@link LiveSyncClientOptions.reseedTimeoutMs} (CTC-281).
+     * The injected callback is a trust boundary like the ws impl: while it runs there is NO socket and
+     * scheduleReconnect is suppressed, so an unbounded await here was the last zero-timer wedge — the
+     * deadline below is the pending timer that upholds the header invariant for the "resyncing" state.
+     * On timeout the attempt is ABANDONED, not cancelled (the callback owns its own I/O bounds — the
+     * replica's seedFromSnapshot aborts its fetch itself): a late settle is discarded via the `settled`
+     * latch, and a late REJECTION is swallowed so it can never surface as an unhandled rejection.
+     */
+    boundedReseed() {
+        const seed = this.reseed();
+        if (this.reseedTimeoutMs <= 0)
+            return seed;
+        void seed.catch(() => { }); // an abandoned attempt's late rejection must never go unhandled
+        return new Promise((resolve, reject) => {
+            let settled = false;
+            const timer = setTimeout(() => {
+                if (settled)
+                    return;
+                settled = true;
+                if (this.reseedTimer === timer)
+                    this.reseedTimer = null;
+                reject(new Error(`reseed did not settle within ${this.reseedTimeoutMs}ms; abandoning (CTC-281)`));
+            }, this.reseedTimeoutMs);
+            this.reseedTimer = timer;
+            const finish = (fn) => {
+                if (settled)
+                    return; // stale settle: the deadline already abandoned this attempt
+                settled = true;
+                clearTimeout(timer);
+                if (this.reseedTimer === timer)
+                    this.reseedTimer = null;
+                fn();
+            };
+            seed.then((cursor) => finish(() => resolve(cursor)), (err) => finish(() => reject(err instanceof Error ? err : new Error(String(err)))));
+        });
     }
     /**
      * Cursor underflow: the deltas we need were evicted from the service's retained change buffer. Close the socket
@@ -347,15 +742,20 @@ export class LiveSyncClient {
         if (this.resyncing)
             return;
         this.resyncing = true;
+        // A full re-seed supersedes any pending gap re-request (it rebuilds from /snapshot wholesale) —
+        // this also covers the server answering a gap re-request with {type:"resync"} (window eviction).
+        this.clearGapState();
         this.setStatus("resyncing");
         this.closeSocket();
+        let reseeded = false;
         try {
             // The reseed runs inside an ACTIVE span so the replica's seed span (the injected reseed IS
             // seedFromSnapshot) auto-parents under this resync span.
             await this.telemetry.withActiveSpan(REPLICA_SPAN.resync, { [CATALYST_ATTR.tenant]: this.accountId }, async () => {
-                const cursor = await this.reseed();
+                const cursor = await this.boundedReseed();
                 this.log("info", `resynced, cursor=${cursor}`);
             });
+            reseeded = true;
         }
         catch (err) {
             this.log("error", "resync reseed failed; will retry on reconnect", err);
@@ -363,8 +763,19 @@ export class LiveSyncClient {
         finally {
             this.resyncing = false;
         }
-        if (!this.stopped)
+        if (this.stopped)
+            return;
+        if (reseeded) {
+            // A completed re-seed reopens immediately — the store is fresh and the endpoint just served us.
             this.openSocket();
+            return;
+        }
+        // A FAILED reseed re-enters the BACKOFF path (CTC-281): the old unconditional reopen made each
+        // gap-escalate → /snapshot-fail → reopen cycle run hot (~30-40s of upgrade + replays + /snapshot
+        // per client, fleet-wide, backoff reset on every open) against exactly the sick server the ticket
+        // covers. scheduleReconnect converges identically once the endpoint recovers — just politely.
+        this.setStatus("reconnecting");
+        this.scheduleReconnect();
     }
     // ── Liveness watchdog (CTC-135) ──
     /** Epoch ms of the last inbound frame (change, pong, or malformed) — null before the first frame.
@@ -373,6 +784,15 @@ export class LiveSyncClient {
     get lastFrameAt() {
         return this._lastFrameAt;
     }
+    /** Epoch ms of the last inbound CHANGE frame (live or replayed; delivered OR gap-dropped), or null
+     *  before the first. Unlike {@link lastFrameAt} — which any bytes stamp, INCLUDING the mirror's
+     *  watchdog auto-pongs — this only moves when the feed actually pushes data. Pairing the two lets a
+     *  stall supervisor separate the three states CTL-1402 conflated: dead socket (both stale),
+     *  healthy-but-unpushed socket (lastFrameAt fresh via pongs, lastChangeFrameAt stale while the
+     *  mirror head advances), and a genuinely quiet feed (same signature, distinguished server-side). */
+    get lastChangeFrameAt() {
+        return this._lastChangeFrameAt;
+    }
     /** Any inbound frame: stamp liveness, cancel a pending pong deadline (it was answered), and postpone
      *  the next ping so a busy stream never sends one. Runs synchronously in onmessage before parsing. */
     onInboundFrame() {
@@ -380,19 +800,24 @@ export class LiveSyncClient {
         this.clearPongDeadline();
         this.armPing();
     }
-    /** (Re)arm the idle-ping timer. No-op when the watchdog is disabled/off or there is no live socket,
-     *  so it is safe to call on every frame. A setTimeout chain (not setInterval): each frame resets it. */
+    /** (Re)arm the idle-ping timer. No-op when the watchdog is off or there is no live socket, so it is
+     *  safe to call on every frame. A setTimeout chain (not setInterval): each frame resets it. A
+     *  DEGRADED watchdog still arms — at {@link DEGRADED_PROBE_MULTIPLIER}x the interval — so detection
+     *  is never permanently off (CTC-281): every live socket always has a probe pending. */
     armPing() {
         this.clearPingTimer();
-        if (this.watchdogDisabled || this.pingIntervalMs <= 0 || this.stopped || !this.ws)
+        if (this.pingIntervalMs <= 0 || this.stopped || !this.ws)
             return;
-        this.pingTimer = setTimeout(() => this.sendPing(), this.pingIntervalMs);
+        const interval = this.watchdogDegraded
+            ? this.pingIntervalMs * DEGRADED_PROBE_MULTIPLIER
+            : this.pingIntervalMs;
+        this.pingTimer = setTimeout(() => this.sendPing(), interval);
     }
     /** The feed has been idle for a full interval: send one liveness ping and start the pong deadline. A
      *  synchronous send throw means the socket is already dead — treat it as an unanswered probe now. */
     sendPing() {
         this.pingTimer = null; // this timer just fired
-        if (this.stopped || this.watchdogDisabled || !this.ws)
+        if (this.stopped || !this.ws)
             return;
         this.pingSentAt = Date.now();
         try {
@@ -419,18 +844,30 @@ export class LiveSyncClient {
         this.onProbeUnanswered();
     }
     /** A ping went unanswered (deadline elapsed or the send threw). If this connection had already proven
-     *  pong capability it is a genuine liveness timeout; otherwise it counts toward the feature-detect —
-     *  after PROBE_FAILURE_LIMIT never-ponged connections the watchdog disables itself for good (an old
-     *  server without auto-pong). Either way, force-reconnect through the existing backoff path. */
+     *  pong capability it is a genuine liveness timeout. If pong capability was proven EARLIER in this
+     *  client's lifetime (CTC-281), a never-ponged connection is the incident signature — a half-open
+     *  socket against a server we KNOW auto-pongs — so it too is a liveness failure and must NEVER count
+     *  toward the feature-detect (during the Jul 17-23 windows, 3 such sockets permanently disabled
+     *  detection). Only while capability is UNPROVEN does the failure count toward the DEGRADE — after
+     *  PROBE_FAILURE_LIMIT never-ponged connections the watchdog backs its probes off to
+     *  DEGRADED_PROBE_MULTIPLIER x pingIntervalMs (an old server without auto-pong costs one bounded
+     *  reconnect per degraded window; a mid-incident restart — per-process latch reset — still detects
+     *  the next half-open within one degraded window, never restart-only; CTC-281). Every path
+     *  force-reconnects through the existing backoff. */
     onProbeUnanswered() {
         if (this.pongObserved) {
             this.log("warn", "liveness timeout: no frame within the pong deadline; reconnecting");
         }
+        else if (this.pongEverObserved) {
+            // A distinct signal from the plain liveness timeout: a PROVEN-pong server delivered zero frames
+            // on a whole connection — the fleet-incident shape (server accepts upgrades, feed is dead).
+            this.log("warn", "liveness timeout on a never-ponged connection against a proven-pong server (half-open or dead feed); reconnecting — watchdog stays armed (CTC-281)");
+        }
         else {
             this.probeFailures += 1;
-            if (this.probeFailures >= PROBE_FAILURE_LIMIT) {
-                this.watchdogDisabled = true;
-                this.log("warn", `liveness watchdog disabled after ${PROBE_FAILURE_LIMIT} unanswered probes (server lacks auto-pong); relying on close/error detection`);
+            if (this.probeFailures >= PROBE_FAILURE_LIMIT && !this.watchdogDegraded) {
+                this.watchdogDegraded = true;
+                this.log("warn", `liveness watchdog degraded after ${PROBE_FAILURE_LIMIT} unanswered probes (server may lack auto-pong); re-probing every ${DEGRADED_PROBE_MULTIPLIER}x pingIntervalMs (CTC-281)`);
             }
         }
         this.forceReconnect();
@@ -459,6 +896,23 @@ export class LiveSyncClient {
     clearLivenessTimers() {
         this.clearPingTimer();
         this.clearPongDeadline();
+        // A pending gap re-request dies with its socket: the timer must not fire against the next one
+        // (whose onopen re-baselines and re-requests from the durable cursor anyway).
+        this.clearGapTimer();
+        // The connect deadline + onerror fallback are per-connection too (CTC-281) — they die with the
+        // socket on both teardown routes (closeSocket and the server-close path), same as the pair above.
+        this.clearConnectTimers();
+    }
+    /** Clear the per-connection connect/open deadline + onerror→onclose fallback (CTC-281). */
+    clearConnectTimers() {
+        if (this.connectTimer != null) {
+            clearTimeout(this.connectTimer);
+            this.connectTimer = null;
+        }
+        if (this.errorFallbackTimer != null) {
+            clearTimeout(this.errorFallbackTimer);
+            this.errorFallbackTimer = null;
+        }
     }
 }
 /** Parse a WS frame (string or ArrayBuffer) into a known server frame, or null for anything malformed. */
@@ -486,6 +940,8 @@ export function parseFrame(data) {
         return parsed;
     if (type === "pong")
         return parsed;
+    if (type === "head")
+        return parsed;
     return null;
 }
 //# sourceMappingURL=live-sync-client.js.map