@abloatai/ablo 0.34.1 → 0.36.0

package/dist/transaction/transport/connectionManager.js

@@ -0,0 +1,673 @@
+/**
+ * Owns the sync engine's connection lifecycle: the state machine that carries a
+ * client from a healthy connection, through a dropout, and back to live. It
+ * watches the browser's online/offline and visibility events, probes real
+ * connectivity and session validity with {@link probeNetwork}, applies retry
+ * backoff with a ceiling and jitter, and runs a watchdog for the cases where
+ * the browser never fires an event (VPN flips, captive portals). When it
+ * decides to recover, it drives the reconnect sequence — bootstrap, then
+ * WebSocket connect — through the {@link ConnectionCallbacks.onReconnect}
+ * callback and reacts to the outcome.
+ *
+ * It deliberately does not perform the bootstrap, local-storage, or object-pool
+ * work itself; that lives in the embedding store, which the manager reaches
+ * only through its callbacks. One manager is created per store, started on the
+ * first successful connect and disposed on teardown.
+ *
+ * This is a plain state machine, not a reactive object: it lives in the
+ * settlement core, which carries no reactivity runtime (ADR 0016). A consumer
+ * that wants to render recovery progress mirrors the transitions through
+ * {@link ConnectionCallbacks.onStateChange} into its own observable state —
+ * which is what the reactive engine's store does.
+ *
+ *   CONNECTED ──(socket drop)──► PROBING_NETWORK ──► RECONNECTING ──► CONNECTED
+ *        │                              │                  │
+ *   (network lost)                      ▼                  ▼
+ *        ▼                        SESSION_EXPIRED       BACKOFF ──► PROBING_NETWORK
+ *     OFFLINE ──(online)──► PROBING_NETWORK
+ *        │
+ *        ▼
+ *   WAITING_FOR_NETWORK
+ *
+ * Three behaviors are worth calling out. A network return or tab focus during a
+ * backoff delay jumps straight to probing instead of waiting out the full
+ * interval. Reaching the retry ceiling while the browser reports offline parks
+ * in `waiting_for_network` rather than hard-reloading a browser that has no
+ * network. And a socket drop (close code 1006) goes straight to probing rather
+ * than the passive `offline` state — 1006 is generated locally and carries no
+ * connectivity signal, so on a healthy machine no online/offline event ever
+ * fires; only a genuine operating-system network loss parks in `offline` to
+ * wait for the `online` event.
+ */
+import { probeNetwork } from './networkProbe.js';
+import { noopLogger } from '../logger.js';
+import { noopSocketObservability, } from '../observability.js';
+// ─── Tunables ─────────────────────────────────────────────────────────────
+const DEFAULT_BACKOFF = {
+    BASE_MS: 2_000,
+    MAX_MS: 30_000,
+    MAX_ATTEMPTS: 8,
+    JITTER: 0.15,
+};
+const ONLINE_DEBOUNCE_MS = 500;
+const WATCHDOG_INTERVAL_MS = 30_000;
+const MAX_STUCK_CYCLES_BEFORE_RELOAD = 6;
+/** Cap on consecutive access-key re-mints before giving up to `auth_blocked`.
+ *  Stops a hot loop if the server keeps reporting the key stale even after a
+ *  "successful" re-mint (clock skew, a mint returning an already-rejected key). */
+const MAX_CREDENTIAL_REFRESH_ATTEMPTS = 3;
+// ─── ConnectionManager ────────────────────────────────────────────────────
+export class ConnectionManager {
+    // Connection state, mirrored to the consumer via `onStateChange`.
+    state = 'connected';
+    offlineSince = null;
+    attempt = 0;
+    lastProbeResult = null;
+    // Private
+    callbacks = null;
+    backoffTimer = null;
+    debounceTimer = null;
+    watchdogTimer = null;
+    stuckCycles = 0;
+    /** Consecutive access-key re-mints in the current recovery cycle; reset on
+     *  reaching `connected`. See {@link MAX_CREDENTIAL_REFRESH_ATTEMPTS}. */
+    credentialRefreshAttempts = 0;
+    disposed = false;
+    baseUrl;
+    getAuthToken;
+    backoff;
+    logger;
+    observability;
+    handleBrowserOnline = null;
+    handleBrowserOffline = null;
+    handleVisibilityChange = null;
+    constructor(options = {}) {
+        this.baseUrl = options.baseUrl;
+        this.getAuthToken = options.getAuthToken;
+        this.backoff = { ...DEFAULT_BACKOFF, ...(options.backoff ?? {}) };
+        this.logger = options.logger ?? noopLogger;
+        this.observability = options.observability ?? noopSocketObservability;
+    }
+    // ── Lifecycle ────────────────────────────────────────────────────────
+    start(callbacks) {
+        this.callbacks = callbacks;
+        this.setupBrowserListeners();
+        this.startWatchdog();
+        this.logger.info('[ConnectionManager] Started');
+    }
+    dispose() {
+        this.disposed = true;
+        this.clearBackoffTimer();
+        this.clearDebounceTimer();
+        if (this.watchdogTimer) {
+            clearInterval(this.watchdogTimer);
+            this.watchdogTimer = null;
+        }
+        this.removeBrowserListeners();
+        this.callbacks = null;
+        this.logger.info('[ConnectionManager] Disposed');
+    }
+    // ── Send events ──────────────────────────────────────────────────────
+    send(event) {
+        if (this.disposed)
+            return;
+        const prevState = this.state;
+        const nextState = this.transition(prevState, event);
+        if (nextState === null)
+            return;
+        this.logger.info('[ConnectionManager] Transition', {
+            from: prevState,
+            to: nextState,
+            event: event.type,
+        });
+        this.state = nextState;
+        if (prevState === 'connected' && nextState !== 'connected') {
+            this.offlineSince = new Date();
+        }
+        if (nextState === 'connected') {
+            this.offlineSince = null;
+            this.attempt = 0;
+            this.stuckCycles = 0;
+        }
+        // Notify the embedding store BEFORE running side effects, so the
+        // UI flips to the new label (e.g. "Reconnecting…") at the same
+        // tick the probe / backoff actually starts. Errors in the consumer
+        // must not break the FSM — wrap defensively.
+        try {
+            this.callbacks?.onStateChange?.(nextState, prevState);
+        }
+        catch (err) {
+            this.logger.debug('[ConnectionManager] onStateChange threw', {
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+        this.onEnterState(nextState, event);
+    }
+    // ── Pure transition ──────────────────────────────────────────────────
+    transition(state, event) {
+        switch (state) {
+            case 'connected':
+                switch (event.type) {
+                    case 'NETWORK_LOST':
+                        // The OS reported the NIC down — park passively in `offline` and
+                        // wait for the `online` event. Probing a downed adapter is wasted
+                        // work.
+                        return 'offline';
+                    case 'WS_DISCONNECTED':
+                        // The socket died (typically code 1006) but the operating-system
+                        // network is almost certainly fine — 1006 is generated locally when
+                        // the TCP connection vanishes and carries no connectivity signal, so
+                        // the browser fires no online/offline event. Probe immediately rather
+                        // than landing in the passive `offline` dead-end (which only escaped
+                        // via the 30s watchdog, long after queued commits rolled back). The
+                        // probe fast-fails if we genuinely are offline → waiting_for_network.
+                        return 'probing_network';
+                    case 'WS_SESSION_ERROR':
+                    case 'BOOTSTRAP_FAILED_SESSION':
+                        return 'session_expired';
+                    case 'WS_HANDSHAKE_FAILED':
+                        return 'probing_network';
+                    case 'TAB_VISIBLE':
+                        return 'probing_network';
+                    default:
+                        return null;
+                }
+            case 'offline':
+                switch (event.type) {
+                    case 'NETWORK_ONLINE':
+                    case 'MANUAL_RETRY':
+                    case 'TAB_VISIBLE':
+                    case 'WS_HANDSHAKE_FAILED':
+                    case 'CREDENTIAL_REFRESHED':
+                        return 'probing_network';
+                    case 'WS_SESSION_ERROR':
+                    case 'BOOTSTRAP_FAILED_SESSION':
+                        return 'session_expired';
+                    default:
+                        return null;
+                }
+            case 'probing_network':
+                switch (event.type) {
+                    case 'PROBE_SUCCESS':
+                        return event.sessionValid ? 'reconnecting' : 'session_expired';
+                    case 'PROBE_CREDENTIAL_STALE':
+                        // Access key expired but the login is fine — re-mint, don't sign out.
+                        return 'refreshing_credential';
+                    case 'PROBE_AUTH_BLOCKED':
+                        return 'auth_blocked';
+                    case 'PROBE_FAILED':
+                        return 'waiting_for_network';
+                    case 'NETWORK_LOST':
+                        return 'offline';
+                    default:
+                        return null;
+                }
+            case 'waiting_for_network':
+                switch (event.type) {
+                    case 'NETWORK_ONLINE':
+                    case 'TAB_VISIBLE':
+                    case 'MANUAL_RETRY':
+                    case 'BACKOFF_ELAPSED':
+                    case 'CREDENTIAL_REFRESHED':
+                        return 'probing_network';
+                    case 'NETWORK_LOST':
+                        return 'offline';
+                    default:
+                        return null;
+                }
+            case 'validating_session':
+                switch (event.type) {
+                    case 'PROBE_SUCCESS':
+                        return event.sessionValid ? 'reconnecting' : 'session_expired';
+                    case 'PROBE_CREDENTIAL_STALE':
+                        return 'refreshing_credential';
+                    case 'PROBE_AUTH_BLOCKED':
+                        return 'auth_blocked';
+                    case 'NETWORK_LOST':
+                        return 'offline';
+                    default:
+                        return null;
+                }
+            case 'refreshing_credential':
+                // Re-minting the short-lived access key (the `ek_`/`rk_`). The login is
+                // presumed valid; this is not a sign-out state.
+                switch (event.type) {
+                    case 'CREDENTIAL_REFRESHED':
+                        // Fresh key in hand — re-probe so we reconnect with it.
+                        return 'probing_network';
+                    case 'BOOTSTRAP_FAILED_SESSION':
+                        // The re-mint hit a genuine 401/403: the long-lived login itself is
+                        // gone. This is the only path from here to sign-out.
+                        return 'session_expired';
+                    case 'RECONNECT_FAILED':
+                        // Couldn't reach the mint endpoint (offline/5xx/throw) — transient.
+                        // Back off and retry; never sign out for a network failure.
+                        return 'backoff';
+                    case 'PROBE_AUTH_BLOCKED':
+                        // Bounded-attempt fallback: the key keeps coming back stale even
+                        // after re-mint (see runRefreshCredential's attempt guard). Stop
+                        // looping without signing out.
+                        return 'auth_blocked';
+                    case 'NETWORK_LOST':
+                        return 'offline';
+                    default:
+                        return null;
+                }
+            case 'reconnecting':
+                switch (event.type) {
+                    case 'RECONNECT_SUCCESS':
+                    case 'WS_CONNECTED':
+                        return 'connected';
+                    case 'RECONNECT_FAILED':
+                    case 'WS_HANDSHAKE_FAILED':
+                        return 'backoff';
+                    case 'BOOTSTRAP_FAILED_SESSION':
+                    case 'WS_SESSION_ERROR':
+                        return 'session_expired';
+                    case 'NETWORK_LOST':
+                        return 'offline';
+                    default:
+                        return null;
+                }
+            case 'backoff':
+                switch (event.type) {
+                    case 'BACKOFF_ELAPSED':
+                    case 'MANUAL_RETRY':
+                    case 'WS_HANDSHAKE_FAILED':
+                        return 'probing_network';
+                    case 'NETWORK_ONLINE':
+                    case 'TAB_VISIBLE':
+                    case 'CREDENTIAL_REFRESHED':
+                        // Network came back (or a fresh credential arrived) while we were
+                        // waiting out a backoff delay — jump straight to probing instead of
+                        // waiting the full exponential interval. Fixes the "doesn't
+                        // retrigger when internet comes back" bug.
+                        return 'probing_network';
+                    case 'NETWORK_LOST':
+                        return 'offline';
+                    case 'WS_SESSION_ERROR':
+                    case 'BOOTSTRAP_FAILED_SESSION':
+                        return 'session_expired';
+                    default:
+                        return null;
+                }
+            case 'auth_blocked':
+                // Reachable, but the data-plane rejected the credential (non-retryable,
+                // non-expiry — e.g. api_key_required, jwt_issuer_untrusted). Don't
+                // auto-reconnect and don't sign out. Allow a manual retry or a
+                // tab-focus / network-return / fresh-credential re-probe (e.g. after a
+                // server deploy or an out-of-band re-mint); a network drop parks
+                // offline; a genuine session error still expires.
+                switch (event.type) {
+                    case 'MANUAL_RETRY':
+                    case 'TAB_VISIBLE':
+                    case 'NETWORK_ONLINE':
+                    case 'CREDENTIAL_REFRESHED':
+                        return 'probing_network';
+                    case 'NETWORK_LOST':
+                        return 'offline';
+                    case 'WS_SESSION_ERROR':
+                    case 'BOOTSTRAP_FAILED_SESSION':
+                        return 'session_expired';
+                    default:
+                        return null;
+                }
+            case 'session_expired':
+                return null; // terminal
+            default:
+                return null;
+        }
+    }
+    // ── Side effects per state ───────────────────────────────────────────
+    onEnterState(state, event) {
+        switch (state) {
+            case 'connected':
+                this.clearBackoffTimer();
+                this.credentialRefreshAttempts = 0;
+                break;
+            case 'offline':
+                this.clearBackoffTimer();
+                this.callbacks?.onDisconnectWebSocket();
+                break;
+            case 'probing_network':
+                // A socket drop (`WS_DISCONNECTED`) lands here directly so recovery
+                // starts immediately. Tear the dead socket down first — that is what
+                // sets the socket's `isManualClose=true` and suppresses its own
+                // reconnect, keeping this machine the single reconnect authority on the
+                // human path. The teardown runs synchronously inside the `disconnected`
+                // emit, before the socket's onclose checks the flag, so the timing
+                // matches the previous `offline`-entry teardown. We gate on the drop
+                // event specifically: the other paths into `probing_network`
+                // (tab-focus re-validation, handshake retry, backoff elapse) must not
+                // tear down a socket that may still be live.
+                if (event.type === 'WS_DISCONNECTED') {
+                    this.callbacks?.onDisconnectWebSocket();
+                }
+                void this.runProbe().catch(this.captureEffectFailure('probe'));
+                break;
+            case 'waiting_for_network':
+                this.scheduleBackoff();
+                break;
+            case 'reconnecting':
+                void this.runReconnect().catch(this.captureEffectFailure('reconnect'));
+                break;
+            case 'refreshing_credential':
+                void this.runRefreshCredential().catch(this.captureEffectFailure('refresh-credential'));
+                break;
+            case 'backoff':
+                this.scheduleBackoff();
+                break;
+            case 'auth_blocked':
+                // Stop here — the server is reachable but rejected the credential (for
+                // example api_key_required or jwt_issuer_untrusted from the data plane).
+                // Neither reconnecting nor re-authenticating fixes it. Drop the socket
+                // and wait for a manual retry or a re-probe. This deliberately does not
+                // call onSessionExpired (no sign-out) and does not reconnect — that is
+                // the whole point of the state.
+                this.clearBackoffTimer();
+                this.callbacks?.onDisconnectWebSocket();
+                this.observability.breadcrumb('Auth blocked — reachable but credential rejected; not reconnecting or signing out', 'sync.offline', 'error');
+                break;
+            case 'session_expired':
+                this.clearBackoffTimer();
+                this.callbacks?.onDisconnectWebSocket();
+                this.callbacks?.onSessionExpired();
+                this.observability.breadcrumb('Session expired — redirecting to signin', 'sync.offline', 'warning');
+                break;
+        }
+    }
+    // ── Async operations ─────────────────────────────────────────────────
+    /**
+     * FSM effects are fire-and-forget by design (`onEnterState` is sync). Each
+     * effect catches its own operational failures and maps them onto an FSM
+     * event, so a rejection here means the machine wedged mid-transition
+     * (`send()` or a transition listener threw) with no retry timer armed past
+     * it — capture it instead of losing the reconnect loop silently.
+     */
+    captureEffectFailure(effect) {
+        return (error) => {
+            this.observability.captureWebSocketError({
+                context: `connection-fsm-${effect}`,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        };
+    }
+    async runProbe() {
+        try {
+            const result = await probeNetwork({
+                baseUrl: this.baseUrl,
+                getAuthToken: this.getAuthToken,
+                logger: this.logger,
+            });
+            this.lastProbeResult = result;
+            // One probe outcome → one event. Exhaustive over ProbeOutcome so a new
+            // outcome can't be silently dropped.
+            switch (result.outcome) {
+                case 'reachable':
+                    this.send({ type: 'PROBE_SUCCESS', sessionValid: true });
+                    break;
+                case 'session_expired':
+                    // Genuine login expiry — terminal. (PROBE_SUCCESS with
+                    // sessionValid:false routes to session_expired in the FSM.)
+                    this.send({ type: 'PROBE_SUCCESS', sessionValid: false });
+                    break;
+                case 'credential_stale':
+                    // Access key expired but the login is fine — re-mint, don't sign out.
+                    this.send({ type: 'PROBE_CREDENTIAL_STALE' });
+                    break;
+                case 'auth_blocked':
+                    this.send({ type: 'PROBE_AUTH_BLOCKED' });
+                    break;
+                case 'unreachable':
+                    this.send({ type: 'PROBE_FAILED' });
+                    break;
+                default: {
+                    const _exhaustive = result.outcome;
+                    void _exhaustive;
+                    this.send({ type: 'PROBE_FAILED' });
+                }
+            }
+        }
+        catch {
+            this.send({ type: 'PROBE_FAILED' });
+        }
+    }
+    /**
+     * Re-mint the short-lived access key on `refreshing_credential`. Delegates to
+     * the `onRefreshCredential` callback (which mints a fresh `ek_`/`rk_` from the
+     * still-valid login and pushes it into the credential source) and maps its
+     * tri-state outcome onto the FSM:
+     *   - `refreshed`     → `CREDENTIAL_REFRESHED` → re-probe & reconnect.
+     *   - `session_error` → `BOOTSTRAP_FAILED_SESSION` → sign out (login is gone).
+     *   - `network_error` → `RECONNECT_FAILED` → back off & retry (never sign out).
+     *
+     * A bounded attempt counter guards against a hot loop where the server keeps
+     * reporting the key stale even after a "successful" re-mint (e.g. a clock skew
+     * or a mint that returns an already-rejected key): after
+     * `MAX_CREDENTIAL_REFRESH_ATTEMPTS` we fall through to `auth_blocked` (stop,
+     * no sign-out) rather than spin. The counter resets once we reach `connected`.
+     *
+     * When no refresher is wired (e.g. a static `apiKey` deployment), we re-probe
+     * directly — the credential source's own scheduler owns refresh there.
+     */
+    async runRefreshCredential() {
+        if (this.credentialRefreshAttempts >= MAX_CREDENTIAL_REFRESH_ATTEMPTS) {
+            this.logger.debug('[ConnectionManager] Access key still stale after repeated re-mints — stopping', { attempts: this.credentialRefreshAttempts });
+            this.credentialRefreshAttempts = 0;
+            this.send({ type: 'PROBE_AUTH_BLOCKED' });
+            return;
+        }
+        this.credentialRefreshAttempts += 1;
+        const refresher = this.callbacks?.onRefreshCredential;
+        if (!refresher) {
+            // No re-mint path wired — re-probe with whatever the credential source
+            // holds (a static-key deployment refreshes out-of-band, if at all).
+            this.send({ type: 'CREDENTIAL_REFRESHED' });
+            return;
+        }
+        try {
+            const result = await refresher();
+            switch (result) {
+                case 'refreshed':
+                    this.send({ type: 'CREDENTIAL_REFRESHED' });
+                    break;
+                case 'session_error':
+                    this.send({ type: 'BOOTSTRAP_FAILED_SESSION' });
+                    break;
+                case 'network_error':
+                    this.send({ type: 'RECONNECT_FAILED' });
+                    break;
+                default: {
+                    const _exhaustive = result;
+                    void _exhaustive;
+                    this.send({ type: 'RECONNECT_FAILED' });
+                }
+            }
+        }
+        catch (error) {
+            // A thrown refresher is transient by contract (offline / mint endpoint
+            // unreachable) — back off and retry, never sign out.
+            this.logger.debug('[ConnectionManager] Credential re-mint threw (transient)', { error });
+            this.send({ type: 'RECONNECT_FAILED' });
+        }
+    }
+    async runReconnect() {
+        if (!this.callbacks)
+            return;
+        try {
+            const result = await this.callbacks.onReconnect();
+            switch (result) {
+                case 'success':
+                    this.send({ type: 'RECONNECT_SUCCESS' });
+                    break;
+                case 'session_error':
+                    this.send({ type: 'BOOTSTRAP_FAILED_SESSION' });
+                    break;
+                case 'network_error':
+                    this.send({ type: 'RECONNECT_FAILED' });
+                    break;
+            }
+        }
+        catch (error) {
+            this.logger.debug('[ConnectionManager] Reconnect threw', { error });
+            this.send({ type: 'RECONNECT_FAILED' });
+        }
+    }
+    // ── Backoff ──────────────────────────────────────────────────────────
+    scheduleBackoff() {
+        this.clearBackoffTimer();
+        if (this.attempt >= this.backoff.MAX_ATTEMPTS) {
+            // If still offline, a hard reload will fail or serve cached and
+            // we'll loop. Park in waiting_for_network and let the `online`
+            // event (or watchdog) restart the cycle when the network comes
+            // back.
+            if (typeof navigator !== 'undefined' && navigator.onLine === false) {
+                this.logger.debug('[ConnectionManager] Max retries while offline — parking until network restored');
+                this.observability.breadcrumb('Max retries — parking offline', 'sync.offline', 'warning');
+                this.attempt = 0;
+                return;
+            }
+            this.logger.debug('[ConnectionManager] Max retries — hard reload');
+            this.observability.breadcrumb('Max retries — hard reload', 'sync.offline', 'error');
+            if (typeof window !== 'undefined') {
+                window.location.reload();
+            }
+            return;
+        }
+        const baseDelay = Math.min(this.backoff.BASE_MS * Math.pow(2, this.attempt), this.backoff.MAX_MS);
+        const jitter = baseDelay * this.backoff.JITTER * (2 * Math.random() - 1);
+        const delay = Math.round(baseDelay + jitter);
+        this.attempt++;
+        this.backoffTimer = setTimeout(() => {
+            this.backoffTimer = null;
+            this.send({ type: 'BACKOFF_ELAPSED' });
+        }, delay);
+    }
+    // ── Browser listeners ────────────────────────────────────────────────
+    setupBrowserListeners() {
+        if (typeof window === 'undefined')
+            return;
+        this.handleBrowserOnline = () => {
+            this.clearDebounceTimer();
+            this.debounceTimer = setTimeout(() => {
+                this.send({ type: 'NETWORK_ONLINE' });
+            }, ONLINE_DEBOUNCE_MS);
+        };
+        this.handleBrowserOffline = () => {
+            this.clearDebounceTimer();
+            this.send({ type: 'NETWORK_LOST' });
+        };
+        this.handleVisibilityChange = () => {
+            if (typeof document !== 'undefined' && document.visibilityState === 'visible') {
+                this.send({ type: 'TAB_VISIBLE' });
+            }
+        };
+        window.addEventListener('online', this.handleBrowserOnline);
+        window.addEventListener('offline', this.handleBrowserOffline);
+        if (typeof document !== 'undefined') {
+            document.addEventListener('visibilitychange', this.handleVisibilityChange);
+        }
+    }
+    removeBrowserListeners() {
+        if (typeof window === 'undefined')
+            return;
+        if (this.handleBrowserOnline) {
+            window.removeEventListener('online', this.handleBrowserOnline);
+        }
+        if (this.handleBrowserOffline) {
+            window.removeEventListener('offline', this.handleBrowserOffline);
+        }
+        if (this.handleVisibilityChange && typeof document !== 'undefined') {
+            document.removeEventListener('visibilitychange', this.handleVisibilityChange);
+        }
+    }
+    // ── Watchdog ─────────────────────────────────────────────────────────
+    startWatchdog() {
+        if (typeof window === 'undefined')
+            return;
+        this.watchdogTimer = setInterval(() => {
+            if (this.disposed)
+                return;
+            // "Stuck" means parked in a non-active recovery state (offline,
+            // waiting_for_network, backoff). This deliberately does not gate on
+            // `navigator.onLine === true`: per MDN, `navigator.onLine` is only
+            // reliable when it returns false ("definitely offline"), and even that
+            // lies after laptop wake or VPN flips. Gating the watchdog on `onLine`
+            // was what stranded the machine in `offline` forever — when the browser
+            // briefly reported offline and never re-fired the `online` event, there
+            // was no escape. The probe itself fast-fails when truly offline, so an
+            // unconditional retry costs nothing in the genuine case.
+            const isStuck = this.state !== 'connected' &&
+                this.state !== 'session_expired' &&
+                this.state !== 'probing_network' &&
+                this.state !== 'refreshing_credential' &&
+                this.state !== 'reconnecting';
+            if (isStuck) {
+                this.stuckCycles++;
+                // Hard-reload gate: only reload when `navigator.onLine` is true.
+                // `onLine === false` is the one direction we trust (MDN: "false
+                // means definitely offline"), and reloading while truly offline
+                // would either serve cached content forever or fail and leave
+                // the user with a broken page.
+                const browserOnline = typeof navigator === 'undefined' || navigator.onLine;
+                if (this.stuckCycles >= MAX_STUCK_CYCLES_BEFORE_RELOAD && browserOnline) {
+                    this.logger.debug('[ConnectionManager] Watchdog: sustained stuck — hard reload');
+                    this.observability.breadcrumb('Watchdog hard reload', 'sync.offline', 'error');
+                    if (typeof window !== 'undefined') {
+                        window.location.reload();
+                    }
+                    return;
+                }
+                this.logger.info('[ConnectionManager] Watchdog: stuck — retry', {
+                    state: this.state,
+                    stuckCycles: this.stuckCycles,
+                    browserOnline,
+                });
+                this.attempt = 0;
+                this.send({ type: 'MANUAL_RETRY' });
+            }
+            else {
+                this.stuckCycles = 0;
+            }
+        }, WATCHDOG_INTERVAL_MS);
+    }
+    // ── UI-friendly views ─────────────────────────────────────────────────
+    get isConnected() { return this.state === 'connected'; }
+    get isOffline() { return this.state === 'offline' || this.state === 'waiting_for_network'; }
+    get isReconnecting() {
+        return (this.state === 'probing_network' ||
+            this.state === 'validating_session' ||
+            this.state === 'refreshing_credential' ||
+            this.state === 'reconnecting' ||
+            this.state === 'backoff');
+    }
+    get isSessionExpired() { return this.state === 'session_expired'; }
+    get offlineDuration() {
+        if (!this.offlineSince)
+            return null;
+        const ms = Date.now() - this.offlineSince.getTime();
+        const seconds = Math.floor(ms / 1000);
+        const minutes = Math.floor(seconds / 60);
+        const hours = Math.floor(minutes / 60);
+        if (seconds < 60)
+            return 'Just now';
+        if (minutes < 60)
+            return `${minutes}m ago`;
+        if (hours < 24)
+            return `${hours}h ago`;
+        return `${Math.floor(hours / 24)}d ago`;
+    }
+    // ── Timer helpers ─────────────────────────────────────────────────────
+    clearBackoffTimer() {
+        if (this.backoffTimer) {
+            clearTimeout(this.backoffTimer);
+            this.backoffTimer = null;
+        }
+    }
+    clearDebounceTimer() {
+        if (this.debounceTimer) {
+            clearTimeout(this.debounceTimer);
+            this.debounceTimer = null;
+        }
+    }
+}