@hienlh/ppm 0.18.13 → 0.18.15

package/src/web/components/remote-desktop/remote-desktop-touch-gesture-tracker.ts

@@ -0,0 +1,195 @@
+/**
+ * Pure(-ish) per-gesture accumulators for `use-remote-desktop-touch.ts` — kept out of the hook
+ * so the DOM-wiring there stays readable and these decision rules live in one place. Each
+ * `track` object is created on touchstart and mutated by the `advance*`/`resolve*` functions as
+ * touchmove/touchend events arrive; nothing here touches the DOM or sends messages directly,
+ * the hook does that with whatever these functions return.
+ */
+import { classifyTwoFingerGesture, isTapGesture, touchDistance, touchMidpoint, type PointerSample } from "./remote-desktop-gesture-classifiers";
+import { fractionFromZoomedPoint, type ZoomPanTransform, type Fraction, type Rect } from "./remote-desktop-coords";
+
+export type RemoteDesktopInputMode = "touch" | "mouse";
+
+export const TAP_MOVE_THRESHOLD_PX = 10;
+export const TAP_MAX_DURATION_MS = 300;
+export const TWO_FINGER_THRESHOLD_PX = 15;
+export const SCALE_MIN = 1;
+export const SCALE_MAX = 4;
+
+function clamp(n: number, min: number, max: number): number {
+  return Math.min(Math.max(n, min), max);
+}
+
+export function clickMessages(xFrac: number, yFrac: number, button: "left" | "right"): Record<string, unknown>[] {
+  return [
+    { type: "pointer", xFrac, yFrac, button, down: true },
+    { type: "pointer", xFrac, yFrac, button, down: false },
+  ];
+}
+
+/* ------------------------------------------------------------------ */
+/*  Single-finger tracking                                             */
+/* ------------------------------------------------------------------ */
+
+export interface SingleTrack {
+  start: PointerSample;
+  last: { x: number; y: number };
+  moved: boolean;
+  /** The displayed video's rect (post letterbox) at gesture start — cached because it does not
+   *  change mid-drag. */
+  rect: Rect;
+}
+
+export function beginSingleTouch(x: number, y: number, rect: Rect): SingleTrack {
+  return { start: { x, y, t: performance.now() }, last: { x, y }, moved: false, rect };
+}
+
+export interface SingleMoveResult {
+  /** Pointer message to send immediately (touch mode: drag-start once, then drag-move). */
+  message?: Record<string, unknown>;
+  /** New virtual-cursor fraction (mouse mode only). */
+  cursor?: Fraction;
+  sendCursorMove?: boolean;
+}
+
+export function advanceSingleTouch(
+  track: SingleTrack,
+  x: number,
+  y: number,
+  mode: RemoteDesktopInputMode,
+  transform: ZoomPanTransform,
+  cursor: Fraction,
+): SingleMoveResult {
+  const dist = Math.hypot(x - track.start.x, y - track.start.y);
+  if (mode === "touch") {
+    if (!track.moved && dist > TAP_MOVE_THRESHOLD_PX) {
+      track.moved = true;
+      const start = fractionFromZoomedPoint(track.start.x, track.start.y, track.rect, transform);
+      track.last = { x, y };
+      return { message: { type: "pointer", xFrac: start.xFrac, yFrac: start.yFrac, button: "left", down: true } };
+    }
+    track.last = { x, y };
+    if (!track.moved) return {};
+    const frac = fractionFromZoomedPoint(x, y, track.rect, transform);
+    return { message: { type: "pointer", xFrac: frac.xFrac, yFrac: frac.yFrac, button: null, down: null } };
+  }
+
+  // Mouse mode: relative drag of the client-side virtual cursor, scaled down by zoom so panning
+  // while zoomed in still moves the cursor the same *host* distance.
+  const dx = (x - track.last.x) / transform.scale;
+  const dy = (y - track.last.y) / transform.scale;
+  if (dist > TAP_MOVE_THRESHOLD_PX) track.moved = true;
+  track.last = { x, y };
+  const next: Fraction = {
+    xFrac: clamp(cursor.xFrac + dx / track.rect.width, 0, 1),
+    yFrac: clamp(cursor.yFrac + dy / track.rect.height, 0, 1),
+  };
+  return { cursor: next, sendCursorMove: true };
+}
+
+export function resolveSingleTouchEnd(
+  track: SingleTrack,
+  x: number,
+  y: number,
+  mode: RemoteDesktopInputMode,
+  transform: ZoomPanTransform,
+  cursor: Fraction,
+): Record<string, unknown>[] {
+  const end: PointerSample = { x, y, t: performance.now() };
+  const tapped = isTapGesture(track.start, end, TAP_MOVE_THRESHOLD_PX, TAP_MAX_DURATION_MS);
+  if (mode === "touch") {
+    const frac = fractionFromZoomedPoint(x, y, track.rect, transform);
+    if (!track.moved && tapped) return clickMessages(frac.xFrac, frac.yFrac, "left");
+    if (track.moved) return [{ type: "pointer", xFrac: frac.xFrac, yFrac: frac.yFrac, button: "left", down: false }];
+    return [];
+  }
+  if (!track.moved && tapped) return clickMessages(cursor.xFrac, cursor.yFrac, "left");
+  return [];
+}
+
+/* ------------------------------------------------------------------ */
+/*  Two-finger tracking                                                 */
+/* ------------------------------------------------------------------ */
+
+export interface TwoFingerTrack {
+  startDistance: number;
+  startMid: { x: number; y: number };
+  lastMid: { x: number; y: number };
+  startScale: number;
+  startTime: number;
+  gesture: "pinch" | "scroll" | null;
+}
+
+export function beginTwoFingerTouch(
+  pa: { x: number; y: number },
+  pb: { x: number; y: number },
+  currentScale: number,
+): TwoFingerTrack {
+  const mid = touchMidpoint(pa, pb);
+  return {
+    startDistance: touchDistance(pa, pb),
+    startMid: mid,
+    lastMid: mid,
+    startScale: currentScale,
+    startTime: performance.now(),
+    gesture: null,
+  };
+}
+
+export interface TwoFingerMoveResult {
+  transform?: ZoomPanTransform;
+  wheelDeltaPx?: number;
+}
+
+/** Locks in "pinch" vs "scroll" the first time either signal (finger spread, or midpoint
+ *  translation) crosses `TWO_FINGER_THRESHOLD_PX`, then keeps applying that gesture for the
+ *  rest of the touch — see `remote-desktop-gesture-classifiers.ts` for why. */
+export function advanceTwoFingerTouch(
+  track: TwoFingerTrack,
+  pa: { x: number; y: number },
+  pb: { x: number; y: number },
+  currentTransform: ZoomPanTransform,
+): TwoFingerMoveResult {
+  const currentDistance = touchDistance(pa, pb);
+  const currentMid = touchMidpoint(pa, pb);
+
+  if (track.gesture === null) {
+    const distDiff = Math.abs(currentDistance - track.startDistance);
+    const midDist = touchDistance(track.startMid, currentMid);
+    if (distDiff > TWO_FINGER_THRESHOLD_PX) {
+      track.gesture = classifyTwoFingerGesture(track.startDistance, currentDistance, TWO_FINGER_THRESHOLD_PX);
+    } else if (midDist > TWO_FINGER_THRESHOLD_PX) {
+      track.gesture = "scroll";
+    } else {
+      track.lastMid = currentMid;
+      return {}; // still ambiguous — wait for more movement before committing
+    }
+  }
+
+  let result: TwoFingerMoveResult;
+  if (track.gesture === "pinch") {
+    const scale = clamp(track.startScale * (currentDistance / track.startDistance), SCALE_MIN, SCALE_MAX);
+    result = { transform: { scale, panX: currentTransform.panX, panY: currentTransform.panY } };
+  } else if (currentTransform.scale > 1.01) {
+    // Panning only makes sense once zoomed in — at scale 1 the same drag scrolls instead.
+    const dx = currentMid.x - track.lastMid.x;
+    const dy = currentMid.y - track.lastMid.y;
+    result = { transform: { scale: currentTransform.scale, panX: currentTransform.panX + dx, panY: currentTransform.panY + dy } };
+  } else {
+    result = { wheelDeltaPx: currentMid.y - track.lastMid.y };
+  }
+  track.lastMid = currentMid;
+  return result;
+}
+
+export function resolveTwoFingerEnd(
+  track: TwoFingerTrack,
+  rect: Rect,
+  transform: ZoomPanTransform,
+): Record<string, unknown>[] {
+  if (track.gesture === null && performance.now() - track.startTime <= TAP_MAX_DURATION_MS) {
+    const frac = fractionFromZoomedPoint(track.lastMid.x, track.lastMid.y, rect, transform);
+    return clickMessages(frac.xFrac, frac.yFrac, "right");
+  }
+  return [];
+}

package/src/web/components/remote-desktop/remote-desktop-window-content.tsx

@@ -0,0 +1,72 @@
+/**
+ * Remote Desktop window body: thin wrapper around `useRemoteDesktopConnection` (WS/nonce/ping/
+ * decode, shared with the mobile viewer) that renders the canvas, overlay states, and wires
+ * pointer/keyboard capture over the connection's `sendMessage`.
+ */
+import { useRef } from "react";
+import { RotateCw, MonitorX, Gauge } from "lucide-react";
+import type { WindowContentProps } from "@/components/floating-window/window-content-registry";
+import { useSettingsStore } from "@/stores/settings-store";
+import { cn } from "@/lib/utils";
+import { useRemoteDesktopConnection } from "./use-remote-desktop-connection";
+import { useRemoteInputCapture } from "./use-remote-input-capture";
+import { RemoteDesktopStatsOverlay } from "./remote-desktop-stats-overlay";
+
+export default function RemoteDesktopWindowContent(_props: WindowContentProps) {
+  const canvasRef = useRef<HTMLCanvasElement | null>(null);
+  const { connState, errorMessage, decoderStatus, decoderErrorMessage, sendMessage, reconnect, getFrameCount, getTotalBytes } =
+    useRemoteDesktopConnection(canvasRef);
+  const statsVisible = useSettingsStore((s) => s.remoteDesktopStatsVisible);
+  const toggleStats = useSettingsStore((s) => s.toggleRemoteDesktopStatsVisible);
+
+  useRemoteInputCapture(canvasRef, sendMessage, connState === "streaming");
+
+  const overlayMessage = decoderStatus === "unsupported"
+    ? decoderErrorMessage
+    : connState === "error"
+      ? errorMessage
+      : connState === "closed"
+        ? "Disconnected"
+        : connState === "connecting"
+          ? "Connecting…"
+          : null;
+
+  return (
+    <div
+      className="relative flex h-full w-full items-center justify-center bg-black"
+      data-testid="remote-desktop-window"
+      data-conn-state={connState}
+    >
+      <canvas ref={canvasRef} data-testid="remote-desktop-canvas" className="max-h-full max-w-full outline-none" />
+      <RemoteDesktopStatsOverlay canvasRef={canvasRef} getFrameCount={getFrameCount} getTotalBytes={getTotalBytes} />
+      {/* Small corner toggle for the stats overlay above — top-right so it never collides with
+          the overlay itself (top-left) or the window's own title bar/controls above this body. */}
+      <button
+        type="button"
+        onClick={toggleStats}
+        aria-label="Toggle stats overlay"
+        aria-pressed={statsVisible}
+        className={cn(
+          "absolute right-1 top-1 z-40 flex size-6 items-center justify-center rounded bg-black/40 text-white/70 hover:bg-black/60 hover:text-white",
+          statsVisible && "bg-primary/70 text-primary-foreground hover:bg-primary/80",
+        )}
+      >
+        <Gauge className="size-3.5" />
+      </button>
+      {overlayMessage && (
+        <div className="absolute inset-0 flex flex-col items-center justify-center gap-3 bg-black/70 text-sm text-white">
+          <MonitorX className="size-6 opacity-70" />
+          <span className="max-w-sm text-center px-4">{overlayMessage}</span>
+          {(connState === "error" || connState === "closed") && decoderStatus !== "unsupported" && (
+            <button
+              onClick={reconnect}
+              className="flex items-center gap-1.5 rounded-md bg-white/10 px-3 py-1.5 hover:bg-white/20"
+            >
+              <RotateCw className="size-3.5" /> Reconnect
+            </button>
+          )}
+        </div>
+      )}
+    </div>
+  );
+}

package/src/web/components/remote-desktop/remote-desktop-ws-url.ts

@@ -0,0 +1,22 @@
+/** Pure URL builder — mirrors `ws-client.ts`'s dev-port bypass so it's unit-testable without
+ *  mounting a real WebSocket or window.location. */
+import { withWsAuth } from "@/lib/ws-auth";
+
+export interface LocationParts {
+  protocol: string;
+  hostname: string;
+  host: string;
+}
+
+export function resolveRemoteDesktopWsUrl(loc: LocationParts, isDev: boolean, devPort = "8081"): string {
+  const authed = withWsAuth("/ws/remote-desktop");
+  const wsProtocol = loc.protocol === "https:" ? "wss:" : "ws:";
+  if (isDev && loc.protocol !== "https:") {
+    // Same bypass as ws-client.ts: Vite's dev proxy has unreliable WS upgrade handling.
+    // Port defaults to 8081 (the normal `bun dev:server` port) but is overridable via
+    // `VITE_DEV_API_PORT` for an alt-port dev stack (see `PPM_DEV_API` in vite.config.ts) —
+    // without this, a second dev stack on e.g. 8082 would always connect to the wrong server.
+    return `ws://${loc.hostname}:${devPort}${authed}`;
+  }
+  return `${wsProtocol}//${loc.host}${authed}`;
+}

package/src/web/components/remote-desktop/use-h264-canvas-decoder.ts

@@ -0,0 +1,175 @@
+/**
+ * WebCodecs `VideoDecoder` wrapper — Annex-B H.264 in, `<canvas>` frames out.
+ *
+ * Two config traps this guards against: (1) the codec string must come from the real
+ * bitstream (`avc1-codec-string.ts` on the server derives it from the actual SPS) — a
+ * hardcoded guess can mismatch the encoder's actual profile/level and make `configure()`
+ * throw; (2) no `description` field is passed, which is what tells WebCodecs to expect
+ * Annex-B (start-code-prefixed) input rather than length-prefixed AVCC — passing a
+ * `description` here would silently break decode of every chunk this session sends.
+ *
+ * Self-healing on decode error: a mobile hardware decoder (confirmed: works on desktop Chrome,
+ * dies on mobile Chromium after ~one GOP) can throw a hard decode error and never recover on its
+ * own — WebCodecs offers no "just skip this frame" API once that happens, the decoder is dead.
+ * Rather than freeze the stream forever, a decode error tears down the errored `VideoDecoder`
+ * and creates a fresh one with the same cached codec string, then waits for the next server
+ * keyframe before feeding it anything (same guard the very first connect already uses) — the
+ * server sends one roughly every second (30-frame GOP @ 30fps), so recovery is bounded to about
+ * that long instead of permanent.
+ */
+import { useCallback, useRef, useState } from "react";
+import { frameTimestampMicros } from "./remote-desktop-frame-timestamp";
+
+export type DecoderStatus = "idle" | "unsupported" | "ready" | "error";
+
+/**
+ * Whether an access unit should be fed to the decoder right now. Decode cannot start on a delta
+ * chunk — true both on the very first connect (before any keyframe has arrived) and after an
+ * auto-recovered decode error, since a freshly (re)created `VideoDecoder` has no prior reference
+ * frame either way. A keyframe is always decodable on its own. Pure so this resync rule is
+ * testable without a real `VideoDecoder`.
+ */
+export function shouldDecodeAccessUnit(hasSeenKeyframeThisInstance: boolean, isKey: boolean): boolean {
+  return isKey || hasSeenKeyframeThisInstance;
+}
+
+export interface UseH264CanvasDecoderResult {
+  status: DecoderStatus;
+  errorMessage: string | null;
+  /** Configure (or reconfigure) the decoder once the server's `{type:"config"}` message
+   *  arrives with a codec string derived from the real SPS. */
+  configure: (codec: string) => Promise<void>;
+  /** Feed one access unit (header byte already stripped by the caller) to the decoder. */
+  decodeAccessUnit: (bytes: Uint8Array, isKey: boolean) => void;
+  reset: () => void;
+  /** Total frames the decoder has handed to `output()` (and drawn, or attempted to) since the
+   *  last `reset()`. Ref-backed, not React state — read it from a poll (e.g. the stats overlay),
+   *  not a render dependency, so every decoded frame doesn't force a re-render. */
+  getFrameCount: () => number;
+}
+
+export function useH264CanvasDecoder(
+  canvasRef: React.RefObject<HTMLCanvasElement | null>,
+): UseH264CanvasDecoderResult {
+  const [status, setStatus] = useState<DecoderStatus>("idle");
+  const [errorMessage, setErrorMessage] = useState<string | null>(null);
+  const decoderRef = useRef<VideoDecoder | null>(null);
+  const ctxRef = useRef<CanvasRenderingContext2D | null>(null);
+  const decodedAnyKeyRef = useRef(false);
+  const frameCountRef = useRef(0);
+  const frameIndexRef = useRef(0); // per-decoder-instance timestamp counter, see decodeAccessUnit
+  const lastCodecRef = useRef<string | null>(null);
+  const activeRef = useRef(false); // false once reset()/unmount has torn this session down
+
+  const fail = useCallback((message: string) => {
+    setStatus("error");
+    setErrorMessage(message);
+  }, []);
+
+  const configure = useCallback(async (codec: string) => {
+    lastCodecRef.current = codec;
+    if (typeof VideoDecoder === "undefined") {
+      setStatus("unsupported");
+      setErrorMessage("This browser has no WebCodecs VideoDecoder — try Chrome, Edge, or Safari 16.4+.");
+      return;
+    }
+    const config: VideoDecoderConfig = { codec, optimizeForLatency: true };
+    try {
+      const support = await VideoDecoder.isConfigSupported(config);
+      if (!support.supported) {
+        setStatus("unsupported");
+        setErrorMessage(`This browser's decoder does not support ${codec}.`);
+        return;
+      }
+    } catch (e) {
+      fail((e as Error).message);
+      return;
+    }
+
+    decoderRef.current?.close();
+    decodedAnyKeyRef.current = false;
+    frameIndexRef.current = 0;
+    const decoder = new VideoDecoder({
+      output: (frame) => {
+        frameCountRef.current += 1;
+        const canvas = canvasRef.current;
+        if (canvas) {
+          if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
+            canvas.width = frame.displayWidth;
+            canvas.height = frame.displayHeight;
+          }
+          if (!ctxRef.current) ctxRef.current = canvas.getContext("2d");
+          ctxRef.current?.drawImage(frame, 0, 0, canvas.width, canvas.height);
+        }
+        frame.close();
+      },
+      error: (e) => recoverOrFail(e.message),
+    });
+    try {
+      decoder.configure(config);
+    } catch (e) {
+      fail((e as Error).message);
+      return;
+    }
+    decoderRef.current = decoder;
+    activeRef.current = true;
+    setStatus("ready");
+    setErrorMessage(null);
+  }, [canvasRef, fail]);
+
+  /** Shared by the decoder's async `error` callback and `decodeAccessUnit`'s synchronous catch
+   *  (a `.decode()` call can also throw directly) — both are "this decoder just broke", handled
+   *  identically: recreate rather than give up, unless the session has already been torn down
+   *  (`reset()`/unmount) or there is no cached codec to recreate with (should not happen — codec
+   *  is cached before any decode is possible). */
+  const recoverOrFail = useCallback((message: string) => {
+    if (!activeRef.current) return; // torn down already — nothing to recover into
+    const codec = lastCodecRef.current;
+    if (!codec) { fail(message); return; }
+    try { decoderRef.current?.close(); } catch { /* discarding the errored instance regardless */ }
+    decoderRef.current = null;
+    void configure(codec); // recreate; configure() itself resets decodedAnyKeyRef, so decode
+    // waits for the next server keyframe instead of feeding a fresh decoder mid-GOP deltas.
+  }, [configure, fail]);
+
+  const decodeAccessUnit = useCallback((bytes: Uint8Array, isKey: boolean) => {
+    const decoder = decoderRef.current;
+    if (!decoder || decoder.state !== "configured") return;
+    if (!shouldDecodeAccessUnit(decodedAnyKeyRef.current, isKey)) return;
+    if (isKey) decodedAnyKeyRef.current = true;
+    try {
+      decoder.decode(new EncodedVideoChunk({
+        type: isKey ? "key" : "delta",
+        timestamp: frameTimestampMicros(frameIndexRef.current++),
+        data: bytes,
+      }));
+    } catch (e) {
+      recoverOrFail((e as Error).message);
+    }
+  }, [recoverOrFail]);
+
+  const reset = useCallback(() => {
+    activeRef.current = false;
+    // Closing an already-errored/half-configured decoder can throw in some browsers; this runs
+    // from a React effect cleanup, and an uncaught throw there crashes the whole tree (no error
+    // boundary catches effect-cleanup errors) — swallow it, we're discarding the decoder anyway.
+    try { decoderRef.current?.close(); } catch { /* discarding regardless */ }
+    decoderRef.current = null;
+    decodedAnyKeyRef.current = false;
+    frameCountRef.current = 0;
+    frameIndexRef.current = 0;
+    setStatus("idle");
+    setErrorMessage(null);
+  }, []);
+
+  const getFrameCount = useCallback(() => frameCountRef.current, []);
+
+  return {
+    status,
+    errorMessage,
+    configure,
+    decodeAccessUnit,
+    reset,
+    getFrameCount,
+  };
+}

package/src/web/components/remote-desktop/use-remote-desktop-available.ts

@@ -0,0 +1,29 @@
+/**
+ * Fetches remote-desktop capabilities once and reports whether the feature should surface in
+ * the UI. Returns `available=false` when the server feature flag is off (the endpoint 404s) or
+ * video capture is unavailable, so the nav entry stays hidden unless `REMOTE_DESKTOP_ENABLED`
+ * is set and ffmpeg can capture.
+ */
+import { useEffect, useState } from "react";
+import { api } from "@/lib/api-client";
+
+interface Capabilities {
+  ffmpegAvailable: boolean;
+  videoAvailable: boolean;
+  inputAvailable: boolean;
+  authRequired: boolean;
+}
+
+export function useRemoteDesktopAvailable(): { available: boolean; authRequired: boolean } {
+  const [caps, setCaps] = useState<Capabilities | null>(null);
+
+  useEffect(() => {
+    let cancelled = false;
+    api.get<Capabilities>("/api/remote-desktop/capabilities")
+      .then((c) => { if (!cancelled) setCaps(c); })
+      .catch(() => { if (!cancelled) setCaps(null); }); // 404 (flag off) or network error — stay hidden
+    return () => { cancelled = true; };
+  }, []);
+
+  return { available: !!caps?.videoAvailable, authRequired: !!caps?.authRequired };
+}

package/src/web/components/remote-desktop/use-remote-desktop-connection.ts

@@ -0,0 +1,147 @@
+/**
+ * Owns one remote-desktop WS connection end to end: mints a session nonce, opens the socket,
+ * feeds binary access units to the H.264 decoder, and answers the server's heartbeat ping.
+ * Factored out of the desktop floating-window body so the mobile full-screen viewer can share
+ * the exact same connection logic instead of a parallel implementation — see
+ * `remote-desktop-window-content.tsx` (desktop) and `remote-desktop-mobile-view.tsx` (mobile),
+ * both now thin callers of this hook.
+ *
+ * Connection is one-shot — a lost connection reports `connState: "closed"`/`"error"` and waits
+ * for the caller to bump `reconnect()`. We deliberately do NOT auto-reconnect on every close (a
+ * flaky link reconnecting in a loop would fight the server's "one session per host" eviction).
+ * The one exception is the tab becoming visible again: browsers throttle the ping
+ * `setInterval` in a backgrounded tab, so a user who glances away can trip the server's
+ * heartbeat timeout — on return we re-ping (or, if the session already died, reconnect once).
+ * That's a discrete user-driven event, not a loop.
+ */
+import { useCallback, useEffect, useRef, useState } from "react";
+import { api } from "@/lib/api-client";
+import { resolveRemoteDesktopWsUrl } from "./remote-desktop-ws-url";
+import { useH264CanvasDecoder, type DecoderStatus } from "./use-h264-canvas-decoder";
+
+export type RemoteDesktopConnState = "connecting" | "streaming" | "error" | "closed";
+
+const PING_INTERVAL_MS = 5_000;
+
+export interface UseRemoteDesktopConnectionResult {
+  connState: RemoteDesktopConnState;
+  errorMessage: string | null;
+  decoderStatus: DecoderStatus;
+  decoderErrorMessage: string | null;
+  sendMessage: (msg: Record<string, unknown>) => void;
+  /** Force a fresh connection (e.g. a manual "Reconnect" button). */
+  reconnect: () => void;
+  /** Total bytes of binary (access-unit) WS messages received since the last (re)connect.
+   *  Ref-backed — poll it (e.g. the stats overlay), don't treat it as a render dependency. Feeds
+   *  the KB/s stat; frame count below feeds fps. */
+  getTotalBytes: () => number;
+  /** Forwards the decoder's own frame counter (see `use-h264-canvas-decoder.ts`). */
+  getFrameCount: () => number;
+}
+
+export function useRemoteDesktopConnection(
+  canvasRef: React.RefObject<HTMLCanvasElement | null>,
+): UseRemoteDesktopConnectionResult {
+  const wsRef = useRef<WebSocket | null>(null);
+  const totalBytesRef = useRef(0);
+  const [connState, setConnState] = useState<RemoteDesktopConnState>("connecting");
+  const [errorMessage, setErrorMessage] = useState<string | null>(null);
+  const [generation, setGeneration] = useState(0); // bump to force a manual reconnect
+
+  const decoder = useH264CanvasDecoder(canvasRef);
+
+  const sendMessage = useCallback((msg: Record<string, unknown>) => {
+    const ws = wsRef.current;
+    if (ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify(msg));
+  }, []);
+
+  useEffect(() => {
+    let cancelled = false;
+    let pingTimer: ReturnType<typeof setInterval> | null = null;
+    setConnState("connecting");
+    setErrorMessage(null);
+    totalBytesRef.current = 0;
+    decoder.reset();
+
+    (async () => {
+      let nonce: string;
+      try {
+        // `wsPath` in the response is always "/ws/remote-desktop" — the client already knows
+        // that path; the response is only consumed for the nonce.
+        const res = await api.post<{ nonce: string; wsPath: string }>("/api/remote-desktop/session");
+        nonce = res.nonce;
+      } catch (e) {
+        if (!cancelled) { setConnState("error"); setErrorMessage((e as Error).message); }
+        return;
+      }
+      if (cancelled) return;
+
+      const url = resolveRemoteDesktopWsUrl(window.location, import.meta.env.DEV, import.meta.env.VITE_DEV_API_PORT);
+      const ws = new WebSocket(url);
+      ws.binaryType = "arraybuffer";
+      wsRef.current = ws;
+
+      ws.onopen = () => {
+        ws.send(JSON.stringify({ type: "auth", nonce }));
+        pingTimer = setInterval(() => sendMessage({ type: "ping" }), PING_INTERVAL_MS);
+      };
+      ws.onmessage = (event) => {
+        if (typeof event.data === "string") {
+          let msg: Record<string, unknown>;
+          try { msg = JSON.parse(event.data); } catch { return; }
+          if (msg.type === "config" && typeof msg.codec === "string") {
+            decoder.configure(msg.codec).then(() => { if (!cancelled) setConnState("streaming"); });
+          } else if (msg.type === "error") {
+            setConnState("error");
+            setErrorMessage(typeof msg.message === "string" ? msg.message : "Server error");
+          }
+          return;
+        }
+        const bytes = new Uint8Array(event.data as ArrayBuffer);
+        if (bytes.length < 1) return;
+        totalBytesRef.current += bytes.length;
+        decoder.decodeAccessUnit(bytes.subarray(1), bytes[0] === 1);
+      };
+      ws.onerror = () => { if (!cancelled) { setConnState("error"); setErrorMessage("WebSocket error"); } };
+      ws.onclose = () => { if (!cancelled) setConnState((s) => (s === "error" ? s : "closed")); };
+    })();
+
+    // A backgrounded tab throttles the ping interval, which can trip the server heartbeat. When
+    // the tab is visible again, re-ping immediately; if the session already died, reconnect once
+    // (bumping generation re-runs this effect). Fires only on the visible transition — not a loop.
+    const onVisibility = () => {
+      if (document.hidden || cancelled) return;
+      const ws = wsRef.current;
+      if (ws && ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "ping" }));
+      else setGeneration((g) => g + 1);
+    };
+    document.addEventListener("visibilitychange", onVisibility);
+
+    return () => {
+      cancelled = true;
+      document.removeEventListener("visibilitychange", onVisibility);
+      if (pingTimer) clearInterval(pingTimer);
+      // A throw here (e.g. closing a socket/decoder mid-teardown) runs inside an effect cleanup,
+      // which no error boundary catches — left unguarded it can blank the entire app on unmount,
+      // not just this component (this is what closing the mobile sheet used to do).
+      try { wsRef.current?.close(); } catch { /* tearing down regardless */ }
+      wsRef.current = null;
+      try { decoder.reset(); } catch { /* tearing down regardless */ }
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps -- reconnect is driven by `generation`
+  }, [generation]);
+
+  const reconnect = useCallback(() => setGeneration((g) => g + 1), []);
+  const getTotalBytes = useCallback(() => totalBytesRef.current, []);
+
+  return {
+    connState,
+    errorMessage,
+    decoderStatus: decoder.status,
+    decoderErrorMessage: decoder.errorMessage,
+    sendMessage,
+    reconnect,
+    getTotalBytes,
+    getFrameCount: decoder.getFrameCount,
+  };
+}

package/src/web/components/remote-desktop/use-remote-desktop-mobile-open-state.ts

@@ -0,0 +1,18 @@
+/**
+ * Global open/close state for the single mobile remote-desktop sheet. There is exactly one
+ * host to control (same as the desktop floating window, which also dedupes to one instance),
+ * so — unlike the mobile explorer's per-path slices — this is just an open flag.
+ */
+import { create } from "zustand";
+
+interface RemoteDesktopMobileOpenState {
+  isOpen: boolean;
+  open(): void;
+  close(): void;
+}
+
+export const useRemoteDesktopMobileOpenState = create<RemoteDesktopMobileOpenState>((set) => ({
+  isOpen: false,
+  open: () => set({ isOpen: true }),
+  close: () => set({ isOpen: false }),
+}));