@hienlh/ppm 0.18.14 → 0.18.15

package/src/web/components/remote-desktop/remote-desktop-ws-url.ts

@@ -0,0 +1,22 @@
+/** Pure URL builder — mirrors `ws-client.ts`'s dev-port bypass so it's unit-testable without
+ *  mounting a real WebSocket or window.location. */
+import { withWsAuth } from "@/lib/ws-auth";
+
+export interface LocationParts {
+  protocol: string;
+  hostname: string;
+  host: string;
+}
+
+export function resolveRemoteDesktopWsUrl(loc: LocationParts, isDev: boolean, devPort = "8081"): string {
+  const authed = withWsAuth("/ws/remote-desktop");
+  const wsProtocol = loc.protocol === "https:" ? "wss:" : "ws:";
+  if (isDev && loc.protocol !== "https:") {
+    // Same bypass as ws-client.ts: Vite's dev proxy has unreliable WS upgrade handling.
+    // Port defaults to 8081 (the normal `bun dev:server` port) but is overridable via
+    // `VITE_DEV_API_PORT` for an alt-port dev stack (see `PPM_DEV_API` in vite.config.ts) —
+    // without this, a second dev stack on e.g. 8082 would always connect to the wrong server.
+    return `ws://${loc.hostname}:${devPort}${authed}`;
+  }
+  return `${wsProtocol}//${loc.host}${authed}`;
+}

package/src/web/components/remote-desktop/use-h264-canvas-decoder.ts

@@ -0,0 +1,175 @@
+/**
+ * WebCodecs `VideoDecoder` wrapper — Annex-B H.264 in, `<canvas>` frames out.
+ *
+ * Two config traps this guards against: (1) the codec string must come from the real
+ * bitstream (`avc1-codec-string.ts` on the server derives it from the actual SPS) — a
+ * hardcoded guess can mismatch the encoder's actual profile/level and make `configure()`
+ * throw; (2) no `description` field is passed, which is what tells WebCodecs to expect
+ * Annex-B (start-code-prefixed) input rather than length-prefixed AVCC — passing a
+ * `description` here would silently break decode of every chunk this session sends.
+ *
+ * Self-healing on decode error: a mobile hardware decoder (confirmed: works on desktop Chrome,
+ * dies on mobile Chromium after ~one GOP) can throw a hard decode error and never recover on its
+ * own — WebCodecs offers no "just skip this frame" API once that happens, the decoder is dead.
+ * Rather than freeze the stream forever, a decode error tears down the errored `VideoDecoder`
+ * and creates a fresh one with the same cached codec string, then waits for the next server
+ * keyframe before feeding it anything (same guard the very first connect already uses) — the
+ * server sends one roughly every second (30-frame GOP @ 30fps), so recovery is bounded to about
+ * that long instead of permanent.
+ */
+import { useCallback, useRef, useState } from "react";
+import { frameTimestampMicros } from "./remote-desktop-frame-timestamp";
+
+export type DecoderStatus = "idle" | "unsupported" | "ready" | "error";
+
+/**
+ * Whether an access unit should be fed to the decoder right now. Decode cannot start on a delta
+ * chunk — true both on the very first connect (before any keyframe has arrived) and after an
+ * auto-recovered decode error, since a freshly (re)created `VideoDecoder` has no prior reference
+ * frame either way. A keyframe is always decodable on its own. Pure so this resync rule is
+ * testable without a real `VideoDecoder`.
+ */
+export function shouldDecodeAccessUnit(hasSeenKeyframeThisInstance: boolean, isKey: boolean): boolean {
+  return isKey || hasSeenKeyframeThisInstance;
+}
+
+export interface UseH264CanvasDecoderResult {
+  status: DecoderStatus;
+  errorMessage: string | null;
+  /** Configure (or reconfigure) the decoder once the server's `{type:"config"}` message
+   *  arrives with a codec string derived from the real SPS. */
+  configure: (codec: string) => Promise<void>;
+  /** Feed one access unit (header byte already stripped by the caller) to the decoder. */
+  decodeAccessUnit: (bytes: Uint8Array, isKey: boolean) => void;
+  reset: () => void;
+  /** Total frames the decoder has handed to `output()` (and drawn, or attempted to) since the
+   *  last `reset()`. Ref-backed, not React state — read it from a poll (e.g. the stats overlay),
+   *  not a render dependency, so every decoded frame doesn't force a re-render. */
+  getFrameCount: () => number;
+}
+
+export function useH264CanvasDecoder(
+  canvasRef: React.RefObject<HTMLCanvasElement | null>,
+): UseH264CanvasDecoderResult {
+  const [status, setStatus] = useState<DecoderStatus>("idle");
+  const [errorMessage, setErrorMessage] = useState<string | null>(null);
+  const decoderRef = useRef<VideoDecoder | null>(null);
+  const ctxRef = useRef<CanvasRenderingContext2D | null>(null);
+  const decodedAnyKeyRef = useRef(false);
+  const frameCountRef = useRef(0);
+  const frameIndexRef = useRef(0); // per-decoder-instance timestamp counter, see decodeAccessUnit
+  const lastCodecRef = useRef<string | null>(null);
+  const activeRef = useRef(false); // false once reset()/unmount has torn this session down
+
+  const fail = useCallback((message: string) => {
+    setStatus("error");
+    setErrorMessage(message);
+  }, []);
+
+  const configure = useCallback(async (codec: string) => {
+    lastCodecRef.current = codec;
+    if (typeof VideoDecoder === "undefined") {
+      setStatus("unsupported");
+      setErrorMessage("This browser has no WebCodecs VideoDecoder — try Chrome, Edge, or Safari 16.4+.");
+      return;
+    }
+    const config: VideoDecoderConfig = { codec, optimizeForLatency: true };
+    try {
+      const support = await VideoDecoder.isConfigSupported(config);
+      if (!support.supported) {
+        setStatus("unsupported");
+        setErrorMessage(`This browser's decoder does not support ${codec}.`);
+        return;
+      }
+    } catch (e) {
+      fail((e as Error).message);
+      return;
+    }
+
+    decoderRef.current?.close();
+    decodedAnyKeyRef.current = false;
+    frameIndexRef.current = 0;
+    const decoder = new VideoDecoder({
+      output: (frame) => {
+        frameCountRef.current += 1;
+        const canvas = canvasRef.current;
+        if (canvas) {
+          if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
+            canvas.width = frame.displayWidth;
+            canvas.height = frame.displayHeight;
+          }
+          if (!ctxRef.current) ctxRef.current = canvas.getContext("2d");
+          ctxRef.current?.drawImage(frame, 0, 0, canvas.width, canvas.height);
+        }
+        frame.close();
+      },
+      error: (e) => recoverOrFail(e.message),
+    });
+    try {
+      decoder.configure(config);
+    } catch (e) {
+      fail((e as Error).message);
+      return;
+    }
+    decoderRef.current = decoder;
+    activeRef.current = true;
+    setStatus("ready");
+    setErrorMessage(null);
+  }, [canvasRef, fail]);
+
+  /** Shared by the decoder's async `error` callback and `decodeAccessUnit`'s synchronous catch
+   *  (a `.decode()` call can also throw directly) — both are "this decoder just broke", handled
+   *  identically: recreate rather than give up, unless the session has already been torn down
+   *  (`reset()`/unmount) or there is no cached codec to recreate with (should not happen — codec
+   *  is cached before any decode is possible). */
+  const recoverOrFail = useCallback((message: string) => {
+    if (!activeRef.current) return; // torn down already — nothing to recover into
+    const codec = lastCodecRef.current;
+    if (!codec) { fail(message); return; }
+    try { decoderRef.current?.close(); } catch { /* discarding the errored instance regardless */ }
+    decoderRef.current = null;
+    void configure(codec); // recreate; configure() itself resets decodedAnyKeyRef, so decode
+    // waits for the next server keyframe instead of feeding a fresh decoder mid-GOP deltas.
+  }, [configure, fail]);
+
+  const decodeAccessUnit = useCallback((bytes: Uint8Array, isKey: boolean) => {
+    const decoder = decoderRef.current;
+    if (!decoder || decoder.state !== "configured") return;
+    if (!shouldDecodeAccessUnit(decodedAnyKeyRef.current, isKey)) return;
+    if (isKey) decodedAnyKeyRef.current = true;
+    try {
+      decoder.decode(new EncodedVideoChunk({
+        type: isKey ? "key" : "delta",
+        timestamp: frameTimestampMicros(frameIndexRef.current++),
+        data: bytes,
+      }));
+    } catch (e) {
+      recoverOrFail((e as Error).message);
+    }
+  }, [recoverOrFail]);
+
+  const reset = useCallback(() => {
+    activeRef.current = false;
+    // Closing an already-errored/half-configured decoder can throw in some browsers; this runs
+    // from a React effect cleanup, and an uncaught throw there crashes the whole tree (no error
+    // boundary catches effect-cleanup errors) — swallow it, we're discarding the decoder anyway.
+    try { decoderRef.current?.close(); } catch { /* discarding regardless */ }
+    decoderRef.current = null;
+    decodedAnyKeyRef.current = false;
+    frameCountRef.current = 0;
+    frameIndexRef.current = 0;
+    setStatus("idle");
+    setErrorMessage(null);
+  }, []);
+
+  const getFrameCount = useCallback(() => frameCountRef.current, []);
+
+  return {
+    status,
+    errorMessage,
+    configure,
+    decodeAccessUnit,
+    reset,
+    getFrameCount,
+  };
+}

package/src/web/components/remote-desktop/use-remote-desktop-available.ts

@@ -0,0 +1,29 @@
+/**
+ * Fetches remote-desktop capabilities once and reports whether the feature should surface in
+ * the UI. Returns `available=false` when the server feature flag is off (the endpoint 404s) or
+ * video capture is unavailable, so the nav entry stays hidden unless `REMOTE_DESKTOP_ENABLED`
+ * is set and ffmpeg can capture.
+ */
+import { useEffect, useState } from "react";
+import { api } from "@/lib/api-client";
+
+interface Capabilities {
+  ffmpegAvailable: boolean;
+  videoAvailable: boolean;
+  inputAvailable: boolean;
+  authRequired: boolean;
+}
+
+export function useRemoteDesktopAvailable(): { available: boolean; authRequired: boolean } {
+  const [caps, setCaps] = useState<Capabilities | null>(null);
+
+  useEffect(() => {
+    let cancelled = false;
+    api.get<Capabilities>("/api/remote-desktop/capabilities")
+      .then((c) => { if (!cancelled) setCaps(c); })
+      .catch(() => { if (!cancelled) setCaps(null); }); // 404 (flag off) or network error — stay hidden
+    return () => { cancelled = true; };
+  }, []);
+
+  return { available: !!caps?.videoAvailable, authRequired: !!caps?.authRequired };
+}

package/src/web/components/remote-desktop/use-remote-desktop-connection.ts

@@ -0,0 +1,147 @@
+/**
+ * Owns one remote-desktop WS connection end to end: mints a session nonce, opens the socket,
+ * feeds binary access units to the H.264 decoder, and answers the server's heartbeat ping.
+ * Factored out of the desktop floating-window body so the mobile full-screen viewer can share
+ * the exact same connection logic instead of a parallel implementation — see
+ * `remote-desktop-window-content.tsx` (desktop) and `remote-desktop-mobile-view.tsx` (mobile),
+ * both now thin callers of this hook.
+ *
+ * Connection is one-shot — a lost connection reports `connState: "closed"`/`"error"` and waits
+ * for the caller to bump `reconnect()`. We deliberately do NOT auto-reconnect on every close (a
+ * flaky link reconnecting in a loop would fight the server's "one session per host" eviction).
+ * The one exception is the tab becoming visible again: browsers throttle the ping
+ * `setInterval` in a backgrounded tab, so a user who glances away can trip the server's
+ * heartbeat timeout — on return we re-ping (or, if the session already died, reconnect once).
+ * That's a discrete user-driven event, not a loop.
+ */
+import { useCallback, useEffect, useRef, useState } from "react";
+import { api } from "@/lib/api-client";
+import { resolveRemoteDesktopWsUrl } from "./remote-desktop-ws-url";
+import { useH264CanvasDecoder, type DecoderStatus } from "./use-h264-canvas-decoder";
+
+export type RemoteDesktopConnState = "connecting" | "streaming" | "error" | "closed";
+
+const PING_INTERVAL_MS = 5_000;
+
+export interface UseRemoteDesktopConnectionResult {
+  connState: RemoteDesktopConnState;
+  errorMessage: string | null;
+  decoderStatus: DecoderStatus;
+  decoderErrorMessage: string | null;
+  sendMessage: (msg: Record<string, unknown>) => void;
+  /** Force a fresh connection (e.g. a manual "Reconnect" button). */
+  reconnect: () => void;
+  /** Total bytes of binary (access-unit) WS messages received since the last (re)connect.
+   *  Ref-backed — poll it (e.g. the stats overlay), don't treat it as a render dependency. Feeds
+   *  the KB/s stat; frame count below feeds fps. */
+  getTotalBytes: () => number;
+  /** Forwards the decoder's own frame counter (see `use-h264-canvas-decoder.ts`). */
+  getFrameCount: () => number;
+}
+
+export function useRemoteDesktopConnection(
+  canvasRef: React.RefObject<HTMLCanvasElement | null>,
+): UseRemoteDesktopConnectionResult {
+  const wsRef = useRef<WebSocket | null>(null);
+  const totalBytesRef = useRef(0);
+  const [connState, setConnState] = useState<RemoteDesktopConnState>("connecting");
+  const [errorMessage, setErrorMessage] = useState<string | null>(null);
+  const [generation, setGeneration] = useState(0); // bump to force a manual reconnect
+
+  const decoder = useH264CanvasDecoder(canvasRef);
+
+  const sendMessage = useCallback((msg: Record<string, unknown>) => {
+    const ws = wsRef.current;
+    if (ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify(msg));
+  }, []);
+
+  useEffect(() => {
+    let cancelled = false;
+    let pingTimer: ReturnType<typeof setInterval> | null = null;
+    setConnState("connecting");
+    setErrorMessage(null);
+    totalBytesRef.current = 0;
+    decoder.reset();
+
+    (async () => {
+      let nonce: string;
+      try {
+        // `wsPath` in the response is always "/ws/remote-desktop" — the client already knows
+        // that path; the response is only consumed for the nonce.
+        const res = await api.post<{ nonce: string; wsPath: string }>("/api/remote-desktop/session");
+        nonce = res.nonce;
+      } catch (e) {
+        if (!cancelled) { setConnState("error"); setErrorMessage((e as Error).message); }
+        return;
+      }
+      if (cancelled) return;
+
+      const url = resolveRemoteDesktopWsUrl(window.location, import.meta.env.DEV, import.meta.env.VITE_DEV_API_PORT);
+      const ws = new WebSocket(url);
+      ws.binaryType = "arraybuffer";
+      wsRef.current = ws;
+
+      ws.onopen = () => {
+        ws.send(JSON.stringify({ type: "auth", nonce }));
+        pingTimer = setInterval(() => sendMessage({ type: "ping" }), PING_INTERVAL_MS);
+      };
+      ws.onmessage = (event) => {
+        if (typeof event.data === "string") {
+          let msg: Record<string, unknown>;
+          try { msg = JSON.parse(event.data); } catch { return; }
+          if (msg.type === "config" && typeof msg.codec === "string") {
+            decoder.configure(msg.codec).then(() => { if (!cancelled) setConnState("streaming"); });
+          } else if (msg.type === "error") {
+            setConnState("error");
+            setErrorMessage(typeof msg.message === "string" ? msg.message : "Server error");
+          }
+          return;
+        }
+        const bytes = new Uint8Array(event.data as ArrayBuffer);
+        if (bytes.length < 1) return;
+        totalBytesRef.current += bytes.length;
+        decoder.decodeAccessUnit(bytes.subarray(1), bytes[0] === 1);
+      };
+      ws.onerror = () => { if (!cancelled) { setConnState("error"); setErrorMessage("WebSocket error"); } };
+      ws.onclose = () => { if (!cancelled) setConnState((s) => (s === "error" ? s : "closed")); };
+    })();
+
+    // A backgrounded tab throttles the ping interval, which can trip the server heartbeat. When
+    // the tab is visible again, re-ping immediately; if the session already died, reconnect once
+    // (bumping generation re-runs this effect). Fires only on the visible transition — not a loop.
+    const onVisibility = () => {
+      if (document.hidden || cancelled) return;
+      const ws = wsRef.current;
+      if (ws && ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "ping" }));
+      else setGeneration((g) => g + 1);
+    };
+    document.addEventListener("visibilitychange", onVisibility);
+
+    return () => {
+      cancelled = true;
+      document.removeEventListener("visibilitychange", onVisibility);
+      if (pingTimer) clearInterval(pingTimer);
+      // A throw here (e.g. closing a socket/decoder mid-teardown) runs inside an effect cleanup,
+      // which no error boundary catches — left unguarded it can blank the entire app on unmount,
+      // not just this component (this is what closing the mobile sheet used to do).
+      try { wsRef.current?.close(); } catch { /* tearing down regardless */ }
+      wsRef.current = null;
+      try { decoder.reset(); } catch { /* tearing down regardless */ }
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps -- reconnect is driven by `generation`
+  }, [generation]);
+
+  const reconnect = useCallback(() => setGeneration((g) => g + 1), []);
+  const getTotalBytes = useCallback(() => totalBytesRef.current, []);
+
+  return {
+    connState,
+    errorMessage,
+    decoderStatus: decoder.status,
+    decoderErrorMessage: decoder.errorMessage,
+    sendMessage,
+    reconnect,
+    getTotalBytes,
+    getFrameCount: decoder.getFrameCount,
+  };
+}

package/src/web/components/remote-desktop/use-remote-desktop-mobile-open-state.ts

@@ -0,0 +1,18 @@
+/**
+ * Global open/close state for the single mobile remote-desktop sheet. There is exactly one
+ * host to control (same as the desktop floating window, which also dedupes to one instance),
+ * so — unlike the mobile explorer's per-path slices — this is just an open flag.
+ */
+import { create } from "zustand";
+
+interface RemoteDesktopMobileOpenState {
+  isOpen: boolean;
+  open(): void;
+  close(): void;
+}
+
+export const useRemoteDesktopMobileOpenState = create<RemoteDesktopMobileOpenState>((set) => ({
+  isOpen: false,
+  open: () => set({ isOpen: true }),
+  close: () => set({ isOpen: false }),
+}));

package/src/web/components/remote-desktop/use-remote-desktop-touch.ts

@@ -0,0 +1,224 @@
+/**
+ * Multi-touch gesture engine for the mobile remote-desktop viewer: pinch-zoom/pan (CSS
+ * transform on the caller's "stage" element) plus tap/drag → pointer + wheel messages over the
+ * shared connection. Two modes (see design doc `research-260907-1538-remote-desktop-mobile-
+ * touch.md`): TOUCH maps a 1-finger drag straight to an absolute drag at the touch point;
+ * MOUSE keeps a client-side virtual cursor that a 1-finger drag moves *relatively*, with a tap
+ * clicking wherever the cursor currently sits. Two-finger gestures are shared by both modes:
+ * tap = right click, pinch = zoom, drag = pan (while zoomed) or scroll wheel (otherwise). The
+ * actual decision rules live in `remote-desktop-touch-gesture-tracker.ts`; this file only wires
+ * DOM touch events to them and to React state.
+ *
+ * All touch listeners are native (`addEventListener`, non-passive) rather than React props, so
+ * `stopPropagation()` here also stops the ancestor `BottomSheet`'s swipe-to-dismiss (a React
+ * synthetic handler) from ever seeing the event — same trick `mobile-explorer-sheet.tsx` uses
+ * for its inline-rename case.
+ */
+import { useEffect, useRef, useState, useCallback } from "react";
+import { letterboxedContentRect, type ZoomPanTransform, type Fraction, type Rect } from "./remote-desktop-coords";
+import {
+  beginSingleTouch, advanceSingleTouch, resolveSingleTouchEnd,
+  beginTwoFingerTouch, advanceTwoFingerTouch, resolveTwoFingerEnd,
+  type SingleTrack, type TwoFingerTrack, type RemoteDesktopInputMode,
+} from "./remote-desktop-touch-gesture-tracker";
+
+export type { RemoteDesktopInputMode };
+
+const IDENTITY_TRANSFORM: ZoomPanTransform = { scale: 1, panX: 0, panY: 0 };
+/** CSS px of two-finger drag per wheel notch (`WHEEL_DELTA` = 120 units, matching a real wheel
+ *  "click"). Tune here if a live test finds scrolling too fast/slow. */
+const WHEEL_PIXELS_PER_NOTCH = 40;
+const WHEEL_DELTA = 120;
+
+export interface UseRemoteDesktopTouchOptions {
+  /** The never-transformed element touch listeners attach to and whose rect is the reference
+   *  frame at `scale=1, pan=0` — must fill the same box the "stage" (canvas) sits in at rest. */
+  containerRef: React.RefObject<HTMLElement | null>;
+  /** The canvas actually being drawn into. Its CSS box is the same size as `containerRef` (it
+   *  fills the stage, which fills the container), but its drawing-buffer size (`.width`/
+   *  `.height`, the real capture resolution) is almost never the same *aspect ratio* — a tap
+   *  must be measured against the letterboxed video rect this produces inside the container,
+   *  not the container's full box, or it lands off by however big the letterbox bars are. */
+  canvasRef: React.RefObject<HTMLCanvasElement | null>;
+  mode: RemoteDesktopInputMode;
+  sendMessage: (msg: Record<string, unknown>) => void;
+  enabled: boolean;
+}
+
+export interface UseRemoteDesktopTouchResult {
+  /** Apply as `transform: translate(panXpx, panYpx) scale(scale)` on the stage element. */
+  transform: ZoomPanTransform;
+  /** Client-side cursor position (mouse mode only) — draw a marker at this fraction inside the
+   *  stage. `null` in touch mode (no persistent cursor concept there). */
+  virtualCursor: Fraction | null;
+  resetZoom: () => void;
+}
+
+export function useRemoteDesktopTouch({
+  containerRef,
+  canvasRef,
+  mode,
+  sendMessage,
+  enabled,
+}: UseRemoteDesktopTouchOptions): UseRemoteDesktopTouchResult {
+  const [transform, setTransform] = useState<ZoomPanTransform>(IDENTITY_TRANSFORM);
+  const [virtualCursor, setVirtualCursor] = useState<Fraction>({ xFrac: 0.5, yFrac: 0.5 });
+  const transformRef = useRef(transform);
+  const cursorRef = useRef(virtualCursor);
+  const modeRef = useRef(mode);
+  modeRef.current = mode;
+
+  const pendingRef = useRef<{ transform: ZoomPanTransform | null; cursor: Fraction | null; sendCursorMove: boolean; wheelPx: number }>(
+    { transform: null, cursor: null, sendCursorMove: false, wheelPx: 0 },
+  );
+  const rafIdRef = useRef(0);
+
+  const flush = useCallback(() => {
+    rafIdRef.current = 0;
+    const p = pendingRef.current;
+    if (p.transform) { transformRef.current = p.transform; setTransform(p.transform); p.transform = null; }
+    if (p.cursor) {
+      cursorRef.current = p.cursor;
+      setVirtualCursor(p.cursor);
+      if (p.sendCursorMove) sendMessage({ type: "pointer", xFrac: p.cursor.xFrac, yFrac: p.cursor.yFrac, button: null, down: null });
+      p.cursor = null;
+      p.sendCursorMove = false;
+    }
+    if (p.wheelPx !== 0) {
+      // Fingers moving up (negative px) reads as "scroll content up" — a positive (forward)
+      // wheel notch, matching a physical wheel rotated away from the user.
+      sendMessage({ type: "wheel", dy: -(p.wheelPx / WHEEL_PIXELS_PER_NOTCH) * WHEEL_DELTA });
+      p.wheelPx = 0;
+    }
+  }, [sendMessage]);
+
+  const schedule = useCallback(() => {
+    if (!rafIdRef.current) rafIdRef.current = requestAnimationFrame(flush);
+  }, [flush]);
+
+  const resetZoom = useCallback(() => {
+    transformRef.current = IDENTITY_TRANSFORM;
+    setTransform(IDENTITY_TRANSFORM);
+  }, []);
+
+  useEffect(() => {
+    const container = containerRef.current;
+    if (!container || !enabled) return;
+
+    let single: SingleTrack | null = null;
+    let two: TwoFingerTrack | null = null;
+    const sendAll = (msgs: Record<string, unknown>[]) => msgs.forEach(sendMessage);
+
+    // The container's rect (never transformed) is only the OUTER box the video is letterboxed
+    // within — `object-fit: contain` (or the matching sizing on the canvas) means the canvas
+    // element's CSS box fills that outer box, but the pixels it actually draws are narrower or
+    // shorter depending on how the capture's aspect ratio compares to the phone's. Without this,
+    // gestures were measured against the full outer box, so a tap landed at the finger's raw
+    // screen position instead of the corresponding video pixel whenever there was a letterbox.
+    const getVideoRect = (): Rect => {
+      const box = container.getBoundingClientRect();
+      const canvas = canvasRef.current;
+      if (!canvas || !canvas.width || !canvas.height) return box;
+      return letterboxedContentRect(box, canvas.width, canvas.height);
+    };
+
+    const onTouchStart = (e: TouchEvent) => {
+      e.preventDefault();
+      e.stopPropagation();
+      const rect = getVideoRect();
+      if (e.touches.length === 1) {
+        two = null;
+        const t = e.touches[0]!;
+        single = beginSingleTouch(t.clientX, t.clientY, rect);
+      } else if (e.touches.length === 2) {
+        single = null;
+        const [a, b] = [e.touches[0]!, e.touches[1]!];
+        two = beginTwoFingerTouch({ x: a.clientX, y: a.clientY }, { x: b.clientX, y: b.clientY }, transformRef.current.scale);
+      } else {
+        // 3+ fingers: not a gesture this viewer understands — drop tracking rather than guess.
+        single = null;
+        two = null;
+      }
+    };
+
+    const onTouchMove = (e: TouchEvent) => {
+      e.preventDefault();
+      e.stopPropagation();
+      if (single && e.touches.length === 1) {
+        const t = e.touches[0]!;
+        const r = advanceSingleTouch(single, t.clientX, t.clientY, modeRef.current, transformRef.current, cursorRef.current);
+        if (r.message) sendMessage(r.message);
+        if (r.cursor) {
+          // Mouse mode is DELTA-based (each call moves the cursor *relative* to `cursorRef`).
+          // rAF coalesces the state-commit + wire-send below to once per frame, but the delta
+          // math itself must not wait for that — updating `cursorRef.current` only inside
+          // `flush()` meant back-to-back touchmove events landing in the same animation frame
+          // each recomputed their delta against the SAME stale base position, so only the last
+          // one "counted" and most of the finger's movement was silently dropped (the host
+          // cursor barely moved). Updating the ref here, synchronously, makes every event
+          // compound on the last regardless of how many land before the next flush.
+          cursorRef.current = r.cursor;
+          pendingRef.current.cursor = r.cursor;
+          pendingRef.current.sendCursorMove = !!r.sendCursorMove;
+          schedule();
+        }
+        return;
+      }
+      if (two && e.touches.length === 2) {
+        const [a, b] = [e.touches[0]!, e.touches[1]!];
+        const r = advanceTwoFingerTouch(two, { x: a.clientX, y: a.clientY }, { x: b.clientX, y: b.clientY }, transformRef.current);
+        if (r.transform) {
+          // Same reasoning as the cursor above: pan is delta-based off `transformRef`, so it
+          // must be updated eagerly too, not just inside the throttled flush.
+          transformRef.current = r.transform;
+          pendingRef.current.transform = r.transform;
+          schedule();
+        }
+        if (r.wheelDeltaPx) { pendingRef.current.wheelPx += r.wheelDeltaPx; schedule(); }
+      }
+    };
+
+    const onTouchEnd = (e: TouchEvent) => {
+      e.preventDefault();
+      e.stopPropagation();
+      const t = e.changedTouches[0];
+      if (single) {
+        const x = t ? t.clientX : single.last.x;
+        const y = t ? t.clientY : single.last.y;
+        sendAll(resolveSingleTouchEnd(single, x, y, modeRef.current, transformRef.current, cursorRef.current));
+        single = null;
+      }
+      if (two) {
+        sendAll(resolveTwoFingerEnd(two, getVideoRect(), transformRef.current));
+        two = null;
+      }
+    };
+
+    const onTouchCancel = (e: TouchEvent) => {
+      e.stopPropagation();
+      // A held button left down on the host would be worse than a dropped click — release it.
+      // Only touch mode ever sends a "down" during a drag (mouse mode only moves the virtual
+      // cursor until a tap), so only touch mode has anything to release here.
+      if (single?.moved && modeRef.current === "touch") {
+        sendAll(resolveSingleTouchEnd(single, single.last.x, single.last.y, "touch", transformRef.current, cursorRef.current));
+      }
+      single = null;
+      two = null;
+    };
+
+    container.addEventListener("touchstart", onTouchStart, { passive: false });
+    container.addEventListener("touchmove", onTouchMove, { passive: false });
+    container.addEventListener("touchend", onTouchEnd, { passive: false });
+    container.addEventListener("touchcancel", onTouchCancel, { passive: false });
+
+    return () => {
+      if (rafIdRef.current) cancelAnimationFrame(rafIdRef.current);
+      container.removeEventListener("touchstart", onTouchStart);
+      container.removeEventListener("touchmove", onTouchMove);
+      container.removeEventListener("touchend", onTouchEnd);
+      container.removeEventListener("touchcancel", onTouchCancel);
+    };
+  }, [containerRef, canvasRef, enabled, sendMessage, schedule]);
+
+  return { transform, virtualCursor: mode === "mouse" ? virtualCursor : null, resetZoom };
+}

package/src/web/components/remote-desktop/use-remote-desktop-virtual-keyboard.ts

@@ -0,0 +1,50 @@
+/**
+ * Forwards keydown/keyup from a hidden `<input>` to the shared connection, so a toolbar
+ * "Keyboard" button can bring up the phone's soft keyboard without a physical keyboard ever
+ * being attached. Mirrors `use-remote-input-capture.ts`'s key handling exactly: `code`
+ * (physical key, layout-independent) is sent, never `key` — the host maps `code` → VK.
+ */
+import { useEffect, useRef, useCallback } from "react";
+
+export interface UseRemoteDesktopVirtualKeyboardResult {
+  inputRef: React.RefObject<HTMLInputElement | null>;
+  /** Focus the hidden input, which brings up the soft keyboard on a touch device. */
+  show: () => void;
+}
+
+export function useRemoteDesktopVirtualKeyboard(
+  sendMessage: (msg: Record<string, unknown>) => void,
+  enabled: boolean,
+): UseRemoteDesktopVirtualKeyboardResult {
+  const inputRef = useRef<HTMLInputElement | null>(null);
+
+  useEffect(() => {
+    const input = inputRef.current;
+    if (!input || !enabled) return;
+
+    // A lost keyup (blur, tab hidden, WS drop) must not leave a modifier logically held on the
+    // host — force a release on every path that could lose the matching keyup.
+    const releaseAll = () => sendMessage({ type: "releaseAll" });
+    const onKeyDown = (e: KeyboardEvent) => { e.preventDefault(); sendMessage({ type: "key", code: e.code, down: true }); };
+    const onKeyUp = (e: KeyboardEvent) => { e.preventDefault(); sendMessage({ type: "key", code: e.code, down: false }); };
+    const onBlur = () => releaseAll();
+    const onVisibilityChange = () => { if (document.hidden) releaseAll(); };
+
+    input.addEventListener("keydown", onKeyDown);
+    input.addEventListener("keyup", onKeyUp);
+    input.addEventListener("blur", onBlur);
+    document.addEventListener("visibilitychange", onVisibilityChange);
+
+    return () => {
+      input.removeEventListener("keydown", onKeyDown);
+      input.removeEventListener("keyup", onKeyUp);
+      input.removeEventListener("blur", onBlur);
+      document.removeEventListener("visibilitychange", onVisibilityChange);
+      releaseAll();
+    };
+  }, [enabled, sendMessage]);
+
+  const show = useCallback(() => inputRef.current?.focus(), []);
+
+  return { inputRef, show };
+}