@desklink/react-native 0.1.1 → 0.3.0

package/src/DesktopView.ios.tsx

@@ -1,14 +1,67 @@
 import * as React from 'react';
-import { Keyboard, StyleSheet, TextInput, View, type GestureResponderEvent, type LayoutChangeEvent } from 'react-native';
-import { RTCView } from 'react-native-webrtc';
+import { Keyboard, StyleSheet, TextInput, View, type GestureResponderEvent, type LayoutChangeEvent, type ViewProps } from 'react-native';
+import { requireNativeViewManager, requireOptionalNativeModule } from 'expo-modules-core';
 import type { DesktopViewProps } from './DesktopView';
+import { hardwareKey, type HardwareKey } from './hardwareKeys';
 import { keyboardDelta } from './keyboardDelta';
+import { requireWebRTC } from './webrtc.ios';
 import { desktopInputEnabled, emitDesktopKeyboard, getDesktopSize, getDesktopStream, markDesktopPresented, nativeDesklink, observeDesktopSurface, registerDesktopView } from './native.ios';
 
 type Point = { x: number; y: number };
-type Gesture = { start: Point; last: Point; time: number; mode: 'pending' | 'hover' | 'pan' | 'armed' | 'drag' | 'two' | 'scroll' | 'pinch' | 'spent'; span: number; focus: Point };
+type Gesture = { stamp: number; start: Point; last: Point; time: number; mode: 'pending' | 'letterbox' | 'hover' | 'pan' | 'armed' | 'drag' | 'two' | 'scroll' | 'pinch' | 'spent'; span: number; focus: Point };
 
-export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0 }: DesktopViewProps) {
+/** Wheel detents accumulate in these steps and never send more than this at once. */
+const MIN_WHEEL_STEP = 0.05;
+const MAX_WHEEL_STEP = 10;
+/** Desktop pixels one wheel detent stands for. */
+const PIXELS_PER_DETENT = 120;
+/** Where the cursor mark's tip sits inside the mark, in points. */
+const CURSOR_HOTSPOT = 3;
+
+type NativeEvent<T> = { nativeEvent: T };
+type PointerEvent = { phase: 'hover' | 'down' | 'move' | 'up' | 'cancel'; x: number; y: number; buttons?: number; timestamp?: number };
+type WheelEvent = { phase: 'begin' | 'move' | 'end'; dx: number; dy: number; x: number; y: number };
+type InputViewProps = ViewProps & {
+    enabled: boolean;
+    onKey: (event: NativeEvent<HardwareKey>) => void;
+    onPointer: (event: NativeEvent<PointerEvent>) => void;
+    onWheel: (event: NativeEvent<WheelEvent>) => void;
+};
+
+/**
+ * The package's native surface (`ios/DesklinkInputView.swift`): it reports the
+ * hardware keyboard and an iPad pointer. A build without it — an app that has
+ * not rebuilt its native project since updating — keeps a plain view, with
+ * touch and the on-screen keyboard only.
+ */
+function loadInputView(): React.ComponentType<InputViewProps> | null {
+    try {
+        return requireOptionalNativeModule('DesklinkInput') ? requireNativeViewManager<InputViewProps>('DesklinkInput') : null;
+    } catch {
+        return null;
+    }
+}
+
+/** A press's desktop button from UIKit's button mask: primary, secondary, then the middle. */
+function pressedButton(mask: number): number {
+    if (mask & 2) return 3;
+    if (mask & 4) return 2;
+    return 1;
+}
+
+/** The native video view, loaded on first render rather than with this module. */
+function loadRTCView(): React.ComponentType<Record<string, unknown>> | null {
+    try {
+        const webrtc = requireWebRTC() as {
+            RTCView?: React.ComponentType<Record<string, unknown>>;
+        };
+        return webrtc.RTCView ?? null;
+    } catch {
+        return null;
+    }
+}
+
+export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, gestures = 'desktop' }: DesktopViewProps) {
     const [revision, refresh] = React.useReducer((n: number) => n + 1, 0);
     const [bounds, setBounds] = React.useState({ width: 0, height: 0 });
     const [size, setSize] = React.useState(() => getDesktopSize(sessionId) ?? { width: 0, height: 0 });
@@ -21,10 +74,17 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
     const input = React.useRef<TextInput>(null);
     const captured = React.useRef(false);
     const gesture = React.useRef<Gesture | null>(null);
+    const wheel = React.useRef({ x: 0, y: 0 });
     const longPress = React.useRef<ReturnType<typeof setTimeout> | null>(null);
     const lastTap = React.useRef<{ time: number; at: Point; point: Point } | null>(null);
+    const chordsDown = React.useRef(new Set<number>());
+    /** Touch timestamps of recent trackpad or mouse presses, which the native view reports and touch leaves alone. */
+    const pressStamps = React.useRef<number[]>([]);
+    const pressed = React.useRef<number | null>(null);
     const [enabled, setEnabled] = React.useState(() => desktopInputEnabled(sessionId));
     const stream = getDesktopStream(sessionId);
+    const RTCView = React.useMemo(loadRTCView, []);
+    const InputView = React.useMemo(loadInputView, []);
     void revision;
 
     React.useEffect(() => {
@@ -36,6 +96,20 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
         const hide = Keyboard.addListener('keyboardWillHide', () => setKeyboardHeight(0));
         return () => { show.remove(); hide.remove(); };
     }, []);
+    React.useEffect(() => {
+        // A held drag belongs to the old meaning: let go of the button rather
+        // than release it somewhere the finger never meant.
+        const held = gesture.current;
+        if (held) {
+            stopTimer();
+            if (held.mode === 'drag') send({ kind: 'release_all' });
+            held.mode = 'spent';
+        }
+        flushWheel(true);
+        if (gestures === 'device') setCursor(null);
+        // `send` and `flushWheel` are stable per render; this runs on profile change.
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [gestures]);
     React.useEffect(() => {
         if (!sessionId) return;
         return registerDesktopView(sessionId, {
@@ -62,10 +136,26 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
         if (!clamp && (px < 0 || py < 0 || px >= size.width || py >= size.height)) return null;
         return { x: Math.max(0, Math.min(size.width - 1, px)), y: Math.max(0, Math.min(size.height - 1, py)) };
     };
+    /** Hold the offset to the picture's edges at a zoom level. */
+    const clampOffset = (x: number, y: number, atScale: number = scale): Point => {
+        const maxX = Math.max(0, (size.width * atScale - bounds.width) / 2);
+        const maxY = Math.max(0, (size.height * atScale - visibleHeight) / 2);
+        return { x: Math.max(-maxX, Math.min(maxX, x)), y: Math.max(-maxY, Math.min(maxY, y)) };
+    };
     const send = (control: Record<string, unknown>) => { if (sessionId) nativeDesklink.sendControl(sessionId, JSON.stringify(control)); };
-    const pointer = (phase: string, at: Point, button?: number) => {
+    const flushWheel = (force: boolean) => {
+        const { x, y } = wheel.current;
+        if (!force && Math.abs(x) < MIN_WHEEL_STEP && Math.abs(y) < MIN_WHEEL_STEP) return;
+        if (!x && !y) return;
+        send({ kind: 'wheel', dx: Math.max(-MAX_WHEEL_STEP, Math.min(MAX_WHEEL_STEP, x)), dy: Math.max(-MAX_WHEEL_STEP, Math.min(MAX_WHEEL_STEP, y)) });
+        wheel.current = { x: 0, y: 0 };
+    };
+    const pointer = (phase: string, at: Point, button?: number, mark = true) => {
         send({ kind: 'pointer', phase, x: at.x, y: at.y, ...(button ? { button } : {}) });
-        setCursor(at);
+        // A device screen shows the touch itself; a cursor would be a second finger that lies.
+        // A trackpad or mouse has the system's own pointer on screen.
+        if (!mark) setCursor(null);
+        else if (gestures !== 'device') setCursor(at);
     };
     const click = (at: Point, button = 1) => { pointer('down', at, button); pointer('up', at, button); };
     const stopTimer = () => { if (longPress.current) clearTimeout(longPress.current); longPress.current = null; };
@@ -78,8 +168,24 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
         if (!enabled) return;
         const touches = event.nativeEvent.touches;
         if (touches.length !== 1) return;
+        // A trackpad or mouse press the native view already reported is its own.
+        const stamp = touches[0].timestamp;
+        if (pressStamps.current.some((pressStamp) => Math.abs(pressStamp - stamp) < 1)) { gesture.current = null; return; }
         const at = { x: touches[0].locationX, y: touches[0].locationY };
-        gesture.current = { start: at, last: at, time: Date.now(), mode: 'pending', span: 0, focus: at };
+        // A device screen is pressed, not pointed at: the finger lands with the
+        // button down. A tap is the same press lifted without moving, and a
+        // hold holds — there is no long press to arm.
+        if (gestures === 'device') {
+            const from = point(at.x, at.y);
+            gesture.current = { stamp, start: at, last: at, time: Date.now(), mode: from ? 'drag' : 'spent', span: 0, focus: at };
+            if (from) pointer('down', from, 1);
+            return;
+        }
+        // Off the picture there is no pointer to carry: the finger waits for a
+        // second one, and one finger alone is ignored. A zoomed picture pans
+        // from anywhere, clamped to its edges.
+        const onPicture = point(at.x, at.y) !== null;
+        gesture.current = { stamp, start: at, last: at, time: Date.now(), mode: onPicture || zoom > 1 ? 'pending' : 'letterbox', span: 0, focus: at };
         stopTimer();
         longPress.current = setTimeout(() => {
             if (gesture.current?.mode === 'pending' && point(at.x, at.y)) gesture.current.mode = 'armed';
@@ -88,7 +194,7 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
     const move = (event: GestureResponderEvent) => {
         if (!enabled) return;
         const g = gesture.current;
-        if (!g) return;
+        if (!g || !scale) return;
         const touches = event.nativeEvent.touches;
         if (touches.length === 2) {
             stopTimer();
@@ -96,34 +202,75 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
             const distance = span(touches);
             if (g.mode !== 'two' && g.mode !== 'pinch' && g.mode !== 'scroll') {
                 if (g.mode === 'drag') send({ kind: 'release_all' });
+                if (g.mode === 'letterbox' && !touches.some((touch) => point(touch.locationX, touch.locationY))) {
+                    // The letterbox is not the desktop: only a second finger
+                    // on the picture starts a gesture.
+                    g.mode = 'spent';
+                    return;
+                }
                 g.mode = 'two'; g.span = distance; g.focus = center; g.start = center;
+                wheel.current = { x: 0, y: 0 };
                 return;
             }
             if (g.mode === 'two') {
                 if (Math.abs(distance - g.span) > 12) g.mode = 'pinch';
-                else if (Math.hypot(center.x - g.start.x, center.y - g.start.y) > 8) g.mode = 'scroll';
+                else if (Math.hypot(center.x - g.start.x, center.y - g.start.y) > 8) {
+                    g.mode = 'scroll';
+                    // The desktop scrolls whatever is under its pointer.
+                    const under = point(center.x, center.y);
+                    if (under) pointer('move', under);
+                }
             }
             if (g.mode === 'pinch' && g.span) {
-                setZoom((value) => Math.max(1, Math.min(2.5 / fit, value * distance / g.span)));
+                const maxZoom = fit > 0 ? 2.5 / fit : 2.5;
+                const next = Math.max(1, Math.min(maxZoom, zoom * distance / g.span));
+                const applied = zoom > 0 ? next / zoom : 1;
+                // The desktop point under the focus stays under it while the
+                // offset follows the fingers, then everything clamps back.
+                const nextScale = fit * next;
+                const ox = (bounds.width - size.width * scale) / 2 + offset.x;
+                const oy = (visibleHeight - size.height * scale) / 2 + offset.y;
+                const baseX = (bounds.width - size.width * nextScale) / 2;
+                const baseY = (visibleHeight - size.height * nextScale) / 2;
+                setZoom(next);
+                setOffset(clampOffset(
+                    center.x - (center.x - ox) * applied - baseX + (center.x - g.focus.x),
+                    center.y - (center.y - oy) * applied - baseY + (center.y - g.focus.y),
+                    nextScale,
+                ));
             } else if (g.mode === 'scroll') {
-                send({ kind: 'wheel', dx: -(center.x - g.focus.x) / scale / 120, dy: -(center.y - g.focus.y) / scale / 120 });
+                // Content follows the fingers: moving them up scrolls the page down.
+                wheel.current.x += -(center.x - g.focus.x) / scale / PIXELS_PER_DETENT;
+                wheel.current.y += -(center.y - g.focus.y) / scale / PIXELS_PER_DETENT;
+                flushWheel(false);
             }
-            g.span = distance; g.focus = center;
+            g.span = distance; g.focus = center; g.last = center;
             return;
         }
-        if (touches.length !== 1 || g.mode === 'spent' || g.mode === 'two' || g.mode === 'pinch' || g.mode === 'scroll') return;
+        if (touches.length !== 1 || g.mode === 'spent' || g.mode === 'letterbox' || g.mode === 'two' || g.mode === 'pinch' || (g.mode === 'scroll' && gestures !== 'browser')) return;
         const at = { x: touches[0].locationX, y: touches[0].locationY };
         if (g.mode === 'pending' && Math.hypot(at.x - g.start.x, at.y - g.start.y) > 8) {
             stopTimer();
-            g.mode = zoom > 1 ? 'pan' : 'hover';
+            // A page scrolls under one finger; the desktop's pointer rides along
+            // so the scroll lands where the finger is.
+            if (gestures === 'browser') {
+                g.mode = 'scroll';
+                const under = point(at.x, at.y);
+                if (under) pointer('move', under);
+            } else g.mode = zoom > 1 ? 'pan' : 'hover';
+        }
+        if (g.mode === 'scroll') {
+            // Content follows the finger: moving it up scrolls the page down.
+            wheel.current.x += -(at.x - g.last.x) / scale / PIXELS_PER_DETENT;
+            wheel.current.y += -(at.y - g.last.y) / scale / PIXELS_PER_DETENT;
+            flushWheel(false);
+            g.last = at; g.focus = at;
+            return;
         }
         if (g.mode === 'pan') {
             const dx = at.x - g.last.x;
             const dy = at.y - g.last.y;
-            setOffset((current) => ({
-                x: Math.max(-Math.max(0, (pictureWidth - bounds.width) / 2), Math.min(Math.max(0, (pictureWidth - bounds.width) / 2), current.x + dx)),
-                y: Math.max(-Math.max(0, (pictureHeight - visibleHeight) / 2), Math.min(Math.max(0, (pictureHeight - visibleHeight) / 2), current.y + dy)),
-            }));
+            setOffset((current) => clampOffset(current.x + dx, current.y + dy));
         }
         if (g.mode === 'armed' && Math.hypot(at.x - g.start.x, at.y - g.start.y) > 8) {
             const from = point(g.start.x, g.start.y, true);
@@ -140,6 +287,8 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
         stopTimer();
         const g = gesture.current;
         if (!g) return;
+        // Leftover wheel below the send threshold still belongs to this scroll.
+        if (g.mode === 'scroll') flushWheel(true);
         if (g.mode === 'two' && Date.now() - g.time < 350) {
             const at = point(g.focus.x, g.focus.y);
             if (at) click(at, 3);
@@ -175,18 +324,86 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
         }
         setKeyboardText(keyboardTextRef.current);
     };
+    const onKey = ({ nativeEvent }: NativeEvent<HardwareKey>) => {
+        if (!enabled || !sessionId) return;
+        const action = hardwareKey(nativeEvent, chordsDown.current, captured.current);
+        if (action === null) return;
+        if ('send' in action) send(action.send);
+        else emitDesktopKeyboard(sessionId, action.capture);
+    };
+    const onPointer = ({ nativeEvent: { phase, x, y, buttons = 0, timestamp = NaN } }: NativeEvent<PointerEvent>) => {
+        if (!enabled) return;
+        if (phase === 'hover') {
+            const at = point(x, y);
+            if (at) pointer('move', at, undefined, false);
+            return;
+        }
+        if (phase === 'down') {
+            pressStamps.current = [...pressStamps.current.slice(-7), timestamp];
+            const button = pressedButton(buttons);
+            // Touch may have seen this press first: it is the pointer's, not a finger's.
+            const g = gesture.current;
+            let held = false;
+            if (g && Math.abs(g.stamp - timestamp) < 1) {
+                stopTimer();
+                gesture.current = null;
+                if (g.mode === 'drag') {
+                    const from = point(g.start.x, g.start.y, true);
+                    held = button === 1;
+                    if (!held && from) pointer('up', from, 1, false);
+                }
+            }
+            // The letterbox is not the desktop: a press there does nothing.
+            const at = point(x, y);
+            if (!at) return;
+            pressed.current = button;
+            if (!held) pointer('down', at, button, false);
+            return;
+        }
+        const button = pressed.current;
+        if (button === null) return;
+        const at = point(x, y, true);
+        if (phase === 'move') { if (at) pointer('move', at, button, false); return; }
+        pressed.current = null;
+        if (at) pointer('up', at, button, false); else send({ kind: 'release_all' });
+    };
+    const onWheel = ({ nativeEvent: { phase, dx, dy, x, y } }: NativeEvent<WheelEvent>) => {
+        if (!enabled || !scale) return;
+        if (phase === 'end') { flushWheel(true); return; }
+        if (phase === 'begin') {
+            // The desktop scrolls whatever is under its pointer.
+            const under = point(x, y);
+            if (under) pointer('move', under, undefined, false);
+        }
+        // Content follows the fingers, as with two fingers on the screen.
+        wheel.current.x += -dx / scale / PIXELS_PER_DETENT;
+        wheel.current.y += -dy / scale / PIXELS_PER_DETENT;
+        flushWheel(false);
+    };
     const onLayout = (event: LayoutChangeEvent) => setBounds(event.nativeEvent.layout);
-    return <View style={[styles.surface, style]} onLayout={onLayout} accessibilityLabel={accessibilityLabel}
-        onStartShouldSetResponder={() => enabled} onMoveShouldSetResponder={() => enabled}
-        onResponderGrant={start} onResponderMove={move} onResponderRelease={end}
-        onResponderTerminate={() => { stopTimer(); if (gesture.current?.mode === 'drag') send({ kind: 'release_all' }); gesture.current = null; }}>
-        {sessionId && stream && size.width > 0 && size.height > 0 ?
+    const surface: ViewProps = {
+        style: [styles.surface, style], onLayout, accessible: accessibilityLabel !== undefined, accessibilityLabel,
+        onStartShouldSetResponder: () => enabled, onMoveShouldSetResponder: () => enabled,
+        onResponderGrant: start, onResponderMove: move, onResponderRelease: end,
+        onResponderTerminate: () => { stopTimer(); if (gesture.current?.mode === 'drag') send({ kind: 'release_all' }); gesture.current = null; },
+    };
+    const children = <>
+        {sessionId && stream && RTCView && size.width > 0 && size.height > 0 ?
             <RTCView pointerEvents="none" streamURL={stream.toURL()} objectFit="contain" style={{ position: 'absolute', left: originX, top: originY, width: pictureWidth, height: pictureHeight }}
                 onDimensionsChange={(event) => { if (event.nativeEvent.width > 0 && event.nativeEvent.height > 0) markDesktopPresented(sessionId); }} /> : placeholder}
-        {cursor && <View pointerEvents="none" style={[styles.cursor, { left: originX + (cursor.x + 0.5) * scale, top: originY + (cursor.y + 0.5) * scale }]} />}
-        <TextInput ref={input} style={styles.input} value={keyboardText} onChangeText={changeText} autoCorrect={false} autoCapitalize="none" submitBehavior="submit"
-            onSubmitEditing={() => { send({ kind: 'key', name: 'Enter', modifiers: [], down: true }); send({ kind: 'key', name: 'Enter', modifiers: [], down: false }); }} />
-    </View>;
+        {cursor && <View pointerEvents="none" accessible={false} style={[styles.cursor, { left: originX + (cursor.x + 0.5) * scale - CURSOR_HOTSPOT, top: originY + (cursor.y + 0.5) * scale - CURSOR_HOTSPOT }]} />}
+        <TextInput ref={input} style={styles.input} value={keyboardText} onChangeText={changeText} autoCorrect={false} autoCapitalize="none" submitBehavior="submit" accessible={false}
+            onSubmitEditing={() => {
+                // While the app holds a sticky modifier, Enter is the app's to
+                // chord: it goes through capture like every other key, so
+                // Ctrl+Enter reaches the desktop instead of typing a newline.
+                if (captured.current && sessionId) emitDesktopKeyboard(sessionId, { key: 'Enter' });
+                else { send({ kind: 'key', name: 'Enter', modifiers: [], down: true }); send({ kind: 'key', name: 'Enter', modifiers: [], down: false }); }
+            }} />
+    </>;
+    return InputView
+        ? <InputView {...surface} enabled={enabled} onKey={onKey} onPointer={onPointer} onWheel={onWheel}>{children}</InputView>
+        : <View {...surface}>{children}</View>;
 }
 
 const styles = StyleSheet.create({

package/src/DesktopView.tsx

@@ -3,6 +3,7 @@ import * as React from 'react';
 import { StyleSheet, View, type StyleProp, type ViewStyle } from 'react-native';
 
 import { desktopAvailable } from './native';
+import type { GestureProfile } from './protocol';
 
 export interface DesktopViewProps {
     /** The session handle from `useDesktopSession`. */
@@ -18,6 +19,11 @@ export interface DesktopViewProps {
      * following the keyboard as it moves.
      */
     keyboardClearance?: number;
+    /**
+     * Which touch meaning the surface uses: a desktop pointer, a browser page
+     * (one finger scrolls), or a device screen (one finger presses and drags).
+     */
+    gestures?: GestureProfile;
 }
 
 interface NativeSurfaceProps {
@@ -26,6 +32,7 @@ interface NativeSurfaceProps {
     accessible?: boolean;
     accessibilityLabel?: string;
     keyboardClearance?: number;
+    gestures?: GestureProfile;
 }
 
 // The Expo view is resolved at module load; on a platform without it the
@@ -42,7 +49,7 @@ const NativeSurface: React.ComponentType<NativeSurfaceProps> | null = desktopAva
  * belong to the application, which mounts this wherever it wants the desktop
  * to appear.
  */
-export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0 }: DesktopViewProps) {
+export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, gestures = 'desktop' }: DesktopViewProps) {
     if (NativeSurface == null || sessionId == null) {
         return (
             <View style={[styles.surface, style]}>
@@ -50,7 +57,7 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
             </View>
         );
     }
-    return <NativeSurface style={[styles.surface, style]} sessionId={sessionId} accessible={accessibilityLabel !== undefined} accessibilityLabel={accessibilityLabel} keyboardClearance={keyboardClearance} />;
+    return <NativeSurface style={[styles.surface, style]} sessionId={sessionId} accessible={accessibilityLabel !== undefined} accessibilityLabel={accessibilityLabel} keyboardClearance={keyboardClearance} gestures={gestures} />;
 }
 
 const styles = StyleSheet.create({

package/src/DesktopView.web.tsx

@@ -1,6 +1,8 @@
 import * as React from 'react';
 import { StyleSheet, View, type StyleProp, type ViewStyle } from 'react-native';
 
+import type { GestureProfile } from './protocol';
+
 export interface DesktopViewProps {
     /** The session handle from `useDesktopSession`. */
     sessionId: string | null;
@@ -14,6 +16,11 @@ export interface DesktopViewProps {
      * controls, in points. While the keyboard is up the picture sits above both.
      */
     keyboardClearance?: number;
+    /**
+     * Which touch meaning the surface uses: a desktop pointer, a browser page
+     * (one finger scrolls), or a device screen (one finger presses and drags).
+     */
+    gestures?: GestureProfile;
 }
 
 /**
@@ -25,19 +32,24 @@ export interface DesktopViewProps {
  * actually mounted rather than at module load, so an application that never
  * opens a desktop does not carry one in its first paint.
  */
-export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0 }: DesktopViewProps) {
+export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, gestures = 'desktop' }: DesktopViewProps) {
     const mounted = React.useCallback(
         (node: unknown) => {
             const element = (node as HTMLElement | null) ?? null;
             if (sessionId === null) return;
             void import('./native').then((platform) => {
                 if (element === null || !platform.desktopAvailable) return;
-                platform.attachSurface(sessionId, element);
+                platform.attachSurface(sessionId, element, gestures);
             });
         },
         [sessionId],
     );
 
+    React.useEffect(() => {
+        if (sessionId === null) return;
+        void import('./native').then((platform) => platform.setGestures(sessionId, gestures));
+    }, [sessionId, gestures]);
+
     React.useEffect(() => {
         if (sessionId === null) return;
         void import('./native').then((platform) => platform.setKeyboardClearance(sessionId, keyboardClearance));

package/src/hardwareKeys.ts

@@ -0,0 +1,61 @@
+import type { ControlMessage } from './protocol';
+
+/** A hardware key as the iOS input view reports it: its HID usage, the characters it types, and what is held. */
+export type HardwareKey = { usage: number; down: boolean; characters: string; modifiers: string[] };
+
+/** What a key press does: a control message, or a key or text for the session's sticky modifiers. */
+export type HardwareKeyAction = { send: ControlMessage } | { capture: { key: string } | { text: string } };
+
+/** Keys with no character of their own, by HID keyboard usage, in the engine's names. */
+const NAMED: Record<number, string> = {
+    0x28: 'Enter', 0x29: 'Escape', 0x2a: 'Backspace', 0x2b: 'Tab', 0x58: 'Enter',
+    0x3a: 'F1', 0x3b: 'F2', 0x3c: 'F3', 0x3d: 'F4', 0x3e: 'F5', 0x3f: 'F6',
+    0x40: 'F7', 0x41: 'F8', 0x42: 'F9', 0x43: 'F10', 0x44: 'F11', 0x45: 'F12',
+    0x4a: 'Home', 0x4b: 'PageUp', 0x4c: 'Delete', 0x4d: 'End', 0x4e: 'PageDown',
+    0x4f: 'ArrowRight', 0x50: 'ArrowLeft', 0x51: 'ArrowDown', 0x52: 'ArrowUp',
+    0xe0: 'Control', 0xe1: 'Shift', 0xe2: 'Alt', 0xe3: 'Meta',
+    0xe4: 'ControlRight', 0xe5: 'ShiftRight', 0xe6: 'AltGraph', 0xe7: 'MetaRight',
+};
+const MODIFIERS = new Set(['Control', 'ControlRight', 'Shift', 'ShiftRight', 'Alt', 'AltGraph', 'Meta', 'MetaRight']);
+
+/** The letter or digit a key stands for where it sits, whatever the layout types: a chord names a key. */
+function chordCharacter(usage: number): string | undefined {
+    if (usage >= 0x04 && usage <= 0x1d) return String.fromCharCode(0x61 + usage - 0x04);
+    if (usage >= 0x1e && usage <= 0x26) return String(usage - 0x1d);
+    if (usage === 0x27) return '0';
+    return undefined;
+}
+
+/** Control characters and the private-use codes UIKit gives keys like F13 type nothing. */
+const TYPES_NOTHING = /[\u0000-\u001f\u007f-]/;
+
+/**
+ * Turn one hardware key into what the desktop receives, as the web and Android
+ * receivers do: named keys and modifiers travel as key events with what is
+ * held, Control and Command chords as the key's own letter or digit, and
+ * anything else as the text it types. `chordsDown` remembers the chord keys
+ * still down, so a chord's key-up is sent even when its modifier let go first.
+ */
+export function hardwareKey(key: HardwareKey, chordsDown: Set<number>, captured: boolean): HardwareKeyAction | null {
+    const { usage, down, modifiers } = key;
+    const name = NAMED[usage];
+    if (name !== undefined) {
+        // A plain key pressed while a sticky modifier waits is the session's to
+        // chord. Only the press is held back: a release still goes, so a key
+        // that was down before the capture cannot stick.
+        if (captured && down && modifiers.length === 0 && !MODIFIERS.has(name)) return { capture: { key: name } };
+        return { send: { kind: 'key', name, modifiers, down } };
+    }
+    const character = chordCharacter(usage);
+    if (character !== undefined) {
+        // Alt alone is not a chord: Option composes a character with the layout.
+        if ((modifiers.includes('Control') || modifiers.includes('Meta')) && (down || chordsDown.has(usage))) {
+            if (down) chordsDown.add(usage);
+            else chordsDown.delete(usage);
+            return { send: { kind: 'key', character, modifiers, down } };
+        }
+        if (!down && chordsDown.delete(usage)) return { send: { kind: 'key', character, modifiers: [], down: false } };
+    }
+    if (!down || key.characters === '' || TYPES_NOTHING.test(key.characters)) return null;
+    return captured ? { capture: { text: key.characters } } : { send: { kind: 'text', text: key.characters } };
+}

package/src/native.ios.ts

@@ -1,5 +1,18 @@
-import { MediaStream, RTCPeerConnection } from 'react-native-webrtc';
+import type { MediaStream, RTCPeerConnection } from 'react-native-webrtc';
 import type { NativeDesklinkModule, NativeSessionEvent } from './native';
+import { requireWebRTC } from './webrtc.ios';
+
+/** The WebRTC stack, loaded the first time a session opens rather than with this module. */
+function loadWebRTC(): { RTCPeerConnection: new (config: unknown) => RTCPeerConnection; MediaStream: new () => MediaStream } | null {
+    try {
+        return requireWebRTC() as {
+            RTCPeerConnection: new (config: unknown) => RTCPeerConnection;
+            MediaStream: new () => MediaStream;
+        };
+    } catch {
+        return null;
+    }
+}
 
 type Session = {
     peer: RTCPeerConnection;
@@ -8,6 +21,7 @@ type Session = {
     input: boolean;
     seq: number;
     closed: boolean;
+    presented: boolean;
     candidates: Array<{ candidate: string; sdpMid: string | null; sdpMLineIndex: number }>;
     remoteSet: boolean;
 };
@@ -18,6 +32,13 @@ const surfaces = new Set<() => void>();
 const sizes = new Map<string, { width: number; height: number }>();
 let nextId = 0;
 
+// RTCView's Metal renderer draws a frame only when its timestamp differs from
+// the last one drawn, and a receiver honouring the engine's zero playout delay
+// stamps every decoded frame with a render time of 0: the view would keep its
+// first frame, or none, while frames go on decoding. Declining the extension
+// keeps real timestamps, at the cost of the receiver's usual jitter buffer.
+const PLAYOUT_DELAY = /^a=extmap:\d+(?:\/\w+)? http:\/\/www\.webrtc\.org\/experiments\/rtp-hdrext\/playout-delay\r?\n/gm;
+
 function emit(sessionId: string, name: NativeSessionEvent['name'], payload: Record<string, unknown> = {}) {
     if (!sessions.has(sessionId) && name !== 'closed') return;
     for (const listener of listeners) listener({ sessionId, name, payload });
@@ -41,6 +62,11 @@ export function observeDesktopSurface(listener: () => void): () => void {
 }
 
 export function markDesktopPresented(id: string) {
+    // A resize or a remount must not re-mark: readiness is the first frame,
+    // and a second mark during `reconnecting` would wrongly read as `live`.
+    const session = sessions.get(id);
+    if (!session || session.presented) return;
+    session.presented = true;
     emit(id, 'presented');
 }
 
@@ -48,6 +74,26 @@ export function desktopInputEnabled(id: string | null): boolean {
     return id != null && sessions.get(id)?.input === true;
 }
 
+/**
+ * The offer with H.264 moved ahead of the other video codecs. For a captured
+ * desktop the engine offers VP9 first and H.264 beside it, and codes whichever
+ * the answer lists first; an answer keeps the offer's order. iOS's WebRTC
+ * decodes H.264 in hardware and VP9 in software, so this receiver asks for
+ * H.264 by reordering the offer it applies. An offer without H.264 is unchanged.
+ */
+export function preferH264(sdp: string): string {
+    const lines = sdp.split('\r\n');
+    const start = lines.findIndex((line) => line.startsWith('m=video '));
+    if (start === -1) return sdp;
+    const next = lines.findIndex((line, index) => index > start && line.startsWith('m='));
+    const section = lines.slice(start + 1, next === -1 ? undefined : next);
+    const h264 = new Set(section.flatMap((line) => /^a=rtpmap:(\d+) H264\//i.exec(line)?.[1] ?? []));
+    const [media, port, proto, ...formats] = lines[start].split(' ');
+    const ordered = [...formats.filter((format) => h264.has(format)), ...formats.filter((format) => !h264.has(format))];
+    lines[start] = [media, port, proto, ...ordered].join(' ');
+    return lines.join('\r\n');
+}
+
 function failure(id: string, error: unknown) {
     emit(id, 'failure', { code: 'transport', message: error instanceof Error ? error.message : String(error) });
 }
@@ -56,8 +102,10 @@ export const nativeDesklink: NativeDesklinkModule = {
     createSession(iceServersJson) {
         const id = `desklink-${++nextId}`;
         try {
-            const peer = new RTCPeerConnection({ iceServers: JSON.parse(iceServersJson) });
-            const session: Session = { peer, channel: null, stream: null, input: false, seq: 0, closed: false, candidates: [], remoteSet: false };
+            const webrtc = loadWebRTC();
+            if (!webrtc) return null;
+            const peer = new webrtc.RTCPeerConnection({ iceServers: JSON.parse(iceServersJson) });
+            const session: Session = { peer, channel: null, stream: null, input: false, seq: 0, closed: false, presented: false, candidates: [], remoteSet: false };
             sessions.set(id, session);
             peer.onicecandidate = (event) => {
                 if (event.candidate) emit(id, 'candidate', {
@@ -72,7 +120,7 @@ export const nativeDesklink: NativeDesklinkModule = {
             };
             peer.ontrack = (event) => {
                 if (session.closed || event.track.kind !== 'video') return;
-                const stream = event.streams[0] ?? new MediaStream();
+                const stream = event.streams[0] ?? new webrtc.MediaStream();
                 if (event.streams.length === 0) stream.addTrack(event.track);
                 session.stream = stream;
                 notifySurface();
@@ -98,7 +146,7 @@ export const nativeDesklink: NativeDesklinkModule = {
         if (type !== 'offer') { failure(id, 'the engine must send an offer'); return false; }
         void (async () => {
             try {
-                await session.peer.setRemoteDescription({ type: 'offer', sdp });
+                await session.peer.setRemoteDescription({ type: 'offer', sdp: preferH264(sdp.replace(PLAYOUT_DELAY, '')) });
                 if (session.closed) return;
                 session.remoteSet = true;
                 for (const candidate of session.candidates.splice(0)) await session.peer.addIceCandidate(candidate);