@desklink/react-native 0.1.0 → 0.2.0

package/README.md

@@ -16,14 +16,11 @@ repository it resolves through the workspace; anywhere else, install it from npm
 npx expo install @desklink/react-native      # or yarn add, then prebuild/rebuild
 ```
 
-Android as a native module, plus the browser, which brings its own WebRTC.
-`expo-module.config.json` declares the native side honestly rather than
-compiling an untested stub for other platforms; `ios/DEFERRED.md` lists exactly
-what an iOS backend has to implement.
-
-It compiles against the WebRTC binding the app already ships
-(`react-native-webrtc`) so there is one `org.webrtc` copy in the binary, and it
-never touches that binding's process-global decoder factory.
+Android uses an Expo native module; iOS uses the app's existing
+`react-native-webrtc` peer connection and `RTCView` from JavaScript. Both need a
+dev client or native build containing that binding. Web uses the browser's WebRTC.
+No second WebRTC binary is linked. The Android module has its own hardware-first
+decoder factory; iOS uses the decoder shipped by `react-native-webrtc`.
 
 ## Use
 
@@ -44,7 +41,7 @@ await desktop.connect();
 // On a deliberate control action, once the picture is live:
 desktop.setInputEnabled(true);
 desktop.showKeyboard();
-desktop.setOrientation('landscape');   // Android: hold the screen on its side; 'auto' to follow the phone
+desktop.setOrientation('landscape');   // Android only; iOS apps configure supported orientations themselves
 await desktop.pasteLocalToRemote(await Clipboard.getStringAsync());
 ```
 
@@ -61,29 +58,30 @@ import { desktopAvailable } from '@desklink/react-native/availability';
 
 ### What the package guarantees
 
-- **Hardware-first decoding.** The session builds its own peer connection with
-  a hardware-first decoder factory for VP9, falling back to software on phones
-  without a VP9 hardware decoder. No other WebRTC user in the app is affected.
+- **VP9 decoding.** Android builds a session-scoped hardware-first decoder
+  factory with software fallback. iOS uses the app's `react-native-webrtc`
+  decoder; neither platform links a second WebRTC stack.
 - **A sharp, zoomable picture.** The desktop fits the view by default; a pinch
   zooms up to 2.5 view pixels per desktop pixel and one finger moves around the
   zoomed desktop. On Android the decoded frame is copied once into the view's
   own texture and drawn with a multi-tap filter when it is shown smaller than
   its size, so a fitted 4K desktop does not alias and a pinch redraws at once.
-  `fitToView()` shows the whole desktop again.
+  iOS uses `RTCView`'s native renderer. `fitToView()` shows the whole desktop again.
 - **A pointer a phone can see.** Once a touch has sent the desktop's pointer
   somewhere, the view draws it there at a readable size, over a picture whose
-  own cursor is a few pixels tall or not captured at all. A mouse keeps its own.
+  own cursor is a few pixels tall or not captured at all. Android hides the
+  extra mark when a mouse is attached, and the `device` profile draws no mark
+  on any platform — the screen being touched shows the finger itself.
 - **The picture above the keyboard.** While the phone's keyboard is up, the
   picture sits above it and above the room the app keeps for its own controls
-  (`keyboardClearance`), moving with the keyboard as it slides. A picture too
-  tall for what is left keeps the pointer, where a tap just put the caret, in
-  sight.
+  (`keyboardClearance`). Android follows the keyboard animation and keeps the
+  pointer in sight; iOS follows the keyboard show/hide bounds.
 - **Contained geometry.** Touch maps through the picture's actual placement.
   A one-finger drag starting in the letterbox does not move the pointer; a
   second finger on the picture can still start a pinch or scroll. A drag that
   leaves the picture is held to its edge rather than released somewhere unseen.
-- **The gestures remote-desktop viewers settled on,** decided natively with a
-  slop threshold: tap to click, and a second tap close by is a double click on
+- **The gestures remote-desktop viewers settled on,** with a slop threshold:
+  tap to click, and a second tap close by is a double click on
   the same point; hold and release for a right click where the finger rested
   (every desktop app's context menu); hold then drag for the left button
   (select text, move a window); on the whole desktop a finger that moves
@@ -91,14 +89,48 @@ import { desktopAvailable } from '@desklink/react-native/availability';
   under the finger while the picture catches up; two fingers scroll the desktop
   under them, or pinch; a quick two-finger tap is a right click too. Scrolling
   is fractional wheel steps, smooth where the desktop supports high-resolution
-  wheels. Under congestion, only the latest unsent pointer position is kept;
-  other input follows that position in order.
+  wheels. Android coalesces pending pointer moves under congestion. The
+  `gestures` prop reshapes one finger for what the picture shows, and can be
+  changed mid-session: `browser` scrolls the page under one finger (a tap
+  still clicks and a hold still right-clicks), `device` presses the screen —
+  the finger lands with the button down, drags with it held and lifts to
+  release, so a tap clicks and a hold holds with no long-press right click,
+  and no drawn pointer — while the default `desktop` is the trackpad above.
+  Two fingers scroll and pinch the same under every profile.
 - **The keys a phone lacks.** `modifiers`, `tapModifier` and `pressKey` give
   sticky Ctrl and Shift: tap arms one for the next key, tap again locks it.
   While one is armed, the next key or character the phone's keyboard types is
   sent as that key's chord, so Ctrl then v is Ctrl+V. The app draws the keys.
-- **Readiness is a rendered frame.** `status: 'live'` is set by the first frame
-  actually presented, not by a track arriving or ICE connecting.
+- **Readiness is a rendered frame.** Android marks the first draw; iOS uses
+  `RTCView`'s native video-dimensions callback, which fires after a decoded
+  picture arrives. Neither marks a track or ICE connection as live.
+- **Connection states, as they happen.** The session status is `idle`,
+  `opening`, `connecting`, `live`, `reconnecting`, `ended` or `failed`. The
+  peer's ICE state and the engine's `session.state` both feed it: a
+  `disconnected` path reads as `reconnecting` within about a second of the
+  drop — never a frozen picture that still says live — and a recovered path
+  reads as `live` again once ICE is `connected`. `failed` stays terminal.
+  Every event carries its engine session id where the carrier preserves it,
+  and the session ignores anything naming a session it no longer holds, so a
+  previous generation's queued offer or revocation cannot corrupt or kill a
+  healthy recovery.
+  Detection is two-layered: ICE and engine `session.state` events, plus the
+  engine's twice-a-second control-channel heartbeat (`{"kind":"ping"}`) — a
+  path that stops heartbeating reads as stalled within ~1.5 s even while ICE
+  consent timers are still making up their minds, and resumed heartbeats read
+  as recovered. Clients on engines that predate the heartbeat simply never
+  arm that watchdog.
+- **ICE restarts before reopens.** When the path drops after frames were
+  shown, the session asks the engine (`session.restart_ice`) to re-offer on
+  the same peer connection: the fresh offer arrives as the usual
+  `session.description` event and is answered as usual, all without reopening
+  the session. The first restart waits ~2 s for blips that heal alone;
+  retries back off across ~60 s, so a 30–60 s outage still recovers when the
+  network returns. Only a terminal `failed` reopens the session, and those
+  reopens back off across ~60 s too instead of spending one attempt while the
+  network is still down. Renegotiation transients from a restart the session
+  asked for itself are expected, not a new outage: they neither flip the
+  status nor trigger another restart for a short grace window.
 - **Control starts off.** `setInputEnabled(true)` enables pointer, keyboard and
   clipboard input; `setInputEnabled(false)` blocks new input, releases held keys
   and buttons, and resets pending gestures. The app decides when to enable it
@@ -122,9 +154,9 @@ interface Signaling {
 ```
 
 `method` is one of `session.open`, `session.description`, `session.candidate`,
-`session.close`; `SessionEvent` is the engine's offer, candidates, state and
-revocation. Pixels and input do **not** use this channel — they are the
-desktop's own WebRTC session.
+`session.restart_ice`, `session.close`; `SessionEvent` is the engine's offer,
+candidates, state and revocation. Pixels and input do **not** use this
+channel — they are the desktop's own WebRTC session.
 
 ## Licence
 

package/android/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'com.android.library'
 apply plugin: 'kotlin-android'
 
 group = 'expo.modules.desklink'
-version = '0.1.0'
+version = '0.1.1'
 
 android {
   namespace 'expo.modules.desklink'

package/android/src/main/java/expo/modules/desklink/DesklinkModule.kt

@@ -126,6 +126,10 @@ class DesklinkModule : Module() {
         view.setKeyboardClearance(clearance ?: 0f)
       }
 
+      Prop("gestures") { view: DesktopView, gestures: String? ->
+        view.setGestures(gestures ?: "desktop")
+      }
+
       OnViewDestroys { view: DesktopView -> view.release() }
 
       OnViewDidUpdateProps { view: DesktopView ->

package/android/src/main/java/expo/modules/desklink/DesktopView.kt

@@ -66,7 +66,9 @@ private const val MIN_WHEEL_STEP = 0.05f
  *  - press and hold: a right click on release, or drag after it to hold the left
  *    button (select text, move a window);
  *  - one finger: move around a zoomed-in desktop; on the whole desktop,
- *    move its pointer, which follows the finger without pressing a button;
+ *    move its pointer, which follows the finger without pressing a button —
+ *    unless `gestures` is `browser`, where it scrolls the page, or `device`,
+ *    where it lands with the button down and drags with it held;
  *  - two fingers: scroll the desktop under them, or pinch to zoom (and move) the
  *    picture; a quick two-finger tap is a right click.
  */
@@ -81,6 +83,23 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
   private var surfaceWidth = 0
   private var surfaceHeight = 0
 
+  /** Which touch meaning this surface uses; a browser page and a device screen answer one finger differently. */
+  var gestures = "desktop"
+    private set
+
+  fun setGestures(value: String) {
+    if (value == gestures) return
+    // A held drag belongs to the old meaning: let go rather than release the
+    // button somewhere the finger never meant.
+    if (gesture == Gesture.DRAG) session?.sendCancel()
+    cancelGesture()
+    gestures = value
+    if (value == "device" && pointerAt != null) {
+      pointerAt = null
+      placePointer()
+    }
+  }
+
   // The picture's placement: surface pixels per desktop pixel, and where the
   // desktop's top-left corner sits in this view. `fitted` keeps a picture that
   // was showing the whole desktop showing it when the view changes size.
@@ -449,7 +468,8 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
   /** Send the desktop's pointer somewhere, and show it there. */
   private fun pointerTo(active: DesktopSession, phase: String, at: Pair<Int, Int>, withButton: Boolean = false) {
     active.sendPointer(phase, at.first, at.second, withButton)
-    if (!mouseInput) showPointer(at)
+    // A device screen shows the touch itself; a mark would be a second finger that lies.
+    if (!mouseInput && gestures != "device") showPointer(at)
   }
 
   private fun showPointer(at: Pair<Int, Int>) {
@@ -519,6 +539,25 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
     }
     when (event.actionMasked) {
       MotionEvent.ACTION_DOWN -> {
+        // A device screen is pressed, not pointed at: the finger lands with the
+        // button down, drags with it held, and lifts to release. A tap is the
+        // same down and up with no move, so it clicks for free, and a hold
+        // holds for free — there is no long press to arm.
+        if (gestures == "device") {
+          downX = event.x
+          downY = event.y
+          lastX = event.x
+          lastY = event.y
+          val at = point(event.x, event.y)
+          if (at == null) {
+            gesture = Gesture.LETTERBOX
+          } else {
+            gesture = Gesture.DRAG
+            lastTapPoint = null
+            pointerTo(active, "down", at)
+          }
+          return true
+        }
         downOnPicture = point(event.x, event.y) != null
         gesture = if (fitted && !downOnPicture) Gesture.LETTERBOX else Gesture.PENDING
         downX = event.x
@@ -573,6 +612,15 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
         Gesture.PENDING -> if (hypot(event.x - downX, event.y - downY) > touchSlop) {
           removeCallbacks(longPress)
           lastTapPoint = null
+          // A page scrolls under one finger; the desktop's pointer rides along
+          // so the scroll lands where the finger is.
+          if (gestures == "browser") {
+            gesture = Gesture.SCROLL
+            lastX = event.x
+            lastY = event.y
+            point(event.x, event.y)?.let { pointerTo(active, "move", it) }
+            return true
+          }
           // The whole desktop has nowhere to move to, so the finger moves the
           // desktop's pointer instead, without a button.
           gesture = if (!fitted) Gesture.PAN else if (downOnPicture) Gesture.HOVER else Gesture.LETTERBOX
@@ -599,6 +647,15 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
         Gesture.DRAG -> dragTo(active, event.x, event.y)
 
         Gesture.TWO, Gesture.PINCH, Gesture.SCROLL -> if (event.pointerCount >= 2) twoFingers(active, event)
+        else if (gesture == Gesture.SCROLL && gestures == "browser") {
+          // One finger dragging a page: content follows it, in pixel deltas
+          // turned to detents like the two-finger scroll.
+          wheelX -= (event.x - lastX) / scale / PIXELS_PER_DETENT
+          wheelY -= (event.y - lastY) / scale / PIXELS_PER_DETENT
+          lastX = event.x
+          lastY = event.y
+          flushWheel(active, force = false)
+        }
 
         else -> {}
       }
@@ -618,6 +675,7 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
         removeCallbacks(longPress)
         when (gesture) {
           Gesture.PENDING -> tap(active, event.x, event.y)
+          Gesture.SCROLL -> flushWheel(active, force = true)
           Gesture.HOVER -> hoverTo(active, event.x, event.y)
           Gesture.ARMED -> point(downX, downY)?.let { rightClick(active, it) }
           Gesture.DRAG -> endDrag(active, event.x, event.y)
@@ -669,7 +727,7 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
 
   private fun rightClick(active: DesktopSession, at: Pair<Int, Int>) {
     active.sendRightClick(at.first, at.second)
-    if (!mouseInput) showPointer(at)
+    if (!mouseInput && gestures != "device") showPointer(at)
   }
 
   private fun twoFingers(active: DesktopSession, event: MotionEvent) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@desklink/react-native",
-    "version": "0.1.0",
+    "version": "0.2.0",
     "license": "Apache-2.0",
     "description": "Render and drive a desktop session from a React Native app",
     "main": "index.ts",
@@ -29,5 +29,8 @@
         "react": ">=19",
         "react-native": ">=0.83",
         "react-native-webrtc": ">=124"
+    },
+    "scripts": {
+        "test:reconnect": "node test/reconnect-flow.mjs"
     }
 }

package/src/DesktopView.ios.tsx

@@ -0,0 +1,227 @@
+import * as React from 'react';
+import { Keyboard, StyleSheet, TextInput, View, type GestureResponderEvent, type LayoutChangeEvent } from 'react-native';
+import { RTCView } from 'react-native-webrtc';
+import type { DesktopViewProps } from './DesktopView';
+import { keyboardDelta } from './keyboardDelta';
+import { desktopInputEnabled, emitDesktopKeyboard, getDesktopSize, getDesktopStream, markDesktopPresented, nativeDesklink, observeDesktopSurface, registerDesktopView } from './native.ios';
+
+type Point = { x: number; y: number };
+type Gesture = { start: Point; last: Point; time: number; mode: 'pending' | 'hover' | 'pan' | 'armed' | 'drag' | 'two' | 'scroll' | 'pinch' | 'spent'; span: number; focus: Point };
+
+export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, gestures = 'desktop' }: DesktopViewProps) {
+    const [revision, refresh] = React.useReducer((n: number) => n + 1, 0);
+    const [bounds, setBounds] = React.useState({ width: 0, height: 0 });
+    const [size, setSize] = React.useState(() => getDesktopSize(sessionId) ?? { width: 0, height: 0 });
+    const [keyboardHeight, setKeyboardHeight] = React.useState(0);
+    const [zoom, setZoom] = React.useState(1);
+    const [offset, setOffset] = React.useState<Point>({ x: 0, y: 0 });
+    const [cursor, setCursor] = React.useState<Point | null>(null);
+    const [keyboardText, setKeyboardText] = React.useState('');
+    const keyboardTextRef = React.useRef('');
+    const input = React.useRef<TextInput>(null);
+    const captured = React.useRef(false);
+    const gesture = React.useRef<Gesture | null>(null);
+    const longPress = React.useRef<ReturnType<typeof setTimeout> | null>(null);
+    const lastTap = React.useRef<{ time: number; at: Point; point: Point } | null>(null);
+    const [enabled, setEnabled] = React.useState(() => desktopInputEnabled(sessionId));
+    const stream = getDesktopStream(sessionId);
+    void revision;
+
+    React.useEffect(() => {
+        setEnabled(desktopInputEnabled(sessionId));
+        return observeDesktopSurface(() => { refresh(); setEnabled(desktopInputEnabled(sessionId)); });
+    }, [sessionId]);
+    React.useEffect(() => {
+        const show = Keyboard.addListener('keyboardWillShow', (event) => setKeyboardHeight(event.endCoordinates.height));
+        const hide = Keyboard.addListener('keyboardWillHide', () => setKeyboardHeight(0));
+        return () => { show.remove(); hide.remove(); };
+    }, []);
+    React.useEffect(() => {
+        if (gestures !== 'device') return;
+        setCursor(null);
+        const held = gesture.current;
+        if (held && (held.mode === 'pending' || held.mode === 'armed')) {
+            stopTimer();
+            held.mode = 'spent';
+        }
+    }, [gestures]);
+    React.useEffect(() => {
+        if (!sessionId) return;
+        return registerDesktopView(sessionId, {
+            show: () => { if (desktopInputEnabled(sessionId)) input.current?.focus(); },
+            hide: () => { input.current?.blur(); Keyboard.dismiss(); },
+            capture: (value) => { captured.current = value; },
+        }, {
+            size: (width, height) => setSize({ width, height }),
+            fit: () => { setZoom(1); setOffset({ x: 0, y: 0 }); },
+        });
+    }, [sessionId]);
+
+    const visibleHeight = Math.max(1, bounds.height - (keyboardHeight ? keyboardHeight + keyboardClearance : 0));
+    const fit = size.width && size.height ? Math.min(bounds.width / size.width, visibleHeight / size.height) : 1;
+    const scale = fit * zoom;
+    const pictureWidth = size.width * scale;
+    const pictureHeight = size.height * scale;
+    const originX = (bounds.width - pictureWidth) / 2 + offset.x;
+    const originY = (visibleHeight - pictureHeight) / 2 + offset.y;
+    const point = (x: number, y: number, clamp = false): Point | null => {
+        if (!size.width || !size.height || !scale) return null;
+        const px = Math.floor((x - originX) / scale);
+        const py = Math.floor((y - originY) / scale);
+        if (!clamp && (px < 0 || py < 0 || px >= size.width || py >= size.height)) return null;
+        return { x: Math.max(0, Math.min(size.width - 1, px)), y: Math.max(0, Math.min(size.height - 1, py)) };
+    };
+    const send = (control: Record<string, unknown>) => { if (sessionId) nativeDesklink.sendControl(sessionId, JSON.stringify(control)); };
+    const pointer = (phase: string, at: Point, button?: number) => {
+        send({ kind: 'pointer', phase, x: at.x, y: at.y, ...(button ? { button } : {}) });
+        // A device screen shows the touch itself; a cursor would be a second finger that lies.
+        if (gestures !== 'device') setCursor(at);
+    };
+    const click = (at: Point, button = 1) => { pointer('down', at, button); pointer('up', at, button); };
+    const stopTimer = () => { if (longPress.current) clearTimeout(longPress.current); longPress.current = null; };
+    const focus = (touches: readonly { locationX: number; locationY: number }[]): Point => ({
+        x: (touches[0].locationX + touches[1].locationX) / 2,
+        y: (touches[0].locationY + touches[1].locationY) / 2,
+    });
+    const span = (touches: readonly { locationX: number; locationY: number }[]) => Math.hypot(touches[0].locationX - touches[1].locationX, touches[0].locationY - touches[1].locationY);
+    const start = (event: GestureResponderEvent) => {
+        if (!enabled) return;
+        const touches = event.nativeEvent.touches;
+        if (touches.length !== 1) return;
+        const at = { x: touches[0].locationX, y: touches[0].locationY };
+        // A device screen is pressed, not pointed at: the finger lands with the
+        // button down. A tap is the same press lifted without moving, and a
+        // hold holds — there is no long press to arm.
+        if (gestures === 'device') {
+            const from = point(at.x, at.y);
+            gesture.current = { start: at, last: at, time: Date.now(), mode: from ? 'drag' : 'spent', span: 0, focus: at };
+            if (from) pointer('down', from, 1);
+            return;
+        }
+        gesture.current = { start: at, last: at, time: Date.now(), mode: 'pending', span: 0, focus: at };
+        stopTimer();
+        longPress.current = setTimeout(() => {
+            if (gesture.current?.mode === 'pending' && point(at.x, at.y)) gesture.current.mode = 'armed';
+        }, 500);
+    };
+    const move = (event: GestureResponderEvent) => {
+        if (!enabled) return;
+        const g = gesture.current;
+        if (!g) return;
+        const touches = event.nativeEvent.touches;
+        if (touches.length === 2) {
+            stopTimer();
+            const center = focus(touches);
+            const distance = span(touches);
+            if (g.mode !== 'two' && g.mode !== 'pinch' && g.mode !== 'scroll') {
+                if (g.mode === 'drag') send({ kind: 'release_all' });
+                g.mode = 'two'; g.span = distance; g.focus = center; g.start = center;
+                return;
+            }
+            if (g.mode === 'two') {
+                if (Math.abs(distance - g.span) > 12) g.mode = 'pinch';
+                else if (Math.hypot(center.x - g.start.x, center.y - g.start.y) > 8) g.mode = 'scroll';
+            }
+            if (g.mode === 'pinch' && g.span) {
+                setZoom((value) => Math.max(1, Math.min(2.5 / fit, value * distance / g.span)));
+            } else if (g.mode === 'scroll') {
+                send({ kind: 'wheel', dx: -(center.x - g.focus.x) / scale / 120, dy: -(center.y - g.focus.y) / scale / 120 });
+            }
+            g.span = distance; g.focus = center; g.last = center;
+            return;
+        }
+        if (touches.length !== 1 || g.mode === 'spent' || g.mode === 'two' || g.mode === 'pinch' || (g.mode === 'scroll' && gestures !== 'browser')) return;
+        const at = { x: touches[0].locationX, y: touches[0].locationY };
+        if (g.mode === 'pending' && Math.hypot(at.x - g.start.x, at.y - g.start.y) > 8) {
+            stopTimer();
+            // A page scrolls under one finger; the desktop's pointer rides along
+            // so the scroll lands where the finger is.
+            if (gestures === 'browser') {
+                g.mode = 'scroll';
+                const under = point(at.x, at.y);
+                if (under) pointer('move', under);
+            } else g.mode = zoom > 1 ? 'pan' : 'hover';
+        }
+        if (g.mode === 'scroll') {
+            // Content follows the finger: moving it up scrolls the page down.
+            send({ kind: 'wheel', dx: -(at.x - g.last.x) / scale / 120, dy: -(at.y - g.last.y) / scale / 120 });
+            g.last = at; g.focus = at;
+            return;
+        }
+        if (g.mode === 'pan') {
+            const dx = at.x - g.last.x;
+            const dy = at.y - g.last.y;
+            setOffset((current) => ({
+                x: Math.max(-Math.max(0, (pictureWidth - bounds.width) / 2), Math.min(Math.max(0, (pictureWidth - bounds.width) / 2), current.x + dx)),
+                y: Math.max(-Math.max(0, (pictureHeight - visibleHeight) / 2), Math.min(Math.max(0, (pictureHeight - visibleHeight) / 2), current.y + dy)),
+            }));
+        }
+        if (g.mode === 'armed' && Math.hypot(at.x - g.start.x, at.y - g.start.y) > 8) {
+            const from = point(g.start.x, g.start.y, true);
+            if (from) pointer('down', from, 1);
+            g.mode = 'drag';
+        }
+        if (g.mode === 'hover' || g.mode === 'drag') {
+            const to = point(at.x, at.y, true);
+            if (to) pointer('move', to, g.mode === 'drag' ? 1 : undefined);
+        }
+        g.last = at;
+    };
+    const end = (event: GestureResponderEvent) => {
+        stopTimer();
+        const g = gesture.current;
+        if (!g) return;
+        if (g.mode === 'two' && Date.now() - g.time < 350) {
+            const at = point(g.focus.x, g.focus.y);
+            if (at) click(at, 3);
+            g.mode = 'spent';
+        }
+        // A remaining finger after a two-finger gesture must not begin a click.
+        if (event.nativeEvent.touches.length > 0) { g.mode = 'spent'; return; }
+        gesture.current = null;
+        if (g.mode === 'drag') { const at = point(g.last.x, g.last.y, true); if (at) pointer('up', at, 1); else send({ kind: 'release_all' }); }
+        else if (g.mode === 'armed') { const at = point(g.start.x, g.start.y); if (at) click(at, 3); }
+        else if (g.mode === 'pending') {
+            const at = point(g.start.x, g.start.y);
+            if (at) {
+                const prev = lastTap.current;
+                const repeated = prev && Date.now() - prev.time < 350 && Math.hypot(prev.at.x - g.start.x, prev.at.y - g.start.y) < 40;
+                click(repeated ? prev.point : at);
+                lastTap.current = { time: Date.now(), at: g.start, point: repeated ? prev.point : at };
+            }
+        }
+    };
+    const changeText = (text: string) => {
+        // Retain only a short suffix so iOS can generate deletion events while
+        // autocorrect and multi-stage input still arrive as committed text.
+        const { removed, added } = keyboardDelta(keyboardTextRef.current, text);
+        keyboardTextRef.current = text.length > 64 ? text.slice(-32) : text;
+        for (let i = 0; i < removed; i++) {
+            if (captured.current && sessionId) emitDesktopKeyboard(sessionId, { key: 'Backspace' });
+            else { send({ kind: 'key', name: 'Backspace', modifiers: [], down: true }); send({ kind: 'key', name: 'Backspace', modifiers: [], down: false }); }
+        }
+        if (added) {
+            if (captured.current && sessionId) emitDesktopKeyboard(sessionId, { text: added });
+            else send({ kind: 'text', text: added });
+        }
+        setKeyboardText(keyboardTextRef.current);
+    };
+    const onLayout = (event: LayoutChangeEvent) => setBounds(event.nativeEvent.layout);
+    return <View style={[styles.surface, style]} onLayout={onLayout} accessibilityLabel={accessibilityLabel}
+        onStartShouldSetResponder={() => enabled} onMoveShouldSetResponder={() => enabled}
+        onResponderGrant={start} onResponderMove={move} onResponderRelease={end}
+        onResponderTerminate={() => { stopTimer(); if (gesture.current?.mode === 'drag') send({ kind: 'release_all' }); gesture.current = null; }}>
+        {sessionId && stream && size.width > 0 && size.height > 0 ?
+            <RTCView pointerEvents="none" streamURL={stream.toURL()} objectFit="contain" style={{ position: 'absolute', left: originX, top: originY, width: pictureWidth, height: pictureHeight }}
+                onDimensionsChange={(event) => { if (event.nativeEvent.width > 0 && event.nativeEvent.height > 0) markDesktopPresented(sessionId); }} /> : placeholder}
+        {cursor && <View pointerEvents="none" style={[styles.cursor, { left: originX + (cursor.x + 0.5) * scale, top: originY + (cursor.y + 0.5) * scale }]} />}
+        <TextInput ref={input} style={styles.input} value={keyboardText} onChangeText={changeText} autoCorrect={false} autoCapitalize="none" submitBehavior="submit"
+            onSubmitEditing={() => { send({ kind: 'key', name: 'Enter', modifiers: [], down: true }); send({ kind: 'key', name: 'Enter', modifiers: [], down: false }); }} />
+    </View>;
+}
+
+const styles = StyleSheet.create({
+    surface: { backgroundColor: '#000', overflow: 'hidden' },
+    cursor: { position: 'absolute', width: 12, height: 18, backgroundColor: '#fff', borderWidth: 2, borderColor: '#000', transform: [{ rotate: '-30deg' }] },
+    input: { position: 'absolute', width: 1, height: 1, opacity: 0, left: 0, bottom: 0 },
+});

package/src/DesktopView.tsx

@@ -3,6 +3,7 @@ import * as React from 'react';
 import { StyleSheet, View, type StyleProp, type ViewStyle } from 'react-native';
 
 import { desktopAvailable } from './native';
+import type { GestureProfile } from './protocol';
 
 export interface DesktopViewProps {
     /** The session handle from `useDesktopSession`. */
@@ -18,6 +19,11 @@ export interface DesktopViewProps {
      * following the keyboard as it moves.
      */
     keyboardClearance?: number;
+    /**
+     * Which touch meaning the surface uses: a desktop pointer, a browser page
+     * (one finger scrolls), or a device screen (one finger presses and drags).
+     */
+    gestures?: GestureProfile;
 }
 
 interface NativeSurfaceProps {
@@ -26,6 +32,7 @@ interface NativeSurfaceProps {
     accessible?: boolean;
     accessibilityLabel?: string;
     keyboardClearance?: number;
+    gestures?: GestureProfile;
 }
 
 // The Expo view is resolved at module load; on a platform without it the
@@ -42,7 +49,7 @@ const NativeSurface: React.ComponentType<NativeSurfaceProps> | null = desktopAva
  * belong to the application, which mounts this wherever it wants the desktop
  * to appear.
  */
-export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0 }: DesktopViewProps) {
+export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, gestures = 'desktop' }: DesktopViewProps) {
     if (NativeSurface == null || sessionId == null) {
         return (
             <View style={[styles.surface, style]}>
@@ -50,7 +57,7 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
             </View>
         );
     }
-    return <NativeSurface style={[styles.surface, style]} sessionId={sessionId} accessible={accessibilityLabel !== undefined} accessibilityLabel={accessibilityLabel} keyboardClearance={keyboardClearance} />;
+    return <NativeSurface style={[styles.surface, style]} sessionId={sessionId} accessible={accessibilityLabel !== undefined} accessibilityLabel={accessibilityLabel} keyboardClearance={keyboardClearance} gestures={gestures} />;
 }
 
 const styles = StyleSheet.create({

package/src/DesktopView.web.tsx

@@ -1,6 +1,8 @@
 import * as React from 'react';
 import { StyleSheet, View, type StyleProp, type ViewStyle } from 'react-native';
 
+import type { GestureProfile } from './protocol';
+
 export interface DesktopViewProps {
     /** The session handle from `useDesktopSession`. */
     sessionId: string | null;
@@ -14,6 +16,11 @@ export interface DesktopViewProps {
      * controls, in points. While the keyboard is up the picture sits above both.
      */
     keyboardClearance?: number;
+    /**
+     * Which touch meaning the surface uses: a desktop pointer, a browser page
+     * (one finger scrolls), or a device screen (one finger presses and drags).
+     */
+    gestures?: GestureProfile;
 }
 
 /**
@@ -25,19 +32,24 @@ export interface DesktopViewProps {
  * actually mounted rather than at module load, so an application that never
  * opens a desktop does not carry one in its first paint.
  */
-export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0 }: DesktopViewProps) {
+export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, gestures = 'desktop' }: DesktopViewProps) {
     const mounted = React.useCallback(
         (node: unknown) => {
             const element = (node as HTMLElement | null) ?? null;
             if (sessionId === null) return;
             void import('./native').then((platform) => {
                 if (element === null || !platform.desktopAvailable) return;
-                platform.attachSurface(sessionId, element);
+                platform.attachSurface(sessionId, element, gestures);
             });
         },
         [sessionId],
     );
 
+    React.useEffect(() => {
+        if (sessionId === null) return;
+        void import('./native').then((platform) => platform.setGestures(sessionId, gestures));
+    }, [sessionId, gestures]);
+
     React.useEffect(() => {
         if (sessionId === null) return;
         void import('./native').then((platform) => platform.setKeyboardClearance(sessionId, keyboardClearance));

package/src/keyboardDelta.ts

@@ -0,0 +1,7 @@
+export function keyboardDelta(previous: string, next: string) {
+    const before = [...previous];
+    const after = [...next];
+    let common = 0;
+    while (common < before.length && common < after.length && before[common] === after[common]) common++;
+    return { removed: before.length - common, added: after.slice(common).join('') };
+}