@desklink/react-native 0.3.0 → 0.3.1

package/README.md

@@ -130,11 +130,12 @@ heat and battery.
   stamps every frame alike, so the iOS peer declines that hint and keeps the
   receiver's usual jitter buffer. `fitToView()` shows the whole desktop again.
 - **A pointer a phone can see.** Once a touch has sent the desktop's pointer
-  somewhere, the view draws it there at a readable size, over a picture whose
-  own cursor is a few pixels tall or not captured at all. Android hides the
-  extra mark when a mouse is attached, iOS when an iPad trackpad or mouse
-  drives it, and the `device` profile draws no mark on any platform — the
-  screen being touched shows the finger itself.
+  somewhere, the view draws an outlined arrow there at a readable size, over a
+  picture whose own cursor is a few pixels tall or not captured at all. On iOS,
+  the arrow tip marks the desktop pointer position. Android hides the extra
+  mark when a mouse is attached, iOS when an iPad trackpad or mouse drives it,
+  and the `device` profile draws no mark on any platform — the screen being
+  touched shows the finger itself.
 - **The picture above the keyboard.** While the phone's keyboard is up, the
   picture sits above it and above the room the app keeps for its own controls
   (`keyboardClearance`). Android follows the keyboard animation and keeps the
@@ -290,15 +291,24 @@ a scroll:
 
 ```sh
 npm run build && cargo build --manifest-path packages/desktop-host/engine/Cargo.toml
-DESKLINK_IOS_MAC=user@mac npm run test:ios --workspace @desklink/react-native
+env -u DISPLAY -u WAYLAND_DISPLAY DESKLINK_IOS_MAC=user@mac npm run test:ios --workspace @desklink/react-native
 ```
 
 The Mac needs Xcode with an iOS simulator runtime, CocoaPods, node and
 [`axe`](https://github.com/cameroncooke/AXe) on its login `PATH`; everything
-the run builds or caches stays in `~/desklink-ios` there. When the selected
+the run builds or caches stays in `~/desklink-ios` there. The run creates a
+uniquely named simulator and deletes it during cleanup. For host desktop isolation, see the
+[private-lab guidance](../../README.md#develop). When the selected
 Xcode has no simulator platform, point `DESKLINK_IOS_XCODE` at one that does;
 `DESKLINK_IOS_RUNTIME` picks the iOS version (for example `18.6`); the
-script's header lists the other settings. To try the example by hand against any bridge:
+script's header lists the other settings. To capture the cursor tip proof, set
+`DESKLINK_IOS_POINTER_PROOF=before` or `after`; the run writes a 4x crop, device
+frame, proof metadata, and host click log under `DESKLINK_IOS_OUT`. Set
+`DESKLINK_IOS_APP` to reuse an app bundle under `DESKLINK_IOS_DIR` when running
+with `DESKLINK_IOS_SKIP_BUILD=1`. Run once with `before` before the change and
+once with `after` after it, on both iPhone and iPad simulators, to compare the
+pointer crop and verify the host click is within two desktop pixels of the tip.
+To try the example by hand against any bridge:
 
 ```sh
 xcrun simctl launch <device> dev.desklink.example -desklinkUrl 'ws://HOST:PORT/desktop?token=TOKEN'

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@desklink/react-native",
-    "version": "0.3.0",
+    "version": "0.3.1",
     "license": "Apache-2.0",
     "description": "Render and drive a desktop session from a React Native app",
     "main": "index.ts",

package/src/DesktopView.ios.spec.tsx

@@ -18,6 +18,7 @@ vi.mock('react-native', () => ({
     Keyboard: { addListener: () => ({ remove: () => undefined }), dismiss: () => undefined },
     StyleSheet: { create: (styles: unknown) => styles },
     TextInput: 'TextInput',
+    Image: 'Image',
     View: 'View',
 }));
 
@@ -204,15 +205,24 @@ describe('DesktopView.ios parity with DesktopView.kt', () => {
     });
 
     it('puts the cursor mark tip on the pointer with a hotspot', async () => {
-        const { renderer, grant, move } = await openView();
+        const { renderer, sent, grant, move, release } = await openView();
         grant([touch(195, 420)]);
         move([touch(250, 430)]);
-        const cursor = renderer.root.findAllByType('View')[1].props.style as unknown[];
+        const image = renderer.root.findByType('Image');
+        const cursor = image.props.style as unknown[];
         const mark = cursor[1] as { left: number; top: number };
         const desktop = { x: Math.floor(250 / SCALE), y: Math.floor((430 - ORIGIN_Y) / SCALE) };
         expect(mark.left).toBeCloseTo((desktop.x + 0.5) * SCALE - 3, 1);
         expect(mark.top).toBeCloseTo(ORIGIN_Y + (desktop.y + 0.5) * SCALE - 3, 1);
-        expect(renderer.root.findAllByType('View')[1].props.accessible).toBe(false);
+        expect(image.props.accessible).toBe(false);
+        expect(image.props.pointerEvents).toBe('none');
+        release();
+        grant([touch(mark.left + 3, mark.top + 3)]);
+        release();
+        expect(sent.filter((message) => message.kind === 'pointer').slice(-2)).toMatchObject([
+            { kind: 'pointer', phase: 'down', ...desktop, button: 1 },
+            { kind: 'pointer', phase: 'up', ...desktop, button: 1 },
+        ]);
     });
 
     it('exposes the surface to VoiceOver only with a label', async () => {

package/src/DesktopView.ios.tsx

@@ -1,5 +1,6 @@
 import * as React from 'react';
-import { Keyboard, StyleSheet, TextInput, View, type GestureResponderEvent, type LayoutChangeEvent, type ViewProps } from 'react-native';
+import { Image, Keyboard, StyleSheet, TextInput, View, type GestureResponderEvent, type LayoutChangeEvent, type ViewProps } from 'react-native';
+import pointerArrow from './pointer.png';
 import { requireNativeViewManager, requireOptionalNativeModule } from 'expo-modules-core';
 import type { DesktopViewProps } from './DesktopView';
 import { hardwareKey, type HardwareKey } from './hardwareKeys';
@@ -391,7 +392,7 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
         {sessionId && stream && RTCView && size.width > 0 && size.height > 0 ?
             <RTCView pointerEvents="none" streamURL={stream.toURL()} objectFit="contain" style={{ position: 'absolute', left: originX, top: originY, width: pictureWidth, height: pictureHeight }}
                 onDimensionsChange={(event) => { if (event.nativeEvent.width > 0 && event.nativeEvent.height > 0) markDesktopPresented(sessionId); }} /> : placeholder}
-        {cursor && <View pointerEvents="none" accessible={false} style={[styles.cursor, { left: originX + (cursor.x + 0.5) * scale - CURSOR_HOTSPOT, top: originY + (cursor.y + 0.5) * scale - CURSOR_HOTSPOT }]} />}
+        {cursor && <Image source={pointerArrow} pointerEvents="none" accessible={false} style={[styles.cursor, { left: originX + (cursor.x + 0.5) * scale - CURSOR_HOTSPOT, top: originY + (cursor.y + 0.5) * scale - CURSOR_HOTSPOT }]} />}
         <TextInput ref={input} style={styles.input} value={keyboardText} onChangeText={changeText} autoCorrect={false} autoCapitalize="none" submitBehavior="submit" accessible={false}
             onSubmitEditing={() => {
                 // While the app holds a sticky modifier, Enter is the app's to
@@ -408,6 +409,6 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
 
 const styles = StyleSheet.create({
     surface: { backgroundColor: '#000', overflow: 'hidden' },
-    cursor: { position: 'absolute', width: 12, height: 18, backgroundColor: '#fff', borderWidth: 2, borderColor: '#000', transform: [{ rotate: '-30deg' }] },
+    cursor: { position: 'absolute', width: 28, height: 28 },
     input: { position: 'absolute', width: 1, height: 1, opacity: 0, left: 0, bottom: 0 },
 });

package/src/pointer.png.d.ts

@@ -0,0 +1,2 @@
+declare const pointer: import('react-native').ImageSourcePropType;
+export default pointer;

package/src/pointer.svg

@@ -0,0 +1,7 @@
+<!-- Same outline and (3, 3) tip hotspot as web and Android. Generate the
+     iOS density assets with rsvg-convert -w SIZE -h SIZE pointer.svg -o FILE:
+     SIZE=28/56/84, FILE=pointer.png/pointer@2x.png/pointer@3x.png. -->
+<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28">
+  <path d="M3 3 L3 22.25 L7.73 17.96 L10.92 25.22 L14 23.9 L10.92 16.86 L17.08 16.86 Z" fill="none" stroke="#fff" stroke-width="2.2" stroke-linejoin="round"/>
+  <path d="M3 3 L3 22.25 L7.73 17.96 L10.92 25.22 L14 23.9 L10.92 16.86 L17.08 16.86 Z" fill="#000"/>
+</svg>

package/src/protocol.ts

@@ -41,6 +41,7 @@ export interface SurfaceGeometry {
 }
 
 export interface SessionOpenRequest {
+    cursor?: 'embedded' | 'hidden';
     permissions: Permission[];
     maxWidth?: number;
     maxHeight?: number;
@@ -52,6 +53,7 @@ export interface SessionOpenRequest {
 }
 
 export interface SessionOpenResult {
+    cursor?: { mode: 'embedded' | 'hidden' | 'metadata' | 'unavailable'; positions: boolean };
     sessionId: string;
     generation: number;
     source: { kind: string; width: number; height: number; origin: { x: number; y: number } };
@@ -69,6 +71,14 @@ export interface RtcCandidate {
     sdpMLineIndex?: number | null;
 }
 
+export interface CursorSample {
+    sessionId: string;
+    x: number;
+    y: number;
+    visible: boolean;
+    timestamp_us: number;
+}
+
 /**
  * Engine → client lifecycle events, already unwrapped from the local protocol.
  *
@@ -81,6 +91,7 @@ export interface RtcCandidate {
 export type SessionEvent =
     | { kind: 'description'; description: RtcDescription; sessionId?: string }
     | { kind: 'candidate'; candidate: RtcCandidate; sessionId?: string }
+    | ({ kind: 'cursor' } & CursorSample)
     | { kind: 'state'; capture: string; transport: string; firstFrame: boolean; sessionId?: string }
     | { kind: 'restoreToken'; token: string; sessionId?: string }
     /**

package/src/useDesktopSession.ts

@@ -4,6 +4,7 @@ import { AppState } from 'react-native';
 import type { NativeDesklinkModule } from './native';
 import type {
     ControlMessage,
+    CursorSample,
     IceServerConfig,
     Permission,
     SessionFailure,
@@ -178,6 +179,7 @@ export interface DesktopSessionOptions {
      */
     authorize: () => Promise<{ signaling: Signaling; session: SessionOpenRequest }>;
     onStateChange?: (snapshot: SessionSnapshot) => void;
+    onCursor?: (sample: CursorSample) => void;
     onError?: (failure: SessionFailure) => void;
     /**
      * The engine refused one input message and the session carries on, such as
@@ -253,6 +255,7 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
      * before the native session exists waits here and is replayed into it.
      */
     const pendingOffer = useRef<{ sdp: string; sessionId?: string } | null>(null);
+    const pendingCursorEvents = useRef<CursorSample[]>([]);
     const pendingCandidates = useRef<Array<{ candidate: string; sdpMid: string | null; sdpMLineIndex: number | null; sessionId?: string }>>([]);
     const attempts = useRef(0);
     /**
@@ -572,6 +575,7 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
         pendingClipboard.current.clear();
         pendingOffer.current = null;
         pendingCandidates.current = [];
+        pendingCursorEvents.current = [];
         restartGraceUntil.current = 0;
         restartCycles.current = 0;
         opened.current = null;
@@ -615,6 +619,7 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
     const establish = useCallback(async () => {
         if (nativeRef.current != null) return;
         const token = ++generationToken.current;
+        pendingCursorEvents.current = [];
         if (!(await loadPlatform())) {
             if (token === generationToken.current) refuse('this build cannot show a desktop surface');
             return;
@@ -672,6 +677,13 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
                         event.candidate.sdpMLineIndex ?? null,
                     );
                     return;
+                case 'cursor':
+                    if (held == null) {
+                        pendingCursorEvents.current.push(event);
+                        return;
+                    }
+                    if (current) optionsRef.current.onCursor?.(event);
+                    return;
                 case 'state':
                     if (!current) return;
                     diagnostics.current.transport = event.transport;
@@ -707,6 +719,7 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
             openedResult = await authorization.signaling.request<SessionOpenResult>('session.open', {
                 permissions: authorization.session.permissions,
                 max_width: authorization.session.maxWidth,
+                ...(authorization.session.cursor === undefined ? {} : { cursor: authorization.session.cursor }),
                 max_height: authorization.session.maxHeight,
                 bitrate_kbps: authorization.session.bitrateKbps,
                 max_fps: authorization.session.maxFps,
@@ -725,6 +738,9 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
         try {
             if (token !== generationToken.current) return;
             opened.current = openedResult;
+            for (const sample of pendingCursorEvents.current.splice(0)) {
+                if (sample.sessionId === openedResult.sessionId) optionsRef.current.onCursor?.(sample);
+            }
             update({ geometry: openedResult.geometry, status: 'connecting' });
             if (token !== generationToken.current) return;