@desklink/react-native 0.4.0 → 0.5.1

package/src/controls.spec.tsx

@@ -0,0 +1,205 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import React from 'react';
+import TestRenderer, { act } from 'react-test-renderer';
+
+(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
+
+/** The key row and the clipboard pill, rendered with `react-native` primitives as host strings. */
+vi.mock('react-native', () => ({
+    StyleSheet: { create: (styles: unknown) => styles, absoluteFill: {} },
+    Linking: { getInitialURL: async () => null, addEventListener: () => ({ remove: () => undefined }) },
+    Platform: { OS: 'ios' },
+    Pressable: 'Pressable',
+    ScrollView: 'ScrollView',
+    StatusBar: 'StatusBar',
+    Text: 'Text',
+    View: 'View',
+    Settings: { get: () => 'ws://localhost/desktop?token=test' },
+}));
+
+vi.mock('react-native-safe-area-context', () => ({
+    SafeAreaProvider: ({ children }: { children: React.ReactNode }) => children,
+    useSafeAreaInsets: () => ({ top: 0, left: 0, bottom: 0, right: 0 }),
+}));
+
+const example = vi.hoisted(() => ({
+    write: vi.fn(),
+    copy: vi.fn(),
+    connect: vi.fn(),
+}));
+vi.mock('expo-clipboard', () => ({ setStringAsync: example.write }));
+vi.mock('@desklink/react-native', async () => ({
+    CONTROL_PERMISSIONS: ['view', 'control', 'clipboard'],
+    ClipboardConfirmation: (await import('./ClipboardConfirmation')).ClipboardConfirmation,
+    ModifierKeys: (await import('./ModifierKeys')).ModifierKeys,
+    DesktopView: 'DesktopView',
+    useDesktopSession: () => ({
+        snapshot: { status: 'live' },
+        nativeId: 'example-session',
+        modifiers: { Control: 'off', Shift: 'off', Alt: 'off', Meta: 'off' },
+        clipboard: null,
+        connect: example.connect,
+        setInputEnabled: vi.fn(),
+        copyRemoteToLocal: example.copy,
+        tapModifier: vi.fn(),
+    }),
+}));
+
+import { KEY_COLORS, ModifierKeys } from './ModifierKeys';
+import { CLIPBOARD_CONFIRMATION_MS, ClipboardConfirmation, PILL_COLORS, describeTransfer } from './ClipboardConfirmation';
+import { NO_MODIFIERS } from './stickyModifiers';
+import type { ClipboardTransfer } from './useDesktopSession';
+import App from '../example/App';
+
+type Style = Record<string, unknown>;
+const flatten = (style: unknown): Style =>
+    Array.isArray(style) ? Object.assign({}, ...style.map(flatten)) : style ? (style as Style) : {};
+
+/** WCAG contrast of two colours, the first drawn with its alpha over `under`. */
+function contrast(top: string, bottom: string, under = '#000000'): number {
+    const rgba = (c: string) => {
+        const m = c.match(/rgba?\(([^)]+)\)/);
+        if (m) { const [r, g, b, a = 1] = m[1].split(',').map(Number); return [r, g, b, a]; }
+        return [1, 3, 5].map((i) => parseInt(c.slice(i, i + 2), 16)).concat(1);
+    };
+    const over = (c: string) => {
+        const [r, g, b, a] = rgba(c);
+        const [ur, ug, ub] = rgba(under);
+        return [r * a + ur * (1 - a), g * a + ug * (1 - a), b * a + ub * (1 - a)];
+    };
+    const lum = (rgb: number[]) => {
+        const [r, g, b] = rgb.map((v) => { const s = v / 255; return s <= 0.03928 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4; });
+        return 0.2126 * r + 0.7152 * g + 0.0722 * b;
+    };
+    const [l1, l2] = [lum(over(top)), lum(over(bottom))].sort((a, b) => b - a);
+    return (l1 + 0.05) / (l2 + 0.05);
+}
+
+describe('ModifierKeys', () => {
+    function render(modifiers = NO_MODIFIERS, onTap = vi.fn()) {
+        let tree!: TestRenderer.ReactTestRenderer;
+        act(() => { tree = TestRenderer.create(<ModifierKeys modifiers={modifiers} onTap={onTap} />); });
+        const key = (name: string) => tree.root.find((node) => node.props.testID === `desklink-key-${name}`);
+        return { key, onTap };
+    }
+
+    it('draws off, latched and locked as three different keys and names each state', () => {
+        const { key } = render({ Control: 'off', Shift: 'once', Alt: 'lock', Meta: 'off' });
+        const look = (name: string) => {
+            const node = key(name);
+            const [, bar] = node.props.children;
+            return { key: flatten(node.props.style), bar: flatten(bar.props.style), said: node.props.accessibilityValue.text };
+        };
+        const off = look('Control');
+        const latched = look('Shift');
+        const locked = look('Alt');
+        expect([off.said, latched.said, locked.said]).toEqual(['off', 'latched', 'locked']);
+        expect(off.key.backgroundColor).not.toBe(latched.key.backgroundColor);
+        // Latched and locked share a fill and differ in shape: the rim and the bar.
+        expect(locked.key.borderColor).not.toBe(latched.key.borderColor);
+        expect(locked.bar.backgroundColor).not.toBe(latched.bar.backgroundColor);
+        expect(key('Control').props.accessibilityState).toEqual({ selected: false });
+        expect(key('Shift').props.accessibilityState).toEqual({ selected: true });
+    });
+
+    it('hands a tap to the session', () => {
+        const { key, onTap } = render();
+        key('Control').props.onPress();
+        expect(onTap).toHaveBeenCalledWith('Control');
+    });
+
+    it('keeps every label at 4.5:1 or better', () => {
+        expect(contrast(KEY_COLORS.offLabel, KEY_COLORS.off)).toBeGreaterThanOrEqual(4.5);
+        expect(contrast(KEY_COLORS.onLabel, KEY_COLORS.on)).toBeGreaterThanOrEqual(4.5);
+    });
+});
+
+describe('ClipboardConfirmation', () => {
+    beforeEach(() => { vi.useFakeTimers(); });
+    afterEach(() => { vi.useRealTimers(); });
+
+    const transfer = (id: number, text = 'hello world'): ClipboardTransfer => ({ id, direction: 'to-desktop', text, truncated: false });
+
+    it('shows each transfer, ignores touches, and goes away after about 1.5 s', () => {
+        let tree!: TestRenderer.ReactTestRenderer;
+        act(() => { tree = TestRenderer.create(<ClipboardConfirmation transfer={null} />); });
+        expect(tree.toJSON()).toBeNull();
+
+        act(() => { tree.update(<ClipboardConfirmation transfer={transfer(1)} />); });
+        const place = tree.toJSON() as TestRenderer.ReactTestRendererJSON;
+        expect(place.props.pointerEvents).toBe('none');
+        expect(flatten(place.props.style).position).toBeUndefined();
+        expect(JSON.stringify(place)).toContain('Sent to desktop · “hello world”');
+
+        act(() => { vi.advanceTimersByTime(CLIPBOARD_CONFIRMATION_MS - 1); });
+        expect(tree.toJSON()).not.toBeNull();
+        act(() => { vi.advanceTimersByTime(1); });
+        expect(tree.toJSON()).toBeNull();
+
+        // The same text again is a new transfer and is confirmed again.
+        act(() => { tree.update(<ClipboardConfirmation transfer={transfer(2)} />); });
+        expect(tree.toJSON()).not.toBeNull();
+    });
+
+    it('describes where the text went and how it starts, with no byte count', () => {
+        expect(describeTransfer({ id: 1, direction: 'to-phone', text: 'a\n  b', truncated: false })).toBe('Copied to phone · “a b”');
+        expect(describeTransfer({ id: 1, direction: 'to-phone', text: 'x'.repeat(2048), truncated: true })).toBe(`Copied to phone · “${'x'.repeat(24)}…”`);
+        expect(describeTransfer({ id: 1, direction: 'to-desktop', text: ' ', truncated: false })).toBe('Sent to desktop');
+    });
+
+    it('reads at 4.5:1 or better over a white or a black desktop', () => {
+        expect(contrast(PILL_COLORS.text, PILL_COLORS.backing, '#ffffff')).toBeGreaterThanOrEqual(4.5);
+        expect(contrast(PILL_COLORS.text, PILL_COLORS.backing, '#000000')).toBeGreaterThanOrEqual(4.5);
+    });
+});
+
+describe('example Copy', () => {
+    it('writes only on a tap and retains a backed accessible failure until the next tap', async () => {
+        example.write.mockReset();
+        example.copy.mockReset();
+        example.copy.mockImplementation(async (write: (text: string) => Promise<void>) => { await write('desktop text'); });
+        let tree!: TestRenderer.ReactTestRenderer;
+        await act(async () => { tree = TestRenderer.create(<App />); });
+        expect(example.write).not.toHaveBeenCalled();
+        const tap = () => tree.root.findByProps({ testID: 'desklink-copy' }).props.onPress();
+
+        example.copy.mockRejectedValueOnce(new Error('clipboard is unavailable for this desktop source'));
+        await act(async () => { tap(); });
+        expect(tree.root.findByProps({ testID: 'desklink-copy-error' }).findByType('Text').props.children)
+            .toBe('Clipboard failed: clipboard is unavailable for this desktop source');
+        expect(example.write).not.toHaveBeenCalled();
+
+        example.write.mockResolvedValue(false);
+        await act(async () => { tap(); });
+        expect(example.write).toHaveBeenCalledWith('desktop text');
+        const error = tree.root.findByProps({ testID: 'desklink-copy-error' });
+        expect(error.props.accessible).toBe(true);
+        expect(error.props.accessibilityRole).toBe('alert');
+        expect(error.props.accessibilityLiveRegion).toBe('polite');
+        const style = flatten(error.props.style);
+        expect(style.paddingHorizontal).toBeGreaterThan(0);
+        expect(style.paddingVertical).toBeGreaterThan(0);
+        expect(contrast('#ffffff', style.backgroundColor as string)).toBeGreaterThanOrEqual(4.5);
+        expect(error.findByType('Text').props.children).toBe('Clipboard failed: The phone refused the clipboard write.');
+        expect(tree.root.findAllByProps({ testID: 'desklink-clipboard' })).toHaveLength(0);
+
+        vi.useFakeTimers();
+        try {
+            act(() => { vi.advanceTimersByTime(5000); });
+            expect(tree.root.findAllByProps({ testID: 'desklink-copy-error' })).toHaveLength(1);
+        } finally { vi.useRealTimers(); }
+
+        example.write.mockRejectedValue(new Error('native write refused'));
+        await act(async () => { tap(); });
+        expect(tree.root.findByProps({ testID: 'desklink-copy-error' }).findByType('Text').props.children).toBe('Clipboard failed: native write refused');
+
+        let finish!: (written: boolean) => void;
+        example.write.mockImplementation(() => new Promise<boolean>((resolve) => { finish = resolve; }));
+        await act(async () => { tap(); });
+        expect(tree.root.findAllByProps({ testID: 'desklink-copy-error' })).toHaveLength(0);
+        expect(tree.root.findAllByProps({ testID: 'desklink-clipboard' })).toHaveLength(0);
+        await act(async () => { finish(true); });
+        expect(tree.root.findAllByProps({ testID: 'desklink-copy-error' })).toHaveLength(0);
+        act(() => { tree.unmount(); });
+    });
+});

package/src/index.ts

@@ -10,12 +10,15 @@ export * from './protocol';
  */
 export type { NativeDesklinkModule, NativeSessionEvent } from './native';
 export { desktopAvailable } from './availability';
-export { DesktopView, type DesktopViewProps } from './DesktopView';
+export { DesktopView, type DesktopViewProps, type EdgeInsets } from './DesktopView';
 export { observeWebKeyboardMotion } from './webKeyboardMotion';
 export type { StickyModifier, StickyModifiers, StickyState } from './stickyModifiers';
+export { ModifierKeys, type ModifierKeysProps } from './ModifierKeys';
+export { ClipboardConfirmation, type ClipboardConfirmationProps } from './ClipboardConfirmation';
 export {
     useDesktopSession,
     CONTROL_PERMISSIONS,
+    type ClipboardTransfer,
     type DesktopSession,
     type DesktopSessionOptions,
 } from './useDesktopSession';

package/src/native.ios.ts

@@ -1,6 +1,7 @@
 import type { MediaStream, RTCPeerConnection } from 'react-native-webrtc';
 import type { NativeDesklinkModule, NativeSessionEvent } from './native';
 import { requireWebRTC } from './webrtc.ios';
+import { videoStats } from './videoStats';
 
 /** The WebRTC stack, loaded the first time a session opens rather than with this module. */
 function loadWebRTC(): { RTCPeerConnection: new (config: unknown) => RTCPeerConnection; MediaStream: new () => MediaStream } | null {
@@ -99,6 +100,10 @@ function failure(id: string, error: unknown) {
 }
 
 export const nativeDesklink: NativeDesklinkModule = {
+    async getStats(id) {
+        const session = sessions.get(id);
+        return session ? videoStats(await session.peer.getStats()) : '[]';
+    },
     createSession(iceServersJson) {
         const id = `desklink-${++nextId}`;
         try {

package/src/native.ts

@@ -45,6 +45,8 @@ export interface NativeDesklinkModule {
     /** Hold the screen in landscape while the desktop is shown, or follow the phone again. */
     setOrientation(mode: 'landscape' | 'auto'): boolean;
     closeSession(id: string): boolean;
+    /** Video reception and codec statistics, excluding network addresses and credentials. */
+    getStats?(id: string): Promise<string>;
     isAvailable(): boolean;
     addListener?(name: 'onSessionEvent', handler: (event: NativeSessionEvent) => void): { remove: () => void };
 }
@@ -57,4 +59,3 @@ export const nativeDesklink: NativeDesklinkModule | null = native;
 
 /** True when this build can show a live desktop surface at all. */
 export const desktopAvailable = native !== null;
-

package/src/native.web.ts

@@ -1,4 +1,5 @@
 import type { NativeDesklinkModule, NativeEventName, NativeSessionEvent } from './native';
+import { videoStats } from './videoStats';
 import type { GestureProfile } from './protocol';
 import { observeWebKeyboardMotion } from './webKeyboardMotion';
 
@@ -898,6 +899,10 @@ const sequence = new Map<string, number>();
 export const desktopAvailable = true;
 
 export const nativeDesklink: NativeDesklinkModule = {
+    async getStats(id) {
+        const session = sessions.get(id);
+        return session ? videoStats(await session.peer.getStats()) : '[]';
+    },
     isAvailable: () => true,
 
     createSession(iceServersJson: string): string | null {

package/src/protocol.ts

@@ -59,7 +59,7 @@ export interface SessionOpenRequest {
 }
 
 export interface SessionOpenResult {
-    cursor?: { mode: 'embedded' | 'hidden' | 'metadata' | 'unavailable'; positions: boolean };
+    cursor?: { mode: 'embedded' | 'hidden' | 'metadata' | 'unavailable'; positions: boolean; source?: 'ext-image-copy-capture-v1'; limitation?: string };
     sessionId: string;
     generation: number;
     source: { kind: string; width: number; height: number; origin: { x: number; y: number } };
@@ -83,6 +83,7 @@ export interface CursorSample {
     y: number;
     visible: boolean;
     timestamp_us: number;
+    hotspot?: { x: number; y: number };
 }
 
 /**

package/src/useDesktopSession.ts

@@ -25,6 +25,14 @@ import {
     type StickyModifiers,
 } from './stickyModifiers';
 
+/** One clipboard transfer that reached its destination; `id` tells two equal transfers apart. */
+export interface ClipboardTransfer {
+    id: number;
+    direction: 'to-phone' | 'to-desktop';
+    text: string;
+    truncated: boolean;
+}
+
 /** How long a clipboard round trip may take before it is reported as lost. */
 const CLIPBOARD_TIMEOUT_MS = 4000;
 
@@ -191,14 +199,16 @@ export interface DesktopSessionOptions {
 
 export interface DesktopSession {
     snapshot: SessionSnapshot;
+    /** Video reception and codec diagnostics, without network addresses. */
+    getStats: () => Promise<Array<Record<string, unknown>>>;
     /** The native session handle, or null when nothing is open. */
     nativeId: string | null;
     connect: () => Promise<void>;
     close: (reason?: string) => Promise<void>;
     showKeyboard: () => void;
     hideKeyboard: () => void;
-    /** Copy the desktop's clipboard to the phone; the caller places it locally. */
-    copyRemoteToLocal: () => Promise<{ text: string; truncated: boolean }>;
+    /** Copy using the caller's local writer, which must reject if the write fails. */
+    copyRemoteToLocal: (writeLocal: (text: string) => Promise<void>) => Promise<{ text: string; truncated: boolean }>;
     /** Send the phone's clipboard text to the desktop. */
     pasteLocalToRemote: (text: string) => Promise<void>;
     /** Release anything the desktop is holding, without ending the session. */
@@ -219,6 +229,8 @@ export interface DesktopSession {
      * because the engine lets go of exactly the modifiers the message names.
      */
     pressKey: (name: string) => () => void;
+    /** The last clipboard transfer that completed, for the app to confirm; null until one does. */
+    clipboard: ClipboardTransfer | null;
 }
 
 const IDLE: SessionSnapshot = {
@@ -243,6 +255,8 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
     const [modifiers, setModifiers] = useState<StickyModifiers>(NO_MODIFIERS);
     /** Read by input as it arrives, which can be faster than a render. */
     const modifiersRef = useRef<StickyModifiers>(NO_MODIFIERS);
+    const [clipboard, setClipboard] = useState<ClipboardTransfer | null>(null);
+    const clipboardCount = useRef(0);
 
     const opened = useRef<SessionOpenResult | null>(null);
     const signaling = useRef<Signaling | null>(null);
@@ -556,6 +570,12 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
         if (id != null) nativeDesklink?.hideKeyboard(id);
     }, []);
 
+    const getStats = useCallback(async (): Promise<Array<Record<string, unknown>>> => {
+        const id = nativeRef.current;
+        if (id == null || !nativeDesklink?.getStats) return [];
+        return JSON.parse(await nativeDesklink.getStats(id));
+    }, []);
+
     const fitToView = useCallback(() => {
         const id = nativeRef.current;
         if (id != null) nativeDesklink?.fitToView(id);
@@ -879,6 +899,14 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
                             return;
                         }
                         if (reply.kind === 'rejected') {
+                            // These session-wide clipboard refusals have no request ID.
+                            // A generic permission rejection can belong to other input.
+                            if (reply.seq === 0 && (reply.code === 'clipboard-unsupported' || reply.code === 'session')) {
+                                for (const resolve of pendingClipboard.current.values()) {
+                                    resolve({ text: '', truncated: false, error: reply.message });
+                                }
+                                pendingClipboard.current.clear();
+                            }
                             diagnostics.current.lastRejection = `${reply.code}: ${reply.message}`;
                             update({ diagnostics: { ...diagnostics.current } });
                             optionsRef.current.onRejected?.({ code: reply.code, message: reply.message });
@@ -929,7 +957,7 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
         };
     }, [transportFailed, onTransportState, cancelRestart, spendRestartAttempt, notePing, teardown, update, pressKey, typeText]);
 
-    const copyRemoteToLocal = useCallback(async () => {
+    const copyRemoteToLocal = useCallback(async (writeLocal: (text: string) => Promise<void>) => {
         if (!inputEnabled.current) throw new Error('Desktop control is off.');
         const id = nativeRef.current;
         if (id == null || nativeDesklink == null) throw new Error('No desktop session is open.');
@@ -943,6 +971,8 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
         nativeDesklink.sendControl(id, JSON.stringify({ kind: 'clipboard_read', request }));
         const reply = await answer;
         if (reply.error != null) throw new Error(reply.error);
+        await writeLocal(reply.text);
+        setClipboard({ id: ++clipboardCount.current, direction: 'to-phone', text: reply.text, truncated: reply.truncated });
         return { text: reply.text, truncated: reply.truncated };
     }, []);
 
@@ -960,6 +990,7 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
         nativeDesklink.sendControl(id, JSON.stringify({ kind: 'clipboard_write', request, text }));
         const reply = await answer;
         if (reply.error != null) throw new Error(reply.error);
+        setClipboard({ id: ++clipboardCount.current, direction: 'to-desktop', text, truncated: false });
     }, []);
 
     const close = useCallback(async (reason = 'closed by the user') => {
@@ -1072,11 +1103,13 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
         releaseHeld,
         setInputEnabled,
         fitToView,
+        getStats,
         setOrientation,
         send,
         modifiers,
         tapModifier,
         pressKey,
+        clipboard,
     }), [
         snapshot,
         nativeId,
@@ -1089,11 +1122,13 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
         releaseHeld,
         setInputEnabled,
         fitToView,
+        getStats,
         setOrientation,
         send,
         modifiers,
         tapModifier,
         pressKey,
+        clipboard,
     ]);
 }
 

package/src/videoStats.ts

@@ -0,0 +1,16 @@
+/** Keep receiver diagnostics useful without disclosing ICE addresses or credentials. */
+export function videoStats(report: { forEach(callback: (stat: Record<string, unknown>) => void): void }): string {
+    const fields = new Set([
+        'id', 'type', 'timestamp', 'kind', 'mediaType', 'codecId', 'mimeType', 'clockRate', 'payloadType',
+        'packetsReceived', 'packetsLost', 'bytesReceived', 'framesReceived', 'framesDecoded',
+        'keyFramesDecoded', 'framesDropped', 'framesPerSecond', 'frameWidth', 'frameHeight',
+        'nackCount', 'pliCount', 'firCount', 'totalDecodeTime', 'decoderImplementation', 'jitterBufferDelay',
+    ]);
+    const stats: Array<Record<string, unknown>> = [];
+    report.forEach((stat) => {
+        if (stat.type === 'inbound-rtp' || stat.type === 'codec') {
+            stats.push(Object.fromEntries(Object.entries(stat).filter(([key]) => fields.has(key))));
+        }
+    });
+    return JSON.stringify(stats);
+}