@desklink/react-native 0.1.0 → 0.2.0

package/src/native.ios.ts

@@ -0,0 +1,186 @@
+import { MediaStream, RTCPeerConnection } from 'react-native-webrtc';
+import type { NativeDesklinkModule, NativeSessionEvent } from './native';
+
+type Session = {
+    peer: RTCPeerConnection;
+    channel: ReturnType<RTCPeerConnection['createDataChannel']> | null;
+    stream: MediaStream | null;
+    input: boolean;
+    seq: number;
+    closed: boolean;
+    candidates: Array<{ candidate: string; sdpMid: string | null; sdpMLineIndex: number }>;
+    remoteSet: boolean;
+};
+
+const sessions = new Map<string, Session>();
+const listeners = new Set<(event: NativeSessionEvent) => void>();
+const surfaces = new Set<() => void>();
+const sizes = new Map<string, { width: number; height: number }>();
+let nextId = 0;
+
+function emit(sessionId: string, name: NativeSessionEvent['name'], payload: Record<string, unknown> = {}) {
+    if (!sessions.has(sessionId) && name !== 'closed') return;
+    for (const listener of listeners) listener({ sessionId, name, payload });
+}
+
+function notifySurface() {
+    for (const listener of surfaces) listener();
+}
+
+export function getDesktopStream(id: string | null): MediaStream | null {
+    return id == null ? null : sessions.get(id)?.stream ?? null;
+}
+
+export function getDesktopSize(id: string | null) {
+    return id == null ? null : sizes.get(id) ?? null;
+}
+
+export function observeDesktopSurface(listener: () => void): () => void {
+    surfaces.add(listener);
+    return () => surfaces.delete(listener);
+}
+
+export function markDesktopPresented(id: string) {
+    emit(id, 'presented');
+}
+
+export function desktopInputEnabled(id: string | null): boolean {
+    return id != null && sessions.get(id)?.input === true;
+}
+
+function failure(id: string, error: unknown) {
+    emit(id, 'failure', { code: 'transport', message: error instanceof Error ? error.message : String(error) });
+}
+
+export const nativeDesklink: NativeDesklinkModule = {
+    createSession(iceServersJson) {
+        const id = `desklink-${++nextId}`;
+        try {
+            const peer = new RTCPeerConnection({ iceServers: JSON.parse(iceServersJson) });
+            const session: Session = { peer, channel: null, stream: null, input: false, seq: 0, closed: false, candidates: [], remoteSet: false };
+            sessions.set(id, session);
+            peer.onicecandidate = (event) => {
+                if (event.candidate) emit(id, 'candidate', {
+                    candidate: event.candidate.candidate,
+                    sdpMid: event.candidate.sdpMid,
+                    sdpMLineIndex: event.candidate.sdpMLineIndex,
+                });
+            };
+            peer.oniceconnectionstatechange = () => {
+                emit(id, 'ice', { state: peer.iceConnectionState.toUpperCase() });
+                if (peer.iceConnectionState === 'failed') failure(id, 'the connection to the desktop was lost');
+            };
+            peer.ontrack = (event) => {
+                if (session.closed || event.track.kind !== 'video') return;
+                const stream = event.streams[0] ?? new MediaStream();
+                if (event.streams.length === 0) stream.addTrack(event.track);
+                session.stream = stream;
+                notifySurface();
+                emit(id, 'track');
+            };
+            peer.ondatachannel = (event) => {
+                if (session.closed) return;
+                const channel = event.channel;
+                session.channel = channel;
+                channel.onopen = () => emit(id, 'channel', { state: 'OPEN' });
+                channel.onclose = () => emit(id, 'channel', { state: 'CLOSED' });
+                channel.onmessage = (message) => emit(id, 'control', { message: message.data });
+            };
+            queueMicrotask(() => emit(id, 'ready'));
+            return id;
+        } catch {
+            return null;
+        }
+    },
+    setRemoteDescription(id, type, sdp) {
+        const session = sessions.get(id);
+        if (!session) return false;
+        if (type !== 'offer') { failure(id, 'the engine must send an offer'); return false; }
+        void (async () => {
+            try {
+                await session.peer.setRemoteDescription({ type: 'offer', sdp });
+                if (session.closed) return;
+                session.remoteSet = true;
+                for (const candidate of session.candidates.splice(0)) await session.peer.addIceCandidate(candidate);
+                const answer = await session.peer.createAnswer();
+                await session.peer.setLocalDescription(answer);
+                if (!session.closed) emit(id, 'answer', { sdp: session.peer.localDescription?.sdp ?? answer.sdp });
+            } catch (error) { if (!session.closed) failure(id, error); }
+        })();
+        return true;
+    },
+    addRemoteCandidate(id, candidate, sdpMid, sdpMLineIndex) {
+        const session = sessions.get(id);
+        if (!session) return false;
+        const ice = { candidate, sdpMid, sdpMLineIndex: sdpMLineIndex ?? 0 };
+        if (!session.remoteSet) session.candidates.push(ice);
+        else void session.peer.addIceCandidate(ice).catch((error) => failure(id, error));
+        return true;
+    },
+    sendControl(id, message) {
+        const session = sessions.get(id);
+        if (!session || session.closed || session.channel?.readyState !== 'open') return false;
+        try {
+            const control = JSON.parse(message) as Record<string, unknown>;
+            if (!session.input && ['pointer', 'wheel', 'key', 'text', 'clipboard_read', 'clipboard_write'].includes(String(control.kind))) return false;
+            session.channel.send(JSON.stringify({ ...control, seq: ++session.seq }));
+            return true;
+        } catch (error) { failure(id, error); return false; }
+    },
+    setInputEnabled(id, enabled) {
+        const session = sessions.get(id);
+        if (!session) return false;
+        session.input = enabled;
+        if (!enabled) nativeDesklink.sendControl(id, '{"kind":"release_all"}');
+        notifySurface();
+        return true;
+    },
+    showKeyboard(id) { keyboardActions.get(id)?.show(); return true; },
+    hideKeyboard(id) { keyboardActions.get(id)?.hide(); return true; },
+    captureKeyboard(id, captured) { keyboardActions.get(id)?.capture(captured); return true; },
+    // iOS rotation belongs to the application's supported orientations; this
+    // receiver cannot override its Info.plist at runtime.
+    setOrientation() { return false; },
+    setSurfaceSize(id, width, height) {
+        if (!sessions.has(id) || width <= 0 || height <= 0) return false;
+        sizes.set(id, { width, height });
+        surfaceActions.get(id)?.size(width, height);
+        return true;
+    },
+    fitToView(id) { surfaceActions.get(id)?.fit(); return true; },
+    closeSession(id) {
+        const session = sessions.get(id);
+        if (!session) return true;
+        session.closed = true;
+        session.channel?.close();
+        session.peer.close();
+        sessions.delete(id);
+        sizes.delete(id);
+        keyboardActions.delete(id);
+        surfaceActions.delete(id);
+        notifySurface();
+        for (const listener of listeners) listener({ sessionId: id, name: 'closed', payload: {} });
+        return true;
+    },
+    isAvailable() { return true; },
+    addListener(_name, handler) { listeners.add(handler); return { remove: () => { listeners.delete(handler); } }; },
+};
+
+export const desktopAvailable = true;
+
+type KeyboardActions = { show: () => void; hide: () => void; capture: (value: boolean) => void };
+type SurfaceActions = { size: (width: number, height: number) => void; fit: () => void };
+const keyboardActions = new Map<string, KeyboardActions>();
+const surfaceActions = new Map<string, SurfaceActions>();
+export function registerDesktopView(id: string, keyboard: KeyboardActions, surface: SurfaceActions): () => void {
+    keyboardActions.set(id, keyboard);
+    surfaceActions.set(id, surface);
+    const size = sizes.get(id);
+    if (size) surface.size(size.width, size.height);
+    return () => {
+        if (keyboardActions.get(id) === keyboard) keyboardActions.delete(id);
+        if (surfaceActions.get(id) === surface) surfaceActions.delete(id);
+        nativeDesklink.sendControl(id, '{"kind":"release_all"}');
+    };
+}
+export function emitDesktopKeyboard(id: string, payload: Record<string, unknown>) { emit(id, 'keyboard', payload); }

package/src/native.ts

@@ -2,14 +2,9 @@ import { Platform } from 'react-native';
 import { requireOptionalNativeModule } from 'expo-modules-core';
 
 
-
 /**
- * The native session module.
- *
- * Everywhere but Android the module is absent, `desktopAvailable` is false, and
- * the package still exports its types plus the web surface — an application
- * renders one screen on every platform and branches on capability, not on
- * platform name.
+ * Android's native session module. Metro resolves `native.ios.ts` instead on
+ * iOS, where the receiver uses the app's react-native-webrtc binding.
  */
 export type NativeEventName =
     | 'ready'

package/src/native.web.ts

@@ -1,4 +1,5 @@
 import type { NativeDesklinkModule, NativeEventName, NativeSessionEvent } from './native';
+import type { GestureProfile } from './protocol';
 import { observeWebKeyboardMotion } from './webKeyboardMotion';
 
 /**
@@ -114,6 +115,8 @@ interface WebSession {
     longPress: ReturnType<typeof setTimeout> | null;
     /** While a sticky modifier waits for its key, typing is the session's to chord. */
     captured: boolean;
+    /** Which touch meaning this surface uses; a browser page and a device screen answer one finger differently. */
+    gestures: GestureProfile;
     inputEnabled: boolean;
     /** The word the keyboard is composing, and how much of it has already gone. */
     composition: string;
@@ -378,8 +381,9 @@ function cancelLongPress(session: WebSession): void {
 function pointer(session: WebSession, phase: 'move' | 'down' | 'up' | 'cancel', at: { x: number; y: number }, button?: number): void {
     control(session, { kind: 'pointer', phase, x: at.x, y: at.y, ...(button === undefined ? {} : { button }), seq: seq(session) });
     // A mouse brings its own cursor; a finger leaves nothing on the glass to
-    // show where the desktop's pointer went, so the mark does.
-    if (session.mouse || phase === 'cancel') return;
+    // show where the desktop's pointer went, so the mark does. A device screen
+    // shows the touch itself, so a mark would be a second finger that lies.
+    if (session.mouse || phase === 'cancel' || session.gestures === 'device') return;
     session.pointerAt = { x: at.x, y: at.y };
     placePointer(session);
 }
@@ -391,13 +395,15 @@ function click(session: WebSession, at: { x: number; y: number }, button: number
 
 /**
  * Touch gestures, the ones mature remote-desktop viewers settled on — the same
- * as the native view's:
+ * as the native view's. The profile reshapes one finger only:
  *
  *  - tap: click where the finger lands; two taps: a double click on one spot;
  *  - press and hold: a right click on release, or drag after it to hold the
  *    left button (select text, move a window);
  *  - one finger: move around a zoomed-in desktop; on the whole desktop,
- *    move its pointer, which follows the finger without pressing a button;
+ *    move its pointer, which follows the finger without pressing a button —
+ *    unless the profile is `browser`, where it scrolls the page, or `device`,
+ *    where it lands with the button down and drags with it held;
  *  - two fingers: scroll the desktop under them, or pinch to zoom (and move)
  *    the picture; a quick two-finger tap is a right click.
  *
@@ -534,6 +540,27 @@ function attachGestures(session: WebSession): () => void {
         }
         session.touches.set(event.pointerId, { x, y });
         if (session.touches.size === 1) {
+            // A device screen is pressed, not pointed at: the finger lands with
+            // the button down, drags with it held, and lifts to release. A tap
+            // is the same down and up with no move, so it clicks for free, and
+            // a hold holds for free — there is no long press to arm.
+            if (session.gestures === 'device') {
+                session.downX = x;
+                session.downY = y;
+                session.lastX = x;
+                session.lastY = y;
+                const at = desktopPoint(session, x, y);
+                if (at === null) {
+                    session.gesture = 'letterbox';
+                    return;
+                }
+                session.gesture = 'drag';
+                session.lastTap = null;
+                pointer(session, 'down', at, 1);
+                session.dragX = at.x;
+                session.dragY = at.y;
+                return;
+            }
             session.downOnPicture = desktopPoint(session, x, y) !== null;
             session.gesture = session.view.fitted && !session.downOnPicture ? 'letterbox' : 'pending';
             session.downX = x;
@@ -586,6 +613,16 @@ function attachGestures(session: WebSession): () => void {
                 if (Math.hypot(x - session.downX, y - session.downY) > TOUCH_SLOP) {
                     cancelLongPress(session);
                     session.lastTap = null;
+                    // A page scrolls under one finger; the desktop's pointer
+                    // rides along so the scroll lands where the finger is.
+                    if (session.gestures === 'browser') {
+                        session.gesture = 'scroll';
+                        session.lastX = x;
+                        session.lastY = y;
+                        const at = desktopPoint(session, x, y);
+                        if (at !== null) pointer(session, 'move', at);
+                        return;
+                    }
                     // The whole desktop has nowhere to move to, so the finger
                     // moves the desktop's pointer instead, without a button.
                     session.gesture = session.view.fitted
@@ -620,6 +657,15 @@ function attachGestures(session: WebSession): () => void {
             case 'pinch':
             case 'scroll':
                 if (session.touches.size >= 2) twoFingers();
+                else if (session.gesture === 'scroll' && session.gestures === 'browser') {
+                    // One finger dragging a page: content follows it, in pixel
+                    // deltas turned to detents like the two-finger scroll.
+                    session.wheelX -= (x - session.lastX) / session.view.scale / PIXELS_PER_DETENT;
+                    session.wheelY -= (y - session.lastY) / session.view.scale / PIXELS_PER_DETENT;
+                    session.lastX = x;
+                    session.lastY = y;
+                    flushWheel(false);
+                }
                 return;
             default:
                 return;
@@ -662,6 +708,7 @@ function attachGestures(session: WebSession): () => void {
             return;
         }
         if (gesture === 'pending') tap(x, y);
+        else if (gesture === 'scroll') flushWheel(true);
         else if (gesture === 'hover') hoverTo(x, y);
         else if (gesture === 'armed') {
             const at = desktopPoint(session, session.downX, session.downY);
@@ -902,6 +949,7 @@ export const nativeDesklink: NativeDesklinkModule = {
             lastTap: null,
             longPress: null,
             captured: false,
+            gestures: 'desktop',
             inputEnabled: false,
             composition: '',
             compositionSent: '',
@@ -959,6 +1007,13 @@ export const nativeDesklink: NativeDesklinkModule = {
                 emit(id, 'failure', { code: 'transport', message: 'the connection to the desktop was lost' });
             }
         };
+        peer.oniceconnectionstatechange = () => {
+            if (sessions.get(id) !== session) return;
+            // Surfaced as it happens — upstream of any failure timeout — so a
+            // short drop reads as reconnecting within about a second, not as
+            // a frozen picture that still says live.
+            emit(id, 'ice', { state: peer.iceConnectionState.toUpperCase() });
+        };
         return id;
     },
 
@@ -1130,9 +1185,35 @@ export const nativeDesklink: NativeDesklinkModule = {
  * surface, and removing the old one is what keeps a re-render from leaving two
  * live videos behind.
  */
-export function attachSurface(id: string, container: HTMLElement | null): void {
+/** Which touch meaning a surface uses; changing it mid-session is safe. */
+export function setGestures(id: string, gestures: GestureProfile): void {
+    const session = sessions.get(id);
+    if (session === undefined) return;
+    if (session.gestures === gestures) return;
+    // A held drag belongs to the old meaning: let go rather than release the
+    // button somewhere the finger never meant.
+    if (session.gesture === 'drag' || session.gesture === 'mouse') endGestureDrag(session);
+    cancelLongPress(session);
+    session.gesture = 'none';
+    session.touches.clear();
+    session.gestures = gestures;
+    // A device screen shows the touch itself: a mark left over from another
+    // profile would be a finger that is not there.
+    if (gestures === 'device' && session.pointerAt !== null) {
+        session.pointerAt = null;
+        placePointer(session);
+    }
+}
+
+/** Release a held button at the last desktop point it was sent to. */
+function endGestureDrag(session: WebSession): void {
+    pointer(session, 'up', { x: session.dragX, y: session.dragY }, 1);
+}
+
+export function attachSurface(id: string, container: HTMLElement | null, gestures: GestureProfile = 'desktop'): void {
     const session = sessions.get(id);
     if (session === undefined) return;
+    session.gestures = gestures;
     session.detach?.();
     session.detach = null;
     if (session.surface !== null) session.surface.remove();

package/src/protocol.ts

@@ -14,10 +14,18 @@
  * only describe the subset a client needs.
  */
 
-export const PROTOCOL_VERSION = 2;
+export const PROTOCOL_VERSION = 3;
 
 export type Permission = 'view' | 'control' | 'clipboard';
 
+/**
+ * Which touch meaning a surface uses. `desktop` is the trackpad-shaped
+ * default; `browser` scrolls a page with one finger; `device` presses the
+ * screen with one finger, like the finger itself was there. The wire messages
+ * are the same under all three — only which gesture sends which changes.
+ */
+export type GestureProfile = 'desktop' | 'browser' | 'device';
+
 export interface IceServerConfig {
     urls: string[];
     username?: string;
@@ -61,13 +69,21 @@ export interface RtcCandidate {
     sdpMLineIndex?: number | null;
 }
 
-/** Engine → client lifecycle events, already unwrapped from the local protocol. */
+/**
+ * Engine → client lifecycle events, already unwrapped from the local protocol.
+ *
+ * `sessionId` names the engine session the event belongs to, when the
+ * carrier preserves it. The hook ignores an event whose id does not match
+ * the session it holds: after an outage, a previous session's queued offer
+ * or revocation can arrive beside the new session's, and applying those to
+ * the new session corrupts or kills a healthy recovery.
+ */
 export type SessionEvent =
-    | { kind: 'description'; description: RtcDescription }
-    | { kind: 'candidate'; candidate: RtcCandidate }
-    | { kind: 'state'; capture: string; transport: string; firstFrame: boolean }
-    | { kind: 'restoreToken'; token: string }
-    | { kind: 'revoked'; reason: string };
+    | { kind: 'description'; description: RtcDescription; sessionId?: string }
+    | { kind: 'candidate'; candidate: RtcCandidate; sessionId?: string }
+    | { kind: 'state'; capture: string; transport: string; firstFrame: boolean; sessionId?: string }
+    | { kind: 'restoreToken'; token: string; sessionId?: string }
+    | { kind: 'revoked'; reason: string; sessionId?: string };
 
 /**
  * The application's own authenticated channel to the host engine.
@@ -139,7 +155,14 @@ export type ControlReply =
     | { kind: 'ack'; seq: number }
     | { kind: 'rejected'; seq: number; code: string; message: string }
     | { kind: 'clipboard'; request: string; text: string; truncated?: boolean; error?: string }
-    | { kind: 'revoked'; reason: string };
+    | { kind: 'revoked'; reason: string }
+    /**
+     * The engine's heartbeat, twice a second while the control channel is
+     * open. A client that has seen one and then sees none for a while reads
+     * the path as stalled, long before ICE consent timers notice. Clients that
+     * predate it simply ignore it.
+     */
+    | { kind: 'ping' };
 
 export function parseControlReply(raw: string): ControlReply | null {
     try {

package/src/stickyModifiers.ts

@@ -1,18 +1,18 @@
 import type { ControlMessage } from './protocol';
 
 /**
- * Ctrl and Shift for a touch keyboard, which has neither.
+ * Desktop modifiers for a touch keyboard.
  *
  * The cycle is the terminal key row's: one tap arms a modifier for the next
  * key, a second tap locks it, a third lets it go. An armed modifier travels
  * with the key it modifies, as that key's own chord, so the desktop never holds
  * a modifier on its own and nothing can be left pressed between keys.
  */
-export type StickyModifier = 'Control' | 'Shift';
+export type StickyModifier = 'Control' | 'Shift' | 'Alt' | 'Meta';
 export type StickyState = 'off' | 'once' | 'lock';
 export type StickyModifiers = Readonly<Record<StickyModifier, StickyState>>;
 
-export const NO_MODIFIERS: StickyModifiers = { Control: 'off', Shift: 'off' };
+export const NO_MODIFIERS: StickyModifiers = { Control: 'off', Shift: 'off', Alt: 'off', Meta: 'off' };
 
 const AFTER_TAP: Record<StickyState, StickyState> = { off: 'once', once: 'lock', lock: 'off' };
 
@@ -22,7 +22,7 @@ export function tapModifier(current: StickyModifiers, name: StickyModifier): Sti
 
 /** The modifiers the next key carries, in the engine's names. */
 export function armedModifiers(current: StickyModifiers): StickyModifier[] {
-    return (['Control', 'Shift'] as const).filter((name) => current[name] !== 'off');
+    return (['Control', 'Shift', 'Alt', 'Meta'] as const).filter((name) => current[name] !== 'off');
 }
 
 /** What stays armed once a key has used the modifiers: a lock stays, the rest let go. */
@@ -30,6 +30,8 @@ export function afterKey(current: StickyModifiers): StickyModifiers {
     return {
         Control: current.Control === 'lock' ? 'lock' : 'off',
         Shift: current.Shift === 'lock' ? 'lock' : 'off',
+        Alt: current.Alt === 'lock' ? 'lock' : 'off',
+        Meta: current.Meta === 'lock' ? 'lock' : 'off',
     };
 }