@limrun/ui 0.17.0-rc.1 → 0.17.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +18 -3
  2. package/dist/{GLTFLoader-DW6wpvyR.mjs → GLTFLoader-B4nnixLv.mjs} +1 -1
  3. package/dist/{GLTFLoader-BKnsBFsj.js → GLTFLoader-GATLvS0U.js} +1 -1
  4. package/dist/components/duo-fold-motion.d.ts +1 -1
  5. package/dist/components/duo-view.d.ts +5 -1
  6. package/dist/{duo-frame-DjfA4gG-.js → duo-frame-CfBILL32.js} +2 -2
  7. package/dist/{duo-frame-BzyYljix.mjs → duo-frame-F3hwc09B.mjs} +406 -406
  8. package/dist/duo-view-BsdKpp8d.mjs +662 -0
  9. package/dist/duo-view-Cy8kBdg9.js +1 -0
  10. package/dist/index.cjs +1 -1
  11. package/dist/index.css +1 -1
  12. package/dist/index.js +2 -2
  13. package/dist/lite.cjs +1 -1
  14. package/dist/lite.js +1 -1
  15. package/dist/{remote-control-CQjSEVDF.mjs → remote-control-BAmNHDdv.mjs} +109 -108
  16. package/dist/{remote-control-DVRXfCR2.js → remote-control-F3h8U5pg.js} +1 -1
  17. package/package.json +5 -1
  18. package/dist/demo.d.ts +0 -1
  19. package/dist/duo-view-COZg8ieG.js +0 -1
  20. package/dist/duo-view-o1yxIzxu.mjs +0 -654
  21. package/index.html +0 -199
  22. package/src/assets/Apple_logo_white.svg +0 -4
  23. package/src/assets/android_boot.gif +0 -0
  24. package/src/assets/android_boot.webp +0 -0
  25. package/src/assets/iphone16pro_black.webp +0 -0
  26. package/src/assets/iphone16pro_black_bg.webp +0 -0
  27. package/src/assets/iphone16pro_black_landscape.webp +0 -0
  28. package/src/assets/iphone16pro_black_landscape_bg.webp +0 -0
  29. package/src/assets/pixel9_black.webp +0 -0
  30. package/src/assets/pixel9_black_landscape.webp +0 -0
  31. package/src/assets/pixel_tablet_landscape.webp +0 -0
  32. package/src/assets/pixel_tablet_portrait.webp +0 -0
  33. package/src/components/device-install/index.ts +0 -1
  34. package/src/components/duo-button-hinge.test.tsx +0 -138
  35. package/src/components/duo-button-hinge.ts +0 -75
  36. package/src/components/duo-controls.tsx +0 -142
  37. package/src/components/duo-flat-frame.tsx +0 -145
  38. package/src/components/duo-flat-geometry.test.ts +0 -63
  39. package/src/components/duo-flat-geometry.ts +0 -97
  40. package/src/components/duo-fold-motion.test.tsx +0 -245
  41. package/src/components/duo-fold-motion.tsx +0 -192
  42. package/src/components/duo-fold-renderer.test.ts +0 -29
  43. package/src/components/duo-fold-renderer.ts +0 -397
  44. package/src/components/duo-fold-timing.test.ts +0 -44
  45. package/src/components/duo-fold-timing.ts +0 -9
  46. package/src/components/duo-frame.tsx +0 -579
  47. package/src/components/duo-hardware.test.ts +0 -61
  48. package/src/components/duo-hardware.ts +0 -61
  49. package/src/components/duo-model.test.ts +0 -243
  50. package/src/components/duo-model.ts +0 -607
  51. package/src/components/duo-screen-handoff.ts +0 -52
  52. package/src/components/duo-view.test.tsx +0 -104
  53. package/src/components/duo-view.tsx +0 -315
  54. package/src/components/inspect-overlay.css +0 -223
  55. package/src/components/inspect-overlay.tsx +0 -437
  56. package/src/components/remote-control-framed.tsx +0 -14
  57. package/src/components/remote-control.css +0 -441
  58. package/src/components/remote-control.test.ts +0 -34
  59. package/src/components/remote-control.tsx +0 -3989
  60. package/src/core/ax-fetcher.test.ts +0 -418
  61. package/src/core/ax-fetcher.ts +0 -387
  62. package/src/core/ax-tree.test.ts +0 -491
  63. package/src/core/ax-tree.ts +0 -416
  64. package/src/core/camera.test.ts +0 -155
  65. package/src/core/camera.ts +0 -133
  66. package/src/core/constants.ts +0 -344
  67. package/src/core/duo.test.ts +0 -142
  68. package/src/core/duo.ts +0 -156
  69. package/src/core/webrtc-messages.ts +0 -148
  70. package/src/demo.tsx +0 -283
  71. package/src/frame-assets.ts +0 -22
  72. package/src/hooks/index.ts +0 -1
  73. package/src/index.ts +0 -29
  74. package/src/lite.test.ts +0 -30
  75. package/src/lite.ts +0 -30
  76. package/src/vite-env.d.ts +0 -1
  77. package/tsconfig.json +0 -26
  78. package/tsconfig.node.json +0 -26
  79. package/vite.config.ts +0 -35
  80. package/vitest.config.ts +0 -24
@@ -1,3989 +0,0 @@
1
- import React, {
2
- useEffect,
3
- useRef,
4
- useMemo,
5
- useState,
6
- forwardRef,
7
- useImperativeHandle,
8
- lazy,
9
- Suspense,
10
- } from 'react';
11
- import { clsx } from 'clsx';
12
- import './remote-control.css';
13
-
14
- import { createDisplayTouchMessage, type DuoState, type DuoOrientation } from '../core/duo';
15
- const DuoView = lazy(() => import('./duo-view'));
16
-
17
- import { ANDROID_KEYS, AMOTION_EVENT, codeMap } from '../core/constants';
18
-
19
- import {
20
- createTouchControlMessage,
21
- createInjectKeycodeMessage,
22
- createSetClipboardMessage,
23
- createTwoFingerTouchControlMessage,
24
- } from '../core/webrtc-messages';
25
- import { AxFetcher, AxStatus, type AndroidElementTreeOptions } from '../core/ax-fetcher';
26
- import { AxElement, AxSnapshot, axElementAtPoint, axSnapshotsEqual } from '../core/ax-tree';
27
- import {
28
- CameraOperationCoordinator,
29
- cameraCaptureConstraints,
30
- createGrantedCameraResult,
31
- parseCameraRequest,
32
- shouldReacquireCamera,
33
- type CameraCaptureInfo,
34
- type CameraRequest,
35
- } from '../core/camera';
36
- import { InspectOverlay, InspectOverlayGeometry, InspectMode } from './inspect-overlay';
37
-
38
- export type { CameraCaptureInfo, CameraFacingMode, CameraRequest, CameraResult } from '../core/camera';
39
-
40
- declare global {
41
- interface Window {
42
- debugRemoteControl?: boolean;
43
- }
44
- }
45
-
46
- export interface RemoteControlProps {
47
- // url is the URL of the instance to connect to.
48
- url: string;
49
-
50
- // token is used to authenticate the connection to the instance.
51
- token: string;
52
-
53
- // className is the class name to apply to the component
54
- // on top of the default styles.
55
- className?: string;
56
-
57
- // sessionId is a unique identifier for the WebRTC session
58
- // with the source to prevent conflicts between other
59
- // users connected to the same source.
60
- // If empty, the component will generate a random one.
61
- sessionId?: string;
62
-
63
- // openUrl is the URL to open in the instance when the
64
- // component is ready.
65
- //
66
- // If not provided, the component will not open any URL.
67
- openUrl?: string;
68
-
69
- // showFrame controls whether to display the device frame
70
- // around the video. Defaults to true.
71
- showFrame?: boolean;
72
-
73
- /** Enables the official iPhone Duo's native displays and interactive folding frame. */
74
- deviceModel?: 'iphone-duo';
75
-
76
- /** Optional prepared Duo GLB, supplied and licensed by the host application. */
77
- duoModelUrl?: string;
78
-
79
- // When true, drops after a working session auto-reconnect instead of
80
- // surfacing the manual "Retry" button. Defaults to false.
81
- autoReconnect?: boolean;
82
-
83
- /**
84
- * Fires once when the component concludes the instance is permanently
85
- * gone rather than suffering a recoverable connection blip.
86
- *
87
- * After a drop, the component probes the instance endpoint over HTTP;
88
- * a 404 means the instance no longer exists (terminated/deleted). When
89
- * that happens the component stops reconnecting and renders a built-in
90
- * "terminated" message in place of the "Retry" button, and invokes this
91
- * callback so the host can react (navigate away, surface its own UI,
92
- * etc.). It is not called for ordinary network drops.
93
- */
94
- onTerminated?: () => void;
95
- /**
96
- * Device frame and boot-logo images. The default export bundles Limrun's
97
- * frames; `@limrun/ui/lite` omits them so a frameless embed stays small.
98
- */
99
- assets?: RemoteControlAssets;
100
- /**
101
- * Coarse connection lifecycle, for status UI outside the component.
102
- */
103
- onConnectionStateChange?: (state: RemoteControlConnectionState) => void;
104
-
105
- /**
106
- * Enable the inspect overlay. When set, the component starts polling the
107
- * accessibility tree and draws boxes over each element on top of the
108
- * video stream.
109
- *
110
- * - `true` — Select mode. Boxes are clickable, click pins a selection
111
- * with action buttons (Tap / Copy selector / Copy id), ESC clears.
112
- * Device input is blocked while in this mode.
113
- * - `'hover-only'` — Boxes follow the cursor as a visual preview. Device
114
- * input still passes through, so you can drive the simulator while
115
- * inspecting.
116
- * - `undefined` / `false` (default) — overlay disabled, no polling.
117
- */
118
- inspectMode?: boolean | 'hover-only';
119
-
120
- /**
121
- * Fires whenever a fresh accessibility snapshot is delivered.
122
- *
123
- * Customers use this to drive their own side panels, agent prompts,
124
- * analytics, etc. The built-in overlay does not require this callback —
125
- * it renders from internal state regardless.
126
- *
127
- * Identical-to-previous snapshots (per `axSnapshotsEqual`) are NOT
128
- * re-emitted, so a stable UI doesn't generate callback noise.
129
- *
130
- * Invoked in a microtask so customer code doesn't run synchronously
131
- * inside React's commit phase.
132
- */
133
- onAxSnapshotChange?: (snapshot: AxSnapshot | null) => void;
134
-
135
- /**
136
- * Fires when the user clicks an overlay element (only emitted when
137
- * `inspectMode === true`). `null` indicates a deselection (ESC, click
138
- * outside any box, or programmatic clear).
139
- *
140
- * The `snapshot` field is the snapshot active at the moment of the
141
- * click — useful for capturing context without races against the next
142
- * poll cycle.
143
- */
144
- onInspectSelectionChange?: (selection: { element: AxElement; snapshot: AxSnapshot } | null) => void;
145
-
146
- /**
147
- * Fires whenever the accessibility subsystem changes coarse-grained
148
- * status. Useful for rendering readiness indicators or error banners in
149
- * a customer-built side panel.
150
- *
151
- * Transitions are deduplicated; no self-loops are emitted. The `error`
152
- * argument is populated when status is `error` or `unavailable`.
153
- *
154
- * Lifecycle: `idle` → `starting` → `ready` (or `unavailable` / `error`).
155
- * Recovery from `error` / `unavailable` is automatic — the fetcher
156
- * keeps polling and transitions back to `ready` on the next success.
157
- */
158
- onAxStatusChange?: (status: AxStatus, error?: string) => void;
159
-
160
- /**
161
- * Base interval (ms) between successful AX-tree fetches.
162
- *
163
- * The fetcher will:
164
- * - Wait `axPollIntervalMs` after a successful fetch with NEW data.
165
- * - Double the wait (up to `axMaxBackoffMs`) when consecutive snapshots
166
- * are byte-identical (e.g. static screen).
167
- * - Wait 5 s when the server reports AX is unavailable.
168
- *
169
- * In addition, after user input (taps, scrolls, keypresses, openUrl,
170
- * terminateApp, orientation flips), the fetcher enters a short
171
- * "activity boost" window (~1.2 s) during which fetches happen at
172
- * ~250 ms regardless of this setting. This captures mid-animation UI
173
- * changes without you having to manually call `refreshAxTree`.
174
- *
175
- * @default 500
176
- */
177
- axPollIntervalMs?: number;
178
-
179
- /**
180
- * Maximum backoff (ms) for the AX-tree polling loop when consecutive
181
- * snapshots are unchanged.
182
- *
183
- * @default 2000
184
- */
185
- axMaxBackoffMs?: number;
186
-
187
- androidElementTreeOptions?: AndroidElementTreeOptions;
188
-
189
- /**
190
- * Fires whenever the iOS simulator's camera demand state changes —
191
- * i.e. an app inside the sim called
192
- * `[AVCaptureSession startRunning]` or `[stopRunning]`. The
193
- * component handles the `navigator.mediaDevices.getUserMedia` prompt
194
- * and SDP plumbing internally; this callback is purely so the host
195
- * UI can render a status indicator ("simulator is using your
196
- * camera", etc.).
197
- *
198
- * `active` reflects whether the sim is currently asking for frames.
199
- * `granted` is set on capture acknowledgements: `true` after a new
200
- * track, reused track, or constraint update; `false` when capture
201
- * fails. The host switches to black frames after a failure.
202
- *
203
- * `camera` (optional, only meaningful when `granted === true`)
204
- * carries what `MediaStreamTrack.getSettings()` reported for the
205
- * active capture: resolution, framerate, device label, facing
206
- * mode. Use it to render a richer status indicator (e.g.
207
- * "Camera · 1920×1080 · 30 fps · FaceTime HD").
208
- *
209
- * iOS instances always fire this callback. Android instances fire it
210
- * when the server supports camera injection (Android 15+); older
211
- * Android instances have no camera-injector path and stay silent.
212
- */
213
- onCameraDemandChange?: (active: boolean, granted?: boolean, camera?: CameraCaptureInfo) => void;
214
-
215
- /**
216
- * Periodically (~1Hz) fires with a snapshot of the outbound
217
- * camera stream's live WebRTC stats — codec, encoder
218
- * implementation, hardware acceleration, encoded fps, bitrate,
219
- * round-trip-time, and the encoder's
220
- * `qualityLimitationReason` (which is what Meet/Zoom use to
221
- * decide whether to show "Bandwidth limited" or "CPU limited"
222
- * banners).
223
- *
224
- * Only fires while the simulator is actively pulling the
225
- * camera AND `getUserMedia` was granted. `null` is emitted
226
- * once when the stream goes back to idle so consumers can
227
- * clear their UI without having to maintain their own
228
- * timeout. The host app does not need to poll `getStats()`
229
- * itself — this is the canonical place.
230
- */
231
- onCameraStats?: (stats: CameraStreamStats | null) => void;
232
-
233
- /**
234
- * When true, capture the user's microphone via
235
- * `getUserMedia({ audio: true })` and stream it live into the
236
- * device's mock microphone. This is a manual override: apps
237
- * that actively consume microphone input also trigger capture
238
- * automatically, like camera demand. The component handles the
239
- * browser permission prompt, WebRTC track attachment, and
240
- * `microphoneStart`/`microphoneStop` signaling. Progress and failures
241
- * are reported via `onMicrophoneStateChange`. Ignored on Android
242
- * servers without camera/mic injection support (Android 14 and older).
243
- */
244
- microphoneEnabled?: boolean;
245
-
246
- /**
247
- * Fires whenever the live-microphone pipeline changes state:
248
- * after a manual or guest-demanded attempt resolves, when the host
249
- * acknowledges or rejects the stream, when the browser revokes the
250
- * track mid-stream, and when capture stops.
251
- */
252
- onMicrophoneStateChange?: (state: MicrophoneState) => void;
253
- }
254
-
255
- /**
256
- * Snapshot of the live-microphone pipeline, reported through
257
- * `onMicrophoneStateChange`.
258
- */
259
- export interface MicrophoneState {
260
- /** True while the browser mic is captured and streaming to the sim. */
261
- active: boolean;
262
- /**
263
- * Human-readable failure from the last attempt, if any (permission
264
- * denied, no device, host rejection, ...).
265
- */
266
- error?: string;
267
- /** Device label of the captured microphone, when available. */
268
- label?: string;
269
- }
270
-
271
- /**
272
- * Live outbound-camera quality snapshot. Derived from
273
- * `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
274
- * once per second while the camera is sending. All fields optional:
275
- * some browsers omit fields (Safari rarely reports
276
- * `encoderImplementation`), and the first sample after camera start
277
- * has no delta-derived numbers yet.
278
- */
279
- export interface CameraStreamStats {
280
- /** "H264", "HEVC"/"H265", "VP9", "VP8", "AV1", etc. (uppercased). */
281
- codec?: string;
282
- /** e.g. "VideoToolbox" (hw), "OpenH264" (sw), "ExternalEncoder". */
283
- encoderImplementation?: string;
284
- /**
285
- * Browser-reported hardware-acceleration hint. Some Chromium
286
- * versions expose this via `powerEfficientEncoder`; we mirror it
287
- * here so consumers don't have to know the spec quirk.
288
- */
289
- hardwareAccelerated?: boolean;
290
- /** Outbound encoded fps over the last sample window. */
291
- framesPerSecond?: number;
292
- /** Cumulative encoded frame count. */
293
- framesEncoded?: number;
294
- /** Outbound encoded bitrate (bits/s) over the last window. */
295
- bitrateBps?: number;
296
- /** Width/height the encoder is currently producing. */
297
- width?: number;
298
- height?: number;
299
- /**
300
- * One of `'none' | 'cpu' | 'bandwidth' | 'other'`. Mirrors
301
- * Meet/Zoom's "limited by …" banners. Anything other than
302
- * `'none'` means the encoder dropped resolution to keep up.
303
- */
304
- qualityLimitationReason?: string;
305
- /** Round-trip time in milliseconds, from the matching RTCP. */
306
- rttMs?: number;
307
- /** Packet-loss percentage over the last sample window (0..100). */
308
- packetsLostPct?: number;
309
- }
310
-
311
- interface ScreenshotData {
312
- dataUri: string;
313
- }
314
-
315
- export interface ImperativeKeyboardEvent {
316
- type: 'keydown' | 'keyup';
317
- code: string; // e.g., "KeyA", "Enter", "ShiftLeft"
318
- shiftKey?: boolean;
319
- altKey?: boolean;
320
- ctrlKey?: boolean;
321
- metaKey?: boolean;
322
- }
323
-
324
- export interface RemoteControlHandle {
325
- openUrl: (url: string) => void;
326
- sendKeyEvent: (event: ImperativeKeyboardEvent) => void;
327
- screenshot: (display?: 'outer' | 'inner') => Promise<ScreenshotData>;
328
- terminateApp: (bundleId: string) => Promise<void>;
329
- reconnect: () => void;
330
-
331
- // Inspect-mode helpers. These are no-ops when inspect mode is disabled or
332
- // the WebSocket isn't open.
333
-
334
- // Force a fresh accessibility-tree fetch outside the normal poll cadence.
335
- refreshAxTree: () => Promise<AxSnapshot>;
336
-
337
- // Pull-based access to the most recent snapshot (the same one passed to
338
- // onAxSnapshotChange). Returns null when no snapshot has arrived yet or
339
- // when inspect mode is off.
340
- getAxSnapshot: () => AxSnapshot | null;
341
-
342
- // Programmatically drive the overlay highlight/selection — useful when a
343
- // customer's own side panel wants to cross-highlight with the overlay.
344
- // Pass `null` to clear.
345
- setInspectHighlight: (element: AxElement | null) => void;
346
- setInspectSelection: (element: AxElement | null) => void;
347
-
348
- // Pull-based access to the current AX subsystem status. Mirrors what
349
- // onAxStatusChange reports, for customers that don't want to subscribe.
350
- getAxStatus: () => AxStatus;
351
- }
352
-
353
- const debugLog = (...args: any[]) => {
354
- if (window.debugRemoteControl) {
355
- console.log(...args);
356
- }
357
- };
358
-
359
- const debugWarn = (...args: any[]) => {
360
- if (window.debugRemoteControl) {
361
- console.warn(...args);
362
- }
363
- };
364
-
365
- // Invokes a customer-provided callback in isolation. A throw from the
366
- // customer's code must NOT propagate back into our state-update flow — that
367
- // would risk corrupting React reconciliation. We log the error to the
368
- // console so the customer can still debug, but otherwise swallow.
369
- const safeInvoke = <Args extends unknown[]>(
370
- label: string,
371
- fn: ((...args: Args) => unknown) | undefined,
372
- ...args: Args
373
- ): void => {
374
- if (!fn) return;
375
- try {
376
- fn(...args);
377
- } catch (err) {
378
- // Surface to the developer regardless of debug flag — this is a bug
379
- // in the customer's handler and they'll want to see it.
380
- // eslint-disable-next-line no-console
381
- console.error(`[RemoteControl] customer callback "${label}" threw:`, err);
382
- }
383
- };
384
-
385
- const motionActionToString = (action: number): string => {
386
- // AMOTION_EVENT is a constants object; find the matching ACTION_* key if present
387
- const match = Object.entries(AMOTION_EVENT).find(
388
- ([key, value]) => key.startsWith('ACTION_') && value === action,
389
- );
390
- return match?.[0] ?? String(action);
391
- };
392
-
393
- type DevicePlatform = 'ios' | 'android';
394
-
395
- const detectPlatform = (url: string): DevicePlatform => {
396
- if (url.includes('/android_')) {
397
- return 'android';
398
- }
399
- // Default to iOS if no Android pattern is found
400
- return 'ios';
401
- };
402
-
403
- // The endpoint may already carry query parameters, and the token may contain URL syntax.
404
- export const withAuthenticationToken = (url: string, token: string): string => {
405
- const endpoint = new URL(url);
406
- endpoint.searchParams.set('token', token);
407
- return endpoint.toString();
408
- };
409
-
410
- export type RemoteControlConnectionState =
411
- | 'connecting'
412
- | 'connected'
413
- | 'reconnecting'
414
- | 'failed'
415
- | 'terminated';
416
-
417
- export type RemoteControlPlatformAssets = {
418
- frame?: string;
419
- frameLandscape?: string;
420
- tabletFrame?: string;
421
- tabletFrameLandscape?: string;
422
- loadingLogo?: string;
423
- };
424
-
425
- export type RemoteControlAssets = Partial<Record<DevicePlatform, RemoteControlPlatformAssets>>;
426
-
427
- type DeviceConfig = {
428
- videoBorderRadiusMultiplier: number;
429
- loadingLogoSize: string;
430
- videoPosition: {
431
- portrait: { heightMultiplier?: number; widthMultiplier?: number };
432
- landscape: { heightMultiplier?: number; widthMultiplier?: number };
433
- };
434
- };
435
-
436
- const ANDROID_TABLET_VIDEO_WIDTH = 1920;
437
- const ANDROID_TABLET_VIDEO_HEIGHT = 1200;
438
- const MAX_CONNECTION_ATTEMPTS = 3;
439
- const CONNECTION_RETRY_DELAY_MS = 1000;
440
- const CONNECTION_SUCCESS_TIMEOUT_MS = 15000;
441
- const ICE_DISCONNECTED_GRACE_MS = 3000;
442
- // Upper bound for the one-shot HTTP probe that disambiguates a terminated
443
- // instance (endpoint 404s) from a transient connection drop.
444
- const TERMINATION_PROBE_TIMEOUT_MS = 4000;
445
-
446
- const isAndroidTabletVideo = (width: number, height: number): boolean =>
447
- (width === ANDROID_TABLET_VIDEO_WIDTH && height === ANDROID_TABLET_VIDEO_HEIGHT) ||
448
- (width === ANDROID_TABLET_VIDEO_HEIGHT && height === ANDROID_TABLET_VIDEO_WIDTH);
449
-
450
- // Device-specific configuration for frame sizing and video positioning
451
- // Video position percentages are relative to the frame image dimensions
452
- const deviceConfig: Record<DevicePlatform, DeviceConfig> = {
453
- ios: {
454
- videoBorderRadiusMultiplier: 0.15,
455
- loadingLogoSize: '20%',
456
- // Video position as percentage of frame dimensions
457
- videoPosition: {
458
- portrait: { heightMultiplier: 0.9678 },
459
- landscape: { widthMultiplier: 0.9678 },
460
- },
461
- },
462
- android: {
463
- videoBorderRadiusMultiplier: 0.13,
464
- loadingLogoSize: '40%',
465
- // Video position as percentage of frame dimensions
466
- videoPosition: {
467
- portrait: { heightMultiplier: 0.967 },
468
- landscape: { widthMultiplier: 0.962 },
469
- },
470
- },
471
- };
472
-
473
- function getAndroidKeycodeAndMeta(event: React.KeyboardEvent): { keycode: number; metaState: number } | null {
474
- const code = event.code;
475
- const keycode = codeMap[code];
476
-
477
- if (!keycode) {
478
- // Use the wrapper for conditional warning
479
- debugWarn(`Unknown event.code: ${code}, key: ${event.key}`);
480
- return null;
481
- }
482
-
483
- let metaState = ANDROID_KEYS.META_NONE;
484
- const isLetter = code >= 'KeyA' && code <= 'KeyZ';
485
- const isCapsLock = event.getModifierState('CapsLock');
486
- const isShiftPressed = event.shiftKey;
487
-
488
- // Determine effective shift state
489
- let effectiveShift = isShiftPressed;
490
- if (isLetter) {
491
- effectiveShift = isShiftPressed !== isCapsLock; // Logical XOR for booleans
492
- }
493
-
494
- // Apply meta states
495
- if (effectiveShift) metaState |= ANDROID_KEYS.META_SHIFT_ON;
496
- if (event.ctrlKey) metaState |= ANDROID_KEYS.META_CTRL_ON;
497
- if (event.altKey) metaState |= ANDROID_KEYS.META_ALT_ON;
498
- if (event.metaKey) metaState |= ANDROID_KEYS.META_META_ON; // Command on Mac, Windows key on Win
499
-
500
- return { keycode, metaState };
501
- }
502
-
503
- export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>(
504
- (
505
- {
506
- className,
507
- url,
508
- token,
509
- sessionId: propSessionId,
510
- openUrl,
511
- showFrame = true,
512
- deviceModel,
513
- duoModelUrl,
514
- autoReconnect = false,
515
- onTerminated,
516
- assets,
517
- onConnectionStateChange,
518
- inspectMode,
519
- onAxSnapshotChange,
520
- onInspectSelectionChange,
521
- onAxStatusChange,
522
- axPollIntervalMs,
523
- axMaxBackoffMs,
524
- androidElementTreeOptions,
525
- onCameraDemandChange,
526
- onCameraStats,
527
- microphoneEnabled = false,
528
- onMicrophoneStateChange,
529
- }: RemoteControlProps,
530
- ref,
531
- ) => {
532
- const containerRef = useRef<HTMLDivElement>(null);
533
- const videoRef = useRef<HTMLVideoElement>(null);
534
- const frameRef = useRef<HTMLImageElement>(null);
535
- const [detectedDuo, setDetectedDuo] = useState(false);
536
- const isDuo = deviceModel === 'iphone-duo' || detectedDuo;
537
- const innerVideoRef = useRef<HTMLVideoElement>(null);
538
- const [duoState, setDuoState] = useState<DuoState | null>(null);
539
- const [duoError, setDuoError] = useState<string>();
540
- const duoRequests = useRef(
541
- new Map<
542
- string,
543
- { resolve: () => void; reject: (error: Error) => void; timer: ReturnType<typeof setTimeout> }
544
- >(),
545
- );
546
- const sendDuoControl = (type: string, parameters: object): Promise<void> =>
547
- new Promise((resolve, reject) => {
548
- const socket = wsRef.current;
549
- if (!socket || socket.readyState !== WebSocket.OPEN) {
550
- reject(new Error('Simulator disconnected'));
551
- return;
552
- }
553
- const id = crypto.randomUUID();
554
- const timer = setTimeout(() => {
555
- duoRequests.current.delete(id);
556
- reject(new Error('Device control timed out'));
557
- }, 5000);
558
- duoRequests.current.set(id, { resolve, reject, timer });
559
- socket.send(JSON.stringify({ type, id, ...parameters }));
560
- });
561
- const [videoLoaded, setVideoLoaded] = useState(false);
562
- const [retryExhausted, setRetryExhausted] = useState(false);
563
- // Set once we've concluded the instance is permanently gone (its
564
- // endpoint 404s after a drop), as opposed to a recoverable blip. When
565
- // set, reconnection stops and a terminated message replaces "Retry".
566
- const [terminated, setTerminated] = useState(false);
567
- const terminatedRef = useRef(false);
568
- // Guards against firing overlapping termination probes.
569
- const terminationProbeInFlightRef = useRef(false);
570
- const [isLandscape, setIsLandscape] = useState(false);
571
- const [useAndroidTabletFrame, setUseAndroidTabletFrame] = useState(false);
572
- const [videoStyle, setVideoStyle] = useState<React.CSSProperties>({});
573
- const wsRef = useRef<WebSocket | null>(null);
574
- const peerConnectionRef = useRef<RTCPeerConnection | null>(null);
575
- const dataChannelRef = useRef<RTCDataChannel | null>(null);
576
- const keepAliveIntervalRef = useRef<number | undefined>(undefined);
577
- const retryTimeoutRef = useRef<number | undefined>(undefined);
578
- const connectionSuccessTimeoutRef = useRef<number | undefined>(undefined);
579
- const requestFrameIntervalRef = useRef<number | undefined>(undefined);
580
- const iceDisconnectedGraceRef = useRef<number | undefined>(undefined);
581
- const connectionGenerationRef = useRef(0);
582
- const connectionAttemptRef = useRef(0);
583
- const controlChannelOpenedRef = useRef(false);
584
- // Mirrored to a ref so stale closures in event handlers see the latest value.
585
- const autoReconnectRef = useRef(autoReconnect);
586
- autoReconnectRef.current = autoReconnect;
587
- // Demand-driven outbound camera state.
588
- //
589
- // The limulator side broadcasts a `cameraRequest` WS message whenever
590
- // an app inside the simulator opens/closes an `AVCaptureSession`.
591
- // We respond by calling `navigator.mediaDevices.getUserMedia(...)`
592
- // and `replaceTrack`ing the result onto a pre-allocated sendonly
593
- // video transceiver. Pre-allocating the transceiver lets us
594
- // attach/detach the local camera without renegotiating the SDP
595
- // (the slot is already in the answer's a=video block, just with
596
- // `inactive`/empty until we install the track).
597
- //
598
- // Refs (not state) because all callers live inside event-handler
599
- // closures and the ref read happens during message processing, not
600
- // during render. We keep both the sender and the active local
601
- // stream so teardown can stop tracks without leaking the camera
602
- // green light.
603
- const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
604
- const outboundLocalStreamRef = useRef<MediaStream | null>(null);
605
- const outboundCameraRequestRef = useRef<CameraRequest | undefined>(undefined);
606
- const cameraOperations = useMemo(() => new CameraOperationCoordinator(), []);
607
- // Mirror the demand-change callback into a ref so the WS message
608
- // handler always sees the freshest customer callback even when the
609
- // parent re-renders mid-session.
610
- const onCameraDemandChangeRef = useRef(onCameraDemandChange);
611
- onCameraDemandChangeRef.current = onCameraDemandChange;
612
- const onCameraStatsRef = useRef(onCameraStats);
613
- onCameraStatsRef.current = onCameraStats;
614
- // Active outbound-camera stats poller. While the camera is
615
- // sending we sample `RTCPeerConnection.getStats()` once per
616
- // second, derive deltas (bitrate, fps, packet loss) from the
617
- // previous sample, and push the result through `onCameraStats`.
618
- // Cleared on teardown so we never leak the timer.
619
- const cameraStatsTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
620
- const cameraStatsPrevRef = useRef<{
621
- timestamp: number;
622
- framesEncoded?: number;
623
- bytesSent?: number;
624
- packetsSent?: number;
625
- packetsLost?: number;
626
- } | null>(null);
627
- // Outbound microphone state (iOS only). The manual prop and guest-driven
628
- // microphoneRequest signals share one pre-allocated audio transceiver.
629
- const microphoneEnabledRef = useRef(microphoneEnabled);
630
- microphoneEnabledRef.current = microphoneEnabled;
631
- const microphoneDemandActiveRef = useRef(false);
632
- const onMicrophoneStateChangeRef = useRef(onMicrophoneStateChange);
633
- onMicrophoneStateChangeRef.current = onMicrophoneStateChange;
634
- const outboundMicSenderRef = useRef<RTCRtpSender | null>(null);
635
- const outboundMicStreamRef = useRef<MediaStream | null>(null);
636
- // Bumped by detach and teardown so an attach suspended on an await
637
- // (the getUserMedia prompt) detects it was superseded and bails.
638
- const micGenerationRef = useRef(0);
639
- // Serializes attach/detach work so their awaits (replaceTrack on the
640
- // shared sender) can never interleave and undo each other's effects.
641
- const micOpQueueRef = useRef<Promise<void>>(Promise.resolve());
642
- // Correlates the in-flight `microphoneStart` with its result so a
643
- // host-side rejection (e.g. no loopback device on the node) can
644
- // tear the capture back down and surface the error.
645
- const micStartPendingIdRef = useRef<string | null>(null);
646
- const firstFrameShownRef = useRef(false);
647
- const pendingScreenshotResolversRef = useRef<
648
- Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
649
- >(new Map());
650
- const pendingScreenshotRejectersRef = useRef<Map<string, (reason?: any) => void>>(new Map());
651
- const pendingTerminateAppResolversRef = useRef<Map<string, () => void>>(new Map());
652
- const pendingTerminateAppRejectersRef = useRef<Map<string, (reason?: any) => void>>(new Map());
653
-
654
- // Map to track active pointers for real touch/mouse single-finger events.
655
- // Key: pointerId (-1 for mouse, touch.identifier for touch), Value: { x: number, y: number }
656
- const activePointers = useRef<Map<number, { x: number; y: number }>>(new Map());
657
-
658
- // Alt/Option modifier state for pinch emulation.
659
- // We use a ref as the source of truth (for synchronous event handler access)
660
- // and state only to trigger re-renders for the visual indicators.
661
- const isAltHeldRef = useRef(false);
662
- const [isAltHeld, setIsAltHeld] = useState(false);
663
-
664
- // State for any two-finger gesture (Alt+mouse simulated or real two-finger touch).
665
- // Tracks positions, video size, source, and pointer IDs (for Android protocol).
666
- type TwoFingerState = {
667
- finger0: { x: number; y: number };
668
- finger1: { x: number; y: number };
669
- videoSize: { width: number; height: number };
670
- // Track source so we know when to clear (Alt release vs touch end)
671
- source: 'alt-mouse' | 'real-touch';
672
- // Pointer IDs for Android (real touch.identifier or simulated -1/-2)
673
- pointerId0: number;
674
- pointerId1: number;
675
- };
676
- const twoFingerStateRef = useRef<TwoFingerState | null>(null);
677
- // Wheel-scroll drag state; lives for a single synthesized gesture.
678
- const wheelStateRef = useRef<{ pos: PointerGeometry; endTimer: number } | null>(null);
679
- // Latest wheel handlers, so the once-attached native listener and the
680
- // pointer machinery always call the current render's closures.
681
- const wheelCallbacksRef = useRef<{ handle: (event: WheelEvent) => void; end: () => void }>({
682
- handle: () => {},
683
- end: () => {},
684
- });
685
-
686
- // Hover point for rendering two-finger indicators when Alt is held.
687
- // Only computed/set when Alt is held to avoid unnecessary re-renders.
688
- type HoverPoint = {
689
- containerX: number;
690
- containerY: number;
691
- mirrorContainerX: number;
692
- mirrorContainerY: number;
693
- videoX: number;
694
- videoY: number;
695
- mirrorVideoX: number;
696
- mirrorVideoY: number;
697
- videoWidth: number;
698
- videoHeight: number;
699
- };
700
- const [hoverPoint, setHoverPoint] = useState<HoverPoint | null>(null);
701
-
702
- // Inspect-mode state.
703
- //
704
- // Lifecycle of `axFetcherRef`:
705
- // - Created in dataChannel.onopen (last step of WebRTC handshake)
706
- // so we know the signaling WS is healthy and the device is
707
- // responsive to control messages.
708
- // - Started immediately if `inspectMode` is already enabled, or
709
- // started later via the sibling useEffect when inspectMode flips on.
710
- // - Stopped + nulled in teardownConnection (WS close / unmount).
711
- //
712
- // Customers can observe readiness via the `onAxStatusChange` callback:
713
- // `starting` fires when start() runs but no snapshot has landed yet;
714
- // `ready` once the first snapshot arrives. Status falls back to
715
- // `unavailable` / `error` if the server can't satisfy AX requests.
716
- const axFetcherRef = useRef<AxFetcher | null>(null);
717
- const [axSnapshot, setAxSnapshot] = useState<AxSnapshot | null>(null);
718
- const [axHighlightedId, setAxHighlightedId] = useState<string | null>(null);
719
- const [axSelectedId, setAxSelectedId] = useState<string | null>(null);
720
- const [overlayGeometry, setOverlayGeometry] = useState<InspectOverlayGeometry | null>(null);
721
- // Viewport-space cursor position used to anchor the inspect InfoCard.
722
- // Throttled to one update per animation frame to avoid React reconciling
723
- // on every native mousemove (~60–120Hz).
724
- const [axCursorPosition, setAxCursorPosition] = useState<{ x: number; y: number } | null>(null);
725
- const cursorPositionRef = useRef<{ x: number; y: number } | null>(null);
726
- const cursorRafIdRef = useRef<number | undefined>(undefined);
727
- const scheduleCursorFlush = (next: { x: number; y: number } | null) => {
728
- cursorPositionRef.current = next;
729
- if (cursorRafIdRef.current !== undefined) return;
730
- cursorRafIdRef.current = window.requestAnimationFrame(() => {
731
- cursorRafIdRef.current = undefined;
732
- setAxCursorPosition(cursorPositionRef.current);
733
- });
734
- };
735
- // Position captured at click-time so the InfoCard "freezes" near where
736
- // the user clicked, even as they move the cursor around afterward. The
737
- // action buttons (Tap / Copy) stay reachable because the card no longer
738
- // chases the cursor while the click target is the active selection.
739
- const [axFrozenCursorPosition, setAxFrozenCursorPosition] = useState<{
740
- x: number;
741
- y: number;
742
- } | null>(null);
743
- // Mirrors for synchronous access from event handlers without stale closures.
744
- const inspectModeRef = useRef<boolean | 'hover-only' | undefined>(inspectMode);
745
- inspectModeRef.current = inspectMode;
746
- const axSnapshotRef = useRef<AxSnapshot | null>(null);
747
- axSnapshotRef.current = axSnapshot;
748
- const onAxSnapshotChangeRef = useRef(onAxSnapshotChange);
749
- onAxSnapshotChangeRef.current = onAxSnapshotChange;
750
- const onInspectSelectionChangeRef = useRef(onInspectSelectionChange);
751
- onInspectSelectionChangeRef.current = onInspectSelectionChange;
752
- const onAxStatusChangeRef = useRef(onAxStatusChange);
753
- onAxStatusChangeRef.current = onAxStatusChange;
754
- const onTerminatedRef = useRef(onTerminated);
755
- onTerminatedRef.current = onTerminated;
756
- const onConnectionStateChangeRef = useRef(onConnectionStateChange);
757
- onConnectionStateChangeRef.current = onConnectionStateChange;
758
- const connectionStateRef = useRef<RemoteControlConnectionState | null>(null);
759
- // Reports each transition once; callers get status text without tracking
760
- // the retry machinery below.
761
- const reportConnectionState = (next: RemoteControlConnectionState) => {
762
- if (connectionStateRef.current === next) return;
763
- connectionStateRef.current = next;
764
- safeInvoke('onConnectionStateChange', onConnectionStateChangeRef.current, next);
765
- };
766
-
767
- const inspectActive = inspectMode === true || inspectMode === 'hover-only';
768
- const inspectModeResolved: InspectMode = inspectMode === 'hover-only' ? 'hover-only' : 'select';
769
-
770
- const sessionId = useMemo(
771
- () =>
772
- propSessionId ||
773
- Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15),
774
- [propSessionId],
775
- );
776
-
777
- const platform = useMemo(() => detectPlatform(url), [url]);
778
- const config = deviceConfig[platform];
779
- const platformAssets = assets?.[platform];
780
- const frameImageSrc =
781
- platform === 'android' && useAndroidTabletFrame ?
782
- isLandscape ? platformAssets?.tabletFrameLandscape
783
- : platformAssets?.tabletFrame
784
- : isLandscape ? platformAssets?.frameLandscape
785
- : platformAssets?.frame;
786
- const loadingLogo = platformAssets?.loadingLogo;
787
- // Without a frame image (the lite entry) the video lays out frameless
788
- // whatever showFrame says.
789
- const frameVisible = !isDuo && showFrame && !!frameImageSrc;
790
-
791
- const updateStatus = (message: string) => {
792
- // Use the wrapper for conditional logging
793
- debugLog(message);
794
- };
795
-
796
- const sendBinaryControlMessage = (data: ArrayBuffer) => {
797
- if (!dataChannelRef.current || dataChannelRef.current.readyState !== 'open') {
798
- return;
799
- }
800
- dataChannelRef.current.send(data);
801
- // Any binary control message is an input event. Bump the AX poller so
802
- // we get a fresh snapshot quickly — the UI almost certainly changed.
803
- axFetcherRef.current?.bumpActivity();
804
- };
805
-
806
- // Pointer ID used by inspect-driven taps. Distinct from human pointers
807
- // (-1 mouse, -2 alt-mirror), the synthesized wheel finger (-3), and our
808
- // touch identifiers so they never interfere with an in-progress drag.
809
- const AX_TAP_POINTER_ID = -10;
810
-
811
- // Send a down+up tap at a viewport-space (clientX/Y) position. The point
812
- // is mapped through the current video letterbox geometry so the
813
- // simulator receives the correct in-stream coordinates regardless of
814
- // how the device frame is sized in the DOM.
815
- const sendTapAtClient = (clientX: number, clientY: number) => {
816
- const ctx = computeVideoMappingContext();
817
- if (!ctx) return;
818
- const geometry = mapClientPointToVideo(ctx, clientX, clientY);
819
- if (!geometry) return;
820
- const { videoX, videoY, videoWidth, videoHeight } = geometry;
821
- const down = createTouchControlMessage(
822
- AMOTION_EVENT.ACTION_DOWN,
823
- AX_TAP_POINTER_ID,
824
- videoWidth,
825
- videoHeight,
826
- videoX,
827
- videoY,
828
- 1.0,
829
- AMOTION_EVENT.BUTTON_PRIMARY,
830
- AMOTION_EVENT.BUTTON_PRIMARY,
831
- );
832
- if (down) sendBinaryControlMessage(down);
833
- window.setTimeout(() => {
834
- const up = createTouchControlMessage(
835
- AMOTION_EVENT.ACTION_UP,
836
- AX_TAP_POINTER_ID,
837
- videoWidth,
838
- videoHeight,
839
- videoX,
840
- videoY,
841
- 0,
842
- AMOTION_EVENT.BUTTON_PRIMARY,
843
- AMOTION_EVENT.BUTTON_PRIMARY,
844
- );
845
- if (up) sendBinaryControlMessage(up);
846
- }, 60);
847
- };
848
-
849
- // Center-of-bounds fallback for programmatic taps when there's no
850
- // user-aimed click position (e.g. customer calls `setInspectSelection`
851
- // followed by their own "tap selected" handler without forwarding a
852
- // pointer position). Maps the element's frame center through the AX
853
- // screen-coordinate space to viewport coords, then delegates to
854
- // sendTapAtClient.
855
- const sendTapAtElementCenter = (element: AxElement, snapshot: AxSnapshot) => {
856
- const ctx = computeVideoMappingContext();
857
- if (!ctx) return;
858
- if (snapshot.screen.width <= 0 || snapshot.screen.height <= 0) return;
859
- const cxAx = element.frame.x + element.frame.width / 2;
860
- const cyAx = element.frame.y + element.frame.height / 2;
861
- // AX screen-fraction → in-video pixel offset → viewport client coord.
862
- const inVideoX = (cxAx / snapshot.screen.width) * ctx.actualWidth;
863
- const inVideoY = (cyAx / snapshot.screen.height) * ctx.actualHeight;
864
- const clientX = ctx.videoRect.left + ctx.offsetX + inVideoX;
865
- const clientY = ctx.videoRect.top + ctx.offsetY + inVideoY;
866
- sendTapAtClient(clientX, clientY);
867
- };
868
-
869
- // Fixed pointer IDs for Alt-simulated two-finger gestures
870
- const ALT_POINTER_ID_PRIMARY = -1;
871
- const ALT_POINTER_ID_MIRROR = -2;
872
- // Synthesized finger for wheel-scroll drags.
873
- const WHEEL_POINTER_ID = -3;
874
-
875
- // Helper to send a single-touch control message (used by both single-finger and Android two-finger paths)
876
- const sendSingleTouch = (
877
- action: number,
878
- pointerId: number,
879
- videoWidth: number,
880
- videoHeight: number,
881
- x: number,
882
- y: number,
883
- ) => {
884
- const message = createTouchControlMessage(
885
- action,
886
- pointerId,
887
- videoWidth,
888
- videoHeight,
889
- x,
890
- y,
891
- 1.0, // pressure
892
- AMOTION_EVENT.BUTTON_PRIMARY,
893
- AMOTION_EVENT.BUTTON_PRIMARY,
894
- );
895
- if (message) {
896
- debugLog('[rc-touch] sendSingleTouch', {
897
- action,
898
- actionName: motionActionToString(action),
899
- pointerId,
900
- x,
901
- y,
902
- video: { width: videoWidth, height: videoHeight },
903
- });
904
- sendBinaryControlMessage(message);
905
- }
906
- };
907
-
908
- // Minimal geometry for single-finger touch events (no mirror/container coords needed).
909
- type PointerGeometry = {
910
- videoX: number;
911
- videoY: number;
912
- videoWidth: number;
913
- videoHeight: number;
914
- };
915
-
916
- const applyPointerEvent = (
917
- pointerId: number,
918
- eventType: 'down' | 'move' | 'up' | 'cancel',
919
- geometry: PointerGeometry | null,
920
- ) => {
921
- if (!geometry) return;
922
- const { videoX, videoY, videoWidth, videoHeight } = geometry;
923
-
924
- let action: number | null = null;
925
- let positionToSend: { x: number; y: number } | null = null;
926
- let pressure = 1.0; // Default pressure
927
- const buttons = AMOTION_EVENT.BUTTON_PRIMARY; // Assume primary button
928
-
929
- switch (eventType) {
930
- case 'down':
931
- // A real pointer going down takes over from any synthesized
932
- // wheel-scroll finger, so the tap isn't sent as a second finger.
933
- if (pointerId !== WHEEL_POINTER_ID) {
934
- wheelCallbacksRef.current.end();
935
- }
936
- // For multi-touch: use ACTION_DOWN for first pointer, ACTION_POINTER_DOWN for additional pointers
937
- const currentPointerCount = activePointers.current.size;
938
- action = currentPointerCount === 0 ? AMOTION_EVENT.ACTION_DOWN : AMOTION_EVENT.ACTION_POINTER_DOWN;
939
- positionToSend = { x: videoX, y: videoY };
940
- activePointers.current.set(pointerId, positionToSend);
941
- if (pointerId === -1) {
942
- // Focus on mouse down
943
- videoRef.current?.focus();
944
- }
945
- break;
946
-
947
- case 'move':
948
- if (activePointers.current.has(pointerId)) {
949
- action = AMOTION_EVENT.ACTION_MOVE;
950
- positionToSend = { x: videoX, y: videoY };
951
- // Update the last known position for this active pointer
952
- activePointers.current.set(pointerId, positionToSend);
953
- }
954
- break;
955
-
956
- case 'up':
957
- case 'cancel': // Treat cancel like up, but use ACTION_CANCEL
958
- if (activePointers.current.has(pointerId)) {
959
- // IMPORTANT: Send the UP/CANCEL at the *last known position* inside the video
960
- positionToSend = activePointers.current.get(pointerId)!;
961
- activePointers.current.delete(pointerId); // Remove pointer as it's no longer active
962
-
963
- if (eventType === 'cancel') {
964
- action = AMOTION_EVENT.ACTION_CANCEL;
965
- } else {
966
- // For multi-touch: use ACTION_UP for last pointer, ACTION_POINTER_UP for non-last pointers
967
- const remainingPointerCount = activePointers.current.size;
968
- action =
969
- remainingPointerCount === 0 ? AMOTION_EVENT.ACTION_UP : AMOTION_EVENT.ACTION_POINTER_UP;
970
- }
971
- }
972
- break;
973
- }
974
-
975
- // Send message if action and position determined
976
- if (action !== null && positionToSend !== null) {
977
- debugLog('[rc-touch][mouse->touch] sending', {
978
- pointerId,
979
- eventType,
980
- action,
981
- actionName: motionActionToString(action),
982
- positionToSend,
983
- video: { width: videoWidth, height: videoHeight },
984
- altHeld: isAltHeldRef.current,
985
- activePointersAfter: Array.from(activePointers.current.entries()).map(([id, pos]) => ({
986
- id,
987
- x: pos.x,
988
- y: pos.y,
989
- })),
990
- });
991
- const message = createTouchControlMessage(
992
- action,
993
- pointerId,
994
- videoWidth,
995
- videoHeight,
996
- positionToSend.x,
997
- positionToSend.y,
998
- pressure,
999
- buttons,
1000
- buttons,
1001
- );
1002
- if (message) {
1003
- debugLog('[rc-touch][mouse->touch] buffer', {
1004
- pointerId,
1005
- actionName: motionActionToString(action),
1006
- byteLength: message.byteLength,
1007
- });
1008
- sendBinaryControlMessage(message);
1009
- }
1010
- } else if (eventType === 'up' || eventType === 'cancel') {
1011
- activePointers.current.delete(pointerId);
1012
- }
1013
- };
1014
-
1015
- // Update Alt modifier state. Only iOS Simulator uses Indigo modifier injection.
1016
- const updateAltHeld = (nextHeld: boolean) => {
1017
- if (isAltHeldRef.current === nextHeld) {
1018
- return;
1019
- }
1020
- isAltHeldRef.current = nextHeld;
1021
- setIsAltHeld(nextHeld);
1022
-
1023
- // Clear hover point when Alt is released to hide indicators immediately.
1024
- if (!nextHeld) {
1025
- setHoverPoint(null);
1026
- }
1027
-
1028
- debugLog('[rc-touch][alt] updateAltHeld', {
1029
- nextHeld,
1030
- activePointerIds: Array.from(activePointers.current.keys()),
1031
- });
1032
-
1033
- // iOS Simulator pinch (Option/Alt+drag) behavior depends on the Option modifier being
1034
- // active on the Indigo HID side. Send Alt key down/up immediately on toggle so the
1035
- // sequence matches Simulator.app (Alt down -> mouse down/drag -> mouse up -> Alt up).
1036
- // This is iOS-specific; Android doesn't use this modifier injection.
1037
- if (platform === 'ios' && dataChannelRef.current && dataChannelRef.current.readyState === 'open') {
1038
- const action = nextHeld ? ANDROID_KEYS.ACTION_DOWN : ANDROID_KEYS.ACTION_UP;
1039
- const message = createInjectKeycodeMessage(
1040
- action,
1041
- ANDROID_KEYS.KEYCODE_ALT_LEFT,
1042
- 0,
1043
- ANDROID_KEYS.META_NONE,
1044
- );
1045
- debugLog('[rc-touch][alt] sending Indigo modifier keycode', {
1046
- action,
1047
- keycode: ANDROID_KEYS.KEYCODE_ALT_LEFT,
1048
- });
1049
- if (message) {
1050
- sendBinaryControlMessage(message);
1051
- }
1052
- }
1053
- };
1054
-
1055
- // Mapping context computed once per DOM event, then reused for each pointer.
1056
- type VideoMappingContext = {
1057
- videoWidth: number;
1058
- videoHeight: number;
1059
- videoRect: DOMRect;
1060
- containerRect: DOMRect;
1061
- actualWidth: number;
1062
- actualHeight: number;
1063
- offsetX: number;
1064
- offsetY: number;
1065
- };
1066
-
1067
- // Compute mapping context from current video/container state (once per event).
1068
- const computeVideoMappingContext = (): VideoMappingContext | null => {
1069
- const video = videoRef.current;
1070
- const container = containerRef.current;
1071
- if (!video || !container) return null;
1072
-
1073
- const videoWidth = video.videoWidth;
1074
- const videoHeight = video.videoHeight;
1075
- if (!videoWidth || !videoHeight) return null;
1076
-
1077
- const videoRect = video.getBoundingClientRect();
1078
- const containerRect = container.getBoundingClientRect();
1079
-
1080
- const displayWidth = videoRect.width;
1081
- const displayHeight = videoRect.height;
1082
- const videoAspectRatio = videoWidth / videoHeight;
1083
- const containerAspectRatio = displayWidth / displayHeight;
1084
-
1085
- let actualWidth = displayWidth;
1086
- let actualHeight = displayHeight;
1087
- if (videoAspectRatio > containerAspectRatio) {
1088
- actualHeight = displayWidth / videoAspectRatio;
1089
- } else {
1090
- actualWidth = displayHeight * videoAspectRatio;
1091
- }
1092
-
1093
- const offsetX = (displayWidth - actualWidth) / 2;
1094
- const offsetY = (displayHeight - actualHeight) / 2;
1095
-
1096
- return {
1097
- videoWidth,
1098
- videoHeight,
1099
- videoRect,
1100
- containerRect,
1101
- actualWidth,
1102
- actualHeight,
1103
- offsetX,
1104
- offsetY,
1105
- };
1106
- };
1107
-
1108
- // Map a client point to video coordinates using a pre-computed context,
1109
- // clamping points outside the rendered video to the nearest point on the video.
1110
- const mapClientPointToVideo = (
1111
- ctx: VideoMappingContext,
1112
- clientX: number,
1113
- clientY: number,
1114
- ): PointerGeometry | null => {
1115
- const relativeX = clientX - ctx.videoRect.left - ctx.offsetX;
1116
- const relativeY = clientY - ctx.videoRect.top - ctx.offsetY;
1117
-
1118
- const clampedRelativeX = Math.max(0, Math.min(ctx.actualWidth, relativeX));
1119
- const clampedRelativeY = Math.max(0, Math.min(ctx.actualHeight, relativeY));
1120
- const videoX = Math.max(
1121
- 0,
1122
- Math.min(ctx.videoWidth, (clampedRelativeX / ctx.actualWidth) * ctx.videoWidth),
1123
- );
1124
- const videoY = Math.max(
1125
- 0,
1126
- Math.min(ctx.videoHeight, (clampedRelativeY / ctx.actualHeight) * ctx.videoHeight),
1127
- );
1128
-
1129
- return {
1130
- videoX,
1131
- videoY,
1132
- videoWidth: ctx.videoWidth,
1133
- videoHeight: ctx.videoHeight,
1134
- };
1135
- };
1136
-
1137
- // Compute full hover point with mirror/container coordinates (for Alt indicator rendering),
1138
- // clamping points outside the rendered video to the nearest point on the video.
1139
- const computeFullHoverPoint = (
1140
- ctx: VideoMappingContext,
1141
- clientX: number,
1142
- clientY: number,
1143
- ): HoverPoint | null => {
1144
- const relativeX = clientX - ctx.videoRect.left - ctx.offsetX;
1145
- const relativeY = clientY - ctx.videoRect.top - ctx.offsetY;
1146
-
1147
- const clampedRelativeX = Math.max(0, Math.min(ctx.actualWidth, relativeX));
1148
- const clampedRelativeY = Math.max(0, Math.min(ctx.actualHeight, relativeY));
1149
- const videoX = Math.max(
1150
- 0,
1151
- Math.min(ctx.videoWidth, (clampedRelativeX / ctx.actualWidth) * ctx.videoWidth),
1152
- );
1153
- const videoY = Math.max(
1154
- 0,
1155
- Math.min(ctx.videoHeight, (clampedRelativeY / ctx.actualHeight) * ctx.videoHeight),
1156
- );
1157
- const mirrorVideoX = ctx.videoWidth - videoX;
1158
- const mirrorVideoY = ctx.videoHeight - videoY;
1159
-
1160
- const contentLeft = ctx.videoRect.left + ctx.offsetX;
1161
- const contentTop = ctx.videoRect.top + ctx.offsetY;
1162
- const containerX = contentLeft - ctx.containerRect.left + clampedRelativeX;
1163
- const containerY = contentTop - ctx.containerRect.top + clampedRelativeY;
1164
- const mirrorContainerX = contentLeft - ctx.containerRect.left + (ctx.actualWidth - clampedRelativeX);
1165
- const mirrorContainerY = contentTop - ctx.containerRect.top + (ctx.actualHeight - clampedRelativeY);
1166
-
1167
- return {
1168
- containerX,
1169
- containerY,
1170
- mirrorContainerX,
1171
- mirrorContainerY,
1172
- videoX,
1173
- videoY,
1174
- mirrorVideoX,
1175
- mirrorVideoY,
1176
- videoWidth: ctx.videoWidth,
1177
- videoHeight: ctx.videoHeight,
1178
- };
1179
- };
1180
-
1181
- // Helper to send a two-finger touch message (iOS-specific type=18 message).
1182
- const sendTwoFingerMessage = (
1183
- action: number,
1184
- videoWidth: number,
1185
- videoHeight: number,
1186
- x0: number,
1187
- y0: number,
1188
- x1: number,
1189
- y1: number,
1190
- ) => {
1191
- const msg = createTwoFingerTouchControlMessage(action, videoWidth, videoHeight, x0, y0, x1, y1);
1192
- debugLog('[rc-touch2] sendTwoFingerMessage (iOS)', {
1193
- actionName: motionActionToString(action),
1194
- video: { width: videoWidth, height: videoHeight },
1195
- p0: { x: x0, y: y0 },
1196
- p1: { x: x1, y: y1 },
1197
- byteLength: msg.byteLength,
1198
- });
1199
- sendBinaryControlMessage(msg);
1200
- };
1201
-
1202
- // Generic two-finger event handler - sends platform-appropriate messages.
1203
- // iOS: uses special two-finger message (type=18)
1204
- // Android: sends two separate single-touch messages with proper action sequencing
1205
- const applyTwoFingerEvent = (
1206
- eventType: 'down' | 'move' | 'up',
1207
- videoWidth: number,
1208
- videoHeight: number,
1209
- x0: number,
1210
- y0: number,
1211
- x1: number,
1212
- y1: number,
1213
- pointerId0: number,
1214
- pointerId1: number,
1215
- ) => {
1216
- debugLog('[rc-touch2] applyTwoFingerEvent', {
1217
- platform,
1218
- eventType,
1219
- video: { width: videoWidth, height: videoHeight },
1220
- p0: { x: x0, y: y0, id: pointerId0 },
1221
- p1: { x: x1, y: y1, id: pointerId1 },
1222
- });
1223
-
1224
- if (eventType === 'down') {
1225
- // A real two-finger gesture takes over from any synthesized
1226
- // wheel-scroll finger.
1227
- wheelCallbacksRef.current.end();
1228
- }
1229
-
1230
- if (platform === 'ios') {
1231
- // iOS: use special two-finger message (type=18)
1232
- const action =
1233
- eventType === 'down' ? AMOTION_EVENT.ACTION_DOWN
1234
- : eventType === 'move' ? AMOTION_EVENT.ACTION_MOVE
1235
- : AMOTION_EVENT.ACTION_UP;
1236
- sendTwoFingerMessage(action, videoWidth, videoHeight, x0, y0, x1, y1);
1237
- } else {
1238
- // Android: send two separate single-touch messages with proper action codes
1239
- // Per scrcpy protocol, each finger is a separate INJECT_TOUCH_EVENT with unique pointerId
1240
- if (eventType === 'down') {
1241
- // First finger down (ACTION_DOWN), then second finger down (ACTION_POINTER_DOWN)
1242
- sendSingleTouch(AMOTION_EVENT.ACTION_DOWN, pointerId0, videoWidth, videoHeight, x0, y0);
1243
- sendSingleTouch(AMOTION_EVENT.ACTION_POINTER_DOWN, pointerId1, videoWidth, videoHeight, x1, y1);
1244
- } else if (eventType === 'move') {
1245
- // Both fingers move (ACTION_MOVE for each)
1246
- sendSingleTouch(AMOTION_EVENT.ACTION_MOVE, pointerId0, videoWidth, videoHeight, x0, y0);
1247
- sendSingleTouch(AMOTION_EVENT.ACTION_MOVE, pointerId1, videoWidth, videoHeight, x1, y1);
1248
- } else {
1249
- // Second finger up (ACTION_POINTER_UP), then first finger up (ACTION_UP)
1250
- sendSingleTouch(AMOTION_EVENT.ACTION_POINTER_UP, pointerId1, videoWidth, videoHeight, x1, y1);
1251
- sendSingleTouch(AMOTION_EVENT.ACTION_UP, pointerId0, videoWidth, videoHeight, x0, y0);
1252
- }
1253
- }
1254
- };
1255
-
1256
- // Update hover point only when Alt is held (to avoid re-renders in normal path).
1257
- const updateHoverPoint = (ctx: VideoMappingContext, clientX: number, clientY: number) => {
1258
- if (!isAltHeldRef.current) {
1259
- // Don't compute or update when Alt isn't held
1260
- if (hoverPoint !== null) {
1261
- setHoverPoint(null);
1262
- }
1263
- return;
1264
- }
1265
- const fullPoint = computeFullHoverPoint(ctx, clientX, clientY);
1266
- setHoverPoint(fullPoint);
1267
- };
1268
-
1269
- // Map clientX/Y to AX screen-coordinate space using the latest snapshot.
1270
- // Returns null if there's no snapshot or the click is outside the video.
1271
- const hitTestAxAtClient = (
1272
- ctx: VideoMappingContext,
1273
- clientX: number,
1274
- clientY: number,
1275
- ): AxElement | null => {
1276
- const snapshot = axSnapshotRef.current;
1277
- if (!snapshot || snapshot.screen.width <= 0 || snapshot.screen.height <= 0) return null;
1278
- const relX = clientX - ctx.videoRect.left - ctx.offsetX;
1279
- const relY = clientY - ctx.videoRect.top - ctx.offsetY;
1280
- if (relX < 0 || relY < 0 || relX > ctx.actualWidth || relY > ctx.actualHeight) return null;
1281
- const axX = (relX / ctx.actualWidth) * snapshot.screen.width;
1282
- const axY = (relY / ctx.actualHeight) * snapshot.screen.height;
1283
- return axElementAtPoint(snapshot, axX, axY);
1284
- };
1285
-
1286
- // Unified handler for both mouse and touch interactions
1287
- const handleInteraction = (event: React.MouseEvent | React.TouchEvent) => {
1288
- if (isDuo) return;
1289
- event.preventDefault();
1290
- event.stopPropagation();
1291
-
1292
- // Compute mapping context once per event (reused for all pointers)
1293
- const ctx = computeVideoMappingContext();
1294
-
1295
- // Inspect-mode handling.
1296
- //
1297
- // We use JS hit-testing (not box-level onMouseEnter/Leave) as the
1298
- // single source of truth for which element is under the cursor — it
1299
- // handles overlapping rectangles deterministically by picking the
1300
- // smallest matching box. The overlay's InspectBox children no longer
1301
- // attach hover handlers; they just paint themselves based on the
1302
- // `highlightedId` prop driven from here.
1303
- //
1304
- // Cursor position is tracked in both modes so the cursor-anchored
1305
- // InfoCard can follow the pointer.
1306
- const isInspecting = inspectModeRef.current === true || inspectModeRef.current === 'hover-only';
1307
- if (isInspecting && !('touches' in event)) {
1308
- if (event.type === 'mousemove') {
1309
- scheduleCursorFlush({ x: event.clientX, y: event.clientY });
1310
- if (ctx) {
1311
- const hit = hitTestAxAtClient(ctx, event.clientX, event.clientY);
1312
- setAxHighlightedId(hit?.id ?? null);
1313
- }
1314
- } else if (event.type === 'mouseleave') {
1315
- scheduleCursorFlush(null);
1316
- setAxHighlightedId(null);
1317
- }
1318
- }
1319
- // Select mode blocks device input — clicks/drags don't reach the
1320
- // simulator. Hover-only mode falls through to the regular path.
1321
- if (inspectModeRef.current === true) {
1322
- return;
1323
- }
1324
-
1325
- // Handle hover point updates for mouse events (only when Alt is held)
1326
- if (!('touches' in event) && ctx) {
1327
- if (event.type === 'mousemove') {
1328
- updateHoverPoint(ctx, event.clientX, event.clientY);
1329
- } else if (event.type === 'mouseleave') {
1330
- setHoverPoint(null);
1331
- }
1332
- // Note: Alt state is tracked via global keydown/keyup listeners, not event.altKey,
1333
- // to ensure consistent behavior across focus transitions.
1334
- }
1335
-
1336
- if (
1337
- !dataChannelRef.current ||
1338
- dataChannelRef.current.readyState !== 'open' ||
1339
- !videoRef.current ||
1340
- !ctx
1341
- ) {
1342
- return;
1343
- }
1344
-
1345
- // --- Event Type Handling ---
1346
-
1347
- if ('touches' in event) {
1348
- // Touch Events - handle both single-finger and two-finger gestures
1349
- const allTouches = event.touches; // All currently active touches
1350
- const changedTouches = event.changedTouches;
1351
-
1352
- let eventType: 'down' | 'move' | 'up' | 'cancel';
1353
- switch (event.type) {
1354
- case 'touchstart':
1355
- eventType = 'down';
1356
- break;
1357
- case 'touchmove':
1358
- eventType = 'move';
1359
- break;
1360
- case 'touchend':
1361
- eventType = 'up';
1362
- break;
1363
- case 'touchcancel':
1364
- eventType = 'cancel';
1365
- break;
1366
- default:
1367
- return;
1368
- }
1369
-
1370
- // Check if we have exactly 2 active touches - route to two-finger logic
1371
- if (allTouches.length === 2) {
1372
- const t0 = allTouches[0];
1373
- const t1 = allTouches[1];
1374
- const g0 = mapClientPointToVideo(ctx, t0.clientX, t0.clientY);
1375
- const g1 = mapClientPointToVideo(ctx, t1.clientX, t1.clientY);
1376
-
1377
- if (!g0 || !g1) return;
1378
-
1379
- if (!twoFingerStateRef.current) {
1380
- // Starting a new two-finger gesture
1381
- twoFingerStateRef.current = {
1382
- finger0: { x: g0.videoX, y: g0.videoY },
1383
- finger1: { x: g1.videoX, y: g1.videoY },
1384
- videoSize: { width: g0.videoWidth, height: g0.videoHeight },
1385
- source: 'real-touch',
1386
- pointerId0: t0.identifier,
1387
- pointerId1: t1.identifier,
1388
- };
1389
- applyTwoFingerEvent(
1390
- 'down',
1391
- g0.videoWidth,
1392
- g0.videoHeight,
1393
- g0.videoX,
1394
- g0.videoY,
1395
- g1.videoX,
1396
- g1.videoY,
1397
- t0.identifier,
1398
- t1.identifier,
1399
- );
1400
- } else if (twoFingerStateRef.current.source === 'real-touch') {
1401
- // Continuing two-finger gesture (move)
1402
- twoFingerStateRef.current.finger0 = { x: g0.videoX, y: g0.videoY };
1403
- twoFingerStateRef.current.finger1 = { x: g1.videoX, y: g1.videoY };
1404
- applyTwoFingerEvent(
1405
- 'move',
1406
- g0.videoWidth,
1407
- g0.videoHeight,
1408
- g0.videoX,
1409
- g0.videoY,
1410
- g1.videoX,
1411
- g1.videoY,
1412
- twoFingerStateRef.current.pointerId0,
1413
- twoFingerStateRef.current.pointerId1,
1414
- );
1415
- }
1416
- } else if (allTouches.length < 2 && twoFingerStateRef.current?.source === 'real-touch') {
1417
- // Finger lifted - end two-finger gesture using last known state
1418
- const state = twoFingerStateRef.current;
1419
- applyTwoFingerEvent(
1420
- 'up',
1421
- state.videoSize.width,
1422
- state.videoSize.height,
1423
- state.finger0.x,
1424
- state.finger0.y,
1425
- state.finger1.x,
1426
- state.finger1.y,
1427
- state.pointerId0,
1428
- state.pointerId1,
1429
- );
1430
- twoFingerStateRef.current = null;
1431
- // Don't process remaining finger - gesture ended
1432
- return;
1433
- } else if (allTouches.length > 2) {
1434
- // 3+ fingers - not supported, ignore
1435
- return;
1436
- } else {
1437
- // Single finger touch (allTouches is 0 or 1)
1438
- // Note: allTouches=0 happens on touchend when last finger lifts
1439
- const touch = changedTouches[0];
1440
- if (touch) {
1441
- const geometry = mapClientPointToVideo(ctx, touch.clientX, touch.clientY);
1442
- applyPointerEvent(touch.identifier, eventType, geometry);
1443
- }
1444
- }
1445
- } else {
1446
- // Mouse Events
1447
- const pointerId = -1; // Primary mouse pointer
1448
- let eventType: 'down' | 'move' | 'up' | 'cancel' | null = null;
1449
-
1450
- // Determine if we're in two-finger mode (Alt+mouse drag)
1451
- const inTwoFingerMode = twoFingerStateRef.current?.source === 'alt-mouse';
1452
-
1453
- switch (event.type) {
1454
- case 'mousedown':
1455
- if (event.button === 0) eventType = 'down';
1456
- break;
1457
- case 'mousemove':
1458
- // Process move if either in two-finger mode or has active pointer (normal drag)
1459
- if (inTwoFingerMode || activePointers.current.has(pointerId)) {
1460
- eventType = 'move';
1461
- }
1462
- break;
1463
- case 'mouseup':
1464
- if (event.button === 0) eventType = 'up';
1465
- break;
1466
- case 'mouseleave':
1467
- // Treat leave like up only if in drag/two-finger mode
1468
- if (inTwoFingerMode || activePointers.current.has(pointerId)) {
1469
- eventType = 'up';
1470
- }
1471
- break;
1472
- }
1473
-
1474
- if (eventType) {
1475
- const geometry = mapClientPointToVideo(ctx, event.clientX, event.clientY);
1476
- if (!geometry) {
1477
- return;
1478
- }
1479
-
1480
- debugLog('[rc-touch][mouse] event', {
1481
- domType: event.type,
1482
- eventType,
1483
- button: event.button,
1484
- buttons: (event as React.MouseEvent).buttons,
1485
- client: { x: event.clientX, y: event.clientY },
1486
- altHeldRef: isAltHeldRef.current,
1487
- inTwoFingerMode,
1488
- geometry: {
1489
- videoX: geometry.videoX,
1490
- videoY: geometry.videoY,
1491
- videoWidth: geometry.videoWidth,
1492
- videoHeight: geometry.videoHeight,
1493
- },
1494
- activePointerIds: Array.from(activePointers.current.keys()),
1495
- });
1496
-
1497
- // Route to two-finger (Alt+mouse) or single-finger path
1498
- if (isAltHeldRef.current || inTwoFingerMode) {
1499
- // Two-finger mode - Alt simulates second finger at mirror position
1500
- handleAltMouseGesture(eventType, geometry);
1501
- } else {
1502
- // Normal single-finger touch
1503
- applyPointerEvent(pointerId, eventType, geometry);
1504
- }
1505
- }
1506
- }
1507
- };
1508
-
1509
- // Handle Alt+mouse gestures (simulated two-finger with mirror position).
1510
- // Works on both iOS and Android - applyTwoFingerEvent handles platform differences.
1511
- const handleAltMouseGesture = (
1512
- eventType: 'down' | 'move' | 'up' | 'cancel',
1513
- geometry: PointerGeometry,
1514
- ) => {
1515
- const { videoX, videoY, videoWidth, videoHeight } = geometry;
1516
- const mirrorX = videoWidth - videoX;
1517
- const mirrorY = videoHeight - videoY;
1518
-
1519
- if (eventType === 'down') {
1520
- // Start two-finger gesture
1521
- twoFingerStateRef.current = {
1522
- finger0: { x: videoX, y: videoY },
1523
- finger1: { x: mirrorX, y: mirrorY },
1524
- videoSize: { width: videoWidth, height: videoHeight },
1525
- source: 'alt-mouse',
1526
- pointerId0: ALT_POINTER_ID_PRIMARY,
1527
- pointerId1: ALT_POINTER_ID_MIRROR,
1528
- };
1529
- videoRef.current?.focus();
1530
- applyTwoFingerEvent(
1531
- 'down',
1532
- videoWidth,
1533
- videoHeight,
1534
- videoX,
1535
- videoY,
1536
- mirrorX,
1537
- mirrorY,
1538
- ALT_POINTER_ID_PRIMARY,
1539
- ALT_POINTER_ID_MIRROR,
1540
- );
1541
- return;
1542
- }
1543
-
1544
- if (eventType === 'move') {
1545
- if (twoFingerStateRef.current?.source === 'alt-mouse') {
1546
- // Update positions
1547
- twoFingerStateRef.current.finger0 = { x: videoX, y: videoY };
1548
- twoFingerStateRef.current.finger1 = { x: mirrorX, y: mirrorY };
1549
- applyTwoFingerEvent(
1550
- 'move',
1551
- videoWidth,
1552
- videoHeight,
1553
- videoX,
1554
- videoY,
1555
- mirrorX,
1556
- mirrorY,
1557
- ALT_POINTER_ID_PRIMARY,
1558
- ALT_POINTER_ID_MIRROR,
1559
- );
1560
- }
1561
- return;
1562
- }
1563
-
1564
- if (eventType === 'up' || eventType === 'cancel') {
1565
- const state = twoFingerStateRef.current;
1566
- if (state?.source === 'alt-mouse') {
1567
- // End gesture at last known positions
1568
- const { finger0, finger1, videoSize } = state;
1569
- applyTwoFingerEvent(
1570
- 'up',
1571
- videoSize.width,
1572
- videoSize.height,
1573
- finger0.x,
1574
- finger0.y,
1575
- finger1.x,
1576
- finger1.y,
1577
- ALT_POINTER_ID_PRIMARY,
1578
- ALT_POINTER_ID_MIRROR,
1579
- );
1580
- twoFingerStateRef.current = null;
1581
- }
1582
- return;
1583
- }
1584
- };
1585
-
1586
- // Mouse wheel / trackpad scrolling over the stream scrolls inside the
1587
- // simulator. iOS has no scroll wheel, so we fake a finger: press down
1588
- // where the cursor is, drag it as wheel events come in, lift when they
1589
- // stop. If the fake finger reaches the screen edge before the user stops
1590
- // scrolling, lift and press again to keep going.
1591
- const WHEEL_END_QUIET_MS = 150;
1592
- const endWheelGesture = () => {
1593
- const state = wheelStateRef.current;
1594
- if (!state) return;
1595
- window.clearTimeout(state.endTimer);
1596
- applyPointerEvent(WHEEL_POINTER_ID, 'up', state.pos);
1597
- wheelStateRef.current = null;
1598
- };
1599
- const handleWheel = (event: WheelEvent) => {
1600
- // Trackpad pinches arrive as ctrl+wheel; don't turn them into scrolls.
1601
- if (event.ctrlKey) return;
1602
- // If the user is already touching/dragging (or holding Alt for the
1603
- // two-finger gesture), let that win - don't add a second fake finger.
1604
- if (isAltHeldRef.current || twoFingerStateRef.current) return;
1605
- const pointerCount = activePointers.current.size;
1606
- if (pointerCount > 1 || (pointerCount === 1 && !activePointers.current.has(WHEEL_POINTER_ID))) return;
1607
-
1608
- const ctx = computeVideoMappingContext();
1609
- if (!ctx) return;
1610
- // Outside the rendered video (frame, letterbox) let the page scroll.
1611
- const relativeX = event.clientX - ctx.videoRect.left - ctx.offsetX;
1612
- const relativeY = event.clientY - ctx.videoRect.top - ctx.offsetY;
1613
- if (relativeX < 0 || relativeX > ctx.actualWidth || relativeY < 0 || relativeY > ctx.actualHeight) {
1614
- return;
1615
- }
1616
- const anchor = mapClientPointToVideo(ctx, event.clientX, event.clientY);
1617
- if (!anchor) return;
1618
- event.preventDefault();
1619
-
1620
- // deltaMode: 0=pixels, 1=lines, 2=pages. Deltas are CSS pixels; convert
1621
- // to stream pixels so a tick scrolls the same visual distance at any
1622
- // window size.
1623
- const scaleX =
1624
- (event.deltaMode === 1 ? 16
1625
- : event.deltaMode === 2 ? ctx.actualWidth
1626
- : 1) *
1627
- (ctx.videoWidth / ctx.actualWidth);
1628
- const scaleY =
1629
- (event.deltaMode === 1 ? 16
1630
- : event.deltaMode === 2 ? ctx.actualHeight
1631
- : 1) *
1632
- (ctx.videoHeight / ctx.actualHeight);
1633
- const dx = -event.deltaX * scaleX;
1634
- const dy = -event.deltaY * scaleY;
1635
- const armEnd = (s: { endTimer: number }) => {
1636
- window.clearTimeout(s.endTimer);
1637
- s.endTimer = window.setTimeout(endWheelGesture, WHEEL_END_QUIET_MS);
1638
- };
1639
- if (dx === 0 && dy === 0) {
1640
- // Some wheel events carry no movement (momentum tails); just keep
1641
- // the gesture alive.
1642
- if (wheelStateRef.current) armEnd(wheelStateRef.current);
1643
- return;
1644
- }
1645
-
1646
- // Where to press the fake finger down: at the cursor, but nudged away
1647
- // from the screen edges so the drag has room to travel.
1648
- const pressPoint = (): PointerGeometry => ({
1649
- videoWidth: anchor.videoWidth,
1650
- videoHeight: anchor.videoHeight,
1651
- videoX:
1652
- Math.abs(dx) > Math.abs(dy) ?
1653
- Math.min(Math.max(anchor.videoX, anchor.videoWidth * 0.2), anchor.videoWidth * 0.8)
1654
- : anchor.videoX,
1655
- videoY:
1656
- Math.abs(dy) >= Math.abs(dx) ?
1657
- Math.min(Math.max(anchor.videoY, anchor.videoHeight * 0.2), anchor.videoHeight * 0.8)
1658
- : anchor.videoY,
1659
- });
1660
-
1661
- const advance = (p: PointerGeometry): PointerGeometry => ({
1662
- ...p,
1663
- videoX: Math.min(Math.max(p.videoX + dx, 0), p.videoWidth),
1664
- videoY: Math.min(Math.max(p.videoY + dy, 0), p.videoHeight),
1665
- });
1666
-
1667
- let state = wheelStateRef.current;
1668
- if (!state) {
1669
- const start = pressPoint();
1670
- applyPointerEvent(WHEEL_POINTER_ID, 'down', start);
1671
- state = { pos: start, endTimer: 0 };
1672
- wheelStateRef.current = state;
1673
- } else {
1674
- window.clearTimeout(state.endTimer);
1675
- }
1676
-
1677
- let pos = state.pos;
1678
- let next = advance(pos);
1679
- if (next.videoX === pos.videoX && next.videoY === pos.videoY) {
1680
- // Finger reached the screen edge: lift it, press again with room to
1681
- // travel, and move right away (a press that never moves would count
1682
- // as a tap).
1683
- applyPointerEvent(WHEEL_POINTER_ID, 'up', pos);
1684
- pos = pressPoint();
1685
- applyPointerEvent(WHEEL_POINTER_ID, 'down', pos);
1686
- next = advance(pos);
1687
- }
1688
- applyPointerEvent(WHEEL_POINTER_ID, 'move', next);
1689
- state.pos = next;
1690
- armEnd(state);
1691
- };
1692
- useEffect(() => {
1693
- wheelCallbacksRef.current = { handle: handleWheel, end: endWheelGesture };
1694
- });
1695
-
1696
- // React attaches onWheel passively, so preventDefault (needed to keep the
1697
- // page from scrolling) requires a native non-passive listener. Cleanup
1698
- // lifts an in-flight synthesized finger before the channel goes away.
1699
- useEffect(() => {
1700
- const container = containerRef.current;
1701
- if (!container) return;
1702
- const listener = (event: WheelEvent) => wheelCallbacksRef.current.handle(event);
1703
- container.addEventListener('wheel', listener, { passive: false });
1704
- return () => {
1705
- wheelCallbacksRef.current.end();
1706
- container.removeEventListener('wheel', listener);
1707
- };
1708
- }, []);
1709
-
1710
- useEffect(() => {
1711
- const handleKeyDown = (event: KeyboardEvent) => {
1712
- if (event.key === 'Alt') {
1713
- updateAltHeld(true);
1714
- }
1715
- };
1716
- const handleKeyUp = (event: KeyboardEvent) => {
1717
- if (event.key === 'Alt') {
1718
- updateAltHeld(false);
1719
- }
1720
- };
1721
- const handleWindowBlur = () => {
1722
- updateAltHeld(false);
1723
- };
1724
- // Use capture phase so these fire before handleKeyboard's stopPropagation
1725
- window.addEventListener('keydown', handleKeyDown, true);
1726
- window.addEventListener('keyup', handleKeyUp, true);
1727
- window.addEventListener('blur', handleWindowBlur);
1728
- return () => {
1729
- window.removeEventListener('keydown', handleKeyDown, true);
1730
- window.removeEventListener('keyup', handleKeyUp, true);
1731
- window.removeEventListener('blur', handleWindowBlur);
1732
- };
1733
- }, []);
1734
-
1735
- const handleKeyboard = (event: React.KeyboardEvent) => {
1736
- event.preventDefault();
1737
- event.stopPropagation();
1738
- const inputFocused =
1739
- document.activeElement === videoRef.current ||
1740
- (isDuo && document.activeElement === event.currentTarget);
1741
- // Use the wrapper for conditional logging
1742
- debugLog('Keyboard event:', {
1743
- type: event.type,
1744
- key: event.key,
1745
- keyCode: event.keyCode,
1746
- code: event.code,
1747
- target: (event.target as HTMLElement).tagName,
1748
- focused: inputFocused,
1749
- });
1750
-
1751
- if (!inputFocused) {
1752
- // Use the wrapper for conditional warning
1753
- debugWarn('Simulator input not focused, skipping keyboard event');
1754
- return;
1755
- }
1756
-
1757
- if (!dataChannelRef.current || dataChannelRef.current.readyState !== 'open') {
1758
- // Use the wrapper for conditional warning
1759
- debugWarn('Data channel not ready for keyboard event:', dataChannelRef.current?.readyState);
1760
- return;
1761
- }
1762
-
1763
- // Handle special shortcuts first (Paste, Menu)
1764
- if (event.type === 'keydown') {
1765
- // Paste (Cmd+V / Ctrl+V)
1766
- if (event.key.toLowerCase() === 'v' && (event.metaKey || event.ctrlKey)) {
1767
- debugLog('Paste shortcut detected');
1768
- navigator.clipboard
1769
- .readText()
1770
- .then((text) => {
1771
- if (text) {
1772
- debugLog(
1773
- 'Pasting text via SET_CLIPBOARD:',
1774
- text.substring(0, 20) + (text.length > 20 ? '...' : ''),
1775
- );
1776
- const message = createSetClipboardMessage(text, true); // paste=true
1777
- sendBinaryControlMessage(message);
1778
- }
1779
- })
1780
- .catch((err) => {
1781
- console.error('Failed to read clipboard contents: ', err);
1782
- });
1783
- return; // Don't process 'v' keycode further
1784
- }
1785
-
1786
- // Menu (Cmd+M / Ctrl+M) - Send down and up immediately
1787
- if (event.key.toLowerCase() === 'm' && (event.metaKey || event.ctrlKey)) {
1788
- debugLog('Menu shortcut detected');
1789
- const messageDown = createInjectKeycodeMessage(
1790
- ANDROID_KEYS.ACTION_DOWN,
1791
- ANDROID_KEYS.MENU,
1792
- 0,
1793
- ANDROID_KEYS.META_NONE, // Modifiers are handled by the shortcut check, not passed down
1794
- );
1795
- sendBinaryControlMessage(messageDown);
1796
- const messageUp = createInjectKeycodeMessage(
1797
- ANDROID_KEYS.ACTION_UP,
1798
- ANDROID_KEYS.MENU,
1799
- 0,
1800
- ANDROID_KEYS.META_NONE,
1801
- );
1802
- sendBinaryControlMessage(messageUp);
1803
- return; // Don't process 'm' keycode further
1804
- }
1805
- }
1806
-
1807
- // Handle general key presses (including Arrows, Enter, Backspace, Delete, Letters, Numbers, Symbols)
1808
- const keyInfo = getAndroidKeycodeAndMeta(event);
1809
-
1810
- if (keyInfo) {
1811
- const { keycode, metaState } = keyInfo;
1812
- const action = event.type === 'keydown' ? ANDROID_KEYS.ACTION_DOWN : ANDROID_KEYS.ACTION_UP;
1813
-
1814
- debugLog(`Sending Keycode: key=${event.key}, code=${keycode}, action=${action}, meta=${metaState}`);
1815
-
1816
- const message = createInjectKeycodeMessage(
1817
- action,
1818
- keycode,
1819
- 0, // repeat count, typically 0 for single presses
1820
- metaState,
1821
- );
1822
- sendBinaryControlMessage(message);
1823
- } else {
1824
- debugLog(`Ignoring unhandled key event: type=${event.type}, key=${event.key}`);
1825
- }
1826
- };
1827
-
1828
- const sendKeepAlive = () => {
1829
- if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
1830
- wsRef.current.send(
1831
- JSON.stringify({
1832
- type: 'keepAlive',
1833
- sessionId: sessionId,
1834
- }),
1835
- );
1836
- }
1837
- };
1838
-
1839
- const startKeepAlive = () => {
1840
- if (keepAliveIntervalRef.current) {
1841
- window.clearInterval(keepAliveIntervalRef.current);
1842
- }
1843
- keepAliveIntervalRef.current = window.setInterval(sendKeepAlive, 10000);
1844
- };
1845
-
1846
- const stopKeepAlive = () => {
1847
- if (keepAliveIntervalRef.current) {
1848
- window.clearInterval(keepAliveIntervalRef.current);
1849
- keepAliveIntervalRef.current = undefined;
1850
- }
1851
- };
1852
-
1853
- const clearScheduledRetry = () => {
1854
- if (retryTimeoutRef.current) {
1855
- window.clearTimeout(retryTimeoutRef.current);
1856
- retryTimeoutRef.current = undefined;
1857
- }
1858
- };
1859
-
1860
- const clearConnectionSuccessTimeout = () => {
1861
- if (connectionSuccessTimeoutRef.current) {
1862
- window.clearTimeout(connectionSuccessTimeoutRef.current);
1863
- connectionSuccessTimeoutRef.current = undefined;
1864
- }
1865
- };
1866
-
1867
- const stopRequestFrameLoop = () => {
1868
- if (requestFrameIntervalRef.current) {
1869
- window.clearInterval(requestFrameIntervalRef.current);
1870
- requestFrameIntervalRef.current = undefined;
1871
- }
1872
- };
1873
-
1874
- const clearIceDisconnectedGrace = () => {
1875
- if (iceDisconnectedGraceRef.current !== undefined) {
1876
- window.clearTimeout(iceDisconnectedGraceRef.current);
1877
- iceDisconnectedGraceRef.current = undefined;
1878
- }
1879
- };
1880
-
1881
- const markFirstFrameShown = () => {
1882
- if (firstFrameShownRef.current) {
1883
- return;
1884
- }
1885
- firstFrameShownRef.current = true;
1886
- stopRequestFrameLoop();
1887
- setVideoLoaded(true);
1888
- reportConnectionState('connected');
1889
- };
1890
-
1891
- // Stop every track on a MediaStream and release the device handle.
1892
- // The browser keeps the camera "on" indicator lit until at least
1893
- // one track on the underlying source is stopped, so we have to
1894
- // call stop() explicitly — closing the peer connection alone
1895
- // won't do it.
1896
- const stopMediaStream = (stream: MediaStream) => {
1897
- for (const track of stream.getTracks()) {
1898
- try {
1899
- track.stop();
1900
- } catch (err) {
1901
- debugWarn('track.stop() failed:', err);
1902
- }
1903
- }
1904
- };
1905
-
1906
- // Stop and forget any currently-attached outbound camera stream.
1907
- // Safe to call when nothing is attached. Does not touch the
1908
- // sender — the caller is responsible for `replaceTrack(null)`
1909
- // separately when it wants the SDP slot itself empty.
1910
- const stopOutboundLocalStream = () => {
1911
- const stream = outboundLocalStreamRef.current;
1912
- if (!stream) return;
1913
- outboundLocalStreamRef.current = null;
1914
- stopMediaStream(stream);
1915
- };
1916
-
1917
- // Runs one attach/detach operation after all previously queued ones,
1918
- // so two operations' awaits never interleave on the shared sender (a
1919
- // stale detach could otherwise clear a fresh attach's track, or send
1920
- // `microphoneStop` after its `microphoneStart`).
1921
- const runMicOp = (op: () => Promise<void>): Promise<void> => {
1922
- const next = micOpQueueRef.current.then(op).catch((err) => {
1923
- debugWarn('microphone operation failed:', err);
1924
- });
1925
- micOpQueueRef.current = next;
1926
- return next;
1927
- };
1928
-
1929
- const isMicrophoneWanted = () => microphoneEnabledRef.current || microphoneDemandActiveRef.current;
1930
-
1931
- const sendMicrophoneStart = (ws: WebSocket) => {
1932
- const id = `ui-mic-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
1933
- micStartPendingIdRef.current = id;
1934
- ws.send(JSON.stringify({ type: 'microphoneStart', id }));
1935
- };
1936
-
1937
- // Capture the user's microphone and stream it to the simulator's
1938
- // mock microphone. Safe to call redundantly: bails when already
1939
- // attached, when the connection isn't up yet (the ICE-connected
1940
- // handler re-invokes it), or when the prop flipped off mid-prompt.
1941
- // Does NOT bump the generation: a redundant call (e.g. the ICE
1942
- // handler firing on both `connected` and `completed`) must not
1943
- // cancel an attach already in flight; only detach/teardown cancel.
1944
- const attachMicrophone = (): Promise<void> => {
1945
- // No platform gate: attachMicrophoneOp bails when the session never
1946
- // allocated a sendonly mic transceiver (non-capable servers).
1947
- return runMicOp(() => attachMicrophoneOp());
1948
- };
1949
-
1950
- const attachMicrophoneOp = async () => {
1951
- const generation = micGenerationRef.current;
1952
- const isCurrent = () => generation === micGenerationRef.current && isMicrophoneWanted();
1953
- const reportMicError = (error: string) => {
1954
- safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
1955
- active: false,
1956
- error,
1957
- });
1958
- };
1959
- if (!isCurrent()) return;
1960
- const ws = wsRef.current;
1961
- const sender = outboundMicSenderRef.current;
1962
- if (!ws || ws.readyState !== WebSocket.OPEN || !sender) {
1963
- // Not connected yet — the post-connect hook retries while the
1964
- // prop is still on.
1965
- return;
1966
- }
1967
- if (outboundMicStreamRef.current) return;
1968
- if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
1969
- // eslint-disable-next-line no-console
1970
- console.warn(
1971
- '[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
1972
- 'getUserMedia requires a secure context (https or http://localhost).',
1973
- );
1974
- reportMicError('Microphone capture requires a secure context (https)');
1975
- return;
1976
- }
1977
- let stream: MediaStream | null = null;
1978
- try {
1979
- stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
1980
- } catch (err) {
1981
- // eslint-disable-next-line no-console
1982
- console.warn('[RemoteControl] microphone getUserMedia denied/failed:', err);
1983
- }
1984
- if (!isCurrent()) {
1985
- if (stream) stopMediaStream(stream);
1986
- return;
1987
- }
1988
- if (!stream) {
1989
- reportMicError('Microphone permission denied or no device available');
1990
- return;
1991
- }
1992
- const audioTrack = stream.getAudioTracks()[0] ?? null;
1993
- if (!audioTrack) {
1994
- stopMediaStream(stream);
1995
- reportMicError('Captured stream has no audio track');
1996
- return;
1997
- }
1998
- try {
1999
- await sender.replaceTrack(audioTrack);
2000
- } catch (err) {
2001
- debugWarn('replaceTrack(audioTrack) failed:', err);
2002
- stopMediaStream(stream);
2003
- reportMicError('Failed to attach the microphone track to the connection');
2004
- return;
2005
- }
2006
- if (!isCurrent()) {
2007
- try {
2008
- await sender.replaceTrack(null);
2009
- } catch (err) {
2010
- debugWarn('replaceTrack(null) on stale mic attach failed:', err);
2011
- }
2012
- stopMediaStream(stream);
2013
- return;
2014
- }
2015
- outboundMicStreamRef.current = stream;
2016
- // If the browser revokes the track later (user clicks Stop in
2017
- // the tab's mic indicator, device unplugged, ...) tear down and
2018
- // tell the host, so the sim's mic goes back to silence instead
2019
- // of freezing on the last buffer.
2020
- audioTrack.onended = () => {
2021
- if (outboundMicStreamRef.current !== stream) return;
2022
- void detachMicrophone('microphone capture ended by the browser');
2023
- };
2024
- sendMicrophoneStart(ws);
2025
- safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
2026
- active: true,
2027
- label: audioTrack.label || undefined,
2028
- });
2029
- };
2030
-
2031
- // Stop the live microphone: drop the track from the sender, stop
2032
- // the capture (turning the browser's mic indicator off) and tell
2033
- // the host to switch its audio source back to silence. Safe to
2034
- // call when nothing is attached.
2035
- const detachMicrophone = (error?: string): Promise<void> => {
2036
- // Bump synchronously (not on the queue) so an attach that is
2037
- // suspended on an await bails at its next checkpoint even while
2038
- // it still owns the queue.
2039
- ++micGenerationRef.current;
2040
- micStartPendingIdRef.current = null;
2041
- return runMicOp(async () => {
2042
- const sender = outboundMicSenderRef.current;
2043
- const stream = outboundMicStreamRef.current;
2044
- outboundMicStreamRef.current = null;
2045
- if (sender && stream) {
2046
- try {
2047
- await sender.replaceTrack(null);
2048
- } catch (err) {
2049
- debugWarn('replaceTrack(null) on mic detach failed:', err);
2050
- }
2051
- }
2052
- if (stream) stopMediaStream(stream);
2053
- const ws = wsRef.current;
2054
- if (stream && ws && ws.readyState === WebSocket.OPEN) {
2055
- const id = `ui-mic-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
2056
- ws.send(JSON.stringify({ type: 'microphoneStop', id }));
2057
- }
2058
- if (stream || error) {
2059
- safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
2060
- active: false,
2061
- ...(error !== undefined && { error }),
2062
- });
2063
- }
2064
- });
2065
- };
2066
-
2067
- // Publish a fresh settings snapshot after every successful capture
2068
- // change. This is status metadata; host output sizing is frame-derived.
2069
- const sendGrantedCameraResult = (ws: WebSocket, videoTrack: MediaStreamTrack) => {
2070
- let result;
2071
- try {
2072
- result = createGrantedCameraResult(videoTrack);
2073
- // eslint-disable-next-line no-console
2074
- console.info(
2075
- `[RemoteControl] camera capture: ${result.camera?.width}x${result.camera?.height}` +
2076
- ` @ ${result.camera?.frameRate ?? '?'}fps` +
2077
- (result.camera?.label ? ` — ${result.camera.label}` : ''),
2078
- );
2079
- } catch (err) {
2080
- debugWarn('getSettings() on outbound camera track failed:', err);
2081
- result = {
2082
- type: 'cameraResult' as const,
2083
- granted: true,
2084
- camera: {
2085
- label: videoTrack.label || undefined,
2086
- },
2087
- };
2088
- }
2089
- ws.send(JSON.stringify(result));
2090
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, result.camera);
2091
- };
2092
-
2093
- // Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
2094
- // into a short uppercase label suitable for a status badge.
2095
- const shortenCodecMime = (mime: string | undefined): string | undefined => {
2096
- if (!mime) return undefined;
2097
- const slash = mime.indexOf('/');
2098
- const tail = slash >= 0 ? mime.slice(slash + 1) : mime;
2099
- const upper = tail.toUpperCase();
2100
- if (upper === 'HEVC') return 'H265';
2101
- return upper;
2102
- };
2103
-
2104
- // Sample outbound-camera stats once and push a normalised
2105
- // snapshot through `onCameraStats`. Skips silently if the sender
2106
- // has been torn down between intervals.
2107
- const sampleOutboundCameraStats = async () => {
2108
- const sender = outboundCameraSenderRef.current;
2109
- const handler = onCameraStatsRef.current;
2110
- if (!sender || !handler) return;
2111
- let report: RTCStatsReport;
2112
- try {
2113
- report = await sender.getStats();
2114
- } catch {
2115
- return;
2116
- }
2117
- let outbound: any | undefined;
2118
- let codecMime: string | undefined;
2119
- let remoteInbound: any | undefined;
2120
- report.forEach((entry: any) => {
2121
- if (entry.type === 'outbound-rtp' && entry.kind === 'video') {
2122
- outbound = entry;
2123
- } else if (entry.type === 'remote-inbound-rtp' && entry.kind === 'video') {
2124
- remoteInbound = entry;
2125
- }
2126
- });
2127
- if (outbound?.codecId) {
2128
- const codec = report.get(outbound.codecId);
2129
- if (codec) codecMime = (codec as any).mimeType;
2130
- }
2131
- if (!outbound) return;
2132
- const now = (outbound.timestamp as number) ?? Date.now();
2133
- const prev = cameraStatsPrevRef.current;
2134
- let fps: number | undefined;
2135
- let bitrate: number | undefined;
2136
- let lossPct: number | undefined;
2137
- if (prev && now > prev.timestamp) {
2138
- const dt = (now - prev.timestamp) / 1000;
2139
- if (typeof outbound.framesEncoded === 'number' && typeof prev.framesEncoded === 'number') {
2140
- fps = Math.max(0, (outbound.framesEncoded - prev.framesEncoded) / dt);
2141
- }
2142
- if (typeof outbound.bytesSent === 'number' && typeof prev.bytesSent === 'number') {
2143
- bitrate = Math.max(0, ((outbound.bytesSent - prev.bytesSent) * 8) / dt);
2144
- }
2145
- if (
2146
- typeof outbound.packetsSent === 'number' &&
2147
- typeof prev.packetsSent === 'number' &&
2148
- remoteInbound &&
2149
- typeof remoteInbound.packetsLost === 'number' &&
2150
- typeof prev.packetsLost === 'number'
2151
- ) {
2152
- const sent = outbound.packetsSent - prev.packetsSent;
2153
- const lost = remoteInbound.packetsLost - prev.packetsLost;
2154
- if (sent > 0) lossPct = Math.max(0, Math.min(100, (lost / sent) * 100));
2155
- }
2156
- }
2157
- cameraStatsPrevRef.current = {
2158
- timestamp: now,
2159
- framesEncoded: outbound.framesEncoded,
2160
- bytesSent: outbound.bytesSent,
2161
- packetsSent: outbound.packetsSent,
2162
- packetsLost: remoteInbound?.packetsLost,
2163
- };
2164
- const stats: CameraStreamStats = {
2165
- codec: shortenCodecMime(codecMime),
2166
- encoderImplementation: outbound.encoderImplementation,
2167
- hardwareAccelerated:
2168
- typeof outbound.powerEfficientEncoder === 'boolean' ? outbound.powerEfficientEncoder : undefined,
2169
- framesPerSecond: fps,
2170
- framesEncoded: outbound.framesEncoded,
2171
- bitrateBps: bitrate,
2172
- width: outbound.frameWidth,
2173
- height: outbound.frameHeight,
2174
- qualityLimitationReason: outbound.qualityLimitationReason,
2175
- rttMs:
2176
- typeof remoteInbound?.roundTripTime === 'number' ? remoteInbound.roundTripTime * 1000 : undefined,
2177
- packetsLostPct: lossPct,
2178
- };
2179
- safeInvoke('onCameraStats', onCameraStatsRef.current, stats);
2180
- };
2181
-
2182
- const startCameraStatsPoller = () => {
2183
- if (cameraStatsTimerRef.current !== null) return;
2184
- cameraStatsPrevRef.current = null;
2185
- // First sample fires after one interval — the very first
2186
- // sample has no deltas to compare against, which is fine: the
2187
- // codec/encoder identity is still useful on its own.
2188
- cameraStatsTimerRef.current = setInterval(() => {
2189
- void sampleOutboundCameraStats();
2190
- }, 1000);
2191
- };
2192
-
2193
- const stopCameraStatsPoller = () => {
2194
- if (cameraStatsTimerRef.current !== null) {
2195
- clearInterval(cameraStatsTimerRef.current);
2196
- cameraStatsTimerRef.current = null;
2197
- }
2198
- cameraStatsPrevRef.current = null;
2199
- safeInvoke('onCameraStats', onCameraStatsRef.current, null);
2200
- };
2201
-
2202
- const handleCameraRequest = (
2203
- request: CameraRequest,
2204
- ws: WebSocket,
2205
- sender: RTCRtpSender,
2206
- isCurrentAttempt: () => boolean,
2207
- ): Promise<void> => {
2208
- // `enqueue` increments its generation synchronously, so suspended
2209
- // permission work becomes stale before the newer operation starts.
2210
- return cameraOperations.enqueue(async ({ isCurrent }) => {
2211
- const isCurrentCameraRequest = () =>
2212
- isCurrent() &&
2213
- isCurrentAttempt() &&
2214
- wsRef.current === ws &&
2215
- outboundCameraSenderRef.current === sender;
2216
- if (!isCurrentCameraRequest()) return;
2217
-
2218
- if (!request.active) {
2219
- const stream = outboundLocalStreamRef.current;
2220
- try {
2221
- await sender.replaceTrack(null);
2222
- } catch (err) {
2223
- debugWarn('replaceTrack(null) on camera detach failed:', err);
2224
- }
2225
- if (outboundLocalStreamRef.current === stream) {
2226
- outboundLocalStreamRef.current = null;
2227
- outboundCameraRequestRef.current = undefined;
2228
- if (stream) stopMediaStream(stream);
2229
- }
2230
- stopCameraStatsPoller();
2231
- if (!isCurrentCameraRequest()) return;
2232
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
2233
- return;
2234
- }
2235
-
2236
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
2237
- const currentStream = outboundLocalStreamRef.current;
2238
- const currentTrack = currentStream?.getVideoTracks()[0];
2239
- const currentRequest = outboundCameraRequestRef.current;
2240
-
2241
- if (
2242
- currentTrack?.readyState === 'live' &&
2243
- currentRequest &&
2244
- !shouldReacquireCamera(currentRequest, request)
2245
- ) {
2246
- outboundCameraRequestRef.current = request;
2247
- sendGrantedCameraResult(ws, currentTrack);
2248
- startCameraStatsPoller();
2249
- return;
2250
- }
2251
-
2252
- // A facing change always restarts browser capture. Releasing first
2253
- // lets mobile browsers open the opposite camera; one-camera laptops
2254
- // naturally return the same available input for the new ideal.
2255
- if (currentStream) {
2256
- try {
2257
- await sender.replaceTrack(null);
2258
- } catch (err) {
2259
- debugWarn('replaceTrack(null) before camera switch failed:', err);
2260
- }
2261
- if (outboundLocalStreamRef.current === currentStream) {
2262
- outboundLocalStreamRef.current = null;
2263
- outboundCameraRequestRef.current = undefined;
2264
- stopMediaStream(currentStream);
2265
- }
2266
- stopCameraStatsPoller();
2267
- if (!isCurrentCameraRequest()) return;
2268
- }
2269
-
2270
- if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
2271
- // eslint-disable-next-line no-console
2272
- console.warn(
2273
- '[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
2274
- 'getUserMedia requires a secure context (https or http://localhost). ' +
2275
- 'Replying cameraResult granted=false so limulator falls back to black frames.',
2276
- );
2277
- if (!isCurrentCameraRequest()) return;
2278
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2279
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2280
- return;
2281
- }
2282
-
2283
- let stream: MediaStream | null = null;
2284
- try {
2285
- stream = await navigator.mediaDevices.getUserMedia({
2286
- video: cameraCaptureConstraints(request),
2287
- audio: false,
2288
- });
2289
- } catch (err) {
2290
- // eslint-disable-next-line no-console
2291
- console.warn('[RemoteControl] getUserMedia denied/failed:', err);
2292
- }
2293
- if (!isCurrentCameraRequest()) {
2294
- if (stream) stopMediaStream(stream);
2295
- return;
2296
- }
2297
- if (!stream) {
2298
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2299
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2300
- return;
2301
- }
2302
-
2303
- const videoTrack = stream.getVideoTracks()[0] ?? null;
2304
- if (!videoTrack) {
2305
- stopMediaStream(stream);
2306
- if (!isCurrentCameraRequest()) return;
2307
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2308
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2309
- return;
2310
- }
2311
-
2312
- try {
2313
- videoTrack.contentHint = 'motion';
2314
- } catch {
2315
- /* older browsers don't support contentHint; ignore */
2316
- }
2317
-
2318
- try {
2319
- await sender.replaceTrack(videoTrack);
2320
- } catch (err) {
2321
- debugWarn('replaceTrack(videoTrack) failed:', err);
2322
- stopMediaStream(stream);
2323
- if (!isCurrentCameraRequest()) return;
2324
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2325
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2326
- return;
2327
- }
2328
- if (!isCurrentCameraRequest()) {
2329
- try {
2330
- await sender.replaceTrack(null);
2331
- } catch (err) {
2332
- debugWarn('replaceTrack(null) for stale camera attach failed:', err);
2333
- }
2334
- stopMediaStream(stream);
2335
- return;
2336
- }
2337
-
2338
- try {
2339
- const params = sender.getParameters();
2340
- if (!params.encodings || params.encodings.length === 0) {
2341
- params.encodings = [{}];
2342
- }
2343
- params.encodings[0].maxBitrate = 8_000_000;
2344
- params.encodings[0].maxFramerate = 30;
2345
- params.degradationPreference = 'maintain-framerate';
2346
- await sender.setParameters(params);
2347
- } catch (err) {
2348
- debugWarn('setParameters on outbound camera failed:', err);
2349
- }
2350
- if (!isCurrentCameraRequest()) {
2351
- try {
2352
- await sender.replaceTrack(null);
2353
- } catch (err) {
2354
- debugWarn('replaceTrack(null) for stale camera finalize failed:', err);
2355
- }
2356
- stopMediaStream(stream);
2357
- return;
2358
- }
2359
-
2360
- outboundLocalStreamRef.current = stream;
2361
- outboundCameraRequestRef.current = request;
2362
-
2363
- videoTrack.onended = () => {
2364
- if (outboundLocalStreamRef.current !== stream) return;
2365
- void cameraOperations
2366
- .enqueueCurrent(async ({ isCurrent: isCurrentEndedCleanup }) => {
2367
- if (
2368
- !isCurrentEndedCleanup() ||
2369
- outboundLocalStreamRef.current !== stream ||
2370
- !isCurrentAttempt() ||
2371
- wsRef.current !== ws ||
2372
- outboundCameraSenderRef.current !== sender
2373
- ) {
2374
- return;
2375
- }
2376
- try {
2377
- await sender.replaceTrack(null);
2378
- } catch (err) {
2379
- debugWarn('replaceTrack(null) after camera ended failed:', err);
2380
- }
2381
- if (
2382
- !isCurrentEndedCleanup() ||
2383
- outboundLocalStreamRef.current !== stream ||
2384
- !isCurrentAttempt() ||
2385
- wsRef.current !== ws ||
2386
- outboundCameraSenderRef.current !== sender
2387
- ) {
2388
- return;
2389
- }
2390
- outboundLocalStreamRef.current = null;
2391
- outboundCameraRequestRef.current = undefined;
2392
- stopMediaStream(stream);
2393
- stopCameraStatsPoller();
2394
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2395
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2396
- })
2397
- .catch((err) => debugWarn('camera ended cleanup failed:', err));
2398
- };
2399
-
2400
- if (!isCurrentCameraRequest()) return;
2401
- sendGrantedCameraResult(ws, videoTrack);
2402
- startCameraStatsPoller();
2403
- });
2404
- };
2405
-
2406
- const teardownConnection = () => {
2407
- for (const request of duoRequests.current.values()) {
2408
- clearTimeout(request.timer);
2409
- request.reject(new Error('Simulator disconnected'));
2410
- }
2411
- duoRequests.current.clear();
2412
- setDuoState(null);
2413
- if (innerVideoRef.current) innerVideoRef.current.srcObject = null;
2414
- clearConnectionSuccessTimeout();
2415
- clearIceDisconnectedGrace();
2416
- stopRequestFrameLoop();
2417
- if (axFetcherRef.current) {
2418
- axFetcherRef.current.stop();
2419
- axFetcherRef.current = null;
2420
- }
2421
- // Drop any active outbound camera before the PC dies so the
2422
- // browser doesn't leave the camera indicator lit between
2423
- // reconnects.
2424
- cameraOperations.invalidate();
2425
- stopCameraStatsPoller();
2426
- stopOutboundLocalStream();
2427
- outboundCameraRequestRef.current = undefined;
2428
- outboundCameraSenderRef.current = null;
2429
- // Same for the live microphone. Manual enablement survives reconnect;
2430
- // guest demand is replayed by the host after it assigns the new session.
2431
- ++micGenerationRef.current;
2432
- microphoneDemandActiveRef.current = false;
2433
- micStartPendingIdRef.current = null;
2434
- if (outboundMicStreamRef.current) {
2435
- stopMediaStream(outboundMicStreamRef.current);
2436
- outboundMicStreamRef.current = null;
2437
- // The capture is gone; report it so consumers don't keep a stale
2438
- // `active: true` across drops (a re-attach after reconnect
2439
- // reports again).
2440
- safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
2441
- active: false,
2442
- });
2443
- }
2444
- outboundMicSenderRef.current = null;
2445
- // A scheduled cursor flush would otherwise call setState on a
2446
- // teardown component once the next frame runs.
2447
- if (cursorRafIdRef.current !== undefined) {
2448
- window.cancelAnimationFrame(cursorRafIdRef.current);
2449
- cursorRafIdRef.current = undefined;
2450
- }
2451
- if (wsRef.current) {
2452
- wsRef.current.onopen = null;
2453
- wsRef.current.onmessage = null;
2454
- wsRef.current.onerror = null;
2455
- wsRef.current.onclose = null;
2456
- wsRef.current.close();
2457
- wsRef.current = null;
2458
- }
2459
- if (peerConnectionRef.current) {
2460
- peerConnectionRef.current.onconnectionstatechange = null;
2461
- peerConnectionRef.current.oniceconnectionstatechange = null;
2462
- peerConnectionRef.current.ontrack = null;
2463
- peerConnectionRef.current.onicecandidate = null;
2464
- peerConnectionRef.current.close();
2465
- peerConnectionRef.current = null;
2466
- }
2467
- if (videoRef.current) {
2468
- videoRef.current.srcObject = null;
2469
- }
2470
- if (dataChannelRef.current) {
2471
- dataChannelRef.current.onopen = null;
2472
- dataChannelRef.current.onclose = null;
2473
- dataChannelRef.current.onerror = null;
2474
- dataChannelRef.current.close();
2475
- dataChannelRef.current = null;
2476
- }
2477
- };
2478
-
2479
- const handleVisibilityChange = () => {
2480
- if (document.hidden) {
2481
- stopKeepAlive();
2482
- } else {
2483
- startKeepAlive();
2484
- }
2485
- };
2486
-
2487
- // Probe the instance endpoint over HTTP to tell a permanent termination
2488
- // apart from a recoverable connection drop. The router serves the same
2489
- // path the WebSocket uses and returns 404 once the instance is gone
2490
- // (deleted / terminated), while a live instance returns something else.
2491
- //
2492
- // Only an authoritative, freshly-fetched 404 counts as terminated. A
2493
- // network outage, a non-404 status, or a CORS-blocked (unreadable)
2494
- // response all resolve to `false` so we keep the normal retry/reconnect
2495
- // behavior and never mistake a connectivity problem for a termination.
2496
- const probeTerminated = async (): Promise<boolean> => {
2497
- // The browser knows it has no network — this is a connectivity drop,
2498
- // not a termination. Don't probe (and don't risk a cached 404).
2499
- if (typeof navigator !== 'undefined' && navigator.onLine === false) {
2500
- return false;
2501
- }
2502
- let probeUrl: string;
2503
- try {
2504
- const parsed = new URL(url);
2505
- parsed.protocol =
2506
- parsed.protocol === 'wss:' ? 'https:'
2507
- : parsed.protocol === 'ws:' ? 'http:'
2508
- : parsed.protocol;
2509
- probeUrl = withAuthenticationToken(parsed.toString(), token);
2510
- } catch {
2511
- return false;
2512
- }
2513
- const controller = new AbortController();
2514
- const timeoutId = window.setTimeout(() => controller.abort(), TERMINATION_PROBE_TIMEOUT_MS);
2515
- try {
2516
- // `no-store` so a prior response can't be served from the HTTP cache
2517
- // (e.g. a stale 404 returned while offline would look like a
2518
- // termination); the probe must always reach the live router.
2519
- const res = await fetch(probeUrl, { method: 'GET', cache: 'no-store', signal: controller.signal });
2520
- return res.status === 404;
2521
- } catch {
2522
- return false;
2523
- } finally {
2524
- window.clearTimeout(timeoutId);
2525
- }
2526
- };
2527
-
2528
- const markTerminated = () => {
2529
- if (terminatedRef.current) return;
2530
- terminatedRef.current = true;
2531
- setTerminated(true);
2532
- setRetryExhausted(false);
2533
- // Invalidate any in-flight/scheduled attempt and stop all activity so
2534
- // we don't keep hammering an endpoint that will never come back.
2535
- connectionGenerationRef.current += 1;
2536
- clearScheduledRetry();
2537
- clearConnectionSuccessTimeout();
2538
- clearIceDisconnectedGrace();
2539
- teardownConnection();
2540
- updateStatus('Instance terminated');
2541
- reportConnectionState('terminated');
2542
- safeInvoke('onTerminated', onTerminatedRef.current);
2543
- };
2544
-
2545
- // Run a one-shot termination check in reaction to a drop. Event-driven
2546
- // (no polling), deduped while one is already in flight, and a no-op once
2547
- // we've concluded the instance is gone.
2548
- const checkTerminated = () => {
2549
- if (terminatedRef.current || terminationProbeInFlightRef.current) return;
2550
- terminationProbeInFlightRef.current = true;
2551
- void probeTerminated().then((isTerminated) => {
2552
- terminationProbeInFlightRef.current = false;
2553
- if (isTerminated) {
2554
- markTerminated();
2555
- }
2556
- });
2557
- };
2558
-
2559
- const scheduleRetry = (reason: string, generation: number) => {
2560
- if (terminatedRef.current) {
2561
- return;
2562
- }
2563
- if (generation !== connectionGenerationRef.current) {
2564
- return;
2565
- }
2566
-
2567
- // A drop just happened. Find out whether the instance is gone for good
2568
- // (404) rather than a recoverable blip; if so the probe stops retries
2569
- // and surfaces the terminated message. Meanwhile the normal retry path
2570
- // below proceeds and is short-circuited once the verdict lands.
2571
- checkTerminated();
2572
-
2573
- if (controlChannelOpenedRef.current) {
2574
- if (!autoReconnectRef.current) {
2575
- updateStatus(`Connection failed after it was established: ${reason}`);
2576
- setRetryExhausted(true);
2577
- reportConnectionState('failed');
2578
- teardownConnection();
2579
- return;
2580
- }
2581
- // Reset so the upcoming retry gets a fresh MAX_CONNECTION_ATTEMPTS budget.
2582
- updateStatus(`Reconnecting after established session dropped: ${reason}`);
2583
- controlChannelOpenedRef.current = false;
2584
- connectionAttemptRef.current = -1;
2585
- }
2586
-
2587
- clearScheduledRetry();
2588
-
2589
- const nextAttempt = connectionAttemptRef.current + 1;
2590
- if (nextAttempt >= MAX_CONNECTION_ATTEMPTS) {
2591
- updateStatus(`Connection failed after ${MAX_CONNECTION_ATTEMPTS} attempts: ${reason}`);
2592
- setRetryExhausted(true);
2593
- reportConnectionState('failed');
2594
- teardownConnection();
2595
- return;
2596
- }
2597
-
2598
- updateStatus(`Retrying connection (${nextAttempt + 1}/${MAX_CONNECTION_ATTEMPTS})`);
2599
- reportConnectionState('reconnecting');
2600
- teardownConnection();
2601
- retryTimeoutRef.current = window.setTimeout(() => {
2602
- retryTimeoutRef.current = undefined;
2603
- if (generation !== connectionGenerationRef.current) {
2604
- return;
2605
- }
2606
- void startAttempt(nextAttempt);
2607
- }, CONNECTION_RETRY_DELAY_MS);
2608
- };
2609
-
2610
- const startAttempt = async (attemptNumber = 0) => {
2611
- // Never reconnect to an instance we've concluded is gone. `start()`
2612
- // clears this when a deliberate fresh connection is requested.
2613
- if (terminatedRef.current) return;
2614
- const generation = connectionGenerationRef.current + 1;
2615
- connectionGenerationRef.current = generation;
2616
- connectionAttemptRef.current = attemptNumber;
2617
- controlChannelOpenedRef.current = false;
2618
- setRetryExhausted(false);
2619
- // A drop after an established session restarts the attempt budget at 0
2620
- // while staying 'reconnecting'; only a fresh start reports 'connecting'.
2621
- if (attemptNumber === 0 && connectionStateRef.current !== 'reconnecting')
2622
- reportConnectionState('connecting');
2623
- clearScheduledRetry();
2624
- clearConnectionSuccessTimeout();
2625
- stopRequestFrameLoop();
2626
- firstFrameShownRef.current = false;
2627
- setVideoLoaded(false);
2628
- teardownConnection();
2629
-
2630
- const isCurrentAttempt = () => generation === connectionGenerationRef.current;
2631
-
2632
- connectionSuccessTimeoutRef.current = window.setTimeout(() => {
2633
- connectionSuccessTimeoutRef.current = undefined;
2634
- if (!isCurrentAttempt() || controlChannelOpenedRef.current) {
2635
- return;
2636
- }
2637
- scheduleRetry('Connection did not succeed within 15 seconds', generation);
2638
- }, CONNECTION_SUCCESS_TIMEOUT_MS);
2639
-
2640
- try {
2641
- const ws = new WebSocket(withAuthenticationToken(url, token));
2642
- wsRef.current = ws;
2643
-
2644
- // Wait for WebSocket to connect
2645
- await new Promise<void>((resolve, reject) => {
2646
- let settled = false;
2647
- const timeoutId = window.setTimeout(() => reject(new Error('WebSocket connection timeout')), 30000);
2648
- const settle = (callback: () => void) => {
2649
- if (settled) {
2650
- return;
2651
- }
2652
- settled = true;
2653
- window.clearTimeout(timeoutId);
2654
- callback();
2655
- };
2656
-
2657
- ws.onopen = () => {
2658
- if (!isCurrentAttempt() || wsRef.current !== ws) {
2659
- return;
2660
- }
2661
- settle(resolve);
2662
- };
2663
-
2664
- ws.onerror = (error) => {
2665
- if (!isCurrentAttempt() || wsRef.current !== ws) {
2666
- return;
2667
- }
2668
- updateStatus('WebSocket error: ' + error);
2669
- settle(() => reject(new Error('WebSocket connection failed')));
2670
- };
2671
-
2672
- ws.onclose = () => {
2673
- if (!isCurrentAttempt() || wsRef.current !== ws) {
2674
- return;
2675
- }
2676
- updateStatus('WebSocket closed');
2677
- settle(() => reject(new Error('WebSocket closed before connection was established')));
2678
- };
2679
- });
2680
- if (!isCurrentAttempt() || wsRef.current !== ws) {
2681
- return;
2682
- }
2683
-
2684
- ws.onerror = (error) => {
2685
- if (!isCurrentAttempt() || wsRef.current !== ws) {
2686
- return;
2687
- }
2688
- updateStatus('WebSocket error: ' + error);
2689
- };
2690
-
2691
- ws.onclose = () => {
2692
- if (!isCurrentAttempt() || wsRef.current !== ws) {
2693
- return;
2694
- }
2695
- updateStatus('WebSocket closed');
2696
- };
2697
-
2698
- // Request RTCConfiguration. The reply doubles as the capability
2699
- // handshake: camera-capable Android servers advertise
2700
- // `cameraSupported: true` alongside the configuration.
2701
- const rtcConfigPromise = new Promise<{
2702
- rtcConfiguration: RTCConfiguration;
2703
- cameraSupported: boolean;
2704
- foldSupported: boolean;
2705
- }>((resolve, reject) => {
2706
- const settle = (callback: () => void) => {
2707
- window.clearTimeout(timeoutId);
2708
- ws.removeEventListener('message', messageHandler);
2709
- callback();
2710
- };
2711
- const timeoutId = window.setTimeout(
2712
- () => settle(() => reject(new Error('RTCConfiguration timeout'))),
2713
- 30000,
2714
- );
2715
-
2716
- const messageHandler = (event: MessageEvent) => {
2717
- if (!isCurrentAttempt() || wsRef.current !== ws) {
2718
- return;
2719
- }
2720
- try {
2721
- const message = JSON.parse(event.data);
2722
- if (
2723
- message.type === 'signalingError' &&
2724
- message.requestType === 'requestRtcConfiguration' &&
2725
- message.sessionId === sessionId
2726
- ) {
2727
- const reason =
2728
- typeof message.error === 'string' ?
2729
- message.error
2730
- : 'Unable to load streaming configuration';
2731
- settle(() => reject(new Error(reason)));
2732
- } else if (message.type === 'rtcConfiguration') {
2733
- settle(() =>
2734
- resolve({
2735
- rtcConfiguration: message.rtcConfiguration,
2736
- cameraSupported: message.cameraSupported === true,
2737
- foldSupported: message.foldSupported === true,
2738
- }),
2739
- );
2740
- }
2741
- } catch (e) {
2742
- console.error('Error handling RTC configuration:', e);
2743
- settle(() => reject(e));
2744
- }
2745
- };
2746
-
2747
- ws.addEventListener('message', messageHandler);
2748
- ws.send(
2749
- JSON.stringify({
2750
- type: 'requestRtcConfiguration',
2751
- sessionId: sessionId,
2752
- }),
2753
- );
2754
- });
2755
-
2756
- const {
2757
- rtcConfiguration: rtcConfig,
2758
- cameraSupported: serverCameraSupported,
2759
- foldSupported,
2760
- } = await rtcConfigPromise;
2761
- if (!isCurrentAttempt() || wsRef.current !== ws) {
2762
- return;
2763
- }
2764
-
2765
- const isDuo = deviceModel === 'iphone-duo' || foldSupported;
2766
- setDetectedDuo(isDuo);
2767
- const peerConnection = new RTCPeerConnection(rtcConfig);
2768
- peerConnectionRef.current = peerConnection;
2769
- peerConnection.addTransceiver('audio', { direction: 'recvonly' });
2770
- const videoTransceiver = peerConnection.addTransceiver('video', { direction: 'recvonly' });
2771
- const duoTransceiver =
2772
- isDuo ? peerConnection.addTransceiver('video', { direction: 'recvonly' }) : null;
2773
-
2774
- // iOS always supports camera/mic injection; Android servers
2775
- // advertise it through `cameraSupported` (Android 15 and newer).
2776
- // Never offer the extra sendonly m-lines to non-capable Android
2777
- // servers — older servers cannot negotiate them safely.
2778
- const canSendMedia = platform === 'ios' || serverCameraSupported;
2779
-
2780
- // Pre-allocate a sendonly slot for the user's camera so the
2781
- // device can pull frames later without renegotiating SDP.
2782
- // Allocating up-front means we don't have to renegotiate the
2783
- // SDP when the device app actually opens its camera session —
2784
- // the codec/SSRC negotiation happens once, here, and turning
2785
- // the camera on/off later is just a `replaceTrack`.
2786
- const outboundCameraTransceiver =
2787
- canSendMedia ?
2788
- peerConnection.addTransceiver('video', {
2789
- direction: 'sendonly',
2790
- })
2791
- : null;
2792
- outboundCameraSenderRef.current = outboundCameraTransceiver?.sender ?? null;
2793
-
2794
- // Same pre-allocation for the live microphone: a sendonly audio
2795
- // slot negotiated once at connection setup, so toggling the mic
2796
- // later is just `replaceTrack` — no SDP renegotiation.
2797
- const outboundMicTransceiver =
2798
- canSendMedia ?
2799
- peerConnection.addTransceiver('audio', {
2800
- direction: 'sendonly',
2801
- })
2802
- : null;
2803
- outboundMicSenderRef.current = outboundMicTransceiver?.sender ?? null;
2804
-
2805
- // As hardware encoder, we use H265 for iOS and VP9 for Android.
2806
- // We make sure these two are the first ones in the list.
2807
- // If not, the fallback is H264 which is also hardware accelerated, although not as good,
2808
- // available on all platforms.
2809
- //
2810
- // The rest is not important.
2811
- if (RTCRtpReceiver.getCapabilities) {
2812
- const capabilities = RTCRtpReceiver.getCapabilities('video');
2813
- if (capabilities && capabilities.codecs) {
2814
- const codecs = capabilities.codecs;
2815
- const sortedCodecs = codecs.sort((a, b) => {
2816
- const getCodecPriority = (codec: { mimeType: string }): number => {
2817
- const mimeType = codec.mimeType.toLowerCase();
2818
- if (mimeType.includes('vp9')) return 1;
2819
- if (mimeType.includes('h265') || mimeType.includes('hevc')) return 2;
2820
- if (mimeType.includes('h264') || mimeType.includes('avc')) return 3;
2821
- return 4; // Everything else
2822
- };
2823
- return getCodecPriority(a) - getCodecPriority(b);
2824
- });
2825
- videoTransceiver.setCodecPreferences(sortedCodecs);
2826
- duoTransceiver?.setCodecPreferences(sortedCodecs);
2827
- debugLog('Set codec preferences:', sortedCodecs.map((c) => c.mimeType).join(', '));
2828
- }
2829
- }
2830
-
2831
- // Pin the outbound camera transceiver's codec order with
2832
- // negotiation-time fallback. Preference order:
2833
- // 1. H.265 / HEVC — VideoToolbox HW on M-series + recent
2834
- // Chrome. ~30% less bitrate at equal quality. If the
2835
- // answerer (limulator) supports HEVC, this wins.
2836
- // 2. H.264 with VideoToolbox-friendly profile-level-ids
2837
- // (42e01f / 42e028 / 640c1f). Universally HW-accelerated
2838
- // on every Mac since 2009. SDP-time fallback when HEVC
2839
- // isn't in the answer.
2840
- // 3. Any other H.264 profile. Often resolves to OpenH264
2841
- // (software) so we keep it explicitly last among H.264
2842
- // to avoid Chrome's default pick.
2843
- // 4. VP9 / VP8 (libvpx software on Mac). Last resort.
2844
- //
2845
- // Caveat: this is *negotiation* fallback. If HEVC HW encoder
2846
- // init fails at session start or gets demoted mid-stream,
2847
- // Chrome falls back to *software* HEVC, not H.264 — no spec
2848
- // hook for runtime re-negotiation. The host-side stats
2849
- // logger will surface this as decoder=ffmpeg / hw=false.
2850
- if (outboundCameraTransceiver && RTCRtpSender.getCapabilities) {
2851
- const senderCaps = RTCRtpSender.getCapabilities('video');
2852
- if (senderCaps && senderCaps.codecs) {
2853
- const vtH264Profiles = new Set(['42e01f', '42e028', '640c1f']);
2854
- const score = (c: { mimeType: string; sdpFmtpLine?: string }): number => {
2855
- const mime = c.mimeType.toLowerCase();
2856
- const fmtp = (c.sdpFmtpLine || '').toLowerCase();
2857
- const profileMatch = fmtp.match(/profile-level-id=([0-9a-f]{6})/);
2858
- const profile = profileMatch ? profileMatch[1] : '';
2859
- if (mime === 'video/h265' || mime === 'video/hevc') return 1;
2860
- if (mime === 'video/h264' && vtH264Profiles.has(profile)) return 2;
2861
- if (mime === 'video/h264') return 3;
2862
- if (mime === 'video/vp9') return 4;
2863
- if (mime === 'video/vp8') return 5;
2864
- if (mime === 'video/rtx' || mime === 'video/red' || mime === 'video/ulpfec') {
2865
- return 0; // Keep RTX/FEC available alongside everything else.
2866
- }
2867
- return 6;
2868
- };
2869
- const sortedSendCodecs = [...senderCaps.codecs].sort((a, b) => score(a) - score(b));
2870
- try {
2871
- outboundCameraTransceiver.setCodecPreferences(sortedSendCodecs);
2872
- debugLog(
2873
- 'Outbound camera codec preferences:',
2874
- sortedSendCodecs
2875
- .map((c) => `${c.mimeType}${c.sdpFmtpLine ? `[${c.sdpFmtpLine}]` : ''}`)
2876
- .join(', '),
2877
- );
2878
- } catch (err) {
2879
- debugWarn('Failed to set outbound camera codec preferences:', err);
2880
- }
2881
- }
2882
- }
2883
-
2884
- const dataChannel = peerConnection.createDataChannel('control', {
2885
- ordered: true,
2886
- negotiated: true,
2887
- id: 1,
2888
- });
2889
- dataChannelRef.current = dataChannel;
2890
-
2891
- dataChannel.onopen = () => {
2892
- if (!isCurrentAttempt() || dataChannelRef.current !== dataChannel || wsRef.current !== ws) {
2893
- return;
2894
- }
2895
- controlChannelOpenedRef.current = true;
2896
- clearConnectionSuccessTimeout();
2897
- updateStatus('Control channel opened');
2898
- if (isDuo) ws.send(JSON.stringify({ type: 'getFoldState', id: 'duo-capabilities' }));
2899
-
2900
- // Spin up the AX fetcher now that we have a stable WS + control
2901
- // channel. The fetcher's send function reuses this WS; it stops
2902
- // sending if the WS dies. start() is called lazily based on the
2903
- // inspectMode prop via a sibling useEffect.
2904
- if (!axFetcherRef.current) {
2905
- axFetcherRef.current = new AxFetcher({
2906
- platform,
2907
- baseIntervalMs: axPollIntervalMs,
2908
- maxBackoffMs: axMaxBackoffMs,
2909
- androidElementTreeOptions,
2910
- send: (payload) => {
2911
- if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return false;
2912
- try {
2913
- wsRef.current.send(JSON.stringify(payload));
2914
- return true;
2915
- } catch {
2916
- return false;
2917
- }
2918
- },
2919
- onSnapshot: (snapshot) => {
2920
- setAxSnapshot((prev) => (axSnapshotsEqual(prev, snapshot) ? prev : snapshot));
2921
- // Defer to a microtask so customer code (which may DOM-write,
2922
- // start expensive work, or itself call back into ref
2923
- // methods) doesn't run synchronously inside our state-setter
2924
- // path. React then has a chance to schedule its render before
2925
- // the customer handler kicks off side-effects.
2926
- queueMicrotask(() => {
2927
- safeInvoke('onAxSnapshotChange', onAxSnapshotChangeRef.current, snapshot);
2928
- });
2929
- },
2930
- onStatusChange: (status, error) => {
2931
- safeInvoke('onAxStatusChange', onAxStatusChangeRef.current, status, error);
2932
- },
2933
- });
2934
- if (inspectModeRef.current === true || inspectModeRef.current === 'hover-only') {
2935
- axFetcherRef.current.start();
2936
- }
2937
- }
2938
- const sendRequestFrame = () => {
2939
- if (
2940
- !isCurrentAttempt() ||
2941
- firstFrameShownRef.current ||
2942
- dataChannelRef.current !== dataChannel ||
2943
- wsRef.current !== ws ||
2944
- ws.readyState !== WebSocket.OPEN
2945
- ) {
2946
- return;
2947
- }
2948
- ws.send(JSON.stringify({ type: 'requestFrame', sessionId: sessionId }));
2949
- };
2950
-
2951
- sendRequestFrame();
2952
- stopRequestFrameLoop();
2953
- requestFrameIntervalRef.current = window.setInterval(() => {
2954
- if (
2955
- !isCurrentAttempt() ||
2956
- firstFrameShownRef.current ||
2957
- dataChannelRef.current !== dataChannel ||
2958
- wsRef.current !== ws ||
2959
- ws.readyState !== WebSocket.OPEN
2960
- ) {
2961
- stopRequestFrameLoop();
2962
- return;
2963
- }
2964
- sendRequestFrame();
2965
- }, 250);
2966
-
2967
- // Send openUrl message if the prop is provided
2968
- if (openUrl) {
2969
- try {
2970
- const decodedUrl = decodeURIComponent(openUrl);
2971
- updateStatus('Opening URL');
2972
- ws.send(
2973
- JSON.stringify({
2974
- type: 'openUrl',
2975
- url: decodedUrl,
2976
- sessionId: sessionId,
2977
- }),
2978
- );
2979
- } catch (error) {
2980
- console.error({ error }, 'Error decoding URL, falling back to the original URL');
2981
- ws.send(
2982
- JSON.stringify({
2983
- type: 'openUrl',
2984
- url: openUrl,
2985
- sessionId: sessionId,
2986
- }),
2987
- );
2988
- }
2989
- // openUrl can take a moment to load the destination — boost
2990
- // AX polling so the overlay refreshes through the transition.
2991
- axFetcherRef.current?.bumpActivity();
2992
- }
2993
- };
2994
-
2995
- dataChannel.onclose = () => {
2996
- if (!isCurrentAttempt() || dataChannelRef.current !== dataChannel) {
2997
- return;
2998
- }
2999
- updateStatus('Control channel closed');
3000
- };
3001
-
3002
- dataChannel.onerror = (error) => {
3003
- if (!isCurrentAttempt() || dataChannelRef.current !== dataChannel) {
3004
- return;
3005
- }
3006
- console.error('Control channel error:', error);
3007
- updateStatus('Control channel error: ' + error);
3008
- };
3009
-
3010
- // Set up connection state monitoring
3011
- peerConnection.onconnectionstatechange = () => {
3012
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
3013
- return;
3014
- }
3015
- updateStatus('Connection state: ' + peerConnection.connectionState);
3016
- if (peerConnection.connectionState === 'failed') {
3017
- scheduleRetry('WebRTC connection entered failed state', generation);
3018
- }
3019
- };
3020
-
3021
- peerConnection.oniceconnectionstatechange = () => {
3022
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
3023
- return;
3024
- }
3025
- const iceState = peerConnection.iceConnectionState;
3026
- updateStatus('ICE state: ' + iceState);
3027
- if (iceState === 'connected' || iceState === 'completed') {
3028
- clearIceDisconnectedGrace();
3029
- // Re-attach the live microphone across reconnects (or apply a
3030
- // prop that was flipped on before the connection was up).
3031
- if (isMicrophoneWanted() && !outboundMicStreamRef.current) {
3032
- void attachMicrophone();
3033
- }
3034
- return;
3035
- }
3036
- if (iceState === 'failed') {
3037
- clearIceDisconnectedGrace();
3038
- scheduleRetry('ICE connection entered failed state', generation);
3039
- return;
3040
- }
3041
- if (
3042
- iceState === 'disconnected' &&
3043
- autoReconnectRef.current &&
3044
- iceDisconnectedGraceRef.current === undefined
3045
- ) {
3046
- // Cap the browser's natural disconnected→failed escalation to recover faster.
3047
- iceDisconnectedGraceRef.current = window.setTimeout(() => {
3048
- iceDisconnectedGraceRef.current = undefined;
3049
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
3050
- return;
3051
- }
3052
- if (peerConnection.iceConnectionState === 'disconnected') {
3053
- scheduleRetry('ICE stayed disconnected past grace period', generation);
3054
- }
3055
- }, ICE_DISCONNECTED_GRACE_MS);
3056
- }
3057
- };
3058
-
3059
- // Set up video handling
3060
- peerConnection.ontrack = (event) => {
3061
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
3062
- return;
3063
- }
3064
- updateStatus('Received remote track: ' + event.track.kind);
3065
- if (event.track.kind === 'video' && videoRef.current) {
3066
- debugLog(`[${new Date().toISOString()}] Video track received:`, event.track);
3067
- const inner =
3068
- event.track.id === 'duo-inner' || event.streams.some((stream) => stream.id === 'duo-inner');
3069
- const target = inner ? innerVideoRef.current : videoRef.current;
3070
- if (target) {
3071
- target.srcObject =
3072
- isDuo ? new MediaStream([event.track]) : event.streams[0] ?? new MediaStream([event.track]);
3073
- void target.play().catch(() => undefined);
3074
- }
3075
- }
3076
- };
3077
-
3078
- // Handle ICE candidates
3079
- peerConnection.onicecandidate = (event) => {
3080
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection || wsRef.current !== ws) {
3081
- return;
3082
- }
3083
- if (event.candidate && ws.readyState === WebSocket.OPEN) {
3084
- const message = {
3085
- type: 'candidate',
3086
- candidate: event.candidate.candidate,
3087
- sdpMid: event.candidate.sdpMid,
3088
- sdpMLineIndex: event.candidate.sdpMLineIndex,
3089
- sessionId: sessionId,
3090
- };
3091
- ws.send(JSON.stringify(message));
3092
- updateStatus('Sent ICE candidate');
3093
- } else {
3094
- updateStatus('ICE candidate gathering completed');
3095
- }
3096
- };
3097
-
3098
- // Handle incoming messages
3099
- ws.onmessage = async (event) => {
3100
- if (!isCurrentAttempt() || wsRef.current !== ws) {
3101
- return;
3102
- }
3103
- let message;
3104
- try {
3105
- message = JSON.parse(event.data);
3106
- } catch (e) {
3107
- debugWarn('Error parsing message:', e);
3108
- return;
3109
- }
3110
- // Inspect-mode responses are routed to the fetcher first so it
3111
- // can resolve in-flight requests regardless of which platform's
3112
- // protocol is in use.
3113
- if (axFetcherRef.current?.handleMessage(message)) {
3114
- return;
3115
- }
3116
- updateStatus('Received: ' + message.type);
3117
- switch (message.type) {
3118
- case 'performActionsResult':
3119
- case 'foldStateResult':
3120
- case 'foldStateChanged': {
3121
- if (!isDuo) break;
3122
- if (message.state) {
3123
- setDuoState(message.state);
3124
- setDuoError(undefined);
3125
- } else if (message.id === 'duo-capabilities')
3126
- setDuoError(message.error || 'This instance does not support native iPhone Duo displays.');
3127
- const request = duoRequests.current.get(message.id);
3128
- if (request) {
3129
- clearTimeout(request.timer);
3130
- duoRequests.current.delete(message.id);
3131
- if (message.error) request.reject(new Error(message.error));
3132
- else request.resolve();
3133
- }
3134
- break;
3135
- }
3136
- case 'signalingError': {
3137
- if (message.requestType !== 'offer' || message.sessionId !== sessionId) {
3138
- break;
3139
- }
3140
- const reason =
3141
- typeof message.error === 'string' ?
3142
- message.error
3143
- : 'Unable to establish the streaming connection';
3144
- updateStatus('Error: ' + reason);
3145
- scheduleRetry(reason, generation);
3146
- break;
3147
- }
3148
- case 'answer':
3149
- if (!peerConnectionRef.current || peerConnectionRef.current !== peerConnection) {
3150
- updateStatus('No peer connection, skipping answer');
3151
- break;
3152
- }
3153
- await peerConnection.setRemoteDescription(
3154
- new RTCSessionDescription({
3155
- type: 'answer',
3156
- sdp: message.sdp,
3157
- }),
3158
- );
3159
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
3160
- return;
3161
- }
3162
- updateStatus('Set remote description');
3163
- break;
3164
- case 'candidate':
3165
- if (!peerConnectionRef.current || peerConnectionRef.current !== peerConnection) {
3166
- updateStatus('No peer connection, skipping candidate');
3167
- break;
3168
- }
3169
- await peerConnection.addIceCandidate(
3170
- new RTCIceCandidate({
3171
- candidate: message.candidate,
3172
- sdpMid: message.sdpMid,
3173
- sdpMLineIndex: message.sdpMLineIndex,
3174
- }),
3175
- );
3176
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
3177
- return;
3178
- }
3179
- updateStatus('Added ICE candidate');
3180
- break;
3181
- case 'screenshot':
3182
- case 'screenshotResult': {
3183
- if (typeof message.id !== 'string') {
3184
- debugWarn('Received invalid screenshot success message:', message);
3185
- break;
3186
- }
3187
- const screenshotError = getScreenshotError(message);
3188
- if (screenshotError) {
3189
- const rejecter = pendingScreenshotRejectersRef.current.get(message.id);
3190
- if (!rejecter) {
3191
- debugWarn(`Received screenshot error for unknown or handled id: ${message.id}`);
3192
- break;
3193
- }
3194
- debugWarn(`Received screenshot error for id ${message.id}: ${screenshotError}`);
3195
- rejecter(new Error(screenshotError));
3196
- pendingScreenshotResolversRef.current.delete(message.id);
3197
- pendingScreenshotRejectersRef.current.delete(message.id);
3198
- break;
3199
- }
3200
- const screenshotData = toScreenshotData(message);
3201
- if (!screenshotData) {
3202
- debugWarn('Received screenshot message without image data:', message);
3203
- break;
3204
- }
3205
- const resolver = pendingScreenshotResolversRef.current.get(message.id);
3206
- if (!resolver) {
3207
- debugWarn(`Received screenshot data for unknown or handled id: ${message.id}`);
3208
- break;
3209
- }
3210
- debugLog(`Received screenshot data for id ${message.id}`);
3211
- resolver(screenshotData);
3212
- pendingScreenshotResolversRef.current.delete(message.id);
3213
- pendingScreenshotRejectersRef.current.delete(message.id);
3214
- break;
3215
- }
3216
- case 'screenshotError':
3217
- if (typeof message.id !== 'string' || typeof message.message !== 'string') {
3218
- debugWarn('Received invalid screenshot error message:', message);
3219
- break;
3220
- }
3221
- const rejecter = pendingScreenshotRejectersRef.current.get(message.id);
3222
- if (!rejecter) {
3223
- debugWarn(`Received screenshot error for unknown or handled id: ${message.id}`);
3224
- break;
3225
- }
3226
- debugWarn(`Received screenshot error for id ${message.id}: ${message.message}`);
3227
- rejecter(new Error(message.message));
3228
- pendingScreenshotResolversRef.current.delete(message.id);
3229
- pendingScreenshotRejectersRef.current.delete(message.id);
3230
- break;
3231
- case 'cameraRequest': {
3232
- // Only camera-capable sessions allocate the sendonly
3233
- // outbound-camera transceiver; without it there is nothing to
3234
- // feed the device, so ignore camera requests entirely — no
3235
- // getUserMedia prompt and no onCameraDemandChange callbacks.
3236
- const sender = outboundCameraSenderRef.current;
3237
- if (!sender) {
3238
- break;
3239
- }
3240
- const request = parseCameraRequest(message);
3241
- if (!request) {
3242
- debugWarn('Received invalid cameraRequest:', message);
3243
- break;
3244
- }
3245
- // eslint-disable-next-line no-console
3246
- console.info('[RemoteControl] cameraRequest received:', request.active, request.facingMode);
3247
- await handleCameraRequest(request, ws, sender, isCurrentAttempt);
3248
- break;
3249
- }
3250
- case 'microphoneRequest': {
3251
- if (!outboundMicSenderRef.current) {
3252
- break;
3253
- }
3254
- const active = message.active === true;
3255
- microphoneDemandActiveRef.current = active;
3256
- // eslint-disable-next-line no-console
3257
- console.info('[RemoteControl] microphoneRequest received, active=', active);
3258
- if (!active) {
3259
- if (!microphoneEnabledRef.current) {
3260
- await detachMicrophone();
3261
- }
3262
- break;
3263
- }
3264
- if (outboundMicStreamRef.current) {
3265
- // A manual stream may already be attached. Re-select it on the
3266
- // host because file playback can have replaced its source.
3267
- sendMicrophoneStart(ws);
3268
- break;
3269
- }
3270
- await attachMicrophone();
3271
- break;
3272
- }
3273
- case 'microphoneStartResult': {
3274
- if (micStartPendingIdRef.current === null || message.id !== micStartPendingIdRef.current) {
3275
- break;
3276
- }
3277
- micStartPendingIdRef.current = null;
3278
- if (typeof message.error === 'string') {
3279
- // Host rejected the stream (e.g. no loopback device
3280
- // assigned on the node). Tear the capture back down so
3281
- // the browser's mic indicator doesn't stay lit for a
3282
- // stream nobody hears.
3283
- // eslint-disable-next-line no-console
3284
- console.warn('[RemoteControl] microphoneStart rejected by host:', message.error);
3285
- void detachMicrophone(message.error);
3286
- }
3287
- break;
3288
- }
3289
- case 'terminateAppResult':
3290
- if (typeof message.id !== 'string') {
3291
- debugWarn('Received invalid terminateApp result message:', message);
3292
- break;
3293
- }
3294
- if (typeof message.error === 'string') {
3295
- const terminateRejecter = pendingTerminateAppRejectersRef.current.get(message.id);
3296
- if (!terminateRejecter) {
3297
- debugWarn(`Received terminateApp error for unknown or handled id: ${message.id}`);
3298
- break;
3299
- }
3300
- debugWarn(`Received terminateApp error for id ${message.id}: ${message.error}`);
3301
- terminateRejecter(new Error(message.error));
3302
- pendingTerminateAppResolversRef.current.delete(message.id);
3303
- pendingTerminateAppRejectersRef.current.delete(message.id);
3304
- break;
3305
- }
3306
- const terminateResolver = pendingTerminateAppResolversRef.current.get(message.id);
3307
- if (!terminateResolver) {
3308
- debugWarn(`Received terminateApp result for unknown or handled id: ${message.id}`);
3309
- break;
3310
- }
3311
- debugLog(`Received terminateApp success for id ${message.id}`);
3312
- terminateResolver();
3313
- pendingTerminateAppResolversRef.current.delete(message.id);
3314
- pendingTerminateAppRejectersRef.current.delete(message.id);
3315
- break;
3316
- default:
3317
- debugWarn(`Received unhandled message type: ${message.type}`, message);
3318
- break;
3319
- }
3320
- };
3321
-
3322
- // Create and send offer
3323
- if (peerConnectionRef.current === peerConnection) {
3324
- const offer = await peerConnection.createOffer({
3325
- offerToReceiveVideo: true,
3326
- offerToReceiveAudio: false,
3327
- });
3328
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
3329
- return;
3330
- }
3331
- await peerConnection.setLocalDescription(offer);
3332
- if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
3333
- return;
3334
- }
3335
-
3336
- if (isCurrentAttempt() && wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
3337
- ws.send(
3338
- JSON.stringify({
3339
- type: 'offer',
3340
- ...(isDuo ? { displayStreams: true } : {}),
3341
- sdp: offer.sdp,
3342
- sessionId: sessionId,
3343
- }),
3344
- );
3345
- }
3346
- updateStatus('Sent offer');
3347
- }
3348
- } catch (e) {
3349
- if (!isCurrentAttempt()) {
3350
- return;
3351
- }
3352
- const reason = e instanceof Error ? e.message : String(e);
3353
- updateStatus('Error: ' + reason);
3354
- scheduleRetry(reason, generation);
3355
- }
3356
- };
3357
-
3358
- const start = () => {
3359
- // A fresh connection (mount, prop change, manual retry, reconnect())
3360
- // clears any prior terminated verdict so we re-evaluate from scratch.
3361
- terminatedRef.current = false;
3362
- setTerminated(false);
3363
- void startAttempt(0);
3364
- };
3365
-
3366
- const stop = () => {
3367
- connectionGenerationRef.current += 1;
3368
- connectionAttemptRef.current = 0;
3369
- controlChannelOpenedRef.current = false;
3370
- clearScheduledRetry();
3371
- clearIceDisconnectedGrace();
3372
- teardownConnection();
3373
- updateStatus('Stopped');
3374
- };
3375
-
3376
- const handleManualRetry = (event: React.MouseEvent<HTMLButtonElement>) => {
3377
- event.stopPropagation();
3378
- start();
3379
- };
3380
-
3381
- // Drive the manual override without interrupting guest-driven demand.
3382
- useEffect(() => {
3383
- if (isMicrophoneWanted()) {
3384
- void attachMicrophone();
3385
- } else {
3386
- void detachMicrophone();
3387
- }
3388
- // eslint-disable-next-line react-hooks/exhaustive-deps
3389
- }, [microphoneEnabled]);
3390
-
3391
- useEffect(() => {
3392
- // Reset video loaded state when connection params change
3393
- setVideoLoaded(false);
3394
-
3395
- // Start connection when component mounts
3396
- start();
3397
-
3398
- // Only start keepAlive if page is visible
3399
- if (!document.hidden) {
3400
- startKeepAlive();
3401
- }
3402
-
3403
- // Add visibility change listener
3404
- document.addEventListener('visibilitychange', handleVisibilityChange);
3405
-
3406
- // Clean up
3407
- return () => {
3408
- stopKeepAlive();
3409
- stop();
3410
- document.removeEventListener('visibilitychange', handleVisibilityChange);
3411
- };
3412
- // Camera attach/detach happens entirely inside the WS message
3413
- // loop now (sendonly transceiver + `replaceTrack`), so the
3414
- // connection effect doesn't need to bounce when the camera
3415
- // turns on/off — no SDP-affecting change.
3416
- // eslint-disable-next-line react-hooks/exhaustive-deps
3417
- }, [url, token, propSessionId, deviceModel]);
3418
-
3419
- // Recompute the inspect-overlay geometry (container-local pixel rect of
3420
- // the actually-rendered video content) from the current mapping context.
3421
- // The InfoCard places itself in viewport coordinates from pointer events
3422
- // directly, so no viewport-space origin is needed in the geometry.
3423
- const recomputeOverlayGeometry = () => {
3424
- const ctx = computeVideoMappingContext();
3425
- if (!ctx) {
3426
- setOverlayGeometry(null);
3427
- return;
3428
- }
3429
- const next: InspectOverlayGeometry = {
3430
- left: ctx.videoRect.left - ctx.containerRect.left + ctx.offsetX,
3431
- top: ctx.videoRect.top - ctx.containerRect.top + ctx.offsetY,
3432
- width: ctx.actualWidth,
3433
- height: ctx.actualHeight,
3434
- };
3435
- setOverlayGeometry((prev) =>
3436
- (
3437
- prev &&
3438
- prev.left === next.left &&
3439
- prev.top === next.top &&
3440
- prev.width === next.width &&
3441
- prev.height === next.height
3442
- ) ?
3443
- prev
3444
- : next,
3445
- );
3446
- };
3447
-
3448
- // Calculate video position and border-radius based on frame dimensions
3449
- useEffect(() => {
3450
- const video = videoRef.current;
3451
- const frame = frameRef.current;
3452
- const container = containerRef.current;
3453
-
3454
- if (!video) return;
3455
-
3456
- const updateVideoPosition = () => {
3457
- // If no frame, just refresh overlay geometry; no inset/letterbox math
3458
- // is needed since the video element is its own size.
3459
- if (!frameVisible || !frame) {
3460
- setVideoStyle({});
3461
- recomputeOverlayGeometry();
3462
- return;
3463
- }
3464
-
3465
- const frameWidth = frame.clientWidth;
3466
- const frameHeight = frame.clientHeight;
3467
-
3468
- if (frameWidth === 0 || frameHeight === 0) return;
3469
-
3470
- // Determine landscape based on video's intrinsic dimensions
3471
- const landscape = video.videoWidth > video.videoHeight;
3472
- setIsLandscape(landscape);
3473
- setUseAndroidTabletFrame(
3474
- platform === 'android' && isAndroidTabletVideo(video.videoWidth, video.videoHeight),
3475
- );
3476
-
3477
- const pos = landscape ? config.videoPosition.landscape : config.videoPosition.portrait;
3478
- let newStyle: React.CSSProperties = {};
3479
- if (pos.heightMultiplier) {
3480
- newStyle.height = `${frameHeight * pos.heightMultiplier}px`;
3481
- // Let the other dimension follow the video stream's intrinsic aspect ratio.
3482
- newStyle.width = 'auto';
3483
- } else if (pos.widthMultiplier) {
3484
- newStyle.width = `${frameWidth * pos.widthMultiplier}px`;
3485
- // Let the other dimension follow the video stream's intrinsic aspect ratio.
3486
- newStyle.height = 'auto';
3487
- }
3488
- newStyle.borderRadius = `${
3489
- landscape ?
3490
- frameHeight * config.videoBorderRadiusMultiplier
3491
- : frameWidth * config.videoBorderRadiusMultiplier
3492
- }px`;
3493
- setVideoStyle(newStyle);
3494
- recomputeOverlayGeometry();
3495
- };
3496
-
3497
- const resizeObserver = new ResizeObserver(() => {
3498
- updateVideoPosition();
3499
- });
3500
-
3501
- if (frame) resizeObserver.observe(frame);
3502
- resizeObserver.observe(video);
3503
- if (container) resizeObserver.observe(container);
3504
-
3505
- // Also update when the frame image loads
3506
- if (frame) frame.addEventListener('load', updateVideoPosition);
3507
-
3508
- // Update when video metadata loads (to get correct intrinsic dimensions)
3509
- video.addEventListener('loadedmetadata', updateVideoPosition);
3510
-
3511
- // IMPORTANT: When the WebRTC stream changes orientation, the intrinsic video size
3512
- // (videoWidth/videoHeight) can change without re-firing 'loadedmetadata'.
3513
- // The <video> element emits 'resize' in that case.
3514
- video.addEventListener('resize', updateVideoPosition);
3515
- // Orientation flips also mean every element's AX frame just changed
3516
- // (portrait↔landscape rotates the layout). Bump so the overlay
3517
- // refreshes immediately rather than waiting out the current poll
3518
- // cycle in a layout that no longer matches the boxes.
3519
- const bumpOnResize = () => axFetcherRef.current?.bumpActivity();
3520
- video.addEventListener('resize', bumpOnResize);
3521
-
3522
- // Initial calculation
3523
- updateVideoPosition();
3524
-
3525
- return () => {
3526
- resizeObserver.disconnect();
3527
- video.removeEventListener('loadedmetadata', updateVideoPosition);
3528
- video.removeEventListener('resize', updateVideoPosition);
3529
- video.removeEventListener('resize', bumpOnResize);
3530
- if (frame) frame.removeEventListener('load', updateVideoPosition);
3531
- };
3532
- }, [config, frameVisible]);
3533
-
3534
- // Start/stop the AX poller and reset inspect state when inspect mode
3535
- // toggles. Connection state is independent: the fetcher gets created on
3536
- // dataChannel.onopen and destroyed on teardown.
3537
- useEffect(() => {
3538
- const fetcher = axFetcherRef.current;
3539
- if (inspectActive) {
3540
- fetcher?.start();
3541
- } else {
3542
- fetcher?.stop();
3543
- setAxSnapshot(null);
3544
- setAxHighlightedId(null);
3545
- setAxSelectedId(null);
3546
- setAxCursorPosition(null);
3547
- setAxFrozenCursorPosition(null);
3548
- cursorPositionRef.current = null;
3549
- if (cursorRafIdRef.current !== undefined) {
3550
- window.cancelAnimationFrame(cursorRafIdRef.current);
3551
- cursorRafIdRef.current = undefined;
3552
- }
3553
- safeInvoke('onAxSnapshotChange', onAxSnapshotChangeRef.current, null);
3554
- }
3555
- }, [inspectActive]);
3556
-
3557
- // Cancel any pending cursor-rAF on unmount so we don't setState on a
3558
- // dead component.
3559
- useEffect(() => {
3560
- return () => {
3561
- if (cursorRafIdRef.current !== undefined) {
3562
- window.cancelAnimationFrame(cursorRafIdRef.current);
3563
- cursorRafIdRef.current = undefined;
3564
- }
3565
- };
3566
- }, []);
3567
-
3568
- // ESC clears overlay selection (Chrome DevTools behavior).
3569
- useEffect(() => {
3570
- if (!inspectActive) return;
3571
- const handleEsc = (e: KeyboardEvent) => {
3572
- if (e.key === 'Escape' && (axSelectedId || axHighlightedId)) {
3573
- setAxSelectedId(null);
3574
- setAxHighlightedId(null);
3575
- setAxFrozenCursorPosition(null);
3576
- safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, null);
3577
- }
3578
- };
3579
- window.addEventListener('keydown', handleEsc);
3580
- return () => window.removeEventListener('keydown', handleEsc);
3581
- }, [inspectActive, axSelectedId, axHighlightedId]);
3582
-
3583
- const handleVideoClick = () => {
3584
- if (videoRef.current) {
3585
- videoRef.current.focus();
3586
- }
3587
- };
3588
-
3589
- // Expose sendOpenUrlCommand via ref
3590
- useImperativeHandle(ref, () => ({
3591
- openUrl: (newUrl: string) => {
3592
- if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
3593
- debugWarn('WebSocket not open, cannot send open_url command via ref.');
3594
- return;
3595
- }
3596
- try {
3597
- const decodedUrl = decodeURIComponent(newUrl);
3598
- updateStatus('Opening URL');
3599
- wsRef.current.send(
3600
- JSON.stringify({
3601
- type: 'openUrl',
3602
- url: decodedUrl,
3603
- sessionId: sessionId,
3604
- }),
3605
- );
3606
- } catch (error) {
3607
- debugWarn('Error decoding or sending URL via ref:', { error, url: newUrl });
3608
- wsRef.current.send(
3609
- JSON.stringify({
3610
- type: 'openUrl',
3611
- url: newUrl,
3612
- sessionId: sessionId,
3613
- }),
3614
- );
3615
- }
3616
- axFetcherRef.current?.bumpActivity();
3617
- },
3618
-
3619
- sendKeyEvent: (event: ImperativeKeyboardEvent) => {
3620
- if (!dataChannelRef.current || dataChannelRef.current.readyState !== 'open') {
3621
- debugWarn('Data channel not ready for imperative key command:', dataChannelRef.current?.readyState);
3622
- return;
3623
- }
3624
-
3625
- const keycode = codeMap[event.code];
3626
- if (!keycode) {
3627
- debugWarn(`Unknown event.code for imperative command: ${event.code}`);
3628
- return;
3629
- }
3630
-
3631
- let metaState = ANDROID_KEYS.META_NONE;
3632
- if (event.shiftKey) metaState |= ANDROID_KEYS.META_SHIFT_ON;
3633
- if (event.altKey) metaState |= ANDROID_KEYS.META_ALT_ON;
3634
- if (event.ctrlKey) metaState |= ANDROID_KEYS.META_CTRL_ON;
3635
- if (event.metaKey) metaState |= ANDROID_KEYS.META_META_ON;
3636
-
3637
- const action = event.type === 'keydown' ? ANDROID_KEYS.ACTION_DOWN : ANDROID_KEYS.ACTION_UP;
3638
-
3639
- debugLog(
3640
- `Sending Imperative Key Command: code=${event.code}, keycode=${keycode}, action=${action}, meta=${metaState}`,
3641
- );
3642
-
3643
- const message = createInjectKeycodeMessage(
3644
- action,
3645
- keycode,
3646
- 0, // repeat count, typically 0 for single presses
3647
- metaState,
3648
- );
3649
- if (message) {
3650
- sendBinaryControlMessage(message);
3651
- }
3652
- },
3653
- screenshot: (display?: 'outer' | 'inner'): Promise<ScreenshotData> => {
3654
- return new Promise<ScreenshotData>((resolve, reject) => {
3655
- if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
3656
- debugWarn('WebSocket not open, cannot send screenshot command.');
3657
- return reject(new Error('WebSocket is not connected or connection is not open.'));
3658
- }
3659
-
3660
- const id = `ui-ss-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
3661
- const request = {
3662
- type: display ? 'screenshotDisplay' : 'screenshot',
3663
- ...(display ? { display } : {}),
3664
- id: id,
3665
- };
3666
-
3667
- pendingScreenshotResolversRef.current.set(id, resolve);
3668
- pendingScreenshotRejectersRef.current.set(id, reject);
3669
-
3670
- debugLog('Sending screenshot request:', request);
3671
- try {
3672
- wsRef.current.send(JSON.stringify(request));
3673
- } catch (err) {
3674
- debugWarn('Failed to send screenshot request immediately:', err);
3675
- pendingScreenshotResolversRef.current.delete(id);
3676
- pendingScreenshotRejectersRef.current.delete(id);
3677
- reject(err);
3678
- return; // Important to return here if send failed synchronously
3679
- }
3680
-
3681
- setTimeout(() => {
3682
- if (pendingScreenshotResolversRef.current.has(id)) {
3683
- debugWarn(`Screenshot request timed out for id ${id}`);
3684
- pendingScreenshotRejectersRef.current.get(id)?.(new Error('Screenshot request timed out'));
3685
- pendingScreenshotResolversRef.current.delete(id);
3686
- pendingScreenshotRejectersRef.current.delete(id);
3687
- }
3688
- }, 30000); // 30-second timeout
3689
- });
3690
- },
3691
- terminateApp: (bundleId: string): Promise<void> => {
3692
- return new Promise<void>((resolve, reject) => {
3693
- if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
3694
- debugWarn('WebSocket not open, cannot send terminateApp command.');
3695
- return reject(new Error('WebSocket is not connected or connection is not open.'));
3696
- }
3697
- const id = `ui-term-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
3698
- const request = {
3699
- type: 'terminateApp',
3700
- id,
3701
- bundleId,
3702
- };
3703
-
3704
- pendingTerminateAppResolversRef.current.set(id, resolve);
3705
- pendingTerminateAppRejectersRef.current.set(id, reject);
3706
-
3707
- debugLog('Sending terminateApp request:', request);
3708
- try {
3709
- wsRef.current.send(JSON.stringify(request));
3710
- } catch (err) {
3711
- debugWarn('Failed to send terminateApp request immediately:', err);
3712
- pendingTerminateAppResolversRef.current.delete(id);
3713
- pendingTerminateAppRejectersRef.current.delete(id);
3714
- reject(err);
3715
- return;
3716
- }
3717
- // Terminating the foreground app drops the user back to the home
3718
- // screen — bump so the overlay reflects the post-terminate state
3719
- // through the SpringBoard transition.
3720
- axFetcherRef.current?.bumpActivity();
3721
-
3722
- setTimeout(() => {
3723
- if (pendingTerminateAppResolversRef.current.has(id)) {
3724
- debugWarn(`terminateApp request timed out for id ${id}`);
3725
- pendingTerminateAppRejectersRef.current.get(id)?.(new Error('terminateApp request timed out'));
3726
- pendingTerminateAppResolversRef.current.delete(id);
3727
- pendingTerminateAppRejectersRef.current.delete(id);
3728
- }
3729
- }, 30000);
3730
- });
3731
- },
3732
- reconnect: () => start(),
3733
-
3734
- refreshAxTree: async (): Promise<AxSnapshot> => {
3735
- const fetcher = axFetcherRef.current;
3736
- if (!fetcher) {
3737
- throw new Error('Inspect mode is not active');
3738
- }
3739
- // The fetcher's refresh() runs the result through the same
3740
- // change-detect path as the poll loop (via deliver()), which calls
3741
- // back into onSnapshot — already wired to setAxSnapshot +
3742
- // onAxSnapshotChange (with safe-invoke). We just return the fetched
3743
- // payload for callers that want it.
3744
- return fetcher.refresh();
3745
- },
3746
-
3747
- getAxSnapshot: () => axSnapshotRef.current,
3748
-
3749
- setInspectHighlight: (element: AxElement | null) => {
3750
- setAxHighlightedId(element?.id ?? null);
3751
- },
3752
-
3753
- setInspectSelection: (element: AxElement | null) => {
3754
- setAxSelectedId(element?.id ?? null);
3755
- // Programmatic selection has no click position — anchor the card at
3756
- // the last known cursor position (if any), otherwise clear.
3757
- // Customer-facing UIs that drive selection from their own panels can
3758
- // call setInspectHighlight separately to move the cursor visual.
3759
- if (element) {
3760
- setAxFrozenCursorPosition(cursorPositionRef.current);
3761
- } else {
3762
- setAxFrozenCursorPosition(null);
3763
- }
3764
- const snapshot = axSnapshotRef.current;
3765
- if (element && snapshot) {
3766
- safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, { element, snapshot });
3767
- } else {
3768
- safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, null);
3769
- }
3770
- },
3771
-
3772
- getAxStatus: () => axFetcherRef.current?.getStatus() ?? 'idle',
3773
- }));
3774
-
3775
- // Show indicators when Alt is held and we have a valid hover point (null when outside)
3776
- const showAltIndicators = isAltHeld && hoverPoint !== null;
3777
-
3778
- return (
3779
- <div
3780
- ref={containerRef}
3781
- className={clsx('rc-container', className)}
3782
- style={{ touchAction: 'none' }} // Keep touchAction none for the container
3783
- // Attach unified handler to all interaction events on the container
3784
- // This helps capture mouseleave correctly even if the video element itself isn't hovered
3785
- onMouseDown={handleInteraction}
3786
- onMouseMove={handleInteraction}
3787
- onMouseUp={handleInteraction}
3788
- onMouseLeave={handleInteraction} // Handle mouse leaving the container
3789
- onTouchStart={handleInteraction}
3790
- onTouchMove={handleInteraction}
3791
- onTouchEnd={handleInteraction}
3792
- onTouchCancel={handleInteraction}
3793
- >
3794
- {showAltIndicators && hoverPoint && (
3795
- <>
3796
- <div
3797
- className="rc-touch-indicator"
3798
- style={{
3799
- left: `${hoverPoint.containerX}px`,
3800
- top: `${hoverPoint.containerY}px`,
3801
- }}
3802
- />
3803
- <div
3804
- className="rc-touch-indicator"
3805
- style={{
3806
- left: `${hoverPoint.mirrorContainerX}px`,
3807
- top: `${hoverPoint.mirrorContainerY}px`,
3808
- }}
3809
- />
3810
- </>
3811
- )}
3812
- {frameVisible && (
3813
- <img
3814
- ref={frameRef}
3815
- src={frameImageSrc}
3816
- alt=""
3817
- className={platform === 'ios' ? clsx('rc-phone-frame', 'rc-phone-frame-ios') : 'rc-phone-frame'}
3818
- draggable={false}
3819
- />
3820
- )}
3821
- <video
3822
- ref={videoRef}
3823
- className={clsx(
3824
- 'rc-video',
3825
- !frameVisible && 'rc-video-frameless',
3826
- !videoLoaded && 'rc-video-loading',
3827
- )}
3828
- style={{
3829
- ...videoStyle,
3830
- ...(isDuo ? { opacity: 0, width: 1, height: 1, pointerEvents: 'none' as const } : {}),
3831
- ...(loadingLogo ?
3832
- {
3833
- backgroundImage: `url("${loadingLogo}")`,
3834
- backgroundRepeat: 'no-repeat',
3835
- backgroundPosition: 'center',
3836
- backgroundSize: config.loadingLogoSize,
3837
- }
3838
- : {}),
3839
- }}
3840
- autoPlay
3841
- playsInline
3842
- muted
3843
- tabIndex={isDuo ? -1 : 0}
3844
- onKeyDown={handleKeyboard}
3845
- onKeyUp={handleKeyboard}
3846
- onClick={handleVideoClick}
3847
- onLoadedData={markFirstFrameShown}
3848
- onFocus={() => {
3849
- if (videoRef.current) {
3850
- videoRef.current.style.outline = 'none';
3851
- }
3852
- }}
3853
- onBlur={() => {
3854
- if (videoRef.current) {
3855
- videoRef.current.style.outline = 'none';
3856
- }
3857
- }}
3858
- />
3859
- {isDuo && (
3860
- <video
3861
- ref={innerVideoRef}
3862
- autoPlay
3863
- playsInline
3864
- muted
3865
- style={{ position: 'absolute', width: 1, height: 1, opacity: 0, pointerEvents: 'none' }}
3866
- />
3867
- )}
3868
- {isDuo && duoState && (
3869
- <Suspense fallback={<div>Loading iPhone Duo…</div>}>
3870
- <DuoView
3871
- modelUrl={duoModelUrl}
3872
- outer={videoRef.current}
3873
- inner={innerVideoRef.current}
3874
- state={duoState}
3875
- setAngle={(angleDegrees) => sendDuoControl('setHingeAngle', { angleDegrees })}
3876
- setOrientation={(orientation: DuoOrientation) =>
3877
- sendDuoControl('setDuoOrientation', { orientation })
3878
- }
3879
- button={(button, down) =>
3880
- sendDuoControl('performActions', {
3881
- actions: [{ type: down ? 'buttonDown' : 'buttonUp', button }],
3882
- })
3883
- }
3884
- touch={(action, screenId, x, y) => {
3885
- const channel = dataChannelRef.current;
3886
- if (channel?.readyState === 'open')
3887
- channel.send(createDisplayTouchMessage(action, screenId, x, y));
3888
- }}
3889
- onKeyDown={handleKeyboard}
3890
- onKeyUp={handleKeyboard}
3891
- />
3892
- </Suspense>
3893
- )}
3894
- {isDuo && duoError && (
3895
- <div role="alert" className="rc-duo-error">
3896
- {duoError}
3897
- </div>
3898
- )}
3899
- {!isDuo && inspectActive && (
3900
- <InspectOverlay
3901
- snapshot={axSnapshot}
3902
- geometry={overlayGeometry}
3903
- highlightedId={axHighlightedId}
3904
- selectedId={axSelectedId}
3905
- mode={inspectModeResolved}
3906
- cursorPosition={axCursorPosition}
3907
- frozenCursorPosition={axFrozenCursorPosition}
3908
- onSelectChange={(element, clickPosition) => {
3909
- setAxSelectedId(element?.id ?? null);
3910
- if (element && clickPosition) {
3911
- setAxFrozenCursorPosition(clickPosition);
3912
- } else if (!element) {
3913
- setAxFrozenCursorPosition(null);
3914
- }
3915
- const snapshot = axSnapshotRef.current;
3916
- if (element && snapshot) {
3917
- safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, {
3918
- element,
3919
- snapshot,
3920
- });
3921
- } else {
3922
- safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, null);
3923
- }
3924
- }}
3925
- onTapElement={(element, tapAt) => {
3926
- // Use the viewport-space position the user originally aimed at
3927
- // (the frozen click position). For containers whose children
3928
- // are absent from the accessibility tree — e.g. iOS UITabBar's
3929
- // home/diagnostics/settings buttons — this taps the specific
3930
- // button the user pointed at instead of the container's
3931
- // averaged center.
3932
- if (tapAt) {
3933
- sendTapAtClient(tapAt.x, tapAt.y);
3934
- return;
3935
- }
3936
- // Fallback (defensive): center of element. Should be
3937
- // unreachable from the InfoCard since it always passes anchor.
3938
- const snapshot = axSnapshotRef.current;
3939
- if (!snapshot) return;
3940
- sendTapAtElementCenter(element, snapshot);
3941
- }}
3942
- />
3943
- )}
3944
- {terminated ?
3945
- <div className="rc-terminated" role="status">
3946
- <div className="rc-terminated-title">Instance terminated</div>
3947
- <div className="rc-terminated-message">
3948
- This instance no longer exists and can&apos;t be reconnected.
3949
- </div>
3950
- </div>
3951
- : retryExhausted && (
3952
- <button type="button" className="rc-retry-button" onClick={handleManualRetry}>
3953
- Retry
3954
- </button>
3955
- )
3956
- }
3957
- </div>
3958
- );
3959
- },
3960
- );
3961
-
3962
- const getScreenshotError = (message: any): string | null => {
3963
- if (typeof message.message === 'string') {
3964
- return message.message;
3965
- }
3966
-
3967
- if (typeof message.error === 'string') {
3968
- return message.error;
3969
- }
3970
-
3971
- return null;
3972
- };
3973
-
3974
- const toScreenshotData = (message: any): ScreenshotData | null => {
3975
- if (typeof message.dataUri === 'string') {
3976
- return { dataUri: message.dataUri };
3977
- }
3978
-
3979
- if (typeof message.base64 === 'string') {
3980
- if (message.base64.startsWith('data:')) {
3981
- return { dataUri: message.base64 };
3982
- }
3983
-
3984
- const mimeType = message.base64.startsWith('/9j/') ? 'image/jpeg' : 'image/png';
3985
- return { dataUri: `data:${mimeType};base64,${message.base64}` };
3986
- }
3987
-
3988
- return null;
3989
- };