@limrun/ui 0.16.0 → 0.17.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -0
- package/dist/GLTFLoader-Cami_vza.mjs +1679 -0
- package/dist/GLTFLoader-DeCLxW_q.js +1 -0
- package/dist/components/duo-button-hinge.d.ts +16 -0
- package/dist/components/duo-controls.d.ts +21 -0
- package/dist/components/duo-flat-frame.d.ts +7 -0
- package/dist/components/duo-flat-geometry.d.ts +52 -0
- package/dist/components/duo-fold-motion.d.ts +27 -0
- package/dist/components/duo-fold-renderer.d.ts +19 -0
- package/dist/components/duo-fold-timing.d.ts +1 -0
- package/dist/components/duo-frame.d.ts +16 -0
- package/dist/components/duo-hardware.d.ts +20 -0
- package/dist/components/duo-loading.d.ts +8 -0
- package/dist/components/duo-model.d.ts +19 -0
- package/dist/components/duo-screen-handoff.d.ts +4 -0
- package/dist/components/duo-view.d.ts +9 -0
- package/dist/components/remote-control.d.ts +5 -1
- package/dist/core/duo.d.ts +63 -0
- package/dist/duo-fold-renderer-BXejNHcY.mjs +257 -0
- package/dist/duo-fold-renderer-DdltAnF7.js +53 -0
- package/dist/duo-frame-Cmw4p2uN.mjs +26767 -0
- package/dist/duo-frame-iMtrvx_a.js +4059 -0
- package/dist/duo-view-BAh-w2Kz.js +1 -0
- package/dist/duo-view-D9Ae00rj.mjs +512 -0
- package/dist/index.cjs +1 -1
- package/dist/index.css +1 -1
- package/dist/index.js +2 -2
- package/dist/lite.cjs +1 -1
- package/dist/lite.js +1 -1
- package/dist/remote-control-B2jZpf3c.js +1 -0
- package/dist/remote-control-DiddxS8l.mjs +2883 -0
- package/package.json +8 -2
- package/dist/demo.d.ts +0 -1
- package/dist/remote-control-BRgbD7yo.mjs +0 -2516
- package/dist/remote-control-VISrP1zI.js +0 -1
- package/index.html +0 -199
- package/src/assets/Apple_logo_white.svg +0 -4
- package/src/assets/android_boot.gif +0 -0
- package/src/assets/android_boot.webp +0 -0
- package/src/assets/iphone16pro_black.webp +0 -0
- package/src/assets/iphone16pro_black_bg.webp +0 -0
- package/src/assets/iphone16pro_black_landscape.webp +0 -0
- package/src/assets/iphone16pro_black_landscape_bg.webp +0 -0
- package/src/assets/pixel9_black.webp +0 -0
- package/src/assets/pixel9_black_landscape.webp +0 -0
- package/src/assets/pixel_tablet_landscape.webp +0 -0
- package/src/assets/pixel_tablet_portrait.webp +0 -0
- package/src/components/device-install/index.ts +0 -1
- package/src/components/inspect-overlay.css +0 -223
- package/src/components/inspect-overlay.tsx +0 -437
- package/src/components/remote-control-framed.tsx +0 -14
- package/src/components/remote-control.css +0 -131
- package/src/components/remote-control.test.ts +0 -34
- package/src/components/remote-control.tsx +0 -3853
- package/src/core/ax-fetcher.test.ts +0 -418
- package/src/core/ax-fetcher.ts +0 -387
- package/src/core/ax-tree.test.ts +0 -491
- package/src/core/ax-tree.ts +0 -416
- package/src/core/camera.test.ts +0 -155
- package/src/core/camera.ts +0 -133
- package/src/core/constants.ts +0 -344
- package/src/core/webrtc-messages.ts +0 -148
- package/src/demo.tsx +0 -269
- package/src/frame-assets.ts +0 -22
- package/src/hooks/index.ts +0 -1
- package/src/index.ts +0 -29
- package/src/lite.test.ts +0 -30
- package/src/lite.ts +0 -30
- package/src/vite-env.d.ts +0 -1
- package/tsconfig.json +0 -26
- package/tsconfig.node.json +0 -26
- package/vite.config.ts +0 -35
- package/vitest.config.ts +0 -24
|
@@ -1,3853 +0,0 @@
|
|
|
1
|
-
import React, { useEffect, useRef, useMemo, useState, forwardRef, useImperativeHandle } from 'react';
|
|
2
|
-
import { clsx } from 'clsx';
|
|
3
|
-
import './remote-control.css';
|
|
4
|
-
|
|
5
|
-
import { ANDROID_KEYS, AMOTION_EVENT, codeMap } from '../core/constants';
|
|
6
|
-
|
|
7
|
-
import {
|
|
8
|
-
createTouchControlMessage,
|
|
9
|
-
createInjectKeycodeMessage,
|
|
10
|
-
createSetClipboardMessage,
|
|
11
|
-
createTwoFingerTouchControlMessage,
|
|
12
|
-
} from '../core/webrtc-messages';
|
|
13
|
-
import { AxFetcher, AxStatus, type AndroidElementTreeOptions } from '../core/ax-fetcher';
|
|
14
|
-
import { AxElement, AxSnapshot, axElementAtPoint, axSnapshotsEqual } from '../core/ax-tree';
|
|
15
|
-
import {
|
|
16
|
-
CameraOperationCoordinator,
|
|
17
|
-
cameraCaptureConstraints,
|
|
18
|
-
createGrantedCameraResult,
|
|
19
|
-
parseCameraRequest,
|
|
20
|
-
shouldReacquireCamera,
|
|
21
|
-
type CameraCaptureInfo,
|
|
22
|
-
type CameraRequest,
|
|
23
|
-
} from '../core/camera';
|
|
24
|
-
import { InspectOverlay, InspectOverlayGeometry, InspectMode } from './inspect-overlay';
|
|
25
|
-
|
|
26
|
-
export type { CameraCaptureInfo, CameraFacingMode, CameraRequest, CameraResult } from '../core/camera';
|
|
27
|
-
|
|
28
|
-
declare global {
|
|
29
|
-
interface Window {
|
|
30
|
-
debugRemoteControl?: boolean;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export interface RemoteControlProps {
|
|
35
|
-
// url is the URL of the instance to connect to.
|
|
36
|
-
url: string;
|
|
37
|
-
|
|
38
|
-
// token is used to authenticate the connection to the instance.
|
|
39
|
-
token: string;
|
|
40
|
-
|
|
41
|
-
// className is the class name to apply to the component
|
|
42
|
-
// on top of the default styles.
|
|
43
|
-
className?: string;
|
|
44
|
-
|
|
45
|
-
// sessionId is a unique identifier for the WebRTC session
|
|
46
|
-
// with the source to prevent conflicts between other
|
|
47
|
-
// users connected to the same source.
|
|
48
|
-
// If empty, the component will generate a random one.
|
|
49
|
-
sessionId?: string;
|
|
50
|
-
|
|
51
|
-
// openUrl is the URL to open in the instance when the
|
|
52
|
-
// component is ready.
|
|
53
|
-
//
|
|
54
|
-
// If not provided, the component will not open any URL.
|
|
55
|
-
openUrl?: string;
|
|
56
|
-
|
|
57
|
-
// showFrame controls whether to display the device frame
|
|
58
|
-
// around the video. Defaults to true.
|
|
59
|
-
showFrame?: boolean;
|
|
60
|
-
|
|
61
|
-
// When true, drops after a working session auto-reconnect instead of
|
|
62
|
-
// surfacing the manual "Retry" button. Defaults to false.
|
|
63
|
-
autoReconnect?: boolean;
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Fires once when the component concludes the instance is permanently
|
|
67
|
-
* gone rather than suffering a recoverable connection blip.
|
|
68
|
-
*
|
|
69
|
-
* After a drop, the component probes the instance endpoint over HTTP;
|
|
70
|
-
* a 404 means the instance no longer exists (terminated/deleted). When
|
|
71
|
-
* that happens the component stops reconnecting and renders a built-in
|
|
72
|
-
* "terminated" message in place of the "Retry" button, and invokes this
|
|
73
|
-
* callback so the host can react (navigate away, surface its own UI,
|
|
74
|
-
* etc.). It is not called for ordinary network drops.
|
|
75
|
-
*/
|
|
76
|
-
onTerminated?: () => void;
|
|
77
|
-
/**
|
|
78
|
-
* Device frame and boot-logo images. The default export bundles Limrun's
|
|
79
|
-
* frames; `@limrun/ui/lite` omits them so a frameless embed stays small.
|
|
80
|
-
*/
|
|
81
|
-
assets?: RemoteControlAssets;
|
|
82
|
-
/**
|
|
83
|
-
* Coarse connection lifecycle, for status UI outside the component.
|
|
84
|
-
*/
|
|
85
|
-
onConnectionStateChange?: (state: RemoteControlConnectionState) => void;
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* Enable the inspect overlay. When set, the component starts polling the
|
|
89
|
-
* accessibility tree and draws boxes over each element on top of the
|
|
90
|
-
* video stream.
|
|
91
|
-
*
|
|
92
|
-
* - `true` — Select mode. Boxes are clickable, click pins a selection
|
|
93
|
-
* with action buttons (Tap / Copy selector / Copy id), ESC clears.
|
|
94
|
-
* Device input is blocked while in this mode.
|
|
95
|
-
* - `'hover-only'` — Boxes follow the cursor as a visual preview. Device
|
|
96
|
-
* input still passes through, so you can drive the simulator while
|
|
97
|
-
* inspecting.
|
|
98
|
-
* - `undefined` / `false` (default) — overlay disabled, no polling.
|
|
99
|
-
*/
|
|
100
|
-
inspectMode?: boolean | 'hover-only';
|
|
101
|
-
|
|
102
|
-
/**
|
|
103
|
-
* Fires whenever a fresh accessibility snapshot is delivered.
|
|
104
|
-
*
|
|
105
|
-
* Customers use this to drive their own side panels, agent prompts,
|
|
106
|
-
* analytics, etc. The built-in overlay does not require this callback —
|
|
107
|
-
* it renders from internal state regardless.
|
|
108
|
-
*
|
|
109
|
-
* Identical-to-previous snapshots (per `axSnapshotsEqual`) are NOT
|
|
110
|
-
* re-emitted, so a stable UI doesn't generate callback noise.
|
|
111
|
-
*
|
|
112
|
-
* Invoked in a microtask so customer code doesn't run synchronously
|
|
113
|
-
* inside React's commit phase.
|
|
114
|
-
*/
|
|
115
|
-
onAxSnapshotChange?: (snapshot: AxSnapshot | null) => void;
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* Fires when the user clicks an overlay element (only emitted when
|
|
119
|
-
* `inspectMode === true`). `null` indicates a deselection (ESC, click
|
|
120
|
-
* outside any box, or programmatic clear).
|
|
121
|
-
*
|
|
122
|
-
* The `snapshot` field is the snapshot active at the moment of the
|
|
123
|
-
* click — useful for capturing context without races against the next
|
|
124
|
-
* poll cycle.
|
|
125
|
-
*/
|
|
126
|
-
onInspectSelectionChange?: (selection: { element: AxElement; snapshot: AxSnapshot } | null) => void;
|
|
127
|
-
|
|
128
|
-
/**
|
|
129
|
-
* Fires whenever the accessibility subsystem changes coarse-grained
|
|
130
|
-
* status. Useful for rendering readiness indicators or error banners in
|
|
131
|
-
* a customer-built side panel.
|
|
132
|
-
*
|
|
133
|
-
* Transitions are deduplicated; no self-loops are emitted. The `error`
|
|
134
|
-
* argument is populated when status is `error` or `unavailable`.
|
|
135
|
-
*
|
|
136
|
-
* Lifecycle: `idle` → `starting` → `ready` (or `unavailable` / `error`).
|
|
137
|
-
* Recovery from `error` / `unavailable` is automatic — the fetcher
|
|
138
|
-
* keeps polling and transitions back to `ready` on the next success.
|
|
139
|
-
*/
|
|
140
|
-
onAxStatusChange?: (status: AxStatus, error?: string) => void;
|
|
141
|
-
|
|
142
|
-
/**
|
|
143
|
-
* Base interval (ms) between successful AX-tree fetches.
|
|
144
|
-
*
|
|
145
|
-
* The fetcher will:
|
|
146
|
-
* - Wait `axPollIntervalMs` after a successful fetch with NEW data.
|
|
147
|
-
* - Double the wait (up to `axMaxBackoffMs`) when consecutive snapshots
|
|
148
|
-
* are byte-identical (e.g. static screen).
|
|
149
|
-
* - Wait 5 s when the server reports AX is unavailable.
|
|
150
|
-
*
|
|
151
|
-
* In addition, after user input (taps, scrolls, keypresses, openUrl,
|
|
152
|
-
* terminateApp, orientation flips), the fetcher enters a short
|
|
153
|
-
* "activity boost" window (~1.2 s) during which fetches happen at
|
|
154
|
-
* ~250 ms regardless of this setting. This captures mid-animation UI
|
|
155
|
-
* changes without you having to manually call `refreshAxTree`.
|
|
156
|
-
*
|
|
157
|
-
* @default 500
|
|
158
|
-
*/
|
|
159
|
-
axPollIntervalMs?: number;
|
|
160
|
-
|
|
161
|
-
/**
|
|
162
|
-
* Maximum backoff (ms) for the AX-tree polling loop when consecutive
|
|
163
|
-
* snapshots are unchanged.
|
|
164
|
-
*
|
|
165
|
-
* @default 2000
|
|
166
|
-
*/
|
|
167
|
-
axMaxBackoffMs?: number;
|
|
168
|
-
|
|
169
|
-
androidElementTreeOptions?: AndroidElementTreeOptions;
|
|
170
|
-
|
|
171
|
-
/**
|
|
172
|
-
* Fires whenever the iOS simulator's camera demand state changes —
|
|
173
|
-
* i.e. an app inside the sim called
|
|
174
|
-
* `[AVCaptureSession startRunning]` or `[stopRunning]`. The
|
|
175
|
-
* component handles the `navigator.mediaDevices.getUserMedia` prompt
|
|
176
|
-
* and SDP plumbing internally; this callback is purely so the host
|
|
177
|
-
* UI can render a status indicator ("simulator is using your
|
|
178
|
-
* camera", etc.).
|
|
179
|
-
*
|
|
180
|
-
* `active` reflects whether the sim is currently asking for frames.
|
|
181
|
-
* `granted` is set on capture acknowledgements: `true` after a new
|
|
182
|
-
* track, reused track, or constraint update; `false` when capture
|
|
183
|
-
* fails. The host switches to black frames after a failure.
|
|
184
|
-
*
|
|
185
|
-
* `camera` (optional, only meaningful when `granted === true`)
|
|
186
|
-
* carries what `MediaStreamTrack.getSettings()` reported for the
|
|
187
|
-
* active capture: resolution, framerate, device label, facing
|
|
188
|
-
* mode. Use it to render a richer status indicator (e.g.
|
|
189
|
-
* "Camera · 1920×1080 · 30 fps · FaceTime HD").
|
|
190
|
-
*
|
|
191
|
-
* iOS instances always fire this callback. Android instances fire it
|
|
192
|
-
* when the server supports camera injection (Android 15+); older
|
|
193
|
-
* Android instances have no camera-injector path and stay silent.
|
|
194
|
-
*/
|
|
195
|
-
onCameraDemandChange?: (active: boolean, granted?: boolean, camera?: CameraCaptureInfo) => void;
|
|
196
|
-
|
|
197
|
-
/**
|
|
198
|
-
* Periodically (~1Hz) fires with a snapshot of the outbound
|
|
199
|
-
* camera stream's live WebRTC stats — codec, encoder
|
|
200
|
-
* implementation, hardware acceleration, encoded fps, bitrate,
|
|
201
|
-
* round-trip-time, and the encoder's
|
|
202
|
-
* `qualityLimitationReason` (which is what Meet/Zoom use to
|
|
203
|
-
* decide whether to show "Bandwidth limited" or "CPU limited"
|
|
204
|
-
* banners).
|
|
205
|
-
*
|
|
206
|
-
* Only fires while the simulator is actively pulling the
|
|
207
|
-
* camera AND `getUserMedia` was granted. `null` is emitted
|
|
208
|
-
* once when the stream goes back to idle so consumers can
|
|
209
|
-
* clear their UI without having to maintain their own
|
|
210
|
-
* timeout. The host app does not need to poll `getStats()`
|
|
211
|
-
* itself — this is the canonical place.
|
|
212
|
-
*/
|
|
213
|
-
onCameraStats?: (stats: CameraStreamStats | null) => void;
|
|
214
|
-
|
|
215
|
-
/**
|
|
216
|
-
* When true, capture the user's microphone via
|
|
217
|
-
* `getUserMedia({ audio: true })` and stream it live into the
|
|
218
|
-
* device's mock microphone. This is a manual override: apps
|
|
219
|
-
* that actively consume microphone input also trigger capture
|
|
220
|
-
* automatically, like camera demand. The component handles the
|
|
221
|
-
* browser permission prompt, WebRTC track attachment, and
|
|
222
|
-
* `microphoneStart`/`microphoneStop` signaling. Progress and failures
|
|
223
|
-
* are reported via `onMicrophoneStateChange`. Ignored on Android
|
|
224
|
-
* servers without camera/mic injection support (Android 14 and older).
|
|
225
|
-
*/
|
|
226
|
-
microphoneEnabled?: boolean;
|
|
227
|
-
|
|
228
|
-
/**
|
|
229
|
-
* Fires whenever the live-microphone pipeline changes state:
|
|
230
|
-
* after a manual or guest-demanded attempt resolves, when the host
|
|
231
|
-
* acknowledges or rejects the stream, when the browser revokes the
|
|
232
|
-
* track mid-stream, and when capture stops.
|
|
233
|
-
*/
|
|
234
|
-
onMicrophoneStateChange?: (state: MicrophoneState) => void;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
/**
|
|
238
|
-
* Snapshot of the live-microphone pipeline, reported through
|
|
239
|
-
* `onMicrophoneStateChange`.
|
|
240
|
-
*/
|
|
241
|
-
export interface MicrophoneState {
|
|
242
|
-
/** True while the browser mic is captured and streaming to the sim. */
|
|
243
|
-
active: boolean;
|
|
244
|
-
/**
|
|
245
|
-
* Human-readable failure from the last attempt, if any (permission
|
|
246
|
-
* denied, no device, host rejection, ...).
|
|
247
|
-
*/
|
|
248
|
-
error?: string;
|
|
249
|
-
/** Device label of the captured microphone, when available. */
|
|
250
|
-
label?: string;
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
/**
|
|
254
|
-
* Live outbound-camera quality snapshot. Derived from
|
|
255
|
-
* `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
|
|
256
|
-
* once per second while the camera is sending. All fields optional:
|
|
257
|
-
* some browsers omit fields (Safari rarely reports
|
|
258
|
-
* `encoderImplementation`), and the first sample after camera start
|
|
259
|
-
* has no delta-derived numbers yet.
|
|
260
|
-
*/
|
|
261
|
-
export interface CameraStreamStats {
|
|
262
|
-
/** "H264", "HEVC"/"H265", "VP9", "VP8", "AV1", etc. (uppercased). */
|
|
263
|
-
codec?: string;
|
|
264
|
-
/** e.g. "VideoToolbox" (hw), "OpenH264" (sw), "ExternalEncoder". */
|
|
265
|
-
encoderImplementation?: string;
|
|
266
|
-
/**
|
|
267
|
-
* Browser-reported hardware-acceleration hint. Some Chromium
|
|
268
|
-
* versions expose this via `powerEfficientEncoder`; we mirror it
|
|
269
|
-
* here so consumers don't have to know the spec quirk.
|
|
270
|
-
*/
|
|
271
|
-
hardwareAccelerated?: boolean;
|
|
272
|
-
/** Outbound encoded fps over the last sample window. */
|
|
273
|
-
framesPerSecond?: number;
|
|
274
|
-
/** Cumulative encoded frame count. */
|
|
275
|
-
framesEncoded?: number;
|
|
276
|
-
/** Outbound encoded bitrate (bits/s) over the last window. */
|
|
277
|
-
bitrateBps?: number;
|
|
278
|
-
/** Width/height the encoder is currently producing. */
|
|
279
|
-
width?: number;
|
|
280
|
-
height?: number;
|
|
281
|
-
/**
|
|
282
|
-
* One of `'none' | 'cpu' | 'bandwidth' | 'other'`. Mirrors
|
|
283
|
-
* Meet/Zoom's "limited by …" banners. Anything other than
|
|
284
|
-
* `'none'` means the encoder dropped resolution to keep up.
|
|
285
|
-
*/
|
|
286
|
-
qualityLimitationReason?: string;
|
|
287
|
-
/** Round-trip time in milliseconds, from the matching RTCP. */
|
|
288
|
-
rttMs?: number;
|
|
289
|
-
/** Packet-loss percentage over the last sample window (0..100). */
|
|
290
|
-
packetsLostPct?: number;
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
interface ScreenshotData {
|
|
294
|
-
dataUri: string;
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
export interface ImperativeKeyboardEvent {
|
|
298
|
-
type: 'keydown' | 'keyup';
|
|
299
|
-
code: string; // e.g., "KeyA", "Enter", "ShiftLeft"
|
|
300
|
-
shiftKey?: boolean;
|
|
301
|
-
altKey?: boolean;
|
|
302
|
-
ctrlKey?: boolean;
|
|
303
|
-
metaKey?: boolean;
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
export interface RemoteControlHandle {
|
|
307
|
-
openUrl: (url: string) => void;
|
|
308
|
-
sendKeyEvent: (event: ImperativeKeyboardEvent) => void;
|
|
309
|
-
screenshot: () => Promise<ScreenshotData>;
|
|
310
|
-
terminateApp: (bundleId: string) => Promise<void>;
|
|
311
|
-
reconnect: () => void;
|
|
312
|
-
|
|
313
|
-
// Inspect-mode helpers. These are no-ops when inspect mode is disabled or
|
|
314
|
-
// the WebSocket isn't open.
|
|
315
|
-
|
|
316
|
-
// Force a fresh accessibility-tree fetch outside the normal poll cadence.
|
|
317
|
-
refreshAxTree: () => Promise<AxSnapshot>;
|
|
318
|
-
|
|
319
|
-
// Pull-based access to the most recent snapshot (the same one passed to
|
|
320
|
-
// onAxSnapshotChange). Returns null when no snapshot has arrived yet or
|
|
321
|
-
// when inspect mode is off.
|
|
322
|
-
getAxSnapshot: () => AxSnapshot | null;
|
|
323
|
-
|
|
324
|
-
// Programmatically drive the overlay highlight/selection — useful when a
|
|
325
|
-
// customer's own side panel wants to cross-highlight with the overlay.
|
|
326
|
-
// Pass `null` to clear.
|
|
327
|
-
setInspectHighlight: (element: AxElement | null) => void;
|
|
328
|
-
setInspectSelection: (element: AxElement | null) => void;
|
|
329
|
-
|
|
330
|
-
// Pull-based access to the current AX subsystem status. Mirrors what
|
|
331
|
-
// onAxStatusChange reports, for customers that don't want to subscribe.
|
|
332
|
-
getAxStatus: () => AxStatus;
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
const debugLog = (...args: any[]) => {
|
|
336
|
-
if (window.debugRemoteControl) {
|
|
337
|
-
console.log(...args);
|
|
338
|
-
}
|
|
339
|
-
};
|
|
340
|
-
|
|
341
|
-
const debugWarn = (...args: any[]) => {
|
|
342
|
-
if (window.debugRemoteControl) {
|
|
343
|
-
console.warn(...args);
|
|
344
|
-
}
|
|
345
|
-
};
|
|
346
|
-
|
|
347
|
-
// Invokes a customer-provided callback in isolation. A throw from the
|
|
348
|
-
// customer's code must NOT propagate back into our state-update flow — that
|
|
349
|
-
// would risk corrupting React reconciliation. We log the error to the
|
|
350
|
-
// console so the customer can still debug, but otherwise swallow.
|
|
351
|
-
const safeInvoke = <Args extends unknown[]>(
|
|
352
|
-
label: string,
|
|
353
|
-
fn: ((...args: Args) => unknown) | undefined,
|
|
354
|
-
...args: Args
|
|
355
|
-
): void => {
|
|
356
|
-
if (!fn) return;
|
|
357
|
-
try {
|
|
358
|
-
fn(...args);
|
|
359
|
-
} catch (err) {
|
|
360
|
-
// Surface to the developer regardless of debug flag — this is a bug
|
|
361
|
-
// in the customer's handler and they'll want to see it.
|
|
362
|
-
// eslint-disable-next-line no-console
|
|
363
|
-
console.error(`[RemoteControl] customer callback "${label}" threw:`, err);
|
|
364
|
-
}
|
|
365
|
-
};
|
|
366
|
-
|
|
367
|
-
const motionActionToString = (action: number): string => {
|
|
368
|
-
// AMOTION_EVENT is a constants object; find the matching ACTION_* key if present
|
|
369
|
-
const match = Object.entries(AMOTION_EVENT).find(
|
|
370
|
-
([key, value]) => key.startsWith('ACTION_') && value === action,
|
|
371
|
-
);
|
|
372
|
-
return match?.[0] ?? String(action);
|
|
373
|
-
};
|
|
374
|
-
|
|
375
|
-
type DevicePlatform = 'ios' | 'android';
|
|
376
|
-
|
|
377
|
-
const detectPlatform = (url: string): DevicePlatform => {
|
|
378
|
-
if (url.includes('/android_')) {
|
|
379
|
-
return 'android';
|
|
380
|
-
}
|
|
381
|
-
// Default to iOS if no Android pattern is found
|
|
382
|
-
return 'ios';
|
|
383
|
-
};
|
|
384
|
-
|
|
385
|
-
// The endpoint may already carry query parameters, and the token may contain URL syntax.
|
|
386
|
-
export const withAuthenticationToken = (url: string, token: string): string => {
|
|
387
|
-
const endpoint = new URL(url);
|
|
388
|
-
endpoint.searchParams.set('token', token);
|
|
389
|
-
return endpoint.toString();
|
|
390
|
-
};
|
|
391
|
-
|
|
392
|
-
export type RemoteControlConnectionState =
|
|
393
|
-
| 'connecting'
|
|
394
|
-
| 'connected'
|
|
395
|
-
| 'reconnecting'
|
|
396
|
-
| 'failed'
|
|
397
|
-
| 'terminated';
|
|
398
|
-
|
|
399
|
-
export type RemoteControlPlatformAssets = {
|
|
400
|
-
frame?: string;
|
|
401
|
-
frameLandscape?: string;
|
|
402
|
-
tabletFrame?: string;
|
|
403
|
-
tabletFrameLandscape?: string;
|
|
404
|
-
loadingLogo?: string;
|
|
405
|
-
};
|
|
406
|
-
|
|
407
|
-
export type RemoteControlAssets = Partial<Record<DevicePlatform, RemoteControlPlatformAssets>>;
|
|
408
|
-
|
|
409
|
-
type DeviceConfig = {
|
|
410
|
-
videoBorderRadiusMultiplier: number;
|
|
411
|
-
loadingLogoSize: string;
|
|
412
|
-
videoPosition: {
|
|
413
|
-
portrait: { heightMultiplier?: number; widthMultiplier?: number };
|
|
414
|
-
landscape: { heightMultiplier?: number; widthMultiplier?: number };
|
|
415
|
-
};
|
|
416
|
-
};
|
|
417
|
-
|
|
418
|
-
const ANDROID_TABLET_VIDEO_WIDTH = 1920;
|
|
419
|
-
const ANDROID_TABLET_VIDEO_HEIGHT = 1200;
|
|
420
|
-
const MAX_CONNECTION_ATTEMPTS = 3;
|
|
421
|
-
const CONNECTION_RETRY_DELAY_MS = 1000;
|
|
422
|
-
const CONNECTION_SUCCESS_TIMEOUT_MS = 15000;
|
|
423
|
-
const ICE_DISCONNECTED_GRACE_MS = 3000;
|
|
424
|
-
// Upper bound for the one-shot HTTP probe that disambiguates a terminated
|
|
425
|
-
// instance (endpoint 404s) from a transient connection drop.
|
|
426
|
-
const TERMINATION_PROBE_TIMEOUT_MS = 4000;
|
|
427
|
-
|
|
428
|
-
const isAndroidTabletVideo = (width: number, height: number): boolean =>
|
|
429
|
-
(width === ANDROID_TABLET_VIDEO_WIDTH && height === ANDROID_TABLET_VIDEO_HEIGHT) ||
|
|
430
|
-
(width === ANDROID_TABLET_VIDEO_HEIGHT && height === ANDROID_TABLET_VIDEO_WIDTH);
|
|
431
|
-
|
|
432
|
-
// Device-specific configuration for frame sizing and video positioning
|
|
433
|
-
// Video position percentages are relative to the frame image dimensions
|
|
434
|
-
const deviceConfig: Record<DevicePlatform, DeviceConfig> = {
|
|
435
|
-
ios: {
|
|
436
|
-
videoBorderRadiusMultiplier: 0.15,
|
|
437
|
-
loadingLogoSize: '20%',
|
|
438
|
-
// Video position as percentage of frame dimensions
|
|
439
|
-
videoPosition: {
|
|
440
|
-
portrait: { heightMultiplier: 0.9678 },
|
|
441
|
-
landscape: { widthMultiplier: 0.9678 },
|
|
442
|
-
},
|
|
443
|
-
},
|
|
444
|
-
android: {
|
|
445
|
-
videoBorderRadiusMultiplier: 0.13,
|
|
446
|
-
loadingLogoSize: '40%',
|
|
447
|
-
// Video position as percentage of frame dimensions
|
|
448
|
-
videoPosition: {
|
|
449
|
-
portrait: { heightMultiplier: 0.967 },
|
|
450
|
-
landscape: { widthMultiplier: 0.962 },
|
|
451
|
-
},
|
|
452
|
-
},
|
|
453
|
-
};
|
|
454
|
-
|
|
455
|
-
function getAndroidKeycodeAndMeta(event: React.KeyboardEvent): { keycode: number; metaState: number } | null {
|
|
456
|
-
const code = event.code;
|
|
457
|
-
const keycode = codeMap[code];
|
|
458
|
-
|
|
459
|
-
if (!keycode) {
|
|
460
|
-
// Use the wrapper for conditional warning
|
|
461
|
-
debugWarn(`Unknown event.code: ${code}, key: ${event.key}`);
|
|
462
|
-
return null;
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
let metaState = ANDROID_KEYS.META_NONE;
|
|
466
|
-
const isLetter = code >= 'KeyA' && code <= 'KeyZ';
|
|
467
|
-
const isCapsLock = event.getModifierState('CapsLock');
|
|
468
|
-
const isShiftPressed = event.shiftKey;
|
|
469
|
-
|
|
470
|
-
// Determine effective shift state
|
|
471
|
-
let effectiveShift = isShiftPressed;
|
|
472
|
-
if (isLetter) {
|
|
473
|
-
effectiveShift = isShiftPressed !== isCapsLock; // Logical XOR for booleans
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
// Apply meta states
|
|
477
|
-
if (effectiveShift) metaState |= ANDROID_KEYS.META_SHIFT_ON;
|
|
478
|
-
if (event.ctrlKey) metaState |= ANDROID_KEYS.META_CTRL_ON;
|
|
479
|
-
if (event.altKey) metaState |= ANDROID_KEYS.META_ALT_ON;
|
|
480
|
-
if (event.metaKey) metaState |= ANDROID_KEYS.META_META_ON; // Command on Mac, Windows key on Win
|
|
481
|
-
|
|
482
|
-
return { keycode, metaState };
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>(
|
|
486
|
-
(
|
|
487
|
-
{
|
|
488
|
-
className,
|
|
489
|
-
url,
|
|
490
|
-
token,
|
|
491
|
-
sessionId: propSessionId,
|
|
492
|
-
openUrl,
|
|
493
|
-
showFrame = true,
|
|
494
|
-
autoReconnect = false,
|
|
495
|
-
onTerminated,
|
|
496
|
-
assets,
|
|
497
|
-
onConnectionStateChange,
|
|
498
|
-
inspectMode,
|
|
499
|
-
onAxSnapshotChange,
|
|
500
|
-
onInspectSelectionChange,
|
|
501
|
-
onAxStatusChange,
|
|
502
|
-
axPollIntervalMs,
|
|
503
|
-
axMaxBackoffMs,
|
|
504
|
-
androidElementTreeOptions,
|
|
505
|
-
onCameraDemandChange,
|
|
506
|
-
onCameraStats,
|
|
507
|
-
microphoneEnabled = false,
|
|
508
|
-
onMicrophoneStateChange,
|
|
509
|
-
}: RemoteControlProps,
|
|
510
|
-
ref,
|
|
511
|
-
) => {
|
|
512
|
-
const containerRef = useRef<HTMLDivElement>(null);
|
|
513
|
-
const videoRef = useRef<HTMLVideoElement>(null);
|
|
514
|
-
const frameRef = useRef<HTMLImageElement>(null);
|
|
515
|
-
const [videoLoaded, setVideoLoaded] = useState(false);
|
|
516
|
-
const [retryExhausted, setRetryExhausted] = useState(false);
|
|
517
|
-
// Set once we've concluded the instance is permanently gone (its
|
|
518
|
-
// endpoint 404s after a drop), as opposed to a recoverable blip. When
|
|
519
|
-
// set, reconnection stops and a terminated message replaces "Retry".
|
|
520
|
-
const [terminated, setTerminated] = useState(false);
|
|
521
|
-
const terminatedRef = useRef(false);
|
|
522
|
-
// Guards against firing overlapping termination probes.
|
|
523
|
-
const terminationProbeInFlightRef = useRef(false);
|
|
524
|
-
const [isLandscape, setIsLandscape] = useState(false);
|
|
525
|
-
const [useAndroidTabletFrame, setUseAndroidTabletFrame] = useState(false);
|
|
526
|
-
const [videoStyle, setVideoStyle] = useState<React.CSSProperties>({});
|
|
527
|
-
const wsRef = useRef<WebSocket | null>(null);
|
|
528
|
-
const peerConnectionRef = useRef<RTCPeerConnection | null>(null);
|
|
529
|
-
const dataChannelRef = useRef<RTCDataChannel | null>(null);
|
|
530
|
-
const keepAliveIntervalRef = useRef<number | undefined>(undefined);
|
|
531
|
-
const retryTimeoutRef = useRef<number | undefined>(undefined);
|
|
532
|
-
const connectionSuccessTimeoutRef = useRef<number | undefined>(undefined);
|
|
533
|
-
const requestFrameIntervalRef = useRef<number | undefined>(undefined);
|
|
534
|
-
const iceDisconnectedGraceRef = useRef<number | undefined>(undefined);
|
|
535
|
-
const connectionGenerationRef = useRef(0);
|
|
536
|
-
const connectionAttemptRef = useRef(0);
|
|
537
|
-
const controlChannelOpenedRef = useRef(false);
|
|
538
|
-
// Mirrored to a ref so stale closures in event handlers see the latest value.
|
|
539
|
-
const autoReconnectRef = useRef(autoReconnect);
|
|
540
|
-
autoReconnectRef.current = autoReconnect;
|
|
541
|
-
// Demand-driven outbound camera state.
|
|
542
|
-
//
|
|
543
|
-
// The limulator side broadcasts a `cameraRequest` WS message whenever
|
|
544
|
-
// an app inside the simulator opens/closes an `AVCaptureSession`.
|
|
545
|
-
// We respond by calling `navigator.mediaDevices.getUserMedia(...)`
|
|
546
|
-
// and `replaceTrack`ing the result onto a pre-allocated sendonly
|
|
547
|
-
// video transceiver. Pre-allocating the transceiver lets us
|
|
548
|
-
// attach/detach the local camera without renegotiating the SDP
|
|
549
|
-
// (the slot is already in the answer's a=video block, just with
|
|
550
|
-
// `inactive`/empty until we install the track).
|
|
551
|
-
//
|
|
552
|
-
// Refs (not state) because all callers live inside event-handler
|
|
553
|
-
// closures and the ref read happens during message processing, not
|
|
554
|
-
// during render. We keep both the sender and the active local
|
|
555
|
-
// stream so teardown can stop tracks without leaking the camera
|
|
556
|
-
// green light.
|
|
557
|
-
const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
|
|
558
|
-
const outboundLocalStreamRef = useRef<MediaStream | null>(null);
|
|
559
|
-
const outboundCameraRequestRef = useRef<CameraRequest | undefined>(undefined);
|
|
560
|
-
const cameraOperations = useMemo(() => new CameraOperationCoordinator(), []);
|
|
561
|
-
// Mirror the demand-change callback into a ref so the WS message
|
|
562
|
-
// handler always sees the freshest customer callback even when the
|
|
563
|
-
// parent re-renders mid-session.
|
|
564
|
-
const onCameraDemandChangeRef = useRef(onCameraDemandChange);
|
|
565
|
-
onCameraDemandChangeRef.current = onCameraDemandChange;
|
|
566
|
-
const onCameraStatsRef = useRef(onCameraStats);
|
|
567
|
-
onCameraStatsRef.current = onCameraStats;
|
|
568
|
-
// Active outbound-camera stats poller. While the camera is
|
|
569
|
-
// sending we sample `RTCPeerConnection.getStats()` once per
|
|
570
|
-
// second, derive deltas (bitrate, fps, packet loss) from the
|
|
571
|
-
// previous sample, and push the result through `onCameraStats`.
|
|
572
|
-
// Cleared on teardown so we never leak the timer.
|
|
573
|
-
const cameraStatsTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
|
574
|
-
const cameraStatsPrevRef = useRef<{
|
|
575
|
-
timestamp: number;
|
|
576
|
-
framesEncoded?: number;
|
|
577
|
-
bytesSent?: number;
|
|
578
|
-
packetsSent?: number;
|
|
579
|
-
packetsLost?: number;
|
|
580
|
-
} | null>(null);
|
|
581
|
-
// Outbound microphone state (iOS only). The manual prop and guest-driven
|
|
582
|
-
// microphoneRequest signals share one pre-allocated audio transceiver.
|
|
583
|
-
const microphoneEnabledRef = useRef(microphoneEnabled);
|
|
584
|
-
microphoneEnabledRef.current = microphoneEnabled;
|
|
585
|
-
const microphoneDemandActiveRef = useRef(false);
|
|
586
|
-
const onMicrophoneStateChangeRef = useRef(onMicrophoneStateChange);
|
|
587
|
-
onMicrophoneStateChangeRef.current = onMicrophoneStateChange;
|
|
588
|
-
const outboundMicSenderRef = useRef<RTCRtpSender | null>(null);
|
|
589
|
-
const outboundMicStreamRef = useRef<MediaStream | null>(null);
|
|
590
|
-
// Bumped by detach and teardown so an attach suspended on an await
|
|
591
|
-
// (the getUserMedia prompt) detects it was superseded and bails.
|
|
592
|
-
const micGenerationRef = useRef(0);
|
|
593
|
-
// Serializes attach/detach work so their awaits (replaceTrack on the
|
|
594
|
-
// shared sender) can never interleave and undo each other's effects.
|
|
595
|
-
const micOpQueueRef = useRef<Promise<void>>(Promise.resolve());
|
|
596
|
-
// Correlates the in-flight `microphoneStart` with its result so a
|
|
597
|
-
// host-side rejection (e.g. no loopback device on the node) can
|
|
598
|
-
// tear the capture back down and surface the error.
|
|
599
|
-
const micStartPendingIdRef = useRef<string | null>(null);
|
|
600
|
-
const firstFrameShownRef = useRef(false);
|
|
601
|
-
const pendingScreenshotResolversRef = useRef<
|
|
602
|
-
Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
|
|
603
|
-
>(new Map());
|
|
604
|
-
const pendingScreenshotRejectersRef = useRef<Map<string, (reason?: any) => void>>(new Map());
|
|
605
|
-
const pendingTerminateAppResolversRef = useRef<Map<string, () => void>>(new Map());
|
|
606
|
-
const pendingTerminateAppRejectersRef = useRef<Map<string, (reason?: any) => void>>(new Map());
|
|
607
|
-
|
|
608
|
-
// Map to track active pointers for real touch/mouse single-finger events.
|
|
609
|
-
// Key: pointerId (-1 for mouse, touch.identifier for touch), Value: { x: number, y: number }
|
|
610
|
-
const activePointers = useRef<Map<number, { x: number; y: number }>>(new Map());
|
|
611
|
-
|
|
612
|
-
// Alt/Option modifier state for pinch emulation.
|
|
613
|
-
// We use a ref as the source of truth (for synchronous event handler access)
|
|
614
|
-
// and state only to trigger re-renders for the visual indicators.
|
|
615
|
-
const isAltHeldRef = useRef(false);
|
|
616
|
-
const [isAltHeld, setIsAltHeld] = useState(false);
|
|
617
|
-
|
|
618
|
-
// State for any two-finger gesture (Alt+mouse simulated or real two-finger touch).
|
|
619
|
-
// Tracks positions, video size, source, and pointer IDs (for Android protocol).
|
|
620
|
-
type TwoFingerState = {
|
|
621
|
-
finger0: { x: number; y: number };
|
|
622
|
-
finger1: { x: number; y: number };
|
|
623
|
-
videoSize: { width: number; height: number };
|
|
624
|
-
// Track source so we know when to clear (Alt release vs touch end)
|
|
625
|
-
source: 'alt-mouse' | 'real-touch';
|
|
626
|
-
// Pointer IDs for Android (real touch.identifier or simulated -1/-2)
|
|
627
|
-
pointerId0: number;
|
|
628
|
-
pointerId1: number;
|
|
629
|
-
};
|
|
630
|
-
const twoFingerStateRef = useRef<TwoFingerState | null>(null);
|
|
631
|
-
// Wheel-scroll drag state; lives for a single synthesized gesture.
|
|
632
|
-
const wheelStateRef = useRef<{ pos: PointerGeometry; endTimer: number } | null>(null);
|
|
633
|
-
// Latest wheel handlers, so the once-attached native listener and the
|
|
634
|
-
// pointer machinery always call the current render's closures.
|
|
635
|
-
const wheelCallbacksRef = useRef<{ handle: (event: WheelEvent) => void; end: () => void }>({
|
|
636
|
-
handle: () => {},
|
|
637
|
-
end: () => {},
|
|
638
|
-
});
|
|
639
|
-
|
|
640
|
-
// Hover point for rendering two-finger indicators when Alt is held.
|
|
641
|
-
// Only computed/set when Alt is held to avoid unnecessary re-renders.
|
|
642
|
-
type HoverPoint = {
|
|
643
|
-
containerX: number;
|
|
644
|
-
containerY: number;
|
|
645
|
-
mirrorContainerX: number;
|
|
646
|
-
mirrorContainerY: number;
|
|
647
|
-
videoX: number;
|
|
648
|
-
videoY: number;
|
|
649
|
-
mirrorVideoX: number;
|
|
650
|
-
mirrorVideoY: number;
|
|
651
|
-
videoWidth: number;
|
|
652
|
-
videoHeight: number;
|
|
653
|
-
};
|
|
654
|
-
const [hoverPoint, setHoverPoint] = useState<HoverPoint | null>(null);
|
|
655
|
-
|
|
656
|
-
// Inspect-mode state.
|
|
657
|
-
//
|
|
658
|
-
// Lifecycle of `axFetcherRef`:
|
|
659
|
-
// - Created in dataChannel.onopen (last step of WebRTC handshake)
|
|
660
|
-
// so we know the signaling WS is healthy and the device is
|
|
661
|
-
// responsive to control messages.
|
|
662
|
-
// - Started immediately if `inspectMode` is already enabled, or
|
|
663
|
-
// started later via the sibling useEffect when inspectMode flips on.
|
|
664
|
-
// - Stopped + nulled in teardownConnection (WS close / unmount).
|
|
665
|
-
//
|
|
666
|
-
// Customers can observe readiness via the `onAxStatusChange` callback:
|
|
667
|
-
// `starting` fires when start() runs but no snapshot has landed yet;
|
|
668
|
-
// `ready` once the first snapshot arrives. Status falls back to
|
|
669
|
-
// `unavailable` / `error` if the server can't satisfy AX requests.
|
|
670
|
-
const axFetcherRef = useRef<AxFetcher | null>(null);
|
|
671
|
-
const [axSnapshot, setAxSnapshot] = useState<AxSnapshot | null>(null);
|
|
672
|
-
const [axHighlightedId, setAxHighlightedId] = useState<string | null>(null);
|
|
673
|
-
const [axSelectedId, setAxSelectedId] = useState<string | null>(null);
|
|
674
|
-
const [overlayGeometry, setOverlayGeometry] = useState<InspectOverlayGeometry | null>(null);
|
|
675
|
-
// Viewport-space cursor position used to anchor the inspect InfoCard.
|
|
676
|
-
// Throttled to one update per animation frame to avoid React reconciling
|
|
677
|
-
// on every native mousemove (~60–120Hz).
|
|
678
|
-
const [axCursorPosition, setAxCursorPosition] = useState<{ x: number; y: number } | null>(null);
|
|
679
|
-
const cursorPositionRef = useRef<{ x: number; y: number } | null>(null);
|
|
680
|
-
const cursorRafIdRef = useRef<number | undefined>(undefined);
|
|
681
|
-
const scheduleCursorFlush = (next: { x: number; y: number } | null) => {
|
|
682
|
-
cursorPositionRef.current = next;
|
|
683
|
-
if (cursorRafIdRef.current !== undefined) return;
|
|
684
|
-
cursorRafIdRef.current = window.requestAnimationFrame(() => {
|
|
685
|
-
cursorRafIdRef.current = undefined;
|
|
686
|
-
setAxCursorPosition(cursorPositionRef.current);
|
|
687
|
-
});
|
|
688
|
-
};
|
|
689
|
-
// Position captured at click-time so the InfoCard "freezes" near where
|
|
690
|
-
// the user clicked, even as they move the cursor around afterward. The
|
|
691
|
-
// action buttons (Tap / Copy) stay reachable because the card no longer
|
|
692
|
-
// chases the cursor while the click target is the active selection.
|
|
693
|
-
const [axFrozenCursorPosition, setAxFrozenCursorPosition] = useState<{
|
|
694
|
-
x: number;
|
|
695
|
-
y: number;
|
|
696
|
-
} | null>(null);
|
|
697
|
-
// Mirrors for synchronous access from event handlers without stale closures.
|
|
698
|
-
const inspectModeRef = useRef<boolean | 'hover-only' | undefined>(inspectMode);
|
|
699
|
-
inspectModeRef.current = inspectMode;
|
|
700
|
-
const axSnapshotRef = useRef<AxSnapshot | null>(null);
|
|
701
|
-
axSnapshotRef.current = axSnapshot;
|
|
702
|
-
const onAxSnapshotChangeRef = useRef(onAxSnapshotChange);
|
|
703
|
-
onAxSnapshotChangeRef.current = onAxSnapshotChange;
|
|
704
|
-
const onInspectSelectionChangeRef = useRef(onInspectSelectionChange);
|
|
705
|
-
onInspectSelectionChangeRef.current = onInspectSelectionChange;
|
|
706
|
-
const onAxStatusChangeRef = useRef(onAxStatusChange);
|
|
707
|
-
onAxStatusChangeRef.current = onAxStatusChange;
|
|
708
|
-
const onTerminatedRef = useRef(onTerminated);
|
|
709
|
-
onTerminatedRef.current = onTerminated;
|
|
710
|
-
const onConnectionStateChangeRef = useRef(onConnectionStateChange);
|
|
711
|
-
onConnectionStateChangeRef.current = onConnectionStateChange;
|
|
712
|
-
const connectionStateRef = useRef<RemoteControlConnectionState | null>(null);
|
|
713
|
-
// Reports each transition once; callers get status text without tracking
|
|
714
|
-
// the retry machinery below.
|
|
715
|
-
const reportConnectionState = (next: RemoteControlConnectionState) => {
|
|
716
|
-
if (connectionStateRef.current === next) return;
|
|
717
|
-
connectionStateRef.current = next;
|
|
718
|
-
safeInvoke('onConnectionStateChange', onConnectionStateChangeRef.current, next);
|
|
719
|
-
};
|
|
720
|
-
|
|
721
|
-
const inspectActive = inspectMode === true || inspectMode === 'hover-only';
|
|
722
|
-
const inspectModeResolved: InspectMode = inspectMode === 'hover-only' ? 'hover-only' : 'select';
|
|
723
|
-
|
|
724
|
-
const sessionId = useMemo(
|
|
725
|
-
() =>
|
|
726
|
-
propSessionId ||
|
|
727
|
-
Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15),
|
|
728
|
-
[propSessionId],
|
|
729
|
-
);
|
|
730
|
-
|
|
731
|
-
const platform = useMemo(() => detectPlatform(url), [url]);
|
|
732
|
-
const config = deviceConfig[platform];
|
|
733
|
-
const platformAssets = assets?.[platform];
|
|
734
|
-
const frameImageSrc =
|
|
735
|
-
platform === 'android' && useAndroidTabletFrame ?
|
|
736
|
-
isLandscape ? platformAssets?.tabletFrameLandscape
|
|
737
|
-
: platformAssets?.tabletFrame
|
|
738
|
-
: isLandscape ? platformAssets?.frameLandscape
|
|
739
|
-
: platformAssets?.frame;
|
|
740
|
-
const loadingLogo = platformAssets?.loadingLogo;
|
|
741
|
-
// Without a frame image (the lite entry) the video lays out frameless
|
|
742
|
-
// whatever showFrame says.
|
|
743
|
-
const frameVisible = showFrame && !!frameImageSrc;
|
|
744
|
-
|
|
745
|
-
const updateStatus = (message: string) => {
|
|
746
|
-
// Use the wrapper for conditional logging
|
|
747
|
-
debugLog(message);
|
|
748
|
-
};
|
|
749
|
-
|
|
750
|
-
const sendBinaryControlMessage = (data: ArrayBuffer) => {
|
|
751
|
-
if (!dataChannelRef.current || dataChannelRef.current.readyState !== 'open') {
|
|
752
|
-
return;
|
|
753
|
-
}
|
|
754
|
-
dataChannelRef.current.send(data);
|
|
755
|
-
// Any binary control message is an input event. Bump the AX poller so
|
|
756
|
-
// we get a fresh snapshot quickly — the UI almost certainly changed.
|
|
757
|
-
axFetcherRef.current?.bumpActivity();
|
|
758
|
-
};
|
|
759
|
-
|
|
760
|
-
// Pointer ID used by inspect-driven taps. Distinct from human pointers
|
|
761
|
-
// (-1 mouse, -2 alt-mirror), the synthesized wheel finger (-3), and our
|
|
762
|
-
// touch identifiers so they never interfere with an in-progress drag.
|
|
763
|
-
const AX_TAP_POINTER_ID = -10;
|
|
764
|
-
|
|
765
|
-
// Send a down+up tap at a viewport-space (clientX/Y) position. The point
|
|
766
|
-
// is mapped through the current video letterbox geometry so the
|
|
767
|
-
// simulator receives the correct in-stream coordinates regardless of
|
|
768
|
-
// how the device frame is sized in the DOM.
|
|
769
|
-
const sendTapAtClient = (clientX: number, clientY: number) => {
|
|
770
|
-
const ctx = computeVideoMappingContext();
|
|
771
|
-
if (!ctx) return;
|
|
772
|
-
const geometry = mapClientPointToVideo(ctx, clientX, clientY);
|
|
773
|
-
if (!geometry) return;
|
|
774
|
-
const { videoX, videoY, videoWidth, videoHeight } = geometry;
|
|
775
|
-
const down = createTouchControlMessage(
|
|
776
|
-
AMOTION_EVENT.ACTION_DOWN,
|
|
777
|
-
AX_TAP_POINTER_ID,
|
|
778
|
-
videoWidth,
|
|
779
|
-
videoHeight,
|
|
780
|
-
videoX,
|
|
781
|
-
videoY,
|
|
782
|
-
1.0,
|
|
783
|
-
AMOTION_EVENT.BUTTON_PRIMARY,
|
|
784
|
-
AMOTION_EVENT.BUTTON_PRIMARY,
|
|
785
|
-
);
|
|
786
|
-
if (down) sendBinaryControlMessage(down);
|
|
787
|
-
window.setTimeout(() => {
|
|
788
|
-
const up = createTouchControlMessage(
|
|
789
|
-
AMOTION_EVENT.ACTION_UP,
|
|
790
|
-
AX_TAP_POINTER_ID,
|
|
791
|
-
videoWidth,
|
|
792
|
-
videoHeight,
|
|
793
|
-
videoX,
|
|
794
|
-
videoY,
|
|
795
|
-
0,
|
|
796
|
-
AMOTION_EVENT.BUTTON_PRIMARY,
|
|
797
|
-
AMOTION_EVENT.BUTTON_PRIMARY,
|
|
798
|
-
);
|
|
799
|
-
if (up) sendBinaryControlMessage(up);
|
|
800
|
-
}, 60);
|
|
801
|
-
};
|
|
802
|
-
|
|
803
|
-
// Center-of-bounds fallback for programmatic taps when there's no
|
|
804
|
-
// user-aimed click position (e.g. customer calls `setInspectSelection`
|
|
805
|
-
// followed by their own "tap selected" handler without forwarding a
|
|
806
|
-
// pointer position). Maps the element's frame center through the AX
|
|
807
|
-
// screen-coordinate space to viewport coords, then delegates to
|
|
808
|
-
// sendTapAtClient.
|
|
809
|
-
const sendTapAtElementCenter = (element: AxElement, snapshot: AxSnapshot) => {
|
|
810
|
-
const ctx = computeVideoMappingContext();
|
|
811
|
-
if (!ctx) return;
|
|
812
|
-
if (snapshot.screen.width <= 0 || snapshot.screen.height <= 0) return;
|
|
813
|
-
const cxAx = element.frame.x + element.frame.width / 2;
|
|
814
|
-
const cyAx = element.frame.y + element.frame.height / 2;
|
|
815
|
-
// AX screen-fraction → in-video pixel offset → viewport client coord.
|
|
816
|
-
const inVideoX = (cxAx / snapshot.screen.width) * ctx.actualWidth;
|
|
817
|
-
const inVideoY = (cyAx / snapshot.screen.height) * ctx.actualHeight;
|
|
818
|
-
const clientX = ctx.videoRect.left + ctx.offsetX + inVideoX;
|
|
819
|
-
const clientY = ctx.videoRect.top + ctx.offsetY + inVideoY;
|
|
820
|
-
sendTapAtClient(clientX, clientY);
|
|
821
|
-
};
|
|
822
|
-
|
|
823
|
-
// Fixed pointer IDs for Alt-simulated two-finger gestures
|
|
824
|
-
const ALT_POINTER_ID_PRIMARY = -1;
|
|
825
|
-
const ALT_POINTER_ID_MIRROR = -2;
|
|
826
|
-
// Synthesized finger for wheel-scroll drags.
|
|
827
|
-
const WHEEL_POINTER_ID = -3;
|
|
828
|
-
|
|
829
|
-
// Helper to send a single-touch control message (used by both single-finger and Android two-finger paths)
|
|
830
|
-
const sendSingleTouch = (
|
|
831
|
-
action: number,
|
|
832
|
-
pointerId: number,
|
|
833
|
-
videoWidth: number,
|
|
834
|
-
videoHeight: number,
|
|
835
|
-
x: number,
|
|
836
|
-
y: number,
|
|
837
|
-
) => {
|
|
838
|
-
const message = createTouchControlMessage(
|
|
839
|
-
action,
|
|
840
|
-
pointerId,
|
|
841
|
-
videoWidth,
|
|
842
|
-
videoHeight,
|
|
843
|
-
x,
|
|
844
|
-
y,
|
|
845
|
-
1.0, // pressure
|
|
846
|
-
AMOTION_EVENT.BUTTON_PRIMARY,
|
|
847
|
-
AMOTION_EVENT.BUTTON_PRIMARY,
|
|
848
|
-
);
|
|
849
|
-
if (message) {
|
|
850
|
-
debugLog('[rc-touch] sendSingleTouch', {
|
|
851
|
-
action,
|
|
852
|
-
actionName: motionActionToString(action),
|
|
853
|
-
pointerId,
|
|
854
|
-
x,
|
|
855
|
-
y,
|
|
856
|
-
video: { width: videoWidth, height: videoHeight },
|
|
857
|
-
});
|
|
858
|
-
sendBinaryControlMessage(message);
|
|
859
|
-
}
|
|
860
|
-
};
|
|
861
|
-
|
|
862
|
-
// Minimal geometry for single-finger touch events (no mirror/container coords needed).
|
|
863
|
-
type PointerGeometry = {
|
|
864
|
-
videoX: number;
|
|
865
|
-
videoY: number;
|
|
866
|
-
videoWidth: number;
|
|
867
|
-
videoHeight: number;
|
|
868
|
-
};
|
|
869
|
-
|
|
870
|
-
const applyPointerEvent = (
|
|
871
|
-
pointerId: number,
|
|
872
|
-
eventType: 'down' | 'move' | 'up' | 'cancel',
|
|
873
|
-
geometry: PointerGeometry | null,
|
|
874
|
-
) => {
|
|
875
|
-
if (!geometry) return;
|
|
876
|
-
const { videoX, videoY, videoWidth, videoHeight } = geometry;
|
|
877
|
-
|
|
878
|
-
let action: number | null = null;
|
|
879
|
-
let positionToSend: { x: number; y: number } | null = null;
|
|
880
|
-
let pressure = 1.0; // Default pressure
|
|
881
|
-
const buttons = AMOTION_EVENT.BUTTON_PRIMARY; // Assume primary button
|
|
882
|
-
|
|
883
|
-
switch (eventType) {
|
|
884
|
-
case 'down':
|
|
885
|
-
// A real pointer going down takes over from any synthesized
|
|
886
|
-
// wheel-scroll finger, so the tap isn't sent as a second finger.
|
|
887
|
-
if (pointerId !== WHEEL_POINTER_ID) {
|
|
888
|
-
wheelCallbacksRef.current.end();
|
|
889
|
-
}
|
|
890
|
-
// For multi-touch: use ACTION_DOWN for first pointer, ACTION_POINTER_DOWN for additional pointers
|
|
891
|
-
const currentPointerCount = activePointers.current.size;
|
|
892
|
-
action = currentPointerCount === 0 ? AMOTION_EVENT.ACTION_DOWN : AMOTION_EVENT.ACTION_POINTER_DOWN;
|
|
893
|
-
positionToSend = { x: videoX, y: videoY };
|
|
894
|
-
activePointers.current.set(pointerId, positionToSend);
|
|
895
|
-
if (pointerId === -1) {
|
|
896
|
-
// Focus on mouse down
|
|
897
|
-
videoRef.current?.focus();
|
|
898
|
-
}
|
|
899
|
-
break;
|
|
900
|
-
|
|
901
|
-
case 'move':
|
|
902
|
-
if (activePointers.current.has(pointerId)) {
|
|
903
|
-
action = AMOTION_EVENT.ACTION_MOVE;
|
|
904
|
-
positionToSend = { x: videoX, y: videoY };
|
|
905
|
-
// Update the last known position for this active pointer
|
|
906
|
-
activePointers.current.set(pointerId, positionToSend);
|
|
907
|
-
}
|
|
908
|
-
break;
|
|
909
|
-
|
|
910
|
-
case 'up':
|
|
911
|
-
case 'cancel': // Treat cancel like up, but use ACTION_CANCEL
|
|
912
|
-
if (activePointers.current.has(pointerId)) {
|
|
913
|
-
// IMPORTANT: Send the UP/CANCEL at the *last known position* inside the video
|
|
914
|
-
positionToSend = activePointers.current.get(pointerId)!;
|
|
915
|
-
activePointers.current.delete(pointerId); // Remove pointer as it's no longer active
|
|
916
|
-
|
|
917
|
-
if (eventType === 'cancel') {
|
|
918
|
-
action = AMOTION_EVENT.ACTION_CANCEL;
|
|
919
|
-
} else {
|
|
920
|
-
// For multi-touch: use ACTION_UP for last pointer, ACTION_POINTER_UP for non-last pointers
|
|
921
|
-
const remainingPointerCount = activePointers.current.size;
|
|
922
|
-
action =
|
|
923
|
-
remainingPointerCount === 0 ? AMOTION_EVENT.ACTION_UP : AMOTION_EVENT.ACTION_POINTER_UP;
|
|
924
|
-
}
|
|
925
|
-
}
|
|
926
|
-
break;
|
|
927
|
-
}
|
|
928
|
-
|
|
929
|
-
// Send message if action and position determined
|
|
930
|
-
if (action !== null && positionToSend !== null) {
|
|
931
|
-
debugLog('[rc-touch][mouse->touch] sending', {
|
|
932
|
-
pointerId,
|
|
933
|
-
eventType,
|
|
934
|
-
action,
|
|
935
|
-
actionName: motionActionToString(action),
|
|
936
|
-
positionToSend,
|
|
937
|
-
video: { width: videoWidth, height: videoHeight },
|
|
938
|
-
altHeld: isAltHeldRef.current,
|
|
939
|
-
activePointersAfter: Array.from(activePointers.current.entries()).map(([id, pos]) => ({
|
|
940
|
-
id,
|
|
941
|
-
x: pos.x,
|
|
942
|
-
y: pos.y,
|
|
943
|
-
})),
|
|
944
|
-
});
|
|
945
|
-
const message = createTouchControlMessage(
|
|
946
|
-
action,
|
|
947
|
-
pointerId,
|
|
948
|
-
videoWidth,
|
|
949
|
-
videoHeight,
|
|
950
|
-
positionToSend.x,
|
|
951
|
-
positionToSend.y,
|
|
952
|
-
pressure,
|
|
953
|
-
buttons,
|
|
954
|
-
buttons,
|
|
955
|
-
);
|
|
956
|
-
if (message) {
|
|
957
|
-
debugLog('[rc-touch][mouse->touch] buffer', {
|
|
958
|
-
pointerId,
|
|
959
|
-
actionName: motionActionToString(action),
|
|
960
|
-
byteLength: message.byteLength,
|
|
961
|
-
});
|
|
962
|
-
sendBinaryControlMessage(message);
|
|
963
|
-
}
|
|
964
|
-
} else if (eventType === 'up' || eventType === 'cancel') {
|
|
965
|
-
activePointers.current.delete(pointerId);
|
|
966
|
-
}
|
|
967
|
-
};
|
|
968
|
-
|
|
969
|
-
// Update Alt modifier state. Only iOS Simulator uses Indigo modifier injection.
|
|
970
|
-
const updateAltHeld = (nextHeld: boolean) => {
|
|
971
|
-
if (isAltHeldRef.current === nextHeld) {
|
|
972
|
-
return;
|
|
973
|
-
}
|
|
974
|
-
isAltHeldRef.current = nextHeld;
|
|
975
|
-
setIsAltHeld(nextHeld);
|
|
976
|
-
|
|
977
|
-
// Clear hover point when Alt is released to hide indicators immediately.
|
|
978
|
-
if (!nextHeld) {
|
|
979
|
-
setHoverPoint(null);
|
|
980
|
-
}
|
|
981
|
-
|
|
982
|
-
debugLog('[rc-touch][alt] updateAltHeld', {
|
|
983
|
-
nextHeld,
|
|
984
|
-
activePointerIds: Array.from(activePointers.current.keys()),
|
|
985
|
-
});
|
|
986
|
-
|
|
987
|
-
// iOS Simulator pinch (Option/Alt+drag) behavior depends on the Option modifier being
|
|
988
|
-
// active on the Indigo HID side. Send Alt key down/up immediately on toggle so the
|
|
989
|
-
// sequence matches Simulator.app (Alt down -> mouse down/drag -> mouse up -> Alt up).
|
|
990
|
-
// This is iOS-specific; Android doesn't use this modifier injection.
|
|
991
|
-
if (platform === 'ios' && dataChannelRef.current && dataChannelRef.current.readyState === 'open') {
|
|
992
|
-
const action = nextHeld ? ANDROID_KEYS.ACTION_DOWN : ANDROID_KEYS.ACTION_UP;
|
|
993
|
-
const message = createInjectKeycodeMessage(
|
|
994
|
-
action,
|
|
995
|
-
ANDROID_KEYS.KEYCODE_ALT_LEFT,
|
|
996
|
-
0,
|
|
997
|
-
ANDROID_KEYS.META_NONE,
|
|
998
|
-
);
|
|
999
|
-
debugLog('[rc-touch][alt] sending Indigo modifier keycode', {
|
|
1000
|
-
action,
|
|
1001
|
-
keycode: ANDROID_KEYS.KEYCODE_ALT_LEFT,
|
|
1002
|
-
});
|
|
1003
|
-
if (message) {
|
|
1004
|
-
sendBinaryControlMessage(message);
|
|
1005
|
-
}
|
|
1006
|
-
}
|
|
1007
|
-
};
|
|
1008
|
-
|
|
1009
|
-
// Mapping context computed once per DOM event, then reused for each pointer.
|
|
1010
|
-
type VideoMappingContext = {
|
|
1011
|
-
videoWidth: number;
|
|
1012
|
-
videoHeight: number;
|
|
1013
|
-
videoRect: DOMRect;
|
|
1014
|
-
containerRect: DOMRect;
|
|
1015
|
-
actualWidth: number;
|
|
1016
|
-
actualHeight: number;
|
|
1017
|
-
offsetX: number;
|
|
1018
|
-
offsetY: number;
|
|
1019
|
-
};
|
|
1020
|
-
|
|
1021
|
-
// Compute mapping context from current video/container state (once per event).
|
|
1022
|
-
const computeVideoMappingContext = (): VideoMappingContext | null => {
|
|
1023
|
-
const video = videoRef.current;
|
|
1024
|
-
const container = containerRef.current;
|
|
1025
|
-
if (!video || !container) return null;
|
|
1026
|
-
|
|
1027
|
-
const videoWidth = video.videoWidth;
|
|
1028
|
-
const videoHeight = video.videoHeight;
|
|
1029
|
-
if (!videoWidth || !videoHeight) return null;
|
|
1030
|
-
|
|
1031
|
-
const videoRect = video.getBoundingClientRect();
|
|
1032
|
-
const containerRect = container.getBoundingClientRect();
|
|
1033
|
-
|
|
1034
|
-
const displayWidth = videoRect.width;
|
|
1035
|
-
const displayHeight = videoRect.height;
|
|
1036
|
-
const videoAspectRatio = videoWidth / videoHeight;
|
|
1037
|
-
const containerAspectRatio = displayWidth / displayHeight;
|
|
1038
|
-
|
|
1039
|
-
let actualWidth = displayWidth;
|
|
1040
|
-
let actualHeight = displayHeight;
|
|
1041
|
-
if (videoAspectRatio > containerAspectRatio) {
|
|
1042
|
-
actualHeight = displayWidth / videoAspectRatio;
|
|
1043
|
-
} else {
|
|
1044
|
-
actualWidth = displayHeight * videoAspectRatio;
|
|
1045
|
-
}
|
|
1046
|
-
|
|
1047
|
-
const offsetX = (displayWidth - actualWidth) / 2;
|
|
1048
|
-
const offsetY = (displayHeight - actualHeight) / 2;
|
|
1049
|
-
|
|
1050
|
-
return {
|
|
1051
|
-
videoWidth,
|
|
1052
|
-
videoHeight,
|
|
1053
|
-
videoRect,
|
|
1054
|
-
containerRect,
|
|
1055
|
-
actualWidth,
|
|
1056
|
-
actualHeight,
|
|
1057
|
-
offsetX,
|
|
1058
|
-
offsetY,
|
|
1059
|
-
};
|
|
1060
|
-
};
|
|
1061
|
-
|
|
1062
|
-
// Map a client point to video coordinates using a pre-computed context,
|
|
1063
|
-
// clamping points outside the rendered video to the nearest point on the video.
|
|
1064
|
-
const mapClientPointToVideo = (
|
|
1065
|
-
ctx: VideoMappingContext,
|
|
1066
|
-
clientX: number,
|
|
1067
|
-
clientY: number,
|
|
1068
|
-
): PointerGeometry | null => {
|
|
1069
|
-
const relativeX = clientX - ctx.videoRect.left - ctx.offsetX;
|
|
1070
|
-
const relativeY = clientY - ctx.videoRect.top - ctx.offsetY;
|
|
1071
|
-
|
|
1072
|
-
const clampedRelativeX = Math.max(0, Math.min(ctx.actualWidth, relativeX));
|
|
1073
|
-
const clampedRelativeY = Math.max(0, Math.min(ctx.actualHeight, relativeY));
|
|
1074
|
-
const videoX = Math.max(
|
|
1075
|
-
0,
|
|
1076
|
-
Math.min(ctx.videoWidth, (clampedRelativeX / ctx.actualWidth) * ctx.videoWidth),
|
|
1077
|
-
);
|
|
1078
|
-
const videoY = Math.max(
|
|
1079
|
-
0,
|
|
1080
|
-
Math.min(ctx.videoHeight, (clampedRelativeY / ctx.actualHeight) * ctx.videoHeight),
|
|
1081
|
-
);
|
|
1082
|
-
|
|
1083
|
-
return {
|
|
1084
|
-
videoX,
|
|
1085
|
-
videoY,
|
|
1086
|
-
videoWidth: ctx.videoWidth,
|
|
1087
|
-
videoHeight: ctx.videoHeight,
|
|
1088
|
-
};
|
|
1089
|
-
};
|
|
1090
|
-
|
|
1091
|
-
// Compute full hover point with mirror/container coordinates (for Alt indicator rendering),
|
|
1092
|
-
// clamping points outside the rendered video to the nearest point on the video.
|
|
1093
|
-
const computeFullHoverPoint = (
|
|
1094
|
-
ctx: VideoMappingContext,
|
|
1095
|
-
clientX: number,
|
|
1096
|
-
clientY: number,
|
|
1097
|
-
): HoverPoint | null => {
|
|
1098
|
-
const relativeX = clientX - ctx.videoRect.left - ctx.offsetX;
|
|
1099
|
-
const relativeY = clientY - ctx.videoRect.top - ctx.offsetY;
|
|
1100
|
-
|
|
1101
|
-
const clampedRelativeX = Math.max(0, Math.min(ctx.actualWidth, relativeX));
|
|
1102
|
-
const clampedRelativeY = Math.max(0, Math.min(ctx.actualHeight, relativeY));
|
|
1103
|
-
const videoX = Math.max(
|
|
1104
|
-
0,
|
|
1105
|
-
Math.min(ctx.videoWidth, (clampedRelativeX / ctx.actualWidth) * ctx.videoWidth),
|
|
1106
|
-
);
|
|
1107
|
-
const videoY = Math.max(
|
|
1108
|
-
0,
|
|
1109
|
-
Math.min(ctx.videoHeight, (clampedRelativeY / ctx.actualHeight) * ctx.videoHeight),
|
|
1110
|
-
);
|
|
1111
|
-
const mirrorVideoX = ctx.videoWidth - videoX;
|
|
1112
|
-
const mirrorVideoY = ctx.videoHeight - videoY;
|
|
1113
|
-
|
|
1114
|
-
const contentLeft = ctx.videoRect.left + ctx.offsetX;
|
|
1115
|
-
const contentTop = ctx.videoRect.top + ctx.offsetY;
|
|
1116
|
-
const containerX = contentLeft - ctx.containerRect.left + clampedRelativeX;
|
|
1117
|
-
const containerY = contentTop - ctx.containerRect.top + clampedRelativeY;
|
|
1118
|
-
const mirrorContainerX = contentLeft - ctx.containerRect.left + (ctx.actualWidth - clampedRelativeX);
|
|
1119
|
-
const mirrorContainerY = contentTop - ctx.containerRect.top + (ctx.actualHeight - clampedRelativeY);
|
|
1120
|
-
|
|
1121
|
-
return {
|
|
1122
|
-
containerX,
|
|
1123
|
-
containerY,
|
|
1124
|
-
mirrorContainerX,
|
|
1125
|
-
mirrorContainerY,
|
|
1126
|
-
videoX,
|
|
1127
|
-
videoY,
|
|
1128
|
-
mirrorVideoX,
|
|
1129
|
-
mirrorVideoY,
|
|
1130
|
-
videoWidth: ctx.videoWidth,
|
|
1131
|
-
videoHeight: ctx.videoHeight,
|
|
1132
|
-
};
|
|
1133
|
-
};
|
|
1134
|
-
|
|
1135
|
-
// Helper to send a two-finger touch message (iOS-specific type=18 message).
|
|
1136
|
-
const sendTwoFingerMessage = (
|
|
1137
|
-
action: number,
|
|
1138
|
-
videoWidth: number,
|
|
1139
|
-
videoHeight: number,
|
|
1140
|
-
x0: number,
|
|
1141
|
-
y0: number,
|
|
1142
|
-
x1: number,
|
|
1143
|
-
y1: number,
|
|
1144
|
-
) => {
|
|
1145
|
-
const msg = createTwoFingerTouchControlMessage(action, videoWidth, videoHeight, x0, y0, x1, y1);
|
|
1146
|
-
debugLog('[rc-touch2] sendTwoFingerMessage (iOS)', {
|
|
1147
|
-
actionName: motionActionToString(action),
|
|
1148
|
-
video: { width: videoWidth, height: videoHeight },
|
|
1149
|
-
p0: { x: x0, y: y0 },
|
|
1150
|
-
p1: { x: x1, y: y1 },
|
|
1151
|
-
byteLength: msg.byteLength,
|
|
1152
|
-
});
|
|
1153
|
-
sendBinaryControlMessage(msg);
|
|
1154
|
-
};
|
|
1155
|
-
|
|
1156
|
-
// Generic two-finger event handler - sends platform-appropriate messages.
|
|
1157
|
-
// iOS: uses special two-finger message (type=18)
|
|
1158
|
-
// Android: sends two separate single-touch messages with proper action sequencing
|
|
1159
|
-
const applyTwoFingerEvent = (
|
|
1160
|
-
eventType: 'down' | 'move' | 'up',
|
|
1161
|
-
videoWidth: number,
|
|
1162
|
-
videoHeight: number,
|
|
1163
|
-
x0: number,
|
|
1164
|
-
y0: number,
|
|
1165
|
-
x1: number,
|
|
1166
|
-
y1: number,
|
|
1167
|
-
pointerId0: number,
|
|
1168
|
-
pointerId1: number,
|
|
1169
|
-
) => {
|
|
1170
|
-
debugLog('[rc-touch2] applyTwoFingerEvent', {
|
|
1171
|
-
platform,
|
|
1172
|
-
eventType,
|
|
1173
|
-
video: { width: videoWidth, height: videoHeight },
|
|
1174
|
-
p0: { x: x0, y: y0, id: pointerId0 },
|
|
1175
|
-
p1: { x: x1, y: y1, id: pointerId1 },
|
|
1176
|
-
});
|
|
1177
|
-
|
|
1178
|
-
if (eventType === 'down') {
|
|
1179
|
-
// A real two-finger gesture takes over from any synthesized
|
|
1180
|
-
// wheel-scroll finger.
|
|
1181
|
-
wheelCallbacksRef.current.end();
|
|
1182
|
-
}
|
|
1183
|
-
|
|
1184
|
-
if (platform === 'ios') {
|
|
1185
|
-
// iOS: use special two-finger message (type=18)
|
|
1186
|
-
const action =
|
|
1187
|
-
eventType === 'down' ? AMOTION_EVENT.ACTION_DOWN
|
|
1188
|
-
: eventType === 'move' ? AMOTION_EVENT.ACTION_MOVE
|
|
1189
|
-
: AMOTION_EVENT.ACTION_UP;
|
|
1190
|
-
sendTwoFingerMessage(action, videoWidth, videoHeight, x0, y0, x1, y1);
|
|
1191
|
-
} else {
|
|
1192
|
-
// Android: send two separate single-touch messages with proper action codes
|
|
1193
|
-
// Per scrcpy protocol, each finger is a separate INJECT_TOUCH_EVENT with unique pointerId
|
|
1194
|
-
if (eventType === 'down') {
|
|
1195
|
-
// First finger down (ACTION_DOWN), then second finger down (ACTION_POINTER_DOWN)
|
|
1196
|
-
sendSingleTouch(AMOTION_EVENT.ACTION_DOWN, pointerId0, videoWidth, videoHeight, x0, y0);
|
|
1197
|
-
sendSingleTouch(AMOTION_EVENT.ACTION_POINTER_DOWN, pointerId1, videoWidth, videoHeight, x1, y1);
|
|
1198
|
-
} else if (eventType === 'move') {
|
|
1199
|
-
// Both fingers move (ACTION_MOVE for each)
|
|
1200
|
-
sendSingleTouch(AMOTION_EVENT.ACTION_MOVE, pointerId0, videoWidth, videoHeight, x0, y0);
|
|
1201
|
-
sendSingleTouch(AMOTION_EVENT.ACTION_MOVE, pointerId1, videoWidth, videoHeight, x1, y1);
|
|
1202
|
-
} else {
|
|
1203
|
-
// Second finger up (ACTION_POINTER_UP), then first finger up (ACTION_UP)
|
|
1204
|
-
sendSingleTouch(AMOTION_EVENT.ACTION_POINTER_UP, pointerId1, videoWidth, videoHeight, x1, y1);
|
|
1205
|
-
sendSingleTouch(AMOTION_EVENT.ACTION_UP, pointerId0, videoWidth, videoHeight, x0, y0);
|
|
1206
|
-
}
|
|
1207
|
-
}
|
|
1208
|
-
};
|
|
1209
|
-
|
|
1210
|
-
// Update hover point only when Alt is held (to avoid re-renders in normal path).
|
|
1211
|
-
const updateHoverPoint = (ctx: VideoMappingContext, clientX: number, clientY: number) => {
|
|
1212
|
-
if (!isAltHeldRef.current) {
|
|
1213
|
-
// Don't compute or update when Alt isn't held
|
|
1214
|
-
if (hoverPoint !== null) {
|
|
1215
|
-
setHoverPoint(null);
|
|
1216
|
-
}
|
|
1217
|
-
return;
|
|
1218
|
-
}
|
|
1219
|
-
const fullPoint = computeFullHoverPoint(ctx, clientX, clientY);
|
|
1220
|
-
setHoverPoint(fullPoint);
|
|
1221
|
-
};
|
|
1222
|
-
|
|
1223
|
-
// Map clientX/Y to AX screen-coordinate space using the latest snapshot.
|
|
1224
|
-
// Returns null if there's no snapshot or the click is outside the video.
|
|
1225
|
-
const hitTestAxAtClient = (
|
|
1226
|
-
ctx: VideoMappingContext,
|
|
1227
|
-
clientX: number,
|
|
1228
|
-
clientY: number,
|
|
1229
|
-
): AxElement | null => {
|
|
1230
|
-
const snapshot = axSnapshotRef.current;
|
|
1231
|
-
if (!snapshot || snapshot.screen.width <= 0 || snapshot.screen.height <= 0) return null;
|
|
1232
|
-
const relX = clientX - ctx.videoRect.left - ctx.offsetX;
|
|
1233
|
-
const relY = clientY - ctx.videoRect.top - ctx.offsetY;
|
|
1234
|
-
if (relX < 0 || relY < 0 || relX > ctx.actualWidth || relY > ctx.actualHeight) return null;
|
|
1235
|
-
const axX = (relX / ctx.actualWidth) * snapshot.screen.width;
|
|
1236
|
-
const axY = (relY / ctx.actualHeight) * snapshot.screen.height;
|
|
1237
|
-
return axElementAtPoint(snapshot, axX, axY);
|
|
1238
|
-
};
|
|
1239
|
-
|
|
1240
|
-
// Unified handler for both mouse and touch interactions
|
|
1241
|
-
const handleInteraction = (event: React.MouseEvent | React.TouchEvent) => {
|
|
1242
|
-
event.preventDefault();
|
|
1243
|
-
event.stopPropagation();
|
|
1244
|
-
|
|
1245
|
-
// Compute mapping context once per event (reused for all pointers)
|
|
1246
|
-
const ctx = computeVideoMappingContext();
|
|
1247
|
-
|
|
1248
|
-
// Inspect-mode handling.
|
|
1249
|
-
//
|
|
1250
|
-
// We use JS hit-testing (not box-level onMouseEnter/Leave) as the
|
|
1251
|
-
// single source of truth for which element is under the cursor — it
|
|
1252
|
-
// handles overlapping rectangles deterministically by picking the
|
|
1253
|
-
// smallest matching box. The overlay's InspectBox children no longer
|
|
1254
|
-
// attach hover handlers; they just paint themselves based on the
|
|
1255
|
-
// `highlightedId` prop driven from here.
|
|
1256
|
-
//
|
|
1257
|
-
// Cursor position is tracked in both modes so the cursor-anchored
|
|
1258
|
-
// InfoCard can follow the pointer.
|
|
1259
|
-
const isInspecting = inspectModeRef.current === true || inspectModeRef.current === 'hover-only';
|
|
1260
|
-
if (isInspecting && !('touches' in event)) {
|
|
1261
|
-
if (event.type === 'mousemove') {
|
|
1262
|
-
scheduleCursorFlush({ x: event.clientX, y: event.clientY });
|
|
1263
|
-
if (ctx) {
|
|
1264
|
-
const hit = hitTestAxAtClient(ctx, event.clientX, event.clientY);
|
|
1265
|
-
setAxHighlightedId(hit?.id ?? null);
|
|
1266
|
-
}
|
|
1267
|
-
} else if (event.type === 'mouseleave') {
|
|
1268
|
-
scheduleCursorFlush(null);
|
|
1269
|
-
setAxHighlightedId(null);
|
|
1270
|
-
}
|
|
1271
|
-
}
|
|
1272
|
-
// Select mode blocks device input — clicks/drags don't reach the
|
|
1273
|
-
// simulator. Hover-only mode falls through to the regular path.
|
|
1274
|
-
if (inspectModeRef.current === true) {
|
|
1275
|
-
return;
|
|
1276
|
-
}
|
|
1277
|
-
|
|
1278
|
-
// Handle hover point updates for mouse events (only when Alt is held)
|
|
1279
|
-
if (!('touches' in event) && ctx) {
|
|
1280
|
-
if (event.type === 'mousemove') {
|
|
1281
|
-
updateHoverPoint(ctx, event.clientX, event.clientY);
|
|
1282
|
-
} else if (event.type === 'mouseleave') {
|
|
1283
|
-
setHoverPoint(null);
|
|
1284
|
-
}
|
|
1285
|
-
// Note: Alt state is tracked via global keydown/keyup listeners, not event.altKey,
|
|
1286
|
-
// to ensure consistent behavior across focus transitions.
|
|
1287
|
-
}
|
|
1288
|
-
|
|
1289
|
-
if (
|
|
1290
|
-
!dataChannelRef.current ||
|
|
1291
|
-
dataChannelRef.current.readyState !== 'open' ||
|
|
1292
|
-
!videoRef.current ||
|
|
1293
|
-
!ctx
|
|
1294
|
-
) {
|
|
1295
|
-
return;
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
|
-
// --- Event Type Handling ---
|
|
1299
|
-
|
|
1300
|
-
if ('touches' in event) {
|
|
1301
|
-
// Touch Events - handle both single-finger and two-finger gestures
|
|
1302
|
-
const allTouches = event.touches; // All currently active touches
|
|
1303
|
-
const changedTouches = event.changedTouches;
|
|
1304
|
-
|
|
1305
|
-
let eventType: 'down' | 'move' | 'up' | 'cancel';
|
|
1306
|
-
switch (event.type) {
|
|
1307
|
-
case 'touchstart':
|
|
1308
|
-
eventType = 'down';
|
|
1309
|
-
break;
|
|
1310
|
-
case 'touchmove':
|
|
1311
|
-
eventType = 'move';
|
|
1312
|
-
break;
|
|
1313
|
-
case 'touchend':
|
|
1314
|
-
eventType = 'up';
|
|
1315
|
-
break;
|
|
1316
|
-
case 'touchcancel':
|
|
1317
|
-
eventType = 'cancel';
|
|
1318
|
-
break;
|
|
1319
|
-
default:
|
|
1320
|
-
return;
|
|
1321
|
-
}
|
|
1322
|
-
|
|
1323
|
-
// Check if we have exactly 2 active touches - route to two-finger logic
|
|
1324
|
-
if (allTouches.length === 2) {
|
|
1325
|
-
const t0 = allTouches[0];
|
|
1326
|
-
const t1 = allTouches[1];
|
|
1327
|
-
const g0 = mapClientPointToVideo(ctx, t0.clientX, t0.clientY);
|
|
1328
|
-
const g1 = mapClientPointToVideo(ctx, t1.clientX, t1.clientY);
|
|
1329
|
-
|
|
1330
|
-
if (!g0 || !g1) return;
|
|
1331
|
-
|
|
1332
|
-
if (!twoFingerStateRef.current) {
|
|
1333
|
-
// Starting a new two-finger gesture
|
|
1334
|
-
twoFingerStateRef.current = {
|
|
1335
|
-
finger0: { x: g0.videoX, y: g0.videoY },
|
|
1336
|
-
finger1: { x: g1.videoX, y: g1.videoY },
|
|
1337
|
-
videoSize: { width: g0.videoWidth, height: g0.videoHeight },
|
|
1338
|
-
source: 'real-touch',
|
|
1339
|
-
pointerId0: t0.identifier,
|
|
1340
|
-
pointerId1: t1.identifier,
|
|
1341
|
-
};
|
|
1342
|
-
applyTwoFingerEvent(
|
|
1343
|
-
'down',
|
|
1344
|
-
g0.videoWidth,
|
|
1345
|
-
g0.videoHeight,
|
|
1346
|
-
g0.videoX,
|
|
1347
|
-
g0.videoY,
|
|
1348
|
-
g1.videoX,
|
|
1349
|
-
g1.videoY,
|
|
1350
|
-
t0.identifier,
|
|
1351
|
-
t1.identifier,
|
|
1352
|
-
);
|
|
1353
|
-
} else if (twoFingerStateRef.current.source === 'real-touch') {
|
|
1354
|
-
// Continuing two-finger gesture (move)
|
|
1355
|
-
twoFingerStateRef.current.finger0 = { x: g0.videoX, y: g0.videoY };
|
|
1356
|
-
twoFingerStateRef.current.finger1 = { x: g1.videoX, y: g1.videoY };
|
|
1357
|
-
applyTwoFingerEvent(
|
|
1358
|
-
'move',
|
|
1359
|
-
g0.videoWidth,
|
|
1360
|
-
g0.videoHeight,
|
|
1361
|
-
g0.videoX,
|
|
1362
|
-
g0.videoY,
|
|
1363
|
-
g1.videoX,
|
|
1364
|
-
g1.videoY,
|
|
1365
|
-
twoFingerStateRef.current.pointerId0,
|
|
1366
|
-
twoFingerStateRef.current.pointerId1,
|
|
1367
|
-
);
|
|
1368
|
-
}
|
|
1369
|
-
} else if (allTouches.length < 2 && twoFingerStateRef.current?.source === 'real-touch') {
|
|
1370
|
-
// Finger lifted - end two-finger gesture using last known state
|
|
1371
|
-
const state = twoFingerStateRef.current;
|
|
1372
|
-
applyTwoFingerEvent(
|
|
1373
|
-
'up',
|
|
1374
|
-
state.videoSize.width,
|
|
1375
|
-
state.videoSize.height,
|
|
1376
|
-
state.finger0.x,
|
|
1377
|
-
state.finger0.y,
|
|
1378
|
-
state.finger1.x,
|
|
1379
|
-
state.finger1.y,
|
|
1380
|
-
state.pointerId0,
|
|
1381
|
-
state.pointerId1,
|
|
1382
|
-
);
|
|
1383
|
-
twoFingerStateRef.current = null;
|
|
1384
|
-
// Don't process remaining finger - gesture ended
|
|
1385
|
-
return;
|
|
1386
|
-
} else if (allTouches.length > 2) {
|
|
1387
|
-
// 3+ fingers - not supported, ignore
|
|
1388
|
-
return;
|
|
1389
|
-
} else {
|
|
1390
|
-
// Single finger touch (allTouches is 0 or 1)
|
|
1391
|
-
// Note: allTouches=0 happens on touchend when last finger lifts
|
|
1392
|
-
const touch = changedTouches[0];
|
|
1393
|
-
if (touch) {
|
|
1394
|
-
const geometry = mapClientPointToVideo(ctx, touch.clientX, touch.clientY);
|
|
1395
|
-
applyPointerEvent(touch.identifier, eventType, geometry);
|
|
1396
|
-
}
|
|
1397
|
-
}
|
|
1398
|
-
} else {
|
|
1399
|
-
// Mouse Events
|
|
1400
|
-
const pointerId = -1; // Primary mouse pointer
|
|
1401
|
-
let eventType: 'down' | 'move' | 'up' | 'cancel' | null = null;
|
|
1402
|
-
|
|
1403
|
-
// Determine if we're in two-finger mode (Alt+mouse drag)
|
|
1404
|
-
const inTwoFingerMode = twoFingerStateRef.current?.source === 'alt-mouse';
|
|
1405
|
-
|
|
1406
|
-
switch (event.type) {
|
|
1407
|
-
case 'mousedown':
|
|
1408
|
-
if (event.button === 0) eventType = 'down';
|
|
1409
|
-
break;
|
|
1410
|
-
case 'mousemove':
|
|
1411
|
-
// Process move if either in two-finger mode or has active pointer (normal drag)
|
|
1412
|
-
if (inTwoFingerMode || activePointers.current.has(pointerId)) {
|
|
1413
|
-
eventType = 'move';
|
|
1414
|
-
}
|
|
1415
|
-
break;
|
|
1416
|
-
case 'mouseup':
|
|
1417
|
-
if (event.button === 0) eventType = 'up';
|
|
1418
|
-
break;
|
|
1419
|
-
case 'mouseleave':
|
|
1420
|
-
// Treat leave like up only if in drag/two-finger mode
|
|
1421
|
-
if (inTwoFingerMode || activePointers.current.has(pointerId)) {
|
|
1422
|
-
eventType = 'up';
|
|
1423
|
-
}
|
|
1424
|
-
break;
|
|
1425
|
-
}
|
|
1426
|
-
|
|
1427
|
-
if (eventType) {
|
|
1428
|
-
const geometry = mapClientPointToVideo(ctx, event.clientX, event.clientY);
|
|
1429
|
-
if (!geometry) {
|
|
1430
|
-
return;
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
debugLog('[rc-touch][mouse] event', {
|
|
1434
|
-
domType: event.type,
|
|
1435
|
-
eventType,
|
|
1436
|
-
button: event.button,
|
|
1437
|
-
buttons: (event as React.MouseEvent).buttons,
|
|
1438
|
-
client: { x: event.clientX, y: event.clientY },
|
|
1439
|
-
altHeldRef: isAltHeldRef.current,
|
|
1440
|
-
inTwoFingerMode,
|
|
1441
|
-
geometry: {
|
|
1442
|
-
videoX: geometry.videoX,
|
|
1443
|
-
videoY: geometry.videoY,
|
|
1444
|
-
videoWidth: geometry.videoWidth,
|
|
1445
|
-
videoHeight: geometry.videoHeight,
|
|
1446
|
-
},
|
|
1447
|
-
activePointerIds: Array.from(activePointers.current.keys()),
|
|
1448
|
-
});
|
|
1449
|
-
|
|
1450
|
-
// Route to two-finger (Alt+mouse) or single-finger path
|
|
1451
|
-
if (isAltHeldRef.current || inTwoFingerMode) {
|
|
1452
|
-
// Two-finger mode - Alt simulates second finger at mirror position
|
|
1453
|
-
handleAltMouseGesture(eventType, geometry);
|
|
1454
|
-
} else {
|
|
1455
|
-
// Normal single-finger touch
|
|
1456
|
-
applyPointerEvent(pointerId, eventType, geometry);
|
|
1457
|
-
}
|
|
1458
|
-
}
|
|
1459
|
-
}
|
|
1460
|
-
};
|
|
1461
|
-
|
|
1462
|
-
// Handle Alt+mouse gestures (simulated two-finger with mirror position).
|
|
1463
|
-
// Works on both iOS and Android - applyTwoFingerEvent handles platform differences.
|
|
1464
|
-
const handleAltMouseGesture = (
|
|
1465
|
-
eventType: 'down' | 'move' | 'up' | 'cancel',
|
|
1466
|
-
geometry: PointerGeometry,
|
|
1467
|
-
) => {
|
|
1468
|
-
const { videoX, videoY, videoWidth, videoHeight } = geometry;
|
|
1469
|
-
const mirrorX = videoWidth - videoX;
|
|
1470
|
-
const mirrorY = videoHeight - videoY;
|
|
1471
|
-
|
|
1472
|
-
if (eventType === 'down') {
|
|
1473
|
-
// Start two-finger gesture
|
|
1474
|
-
twoFingerStateRef.current = {
|
|
1475
|
-
finger0: { x: videoX, y: videoY },
|
|
1476
|
-
finger1: { x: mirrorX, y: mirrorY },
|
|
1477
|
-
videoSize: { width: videoWidth, height: videoHeight },
|
|
1478
|
-
source: 'alt-mouse',
|
|
1479
|
-
pointerId0: ALT_POINTER_ID_PRIMARY,
|
|
1480
|
-
pointerId1: ALT_POINTER_ID_MIRROR,
|
|
1481
|
-
};
|
|
1482
|
-
videoRef.current?.focus();
|
|
1483
|
-
applyTwoFingerEvent(
|
|
1484
|
-
'down',
|
|
1485
|
-
videoWidth,
|
|
1486
|
-
videoHeight,
|
|
1487
|
-
videoX,
|
|
1488
|
-
videoY,
|
|
1489
|
-
mirrorX,
|
|
1490
|
-
mirrorY,
|
|
1491
|
-
ALT_POINTER_ID_PRIMARY,
|
|
1492
|
-
ALT_POINTER_ID_MIRROR,
|
|
1493
|
-
);
|
|
1494
|
-
return;
|
|
1495
|
-
}
|
|
1496
|
-
|
|
1497
|
-
if (eventType === 'move') {
|
|
1498
|
-
if (twoFingerStateRef.current?.source === 'alt-mouse') {
|
|
1499
|
-
// Update positions
|
|
1500
|
-
twoFingerStateRef.current.finger0 = { x: videoX, y: videoY };
|
|
1501
|
-
twoFingerStateRef.current.finger1 = { x: mirrorX, y: mirrorY };
|
|
1502
|
-
applyTwoFingerEvent(
|
|
1503
|
-
'move',
|
|
1504
|
-
videoWidth,
|
|
1505
|
-
videoHeight,
|
|
1506
|
-
videoX,
|
|
1507
|
-
videoY,
|
|
1508
|
-
mirrorX,
|
|
1509
|
-
mirrorY,
|
|
1510
|
-
ALT_POINTER_ID_PRIMARY,
|
|
1511
|
-
ALT_POINTER_ID_MIRROR,
|
|
1512
|
-
);
|
|
1513
|
-
}
|
|
1514
|
-
return;
|
|
1515
|
-
}
|
|
1516
|
-
|
|
1517
|
-
if (eventType === 'up' || eventType === 'cancel') {
|
|
1518
|
-
const state = twoFingerStateRef.current;
|
|
1519
|
-
if (state?.source === 'alt-mouse') {
|
|
1520
|
-
// End gesture at last known positions
|
|
1521
|
-
const { finger0, finger1, videoSize } = state;
|
|
1522
|
-
applyTwoFingerEvent(
|
|
1523
|
-
'up',
|
|
1524
|
-
videoSize.width,
|
|
1525
|
-
videoSize.height,
|
|
1526
|
-
finger0.x,
|
|
1527
|
-
finger0.y,
|
|
1528
|
-
finger1.x,
|
|
1529
|
-
finger1.y,
|
|
1530
|
-
ALT_POINTER_ID_PRIMARY,
|
|
1531
|
-
ALT_POINTER_ID_MIRROR,
|
|
1532
|
-
);
|
|
1533
|
-
twoFingerStateRef.current = null;
|
|
1534
|
-
}
|
|
1535
|
-
return;
|
|
1536
|
-
}
|
|
1537
|
-
};
|
|
1538
|
-
|
|
1539
|
-
// Mouse wheel / trackpad scrolling over the stream scrolls inside the
|
|
1540
|
-
// simulator. iOS has no scroll wheel, so we fake a finger: press down
|
|
1541
|
-
// where the cursor is, drag it as wheel events come in, lift when they
|
|
1542
|
-
// stop. If the fake finger reaches the screen edge before the user stops
|
|
1543
|
-
// scrolling, lift and press again to keep going.
|
|
1544
|
-
const WHEEL_END_QUIET_MS = 150;
|
|
1545
|
-
const endWheelGesture = () => {
|
|
1546
|
-
const state = wheelStateRef.current;
|
|
1547
|
-
if (!state) return;
|
|
1548
|
-
window.clearTimeout(state.endTimer);
|
|
1549
|
-
applyPointerEvent(WHEEL_POINTER_ID, 'up', state.pos);
|
|
1550
|
-
wheelStateRef.current = null;
|
|
1551
|
-
};
|
|
1552
|
-
const handleWheel = (event: WheelEvent) => {
|
|
1553
|
-
// Trackpad pinches arrive as ctrl+wheel; don't turn them into scrolls.
|
|
1554
|
-
if (event.ctrlKey) return;
|
|
1555
|
-
// If the user is already touching/dragging (or holding Alt for the
|
|
1556
|
-
// two-finger gesture), let that win - don't add a second fake finger.
|
|
1557
|
-
if (isAltHeldRef.current || twoFingerStateRef.current) return;
|
|
1558
|
-
const pointerCount = activePointers.current.size;
|
|
1559
|
-
if (pointerCount > 1 || (pointerCount === 1 && !activePointers.current.has(WHEEL_POINTER_ID))) return;
|
|
1560
|
-
|
|
1561
|
-
const ctx = computeVideoMappingContext();
|
|
1562
|
-
if (!ctx) return;
|
|
1563
|
-
// Outside the rendered video (frame, letterbox) let the page scroll.
|
|
1564
|
-
const relativeX = event.clientX - ctx.videoRect.left - ctx.offsetX;
|
|
1565
|
-
const relativeY = event.clientY - ctx.videoRect.top - ctx.offsetY;
|
|
1566
|
-
if (relativeX < 0 || relativeX > ctx.actualWidth || relativeY < 0 || relativeY > ctx.actualHeight) {
|
|
1567
|
-
return;
|
|
1568
|
-
}
|
|
1569
|
-
const anchor = mapClientPointToVideo(ctx, event.clientX, event.clientY);
|
|
1570
|
-
if (!anchor) return;
|
|
1571
|
-
event.preventDefault();
|
|
1572
|
-
|
|
1573
|
-
// deltaMode: 0=pixels, 1=lines, 2=pages. Deltas are CSS pixels; convert
|
|
1574
|
-
// to stream pixels so a tick scrolls the same visual distance at any
|
|
1575
|
-
// window size.
|
|
1576
|
-
const scaleX =
|
|
1577
|
-
(event.deltaMode === 1 ? 16
|
|
1578
|
-
: event.deltaMode === 2 ? ctx.actualWidth
|
|
1579
|
-
: 1) *
|
|
1580
|
-
(ctx.videoWidth / ctx.actualWidth);
|
|
1581
|
-
const scaleY =
|
|
1582
|
-
(event.deltaMode === 1 ? 16
|
|
1583
|
-
: event.deltaMode === 2 ? ctx.actualHeight
|
|
1584
|
-
: 1) *
|
|
1585
|
-
(ctx.videoHeight / ctx.actualHeight);
|
|
1586
|
-
const dx = -event.deltaX * scaleX;
|
|
1587
|
-
const dy = -event.deltaY * scaleY;
|
|
1588
|
-
const armEnd = (s: { endTimer: number }) => {
|
|
1589
|
-
window.clearTimeout(s.endTimer);
|
|
1590
|
-
s.endTimer = window.setTimeout(endWheelGesture, WHEEL_END_QUIET_MS);
|
|
1591
|
-
};
|
|
1592
|
-
if (dx === 0 && dy === 0) {
|
|
1593
|
-
// Some wheel events carry no movement (momentum tails); just keep
|
|
1594
|
-
// the gesture alive.
|
|
1595
|
-
if (wheelStateRef.current) armEnd(wheelStateRef.current);
|
|
1596
|
-
return;
|
|
1597
|
-
}
|
|
1598
|
-
|
|
1599
|
-
// Where to press the fake finger down: at the cursor, but nudged away
|
|
1600
|
-
// from the screen edges so the drag has room to travel.
|
|
1601
|
-
const pressPoint = (): PointerGeometry => ({
|
|
1602
|
-
videoWidth: anchor.videoWidth,
|
|
1603
|
-
videoHeight: anchor.videoHeight,
|
|
1604
|
-
videoX:
|
|
1605
|
-
Math.abs(dx) > Math.abs(dy) ?
|
|
1606
|
-
Math.min(Math.max(anchor.videoX, anchor.videoWidth * 0.2), anchor.videoWidth * 0.8)
|
|
1607
|
-
: anchor.videoX,
|
|
1608
|
-
videoY:
|
|
1609
|
-
Math.abs(dy) >= Math.abs(dx) ?
|
|
1610
|
-
Math.min(Math.max(anchor.videoY, anchor.videoHeight * 0.2), anchor.videoHeight * 0.8)
|
|
1611
|
-
: anchor.videoY,
|
|
1612
|
-
});
|
|
1613
|
-
|
|
1614
|
-
const advance = (p: PointerGeometry): PointerGeometry => ({
|
|
1615
|
-
...p,
|
|
1616
|
-
videoX: Math.min(Math.max(p.videoX + dx, 0), p.videoWidth),
|
|
1617
|
-
videoY: Math.min(Math.max(p.videoY + dy, 0), p.videoHeight),
|
|
1618
|
-
});
|
|
1619
|
-
|
|
1620
|
-
let state = wheelStateRef.current;
|
|
1621
|
-
if (!state) {
|
|
1622
|
-
const start = pressPoint();
|
|
1623
|
-
applyPointerEvent(WHEEL_POINTER_ID, 'down', start);
|
|
1624
|
-
state = { pos: start, endTimer: 0 };
|
|
1625
|
-
wheelStateRef.current = state;
|
|
1626
|
-
} else {
|
|
1627
|
-
window.clearTimeout(state.endTimer);
|
|
1628
|
-
}
|
|
1629
|
-
|
|
1630
|
-
let pos = state.pos;
|
|
1631
|
-
let next = advance(pos);
|
|
1632
|
-
if (next.videoX === pos.videoX && next.videoY === pos.videoY) {
|
|
1633
|
-
// Finger reached the screen edge: lift it, press again with room to
|
|
1634
|
-
// travel, and move right away (a press that never moves would count
|
|
1635
|
-
// as a tap).
|
|
1636
|
-
applyPointerEvent(WHEEL_POINTER_ID, 'up', pos);
|
|
1637
|
-
pos = pressPoint();
|
|
1638
|
-
applyPointerEvent(WHEEL_POINTER_ID, 'down', pos);
|
|
1639
|
-
next = advance(pos);
|
|
1640
|
-
}
|
|
1641
|
-
applyPointerEvent(WHEEL_POINTER_ID, 'move', next);
|
|
1642
|
-
state.pos = next;
|
|
1643
|
-
armEnd(state);
|
|
1644
|
-
};
|
|
1645
|
-
useEffect(() => {
|
|
1646
|
-
wheelCallbacksRef.current = { handle: handleWheel, end: endWheelGesture };
|
|
1647
|
-
});
|
|
1648
|
-
|
|
1649
|
-
// React attaches onWheel passively, so preventDefault (needed to keep the
|
|
1650
|
-
// page from scrolling) requires a native non-passive listener. Cleanup
|
|
1651
|
-
// lifts an in-flight synthesized finger before the channel goes away.
|
|
1652
|
-
useEffect(() => {
|
|
1653
|
-
const container = containerRef.current;
|
|
1654
|
-
if (!container) return;
|
|
1655
|
-
const listener = (event: WheelEvent) => wheelCallbacksRef.current.handle(event);
|
|
1656
|
-
container.addEventListener('wheel', listener, { passive: false });
|
|
1657
|
-
return () => {
|
|
1658
|
-
wheelCallbacksRef.current.end();
|
|
1659
|
-
container.removeEventListener('wheel', listener);
|
|
1660
|
-
};
|
|
1661
|
-
}, []);
|
|
1662
|
-
|
|
1663
|
-
useEffect(() => {
|
|
1664
|
-
const handleKeyDown = (event: KeyboardEvent) => {
|
|
1665
|
-
if (event.key === 'Alt') {
|
|
1666
|
-
updateAltHeld(true);
|
|
1667
|
-
}
|
|
1668
|
-
};
|
|
1669
|
-
const handleKeyUp = (event: KeyboardEvent) => {
|
|
1670
|
-
if (event.key === 'Alt') {
|
|
1671
|
-
updateAltHeld(false);
|
|
1672
|
-
}
|
|
1673
|
-
};
|
|
1674
|
-
const handleWindowBlur = () => {
|
|
1675
|
-
updateAltHeld(false);
|
|
1676
|
-
};
|
|
1677
|
-
// Use capture phase so these fire before handleKeyboard's stopPropagation
|
|
1678
|
-
window.addEventListener('keydown', handleKeyDown, true);
|
|
1679
|
-
window.addEventListener('keyup', handleKeyUp, true);
|
|
1680
|
-
window.addEventListener('blur', handleWindowBlur);
|
|
1681
|
-
return () => {
|
|
1682
|
-
window.removeEventListener('keydown', handleKeyDown, true);
|
|
1683
|
-
window.removeEventListener('keyup', handleKeyUp, true);
|
|
1684
|
-
window.removeEventListener('blur', handleWindowBlur);
|
|
1685
|
-
};
|
|
1686
|
-
}, []);
|
|
1687
|
-
|
|
1688
|
-
const handleKeyboard = (event: React.KeyboardEvent) => {
|
|
1689
|
-
event.preventDefault();
|
|
1690
|
-
event.stopPropagation();
|
|
1691
|
-
// Use the wrapper for conditional logging
|
|
1692
|
-
debugLog('Keyboard event:', {
|
|
1693
|
-
type: event.type,
|
|
1694
|
-
key: event.key,
|
|
1695
|
-
keyCode: event.keyCode,
|
|
1696
|
-
code: event.code,
|
|
1697
|
-
target: (event.target as HTMLElement).tagName,
|
|
1698
|
-
focused: document.activeElement === videoRef.current,
|
|
1699
|
-
});
|
|
1700
|
-
|
|
1701
|
-
if (document.activeElement !== videoRef.current) {
|
|
1702
|
-
// Use the wrapper for conditional warning
|
|
1703
|
-
debugWarn('Video element not focused, skipping keyboard event');
|
|
1704
|
-
return;
|
|
1705
|
-
}
|
|
1706
|
-
|
|
1707
|
-
if (!dataChannelRef.current || dataChannelRef.current.readyState !== 'open') {
|
|
1708
|
-
// Use the wrapper for conditional warning
|
|
1709
|
-
debugWarn('Data channel not ready for keyboard event:', dataChannelRef.current?.readyState);
|
|
1710
|
-
return;
|
|
1711
|
-
}
|
|
1712
|
-
|
|
1713
|
-
// Handle special shortcuts first (Paste, Menu)
|
|
1714
|
-
if (event.type === 'keydown') {
|
|
1715
|
-
// Paste (Cmd+V / Ctrl+V)
|
|
1716
|
-
if (event.key.toLowerCase() === 'v' && (event.metaKey || event.ctrlKey)) {
|
|
1717
|
-
debugLog('Paste shortcut detected');
|
|
1718
|
-
navigator.clipboard
|
|
1719
|
-
.readText()
|
|
1720
|
-
.then((text) => {
|
|
1721
|
-
if (text) {
|
|
1722
|
-
debugLog(
|
|
1723
|
-
'Pasting text via SET_CLIPBOARD:',
|
|
1724
|
-
text.substring(0, 20) + (text.length > 20 ? '...' : ''),
|
|
1725
|
-
);
|
|
1726
|
-
const message = createSetClipboardMessage(text, true); // paste=true
|
|
1727
|
-
sendBinaryControlMessage(message);
|
|
1728
|
-
}
|
|
1729
|
-
})
|
|
1730
|
-
.catch((err) => {
|
|
1731
|
-
console.error('Failed to read clipboard contents: ', err);
|
|
1732
|
-
});
|
|
1733
|
-
return; // Don't process 'v' keycode further
|
|
1734
|
-
}
|
|
1735
|
-
|
|
1736
|
-
// Menu (Cmd+M / Ctrl+M) - Send down and up immediately
|
|
1737
|
-
if (event.key.toLowerCase() === 'm' && (event.metaKey || event.ctrlKey)) {
|
|
1738
|
-
debugLog('Menu shortcut detected');
|
|
1739
|
-
const messageDown = createInjectKeycodeMessage(
|
|
1740
|
-
ANDROID_KEYS.ACTION_DOWN,
|
|
1741
|
-
ANDROID_KEYS.MENU,
|
|
1742
|
-
0,
|
|
1743
|
-
ANDROID_KEYS.META_NONE, // Modifiers are handled by the shortcut check, not passed down
|
|
1744
|
-
);
|
|
1745
|
-
sendBinaryControlMessage(messageDown);
|
|
1746
|
-
const messageUp = createInjectKeycodeMessage(
|
|
1747
|
-
ANDROID_KEYS.ACTION_UP,
|
|
1748
|
-
ANDROID_KEYS.MENU,
|
|
1749
|
-
0,
|
|
1750
|
-
ANDROID_KEYS.META_NONE,
|
|
1751
|
-
);
|
|
1752
|
-
sendBinaryControlMessage(messageUp);
|
|
1753
|
-
return; // Don't process 'm' keycode further
|
|
1754
|
-
}
|
|
1755
|
-
}
|
|
1756
|
-
|
|
1757
|
-
// Handle general key presses (including Arrows, Enter, Backspace, Delete, Letters, Numbers, Symbols)
|
|
1758
|
-
const keyInfo = getAndroidKeycodeAndMeta(event);
|
|
1759
|
-
|
|
1760
|
-
if (keyInfo) {
|
|
1761
|
-
const { keycode, metaState } = keyInfo;
|
|
1762
|
-
const action = event.type === 'keydown' ? ANDROID_KEYS.ACTION_DOWN : ANDROID_KEYS.ACTION_UP;
|
|
1763
|
-
|
|
1764
|
-
debugLog(`Sending Keycode: key=${event.key}, code=${keycode}, action=${action}, meta=${metaState}`);
|
|
1765
|
-
|
|
1766
|
-
const message = createInjectKeycodeMessage(
|
|
1767
|
-
action,
|
|
1768
|
-
keycode,
|
|
1769
|
-
0, // repeat count, typically 0 for single presses
|
|
1770
|
-
metaState,
|
|
1771
|
-
);
|
|
1772
|
-
sendBinaryControlMessage(message);
|
|
1773
|
-
} else {
|
|
1774
|
-
debugLog(`Ignoring unhandled key event: type=${event.type}, key=${event.key}`);
|
|
1775
|
-
}
|
|
1776
|
-
};
|
|
1777
|
-
|
|
1778
|
-
const sendKeepAlive = () => {
|
|
1779
|
-
if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
|
|
1780
|
-
wsRef.current.send(
|
|
1781
|
-
JSON.stringify({
|
|
1782
|
-
type: 'keepAlive',
|
|
1783
|
-
sessionId: sessionId,
|
|
1784
|
-
}),
|
|
1785
|
-
);
|
|
1786
|
-
}
|
|
1787
|
-
};
|
|
1788
|
-
|
|
1789
|
-
const startKeepAlive = () => {
|
|
1790
|
-
if (keepAliveIntervalRef.current) {
|
|
1791
|
-
window.clearInterval(keepAliveIntervalRef.current);
|
|
1792
|
-
}
|
|
1793
|
-
keepAliveIntervalRef.current = window.setInterval(sendKeepAlive, 10000);
|
|
1794
|
-
};
|
|
1795
|
-
|
|
1796
|
-
const stopKeepAlive = () => {
|
|
1797
|
-
if (keepAliveIntervalRef.current) {
|
|
1798
|
-
window.clearInterval(keepAliveIntervalRef.current);
|
|
1799
|
-
keepAliveIntervalRef.current = undefined;
|
|
1800
|
-
}
|
|
1801
|
-
};
|
|
1802
|
-
|
|
1803
|
-
const clearScheduledRetry = () => {
|
|
1804
|
-
if (retryTimeoutRef.current) {
|
|
1805
|
-
window.clearTimeout(retryTimeoutRef.current);
|
|
1806
|
-
retryTimeoutRef.current = undefined;
|
|
1807
|
-
}
|
|
1808
|
-
};
|
|
1809
|
-
|
|
1810
|
-
const clearConnectionSuccessTimeout = () => {
|
|
1811
|
-
if (connectionSuccessTimeoutRef.current) {
|
|
1812
|
-
window.clearTimeout(connectionSuccessTimeoutRef.current);
|
|
1813
|
-
connectionSuccessTimeoutRef.current = undefined;
|
|
1814
|
-
}
|
|
1815
|
-
};
|
|
1816
|
-
|
|
1817
|
-
const stopRequestFrameLoop = () => {
|
|
1818
|
-
if (requestFrameIntervalRef.current) {
|
|
1819
|
-
window.clearInterval(requestFrameIntervalRef.current);
|
|
1820
|
-
requestFrameIntervalRef.current = undefined;
|
|
1821
|
-
}
|
|
1822
|
-
};
|
|
1823
|
-
|
|
1824
|
-
const clearIceDisconnectedGrace = () => {
|
|
1825
|
-
if (iceDisconnectedGraceRef.current !== undefined) {
|
|
1826
|
-
window.clearTimeout(iceDisconnectedGraceRef.current);
|
|
1827
|
-
iceDisconnectedGraceRef.current = undefined;
|
|
1828
|
-
}
|
|
1829
|
-
};
|
|
1830
|
-
|
|
1831
|
-
const markFirstFrameShown = () => {
|
|
1832
|
-
if (firstFrameShownRef.current) {
|
|
1833
|
-
return;
|
|
1834
|
-
}
|
|
1835
|
-
firstFrameShownRef.current = true;
|
|
1836
|
-
stopRequestFrameLoop();
|
|
1837
|
-
setVideoLoaded(true);
|
|
1838
|
-
reportConnectionState('connected');
|
|
1839
|
-
};
|
|
1840
|
-
|
|
1841
|
-
// Stop every track on a MediaStream and release the device handle.
|
|
1842
|
-
// The browser keeps the camera "on" indicator lit until at least
|
|
1843
|
-
// one track on the underlying source is stopped, so we have to
|
|
1844
|
-
// call stop() explicitly — closing the peer connection alone
|
|
1845
|
-
// won't do it.
|
|
1846
|
-
const stopMediaStream = (stream: MediaStream) => {
|
|
1847
|
-
for (const track of stream.getTracks()) {
|
|
1848
|
-
try {
|
|
1849
|
-
track.stop();
|
|
1850
|
-
} catch (err) {
|
|
1851
|
-
debugWarn('track.stop() failed:', err);
|
|
1852
|
-
}
|
|
1853
|
-
}
|
|
1854
|
-
};
|
|
1855
|
-
|
|
1856
|
-
// Stop and forget any currently-attached outbound camera stream.
|
|
1857
|
-
// Safe to call when nothing is attached. Does not touch the
|
|
1858
|
-
// sender — the caller is responsible for `replaceTrack(null)`
|
|
1859
|
-
// separately when it wants the SDP slot itself empty.
|
|
1860
|
-
const stopOutboundLocalStream = () => {
|
|
1861
|
-
const stream = outboundLocalStreamRef.current;
|
|
1862
|
-
if (!stream) return;
|
|
1863
|
-
outboundLocalStreamRef.current = null;
|
|
1864
|
-
stopMediaStream(stream);
|
|
1865
|
-
};
|
|
1866
|
-
|
|
1867
|
-
// Runs one attach/detach operation after all previously queued ones,
|
|
1868
|
-
// so two operations' awaits never interleave on the shared sender (a
|
|
1869
|
-
// stale detach could otherwise clear a fresh attach's track, or send
|
|
1870
|
-
// `microphoneStop` after its `microphoneStart`).
|
|
1871
|
-
const runMicOp = (op: () => Promise<void>): Promise<void> => {
|
|
1872
|
-
const next = micOpQueueRef.current.then(op).catch((err) => {
|
|
1873
|
-
debugWarn('microphone operation failed:', err);
|
|
1874
|
-
});
|
|
1875
|
-
micOpQueueRef.current = next;
|
|
1876
|
-
return next;
|
|
1877
|
-
};
|
|
1878
|
-
|
|
1879
|
-
const isMicrophoneWanted = () => microphoneEnabledRef.current || microphoneDemandActiveRef.current;
|
|
1880
|
-
|
|
1881
|
-
const sendMicrophoneStart = (ws: WebSocket) => {
|
|
1882
|
-
const id = `ui-mic-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
|
|
1883
|
-
micStartPendingIdRef.current = id;
|
|
1884
|
-
ws.send(JSON.stringify({ type: 'microphoneStart', id }));
|
|
1885
|
-
};
|
|
1886
|
-
|
|
1887
|
-
// Capture the user's microphone and stream it to the simulator's
|
|
1888
|
-
// mock microphone. Safe to call redundantly: bails when already
|
|
1889
|
-
// attached, when the connection isn't up yet (the ICE-connected
|
|
1890
|
-
// handler re-invokes it), or when the prop flipped off mid-prompt.
|
|
1891
|
-
// Does NOT bump the generation: a redundant call (e.g. the ICE
|
|
1892
|
-
// handler firing on both `connected` and `completed`) must not
|
|
1893
|
-
// cancel an attach already in flight; only detach/teardown cancel.
|
|
1894
|
-
const attachMicrophone = (): Promise<void> => {
|
|
1895
|
-
// No platform gate: attachMicrophoneOp bails when the session never
|
|
1896
|
-
// allocated a sendonly mic transceiver (non-capable servers).
|
|
1897
|
-
return runMicOp(() => attachMicrophoneOp());
|
|
1898
|
-
};
|
|
1899
|
-
|
|
1900
|
-
const attachMicrophoneOp = async () => {
|
|
1901
|
-
const generation = micGenerationRef.current;
|
|
1902
|
-
const isCurrent = () => generation === micGenerationRef.current && isMicrophoneWanted();
|
|
1903
|
-
const reportMicError = (error: string) => {
|
|
1904
|
-
safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
|
|
1905
|
-
active: false,
|
|
1906
|
-
error,
|
|
1907
|
-
});
|
|
1908
|
-
};
|
|
1909
|
-
if (!isCurrent()) return;
|
|
1910
|
-
const ws = wsRef.current;
|
|
1911
|
-
const sender = outboundMicSenderRef.current;
|
|
1912
|
-
if (!ws || ws.readyState !== WebSocket.OPEN || !sender) {
|
|
1913
|
-
// Not connected yet — the post-connect hook retries while the
|
|
1914
|
-
// prop is still on.
|
|
1915
|
-
return;
|
|
1916
|
-
}
|
|
1917
|
-
if (outboundMicStreamRef.current) return;
|
|
1918
|
-
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
1919
|
-
// eslint-disable-next-line no-console
|
|
1920
|
-
console.warn(
|
|
1921
|
-
'[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
|
|
1922
|
-
'getUserMedia requires a secure context (https or http://localhost).',
|
|
1923
|
-
);
|
|
1924
|
-
reportMicError('Microphone capture requires a secure context (https)');
|
|
1925
|
-
return;
|
|
1926
|
-
}
|
|
1927
|
-
let stream: MediaStream | null = null;
|
|
1928
|
-
try {
|
|
1929
|
-
stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
|
1930
|
-
} catch (err) {
|
|
1931
|
-
// eslint-disable-next-line no-console
|
|
1932
|
-
console.warn('[RemoteControl] microphone getUserMedia denied/failed:', err);
|
|
1933
|
-
}
|
|
1934
|
-
if (!isCurrent()) {
|
|
1935
|
-
if (stream) stopMediaStream(stream);
|
|
1936
|
-
return;
|
|
1937
|
-
}
|
|
1938
|
-
if (!stream) {
|
|
1939
|
-
reportMicError('Microphone permission denied or no device available');
|
|
1940
|
-
return;
|
|
1941
|
-
}
|
|
1942
|
-
const audioTrack = stream.getAudioTracks()[0] ?? null;
|
|
1943
|
-
if (!audioTrack) {
|
|
1944
|
-
stopMediaStream(stream);
|
|
1945
|
-
reportMicError('Captured stream has no audio track');
|
|
1946
|
-
return;
|
|
1947
|
-
}
|
|
1948
|
-
try {
|
|
1949
|
-
await sender.replaceTrack(audioTrack);
|
|
1950
|
-
} catch (err) {
|
|
1951
|
-
debugWarn('replaceTrack(audioTrack) failed:', err);
|
|
1952
|
-
stopMediaStream(stream);
|
|
1953
|
-
reportMicError('Failed to attach the microphone track to the connection');
|
|
1954
|
-
return;
|
|
1955
|
-
}
|
|
1956
|
-
if (!isCurrent()) {
|
|
1957
|
-
try {
|
|
1958
|
-
await sender.replaceTrack(null);
|
|
1959
|
-
} catch (err) {
|
|
1960
|
-
debugWarn('replaceTrack(null) on stale mic attach failed:', err);
|
|
1961
|
-
}
|
|
1962
|
-
stopMediaStream(stream);
|
|
1963
|
-
return;
|
|
1964
|
-
}
|
|
1965
|
-
outboundMicStreamRef.current = stream;
|
|
1966
|
-
// If the browser revokes the track later (user clicks Stop in
|
|
1967
|
-
// the tab's mic indicator, device unplugged, ...) tear down and
|
|
1968
|
-
// tell the host, so the sim's mic goes back to silence instead
|
|
1969
|
-
// of freezing on the last buffer.
|
|
1970
|
-
audioTrack.onended = () => {
|
|
1971
|
-
if (outboundMicStreamRef.current !== stream) return;
|
|
1972
|
-
void detachMicrophone('microphone capture ended by the browser');
|
|
1973
|
-
};
|
|
1974
|
-
sendMicrophoneStart(ws);
|
|
1975
|
-
safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
|
|
1976
|
-
active: true,
|
|
1977
|
-
label: audioTrack.label || undefined,
|
|
1978
|
-
});
|
|
1979
|
-
};
|
|
1980
|
-
|
|
1981
|
-
// Stop the live microphone: drop the track from the sender, stop
|
|
1982
|
-
// the capture (turning the browser's mic indicator off) and tell
|
|
1983
|
-
// the host to switch its audio source back to silence. Safe to
|
|
1984
|
-
// call when nothing is attached.
|
|
1985
|
-
const detachMicrophone = (error?: string): Promise<void> => {
|
|
1986
|
-
// Bump synchronously (not on the queue) so an attach that is
|
|
1987
|
-
// suspended on an await bails at its next checkpoint even while
|
|
1988
|
-
// it still owns the queue.
|
|
1989
|
-
++micGenerationRef.current;
|
|
1990
|
-
micStartPendingIdRef.current = null;
|
|
1991
|
-
return runMicOp(async () => {
|
|
1992
|
-
const sender = outboundMicSenderRef.current;
|
|
1993
|
-
const stream = outboundMicStreamRef.current;
|
|
1994
|
-
outboundMicStreamRef.current = null;
|
|
1995
|
-
if (sender && stream) {
|
|
1996
|
-
try {
|
|
1997
|
-
await sender.replaceTrack(null);
|
|
1998
|
-
} catch (err) {
|
|
1999
|
-
debugWarn('replaceTrack(null) on mic detach failed:', err);
|
|
2000
|
-
}
|
|
2001
|
-
}
|
|
2002
|
-
if (stream) stopMediaStream(stream);
|
|
2003
|
-
const ws = wsRef.current;
|
|
2004
|
-
if (stream && ws && ws.readyState === WebSocket.OPEN) {
|
|
2005
|
-
const id = `ui-mic-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
|
|
2006
|
-
ws.send(JSON.stringify({ type: 'microphoneStop', id }));
|
|
2007
|
-
}
|
|
2008
|
-
if (stream || error) {
|
|
2009
|
-
safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
|
|
2010
|
-
active: false,
|
|
2011
|
-
...(error !== undefined && { error }),
|
|
2012
|
-
});
|
|
2013
|
-
}
|
|
2014
|
-
});
|
|
2015
|
-
};
|
|
2016
|
-
|
|
2017
|
-
// Publish a fresh settings snapshot after every successful capture
|
|
2018
|
-
// change. This is status metadata; host output sizing is frame-derived.
|
|
2019
|
-
const sendGrantedCameraResult = (ws: WebSocket, videoTrack: MediaStreamTrack) => {
|
|
2020
|
-
let result;
|
|
2021
|
-
try {
|
|
2022
|
-
result = createGrantedCameraResult(videoTrack);
|
|
2023
|
-
// eslint-disable-next-line no-console
|
|
2024
|
-
console.info(
|
|
2025
|
-
`[RemoteControl] camera capture: ${result.camera?.width}x${result.camera?.height}` +
|
|
2026
|
-
` @ ${result.camera?.frameRate ?? '?'}fps` +
|
|
2027
|
-
(result.camera?.label ? ` — ${result.camera.label}` : ''),
|
|
2028
|
-
);
|
|
2029
|
-
} catch (err) {
|
|
2030
|
-
debugWarn('getSettings() on outbound camera track failed:', err);
|
|
2031
|
-
result = {
|
|
2032
|
-
type: 'cameraResult' as const,
|
|
2033
|
-
granted: true,
|
|
2034
|
-
camera: {
|
|
2035
|
-
label: videoTrack.label || undefined,
|
|
2036
|
-
},
|
|
2037
|
-
};
|
|
2038
|
-
}
|
|
2039
|
-
ws.send(JSON.stringify(result));
|
|
2040
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, result.camera);
|
|
2041
|
-
};
|
|
2042
|
-
|
|
2043
|
-
// Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
|
|
2044
|
-
// into a short uppercase label suitable for a status badge.
|
|
2045
|
-
const shortenCodecMime = (mime: string | undefined): string | undefined => {
|
|
2046
|
-
if (!mime) return undefined;
|
|
2047
|
-
const slash = mime.indexOf('/');
|
|
2048
|
-
const tail = slash >= 0 ? mime.slice(slash + 1) : mime;
|
|
2049
|
-
const upper = tail.toUpperCase();
|
|
2050
|
-
if (upper === 'HEVC') return 'H265';
|
|
2051
|
-
return upper;
|
|
2052
|
-
};
|
|
2053
|
-
|
|
2054
|
-
// Sample outbound-camera stats once and push a normalised
|
|
2055
|
-
// snapshot through `onCameraStats`. Skips silently if the sender
|
|
2056
|
-
// has been torn down between intervals.
|
|
2057
|
-
const sampleOutboundCameraStats = async () => {
|
|
2058
|
-
const sender = outboundCameraSenderRef.current;
|
|
2059
|
-
const handler = onCameraStatsRef.current;
|
|
2060
|
-
if (!sender || !handler) return;
|
|
2061
|
-
let report: RTCStatsReport;
|
|
2062
|
-
try {
|
|
2063
|
-
report = await sender.getStats();
|
|
2064
|
-
} catch {
|
|
2065
|
-
return;
|
|
2066
|
-
}
|
|
2067
|
-
let outbound: any | undefined;
|
|
2068
|
-
let codecMime: string | undefined;
|
|
2069
|
-
let remoteInbound: any | undefined;
|
|
2070
|
-
report.forEach((entry: any) => {
|
|
2071
|
-
if (entry.type === 'outbound-rtp' && entry.kind === 'video') {
|
|
2072
|
-
outbound = entry;
|
|
2073
|
-
} else if (entry.type === 'remote-inbound-rtp' && entry.kind === 'video') {
|
|
2074
|
-
remoteInbound = entry;
|
|
2075
|
-
}
|
|
2076
|
-
});
|
|
2077
|
-
if (outbound?.codecId) {
|
|
2078
|
-
const codec = report.get(outbound.codecId);
|
|
2079
|
-
if (codec) codecMime = (codec as any).mimeType;
|
|
2080
|
-
}
|
|
2081
|
-
if (!outbound) return;
|
|
2082
|
-
const now = (outbound.timestamp as number) ?? Date.now();
|
|
2083
|
-
const prev = cameraStatsPrevRef.current;
|
|
2084
|
-
let fps: number | undefined;
|
|
2085
|
-
let bitrate: number | undefined;
|
|
2086
|
-
let lossPct: number | undefined;
|
|
2087
|
-
if (prev && now > prev.timestamp) {
|
|
2088
|
-
const dt = (now - prev.timestamp) / 1000;
|
|
2089
|
-
if (typeof outbound.framesEncoded === 'number' && typeof prev.framesEncoded === 'number') {
|
|
2090
|
-
fps = Math.max(0, (outbound.framesEncoded - prev.framesEncoded) / dt);
|
|
2091
|
-
}
|
|
2092
|
-
if (typeof outbound.bytesSent === 'number' && typeof prev.bytesSent === 'number') {
|
|
2093
|
-
bitrate = Math.max(0, ((outbound.bytesSent - prev.bytesSent) * 8) / dt);
|
|
2094
|
-
}
|
|
2095
|
-
if (
|
|
2096
|
-
typeof outbound.packetsSent === 'number' &&
|
|
2097
|
-
typeof prev.packetsSent === 'number' &&
|
|
2098
|
-
remoteInbound &&
|
|
2099
|
-
typeof remoteInbound.packetsLost === 'number' &&
|
|
2100
|
-
typeof prev.packetsLost === 'number'
|
|
2101
|
-
) {
|
|
2102
|
-
const sent = outbound.packetsSent - prev.packetsSent;
|
|
2103
|
-
const lost = remoteInbound.packetsLost - prev.packetsLost;
|
|
2104
|
-
if (sent > 0) lossPct = Math.max(0, Math.min(100, (lost / sent) * 100));
|
|
2105
|
-
}
|
|
2106
|
-
}
|
|
2107
|
-
cameraStatsPrevRef.current = {
|
|
2108
|
-
timestamp: now,
|
|
2109
|
-
framesEncoded: outbound.framesEncoded,
|
|
2110
|
-
bytesSent: outbound.bytesSent,
|
|
2111
|
-
packetsSent: outbound.packetsSent,
|
|
2112
|
-
packetsLost: remoteInbound?.packetsLost,
|
|
2113
|
-
};
|
|
2114
|
-
const stats: CameraStreamStats = {
|
|
2115
|
-
codec: shortenCodecMime(codecMime),
|
|
2116
|
-
encoderImplementation: outbound.encoderImplementation,
|
|
2117
|
-
hardwareAccelerated:
|
|
2118
|
-
typeof outbound.powerEfficientEncoder === 'boolean' ? outbound.powerEfficientEncoder : undefined,
|
|
2119
|
-
framesPerSecond: fps,
|
|
2120
|
-
framesEncoded: outbound.framesEncoded,
|
|
2121
|
-
bitrateBps: bitrate,
|
|
2122
|
-
width: outbound.frameWidth,
|
|
2123
|
-
height: outbound.frameHeight,
|
|
2124
|
-
qualityLimitationReason: outbound.qualityLimitationReason,
|
|
2125
|
-
rttMs:
|
|
2126
|
-
typeof remoteInbound?.roundTripTime === 'number' ? remoteInbound.roundTripTime * 1000 : undefined,
|
|
2127
|
-
packetsLostPct: lossPct,
|
|
2128
|
-
};
|
|
2129
|
-
safeInvoke('onCameraStats', onCameraStatsRef.current, stats);
|
|
2130
|
-
};
|
|
2131
|
-
|
|
2132
|
-
const startCameraStatsPoller = () => {
|
|
2133
|
-
if (cameraStatsTimerRef.current !== null) return;
|
|
2134
|
-
cameraStatsPrevRef.current = null;
|
|
2135
|
-
// First sample fires after one interval — the very first
|
|
2136
|
-
// sample has no deltas to compare against, which is fine: the
|
|
2137
|
-
// codec/encoder identity is still useful on its own.
|
|
2138
|
-
cameraStatsTimerRef.current = setInterval(() => {
|
|
2139
|
-
void sampleOutboundCameraStats();
|
|
2140
|
-
}, 1000);
|
|
2141
|
-
};
|
|
2142
|
-
|
|
2143
|
-
const stopCameraStatsPoller = () => {
|
|
2144
|
-
if (cameraStatsTimerRef.current !== null) {
|
|
2145
|
-
clearInterval(cameraStatsTimerRef.current);
|
|
2146
|
-
cameraStatsTimerRef.current = null;
|
|
2147
|
-
}
|
|
2148
|
-
cameraStatsPrevRef.current = null;
|
|
2149
|
-
safeInvoke('onCameraStats', onCameraStatsRef.current, null);
|
|
2150
|
-
};
|
|
2151
|
-
|
|
2152
|
-
const handleCameraRequest = (
|
|
2153
|
-
request: CameraRequest,
|
|
2154
|
-
ws: WebSocket,
|
|
2155
|
-
sender: RTCRtpSender,
|
|
2156
|
-
isCurrentAttempt: () => boolean,
|
|
2157
|
-
): Promise<void> => {
|
|
2158
|
-
// `enqueue` increments its generation synchronously, so suspended
|
|
2159
|
-
// permission work becomes stale before the newer operation starts.
|
|
2160
|
-
return cameraOperations.enqueue(async ({ isCurrent }) => {
|
|
2161
|
-
const isCurrentCameraRequest = () =>
|
|
2162
|
-
isCurrent() &&
|
|
2163
|
-
isCurrentAttempt() &&
|
|
2164
|
-
wsRef.current === ws &&
|
|
2165
|
-
outboundCameraSenderRef.current === sender;
|
|
2166
|
-
if (!isCurrentCameraRequest()) return;
|
|
2167
|
-
|
|
2168
|
-
if (!request.active) {
|
|
2169
|
-
const stream = outboundLocalStreamRef.current;
|
|
2170
|
-
try {
|
|
2171
|
-
await sender.replaceTrack(null);
|
|
2172
|
-
} catch (err) {
|
|
2173
|
-
debugWarn('replaceTrack(null) on camera detach failed:', err);
|
|
2174
|
-
}
|
|
2175
|
-
if (outboundLocalStreamRef.current === stream) {
|
|
2176
|
-
outboundLocalStreamRef.current = null;
|
|
2177
|
-
outboundCameraRequestRef.current = undefined;
|
|
2178
|
-
if (stream) stopMediaStream(stream);
|
|
2179
|
-
}
|
|
2180
|
-
stopCameraStatsPoller();
|
|
2181
|
-
if (!isCurrentCameraRequest()) return;
|
|
2182
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
|
|
2183
|
-
return;
|
|
2184
|
-
}
|
|
2185
|
-
|
|
2186
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
|
|
2187
|
-
const currentStream = outboundLocalStreamRef.current;
|
|
2188
|
-
const currentTrack = currentStream?.getVideoTracks()[0];
|
|
2189
|
-
const currentRequest = outboundCameraRequestRef.current;
|
|
2190
|
-
|
|
2191
|
-
if (
|
|
2192
|
-
currentTrack?.readyState === 'live' &&
|
|
2193
|
-
currentRequest &&
|
|
2194
|
-
!shouldReacquireCamera(currentRequest, request)
|
|
2195
|
-
) {
|
|
2196
|
-
outboundCameraRequestRef.current = request;
|
|
2197
|
-
sendGrantedCameraResult(ws, currentTrack);
|
|
2198
|
-
startCameraStatsPoller();
|
|
2199
|
-
return;
|
|
2200
|
-
}
|
|
2201
|
-
|
|
2202
|
-
// A facing change always restarts browser capture. Releasing first
|
|
2203
|
-
// lets mobile browsers open the opposite camera; one-camera laptops
|
|
2204
|
-
// naturally return the same available input for the new ideal.
|
|
2205
|
-
if (currentStream) {
|
|
2206
|
-
try {
|
|
2207
|
-
await sender.replaceTrack(null);
|
|
2208
|
-
} catch (err) {
|
|
2209
|
-
debugWarn('replaceTrack(null) before camera switch failed:', err);
|
|
2210
|
-
}
|
|
2211
|
-
if (outboundLocalStreamRef.current === currentStream) {
|
|
2212
|
-
outboundLocalStreamRef.current = null;
|
|
2213
|
-
outboundCameraRequestRef.current = undefined;
|
|
2214
|
-
stopMediaStream(currentStream);
|
|
2215
|
-
}
|
|
2216
|
-
stopCameraStatsPoller();
|
|
2217
|
-
if (!isCurrentCameraRequest()) return;
|
|
2218
|
-
}
|
|
2219
|
-
|
|
2220
|
-
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
2221
|
-
// eslint-disable-next-line no-console
|
|
2222
|
-
console.warn(
|
|
2223
|
-
'[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
|
|
2224
|
-
'getUserMedia requires a secure context (https or http://localhost). ' +
|
|
2225
|
-
'Replying cameraResult granted=false so limulator falls back to black frames.',
|
|
2226
|
-
);
|
|
2227
|
-
if (!isCurrentCameraRequest()) return;
|
|
2228
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2229
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2230
|
-
return;
|
|
2231
|
-
}
|
|
2232
|
-
|
|
2233
|
-
let stream: MediaStream | null = null;
|
|
2234
|
-
try {
|
|
2235
|
-
stream = await navigator.mediaDevices.getUserMedia({
|
|
2236
|
-
video: cameraCaptureConstraints(request),
|
|
2237
|
-
audio: false,
|
|
2238
|
-
});
|
|
2239
|
-
} catch (err) {
|
|
2240
|
-
// eslint-disable-next-line no-console
|
|
2241
|
-
console.warn('[RemoteControl] getUserMedia denied/failed:', err);
|
|
2242
|
-
}
|
|
2243
|
-
if (!isCurrentCameraRequest()) {
|
|
2244
|
-
if (stream) stopMediaStream(stream);
|
|
2245
|
-
return;
|
|
2246
|
-
}
|
|
2247
|
-
if (!stream) {
|
|
2248
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2249
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2250
|
-
return;
|
|
2251
|
-
}
|
|
2252
|
-
|
|
2253
|
-
const videoTrack = stream.getVideoTracks()[0] ?? null;
|
|
2254
|
-
if (!videoTrack) {
|
|
2255
|
-
stopMediaStream(stream);
|
|
2256
|
-
if (!isCurrentCameraRequest()) return;
|
|
2257
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2258
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2259
|
-
return;
|
|
2260
|
-
}
|
|
2261
|
-
|
|
2262
|
-
try {
|
|
2263
|
-
videoTrack.contentHint = 'motion';
|
|
2264
|
-
} catch {
|
|
2265
|
-
/* older browsers don't support contentHint; ignore */
|
|
2266
|
-
}
|
|
2267
|
-
|
|
2268
|
-
try {
|
|
2269
|
-
await sender.replaceTrack(videoTrack);
|
|
2270
|
-
} catch (err) {
|
|
2271
|
-
debugWarn('replaceTrack(videoTrack) failed:', err);
|
|
2272
|
-
stopMediaStream(stream);
|
|
2273
|
-
if (!isCurrentCameraRequest()) return;
|
|
2274
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2275
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2276
|
-
return;
|
|
2277
|
-
}
|
|
2278
|
-
if (!isCurrentCameraRequest()) {
|
|
2279
|
-
try {
|
|
2280
|
-
await sender.replaceTrack(null);
|
|
2281
|
-
} catch (err) {
|
|
2282
|
-
debugWarn('replaceTrack(null) for stale camera attach failed:', err);
|
|
2283
|
-
}
|
|
2284
|
-
stopMediaStream(stream);
|
|
2285
|
-
return;
|
|
2286
|
-
}
|
|
2287
|
-
|
|
2288
|
-
try {
|
|
2289
|
-
const params = sender.getParameters();
|
|
2290
|
-
if (!params.encodings || params.encodings.length === 0) {
|
|
2291
|
-
params.encodings = [{}];
|
|
2292
|
-
}
|
|
2293
|
-
params.encodings[0].maxBitrate = 8_000_000;
|
|
2294
|
-
params.encodings[0].maxFramerate = 30;
|
|
2295
|
-
params.degradationPreference = 'maintain-framerate';
|
|
2296
|
-
await sender.setParameters(params);
|
|
2297
|
-
} catch (err) {
|
|
2298
|
-
debugWarn('setParameters on outbound camera failed:', err);
|
|
2299
|
-
}
|
|
2300
|
-
if (!isCurrentCameraRequest()) {
|
|
2301
|
-
try {
|
|
2302
|
-
await sender.replaceTrack(null);
|
|
2303
|
-
} catch (err) {
|
|
2304
|
-
debugWarn('replaceTrack(null) for stale camera finalize failed:', err);
|
|
2305
|
-
}
|
|
2306
|
-
stopMediaStream(stream);
|
|
2307
|
-
return;
|
|
2308
|
-
}
|
|
2309
|
-
|
|
2310
|
-
outboundLocalStreamRef.current = stream;
|
|
2311
|
-
outboundCameraRequestRef.current = request;
|
|
2312
|
-
|
|
2313
|
-
videoTrack.onended = () => {
|
|
2314
|
-
if (outboundLocalStreamRef.current !== stream) return;
|
|
2315
|
-
void cameraOperations
|
|
2316
|
-
.enqueueCurrent(async ({ isCurrent: isCurrentEndedCleanup }) => {
|
|
2317
|
-
if (
|
|
2318
|
-
!isCurrentEndedCleanup() ||
|
|
2319
|
-
outboundLocalStreamRef.current !== stream ||
|
|
2320
|
-
!isCurrentAttempt() ||
|
|
2321
|
-
wsRef.current !== ws ||
|
|
2322
|
-
outboundCameraSenderRef.current !== sender
|
|
2323
|
-
) {
|
|
2324
|
-
return;
|
|
2325
|
-
}
|
|
2326
|
-
try {
|
|
2327
|
-
await sender.replaceTrack(null);
|
|
2328
|
-
} catch (err) {
|
|
2329
|
-
debugWarn('replaceTrack(null) after camera ended failed:', err);
|
|
2330
|
-
}
|
|
2331
|
-
if (
|
|
2332
|
-
!isCurrentEndedCleanup() ||
|
|
2333
|
-
outboundLocalStreamRef.current !== stream ||
|
|
2334
|
-
!isCurrentAttempt() ||
|
|
2335
|
-
wsRef.current !== ws ||
|
|
2336
|
-
outboundCameraSenderRef.current !== sender
|
|
2337
|
-
) {
|
|
2338
|
-
return;
|
|
2339
|
-
}
|
|
2340
|
-
outboundLocalStreamRef.current = null;
|
|
2341
|
-
outboundCameraRequestRef.current = undefined;
|
|
2342
|
-
stopMediaStream(stream);
|
|
2343
|
-
stopCameraStatsPoller();
|
|
2344
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2345
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2346
|
-
})
|
|
2347
|
-
.catch((err) => debugWarn('camera ended cleanup failed:', err));
|
|
2348
|
-
};
|
|
2349
|
-
|
|
2350
|
-
if (!isCurrentCameraRequest()) return;
|
|
2351
|
-
sendGrantedCameraResult(ws, videoTrack);
|
|
2352
|
-
startCameraStatsPoller();
|
|
2353
|
-
});
|
|
2354
|
-
};
|
|
2355
|
-
|
|
2356
|
-
const teardownConnection = () => {
|
|
2357
|
-
clearConnectionSuccessTimeout();
|
|
2358
|
-
clearIceDisconnectedGrace();
|
|
2359
|
-
stopRequestFrameLoop();
|
|
2360
|
-
if (axFetcherRef.current) {
|
|
2361
|
-
axFetcherRef.current.stop();
|
|
2362
|
-
axFetcherRef.current = null;
|
|
2363
|
-
}
|
|
2364
|
-
// Drop any active outbound camera before the PC dies so the
|
|
2365
|
-
// browser doesn't leave the camera indicator lit between
|
|
2366
|
-
// reconnects.
|
|
2367
|
-
cameraOperations.invalidate();
|
|
2368
|
-
stopCameraStatsPoller();
|
|
2369
|
-
stopOutboundLocalStream();
|
|
2370
|
-
outboundCameraRequestRef.current = undefined;
|
|
2371
|
-
outboundCameraSenderRef.current = null;
|
|
2372
|
-
// Same for the live microphone. Manual enablement survives reconnect;
|
|
2373
|
-
// guest demand is replayed by the host after it assigns the new session.
|
|
2374
|
-
++micGenerationRef.current;
|
|
2375
|
-
microphoneDemandActiveRef.current = false;
|
|
2376
|
-
micStartPendingIdRef.current = null;
|
|
2377
|
-
if (outboundMicStreamRef.current) {
|
|
2378
|
-
stopMediaStream(outboundMicStreamRef.current);
|
|
2379
|
-
outboundMicStreamRef.current = null;
|
|
2380
|
-
// The capture is gone; report it so consumers don't keep a stale
|
|
2381
|
-
// `active: true` across drops (a re-attach after reconnect
|
|
2382
|
-
// reports again).
|
|
2383
|
-
safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
|
|
2384
|
-
active: false,
|
|
2385
|
-
});
|
|
2386
|
-
}
|
|
2387
|
-
outboundMicSenderRef.current = null;
|
|
2388
|
-
// A scheduled cursor flush would otherwise call setState on a
|
|
2389
|
-
// teardown component once the next frame runs.
|
|
2390
|
-
if (cursorRafIdRef.current !== undefined) {
|
|
2391
|
-
window.cancelAnimationFrame(cursorRafIdRef.current);
|
|
2392
|
-
cursorRafIdRef.current = undefined;
|
|
2393
|
-
}
|
|
2394
|
-
if (wsRef.current) {
|
|
2395
|
-
wsRef.current.onopen = null;
|
|
2396
|
-
wsRef.current.onmessage = null;
|
|
2397
|
-
wsRef.current.onerror = null;
|
|
2398
|
-
wsRef.current.onclose = null;
|
|
2399
|
-
wsRef.current.close();
|
|
2400
|
-
wsRef.current = null;
|
|
2401
|
-
}
|
|
2402
|
-
if (peerConnectionRef.current) {
|
|
2403
|
-
peerConnectionRef.current.onconnectionstatechange = null;
|
|
2404
|
-
peerConnectionRef.current.oniceconnectionstatechange = null;
|
|
2405
|
-
peerConnectionRef.current.ontrack = null;
|
|
2406
|
-
peerConnectionRef.current.onicecandidate = null;
|
|
2407
|
-
peerConnectionRef.current.close();
|
|
2408
|
-
peerConnectionRef.current = null;
|
|
2409
|
-
}
|
|
2410
|
-
if (videoRef.current) {
|
|
2411
|
-
videoRef.current.srcObject = null;
|
|
2412
|
-
}
|
|
2413
|
-
if (dataChannelRef.current) {
|
|
2414
|
-
dataChannelRef.current.onopen = null;
|
|
2415
|
-
dataChannelRef.current.onclose = null;
|
|
2416
|
-
dataChannelRef.current.onerror = null;
|
|
2417
|
-
dataChannelRef.current.close();
|
|
2418
|
-
dataChannelRef.current = null;
|
|
2419
|
-
}
|
|
2420
|
-
};
|
|
2421
|
-
|
|
2422
|
-
const handleVisibilityChange = () => {
|
|
2423
|
-
if (document.hidden) {
|
|
2424
|
-
stopKeepAlive();
|
|
2425
|
-
} else {
|
|
2426
|
-
startKeepAlive();
|
|
2427
|
-
}
|
|
2428
|
-
};
|
|
2429
|
-
|
|
2430
|
-
// Probe the instance endpoint over HTTP to tell a permanent termination
|
|
2431
|
-
// apart from a recoverable connection drop. The router serves the same
|
|
2432
|
-
// path the WebSocket uses and returns 404 once the instance is gone
|
|
2433
|
-
// (deleted / terminated), while a live instance returns something else.
|
|
2434
|
-
//
|
|
2435
|
-
// Only an authoritative, freshly-fetched 404 counts as terminated. A
|
|
2436
|
-
// network outage, a non-404 status, or a CORS-blocked (unreadable)
|
|
2437
|
-
// response all resolve to `false` so we keep the normal retry/reconnect
|
|
2438
|
-
// behavior and never mistake a connectivity problem for a termination.
|
|
2439
|
-
const probeTerminated = async (): Promise<boolean> => {
|
|
2440
|
-
// The browser knows it has no network — this is a connectivity drop,
|
|
2441
|
-
// not a termination. Don't probe (and don't risk a cached 404).
|
|
2442
|
-
if (typeof navigator !== 'undefined' && navigator.onLine === false) {
|
|
2443
|
-
return false;
|
|
2444
|
-
}
|
|
2445
|
-
let probeUrl: string;
|
|
2446
|
-
try {
|
|
2447
|
-
const parsed = new URL(url);
|
|
2448
|
-
parsed.protocol =
|
|
2449
|
-
parsed.protocol === 'wss:' ? 'https:'
|
|
2450
|
-
: parsed.protocol === 'ws:' ? 'http:'
|
|
2451
|
-
: parsed.protocol;
|
|
2452
|
-
probeUrl = withAuthenticationToken(parsed.toString(), token);
|
|
2453
|
-
} catch {
|
|
2454
|
-
return false;
|
|
2455
|
-
}
|
|
2456
|
-
const controller = new AbortController();
|
|
2457
|
-
const timeoutId = window.setTimeout(() => controller.abort(), TERMINATION_PROBE_TIMEOUT_MS);
|
|
2458
|
-
try {
|
|
2459
|
-
// `no-store` so a prior response can't be served from the HTTP cache
|
|
2460
|
-
// (e.g. a stale 404 returned while offline would look like a
|
|
2461
|
-
// termination); the probe must always reach the live router.
|
|
2462
|
-
const res = await fetch(probeUrl, { method: 'GET', cache: 'no-store', signal: controller.signal });
|
|
2463
|
-
return res.status === 404;
|
|
2464
|
-
} catch {
|
|
2465
|
-
return false;
|
|
2466
|
-
} finally {
|
|
2467
|
-
window.clearTimeout(timeoutId);
|
|
2468
|
-
}
|
|
2469
|
-
};
|
|
2470
|
-
|
|
2471
|
-
const markTerminated = () => {
|
|
2472
|
-
if (terminatedRef.current) return;
|
|
2473
|
-
terminatedRef.current = true;
|
|
2474
|
-
setTerminated(true);
|
|
2475
|
-
setRetryExhausted(false);
|
|
2476
|
-
// Invalidate any in-flight/scheduled attempt and stop all activity so
|
|
2477
|
-
// we don't keep hammering an endpoint that will never come back.
|
|
2478
|
-
connectionGenerationRef.current += 1;
|
|
2479
|
-
clearScheduledRetry();
|
|
2480
|
-
clearConnectionSuccessTimeout();
|
|
2481
|
-
clearIceDisconnectedGrace();
|
|
2482
|
-
teardownConnection();
|
|
2483
|
-
updateStatus('Instance terminated');
|
|
2484
|
-
reportConnectionState('terminated');
|
|
2485
|
-
safeInvoke('onTerminated', onTerminatedRef.current);
|
|
2486
|
-
};
|
|
2487
|
-
|
|
2488
|
-
// Run a one-shot termination check in reaction to a drop. Event-driven
|
|
2489
|
-
// (no polling), deduped while one is already in flight, and a no-op once
|
|
2490
|
-
// we've concluded the instance is gone.
|
|
2491
|
-
const checkTerminated = () => {
|
|
2492
|
-
if (terminatedRef.current || terminationProbeInFlightRef.current) return;
|
|
2493
|
-
terminationProbeInFlightRef.current = true;
|
|
2494
|
-
void probeTerminated().then((isTerminated) => {
|
|
2495
|
-
terminationProbeInFlightRef.current = false;
|
|
2496
|
-
if (isTerminated) {
|
|
2497
|
-
markTerminated();
|
|
2498
|
-
}
|
|
2499
|
-
});
|
|
2500
|
-
};
|
|
2501
|
-
|
|
2502
|
-
const scheduleRetry = (reason: string, generation: number) => {
|
|
2503
|
-
if (terminatedRef.current) {
|
|
2504
|
-
return;
|
|
2505
|
-
}
|
|
2506
|
-
if (generation !== connectionGenerationRef.current) {
|
|
2507
|
-
return;
|
|
2508
|
-
}
|
|
2509
|
-
|
|
2510
|
-
// A drop just happened. Find out whether the instance is gone for good
|
|
2511
|
-
// (404) rather than a recoverable blip; if so the probe stops retries
|
|
2512
|
-
// and surfaces the terminated message. Meanwhile the normal retry path
|
|
2513
|
-
// below proceeds and is short-circuited once the verdict lands.
|
|
2514
|
-
checkTerminated();
|
|
2515
|
-
|
|
2516
|
-
if (controlChannelOpenedRef.current) {
|
|
2517
|
-
if (!autoReconnectRef.current) {
|
|
2518
|
-
updateStatus(`Connection failed after it was established: ${reason}`);
|
|
2519
|
-
setRetryExhausted(true);
|
|
2520
|
-
reportConnectionState('failed');
|
|
2521
|
-
teardownConnection();
|
|
2522
|
-
return;
|
|
2523
|
-
}
|
|
2524
|
-
// Reset so the upcoming retry gets a fresh MAX_CONNECTION_ATTEMPTS budget.
|
|
2525
|
-
updateStatus(`Reconnecting after established session dropped: ${reason}`);
|
|
2526
|
-
controlChannelOpenedRef.current = false;
|
|
2527
|
-
connectionAttemptRef.current = -1;
|
|
2528
|
-
}
|
|
2529
|
-
|
|
2530
|
-
clearScheduledRetry();
|
|
2531
|
-
|
|
2532
|
-
const nextAttempt = connectionAttemptRef.current + 1;
|
|
2533
|
-
if (nextAttempt >= MAX_CONNECTION_ATTEMPTS) {
|
|
2534
|
-
updateStatus(`Connection failed after ${MAX_CONNECTION_ATTEMPTS} attempts: ${reason}`);
|
|
2535
|
-
setRetryExhausted(true);
|
|
2536
|
-
reportConnectionState('failed');
|
|
2537
|
-
teardownConnection();
|
|
2538
|
-
return;
|
|
2539
|
-
}
|
|
2540
|
-
|
|
2541
|
-
updateStatus(`Retrying connection (${nextAttempt + 1}/${MAX_CONNECTION_ATTEMPTS})`);
|
|
2542
|
-
reportConnectionState('reconnecting');
|
|
2543
|
-
teardownConnection();
|
|
2544
|
-
retryTimeoutRef.current = window.setTimeout(() => {
|
|
2545
|
-
retryTimeoutRef.current = undefined;
|
|
2546
|
-
if (generation !== connectionGenerationRef.current) {
|
|
2547
|
-
return;
|
|
2548
|
-
}
|
|
2549
|
-
void startAttempt(nextAttempt);
|
|
2550
|
-
}, CONNECTION_RETRY_DELAY_MS);
|
|
2551
|
-
};
|
|
2552
|
-
|
|
2553
|
-
const startAttempt = async (attemptNumber = 0) => {
|
|
2554
|
-
// Never reconnect to an instance we've concluded is gone. `start()`
|
|
2555
|
-
// clears this when a deliberate fresh connection is requested.
|
|
2556
|
-
if (terminatedRef.current) return;
|
|
2557
|
-
const generation = connectionGenerationRef.current + 1;
|
|
2558
|
-
connectionGenerationRef.current = generation;
|
|
2559
|
-
connectionAttemptRef.current = attemptNumber;
|
|
2560
|
-
controlChannelOpenedRef.current = false;
|
|
2561
|
-
setRetryExhausted(false);
|
|
2562
|
-
// A drop after an established session restarts the attempt budget at 0
|
|
2563
|
-
// while staying 'reconnecting'; only a fresh start reports 'connecting'.
|
|
2564
|
-
if (attemptNumber === 0 && connectionStateRef.current !== 'reconnecting')
|
|
2565
|
-
reportConnectionState('connecting');
|
|
2566
|
-
clearScheduledRetry();
|
|
2567
|
-
clearConnectionSuccessTimeout();
|
|
2568
|
-
stopRequestFrameLoop();
|
|
2569
|
-
firstFrameShownRef.current = false;
|
|
2570
|
-
setVideoLoaded(false);
|
|
2571
|
-
teardownConnection();
|
|
2572
|
-
|
|
2573
|
-
const isCurrentAttempt = () => generation === connectionGenerationRef.current;
|
|
2574
|
-
|
|
2575
|
-
connectionSuccessTimeoutRef.current = window.setTimeout(() => {
|
|
2576
|
-
connectionSuccessTimeoutRef.current = undefined;
|
|
2577
|
-
if (!isCurrentAttempt() || controlChannelOpenedRef.current) {
|
|
2578
|
-
return;
|
|
2579
|
-
}
|
|
2580
|
-
scheduleRetry('Connection did not succeed within 15 seconds', generation);
|
|
2581
|
-
}, CONNECTION_SUCCESS_TIMEOUT_MS);
|
|
2582
|
-
|
|
2583
|
-
try {
|
|
2584
|
-
const ws = new WebSocket(withAuthenticationToken(url, token));
|
|
2585
|
-
wsRef.current = ws;
|
|
2586
|
-
|
|
2587
|
-
// Wait for WebSocket to connect
|
|
2588
|
-
await new Promise<void>((resolve, reject) => {
|
|
2589
|
-
let settled = false;
|
|
2590
|
-
const timeoutId = window.setTimeout(() => reject(new Error('WebSocket connection timeout')), 30000);
|
|
2591
|
-
const settle = (callback: () => void) => {
|
|
2592
|
-
if (settled) {
|
|
2593
|
-
return;
|
|
2594
|
-
}
|
|
2595
|
-
settled = true;
|
|
2596
|
-
window.clearTimeout(timeoutId);
|
|
2597
|
-
callback();
|
|
2598
|
-
};
|
|
2599
|
-
|
|
2600
|
-
ws.onopen = () => {
|
|
2601
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2602
|
-
return;
|
|
2603
|
-
}
|
|
2604
|
-
settle(resolve);
|
|
2605
|
-
};
|
|
2606
|
-
|
|
2607
|
-
ws.onerror = (error) => {
|
|
2608
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2609
|
-
return;
|
|
2610
|
-
}
|
|
2611
|
-
updateStatus('WebSocket error: ' + error);
|
|
2612
|
-
settle(() => reject(new Error('WebSocket connection failed')));
|
|
2613
|
-
};
|
|
2614
|
-
|
|
2615
|
-
ws.onclose = () => {
|
|
2616
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2617
|
-
return;
|
|
2618
|
-
}
|
|
2619
|
-
updateStatus('WebSocket closed');
|
|
2620
|
-
settle(() => reject(new Error('WebSocket closed before connection was established')));
|
|
2621
|
-
};
|
|
2622
|
-
});
|
|
2623
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2624
|
-
return;
|
|
2625
|
-
}
|
|
2626
|
-
|
|
2627
|
-
ws.onerror = (error) => {
|
|
2628
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2629
|
-
return;
|
|
2630
|
-
}
|
|
2631
|
-
updateStatus('WebSocket error: ' + error);
|
|
2632
|
-
};
|
|
2633
|
-
|
|
2634
|
-
ws.onclose = () => {
|
|
2635
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2636
|
-
return;
|
|
2637
|
-
}
|
|
2638
|
-
updateStatus('WebSocket closed');
|
|
2639
|
-
};
|
|
2640
|
-
|
|
2641
|
-
// Request RTCConfiguration. The reply doubles as the capability
|
|
2642
|
-
// handshake: camera-capable Android servers advertise
|
|
2643
|
-
// `cameraSupported: true` alongside the configuration.
|
|
2644
|
-
const rtcConfigPromise = new Promise<{
|
|
2645
|
-
rtcConfiguration: RTCConfiguration;
|
|
2646
|
-
cameraSupported: boolean;
|
|
2647
|
-
}>((resolve, reject) => {
|
|
2648
|
-
const settle = (callback: () => void) => {
|
|
2649
|
-
window.clearTimeout(timeoutId);
|
|
2650
|
-
ws.removeEventListener('message', messageHandler);
|
|
2651
|
-
callback();
|
|
2652
|
-
};
|
|
2653
|
-
const timeoutId = window.setTimeout(
|
|
2654
|
-
() => settle(() => reject(new Error('RTCConfiguration timeout'))),
|
|
2655
|
-
30000,
|
|
2656
|
-
);
|
|
2657
|
-
|
|
2658
|
-
const messageHandler = (event: MessageEvent) => {
|
|
2659
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2660
|
-
return;
|
|
2661
|
-
}
|
|
2662
|
-
try {
|
|
2663
|
-
const message = JSON.parse(event.data);
|
|
2664
|
-
if (
|
|
2665
|
-
message.type === 'signalingError' &&
|
|
2666
|
-
message.requestType === 'requestRtcConfiguration' &&
|
|
2667
|
-
message.sessionId === sessionId
|
|
2668
|
-
) {
|
|
2669
|
-
const reason =
|
|
2670
|
-
typeof message.error === 'string' ?
|
|
2671
|
-
message.error
|
|
2672
|
-
: 'Unable to load streaming configuration';
|
|
2673
|
-
settle(() => reject(new Error(reason)));
|
|
2674
|
-
} else if (message.type === 'rtcConfiguration') {
|
|
2675
|
-
settle(() =>
|
|
2676
|
-
resolve({
|
|
2677
|
-
rtcConfiguration: message.rtcConfiguration,
|
|
2678
|
-
cameraSupported: message.cameraSupported === true,
|
|
2679
|
-
}),
|
|
2680
|
-
);
|
|
2681
|
-
}
|
|
2682
|
-
} catch (e) {
|
|
2683
|
-
console.error('Error handling RTC configuration:', e);
|
|
2684
|
-
settle(() => reject(e));
|
|
2685
|
-
}
|
|
2686
|
-
};
|
|
2687
|
-
|
|
2688
|
-
ws.addEventListener('message', messageHandler);
|
|
2689
|
-
ws.send(
|
|
2690
|
-
JSON.stringify({
|
|
2691
|
-
type: 'requestRtcConfiguration',
|
|
2692
|
-
sessionId: sessionId,
|
|
2693
|
-
}),
|
|
2694
|
-
);
|
|
2695
|
-
});
|
|
2696
|
-
|
|
2697
|
-
const { rtcConfiguration: rtcConfig, cameraSupported: serverCameraSupported } =
|
|
2698
|
-
await rtcConfigPromise;
|
|
2699
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2700
|
-
return;
|
|
2701
|
-
}
|
|
2702
|
-
|
|
2703
|
-
const peerConnection = new RTCPeerConnection(rtcConfig);
|
|
2704
|
-
peerConnectionRef.current = peerConnection;
|
|
2705
|
-
peerConnection.addTransceiver('audio', { direction: 'recvonly' });
|
|
2706
|
-
const videoTransceiver = peerConnection.addTransceiver('video', { direction: 'recvonly' });
|
|
2707
|
-
|
|
2708
|
-
// iOS always supports camera/mic injection; Android servers
|
|
2709
|
-
// advertise it through `cameraSupported` (Android 15 and newer).
|
|
2710
|
-
// Never offer the extra sendonly m-lines to non-capable Android
|
|
2711
|
-
// servers — older servers cannot negotiate them safely.
|
|
2712
|
-
const canSendMedia = platform === 'ios' || serverCameraSupported;
|
|
2713
|
-
|
|
2714
|
-
// Pre-allocate a sendonly slot for the user's camera so the
|
|
2715
|
-
// device can pull frames later without renegotiating SDP.
|
|
2716
|
-
// Allocating up-front means we don't have to renegotiate the
|
|
2717
|
-
// SDP when the device app actually opens its camera session —
|
|
2718
|
-
// the codec/SSRC negotiation happens once, here, and turning
|
|
2719
|
-
// the camera on/off later is just a `replaceTrack`.
|
|
2720
|
-
const outboundCameraTransceiver =
|
|
2721
|
-
canSendMedia ?
|
|
2722
|
-
peerConnection.addTransceiver('video', {
|
|
2723
|
-
direction: 'sendonly',
|
|
2724
|
-
})
|
|
2725
|
-
: null;
|
|
2726
|
-
outboundCameraSenderRef.current = outboundCameraTransceiver?.sender ?? null;
|
|
2727
|
-
|
|
2728
|
-
// Same pre-allocation for the live microphone: a sendonly audio
|
|
2729
|
-
// slot negotiated once at connection setup, so toggling the mic
|
|
2730
|
-
// later is just `replaceTrack` — no SDP renegotiation.
|
|
2731
|
-
const outboundMicTransceiver =
|
|
2732
|
-
canSendMedia ?
|
|
2733
|
-
peerConnection.addTransceiver('audio', {
|
|
2734
|
-
direction: 'sendonly',
|
|
2735
|
-
})
|
|
2736
|
-
: null;
|
|
2737
|
-
outboundMicSenderRef.current = outboundMicTransceiver?.sender ?? null;
|
|
2738
|
-
|
|
2739
|
-
// As hardware encoder, we use H265 for iOS and VP9 for Android.
|
|
2740
|
-
// We make sure these two are the first ones in the list.
|
|
2741
|
-
// If not, the fallback is H264 which is also hardware accelerated, although not as good,
|
|
2742
|
-
// available on all platforms.
|
|
2743
|
-
//
|
|
2744
|
-
// The rest is not important.
|
|
2745
|
-
if (RTCRtpReceiver.getCapabilities) {
|
|
2746
|
-
const capabilities = RTCRtpReceiver.getCapabilities('video');
|
|
2747
|
-
if (capabilities && capabilities.codecs) {
|
|
2748
|
-
const codecs = capabilities.codecs;
|
|
2749
|
-
const sortedCodecs = codecs.sort((a, b) => {
|
|
2750
|
-
const getCodecPriority = (codec: { mimeType: string }): number => {
|
|
2751
|
-
const mimeType = codec.mimeType.toLowerCase();
|
|
2752
|
-
if (mimeType.includes('vp9')) return 1;
|
|
2753
|
-
if (mimeType.includes('h265') || mimeType.includes('hevc')) return 2;
|
|
2754
|
-
if (mimeType.includes('h264') || mimeType.includes('avc')) return 3;
|
|
2755
|
-
return 4; // Everything else
|
|
2756
|
-
};
|
|
2757
|
-
return getCodecPriority(a) - getCodecPriority(b);
|
|
2758
|
-
});
|
|
2759
|
-
videoTransceiver.setCodecPreferences(sortedCodecs);
|
|
2760
|
-
debugLog('Set codec preferences:', sortedCodecs.map((c) => c.mimeType).join(', '));
|
|
2761
|
-
}
|
|
2762
|
-
}
|
|
2763
|
-
|
|
2764
|
-
// Pin the outbound camera transceiver's codec order with
|
|
2765
|
-
// negotiation-time fallback. Preference order:
|
|
2766
|
-
// 1. H.265 / HEVC — VideoToolbox HW on M-series + recent
|
|
2767
|
-
// Chrome. ~30% less bitrate at equal quality. If the
|
|
2768
|
-
// answerer (limulator) supports HEVC, this wins.
|
|
2769
|
-
// 2. H.264 with VideoToolbox-friendly profile-level-ids
|
|
2770
|
-
// (42e01f / 42e028 / 640c1f). Universally HW-accelerated
|
|
2771
|
-
// on every Mac since 2009. SDP-time fallback when HEVC
|
|
2772
|
-
// isn't in the answer.
|
|
2773
|
-
// 3. Any other H.264 profile. Often resolves to OpenH264
|
|
2774
|
-
// (software) so we keep it explicitly last among H.264
|
|
2775
|
-
// to avoid Chrome's default pick.
|
|
2776
|
-
// 4. VP9 / VP8 (libvpx software on Mac). Last resort.
|
|
2777
|
-
//
|
|
2778
|
-
// Caveat: this is *negotiation* fallback. If HEVC HW encoder
|
|
2779
|
-
// init fails at session start or gets demoted mid-stream,
|
|
2780
|
-
// Chrome falls back to *software* HEVC, not H.264 — no spec
|
|
2781
|
-
// hook for runtime re-negotiation. The host-side stats
|
|
2782
|
-
// logger will surface this as decoder=ffmpeg / hw=false.
|
|
2783
|
-
if (outboundCameraTransceiver && RTCRtpSender.getCapabilities) {
|
|
2784
|
-
const senderCaps = RTCRtpSender.getCapabilities('video');
|
|
2785
|
-
if (senderCaps && senderCaps.codecs) {
|
|
2786
|
-
const vtH264Profiles = new Set(['42e01f', '42e028', '640c1f']);
|
|
2787
|
-
const score = (c: { mimeType: string; sdpFmtpLine?: string }): number => {
|
|
2788
|
-
const mime = c.mimeType.toLowerCase();
|
|
2789
|
-
const fmtp = (c.sdpFmtpLine || '').toLowerCase();
|
|
2790
|
-
const profileMatch = fmtp.match(/profile-level-id=([0-9a-f]{6})/);
|
|
2791
|
-
const profile = profileMatch ? profileMatch[1] : '';
|
|
2792
|
-
if (mime === 'video/h265' || mime === 'video/hevc') return 1;
|
|
2793
|
-
if (mime === 'video/h264' && vtH264Profiles.has(profile)) return 2;
|
|
2794
|
-
if (mime === 'video/h264') return 3;
|
|
2795
|
-
if (mime === 'video/vp9') return 4;
|
|
2796
|
-
if (mime === 'video/vp8') return 5;
|
|
2797
|
-
if (mime === 'video/rtx' || mime === 'video/red' || mime === 'video/ulpfec') {
|
|
2798
|
-
return 0; // Keep RTX/FEC available alongside everything else.
|
|
2799
|
-
}
|
|
2800
|
-
return 6;
|
|
2801
|
-
};
|
|
2802
|
-
const sortedSendCodecs = [...senderCaps.codecs].sort((a, b) => score(a) - score(b));
|
|
2803
|
-
try {
|
|
2804
|
-
outboundCameraTransceiver.setCodecPreferences(sortedSendCodecs);
|
|
2805
|
-
debugLog(
|
|
2806
|
-
'Outbound camera codec preferences:',
|
|
2807
|
-
sortedSendCodecs
|
|
2808
|
-
.map((c) => `${c.mimeType}${c.sdpFmtpLine ? `[${c.sdpFmtpLine}]` : ''}`)
|
|
2809
|
-
.join(', '),
|
|
2810
|
-
);
|
|
2811
|
-
} catch (err) {
|
|
2812
|
-
debugWarn('Failed to set outbound camera codec preferences:', err);
|
|
2813
|
-
}
|
|
2814
|
-
}
|
|
2815
|
-
}
|
|
2816
|
-
|
|
2817
|
-
const dataChannel = peerConnection.createDataChannel('control', {
|
|
2818
|
-
ordered: true,
|
|
2819
|
-
negotiated: true,
|
|
2820
|
-
id: 1,
|
|
2821
|
-
});
|
|
2822
|
-
dataChannelRef.current = dataChannel;
|
|
2823
|
-
|
|
2824
|
-
dataChannel.onopen = () => {
|
|
2825
|
-
if (!isCurrentAttempt() || dataChannelRef.current !== dataChannel || wsRef.current !== ws) {
|
|
2826
|
-
return;
|
|
2827
|
-
}
|
|
2828
|
-
controlChannelOpenedRef.current = true;
|
|
2829
|
-
clearConnectionSuccessTimeout();
|
|
2830
|
-
updateStatus('Control channel opened');
|
|
2831
|
-
|
|
2832
|
-
// Spin up the AX fetcher now that we have a stable WS + control
|
|
2833
|
-
// channel. The fetcher's send function reuses this WS; it stops
|
|
2834
|
-
// sending if the WS dies. start() is called lazily based on the
|
|
2835
|
-
// inspectMode prop via a sibling useEffect.
|
|
2836
|
-
if (!axFetcherRef.current) {
|
|
2837
|
-
axFetcherRef.current = new AxFetcher({
|
|
2838
|
-
platform,
|
|
2839
|
-
baseIntervalMs: axPollIntervalMs,
|
|
2840
|
-
maxBackoffMs: axMaxBackoffMs,
|
|
2841
|
-
androidElementTreeOptions,
|
|
2842
|
-
send: (payload) => {
|
|
2843
|
-
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return false;
|
|
2844
|
-
try {
|
|
2845
|
-
wsRef.current.send(JSON.stringify(payload));
|
|
2846
|
-
return true;
|
|
2847
|
-
} catch {
|
|
2848
|
-
return false;
|
|
2849
|
-
}
|
|
2850
|
-
},
|
|
2851
|
-
onSnapshot: (snapshot) => {
|
|
2852
|
-
setAxSnapshot((prev) => (axSnapshotsEqual(prev, snapshot) ? prev : snapshot));
|
|
2853
|
-
// Defer to a microtask so customer code (which may DOM-write,
|
|
2854
|
-
// start expensive work, or itself call back into ref
|
|
2855
|
-
// methods) doesn't run synchronously inside our state-setter
|
|
2856
|
-
// path. React then has a chance to schedule its render before
|
|
2857
|
-
// the customer handler kicks off side-effects.
|
|
2858
|
-
queueMicrotask(() => {
|
|
2859
|
-
safeInvoke('onAxSnapshotChange', onAxSnapshotChangeRef.current, snapshot);
|
|
2860
|
-
});
|
|
2861
|
-
},
|
|
2862
|
-
onStatusChange: (status, error) => {
|
|
2863
|
-
safeInvoke('onAxStatusChange', onAxStatusChangeRef.current, status, error);
|
|
2864
|
-
},
|
|
2865
|
-
});
|
|
2866
|
-
if (inspectModeRef.current === true || inspectModeRef.current === 'hover-only') {
|
|
2867
|
-
axFetcherRef.current.start();
|
|
2868
|
-
}
|
|
2869
|
-
}
|
|
2870
|
-
const sendRequestFrame = () => {
|
|
2871
|
-
if (
|
|
2872
|
-
!isCurrentAttempt() ||
|
|
2873
|
-
firstFrameShownRef.current ||
|
|
2874
|
-
dataChannelRef.current !== dataChannel ||
|
|
2875
|
-
wsRef.current !== ws ||
|
|
2876
|
-
ws.readyState !== WebSocket.OPEN
|
|
2877
|
-
) {
|
|
2878
|
-
return;
|
|
2879
|
-
}
|
|
2880
|
-
ws.send(JSON.stringify({ type: 'requestFrame', sessionId: sessionId }));
|
|
2881
|
-
};
|
|
2882
|
-
|
|
2883
|
-
sendRequestFrame();
|
|
2884
|
-
stopRequestFrameLoop();
|
|
2885
|
-
requestFrameIntervalRef.current = window.setInterval(() => {
|
|
2886
|
-
if (
|
|
2887
|
-
!isCurrentAttempt() ||
|
|
2888
|
-
firstFrameShownRef.current ||
|
|
2889
|
-
dataChannelRef.current !== dataChannel ||
|
|
2890
|
-
wsRef.current !== ws ||
|
|
2891
|
-
ws.readyState !== WebSocket.OPEN
|
|
2892
|
-
) {
|
|
2893
|
-
stopRequestFrameLoop();
|
|
2894
|
-
return;
|
|
2895
|
-
}
|
|
2896
|
-
sendRequestFrame();
|
|
2897
|
-
}, 250);
|
|
2898
|
-
|
|
2899
|
-
// Send openUrl message if the prop is provided
|
|
2900
|
-
if (openUrl) {
|
|
2901
|
-
try {
|
|
2902
|
-
const decodedUrl = decodeURIComponent(openUrl);
|
|
2903
|
-
updateStatus('Opening URL');
|
|
2904
|
-
ws.send(
|
|
2905
|
-
JSON.stringify({
|
|
2906
|
-
type: 'openUrl',
|
|
2907
|
-
url: decodedUrl,
|
|
2908
|
-
sessionId: sessionId,
|
|
2909
|
-
}),
|
|
2910
|
-
);
|
|
2911
|
-
} catch (error) {
|
|
2912
|
-
console.error({ error }, 'Error decoding URL, falling back to the original URL');
|
|
2913
|
-
ws.send(
|
|
2914
|
-
JSON.stringify({
|
|
2915
|
-
type: 'openUrl',
|
|
2916
|
-
url: openUrl,
|
|
2917
|
-
sessionId: sessionId,
|
|
2918
|
-
}),
|
|
2919
|
-
);
|
|
2920
|
-
}
|
|
2921
|
-
// openUrl can take a moment to load the destination — boost
|
|
2922
|
-
// AX polling so the overlay refreshes through the transition.
|
|
2923
|
-
axFetcherRef.current?.bumpActivity();
|
|
2924
|
-
}
|
|
2925
|
-
};
|
|
2926
|
-
|
|
2927
|
-
dataChannel.onclose = () => {
|
|
2928
|
-
if (!isCurrentAttempt() || dataChannelRef.current !== dataChannel) {
|
|
2929
|
-
return;
|
|
2930
|
-
}
|
|
2931
|
-
updateStatus('Control channel closed');
|
|
2932
|
-
};
|
|
2933
|
-
|
|
2934
|
-
dataChannel.onerror = (error) => {
|
|
2935
|
-
if (!isCurrentAttempt() || dataChannelRef.current !== dataChannel) {
|
|
2936
|
-
return;
|
|
2937
|
-
}
|
|
2938
|
-
console.error('Control channel error:', error);
|
|
2939
|
-
updateStatus('Control channel error: ' + error);
|
|
2940
|
-
};
|
|
2941
|
-
|
|
2942
|
-
// Set up connection state monitoring
|
|
2943
|
-
peerConnection.onconnectionstatechange = () => {
|
|
2944
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
|
|
2945
|
-
return;
|
|
2946
|
-
}
|
|
2947
|
-
updateStatus('Connection state: ' + peerConnection.connectionState);
|
|
2948
|
-
if (peerConnection.connectionState === 'failed') {
|
|
2949
|
-
scheduleRetry('WebRTC connection entered failed state', generation);
|
|
2950
|
-
}
|
|
2951
|
-
};
|
|
2952
|
-
|
|
2953
|
-
peerConnection.oniceconnectionstatechange = () => {
|
|
2954
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
|
|
2955
|
-
return;
|
|
2956
|
-
}
|
|
2957
|
-
const iceState = peerConnection.iceConnectionState;
|
|
2958
|
-
updateStatus('ICE state: ' + iceState);
|
|
2959
|
-
if (iceState === 'connected' || iceState === 'completed') {
|
|
2960
|
-
clearIceDisconnectedGrace();
|
|
2961
|
-
// Re-attach the live microphone across reconnects (or apply a
|
|
2962
|
-
// prop that was flipped on before the connection was up).
|
|
2963
|
-
if (isMicrophoneWanted() && !outboundMicStreamRef.current) {
|
|
2964
|
-
void attachMicrophone();
|
|
2965
|
-
}
|
|
2966
|
-
return;
|
|
2967
|
-
}
|
|
2968
|
-
if (iceState === 'failed') {
|
|
2969
|
-
clearIceDisconnectedGrace();
|
|
2970
|
-
scheduleRetry('ICE connection entered failed state', generation);
|
|
2971
|
-
return;
|
|
2972
|
-
}
|
|
2973
|
-
if (
|
|
2974
|
-
iceState === 'disconnected' &&
|
|
2975
|
-
autoReconnectRef.current &&
|
|
2976
|
-
iceDisconnectedGraceRef.current === undefined
|
|
2977
|
-
) {
|
|
2978
|
-
// Cap the browser's natural disconnected→failed escalation to recover faster.
|
|
2979
|
-
iceDisconnectedGraceRef.current = window.setTimeout(() => {
|
|
2980
|
-
iceDisconnectedGraceRef.current = undefined;
|
|
2981
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
|
|
2982
|
-
return;
|
|
2983
|
-
}
|
|
2984
|
-
if (peerConnection.iceConnectionState === 'disconnected') {
|
|
2985
|
-
scheduleRetry('ICE stayed disconnected past grace period', generation);
|
|
2986
|
-
}
|
|
2987
|
-
}, ICE_DISCONNECTED_GRACE_MS);
|
|
2988
|
-
}
|
|
2989
|
-
};
|
|
2990
|
-
|
|
2991
|
-
// Set up video handling
|
|
2992
|
-
peerConnection.ontrack = (event) => {
|
|
2993
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
|
|
2994
|
-
return;
|
|
2995
|
-
}
|
|
2996
|
-
updateStatus('Received remote track: ' + event.track.kind);
|
|
2997
|
-
if (event.track.kind === 'video' && videoRef.current) {
|
|
2998
|
-
debugLog(`[${new Date().toISOString()}] Video track received:`, event.track);
|
|
2999
|
-
videoRef.current.srcObject = event.streams[0];
|
|
3000
|
-
}
|
|
3001
|
-
};
|
|
3002
|
-
|
|
3003
|
-
// Handle ICE candidates
|
|
3004
|
-
peerConnection.onicecandidate = (event) => {
|
|
3005
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection || wsRef.current !== ws) {
|
|
3006
|
-
return;
|
|
3007
|
-
}
|
|
3008
|
-
if (event.candidate && ws.readyState === WebSocket.OPEN) {
|
|
3009
|
-
const message = {
|
|
3010
|
-
type: 'candidate',
|
|
3011
|
-
candidate: event.candidate.candidate,
|
|
3012
|
-
sdpMid: event.candidate.sdpMid,
|
|
3013
|
-
sdpMLineIndex: event.candidate.sdpMLineIndex,
|
|
3014
|
-
sessionId: sessionId,
|
|
3015
|
-
};
|
|
3016
|
-
ws.send(JSON.stringify(message));
|
|
3017
|
-
updateStatus('Sent ICE candidate');
|
|
3018
|
-
} else {
|
|
3019
|
-
updateStatus('ICE candidate gathering completed');
|
|
3020
|
-
}
|
|
3021
|
-
};
|
|
3022
|
-
|
|
3023
|
-
// Handle incoming messages
|
|
3024
|
-
ws.onmessage = async (event) => {
|
|
3025
|
-
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
3026
|
-
return;
|
|
3027
|
-
}
|
|
3028
|
-
let message;
|
|
3029
|
-
try {
|
|
3030
|
-
message = JSON.parse(event.data);
|
|
3031
|
-
} catch (e) {
|
|
3032
|
-
debugWarn('Error parsing message:', e);
|
|
3033
|
-
return;
|
|
3034
|
-
}
|
|
3035
|
-
// Inspect-mode responses are routed to the fetcher first so it
|
|
3036
|
-
// can resolve in-flight requests regardless of which platform's
|
|
3037
|
-
// protocol is in use.
|
|
3038
|
-
if (axFetcherRef.current?.handleMessage(message)) {
|
|
3039
|
-
return;
|
|
3040
|
-
}
|
|
3041
|
-
updateStatus('Received: ' + message.type);
|
|
3042
|
-
switch (message.type) {
|
|
3043
|
-
case 'signalingError': {
|
|
3044
|
-
if (message.requestType !== 'offer' || message.sessionId !== sessionId) {
|
|
3045
|
-
break;
|
|
3046
|
-
}
|
|
3047
|
-
const reason =
|
|
3048
|
-
typeof message.error === 'string' ?
|
|
3049
|
-
message.error
|
|
3050
|
-
: 'Unable to establish the streaming connection';
|
|
3051
|
-
updateStatus('Error: ' + reason);
|
|
3052
|
-
scheduleRetry(reason, generation);
|
|
3053
|
-
break;
|
|
3054
|
-
}
|
|
3055
|
-
case 'answer':
|
|
3056
|
-
if (!peerConnectionRef.current || peerConnectionRef.current !== peerConnection) {
|
|
3057
|
-
updateStatus('No peer connection, skipping answer');
|
|
3058
|
-
break;
|
|
3059
|
-
}
|
|
3060
|
-
await peerConnection.setRemoteDescription(
|
|
3061
|
-
new RTCSessionDescription({
|
|
3062
|
-
type: 'answer',
|
|
3063
|
-
sdp: message.sdp,
|
|
3064
|
-
}),
|
|
3065
|
-
);
|
|
3066
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
|
|
3067
|
-
return;
|
|
3068
|
-
}
|
|
3069
|
-
updateStatus('Set remote description');
|
|
3070
|
-
break;
|
|
3071
|
-
case 'candidate':
|
|
3072
|
-
if (!peerConnectionRef.current || peerConnectionRef.current !== peerConnection) {
|
|
3073
|
-
updateStatus('No peer connection, skipping candidate');
|
|
3074
|
-
break;
|
|
3075
|
-
}
|
|
3076
|
-
await peerConnection.addIceCandidate(
|
|
3077
|
-
new RTCIceCandidate({
|
|
3078
|
-
candidate: message.candidate,
|
|
3079
|
-
sdpMid: message.sdpMid,
|
|
3080
|
-
sdpMLineIndex: message.sdpMLineIndex,
|
|
3081
|
-
}),
|
|
3082
|
-
);
|
|
3083
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
|
|
3084
|
-
return;
|
|
3085
|
-
}
|
|
3086
|
-
updateStatus('Added ICE candidate');
|
|
3087
|
-
break;
|
|
3088
|
-
case 'screenshot':
|
|
3089
|
-
case 'screenshotResult': {
|
|
3090
|
-
if (typeof message.id !== 'string') {
|
|
3091
|
-
debugWarn('Received invalid screenshot success message:', message);
|
|
3092
|
-
break;
|
|
3093
|
-
}
|
|
3094
|
-
const screenshotError = getScreenshotError(message);
|
|
3095
|
-
if (screenshotError) {
|
|
3096
|
-
const rejecter = pendingScreenshotRejectersRef.current.get(message.id);
|
|
3097
|
-
if (!rejecter) {
|
|
3098
|
-
debugWarn(`Received screenshot error for unknown or handled id: ${message.id}`);
|
|
3099
|
-
break;
|
|
3100
|
-
}
|
|
3101
|
-
debugWarn(`Received screenshot error for id ${message.id}: ${screenshotError}`);
|
|
3102
|
-
rejecter(new Error(screenshotError));
|
|
3103
|
-
pendingScreenshotResolversRef.current.delete(message.id);
|
|
3104
|
-
pendingScreenshotRejectersRef.current.delete(message.id);
|
|
3105
|
-
break;
|
|
3106
|
-
}
|
|
3107
|
-
const screenshotData = toScreenshotData(message);
|
|
3108
|
-
if (!screenshotData) {
|
|
3109
|
-
debugWarn('Received screenshot message without image data:', message);
|
|
3110
|
-
break;
|
|
3111
|
-
}
|
|
3112
|
-
const resolver = pendingScreenshotResolversRef.current.get(message.id);
|
|
3113
|
-
if (!resolver) {
|
|
3114
|
-
debugWarn(`Received screenshot data for unknown or handled id: ${message.id}`);
|
|
3115
|
-
break;
|
|
3116
|
-
}
|
|
3117
|
-
debugLog(`Received screenshot data for id ${message.id}`);
|
|
3118
|
-
resolver(screenshotData);
|
|
3119
|
-
pendingScreenshotResolversRef.current.delete(message.id);
|
|
3120
|
-
pendingScreenshotRejectersRef.current.delete(message.id);
|
|
3121
|
-
break;
|
|
3122
|
-
}
|
|
3123
|
-
case 'screenshotError':
|
|
3124
|
-
if (typeof message.id !== 'string' || typeof message.message !== 'string') {
|
|
3125
|
-
debugWarn('Received invalid screenshot error message:', message);
|
|
3126
|
-
break;
|
|
3127
|
-
}
|
|
3128
|
-
const rejecter = pendingScreenshotRejectersRef.current.get(message.id);
|
|
3129
|
-
if (!rejecter) {
|
|
3130
|
-
debugWarn(`Received screenshot error for unknown or handled id: ${message.id}`);
|
|
3131
|
-
break;
|
|
3132
|
-
}
|
|
3133
|
-
debugWarn(`Received screenshot error for id ${message.id}: ${message.message}`);
|
|
3134
|
-
rejecter(new Error(message.message));
|
|
3135
|
-
pendingScreenshotResolversRef.current.delete(message.id);
|
|
3136
|
-
pendingScreenshotRejectersRef.current.delete(message.id);
|
|
3137
|
-
break;
|
|
3138
|
-
case 'cameraRequest': {
|
|
3139
|
-
// Only camera-capable sessions allocate the sendonly
|
|
3140
|
-
// outbound-camera transceiver; without it there is nothing to
|
|
3141
|
-
// feed the device, so ignore camera requests entirely — no
|
|
3142
|
-
// getUserMedia prompt and no onCameraDemandChange callbacks.
|
|
3143
|
-
const sender = outboundCameraSenderRef.current;
|
|
3144
|
-
if (!sender) {
|
|
3145
|
-
break;
|
|
3146
|
-
}
|
|
3147
|
-
const request = parseCameraRequest(message);
|
|
3148
|
-
if (!request) {
|
|
3149
|
-
debugWarn('Received invalid cameraRequest:', message);
|
|
3150
|
-
break;
|
|
3151
|
-
}
|
|
3152
|
-
// eslint-disable-next-line no-console
|
|
3153
|
-
console.info('[RemoteControl] cameraRequest received:', request.active, request.facingMode);
|
|
3154
|
-
await handleCameraRequest(request, ws, sender, isCurrentAttempt);
|
|
3155
|
-
break;
|
|
3156
|
-
}
|
|
3157
|
-
case 'microphoneRequest': {
|
|
3158
|
-
if (!outboundMicSenderRef.current) {
|
|
3159
|
-
break;
|
|
3160
|
-
}
|
|
3161
|
-
const active = message.active === true;
|
|
3162
|
-
microphoneDemandActiveRef.current = active;
|
|
3163
|
-
// eslint-disable-next-line no-console
|
|
3164
|
-
console.info('[RemoteControl] microphoneRequest received, active=', active);
|
|
3165
|
-
if (!active) {
|
|
3166
|
-
if (!microphoneEnabledRef.current) {
|
|
3167
|
-
await detachMicrophone();
|
|
3168
|
-
}
|
|
3169
|
-
break;
|
|
3170
|
-
}
|
|
3171
|
-
if (outboundMicStreamRef.current) {
|
|
3172
|
-
// A manual stream may already be attached. Re-select it on the
|
|
3173
|
-
// host because file playback can have replaced its source.
|
|
3174
|
-
sendMicrophoneStart(ws);
|
|
3175
|
-
break;
|
|
3176
|
-
}
|
|
3177
|
-
await attachMicrophone();
|
|
3178
|
-
break;
|
|
3179
|
-
}
|
|
3180
|
-
case 'microphoneStartResult': {
|
|
3181
|
-
if (micStartPendingIdRef.current === null || message.id !== micStartPendingIdRef.current) {
|
|
3182
|
-
break;
|
|
3183
|
-
}
|
|
3184
|
-
micStartPendingIdRef.current = null;
|
|
3185
|
-
if (typeof message.error === 'string') {
|
|
3186
|
-
// Host rejected the stream (e.g. no loopback device
|
|
3187
|
-
// assigned on the node). Tear the capture back down so
|
|
3188
|
-
// the browser's mic indicator doesn't stay lit for a
|
|
3189
|
-
// stream nobody hears.
|
|
3190
|
-
// eslint-disable-next-line no-console
|
|
3191
|
-
console.warn('[RemoteControl] microphoneStart rejected by host:', message.error);
|
|
3192
|
-
void detachMicrophone(message.error);
|
|
3193
|
-
}
|
|
3194
|
-
break;
|
|
3195
|
-
}
|
|
3196
|
-
case 'terminateAppResult':
|
|
3197
|
-
if (typeof message.id !== 'string') {
|
|
3198
|
-
debugWarn('Received invalid terminateApp result message:', message);
|
|
3199
|
-
break;
|
|
3200
|
-
}
|
|
3201
|
-
if (typeof message.error === 'string') {
|
|
3202
|
-
const terminateRejecter = pendingTerminateAppRejectersRef.current.get(message.id);
|
|
3203
|
-
if (!terminateRejecter) {
|
|
3204
|
-
debugWarn(`Received terminateApp error for unknown or handled id: ${message.id}`);
|
|
3205
|
-
break;
|
|
3206
|
-
}
|
|
3207
|
-
debugWarn(`Received terminateApp error for id ${message.id}: ${message.error}`);
|
|
3208
|
-
terminateRejecter(new Error(message.error));
|
|
3209
|
-
pendingTerminateAppResolversRef.current.delete(message.id);
|
|
3210
|
-
pendingTerminateAppRejectersRef.current.delete(message.id);
|
|
3211
|
-
break;
|
|
3212
|
-
}
|
|
3213
|
-
const terminateResolver = pendingTerminateAppResolversRef.current.get(message.id);
|
|
3214
|
-
if (!terminateResolver) {
|
|
3215
|
-
debugWarn(`Received terminateApp result for unknown or handled id: ${message.id}`);
|
|
3216
|
-
break;
|
|
3217
|
-
}
|
|
3218
|
-
debugLog(`Received terminateApp success for id ${message.id}`);
|
|
3219
|
-
terminateResolver();
|
|
3220
|
-
pendingTerminateAppResolversRef.current.delete(message.id);
|
|
3221
|
-
pendingTerminateAppRejectersRef.current.delete(message.id);
|
|
3222
|
-
break;
|
|
3223
|
-
default:
|
|
3224
|
-
debugWarn(`Received unhandled message type: ${message.type}`, message);
|
|
3225
|
-
break;
|
|
3226
|
-
}
|
|
3227
|
-
};
|
|
3228
|
-
|
|
3229
|
-
// Create and send offer
|
|
3230
|
-
if (peerConnectionRef.current === peerConnection) {
|
|
3231
|
-
const offer = await peerConnection.createOffer({
|
|
3232
|
-
offerToReceiveVideo: true,
|
|
3233
|
-
offerToReceiveAudio: false,
|
|
3234
|
-
});
|
|
3235
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
|
|
3236
|
-
return;
|
|
3237
|
-
}
|
|
3238
|
-
await peerConnection.setLocalDescription(offer);
|
|
3239
|
-
if (!isCurrentAttempt() || peerConnectionRef.current !== peerConnection) {
|
|
3240
|
-
return;
|
|
3241
|
-
}
|
|
3242
|
-
|
|
3243
|
-
if (isCurrentAttempt() && wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
|
|
3244
|
-
ws.send(
|
|
3245
|
-
JSON.stringify({
|
|
3246
|
-
type: 'offer',
|
|
3247
|
-
sdp: offer.sdp,
|
|
3248
|
-
sessionId: sessionId,
|
|
3249
|
-
}),
|
|
3250
|
-
);
|
|
3251
|
-
}
|
|
3252
|
-
updateStatus('Sent offer');
|
|
3253
|
-
}
|
|
3254
|
-
} catch (e) {
|
|
3255
|
-
if (!isCurrentAttempt()) {
|
|
3256
|
-
return;
|
|
3257
|
-
}
|
|
3258
|
-
const reason = e instanceof Error ? e.message : String(e);
|
|
3259
|
-
updateStatus('Error: ' + reason);
|
|
3260
|
-
scheduleRetry(reason, generation);
|
|
3261
|
-
}
|
|
3262
|
-
};
|
|
3263
|
-
|
|
3264
|
-
const start = () => {
|
|
3265
|
-
// A fresh connection (mount, prop change, manual retry, reconnect())
|
|
3266
|
-
// clears any prior terminated verdict so we re-evaluate from scratch.
|
|
3267
|
-
terminatedRef.current = false;
|
|
3268
|
-
setTerminated(false);
|
|
3269
|
-
void startAttempt(0);
|
|
3270
|
-
};
|
|
3271
|
-
|
|
3272
|
-
const stop = () => {
|
|
3273
|
-
connectionGenerationRef.current += 1;
|
|
3274
|
-
connectionAttemptRef.current = 0;
|
|
3275
|
-
controlChannelOpenedRef.current = false;
|
|
3276
|
-
clearScheduledRetry();
|
|
3277
|
-
clearIceDisconnectedGrace();
|
|
3278
|
-
teardownConnection();
|
|
3279
|
-
updateStatus('Stopped');
|
|
3280
|
-
};
|
|
3281
|
-
|
|
3282
|
-
const handleManualRetry = (event: React.MouseEvent<HTMLButtonElement>) => {
|
|
3283
|
-
event.stopPropagation();
|
|
3284
|
-
start();
|
|
3285
|
-
};
|
|
3286
|
-
|
|
3287
|
-
// Drive the manual override without interrupting guest-driven demand.
|
|
3288
|
-
useEffect(() => {
|
|
3289
|
-
if (isMicrophoneWanted()) {
|
|
3290
|
-
void attachMicrophone();
|
|
3291
|
-
} else {
|
|
3292
|
-
void detachMicrophone();
|
|
3293
|
-
}
|
|
3294
|
-
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
3295
|
-
}, [microphoneEnabled]);
|
|
3296
|
-
|
|
3297
|
-
useEffect(() => {
|
|
3298
|
-
// Reset video loaded state when connection params change
|
|
3299
|
-
setVideoLoaded(false);
|
|
3300
|
-
|
|
3301
|
-
// Start connection when component mounts
|
|
3302
|
-
start();
|
|
3303
|
-
|
|
3304
|
-
// Only start keepAlive if page is visible
|
|
3305
|
-
if (!document.hidden) {
|
|
3306
|
-
startKeepAlive();
|
|
3307
|
-
}
|
|
3308
|
-
|
|
3309
|
-
// Add visibility change listener
|
|
3310
|
-
document.addEventListener('visibilitychange', handleVisibilityChange);
|
|
3311
|
-
|
|
3312
|
-
// Clean up
|
|
3313
|
-
return () => {
|
|
3314
|
-
stopKeepAlive();
|
|
3315
|
-
stop();
|
|
3316
|
-
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
|
3317
|
-
};
|
|
3318
|
-
// Camera attach/detach happens entirely inside the WS message
|
|
3319
|
-
// loop now (sendonly transceiver + `replaceTrack`), so the
|
|
3320
|
-
// connection effect doesn't need to bounce when the camera
|
|
3321
|
-
// turns on/off — no SDP-affecting change.
|
|
3322
|
-
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
3323
|
-
}, [url, token, propSessionId]);
|
|
3324
|
-
|
|
3325
|
-
// Recompute the inspect-overlay geometry (container-local pixel rect of
|
|
3326
|
-
// the actually-rendered video content) from the current mapping context.
|
|
3327
|
-
// The InfoCard places itself in viewport coordinates from pointer events
|
|
3328
|
-
// directly, so no viewport-space origin is needed in the geometry.
|
|
3329
|
-
const recomputeOverlayGeometry = () => {
|
|
3330
|
-
const ctx = computeVideoMappingContext();
|
|
3331
|
-
if (!ctx) {
|
|
3332
|
-
setOverlayGeometry(null);
|
|
3333
|
-
return;
|
|
3334
|
-
}
|
|
3335
|
-
const next: InspectOverlayGeometry = {
|
|
3336
|
-
left: ctx.videoRect.left - ctx.containerRect.left + ctx.offsetX,
|
|
3337
|
-
top: ctx.videoRect.top - ctx.containerRect.top + ctx.offsetY,
|
|
3338
|
-
width: ctx.actualWidth,
|
|
3339
|
-
height: ctx.actualHeight,
|
|
3340
|
-
};
|
|
3341
|
-
setOverlayGeometry((prev) =>
|
|
3342
|
-
(
|
|
3343
|
-
prev &&
|
|
3344
|
-
prev.left === next.left &&
|
|
3345
|
-
prev.top === next.top &&
|
|
3346
|
-
prev.width === next.width &&
|
|
3347
|
-
prev.height === next.height
|
|
3348
|
-
) ?
|
|
3349
|
-
prev
|
|
3350
|
-
: next,
|
|
3351
|
-
);
|
|
3352
|
-
};
|
|
3353
|
-
|
|
3354
|
-
// Calculate video position and border-radius based on frame dimensions
|
|
3355
|
-
useEffect(() => {
|
|
3356
|
-
const video = videoRef.current;
|
|
3357
|
-
const frame = frameRef.current;
|
|
3358
|
-
const container = containerRef.current;
|
|
3359
|
-
|
|
3360
|
-
if (!video) return;
|
|
3361
|
-
|
|
3362
|
-
const updateVideoPosition = () => {
|
|
3363
|
-
// If no frame, just refresh overlay geometry; no inset/letterbox math
|
|
3364
|
-
// is needed since the video element is its own size.
|
|
3365
|
-
if (!frameVisible || !frame) {
|
|
3366
|
-
setVideoStyle({});
|
|
3367
|
-
recomputeOverlayGeometry();
|
|
3368
|
-
return;
|
|
3369
|
-
}
|
|
3370
|
-
|
|
3371
|
-
const frameWidth = frame.clientWidth;
|
|
3372
|
-
const frameHeight = frame.clientHeight;
|
|
3373
|
-
|
|
3374
|
-
if (frameWidth === 0 || frameHeight === 0) return;
|
|
3375
|
-
|
|
3376
|
-
// Determine landscape based on video's intrinsic dimensions
|
|
3377
|
-
const landscape = video.videoWidth > video.videoHeight;
|
|
3378
|
-
setIsLandscape(landscape);
|
|
3379
|
-
setUseAndroidTabletFrame(
|
|
3380
|
-
platform === 'android' && isAndroidTabletVideo(video.videoWidth, video.videoHeight),
|
|
3381
|
-
);
|
|
3382
|
-
|
|
3383
|
-
const pos = landscape ? config.videoPosition.landscape : config.videoPosition.portrait;
|
|
3384
|
-
let newStyle: React.CSSProperties = {};
|
|
3385
|
-
if (pos.heightMultiplier) {
|
|
3386
|
-
newStyle.height = `${frameHeight * pos.heightMultiplier}px`;
|
|
3387
|
-
// Let the other dimension follow the video stream's intrinsic aspect ratio.
|
|
3388
|
-
newStyle.width = 'auto';
|
|
3389
|
-
} else if (pos.widthMultiplier) {
|
|
3390
|
-
newStyle.width = `${frameWidth * pos.widthMultiplier}px`;
|
|
3391
|
-
// Let the other dimension follow the video stream's intrinsic aspect ratio.
|
|
3392
|
-
newStyle.height = 'auto';
|
|
3393
|
-
}
|
|
3394
|
-
newStyle.borderRadius = `${
|
|
3395
|
-
landscape ?
|
|
3396
|
-
frameHeight * config.videoBorderRadiusMultiplier
|
|
3397
|
-
: frameWidth * config.videoBorderRadiusMultiplier
|
|
3398
|
-
}px`;
|
|
3399
|
-
setVideoStyle(newStyle);
|
|
3400
|
-
recomputeOverlayGeometry();
|
|
3401
|
-
};
|
|
3402
|
-
|
|
3403
|
-
const resizeObserver = new ResizeObserver(() => {
|
|
3404
|
-
updateVideoPosition();
|
|
3405
|
-
});
|
|
3406
|
-
|
|
3407
|
-
if (frame) resizeObserver.observe(frame);
|
|
3408
|
-
resizeObserver.observe(video);
|
|
3409
|
-
if (container) resizeObserver.observe(container);
|
|
3410
|
-
|
|
3411
|
-
// Also update when the frame image loads
|
|
3412
|
-
if (frame) frame.addEventListener('load', updateVideoPosition);
|
|
3413
|
-
|
|
3414
|
-
// Update when video metadata loads (to get correct intrinsic dimensions)
|
|
3415
|
-
video.addEventListener('loadedmetadata', updateVideoPosition);
|
|
3416
|
-
|
|
3417
|
-
// IMPORTANT: When the WebRTC stream changes orientation, the intrinsic video size
|
|
3418
|
-
// (videoWidth/videoHeight) can change without re-firing 'loadedmetadata'.
|
|
3419
|
-
// The <video> element emits 'resize' in that case.
|
|
3420
|
-
video.addEventListener('resize', updateVideoPosition);
|
|
3421
|
-
// Orientation flips also mean every element's AX frame just changed
|
|
3422
|
-
// (portrait↔landscape rotates the layout). Bump so the overlay
|
|
3423
|
-
// refreshes immediately rather than waiting out the current poll
|
|
3424
|
-
// cycle in a layout that no longer matches the boxes.
|
|
3425
|
-
const bumpOnResize = () => axFetcherRef.current?.bumpActivity();
|
|
3426
|
-
video.addEventListener('resize', bumpOnResize);
|
|
3427
|
-
|
|
3428
|
-
// Initial calculation
|
|
3429
|
-
updateVideoPosition();
|
|
3430
|
-
|
|
3431
|
-
return () => {
|
|
3432
|
-
resizeObserver.disconnect();
|
|
3433
|
-
video.removeEventListener('loadedmetadata', updateVideoPosition);
|
|
3434
|
-
video.removeEventListener('resize', updateVideoPosition);
|
|
3435
|
-
video.removeEventListener('resize', bumpOnResize);
|
|
3436
|
-
if (frame) frame.removeEventListener('load', updateVideoPosition);
|
|
3437
|
-
};
|
|
3438
|
-
}, [config, frameVisible]);
|
|
3439
|
-
|
|
3440
|
-
// Start/stop the AX poller and reset inspect state when inspect mode
|
|
3441
|
-
// toggles. Connection state is independent: the fetcher gets created on
|
|
3442
|
-
// dataChannel.onopen and destroyed on teardown.
|
|
3443
|
-
useEffect(() => {
|
|
3444
|
-
const fetcher = axFetcherRef.current;
|
|
3445
|
-
if (inspectActive) {
|
|
3446
|
-
fetcher?.start();
|
|
3447
|
-
} else {
|
|
3448
|
-
fetcher?.stop();
|
|
3449
|
-
setAxSnapshot(null);
|
|
3450
|
-
setAxHighlightedId(null);
|
|
3451
|
-
setAxSelectedId(null);
|
|
3452
|
-
setAxCursorPosition(null);
|
|
3453
|
-
setAxFrozenCursorPosition(null);
|
|
3454
|
-
cursorPositionRef.current = null;
|
|
3455
|
-
if (cursorRafIdRef.current !== undefined) {
|
|
3456
|
-
window.cancelAnimationFrame(cursorRafIdRef.current);
|
|
3457
|
-
cursorRafIdRef.current = undefined;
|
|
3458
|
-
}
|
|
3459
|
-
safeInvoke('onAxSnapshotChange', onAxSnapshotChangeRef.current, null);
|
|
3460
|
-
}
|
|
3461
|
-
}, [inspectActive]);
|
|
3462
|
-
|
|
3463
|
-
// Cancel any pending cursor-rAF on unmount so we don't setState on a
|
|
3464
|
-
// dead component.
|
|
3465
|
-
useEffect(() => {
|
|
3466
|
-
return () => {
|
|
3467
|
-
if (cursorRafIdRef.current !== undefined) {
|
|
3468
|
-
window.cancelAnimationFrame(cursorRafIdRef.current);
|
|
3469
|
-
cursorRafIdRef.current = undefined;
|
|
3470
|
-
}
|
|
3471
|
-
};
|
|
3472
|
-
}, []);
|
|
3473
|
-
|
|
3474
|
-
// ESC clears overlay selection (Chrome DevTools behavior).
|
|
3475
|
-
useEffect(() => {
|
|
3476
|
-
if (!inspectActive) return;
|
|
3477
|
-
const handleEsc = (e: KeyboardEvent) => {
|
|
3478
|
-
if (e.key === 'Escape' && (axSelectedId || axHighlightedId)) {
|
|
3479
|
-
setAxSelectedId(null);
|
|
3480
|
-
setAxHighlightedId(null);
|
|
3481
|
-
setAxFrozenCursorPosition(null);
|
|
3482
|
-
safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, null);
|
|
3483
|
-
}
|
|
3484
|
-
};
|
|
3485
|
-
window.addEventListener('keydown', handleEsc);
|
|
3486
|
-
return () => window.removeEventListener('keydown', handleEsc);
|
|
3487
|
-
}, [inspectActive, axSelectedId, axHighlightedId]);
|
|
3488
|
-
|
|
3489
|
-
const handleVideoClick = () => {
|
|
3490
|
-
if (videoRef.current) {
|
|
3491
|
-
videoRef.current.focus();
|
|
3492
|
-
}
|
|
3493
|
-
};
|
|
3494
|
-
|
|
3495
|
-
// Expose sendOpenUrlCommand via ref
|
|
3496
|
-
useImperativeHandle(ref, () => ({
|
|
3497
|
-
openUrl: (newUrl: string) => {
|
|
3498
|
-
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
|
|
3499
|
-
debugWarn('WebSocket not open, cannot send open_url command via ref.');
|
|
3500
|
-
return;
|
|
3501
|
-
}
|
|
3502
|
-
try {
|
|
3503
|
-
const decodedUrl = decodeURIComponent(newUrl);
|
|
3504
|
-
updateStatus('Opening URL');
|
|
3505
|
-
wsRef.current.send(
|
|
3506
|
-
JSON.stringify({
|
|
3507
|
-
type: 'openUrl',
|
|
3508
|
-
url: decodedUrl,
|
|
3509
|
-
sessionId: sessionId,
|
|
3510
|
-
}),
|
|
3511
|
-
);
|
|
3512
|
-
} catch (error) {
|
|
3513
|
-
debugWarn('Error decoding or sending URL via ref:', { error, url: newUrl });
|
|
3514
|
-
wsRef.current.send(
|
|
3515
|
-
JSON.stringify({
|
|
3516
|
-
type: 'openUrl',
|
|
3517
|
-
url: newUrl,
|
|
3518
|
-
sessionId: sessionId,
|
|
3519
|
-
}),
|
|
3520
|
-
);
|
|
3521
|
-
}
|
|
3522
|
-
axFetcherRef.current?.bumpActivity();
|
|
3523
|
-
},
|
|
3524
|
-
|
|
3525
|
-
sendKeyEvent: (event: ImperativeKeyboardEvent) => {
|
|
3526
|
-
if (!dataChannelRef.current || dataChannelRef.current.readyState !== 'open') {
|
|
3527
|
-
debugWarn('Data channel not ready for imperative key command:', dataChannelRef.current?.readyState);
|
|
3528
|
-
return;
|
|
3529
|
-
}
|
|
3530
|
-
|
|
3531
|
-
const keycode = codeMap[event.code];
|
|
3532
|
-
if (!keycode) {
|
|
3533
|
-
debugWarn(`Unknown event.code for imperative command: ${event.code}`);
|
|
3534
|
-
return;
|
|
3535
|
-
}
|
|
3536
|
-
|
|
3537
|
-
let metaState = ANDROID_KEYS.META_NONE;
|
|
3538
|
-
if (event.shiftKey) metaState |= ANDROID_KEYS.META_SHIFT_ON;
|
|
3539
|
-
if (event.altKey) metaState |= ANDROID_KEYS.META_ALT_ON;
|
|
3540
|
-
if (event.ctrlKey) metaState |= ANDROID_KEYS.META_CTRL_ON;
|
|
3541
|
-
if (event.metaKey) metaState |= ANDROID_KEYS.META_META_ON;
|
|
3542
|
-
|
|
3543
|
-
const action = event.type === 'keydown' ? ANDROID_KEYS.ACTION_DOWN : ANDROID_KEYS.ACTION_UP;
|
|
3544
|
-
|
|
3545
|
-
debugLog(
|
|
3546
|
-
`Sending Imperative Key Command: code=${event.code}, keycode=${keycode}, action=${action}, meta=${metaState}`,
|
|
3547
|
-
);
|
|
3548
|
-
|
|
3549
|
-
const message = createInjectKeycodeMessage(
|
|
3550
|
-
action,
|
|
3551
|
-
keycode,
|
|
3552
|
-
0, // repeat count, typically 0 for single presses
|
|
3553
|
-
metaState,
|
|
3554
|
-
);
|
|
3555
|
-
if (message) {
|
|
3556
|
-
sendBinaryControlMessage(message);
|
|
3557
|
-
}
|
|
3558
|
-
},
|
|
3559
|
-
screenshot: (): Promise<ScreenshotData> => {
|
|
3560
|
-
return new Promise<ScreenshotData>((resolve, reject) => {
|
|
3561
|
-
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
|
|
3562
|
-
debugWarn('WebSocket not open, cannot send screenshot command.');
|
|
3563
|
-
return reject(new Error('WebSocket is not connected or connection is not open.'));
|
|
3564
|
-
}
|
|
3565
|
-
|
|
3566
|
-
const id = `ui-ss-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
|
|
3567
|
-
const request = {
|
|
3568
|
-
type: 'screenshot', // Matches the type expected by instance API
|
|
3569
|
-
id: id,
|
|
3570
|
-
};
|
|
3571
|
-
|
|
3572
|
-
pendingScreenshotResolversRef.current.set(id, resolve);
|
|
3573
|
-
pendingScreenshotRejectersRef.current.set(id, reject);
|
|
3574
|
-
|
|
3575
|
-
debugLog('Sending screenshot request:', request);
|
|
3576
|
-
try {
|
|
3577
|
-
wsRef.current.send(JSON.stringify(request));
|
|
3578
|
-
} catch (err) {
|
|
3579
|
-
debugWarn('Failed to send screenshot request immediately:', err);
|
|
3580
|
-
pendingScreenshotResolversRef.current.delete(id);
|
|
3581
|
-
pendingScreenshotRejectersRef.current.delete(id);
|
|
3582
|
-
reject(err);
|
|
3583
|
-
return; // Important to return here if send failed synchronously
|
|
3584
|
-
}
|
|
3585
|
-
|
|
3586
|
-
setTimeout(() => {
|
|
3587
|
-
if (pendingScreenshotResolversRef.current.has(id)) {
|
|
3588
|
-
debugWarn(`Screenshot request timed out for id ${id}`);
|
|
3589
|
-
pendingScreenshotRejectersRef.current.get(id)?.(new Error('Screenshot request timed out'));
|
|
3590
|
-
pendingScreenshotResolversRef.current.delete(id);
|
|
3591
|
-
pendingScreenshotRejectersRef.current.delete(id);
|
|
3592
|
-
}
|
|
3593
|
-
}, 30000); // 30-second timeout
|
|
3594
|
-
});
|
|
3595
|
-
},
|
|
3596
|
-
terminateApp: (bundleId: string): Promise<void> => {
|
|
3597
|
-
return new Promise<void>((resolve, reject) => {
|
|
3598
|
-
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
|
|
3599
|
-
debugWarn('WebSocket not open, cannot send terminateApp command.');
|
|
3600
|
-
return reject(new Error('WebSocket is not connected or connection is not open.'));
|
|
3601
|
-
}
|
|
3602
|
-
const id = `ui-term-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
|
|
3603
|
-
const request = {
|
|
3604
|
-
type: 'terminateApp',
|
|
3605
|
-
id,
|
|
3606
|
-
bundleId,
|
|
3607
|
-
};
|
|
3608
|
-
|
|
3609
|
-
pendingTerminateAppResolversRef.current.set(id, resolve);
|
|
3610
|
-
pendingTerminateAppRejectersRef.current.set(id, reject);
|
|
3611
|
-
|
|
3612
|
-
debugLog('Sending terminateApp request:', request);
|
|
3613
|
-
try {
|
|
3614
|
-
wsRef.current.send(JSON.stringify(request));
|
|
3615
|
-
} catch (err) {
|
|
3616
|
-
debugWarn('Failed to send terminateApp request immediately:', err);
|
|
3617
|
-
pendingTerminateAppResolversRef.current.delete(id);
|
|
3618
|
-
pendingTerminateAppRejectersRef.current.delete(id);
|
|
3619
|
-
reject(err);
|
|
3620
|
-
return;
|
|
3621
|
-
}
|
|
3622
|
-
// Terminating the foreground app drops the user back to the home
|
|
3623
|
-
// screen — bump so the overlay reflects the post-terminate state
|
|
3624
|
-
// through the SpringBoard transition.
|
|
3625
|
-
axFetcherRef.current?.bumpActivity();
|
|
3626
|
-
|
|
3627
|
-
setTimeout(() => {
|
|
3628
|
-
if (pendingTerminateAppResolversRef.current.has(id)) {
|
|
3629
|
-
debugWarn(`terminateApp request timed out for id ${id}`);
|
|
3630
|
-
pendingTerminateAppRejectersRef.current.get(id)?.(new Error('terminateApp request timed out'));
|
|
3631
|
-
pendingTerminateAppResolversRef.current.delete(id);
|
|
3632
|
-
pendingTerminateAppRejectersRef.current.delete(id);
|
|
3633
|
-
}
|
|
3634
|
-
}, 30000);
|
|
3635
|
-
});
|
|
3636
|
-
},
|
|
3637
|
-
reconnect: () => start(),
|
|
3638
|
-
|
|
3639
|
-
refreshAxTree: async (): Promise<AxSnapshot> => {
|
|
3640
|
-
const fetcher = axFetcherRef.current;
|
|
3641
|
-
if (!fetcher) {
|
|
3642
|
-
throw new Error('Inspect mode is not active');
|
|
3643
|
-
}
|
|
3644
|
-
// The fetcher's refresh() runs the result through the same
|
|
3645
|
-
// change-detect path as the poll loop (via deliver()), which calls
|
|
3646
|
-
// back into onSnapshot — already wired to setAxSnapshot +
|
|
3647
|
-
// onAxSnapshotChange (with safe-invoke). We just return the fetched
|
|
3648
|
-
// payload for callers that want it.
|
|
3649
|
-
return fetcher.refresh();
|
|
3650
|
-
},
|
|
3651
|
-
|
|
3652
|
-
getAxSnapshot: () => axSnapshotRef.current,
|
|
3653
|
-
|
|
3654
|
-
setInspectHighlight: (element: AxElement | null) => {
|
|
3655
|
-
setAxHighlightedId(element?.id ?? null);
|
|
3656
|
-
},
|
|
3657
|
-
|
|
3658
|
-
setInspectSelection: (element: AxElement | null) => {
|
|
3659
|
-
setAxSelectedId(element?.id ?? null);
|
|
3660
|
-
// Programmatic selection has no click position — anchor the card at
|
|
3661
|
-
// the last known cursor position (if any), otherwise clear.
|
|
3662
|
-
// Customer-facing UIs that drive selection from their own panels can
|
|
3663
|
-
// call setInspectHighlight separately to move the cursor visual.
|
|
3664
|
-
if (element) {
|
|
3665
|
-
setAxFrozenCursorPosition(cursorPositionRef.current);
|
|
3666
|
-
} else {
|
|
3667
|
-
setAxFrozenCursorPosition(null);
|
|
3668
|
-
}
|
|
3669
|
-
const snapshot = axSnapshotRef.current;
|
|
3670
|
-
if (element && snapshot) {
|
|
3671
|
-
safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, { element, snapshot });
|
|
3672
|
-
} else {
|
|
3673
|
-
safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, null);
|
|
3674
|
-
}
|
|
3675
|
-
},
|
|
3676
|
-
|
|
3677
|
-
getAxStatus: () => axFetcherRef.current?.getStatus() ?? 'idle',
|
|
3678
|
-
}));
|
|
3679
|
-
|
|
3680
|
-
// Show indicators when Alt is held and we have a valid hover point (null when outside)
|
|
3681
|
-
const showAltIndicators = isAltHeld && hoverPoint !== null;
|
|
3682
|
-
|
|
3683
|
-
return (
|
|
3684
|
-
<div
|
|
3685
|
-
ref={containerRef}
|
|
3686
|
-
className={clsx('rc-container', className)}
|
|
3687
|
-
style={{ touchAction: 'none' }} // Keep touchAction none for the container
|
|
3688
|
-
// Attach unified handler to all interaction events on the container
|
|
3689
|
-
// This helps capture mouseleave correctly even if the video element itself isn't hovered
|
|
3690
|
-
onMouseDown={handleInteraction}
|
|
3691
|
-
onMouseMove={handleInteraction}
|
|
3692
|
-
onMouseUp={handleInteraction}
|
|
3693
|
-
onMouseLeave={handleInteraction} // Handle mouse leaving the container
|
|
3694
|
-
onTouchStart={handleInteraction}
|
|
3695
|
-
onTouchMove={handleInteraction}
|
|
3696
|
-
onTouchEnd={handleInteraction}
|
|
3697
|
-
onTouchCancel={handleInteraction}
|
|
3698
|
-
>
|
|
3699
|
-
{showAltIndicators && hoverPoint && (
|
|
3700
|
-
<>
|
|
3701
|
-
<div
|
|
3702
|
-
className="rc-touch-indicator"
|
|
3703
|
-
style={{
|
|
3704
|
-
left: `${hoverPoint.containerX}px`,
|
|
3705
|
-
top: `${hoverPoint.containerY}px`,
|
|
3706
|
-
}}
|
|
3707
|
-
/>
|
|
3708
|
-
<div
|
|
3709
|
-
className="rc-touch-indicator"
|
|
3710
|
-
style={{
|
|
3711
|
-
left: `${hoverPoint.mirrorContainerX}px`,
|
|
3712
|
-
top: `${hoverPoint.mirrorContainerY}px`,
|
|
3713
|
-
}}
|
|
3714
|
-
/>
|
|
3715
|
-
</>
|
|
3716
|
-
)}
|
|
3717
|
-
{frameVisible && (
|
|
3718
|
-
<img
|
|
3719
|
-
ref={frameRef}
|
|
3720
|
-
src={frameImageSrc}
|
|
3721
|
-
alt=""
|
|
3722
|
-
className={platform === 'ios' ? clsx('rc-phone-frame', 'rc-phone-frame-ios') : 'rc-phone-frame'}
|
|
3723
|
-
draggable={false}
|
|
3724
|
-
/>
|
|
3725
|
-
)}
|
|
3726
|
-
<video
|
|
3727
|
-
ref={videoRef}
|
|
3728
|
-
className={clsx(
|
|
3729
|
-
'rc-video',
|
|
3730
|
-
!frameVisible && 'rc-video-frameless',
|
|
3731
|
-
!videoLoaded && 'rc-video-loading',
|
|
3732
|
-
)}
|
|
3733
|
-
style={{
|
|
3734
|
-
...videoStyle,
|
|
3735
|
-
...(loadingLogo ?
|
|
3736
|
-
{
|
|
3737
|
-
backgroundImage: `url("${loadingLogo}")`,
|
|
3738
|
-
backgroundRepeat: 'no-repeat',
|
|
3739
|
-
backgroundPosition: 'center',
|
|
3740
|
-
backgroundSize: config.loadingLogoSize,
|
|
3741
|
-
}
|
|
3742
|
-
: {}),
|
|
3743
|
-
}}
|
|
3744
|
-
autoPlay
|
|
3745
|
-
playsInline
|
|
3746
|
-
muted
|
|
3747
|
-
tabIndex={0}
|
|
3748
|
-
onKeyDown={handleKeyboard}
|
|
3749
|
-
onKeyUp={handleKeyboard}
|
|
3750
|
-
onClick={handleVideoClick}
|
|
3751
|
-
onLoadedData={markFirstFrameShown}
|
|
3752
|
-
onFocus={() => {
|
|
3753
|
-
if (videoRef.current) {
|
|
3754
|
-
videoRef.current.style.outline = 'none';
|
|
3755
|
-
}
|
|
3756
|
-
}}
|
|
3757
|
-
onBlur={() => {
|
|
3758
|
-
if (videoRef.current) {
|
|
3759
|
-
videoRef.current.style.outline = 'none';
|
|
3760
|
-
}
|
|
3761
|
-
}}
|
|
3762
|
-
/>
|
|
3763
|
-
{inspectActive && (
|
|
3764
|
-
<InspectOverlay
|
|
3765
|
-
snapshot={axSnapshot}
|
|
3766
|
-
geometry={overlayGeometry}
|
|
3767
|
-
highlightedId={axHighlightedId}
|
|
3768
|
-
selectedId={axSelectedId}
|
|
3769
|
-
mode={inspectModeResolved}
|
|
3770
|
-
cursorPosition={axCursorPosition}
|
|
3771
|
-
frozenCursorPosition={axFrozenCursorPosition}
|
|
3772
|
-
onSelectChange={(element, clickPosition) => {
|
|
3773
|
-
setAxSelectedId(element?.id ?? null);
|
|
3774
|
-
if (element && clickPosition) {
|
|
3775
|
-
setAxFrozenCursorPosition(clickPosition);
|
|
3776
|
-
} else if (!element) {
|
|
3777
|
-
setAxFrozenCursorPosition(null);
|
|
3778
|
-
}
|
|
3779
|
-
const snapshot = axSnapshotRef.current;
|
|
3780
|
-
if (element && snapshot) {
|
|
3781
|
-
safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, {
|
|
3782
|
-
element,
|
|
3783
|
-
snapshot,
|
|
3784
|
-
});
|
|
3785
|
-
} else {
|
|
3786
|
-
safeInvoke('onInspectSelectionChange', onInspectSelectionChangeRef.current, null);
|
|
3787
|
-
}
|
|
3788
|
-
}}
|
|
3789
|
-
onTapElement={(element, tapAt) => {
|
|
3790
|
-
// Use the viewport-space position the user originally aimed at
|
|
3791
|
-
// (the frozen click position). For containers whose children
|
|
3792
|
-
// are absent from the accessibility tree — e.g. iOS UITabBar's
|
|
3793
|
-
// home/diagnostics/settings buttons — this taps the specific
|
|
3794
|
-
// button the user pointed at instead of the container's
|
|
3795
|
-
// averaged center.
|
|
3796
|
-
if (tapAt) {
|
|
3797
|
-
sendTapAtClient(tapAt.x, tapAt.y);
|
|
3798
|
-
return;
|
|
3799
|
-
}
|
|
3800
|
-
// Fallback (defensive): center of element. Should be
|
|
3801
|
-
// unreachable from the InfoCard since it always passes anchor.
|
|
3802
|
-
const snapshot = axSnapshotRef.current;
|
|
3803
|
-
if (!snapshot) return;
|
|
3804
|
-
sendTapAtElementCenter(element, snapshot);
|
|
3805
|
-
}}
|
|
3806
|
-
/>
|
|
3807
|
-
)}
|
|
3808
|
-
{terminated ?
|
|
3809
|
-
<div className="rc-terminated" role="status">
|
|
3810
|
-
<div className="rc-terminated-title">Instance terminated</div>
|
|
3811
|
-
<div className="rc-terminated-message">
|
|
3812
|
-
This instance no longer exists and can't be reconnected.
|
|
3813
|
-
</div>
|
|
3814
|
-
</div>
|
|
3815
|
-
: retryExhausted && (
|
|
3816
|
-
<button type="button" className="rc-retry-button" onClick={handleManualRetry}>
|
|
3817
|
-
Retry
|
|
3818
|
-
</button>
|
|
3819
|
-
)
|
|
3820
|
-
}
|
|
3821
|
-
</div>
|
|
3822
|
-
);
|
|
3823
|
-
},
|
|
3824
|
-
);
|
|
3825
|
-
|
|
3826
|
-
const getScreenshotError = (message: any): string | null => {
|
|
3827
|
-
if (typeof message.message === 'string') {
|
|
3828
|
-
return message.message;
|
|
3829
|
-
}
|
|
3830
|
-
|
|
3831
|
-
if (typeof message.error === 'string') {
|
|
3832
|
-
return message.error;
|
|
3833
|
-
}
|
|
3834
|
-
|
|
3835
|
-
return null;
|
|
3836
|
-
};
|
|
3837
|
-
|
|
3838
|
-
const toScreenshotData = (message: any): ScreenshotData | null => {
|
|
3839
|
-
if (typeof message.dataUri === 'string') {
|
|
3840
|
-
return { dataUri: message.dataUri };
|
|
3841
|
-
}
|
|
3842
|
-
|
|
3843
|
-
if (typeof message.base64 === 'string') {
|
|
3844
|
-
if (message.base64.startsWith('data:')) {
|
|
3845
|
-
return { dataUri: message.base64 };
|
|
3846
|
-
}
|
|
3847
|
-
|
|
3848
|
-
const mimeType = message.base64.startsWith('/9j/') ? 'image/jpeg' : 'image/png';
|
|
3849
|
-
return { dataUri: `data:${mimeType};base64,${message.base64}` };
|
|
3850
|
-
}
|
|
3851
|
-
|
|
3852
|
-
return null;
|
|
3853
|
-
};
|