@limrun/ui 0.15.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,17 +18,28 @@ import {
18
18
  createSetClipboardMessage,
19
19
  createTwoFingerTouchControlMessage,
20
20
  } from '../core/webrtc-messages';
21
- import { AxFetcher, AxStatus } from '../core/ax-fetcher';
21
+ import { AxFetcher, AxStatus, type AndroidElementTreeOptions } from '../core/ax-fetcher';
22
22
  import { AxElement, AxSnapshot, axElementAtPoint, axSnapshotsEqual } from '../core/ax-tree';
23
+ import {
24
+ CameraOperationCoordinator,
25
+ cameraCaptureConstraints,
26
+ createGrantedCameraResult,
27
+ parseCameraRequest,
28
+ shouldReacquireCamera,
29
+ type CameraCaptureInfo,
30
+ type CameraRequest,
31
+ } from '../core/camera';
23
32
  import { InspectOverlay, InspectOverlayGeometry, InspectMode } from './inspect-overlay';
24
33
 
34
+ export type { CameraCaptureInfo, CameraFacingMode, CameraRequest, CameraResult } from '../core/camera';
35
+
25
36
  declare global {
26
37
  interface Window {
27
38
  debugRemoteControl?: boolean;
28
39
  }
29
40
  }
30
41
 
31
- interface RemoteControlProps {
42
+ export interface RemoteControlProps {
32
43
  // url is the URL of the instance to connect to.
33
44
  url: string;
34
45
 
@@ -154,6 +165,8 @@ interface RemoteControlProps {
154
165
  */
155
166
  axMaxBackoffMs?: number;
156
167
 
168
+ androidElementTreeOptions?: AndroidElementTreeOptions;
169
+
157
170
  /**
158
171
  * Fires whenever the iOS simulator's camera demand state changes —
159
172
  * i.e. an app inside the sim called
@@ -163,12 +176,10 @@ interface RemoteControlProps {
163
176
  * UI can render a status indicator ("simulator is using your
164
177
  * camera", etc.).
165
178
  *
166
- * `active` reflects whether the sim is currently asking for
167
- * frames. `granted` is set only on the call that follows a
168
- * `getUserMedia` attempt: `true` when the user accepted the
169
- * browser prompt, `false` when they denied or the call failed
170
- * (in which case the limulator side switches to a black-frame
171
- * fallback so the app keeps ticking).
179
+ * `active` reflects whether the sim is currently asking for frames.
180
+ * `granted` is set on capture acknowledgements: `true` after a new
181
+ * track, reused track, or constraint update; `false` when capture
182
+ * fails. The host switches to black frames after a failure.
172
183
  *
173
184
  * `camera` (optional, only meaningful when `granted === true`)
174
185
  * carries what `MediaStreamTrack.getSettings()` reported for the
@@ -200,41 +211,6 @@ interface RemoteControlProps {
200
211
  */
201
212
  onCameraStats?: (stats: CameraStreamStats | null) => void;
202
213
 
203
- /**
204
- * Optional resolution cap for outbound camera capture.
205
- *
206
- * - `'auto'` (default): no extra constraint. The browser captures
207
- * at its webcam's native max; WebRTC's quality scaler may step
208
- * down resolution on the encoder side under bandwidth pressure
209
- * while still feeding the simulator at the pool's native size.
210
- * - `'1080p'` / `'720p'` / `'480p'`: hard cap applied via
211
- * `getUserMedia` constraints (for new captures) and
212
- * `track.applyConstraints` (for the currently-active track),
213
- * matching the way Meet/Zoom expose a "Send resolution" picker.
214
- *
215
- * Bumping or lowering the cap mid-stream is supported; the
216
- * change takes effect within a frame or two as the webcam
217
- * re-negotiates.
218
- */
219
- cameraResolutionCap?: CameraResolutionCap;
220
- /**
221
- * Aspect ratio the simulator's virtual camera should report to apps.
222
- * Picking a value here triggers a `cameraAspect` WS message to the
223
- * host, which rebuilds its IOSurface ring at the matching dimensions
224
- * (16:9 → 1920×1080, 4:3 → 1440×1080, 1:1 → 1080×1080, 9:16 →
225
- * 1080×1920) and signals the in-sim dylib to re-handshake. iOS apps
226
- * see CMSampleBuffers at the new dimensions within a frame or two.
227
- *
228
- * The browser still captures whatever the webcam offers; the host
229
- * aspect-fills (cover, center-crop) into the new pool. Switching
230
- * aspect at runtime is intentionally cheap so users can A/B preview
231
- * styles without restarting the simulator.
232
- *
233
- * `undefined` leaves the pool untouched (the host's boot default —
234
- * 16:9 / 1920×1080 — applies).
235
- */
236
- cameraAspect?: CameraAspect;
237
-
238
214
  /**
239
215
  * When true, capture the user's microphone via
240
216
  * `getUserMedia({ audio: true })` and stream it live into the
@@ -273,37 +249,6 @@ export interface MicrophoneState {
273
249
  label?: string;
274
250
  }
275
251
 
276
- /**
277
- * Resolution caps a host app can request on the outbound camera.
278
- * `'auto'` is "let the browser decide" (no constraints beyond the
279
- * 30 fps ceiling); the other options clamp width/height to the
280
- * named target. Aspect ratio is preserved.
281
- */
282
- export type CameraResolutionCap = 'auto' | '1080p' | '720p' | '480p';
283
-
284
- /**
285
- * Aspect ratios exposed to the operator for the simulated camera.
286
- * The host maps each label to concrete IOSurface dimensions; values
287
- * the host doesn't recognise are silently ignored.
288
- */
289
- export type CameraAspect = '16:9' | '4:3' | '1:1' | '9:16';
290
-
291
- /**
292
- * Snapshot of the browser's webcam capture, mirrored from
293
- * `MediaStreamTrack.getSettings()`. Forwarded to the host alongside
294
- * `cameraResult` and exposed to the host app via
295
- * `onCameraDemandChange` so it can render a status indicator without
296
- * having to call `getStats()` itself.
297
- */
298
- export interface CameraCaptureInfo {
299
- width?: number;
300
- height?: number;
301
- frameRate?: number;
302
- deviceId?: string;
303
- label?: string;
304
- facingMode?: string;
305
- }
306
-
307
252
  /**
308
253
  * Live outbound-camera quality snapshot. Derived from
309
254
  * `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
@@ -551,10 +496,9 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
551
496
  onAxStatusChange,
552
497
  axPollIntervalMs,
553
498
  axMaxBackoffMs,
499
+ androidElementTreeOptions,
554
500
  onCameraDemandChange,
555
501
  onCameraStats,
556
- cameraResolutionCap = 'auto',
557
- cameraAspect,
558
502
  microphoneEnabled = false,
559
503
  onMicrophoneStateChange,
560
504
  }: RemoteControlProps,
@@ -607,20 +551,8 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
607
551
  // green light.
608
552
  const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
609
553
  const outboundLocalStreamRef = useRef<MediaStream | null>(null);
610
- // Bumped on every `cameraRequest` so a handler suspended on an
611
- // await (e.g. the getUserMedia prompt) can detect it was superseded
612
- // by a newer request and bail instead of re-attaching the camera.
613
- const cameraRequestGenerationRef = useRef(0);
614
- const cameraResolutionCapRef = useRef(cameraResolutionCap);
615
- cameraResolutionCapRef.current = cameraResolutionCap;
616
- // The aspect prop also rides a ref so the WS `onopen` reconnect
617
- // path can replay the operator's last pick without depending on
618
- // an in-flight render cycle. We mutate it inline (same render-
619
- // time pattern as the cap ref) so a parent prop change is visible
620
- // to closures captured during the next render; the useEffect
621
- // below handles the actual "send to host" side-effect.
622
- const cameraAspectRef = useRef<CameraAspect | undefined>(cameraAspect);
623
- cameraAspectRef.current = cameraAspect;
554
+ const outboundCameraRequestRef = useRef<CameraRequest | undefined>(undefined);
555
+ const cameraOperations = useMemo(() => new CameraOperationCoordinator(), []);
624
556
  // Mirror the demand-change callback into a ref so the WS message
625
557
  // handler always sees the freshest customer callback even when the
626
558
  // parent re-renders mid-session.
@@ -2055,32 +1987,30 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2055
1987
  });
2056
1988
  };
2057
1989
 
2058
- // Translate the user-facing resolution cap into a
2059
- // MediaTrackConstraints fragment we can feed `getUserMedia`
2060
- // or `applyConstraints`. Always returns a 30 fps ceiling so
2061
- // a 60 fps webcam doesn't double our encoder cost for no
2062
- // visible win (the simulator pool is paced at 30).
2063
- //
2064
- // We use `ideal` rather than `max` so that a webcam with a
2065
- // native 720p mode still hands us 720p when the user picks
2066
- // 1080p — instead of refusing the constraint outright. The
2067
- // browser's NotReadableError on a too-strict `max` is the
2068
- // most common camera-permission gotcha in the wild.
2069
- const cameraCapToConstraints = (cap: CameraResolutionCap): MediaTrackConstraints => {
2070
- const base: MediaTrackConstraints = {
2071
- frameRate: { ideal: 30, max: 30 },
2072
- };
2073
- switch (cap) {
2074
- case '1080p':
2075
- return { ...base, width: { ideal: 1920 }, height: { ideal: 1080 } };
2076
- case '720p':
2077
- return { ...base, width: { ideal: 1280 }, height: { ideal: 720 } };
2078
- case '480p':
2079
- return { ...base, width: { ideal: 854 }, height: { ideal: 480 } };
2080
- case 'auto':
2081
- default:
2082
- return base;
1990
+ // Publish a fresh settings snapshot after every successful capture
1991
+ // change. This is status metadata; host output sizing is frame-derived.
1992
+ const sendGrantedCameraResult = (ws: WebSocket, videoTrack: MediaStreamTrack) => {
1993
+ let result;
1994
+ try {
1995
+ result = createGrantedCameraResult(videoTrack);
1996
+ // eslint-disable-next-line no-console
1997
+ console.info(
1998
+ `[RemoteControl] camera capture: ${result.camera?.width}x${result.camera?.height}` +
1999
+ ` @ ${result.camera?.frameRate ?? '?'}fps` +
2000
+ (result.camera?.label ? ` — ${result.camera.label}` : ''),
2001
+ );
2002
+ } catch (err) {
2003
+ debugWarn('getSettings() on outbound camera track failed:', err);
2004
+ result = {
2005
+ type: 'cameraResult' as const,
2006
+ granted: true,
2007
+ camera: {
2008
+ label: videoTrack.label || undefined,
2009
+ },
2010
+ };
2083
2011
  }
2012
+ ws.send(JSON.stringify(result));
2013
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, result.camera);
2084
2014
  };
2085
2015
 
2086
2016
  // Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
@@ -2192,6 +2122,210 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2192
2122
  safeInvoke('onCameraStats', onCameraStatsRef.current, null);
2193
2123
  };
2194
2124
 
2125
+ const handleCameraRequest = (
2126
+ request: CameraRequest,
2127
+ ws: WebSocket,
2128
+ sender: RTCRtpSender,
2129
+ isCurrentAttempt: () => boolean,
2130
+ ): Promise<void> => {
2131
+ // `enqueue` increments its generation synchronously, so suspended
2132
+ // permission work becomes stale before the newer operation starts.
2133
+ return cameraOperations.enqueue(async ({ isCurrent }) => {
2134
+ const isCurrentCameraRequest = () =>
2135
+ isCurrent() &&
2136
+ isCurrentAttempt() &&
2137
+ wsRef.current === ws &&
2138
+ outboundCameraSenderRef.current === sender;
2139
+ if (!isCurrentCameraRequest()) return;
2140
+
2141
+ if (!request.active) {
2142
+ const stream = outboundLocalStreamRef.current;
2143
+ try {
2144
+ await sender.replaceTrack(null);
2145
+ } catch (err) {
2146
+ debugWarn('replaceTrack(null) on camera detach failed:', err);
2147
+ }
2148
+ if (outboundLocalStreamRef.current === stream) {
2149
+ outboundLocalStreamRef.current = null;
2150
+ outboundCameraRequestRef.current = undefined;
2151
+ if (stream) stopMediaStream(stream);
2152
+ }
2153
+ stopCameraStatsPoller();
2154
+ if (!isCurrentCameraRequest()) return;
2155
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
2156
+ return;
2157
+ }
2158
+
2159
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
2160
+ const currentStream = outboundLocalStreamRef.current;
2161
+ const currentTrack = currentStream?.getVideoTracks()[0];
2162
+ const currentRequest = outboundCameraRequestRef.current;
2163
+
2164
+ if (
2165
+ currentTrack?.readyState === 'live' &&
2166
+ currentRequest &&
2167
+ !shouldReacquireCamera(currentRequest, request)
2168
+ ) {
2169
+ outboundCameraRequestRef.current = request;
2170
+ sendGrantedCameraResult(ws, currentTrack);
2171
+ startCameraStatsPoller();
2172
+ return;
2173
+ }
2174
+
2175
+ // A facing change always restarts browser capture. Releasing first
2176
+ // lets mobile browsers open the opposite camera; one-camera laptops
2177
+ // naturally return the same available input for the new ideal.
2178
+ if (currentStream) {
2179
+ try {
2180
+ await sender.replaceTrack(null);
2181
+ } catch (err) {
2182
+ debugWarn('replaceTrack(null) before camera switch failed:', err);
2183
+ }
2184
+ if (outboundLocalStreamRef.current === currentStream) {
2185
+ outboundLocalStreamRef.current = null;
2186
+ outboundCameraRequestRef.current = undefined;
2187
+ stopMediaStream(currentStream);
2188
+ }
2189
+ stopCameraStatsPoller();
2190
+ if (!isCurrentCameraRequest()) return;
2191
+ }
2192
+
2193
+ if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
2194
+ // eslint-disable-next-line no-console
2195
+ console.warn(
2196
+ '[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
2197
+ 'getUserMedia requires a secure context (https or http://localhost). ' +
2198
+ 'Replying cameraResult granted=false so limulator falls back to black frames.',
2199
+ );
2200
+ if (!isCurrentCameraRequest()) return;
2201
+ ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2202
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2203
+ return;
2204
+ }
2205
+
2206
+ let stream: MediaStream | null = null;
2207
+ try {
2208
+ stream = await navigator.mediaDevices.getUserMedia({
2209
+ video: cameraCaptureConstraints(request),
2210
+ audio: false,
2211
+ });
2212
+ } catch (err) {
2213
+ // eslint-disable-next-line no-console
2214
+ console.warn('[RemoteControl] getUserMedia denied/failed:', err);
2215
+ }
2216
+ if (!isCurrentCameraRequest()) {
2217
+ if (stream) stopMediaStream(stream);
2218
+ return;
2219
+ }
2220
+ if (!stream) {
2221
+ ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2222
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2223
+ return;
2224
+ }
2225
+
2226
+ const videoTrack = stream.getVideoTracks()[0] ?? null;
2227
+ if (!videoTrack) {
2228
+ stopMediaStream(stream);
2229
+ if (!isCurrentCameraRequest()) return;
2230
+ ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2231
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2232
+ return;
2233
+ }
2234
+
2235
+ try {
2236
+ videoTrack.contentHint = 'motion';
2237
+ } catch {
2238
+ /* older browsers don't support contentHint; ignore */
2239
+ }
2240
+
2241
+ try {
2242
+ await sender.replaceTrack(videoTrack);
2243
+ } catch (err) {
2244
+ debugWarn('replaceTrack(videoTrack) failed:', err);
2245
+ stopMediaStream(stream);
2246
+ if (!isCurrentCameraRequest()) return;
2247
+ ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2248
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2249
+ return;
2250
+ }
2251
+ if (!isCurrentCameraRequest()) {
2252
+ try {
2253
+ await sender.replaceTrack(null);
2254
+ } catch (err) {
2255
+ debugWarn('replaceTrack(null) for stale camera attach failed:', err);
2256
+ }
2257
+ stopMediaStream(stream);
2258
+ return;
2259
+ }
2260
+
2261
+ try {
2262
+ const params = sender.getParameters();
2263
+ if (!params.encodings || params.encodings.length === 0) {
2264
+ params.encodings = [{}];
2265
+ }
2266
+ params.encodings[0].maxBitrate = 8_000_000;
2267
+ params.encodings[0].maxFramerate = 30;
2268
+ params.degradationPreference = 'maintain-framerate';
2269
+ await sender.setParameters(params);
2270
+ } catch (err) {
2271
+ debugWarn('setParameters on outbound camera failed:', err);
2272
+ }
2273
+ if (!isCurrentCameraRequest()) {
2274
+ try {
2275
+ await sender.replaceTrack(null);
2276
+ } catch (err) {
2277
+ debugWarn('replaceTrack(null) for stale camera finalize failed:', err);
2278
+ }
2279
+ stopMediaStream(stream);
2280
+ return;
2281
+ }
2282
+
2283
+ outboundLocalStreamRef.current = stream;
2284
+ outboundCameraRequestRef.current = request;
2285
+
2286
+ videoTrack.onended = () => {
2287
+ if (outboundLocalStreamRef.current !== stream) return;
2288
+ void cameraOperations
2289
+ .enqueueCurrent(async ({ isCurrent: isCurrentEndedCleanup }) => {
2290
+ if (
2291
+ !isCurrentEndedCleanup() ||
2292
+ outboundLocalStreamRef.current !== stream ||
2293
+ !isCurrentAttempt() ||
2294
+ wsRef.current !== ws ||
2295
+ outboundCameraSenderRef.current !== sender
2296
+ ) {
2297
+ return;
2298
+ }
2299
+ try {
2300
+ await sender.replaceTrack(null);
2301
+ } catch (err) {
2302
+ debugWarn('replaceTrack(null) after camera ended failed:', err);
2303
+ }
2304
+ if (
2305
+ !isCurrentEndedCleanup() ||
2306
+ outboundLocalStreamRef.current !== stream ||
2307
+ !isCurrentAttempt() ||
2308
+ wsRef.current !== ws ||
2309
+ outboundCameraSenderRef.current !== sender
2310
+ ) {
2311
+ return;
2312
+ }
2313
+ outboundLocalStreamRef.current = null;
2314
+ outboundCameraRequestRef.current = undefined;
2315
+ stopMediaStream(stream);
2316
+ stopCameraStatsPoller();
2317
+ ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2318
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2319
+ })
2320
+ .catch((err) => debugWarn('camera ended cleanup failed:', err));
2321
+ };
2322
+
2323
+ if (!isCurrentCameraRequest()) return;
2324
+ sendGrantedCameraResult(ws, videoTrack);
2325
+ startCameraStatsPoller();
2326
+ });
2327
+ };
2328
+
2195
2329
  const teardownConnection = () => {
2196
2330
  clearConnectionSuccessTimeout();
2197
2331
  clearIceDisconnectedGrace();
@@ -2203,8 +2337,10 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2203
2337
  // Drop any active outbound camera before the PC dies so the
2204
2338
  // browser doesn't leave the camera indicator lit between
2205
2339
  // reconnects.
2340
+ cameraOperations.invalidate();
2206
2341
  stopCameraStatsPoller();
2207
2342
  stopOutboundLocalStream();
2343
+ outboundCameraRequestRef.current = undefined;
2208
2344
  outboundCameraSenderRef.current = null;
2209
2345
  // Same for the live microphone. Manual enablement survives reconnect;
2210
2346
  // guest demand is replayed by the host after it assigns the new session.
@@ -2430,21 +2566,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2430
2566
  if (!isCurrentAttempt() || wsRef.current !== ws) {
2431
2567
  return;
2432
2568
  }
2433
- // Replay the saved camera aspect for fresh connections
2434
- // (initial mount, autoreconnect, sim reboot). The host's
2435
- // streamer starts at its boot default (16:9 / 1920×1080)
2436
- // and a missing message would mean the operator's pick
2437
- // silently reverts on reconnect. We always send something
2438
- // when the prop is set, even if it matches the host
2439
- // default — the host short-circuits no-op rebuilds.
2440
- const initialAspect = cameraAspectRef.current;
2441
- if (initialAspect) {
2442
- try {
2443
- ws.send(JSON.stringify({ type: 'cameraAspect', aspect: initialAspect }));
2444
- } catch (err) {
2445
- debugWarn('initial cameraAspect send failed:', err);
2446
- }
2447
- }
2448
2569
  settle(resolve);
2449
2570
  };
2450
2571
 
@@ -2663,6 +2784,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2663
2784
  platform,
2664
2785
  baseIntervalMs: axPollIntervalMs,
2665
2786
  maxBackoffMs: axMaxBackoffMs,
2787
+ androidElementTreeOptions,
2666
2788
  send: (payload) => {
2667
2789
  if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return false;
2668
2790
  try {
@@ -2952,235 +3074,18 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2952
3074
  // outbound-camera transceiver; without it there is nothing to
2953
3075
  // feed the device, so ignore camera requests entirely — no
2954
3076
  // getUserMedia prompt and no onCameraDemandChange callbacks.
2955
- if (!outboundCameraSenderRef.current) {
2956
- break;
2957
- }
2958
- const active = message.active === true;
2959
- // Bump up front so any earlier in-flight handler bails.
2960
- const cameraGeneration = ++cameraRequestGenerationRef.current;
2961
- const isCurrentCameraRequest = () =>
2962
- isCurrentAttempt() &&
2963
- wsRef.current === ws &&
2964
- cameraGeneration === cameraRequestGenerationRef.current;
2965
- // Log unconditionally — this is one of the few places we
2966
- // hand control to the browser's permission UI, and a
2967
- // silent failure here looks identical to "limulator
2968
- // never asked" from the user's point of view.
2969
- // eslint-disable-next-line no-console
2970
- console.info('[RemoteControl] cameraRequest received, active=', active);
2971
- if (!active) {
2972
- // Sim no longer wants frames. Drop our local track and
2973
- // shut the browser's camera green light off.
2974
- const sender = outboundCameraSenderRef.current;
2975
- if (sender) {
2976
- try {
2977
- await sender.replaceTrack(null);
2978
- } catch (err) {
2979
- debugWarn('replaceTrack(null) on camera detach failed:', err);
2980
- }
2981
- }
2982
- stopCameraStatsPoller();
2983
- stopOutboundLocalStream();
2984
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
2985
- break;
2986
- }
2987
- // Sim is asking for camera. Ask the browser; the user's
2988
- // prompt response is reported back to the host via a
2989
- // `cameraResult` message so it can swap to a
2990
- // black-frame fallback on denial.
2991
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
2992
- if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
2993
- // Likely an insecure context (http on a non-localhost
2994
- // origin) — Chrome strips `mediaDevices` off
2995
- // `navigator` in that case and the only signal is
2996
- // this undefined check.
2997
- // eslint-disable-next-line no-console
2998
- console.warn(
2999
- '[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
3000
- 'getUserMedia requires a secure context (https or http://localhost). ' +
3001
- 'Replying cameraResult granted=false so limulator falls back to black frames.',
3002
- );
3003
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
3004
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
3005
- break;
3006
- }
3007
- let stream: MediaStream | null = null;
3008
- try {
3009
- // Capture at the webcam's *native* resolution by
3010
- // default (no width/height constraints). When the
3011
- // host app has picked an explicit cap via
3012
- // `cameraResolutionCap`, we honour it here. Frame
3013
- // rate is always capped to 30 to match the
3014
- // simulator pool's pacing.
3015
- stream = await navigator.mediaDevices.getUserMedia({
3016
- video: cameraCapToConstraints(cameraResolutionCapRef.current),
3017
- audio: false,
3018
- });
3019
- } catch (err) {
3020
- // Surface unconditionally — the user is the only one
3021
- // who can fix this (permission denied, no device,
3022
- // dismissed prompt, etc.). `debugWarn` alone would
3023
- // hide it in normal builds.
3024
- // eslint-disable-next-line no-console
3025
- console.warn('[RemoteControl] getUserMedia denied/failed:', err);
3026
- }
3027
- if (!isCurrentCameraRequest()) {
3028
- // Superseded during the prompt; drop the stream we got.
3029
- if (stream) stopMediaStream(stream);
3030
- return;
3031
- }
3032
- if (!stream) {
3033
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
3034
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
3035
- break;
3036
- }
3037
- // Replace any previous local stream (e.g. an earlier
3038
- // cameraRequest that resolved with a different device)
3039
- // before we install the new tracks.
3040
- stopOutboundLocalStream();
3041
- outboundLocalStreamRef.current = stream;
3042
3077
  const sender = outboundCameraSenderRef.current;
3043
- const videoTrack = stream.getVideoTracks()[0] ?? null;
3044
- if (!sender || !videoTrack) {
3045
- if (stream) stopMediaStream(stream);
3046
- outboundLocalStreamRef.current = null;
3047
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
3048
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
3078
+ if (!sender) {
3049
3079
  break;
3050
3080
  }
3051
- try {
3052
- await sender.replaceTrack(videoTrack);
3053
- } catch (err) {
3054
- debugWarn('replaceTrack(videoTrack) failed:', err);
3055
- stopMediaStream(stream);
3056
- outboundLocalStreamRef.current = null;
3057
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
3058
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
3081
+ const request = parseCameraRequest(message);
3082
+ if (!request) {
3083
+ debugWarn('Received invalid cameraRequest:', message);
3059
3084
  break;
3060
3085
  }
3061
- if (!isCurrentCameraRequest()) {
3062
- // Superseded mid-attach; detach and skip the ACK/poller.
3063
- try {
3064
- await sender.replaceTrack(null);
3065
- } catch (err) {
3066
- debugWarn('replaceTrack(null) on stale camera attach failed:', err);
3067
- }
3068
- stopMediaStream(stream);
3069
- if (outboundLocalStreamRef.current === stream) {
3070
- outboundLocalStreamRef.current = null;
3071
- }
3072
- return;
3073
- }
3074
- // Tell the encoder this is real motion content (a
3075
- // physical camera), not text/slides. With `'motion'`
3076
- // VideoToolbox / libvpx pick latency-friendly tuning
3077
- // (no extra B-frames, shorter GoP smoothing). Set
3078
- // before the first frame so the encoder init reads it.
3079
- try {
3080
- videoTrack.contentHint = 'motion';
3081
- } catch {
3082
- /* older browsers don't support contentHint; ignore */
3083
- }
3084
- // Bound the outbound bitrate generously and let
3085
- // WebRTC's congestion control (BWE) find the floor.
3086
- // 8 Mbps is comfortable for native 1080p30 webcam
3087
- // content over LAN — the encoder will use far less when
3088
- // the scene is static, and BWE will throttle if a
3089
- // hop is constrained. Pair with
3090
- // `maintain-framerate` so the quality scaler steps
3091
- // down resolution before dropping frames, matching how
3092
- // Meet/Zoom degrade.
3093
- try {
3094
- const params = sender.getParameters();
3095
- if (!params.encodings || params.encodings.length === 0) {
3096
- params.encodings = [{}];
3097
- }
3098
- params.encodings[0].maxBitrate = 8_000_000;
3099
- params.encodings[0].maxFramerate = 30;
3100
- params.degradationPreference = 'maintain-framerate';
3101
- await sender.setParameters(params);
3102
- } catch (err) {
3103
- debugWarn('setParameters on outbound camera failed:', err);
3104
- }
3105
- // Surface the actual captured geometry to the host so
3106
- // it can size its IOSurface pool / picture-format
3107
- // metadata to match. `getSettings()` returns what the
3108
- // browser actually picked — which may differ from any
3109
- // hints we sent in the constraints — including the
3110
- // selected `deviceId` / `label` (useful when a user has
3111
- // multiple cameras and we eventually expose a picker).
3112
- let cameraMetadata: {
3113
- width?: number;
3114
- height?: number;
3115
- frameRate?: number;
3116
- deviceId?: string;
3117
- label?: string;
3118
- facingMode?: string;
3119
- } = {};
3120
- try {
3121
- const settings = videoTrack.getSettings();
3122
- cameraMetadata = {
3123
- width: settings.width,
3124
- height: settings.height,
3125
- frameRate: settings.frameRate,
3126
- deviceId: settings.deviceId,
3127
- label: videoTrack.label || undefined,
3128
- facingMode: settings.facingMode,
3129
- };
3130
- // eslint-disable-next-line no-console
3131
- console.info(
3132
- `[RemoteControl] camera capture: ${cameraMetadata.width}x${cameraMetadata.height}` +
3133
- ` @ ${cameraMetadata.frameRate ?? '?'}fps` +
3134
- (cameraMetadata.label ? ` — ${cameraMetadata.label}` : ''),
3135
- );
3136
- } catch (err) {
3137
- debugWarn('getSettings() on outbound camera track failed:', err);
3138
- }
3139
- // If the browser revokes the track later (extension,
3140
- // user clicks Stop in the camera tab UI, etc.), notify
3141
- // the host so it can switch to black frames.
3142
- videoTrack.onended = () => {
3143
- if (outboundLocalStreamRef.current !== stream) return;
3144
- stopCameraStatsPoller();
3145
- stopOutboundLocalStream();
3146
- if (wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
3147
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
3148
- }
3149
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
3150
- };
3151
- if (!isCurrentCameraRequest()) {
3152
- // Superseded during setParameters; detach and bail.
3153
- videoTrack.onended = null;
3154
- try {
3155
- await sender.replaceTrack(null);
3156
- } catch (err) {
3157
- debugWarn('replaceTrack(null) on stale camera finalize failed:', err);
3158
- }
3159
- stopMediaStream(stream);
3160
- if (outboundLocalStreamRef.current === stream) {
3161
- outboundLocalStreamRef.current = null;
3162
- }
3163
- return;
3164
- }
3165
- ws.send(
3166
- JSON.stringify({
3167
- type: 'cameraResult',
3168
- granted: true,
3169
- // Forward what the browser actually captured so the
3170
- // host can size its IOSurface pool, log the device,
3171
- // and (eventually) surface a status pill / picker.
3172
- camera: cameraMetadata,
3173
- }),
3174
- );
3175
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, cameraMetadata);
3176
- // Kick off the per-second outbound stats sampler. We do
3177
- // this *after* `setParameters` so the first sample
3178
- // already sees the encoder under its final bitrate /
3179
- // degradation policy, and *after* the host-side
3180
- // attachInboundTrack will have wired up (the
3181
- // cameraResult ACK is what triggers it on the host),
3182
- // so framesEncoded starts climbing immediately.
3183
- startCameraStatsPoller();
3086
+ // eslint-disable-next-line no-console
3087
+ console.info('[RemoteControl] cameraRequest received:', request.active, request.facingMode);
3088
+ await handleCameraRequest(request, ws, sender, isCurrentAttempt);
3184
3089
  break;
3185
3090
  }
3186
3091
  case 'microphoneRequest': {
@@ -3313,41 +3218,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
3313
3218
  start();
3314
3219
  };
3315
3220
 
3316
- // Re-apply the resolution cap on the currently-sending track
3317
- // whenever the host app changes its preference. Skips when no
3318
- // camera is active — the next `getUserMedia` will pick up the
3319
- // new value via `cameraCapToConstraints` automatically.
3320
- useEffect(() => {
3321
- const stream = outboundLocalStreamRef.current;
3322
- if (!stream) return;
3323
- const track = stream.getVideoTracks()[0];
3324
- if (!track) return;
3325
- const constraints = cameraCapToConstraints(cameraResolutionCap);
3326
- track.applyConstraints(constraints).catch((err) => {
3327
- debugWarn('applyConstraints for new camera cap failed:', err);
3328
- });
3329
- // eslint-disable-next-line react-hooks/exhaustive-deps
3330
- }, [cameraResolutionCap]);
3331
-
3332
- // Push the aspect preference to the host whenever it changes.
3333
- // The host rebuilds its IOSurface pool and bumps `pool_generation`
3334
- // on its end; the dylib re-handshakes on next sem_wait. No
3335
- // peer-connection renegotiation is needed — the aspect change is
3336
- // purely about the pixel buffer dimensions iOS apps observe, not
3337
- // about WebRTC track layout. The `cameraAspectRef` is mirrored
3338
- // higher up so the WS `onopen` reconnect path can replay the
3339
- // latest value on fresh connections.
3340
- useEffect(() => {
3341
- if (!cameraAspect) return;
3342
- const ws = wsRef.current;
3343
- if (!ws || ws.readyState !== WebSocket.OPEN) return;
3344
- try {
3345
- ws.send(JSON.stringify({ type: 'cameraAspect', aspect: cameraAspect }));
3346
- } catch (err) {
3347
- debugWarn('cameraAspect send failed:', err);
3348
- }
3349
- }, [cameraAspect]);
3350
-
3351
3221
  // Drive the manual override without interrupting guest-driven demand.
3352
3222
  useEffect(() => {
3353
3223
  if (isMicrophoneWanted()) {