@limrun/ui 0.13.4 → 0.13.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -140,157 +140,6 @@ interface RemoteControlProps {
140
140
  * @default 2000
141
141
  */
142
142
  axMaxBackoffMs?: number;
143
-
144
- /**
145
- * Fires whenever the iOS simulator's camera demand state changes —
146
- * i.e. an app inside the sim called
147
- * `[AVCaptureSession startRunning]` or `[stopRunning]`. The
148
- * component handles the `navigator.mediaDevices.getUserMedia` prompt
149
- * and SDP plumbing internally; this callback is purely so the host
150
- * UI can render a status indicator ("simulator is using your
151
- * camera", etc.).
152
- *
153
- * `active` reflects whether the sim is currently asking for
154
- * frames. `granted` is set only on the call that follows a
155
- * `getUserMedia` attempt: `true` when the user accepted the
156
- * browser prompt, `false` when they denied or the call failed
157
- * (in which case the limulator side switches to a black-frame
158
- * fallback so the app keeps ticking).
159
- *
160
- * `camera` (optional, only meaningful when `granted === true`)
161
- * carries what `MediaStreamTrack.getSettings()` reported for the
162
- * active capture: resolution, framerate, device label, facing
163
- * mode. Use it to render a richer status indicator (e.g.
164
- * "Camera · 1920×1080 · 30 fps · FaceTime HD").
165
- *
166
- * Only iOS instances ever fire this callback; Android instances
167
- * have no camera-injector path and stay silent.
168
- */
169
- onCameraDemandChange?: (active: boolean, granted?: boolean, camera?: CameraCaptureInfo) => void;
170
-
171
- /**
172
- * Periodically (~1Hz) fires with a snapshot of the outbound
173
- * camera stream's live WebRTC stats — codec, encoder
174
- * implementation, hardware acceleration, encoded fps, bitrate,
175
- * round-trip-time, and the encoder's
176
- * `qualityLimitationReason` (which is what Meet/Zoom use to
177
- * decide whether to show "Bandwidth limited" or "CPU limited"
178
- * banners).
179
- *
180
- * Only fires while the simulator is actively pulling the
181
- * camera AND `getUserMedia` was granted. `null` is emitted
182
- * once when the stream goes back to idle so consumers can
183
- * clear their UI without having to maintain their own
184
- * timeout. The host app does not need to poll `getStats()`
185
- * itself — this is the canonical place.
186
- */
187
- onCameraStats?: (stats: CameraStreamStats | null) => void;
188
-
189
- /**
190
- * Optional resolution cap for outbound camera capture.
191
- *
192
- * - `'auto'` (default): no extra constraint. The browser captures
193
- * at its webcam's native max; WebRTC's quality scaler may step
194
- * down resolution on the encoder side under bandwidth pressure
195
- * while still feeding the simulator at the pool's native size.
196
- * - `'1080p'` / `'720p'` / `'480p'`: hard cap applied via
197
- * `getUserMedia` constraints (for new captures) and
198
- * `track.applyConstraints` (for the currently-active track),
199
- * matching the way Meet/Zoom expose a "Send resolution" picker.
200
- *
201
- * Bumping or lowering the cap mid-stream is supported; the
202
- * change takes effect within a frame or two as the webcam
203
- * re-negotiates.
204
- */
205
- cameraResolutionCap?: CameraResolutionCap;
206
- /**
207
- * Aspect ratio the simulator's virtual camera should report to apps.
208
- * Picking a value here triggers a `cameraAspect` WS message to the
209
- * host, which rebuilds its IOSurface ring at the matching dimensions
210
- * (16:9 → 1920×1080, 4:3 → 1440×1080, 1:1 → 1080×1080, 9:16 →
211
- * 1080×1920) and signals the in-sim dylib to re-handshake. iOS apps
212
- * see CMSampleBuffers at the new dimensions within a frame or two.
213
- *
214
- * The browser still captures whatever the webcam offers; the host
215
- * aspect-fills (cover, center-crop) into the new pool. Switching
216
- * aspect at runtime is intentionally cheap so users can A/B preview
217
- * styles without restarting the simulator.
218
- *
219
- * `undefined` leaves the pool untouched (the host's boot default —
220
- * 16:9 / 1920×1080 — applies).
221
- */
222
- cameraAspect?: CameraAspect;
223
- }
224
-
225
- /**
226
- * Resolution caps a host app can request on the outbound camera.
227
- * `'auto'` is "let the browser decide" (no constraints beyond the
228
- * 30 fps ceiling); the other options clamp width/height to the
229
- * named target. Aspect ratio is preserved.
230
- */
231
- export type CameraResolutionCap = 'auto' | '1080p' | '720p' | '480p';
232
-
233
- /**
234
- * Aspect ratios exposed to the operator for the simulated camera.
235
- * The host maps each label to concrete IOSurface dimensions; values
236
- * the host doesn't recognise are silently ignored.
237
- */
238
- export type CameraAspect = '16:9' | '4:3' | '1:1' | '9:16';
239
-
240
- /**
241
- * Snapshot of the browser's webcam capture, mirrored from
242
- * `MediaStreamTrack.getSettings()`. Forwarded to the host alongside
243
- * `cameraResult` and exposed to the host app via
244
- * `onCameraDemandChange` so it can render a status indicator without
245
- * having to call `getStats()` itself.
246
- */
247
- export interface CameraCaptureInfo {
248
- width?: number;
249
- height?: number;
250
- frameRate?: number;
251
- deviceId?: string;
252
- label?: string;
253
- facingMode?: string;
254
- }
255
-
256
- /**
257
- * Live outbound-camera quality snapshot. Derived from
258
- * `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
259
- * once per second while the camera is sending. All fields optional:
260
- * some browsers omit fields (Safari rarely reports
261
- * `encoderImplementation`), and the first sample after camera start
262
- * has no delta-derived numbers yet.
263
- */
264
- export interface CameraStreamStats {
265
- /** "H264", "HEVC"/"H265", "VP9", "VP8", "AV1", etc. (uppercased). */
266
- codec?: string;
267
- /** e.g. "VideoToolbox" (hw), "OpenH264" (sw), "ExternalEncoder". */
268
- encoderImplementation?: string;
269
- /**
270
- * Browser-reported hardware-acceleration hint. Some Chromium
271
- * versions expose this via `powerEfficientEncoder`; we mirror it
272
- * here so consumers don't have to know the spec quirk.
273
- */
274
- hardwareAccelerated?: boolean;
275
- /** Outbound encoded fps over the last sample window. */
276
- framesPerSecond?: number;
277
- /** Cumulative encoded frame count. */
278
- framesEncoded?: number;
279
- /** Outbound encoded bitrate (bits/s) over the last window. */
280
- bitrateBps?: number;
281
- /** Width/height the encoder is currently producing. */
282
- width?: number;
283
- height?: number;
284
- /**
285
- * One of `'none' | 'cpu' | 'bandwidth' | 'other'`. Mirrors
286
- * Meet/Zoom's "limited by …" banners. Anything other than
287
- * `'none'` means the encoder dropped resolution to keep up.
288
- */
289
- qualityLimitationReason?: string;
290
- /** Round-trip time in milliseconds, from the matching RTCP. */
291
- rttMs?: number;
292
- /** Packet-loss percentage over the last sample window (0..100). */
293
- packetsLostPct?: number;
294
143
  }
295
144
 
296
145
  interface ScreenshotData {
@@ -489,10 +338,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
489
338
  onAxStatusChange,
490
339
  axPollIntervalMs,
491
340
  axMaxBackoffMs,
492
- onCameraDemandChange,
493
- onCameraStats,
494
- cameraResolutionCap = 'auto',
495
- cameraAspect,
496
341
  }: RemoteControlProps,
497
342
  ref,
498
343
  ) => {
@@ -518,58 +363,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
518
363
  // Mirrored to a ref so stale closures in event handlers see the latest value.
519
364
  const autoReconnectRef = useRef(autoReconnect);
520
365
  autoReconnectRef.current = autoReconnect;
521
- // Demand-driven outbound camera state.
522
- //
523
- // The limulator side broadcasts a `cameraRequest` WS message whenever
524
- // an app inside the simulator opens/closes an `AVCaptureSession`.
525
- // We respond by calling `navigator.mediaDevices.getUserMedia(...)`
526
- // and `replaceTrack`ing the result onto a pre-allocated sendonly
527
- // video transceiver. Pre-allocating the transceiver lets us
528
- // attach/detach the local camera without renegotiating the SDP
529
- // (the slot is already in the answer's a=video block, just with
530
- // `inactive`/empty until we install the track).
531
- //
532
- // Refs (not state) because all callers live inside event-handler
533
- // closures and the ref read happens during message processing, not
534
- // during render. We keep both the sender and the active local
535
- // stream so teardown can stop tracks without leaking the camera
536
- // green light.
537
- const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
538
- const outboundLocalStreamRef = useRef<MediaStream | null>(null);
539
- // Bumped on every `cameraRequest` so a handler suspended on an
540
- // await (e.g. the getUserMedia prompt) can detect it was superseded
541
- // by a newer request and bail instead of re-attaching the camera.
542
- const cameraRequestGenerationRef = useRef(0);
543
- const cameraResolutionCapRef = useRef(cameraResolutionCap);
544
- cameraResolutionCapRef.current = cameraResolutionCap;
545
- // The aspect prop also rides a ref so the WS `onopen` reconnect
546
- // path can replay the operator's last pick without depending on
547
- // an in-flight render cycle. We mutate it inline (same render-
548
- // time pattern as the cap ref) so a parent prop change is visible
549
- // to closures captured during the next render; the useEffect
550
- // below handles the actual "send to host" side-effect.
551
- const cameraAspectRef = useRef<CameraAspect | undefined>(cameraAspect);
552
- cameraAspectRef.current = cameraAspect;
553
- // Mirror the demand-change callback into a ref so the WS message
554
- // handler always sees the freshest customer callback even when the
555
- // parent re-renders mid-session.
556
- const onCameraDemandChangeRef = useRef(onCameraDemandChange);
557
- onCameraDemandChangeRef.current = onCameraDemandChange;
558
- const onCameraStatsRef = useRef(onCameraStats);
559
- onCameraStatsRef.current = onCameraStats;
560
- // Active outbound-camera stats poller. While the camera is
561
- // sending we sample `RTCPeerConnection.getStats()` once per
562
- // second, derive deltas (bitrate, fps, packet loss) from the
563
- // previous sample, and push the result through `onCameraStats`.
564
- // Cleared on teardown so we never leak the timer.
565
- const cameraStatsTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
566
- const cameraStatsPrevRef = useRef<{
567
- timestamp: number;
568
- framesEncoded?: number;
569
- bytesSent?: number;
570
- packetsSent?: number;
571
- packetsLost?: number;
572
- } | null>(null);
573
366
  const firstFrameShownRef = useRef(false);
574
367
  const pendingScreenshotResolversRef = useRef<
575
368
  Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
@@ -1642,169 +1435,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
1642
1435
  setVideoLoaded(true);
1643
1436
  };
1644
1437
 
1645
- // Stop every track on a MediaStream and release the device handle.
1646
- // The browser keeps the camera "on" indicator lit until at least
1647
- // one track on the underlying source is stopped, so we have to
1648
- // call stop() explicitly — closing the peer connection alone
1649
- // won't do it.
1650
- const stopMediaStream = (stream: MediaStream) => {
1651
- for (const track of stream.getTracks()) {
1652
- try {
1653
- track.stop();
1654
- } catch (err) {
1655
- debugWarn('track.stop() failed:', err);
1656
- }
1657
- }
1658
- };
1659
-
1660
- // Stop and forget any currently-attached outbound camera stream.
1661
- // Safe to call when nothing is attached. Does not touch the
1662
- // sender — the caller is responsible for `replaceTrack(null)`
1663
- // separately when it wants the SDP slot itself empty.
1664
- const stopOutboundLocalStream = () => {
1665
- const stream = outboundLocalStreamRef.current;
1666
- if (!stream) return;
1667
- outboundLocalStreamRef.current = null;
1668
- stopMediaStream(stream);
1669
- };
1670
-
1671
- // Translate the user-facing resolution cap into a
1672
- // MediaTrackConstraints fragment we can feed `getUserMedia`
1673
- // or `applyConstraints`. Always returns a 30 fps ceiling so
1674
- // a 60 fps webcam doesn't double our encoder cost for no
1675
- // visible win (the simulator pool is paced at 30).
1676
- //
1677
- // We use `ideal` rather than `max` so that a webcam with a
1678
- // native 720p mode still hands us 720p when the user picks
1679
- // 1080p — instead of refusing the constraint outright. The
1680
- // browser's NotReadableError on a too-strict `max` is the
1681
- // most common camera-permission gotcha in the wild.
1682
- const cameraCapToConstraints = (cap: CameraResolutionCap): MediaTrackConstraints => {
1683
- const base: MediaTrackConstraints = {
1684
- frameRate: { ideal: 30, max: 30 },
1685
- };
1686
- switch (cap) {
1687
- case '1080p':
1688
- return { ...base, width: { ideal: 1920 }, height: { ideal: 1080 } };
1689
- case '720p':
1690
- return { ...base, width: { ideal: 1280 }, height: { ideal: 720 } };
1691
- case '480p':
1692
- return { ...base, width: { ideal: 854 }, height: { ideal: 480 } };
1693
- case 'auto':
1694
- default:
1695
- return base;
1696
- }
1697
- };
1698
-
1699
- // Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
1700
- // into a short uppercase label suitable for a status badge.
1701
- const shortenCodecMime = (mime: string | undefined): string | undefined => {
1702
- if (!mime) return undefined;
1703
- const slash = mime.indexOf('/');
1704
- const tail = slash >= 0 ? mime.slice(slash + 1) : mime;
1705
- const upper = tail.toUpperCase();
1706
- if (upper === 'HEVC') return 'H265';
1707
- return upper;
1708
- };
1709
-
1710
- // Sample outbound-camera stats once and push a normalised
1711
- // snapshot through `onCameraStats`. Skips silently if the sender
1712
- // has been torn down between intervals.
1713
- const sampleOutboundCameraStats = async () => {
1714
- const sender = outboundCameraSenderRef.current;
1715
- const handler = onCameraStatsRef.current;
1716
- if (!sender || !handler) return;
1717
- let report: RTCStatsReport;
1718
- try {
1719
- report = await sender.getStats();
1720
- } catch {
1721
- return;
1722
- }
1723
- let outbound: any | undefined;
1724
- let codecMime: string | undefined;
1725
- let remoteInbound: any | undefined;
1726
- report.forEach((entry: any) => {
1727
- if (entry.type === 'outbound-rtp' && entry.kind === 'video') {
1728
- outbound = entry;
1729
- } else if (entry.type === 'remote-inbound-rtp' && entry.kind === 'video') {
1730
- remoteInbound = entry;
1731
- }
1732
- });
1733
- if (outbound?.codecId) {
1734
- const codec = report.get(outbound.codecId);
1735
- if (codec) codecMime = (codec as any).mimeType;
1736
- }
1737
- if (!outbound) return;
1738
- const now = (outbound.timestamp as number) ?? Date.now();
1739
- const prev = cameraStatsPrevRef.current;
1740
- let fps: number | undefined;
1741
- let bitrate: number | undefined;
1742
- let lossPct: number | undefined;
1743
- if (prev && now > prev.timestamp) {
1744
- const dt = (now - prev.timestamp) / 1000;
1745
- if (typeof outbound.framesEncoded === 'number' && typeof prev.framesEncoded === 'number') {
1746
- fps = Math.max(0, (outbound.framesEncoded - prev.framesEncoded) / dt);
1747
- }
1748
- if (typeof outbound.bytesSent === 'number' && typeof prev.bytesSent === 'number') {
1749
- bitrate = Math.max(0, ((outbound.bytesSent - prev.bytesSent) * 8) / dt);
1750
- }
1751
- if (
1752
- typeof outbound.packetsSent === 'number' &&
1753
- typeof prev.packetsSent === 'number' &&
1754
- remoteInbound &&
1755
- typeof remoteInbound.packetsLost === 'number' &&
1756
- typeof prev.packetsLost === 'number'
1757
- ) {
1758
- const sent = outbound.packetsSent - prev.packetsSent;
1759
- const lost = remoteInbound.packetsLost - prev.packetsLost;
1760
- if (sent > 0) lossPct = Math.max(0, Math.min(100, (lost / sent) * 100));
1761
- }
1762
- }
1763
- cameraStatsPrevRef.current = {
1764
- timestamp: now,
1765
- framesEncoded: outbound.framesEncoded,
1766
- bytesSent: outbound.bytesSent,
1767
- packetsSent: outbound.packetsSent,
1768
- packetsLost: remoteInbound?.packetsLost,
1769
- };
1770
- const stats: CameraStreamStats = {
1771
- codec: shortenCodecMime(codecMime),
1772
- encoderImplementation: outbound.encoderImplementation,
1773
- hardwareAccelerated:
1774
- typeof outbound.powerEfficientEncoder === 'boolean' ? outbound.powerEfficientEncoder : undefined,
1775
- framesPerSecond: fps,
1776
- framesEncoded: outbound.framesEncoded,
1777
- bitrateBps: bitrate,
1778
- width: outbound.frameWidth,
1779
- height: outbound.frameHeight,
1780
- qualityLimitationReason: outbound.qualityLimitationReason,
1781
- rttMs:
1782
- typeof remoteInbound?.roundTripTime === 'number' ? remoteInbound.roundTripTime * 1000 : undefined,
1783
- packetsLostPct: lossPct,
1784
- };
1785
- safeInvoke('onCameraStats', onCameraStatsRef.current, stats);
1786
- };
1787
-
1788
- const startCameraStatsPoller = () => {
1789
- if (cameraStatsTimerRef.current !== null) return;
1790
- cameraStatsPrevRef.current = null;
1791
- // First sample fires after one interval — the very first
1792
- // sample has no deltas to compare against, which is fine: the
1793
- // codec/encoder identity is still useful on its own.
1794
- cameraStatsTimerRef.current = setInterval(() => {
1795
- void sampleOutboundCameraStats();
1796
- }, 1000);
1797
- };
1798
-
1799
- const stopCameraStatsPoller = () => {
1800
- if (cameraStatsTimerRef.current !== null) {
1801
- clearInterval(cameraStatsTimerRef.current);
1802
- cameraStatsTimerRef.current = null;
1803
- }
1804
- cameraStatsPrevRef.current = null;
1805
- safeInvoke('onCameraStats', onCameraStatsRef.current, null);
1806
- };
1807
-
1808
1438
  const teardownConnection = () => {
1809
1439
  clearConnectionSuccessTimeout();
1810
1440
  clearIceDisconnectedGrace();
@@ -1813,12 +1443,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
1813
1443
  axFetcherRef.current.stop();
1814
1444
  axFetcherRef.current = null;
1815
1445
  }
1816
- // Drop any active outbound camera before the PC dies so the
1817
- // browser doesn't leave the camera indicator lit between
1818
- // reconnects.
1819
- stopCameraStatsPoller();
1820
- stopOutboundLocalStream();
1821
- outboundCameraSenderRef.current = null;
1822
1446
  // A scheduled cursor flush would otherwise call setState on a
1823
1447
  // teardown component once the next frame runs.
1824
1448
  if (cursorRafIdRef.current !== undefined) {
@@ -1944,21 +1568,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
1944
1568
  if (!isCurrentAttempt() || wsRef.current !== ws) {
1945
1569
  return;
1946
1570
  }
1947
- // Replay the saved camera aspect for fresh connections
1948
- // (initial mount, autoreconnect, sim reboot). The host's
1949
- // streamer starts at its boot default (16:9 / 1920×1080)
1950
- // and a missing message would mean the operator's pick
1951
- // silently reverts on reconnect. We always send something
1952
- // when the prop is set, even if it matches the host
1953
- // default — the host short-circuits no-op rebuilds.
1954
- const initialAspect = cameraAspectRef.current;
1955
- if (initialAspect) {
1956
- try {
1957
- ws.send(JSON.stringify({ type: 'cameraAspect', aspect: initialAspect }));
1958
- } catch (err) {
1959
- debugWarn('initial cameraAspect send failed:', err);
1960
- }
1961
- }
1962
1571
  settle(resolve);
1963
1572
  };
1964
1573
 
@@ -2035,19 +1644,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2035
1644
  peerConnection.addTransceiver('audio', { direction: 'recvonly' });
2036
1645
  const videoTransceiver = peerConnection.addTransceiver('video', { direction: 'recvonly' });
2037
1646
 
2038
- // Pre-allocate a sendonly video slot for the user's camera.
2039
- // The track stays `null` until the limulator side asks for one
2040
- // (via the `cameraRequest` WS message), at which point we call
2041
- // `replaceTrack` with a `getUserMedia` result. Allocating
2042
- // up-front means we don't have to renegotiate the SDP when the
2043
- // simulator app actually opens its `AVCaptureSession` — the
2044
- // codec/SSRC negotiation happens once, here, and turning the
2045
- // camera on/off later is just a track-replace.
2046
- const outboundCameraTransceiver = peerConnection.addTransceiver('video', {
2047
- direction: 'sendonly',
2048
- });
2049
- outboundCameraSenderRef.current = outboundCameraTransceiver.sender;
2050
-
2051
1647
  // As hardware encoder, we use H265 for iOS and VP9 for Android.
2052
1648
  // We make sure these two are the first ones in the list.
2053
1649
  // If not, the fallback is H264 which is also hardware accelerated, although not as good,
@@ -2073,59 +1669,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2073
1669
  }
2074
1670
  }
2075
1671
 
2076
- // Pin the outbound camera transceiver's codec order with
2077
- // negotiation-time fallback. Preference order:
2078
- // 1. H.265 / HEVC — VideoToolbox HW on M-series + recent
2079
- // Chrome. ~30% less bitrate at equal quality. If the
2080
- // answerer (limulator) supports HEVC, this wins.
2081
- // 2. H.264 with VideoToolbox-friendly profile-level-ids
2082
- // (42e01f / 42e028 / 640c1f). Universally HW-accelerated
2083
- // on every Mac since 2009. SDP-time fallback when HEVC
2084
- // isn't in the answer.
2085
- // 3. Any other H.264 profile. Often resolves to OpenH264
2086
- // (software) so we keep it explicitly last among H.264
2087
- // to avoid Chrome's default pick.
2088
- // 4. VP9 / VP8 (libvpx software on Mac). Last resort.
2089
- //
2090
- // Caveat: this is *negotiation* fallback. If HEVC HW encoder
2091
- // init fails at session start or gets demoted mid-stream,
2092
- // Chrome falls back to *software* HEVC, not H.264 — no spec
2093
- // hook for runtime re-negotiation. The host-side stats
2094
- // logger will surface this as decoder=ffmpeg / hw=false.
2095
- if (RTCRtpSender.getCapabilities) {
2096
- const senderCaps = RTCRtpSender.getCapabilities('video');
2097
- if (senderCaps && senderCaps.codecs) {
2098
- const vtH264Profiles = new Set(['42e01f', '42e028', '640c1f']);
2099
- const score = (c: { mimeType: string; sdpFmtpLine?: string }): number => {
2100
- const mime = c.mimeType.toLowerCase();
2101
- const fmtp = (c.sdpFmtpLine || '').toLowerCase();
2102
- const profileMatch = fmtp.match(/profile-level-id=([0-9a-f]{6})/);
2103
- const profile = profileMatch ? profileMatch[1] : '';
2104
- if (mime === 'video/h265' || mime === 'video/hevc') return 1;
2105
- if (mime === 'video/h264' && vtH264Profiles.has(profile)) return 2;
2106
- if (mime === 'video/h264') return 3;
2107
- if (mime === 'video/vp9') return 4;
2108
- if (mime === 'video/vp8') return 5;
2109
- if (mime === 'video/rtx' || mime === 'video/red' || mime === 'video/ulpfec') {
2110
- return 0; // Keep RTX/FEC available alongside everything else.
2111
- }
2112
- return 6;
2113
- };
2114
- const sortedSendCodecs = [...senderCaps.codecs].sort((a, b) => score(a) - score(b));
2115
- try {
2116
- outboundCameraTransceiver.setCodecPreferences(sortedSendCodecs);
2117
- debugLog(
2118
- 'Outbound camera codec preferences:',
2119
- sortedSendCodecs
2120
- .map((c) => `${c.mimeType}${c.sdpFmtpLine ? `[${c.sdpFmtpLine}]` : ''}`)
2121
- .join(', '),
2122
- );
2123
- } catch (err) {
2124
- debugWarn('Failed to set outbound camera codec preferences:', err);
2125
- }
2126
- }
2127
- }
2128
-
2129
1672
  const dataChannel = peerConnection.createDataChannel('control', {
2130
1673
  ordered: true,
2131
1674
  negotiated: true,
@@ -2429,235 +1972,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2429
1972
  pendingScreenshotResolversRef.current.delete(message.id);
2430
1973
  pendingScreenshotRejectersRef.current.delete(message.id);
2431
1974
  break;
2432
- case 'cameraRequest': {
2433
- const active = message.active === true;
2434
- // Bump up front so any earlier in-flight handler bails.
2435
- const cameraGeneration = ++cameraRequestGenerationRef.current;
2436
- const isCurrentCameraRequest = () =>
2437
- isCurrentAttempt() &&
2438
- wsRef.current === ws &&
2439
- cameraGeneration === cameraRequestGenerationRef.current;
2440
- // Log unconditionally — this is one of the few places we
2441
- // hand control to the browser's permission UI, and a
2442
- // silent failure here looks identical to "limulator
2443
- // never asked" from the user's point of view.
2444
- // eslint-disable-next-line no-console
2445
- console.info('[RemoteControl] cameraRequest received, active=', active);
2446
- if (!active) {
2447
- // Sim no longer wants frames. Drop our local track and
2448
- // shut the browser's camera green light off.
2449
- const sender = outboundCameraSenderRef.current;
2450
- if (sender) {
2451
- try {
2452
- await sender.replaceTrack(null);
2453
- } catch (err) {
2454
- debugWarn('replaceTrack(null) on camera detach failed:', err);
2455
- }
2456
- }
2457
- stopCameraStatsPoller();
2458
- stopOutboundLocalStream();
2459
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
2460
- break;
2461
- }
2462
- // Sim is asking for camera. Ask the browser; the user's
2463
- // prompt response is reported back to the host via a
2464
- // `cameraResult` message so it can swap to a
2465
- // black-frame fallback on denial.
2466
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
2467
- if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
2468
- // Likely an insecure context (http on a non-localhost
2469
- // origin) — Chrome strips `mediaDevices` off
2470
- // `navigator` in that case and the only signal is
2471
- // this undefined check.
2472
- // eslint-disable-next-line no-console
2473
- console.warn(
2474
- '[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
2475
- 'getUserMedia requires a secure context (https or http://localhost). ' +
2476
- 'Replying cameraResult granted=false so limulator falls back to black frames.',
2477
- );
2478
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2479
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2480
- break;
2481
- }
2482
- let stream: MediaStream | null = null;
2483
- try {
2484
- // Capture at the webcam's *native* resolution by
2485
- // default (no width/height constraints). When the
2486
- // host app has picked an explicit cap via
2487
- // `cameraResolutionCap`, we honour it here. Frame
2488
- // rate is always capped to 30 to match the
2489
- // simulator pool's pacing.
2490
- stream = await navigator.mediaDevices.getUserMedia({
2491
- video: cameraCapToConstraints(cameraResolutionCapRef.current),
2492
- audio: false,
2493
- });
2494
- } catch (err) {
2495
- // Surface unconditionally — the user is the only one
2496
- // who can fix this (permission denied, no device,
2497
- // dismissed prompt, etc.). `debugWarn` alone would
2498
- // hide it in normal builds.
2499
- // eslint-disable-next-line no-console
2500
- console.warn('[RemoteControl] getUserMedia denied/failed:', err);
2501
- }
2502
- if (!isCurrentCameraRequest()) {
2503
- // Superseded during the prompt; drop the stream we got.
2504
- if (stream) stopMediaStream(stream);
2505
- return;
2506
- }
2507
- if (!stream) {
2508
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2509
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2510
- break;
2511
- }
2512
- // Replace any previous local stream (e.g. an earlier
2513
- // cameraRequest that resolved with a different device)
2514
- // before we install the new tracks.
2515
- stopOutboundLocalStream();
2516
- outboundLocalStreamRef.current = stream;
2517
- const sender = outboundCameraSenderRef.current;
2518
- const videoTrack = stream.getVideoTracks()[0] ?? null;
2519
- if (!sender || !videoTrack) {
2520
- if (stream) stopMediaStream(stream);
2521
- outboundLocalStreamRef.current = null;
2522
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2523
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2524
- break;
2525
- }
2526
- try {
2527
- await sender.replaceTrack(videoTrack);
2528
- } catch (err) {
2529
- debugWarn('replaceTrack(videoTrack) failed:', err);
2530
- stopMediaStream(stream);
2531
- outboundLocalStreamRef.current = null;
2532
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2533
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2534
- break;
2535
- }
2536
- if (!isCurrentCameraRequest()) {
2537
- // Superseded mid-attach; detach and skip the ACK/poller.
2538
- try {
2539
- await sender.replaceTrack(null);
2540
- } catch (err) {
2541
- debugWarn('replaceTrack(null) on stale camera attach failed:', err);
2542
- }
2543
- stopMediaStream(stream);
2544
- if (outboundLocalStreamRef.current === stream) {
2545
- outboundLocalStreamRef.current = null;
2546
- }
2547
- return;
2548
- }
2549
- // Tell the encoder this is real motion content (a
2550
- // physical camera), not text/slides. With `'motion'`
2551
- // VideoToolbox / libvpx pick latency-friendly tuning
2552
- // (no extra B-frames, shorter GoP smoothing). Set
2553
- // before the first frame so the encoder init reads it.
2554
- try {
2555
- videoTrack.contentHint = 'motion';
2556
- } catch {
2557
- /* older browsers don't support contentHint; ignore */
2558
- }
2559
- // Bound the outbound bitrate generously and let
2560
- // WebRTC's congestion control (BWE) find the floor.
2561
- // 8 Mbps is comfortable for native 1080p30 webcam
2562
- // content over LAN — the encoder will use far less when
2563
- // the scene is static, and BWE will throttle if a
2564
- // hop is constrained. Pair with
2565
- // `maintain-framerate` so the quality scaler steps
2566
- // down resolution before dropping frames, matching how
2567
- // Meet/Zoom degrade.
2568
- try {
2569
- const params = sender.getParameters();
2570
- if (!params.encodings || params.encodings.length === 0) {
2571
- params.encodings = [{}];
2572
- }
2573
- params.encodings[0].maxBitrate = 8_000_000;
2574
- params.encodings[0].maxFramerate = 30;
2575
- params.degradationPreference = 'maintain-framerate';
2576
- await sender.setParameters(params);
2577
- } catch (err) {
2578
- debugWarn('setParameters on outbound camera failed:', err);
2579
- }
2580
- // Surface the actual captured geometry to the host so
2581
- // it can size its IOSurface pool / picture-format
2582
- // metadata to match. `getSettings()` returns what the
2583
- // browser actually picked — which may differ from any
2584
- // hints we sent in the constraints — including the
2585
- // selected `deviceId` / `label` (useful when a user has
2586
- // multiple cameras and we eventually expose a picker).
2587
- let cameraMetadata: {
2588
- width?: number;
2589
- height?: number;
2590
- frameRate?: number;
2591
- deviceId?: string;
2592
- label?: string;
2593
- facingMode?: string;
2594
- } = {};
2595
- try {
2596
- const settings = videoTrack.getSettings();
2597
- cameraMetadata = {
2598
- width: settings.width,
2599
- height: settings.height,
2600
- frameRate: settings.frameRate,
2601
- deviceId: settings.deviceId,
2602
- label: videoTrack.label || undefined,
2603
- facingMode: settings.facingMode,
2604
- };
2605
- // eslint-disable-next-line no-console
2606
- console.info(
2607
- `[RemoteControl] camera capture: ${cameraMetadata.width}x${cameraMetadata.height}` +
2608
- ` @ ${cameraMetadata.frameRate ?? '?'}fps` +
2609
- (cameraMetadata.label ? ` — ${cameraMetadata.label}` : ''),
2610
- );
2611
- } catch (err) {
2612
- debugWarn('getSettings() on outbound camera track failed:', err);
2613
- }
2614
- // If the browser revokes the track later (extension,
2615
- // user clicks Stop in the camera tab UI, etc.), notify
2616
- // the host so it can switch to black frames.
2617
- videoTrack.onended = () => {
2618
- if (outboundLocalStreamRef.current !== stream) return;
2619
- stopCameraStatsPoller();
2620
- stopOutboundLocalStream();
2621
- if (wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
2622
- ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2623
- }
2624
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2625
- };
2626
- if (!isCurrentCameraRequest()) {
2627
- // Superseded during setParameters; detach and bail.
2628
- videoTrack.onended = null;
2629
- try {
2630
- await sender.replaceTrack(null);
2631
- } catch (err) {
2632
- debugWarn('replaceTrack(null) on stale camera finalize failed:', err);
2633
- }
2634
- stopMediaStream(stream);
2635
- if (outboundLocalStreamRef.current === stream) {
2636
- outboundLocalStreamRef.current = null;
2637
- }
2638
- return;
2639
- }
2640
- ws.send(
2641
- JSON.stringify({
2642
- type: 'cameraResult',
2643
- granted: true,
2644
- // Forward what the browser actually captured so the
2645
- // host can size its IOSurface pool, log the device,
2646
- // and (eventually) surface a status pill / picker.
2647
- camera: cameraMetadata,
2648
- }),
2649
- );
2650
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, cameraMetadata);
2651
- // Kick off the per-second outbound stats sampler. We do
2652
- // this *after* `setParameters` so the first sample
2653
- // already sees the encoder under its final bitrate /
2654
- // degradation policy, and *after* the host-side
2655
- // attachInboundTrack will have wired up (the
2656
- // cameraResult ACK is what triggers it on the host),
2657
- // so framesEncoded starts climbing immediately.
2658
- startCameraStatsPoller();
2659
- break;
2660
- }
2661
1975
  case 'terminateAppResult':
2662
1976
  if (typeof message.id !== 'string') {
2663
1977
  debugWarn('Received invalid terminateApp result message:', message);
@@ -2745,41 +2059,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2745
2059
  start();
2746
2060
  };
2747
2061
 
2748
- // Re-apply the resolution cap on the currently-sending track
2749
- // whenever the host app changes its preference. Skips when no
2750
- // camera is active — the next `getUserMedia` will pick up the
2751
- // new value via `cameraCapToConstraints` automatically.
2752
- useEffect(() => {
2753
- const stream = outboundLocalStreamRef.current;
2754
- if (!stream) return;
2755
- const track = stream.getVideoTracks()[0];
2756
- if (!track) return;
2757
- const constraints = cameraCapToConstraints(cameraResolutionCap);
2758
- track.applyConstraints(constraints).catch((err) => {
2759
- debugWarn('applyConstraints for new camera cap failed:', err);
2760
- });
2761
- // eslint-disable-next-line react-hooks/exhaustive-deps
2762
- }, [cameraResolutionCap]);
2763
-
2764
- // Push the aspect preference to the host whenever it changes.
2765
- // The host rebuilds its IOSurface pool and bumps `pool_generation`
2766
- // on its end; the dylib re-handshakes on next sem_wait. No
2767
- // peer-connection renegotiation is needed — the aspect change is
2768
- // purely about the pixel buffer dimensions iOS apps observe, not
2769
- // about WebRTC track layout. The `cameraAspectRef` is mirrored
2770
- // higher up so the WS `onopen` reconnect path can replay the
2771
- // latest value on fresh connections.
2772
- useEffect(() => {
2773
- if (!cameraAspect) return;
2774
- const ws = wsRef.current;
2775
- if (!ws || ws.readyState !== WebSocket.OPEN) return;
2776
- try {
2777
- ws.send(JSON.stringify({ type: 'cameraAspect', aspect: cameraAspect }));
2778
- } catch (err) {
2779
- debugWarn('cameraAspect send failed:', err);
2780
- }
2781
- }, [cameraAspect]);
2782
-
2783
2062
  useEffect(() => {
2784
2063
  // Reset video loaded state when connection params change
2785
2064
  setVideoLoaded(false);
@@ -2801,11 +2080,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2801
2080
  stop();
2802
2081
  document.removeEventListener('visibilitychange', handleVisibilityChange);
2803
2082
  };
2804
- // Camera attach/detach happens entirely inside the WS message
2805
- // loop now (sendonly transceiver + `replaceTrack`), so the
2806
- // connection effect doesn't need to bounce when the camera
2807
- // turns on/off — no SDP-affecting change.
2808
- // eslint-disable-next-line react-hooks/exhaustive-deps
2809
2083
  }, [url, token, propSessionId]);
2810
2084
 
2811
2085
  // Recompute the inspect-overlay geometry (container-local pixel rect of