@limrun/ui 0.9.0-rc.14 → 0.9.0-rc.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/components/remote-control.d.ts +125 -1
  2. package/dist/core/device-install/apple/provisioning.d.ts +55 -4
  3. package/dist/core/device-install/operations/limbuild-client.d.ts +15 -1
  4. package/dist/core/device-install/storage/browser-storage.d.ts +9 -5
  5. package/dist/core/device-install/types.d.ts +4 -1
  6. package/dist/device-install/index.cjs +1 -1
  7. package/dist/device-install/index.js +70 -64
  8. package/dist/device-install/react.cjs +1 -1
  9. package/dist/device-install/react.js +1 -1
  10. package/dist/device-install-dialog-DY35un0b.js +9 -0
  11. package/dist/device-install-dialog-chNLeiiL.mjs +2000 -0
  12. package/dist/device-install-dialog.css +1 -1
  13. package/dist/hooks/use-device-install.d.ts +5 -1
  14. package/dist/index.cjs +1 -1
  15. package/dist/index.js +1286 -1116
  16. package/dist/use-device-install-BIrl0v-k.js +31 -0
  17. package/dist/{use-device-install-sDVvby1V.mjs → use-device-install-LGfEdqyM.mjs} +4388 -4271
  18. package/package.json +4 -2
  19. package/src/components/device-install/device-install-dialog.css +29 -0
  20. package/src/components/device-install/device-install-dialog.tsx +91 -30
  21. package/src/components/remote-control.tsx +535 -5
  22. package/src/core/device-install/apple/provisioning.test.ts +84 -0
  23. package/src/core/device-install/apple/provisioning.ts +91 -7
  24. package/src/core/device-install/operations/limbuild-client.ts +32 -2
  25. package/src/core/device-install/storage/browser-storage.ts +29 -14
  26. package/src/core/device-install/types.ts +5 -1
  27. package/src/hooks/use-device-install.ts +135 -59
  28. package/dist/device-install-dialog-CjH25hnN.js +0 -2
  29. package/dist/device-install-dialog-W5Xv9kWL.mjs +0 -443
  30. package/dist/use-device-install-Y1u6vIBB.js +0 -31
@@ -157,10 +157,144 @@ interface RemoteControlProps {
157
157
  * (in which case the limulator side switches to a black-frame
158
158
  * fallback so the app keeps ticking).
159
159
  *
160
+ * `camera` (optional, only meaningful when `granted === true`)
161
+ * carries what `MediaStreamTrack.getSettings()` reported for the
162
+ * active capture: resolution, framerate, device label, facing
163
+ * mode. Use it to render a richer status indicator (e.g.
164
+ * "Camera · 1920×1080 · 30 fps · FaceTime HD").
165
+ *
160
166
  * Only iOS instances ever fire this callback; Android instances
161
167
  * have no camera-injector path and stay silent.
162
168
  */
163
- onCameraDemandChange?: (active: boolean, granted?: boolean) => void;
169
+ onCameraDemandChange?: (
170
+ active: boolean,
171
+ granted?: boolean,
172
+ camera?: CameraCaptureInfo,
173
+ ) => void;
174
+
175
+ /**
176
+ * Periodically (~1Hz) fires with a snapshot of the outbound
177
+ * camera stream's live WebRTC stats — codec, encoder
178
+ * implementation, hardware acceleration, encoded fps, bitrate,
179
+ * round-trip-time, and the encoder's
180
+ * `qualityLimitationReason` (which is what Meet/Zoom use to
181
+ * decide whether to show "Bandwidth limited" or "CPU limited"
182
+ * banners).
183
+ *
184
+ * Only fires while the simulator is actively pulling the
185
+ * camera AND `getUserMedia` was granted. `null` is emitted
186
+ * once when the stream goes back to idle so consumers can
187
+ * clear their UI without having to maintain their own
188
+ * timeout. The host app does not need to poll `getStats()`
189
+ * itself — this is the canonical place.
190
+ */
191
+ onCameraStats?: (stats: CameraStreamStats | null) => void;
192
+
193
+ /**
194
+ * Optional resolution cap for outbound camera capture.
195
+ *
196
+ * - `'auto'` (default): no extra constraint. The browser captures
197
+ * at its webcam's native max; WebRTC's quality scaler may step
198
+ * down resolution on the encoder side under bandwidth pressure
199
+ * while still feeding the simulator at the pool's native size.
200
+ * - `'1080p'` / `'720p'` / `'480p'`: hard cap applied via
201
+ * `getUserMedia` constraints (for new captures) and
202
+ * `track.applyConstraints` (for the currently-active track),
203
+ * matching the way Meet/Zoom expose a "Send resolution" picker.
204
+ *
205
+ * Bumping or lowering the cap mid-stream is supported; the
206
+ * change takes effect within a frame or two as the webcam
207
+ * re-negotiates.
208
+ */
209
+ cameraResolutionCap?: CameraResolutionCap;
210
+ /**
211
+ * Aspect ratio the simulator's virtual camera should report to apps.
212
+ * Picking a value here triggers a `cameraAspect` WS message to the
213
+ * host, which rebuilds its IOSurface ring at the matching dimensions
214
+ * (16:9 → 1920×1080, 4:3 → 1440×1080, 1:1 → 1080×1080, 9:16 →
215
+ * 1080×1920) and signals the in-sim dylib to re-handshake. iOS apps
216
+ * see CMSampleBuffers at the new dimensions within a frame or two.
217
+ *
218
+ * The browser still captures whatever the webcam offers; the host
219
+ * aspect-fills (cover, center-crop) into the new pool. Switching
220
+ * aspect at runtime is intentionally cheap so users can A/B preview
221
+ * styles without restarting the simulator.
222
+ *
223
+ * `undefined` leaves the pool untouched (the host's boot default —
224
+ * 16:9 / 1920×1080 — applies).
225
+ */
226
+ cameraAspect?: CameraAspect;
227
+ }
228
+
229
+ /**
230
+ * Resolution caps a host app can request on the outbound camera.
231
+ * `'auto'` is "let the browser decide" (no constraints beyond the
232
+ * 30 fps ceiling); the other options clamp width/height to the
233
+ * named target. Aspect ratio is preserved.
234
+ */
235
+ export type CameraResolutionCap = 'auto' | '1080p' | '720p' | '480p';
236
+
237
+ /**
238
+ * Aspect ratios exposed to the operator for the simulated camera.
239
+ * The host maps each label to concrete IOSurface dimensions; values
240
+ * the host doesn't recognise are silently ignored.
241
+ */
242
+ export type CameraAspect = '16:9' | '4:3' | '1:1' | '9:16';
243
+
244
+ /**
245
+ * Snapshot of the browser's webcam capture, mirrored from
246
+ * `MediaStreamTrack.getSettings()`. Forwarded to the host alongside
247
+ * `cameraResult` and exposed to the host app via
248
+ * `onCameraDemandChange` so it can render a status indicator without
249
+ * having to call `getStats()` itself.
250
+ */
251
+ export interface CameraCaptureInfo {
252
+ width?: number;
253
+ height?: number;
254
+ frameRate?: number;
255
+ deviceId?: string;
256
+ label?: string;
257
+ facingMode?: string;
258
+ }
259
+
260
+ /**
261
+ * Live outbound-camera quality snapshot. Derived from
262
+ * `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
263
+ * once per second while the camera is sending. All fields optional:
264
+ * some browsers omit fields (Safari rarely reports
265
+ * `encoderImplementation`), and the first sample after camera start
266
+ * has no delta-derived numbers yet.
267
+ */
268
+ export interface CameraStreamStats {
269
+ /** "H264", "HEVC"/"H265", "VP9", "VP8", "AV1", etc. (uppercased). */
270
+ codec?: string;
271
+ /** e.g. "VideoToolbox" (hw), "OpenH264" (sw), "ExternalEncoder". */
272
+ encoderImplementation?: string;
273
+ /**
274
+ * Browser-reported hardware-acceleration hint. Some Chromium
275
+ * versions expose this via `powerEfficientEncoder`; we mirror it
276
+ * here so consumers don't have to know the spec quirk.
277
+ */
278
+ hardwareAccelerated?: boolean;
279
+ /** Outbound encoded fps over the last sample window. */
280
+ framesPerSecond?: number;
281
+ /** Cumulative encoded frame count. */
282
+ framesEncoded?: number;
283
+ /** Outbound encoded bitrate (bits/s) over the last window. */
284
+ bitrateBps?: number;
285
+ /** Width/height the encoder is currently producing. */
286
+ width?: number;
287
+ height?: number;
288
+ /**
289
+ * One of `'none' | 'cpu' | 'bandwidth' | 'other'`. Mirrors
290
+ * Meet/Zoom's "limited by …" banners. Anything other than
291
+ * `'none'` means the encoder dropped resolution to keep up.
292
+ */
293
+ qualityLimitationReason?: string;
294
+ /** Round-trip time in milliseconds, from the matching RTCP. */
295
+ rttMs?: number;
296
+ /** Packet-loss percentage over the last sample window (0..100). */
297
+ packetsLostPct?: number;
164
298
  }
165
299
 
166
300
  interface ScreenshotData {
@@ -360,6 +494,9 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
360
494
  axPollIntervalMs,
361
495
  axMaxBackoffMs,
362
496
  onCameraDemandChange,
497
+ onCameraStats,
498
+ cameraResolutionCap = 'auto',
499
+ cameraAspect,
363
500
  }: RemoteControlProps,
364
501
  ref,
365
502
  ) => {
@@ -403,11 +540,36 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
403
540
  // green light.
404
541
  const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
405
542
  const outboundLocalStreamRef = useRef<MediaStream | null>(null);
543
+ const cameraResolutionCapRef = useRef(cameraResolutionCap);
544
+ cameraResolutionCapRef.current = cameraResolutionCap;
545
+ // The aspect prop also rides a ref so the WS `onopen` reconnect
546
+ // path can replay the operator's last pick without depending on
547
+ // an in-flight render cycle. We mutate it inline (same render-
548
+ // time pattern as the cap ref) so a parent prop change is visible
549
+ // to closures captured during the next render; the useEffect
550
+ // below handles the actual "send to host" side-effect.
551
+ const cameraAspectRef = useRef<CameraAspect | undefined>(cameraAspect);
552
+ cameraAspectRef.current = cameraAspect;
406
553
  // Mirror the demand-change callback into a ref so the WS message
407
554
  // handler always sees the freshest customer callback even when the
408
555
  // parent re-renders mid-session.
409
556
  const onCameraDemandChangeRef = useRef(onCameraDemandChange);
410
557
  onCameraDemandChangeRef.current = onCameraDemandChange;
558
+ const onCameraStatsRef = useRef(onCameraStats);
559
+ onCameraStatsRef.current = onCameraStats;
560
+ // Active outbound-camera stats poller. While the camera is
561
+ // sending we sample `RTCPeerConnection.getStats()` once per
562
+ // second, derive deltas (bitrate, fps, packet loss) from the
563
+ // previous sample, and push the result through `onCameraStats`.
564
+ // Cleared on teardown so we never leak the timer.
565
+ const cameraStatsTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
566
+ const cameraStatsPrevRef = useRef<{
567
+ timestamp: number;
568
+ framesEncoded?: number;
569
+ bytesSent?: number;
570
+ packetsSent?: number;
571
+ packetsLost?: number;
572
+ } | null>(null);
411
573
  const firstFrameShownRef = useRef(false);
412
574
  const pendingScreenshotResolversRef = useRef<
413
575
  Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
@@ -1506,6 +1668,150 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
1506
1668
  stopMediaStream(stream);
1507
1669
  };
1508
1670
 
1671
+ // Translate the user-facing resolution cap into a
1672
+ // MediaTrackConstraints fragment we can feed `getUserMedia`
1673
+ // or `applyConstraints`. Always returns a 30 fps ceiling so
1674
+ // a 60 fps webcam doesn't double our encoder cost for no
1675
+ // visible win (the simulator pool is paced at 30).
1676
+ //
1677
+ // We use `ideal` rather than `max` so that a webcam with a
1678
+ // native 720p mode still hands us 720p when the user picks
1679
+ // 1080p — instead of refusing the constraint outright. The
1680
+ // browser's NotReadableError on a too-strict `max` is the
1681
+ // most common camera-permission gotcha in the wild.
1682
+ const cameraCapToConstraints = (cap: CameraResolutionCap): MediaTrackConstraints => {
1683
+ const base: MediaTrackConstraints = {
1684
+ frameRate: { ideal: 30, max: 30 },
1685
+ };
1686
+ switch (cap) {
1687
+ case '1080p':
1688
+ return { ...base, width: { ideal: 1920 }, height: { ideal: 1080 } };
1689
+ case '720p':
1690
+ return { ...base, width: { ideal: 1280 }, height: { ideal: 720 } };
1691
+ case '480p':
1692
+ return { ...base, width: { ideal: 854 }, height: { ideal: 480 } };
1693
+ case 'auto':
1694
+ default:
1695
+ return base;
1696
+ }
1697
+ };
1698
+
1699
+ // Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
1700
+ // into a short uppercase label suitable for a status badge.
1701
+ const shortenCodecMime = (mime: string | undefined): string | undefined => {
1702
+ if (!mime) return undefined;
1703
+ const slash = mime.indexOf('/');
1704
+ const tail = slash >= 0 ? mime.slice(slash + 1) : mime;
1705
+ const upper = tail.toUpperCase();
1706
+ if (upper === 'HEVC') return 'H265';
1707
+ return upper;
1708
+ };
1709
+
1710
+ // Sample outbound-camera stats once and push a normalised
1711
+ // snapshot through `onCameraStats`. Skips silently if the sender
1712
+ // has been torn down between intervals.
1713
+ const sampleOutboundCameraStats = async () => {
1714
+ const sender = outboundCameraSenderRef.current;
1715
+ const handler = onCameraStatsRef.current;
1716
+ if (!sender || !handler) return;
1717
+ let report: RTCStatsReport;
1718
+ try {
1719
+ report = await sender.getStats();
1720
+ } catch {
1721
+ return;
1722
+ }
1723
+ let outbound: any | undefined;
1724
+ let codecMime: string | undefined;
1725
+ let remoteInbound: any | undefined;
1726
+ report.forEach((entry: any) => {
1727
+ if (entry.type === 'outbound-rtp' && entry.kind === 'video') {
1728
+ outbound = entry;
1729
+ } else if (entry.type === 'remote-inbound-rtp' && entry.kind === 'video') {
1730
+ remoteInbound = entry;
1731
+ }
1732
+ });
1733
+ if (outbound?.codecId) {
1734
+ const codec = report.get(outbound.codecId);
1735
+ if (codec) codecMime = (codec as any).mimeType;
1736
+ }
1737
+ if (!outbound) return;
1738
+ const now = (outbound.timestamp as number) ?? Date.now();
1739
+ const prev = cameraStatsPrevRef.current;
1740
+ let fps: number | undefined;
1741
+ let bitrate: number | undefined;
1742
+ let lossPct: number | undefined;
1743
+ if (prev && now > prev.timestamp) {
1744
+ const dt = (now - prev.timestamp) / 1000;
1745
+ if (
1746
+ typeof outbound.framesEncoded === 'number' &&
1747
+ typeof prev.framesEncoded === 'number'
1748
+ ) {
1749
+ fps = Math.max(0, (outbound.framesEncoded - prev.framesEncoded) / dt);
1750
+ }
1751
+ if (typeof outbound.bytesSent === 'number' && typeof prev.bytesSent === 'number') {
1752
+ bitrate = Math.max(0, ((outbound.bytesSent - prev.bytesSent) * 8) / dt);
1753
+ }
1754
+ if (
1755
+ typeof outbound.packetsSent === 'number' &&
1756
+ typeof prev.packetsSent === 'number' &&
1757
+ remoteInbound &&
1758
+ typeof remoteInbound.packetsLost === 'number' &&
1759
+ typeof prev.packetsLost === 'number'
1760
+ ) {
1761
+ const sent = outbound.packetsSent - prev.packetsSent;
1762
+ const lost = remoteInbound.packetsLost - prev.packetsLost;
1763
+ if (sent > 0) lossPct = Math.max(0, Math.min(100, (lost / sent) * 100));
1764
+ }
1765
+ }
1766
+ cameraStatsPrevRef.current = {
1767
+ timestamp: now,
1768
+ framesEncoded: outbound.framesEncoded,
1769
+ bytesSent: outbound.bytesSent,
1770
+ packetsSent: outbound.packetsSent,
1771
+ packetsLost: remoteInbound?.packetsLost,
1772
+ };
1773
+ const stats: CameraStreamStats = {
1774
+ codec: shortenCodecMime(codecMime),
1775
+ encoderImplementation: outbound.encoderImplementation,
1776
+ hardwareAccelerated:
1777
+ typeof outbound.powerEfficientEncoder === 'boolean'
1778
+ ? outbound.powerEfficientEncoder
1779
+ : undefined,
1780
+ framesPerSecond: fps,
1781
+ framesEncoded: outbound.framesEncoded,
1782
+ bitrateBps: bitrate,
1783
+ width: outbound.frameWidth,
1784
+ height: outbound.frameHeight,
1785
+ qualityLimitationReason: outbound.qualityLimitationReason,
1786
+ rttMs:
1787
+ typeof remoteInbound?.roundTripTime === 'number'
1788
+ ? remoteInbound.roundTripTime * 1000
1789
+ : undefined,
1790
+ packetsLostPct: lossPct,
1791
+ };
1792
+ safeInvoke('onCameraStats', onCameraStatsRef.current, stats);
1793
+ };
1794
+
1795
+ const startCameraStatsPoller = () => {
1796
+ if (cameraStatsTimerRef.current !== null) return;
1797
+ cameraStatsPrevRef.current = null;
1798
+ // First sample fires after one interval — the very first
1799
+ // sample has no deltas to compare against, which is fine: the
1800
+ // codec/encoder identity is still useful on its own.
1801
+ cameraStatsTimerRef.current = setInterval(() => {
1802
+ void sampleOutboundCameraStats();
1803
+ }, 1000);
1804
+ };
1805
+
1806
+ const stopCameraStatsPoller = () => {
1807
+ if (cameraStatsTimerRef.current !== null) {
1808
+ clearInterval(cameraStatsTimerRef.current);
1809
+ cameraStatsTimerRef.current = null;
1810
+ }
1811
+ cameraStatsPrevRef.current = null;
1812
+ safeInvoke('onCameraStats', onCameraStatsRef.current, null);
1813
+ };
1814
+
1509
1815
  const teardownConnection = () => {
1510
1816
  clearConnectionSuccessTimeout();
1511
1817
  clearIceDisconnectedGrace();
@@ -1517,6 +1823,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
1517
1823
  // Drop any active outbound camera before the PC dies so the
1518
1824
  // browser doesn't leave the camera indicator lit between
1519
1825
  // reconnects.
1826
+ stopCameraStatsPoller();
1520
1827
  stopOutboundLocalStream();
1521
1828
  outboundCameraSenderRef.current = null;
1522
1829
  // A scheduled cursor flush would otherwise call setState on a
@@ -1644,6 +1951,21 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
1644
1951
  if (!isCurrentAttempt() || wsRef.current !== ws) {
1645
1952
  return;
1646
1953
  }
1954
+ // Replay the saved camera aspect for fresh connections
1955
+ // (initial mount, autoreconnect, sim reboot). The host's
1956
+ // streamer starts at its boot default (16:9 / 1920×1080)
1957
+ // and a missing message would mean the operator's pick
1958
+ // silently reverts on reconnect. We always send something
1959
+ // when the prop is set, even if it matches the host
1960
+ // default — the host short-circuits no-op rebuilds.
1961
+ const initialAspect = cameraAspectRef.current;
1962
+ if (initialAspect) {
1963
+ try {
1964
+ ws.send(JSON.stringify({ type: 'cameraAspect', aspect: initialAspect }));
1965
+ } catch (err) {
1966
+ debugWarn('initial cameraAspect send failed:', err);
1967
+ }
1968
+ }
1647
1969
  settle(resolve);
1648
1970
  };
1649
1971
 
@@ -1758,6 +2080,59 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
1758
2080
  }
1759
2081
  }
1760
2082
 
2083
+ // Pin the outbound camera transceiver's codec order with
2084
+ // negotiation-time fallback. Preference order:
2085
+ // 1. H.265 / HEVC — VideoToolbox HW on M-series + recent
2086
+ // Chrome. ~30% less bitrate at equal quality. If the
2087
+ // answerer (limulator) supports HEVC, this wins.
2088
+ // 2. H.264 with VideoToolbox-friendly profile-level-ids
2089
+ // (42e01f / 42e028 / 640c1f). Universally HW-accelerated
2090
+ // on every Mac since 2009. SDP-time fallback when HEVC
2091
+ // isn't in the answer.
2092
+ // 3. Any other H.264 profile. Often resolves to OpenH264
2093
+ // (software) so we keep it explicitly last among H.264
2094
+ // to avoid Chrome's default pick.
2095
+ // 4. VP9 / VP8 (libvpx software on Mac). Last resort.
2096
+ //
2097
+ // Caveat: this is *negotiation* fallback. If HEVC HW encoder
2098
+ // init fails at session start or gets demoted mid-stream,
2099
+ // Chrome falls back to *software* HEVC, not H.264 — no spec
2100
+ // hook for runtime re-negotiation. The host-side stats
2101
+ // logger will surface this as decoder=ffmpeg / hw=false.
2102
+ if (RTCRtpSender.getCapabilities) {
2103
+ const senderCaps = RTCRtpSender.getCapabilities('video');
2104
+ if (senderCaps && senderCaps.codecs) {
2105
+ const vtH264Profiles = new Set(['42e01f', '42e028', '640c1f']);
2106
+ const score = (c: { mimeType: string; sdpFmtpLine?: string }): number => {
2107
+ const mime = c.mimeType.toLowerCase();
2108
+ const fmtp = (c.sdpFmtpLine || '').toLowerCase();
2109
+ const profileMatch = fmtp.match(/profile-level-id=([0-9a-f]{6})/);
2110
+ const profile = profileMatch ? profileMatch[1] : '';
2111
+ if (mime === 'video/h265' || mime === 'video/hevc') return 1;
2112
+ if (mime === 'video/h264' && vtH264Profiles.has(profile)) return 2;
2113
+ if (mime === 'video/h264') return 3;
2114
+ if (mime === 'video/vp9') return 4;
2115
+ if (mime === 'video/vp8') return 5;
2116
+ if (mime === 'video/rtx' || mime === 'video/red' || mime === 'video/ulpfec') {
2117
+ return 0; // Keep RTX/FEC available alongside everything else.
2118
+ }
2119
+ return 6;
2120
+ };
2121
+ const sortedSendCodecs = [...senderCaps.codecs].sort((a, b) => score(a) - score(b));
2122
+ try {
2123
+ outboundCameraTransceiver.setCodecPreferences(sortedSendCodecs);
2124
+ debugLog(
2125
+ 'Outbound camera codec preferences:',
2126
+ sortedSendCodecs
2127
+ .map((c) => `${c.mimeType}${c.sdpFmtpLine ? `[${c.sdpFmtpLine}]` : ''}`)
2128
+ .join(', '),
2129
+ );
2130
+ } catch (err) {
2131
+ debugWarn('Failed to set outbound camera codec preferences:', err);
2132
+ }
2133
+ }
2134
+ }
2135
+
1761
2136
  const dataChannel = peerConnection.createDataChannel('control', {
1762
2137
  ordered: true,
1763
2138
  negotiated: true,
@@ -2063,6 +2438,12 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2063
2438
  break;
2064
2439
  case 'cameraRequest': {
2065
2440
  const active = message.active === true;
2441
+ // Log unconditionally — this is one of the few places we
2442
+ // hand control to the browser's permission UI, and a
2443
+ // silent failure here looks identical to "limulator
2444
+ // never asked" from the user's point of view.
2445
+ // eslint-disable-next-line no-console
2446
+ console.info('[RemoteControl] cameraRequest received, active=', active);
2066
2447
  if (!active) {
2067
2448
  // Sim no longer wants frames. Drop our local track and
2068
2449
  // shut the browser's camera green light off.
@@ -2074,6 +2455,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2074
2455
  debugWarn('replaceTrack(null) on camera detach failed:', err);
2075
2456
  }
2076
2457
  }
2458
+ stopCameraStatsPoller();
2077
2459
  stopOutboundLocalStream();
2078
2460
  safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
2079
2461
  break;
@@ -2083,14 +2465,40 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2083
2465
  // `cameraResult` message so it can swap to a
2084
2466
  // black-frame fallback on denial.
2085
2467
  safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
2468
+ if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
2469
+ // Likely an insecure context (http on a non-localhost
2470
+ // origin) — Chrome strips `mediaDevices` off
2471
+ // `navigator` in that case and the only signal is
2472
+ // this undefined check.
2473
+ // eslint-disable-next-line no-console
2474
+ console.warn(
2475
+ '[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
2476
+ 'getUserMedia requires a secure context (https or http://localhost). ' +
2477
+ 'Replying cameraResult granted=false so limulator falls back to black frames.',
2478
+ );
2479
+ ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2480
+ safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2481
+ break;
2482
+ }
2086
2483
  let stream: MediaStream | null = null;
2087
2484
  try {
2485
+ // Capture at the webcam's *native* resolution by
2486
+ // default (no width/height constraints). When the
2487
+ // host app has picked an explicit cap via
2488
+ // `cameraResolutionCap`, we honour it here. Frame
2489
+ // rate is always capped to 30 to match the
2490
+ // simulator pool's pacing.
2088
2491
  stream = await navigator.mediaDevices.getUserMedia({
2089
- video: true,
2492
+ video: cameraCapToConstraints(cameraResolutionCapRef.current),
2090
2493
  audio: false,
2091
2494
  });
2092
2495
  } catch (err) {
2093
- debugWarn('getUserMedia denied/failed:', err);
2496
+ // Surface unconditionally — the user is the only one
2497
+ // who can fix this (permission denied, no device,
2498
+ // dismissed prompt, etc.). `debugWarn` alone would
2499
+ // hide it in normal builds.
2500
+ // eslint-disable-next-line no-console
2501
+ console.warn('[RemoteControl] getUserMedia denied/failed:', err);
2094
2502
  }
2095
2503
  if (!isCurrentAttempt() || wsRef.current !== ws) {
2096
2504
  if (stream) stopMediaStream(stream);
@@ -2125,19 +2533,106 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2125
2533
  safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2126
2534
  break;
2127
2535
  }
2536
+ // Tell the encoder this is real motion content (a
2537
+ // physical camera), not text/slides. With `'motion'`
2538
+ // VideoToolbox / libvpx pick latency-friendly tuning
2539
+ // (no extra B-frames, shorter GoP smoothing). Set
2540
+ // before the first frame so the encoder init reads it.
2541
+ try {
2542
+ videoTrack.contentHint = 'motion';
2543
+ } catch {
2544
+ /* older browsers don't support contentHint; ignore */
2545
+ }
2546
+ // Bound the outbound bitrate generously and let
2547
+ // WebRTC's congestion control (BWE) find the floor.
2548
+ // 8 Mbps is comfortable for native 1080p30 webcam
2549
+ // content over LAN — the encoder will use far less when
2550
+ // the scene is static, and BWE will throttle if a
2551
+ // hop is constrained. Pair with
2552
+ // `maintain-framerate` so the quality scaler steps
2553
+ // down resolution before dropping frames, matching how
2554
+ // Meet/Zoom degrade.
2555
+ try {
2556
+ const params = sender.getParameters();
2557
+ if (!params.encodings || params.encodings.length === 0) {
2558
+ params.encodings = [{}];
2559
+ }
2560
+ params.encodings[0].maxBitrate = 8_000_000;
2561
+ params.encodings[0].maxFramerate = 30;
2562
+ params.degradationPreference = 'maintain-framerate';
2563
+ await sender.setParameters(params);
2564
+ } catch (err) {
2565
+ debugWarn('setParameters on outbound camera failed:', err);
2566
+ }
2567
+ // Surface the actual captured geometry to the host so
2568
+ // it can size its IOSurface pool / picture-format
2569
+ // metadata to match. `getSettings()` returns what the
2570
+ // browser actually picked — which may differ from any
2571
+ // hints we sent in the constraints — including the
2572
+ // selected `deviceId` / `label` (useful when a user has
2573
+ // multiple cameras and we eventually expose a picker).
2574
+ let cameraMetadata: {
2575
+ width?: number;
2576
+ height?: number;
2577
+ frameRate?: number;
2578
+ deviceId?: string;
2579
+ label?: string;
2580
+ facingMode?: string;
2581
+ } = {};
2582
+ try {
2583
+ const settings = videoTrack.getSettings();
2584
+ cameraMetadata = {
2585
+ width: settings.width,
2586
+ height: settings.height,
2587
+ frameRate: settings.frameRate,
2588
+ deviceId: settings.deviceId,
2589
+ label: videoTrack.label || undefined,
2590
+ facingMode: settings.facingMode,
2591
+ };
2592
+ // eslint-disable-next-line no-console
2593
+ console.info(
2594
+ `[RemoteControl] camera capture: ${cameraMetadata.width}x${cameraMetadata.height}` +
2595
+ ` @ ${cameraMetadata.frameRate ?? '?'}fps` +
2596
+ (cameraMetadata.label ? ` — ${cameraMetadata.label}` : ''),
2597
+ );
2598
+ } catch (err) {
2599
+ debugWarn('getSettings() on outbound camera track failed:', err);
2600
+ }
2128
2601
  // If the browser revokes the track later (extension,
2129
2602
  // user clicks Stop in the camera tab UI, etc.), notify
2130
2603
  // the host so it can switch to black frames.
2131
2604
  videoTrack.onended = () => {
2132
2605
  if (outboundLocalStreamRef.current !== stream) return;
2606
+ stopCameraStatsPoller();
2133
2607
  stopOutboundLocalStream();
2134
2608
  if (wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
2135
2609
  ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
2136
2610
  }
2137
2611
  safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
2138
2612
  };
2139
- ws.send(JSON.stringify({ type: 'cameraResult', granted: true }));
2140
- safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true);
2613
+ ws.send(JSON.stringify({
2614
+ type: 'cameraResult',
2615
+ granted: true,
2616
+ // Forward what the browser actually captured so the
2617
+ // host can size its IOSurface pool, log the device,
2618
+ // and (eventually) surface a status pill / picker.
2619
+ camera: cameraMetadata,
2620
+ }));
2621
+ safeInvoke(
2622
+ 'onCameraDemandChange',
2623
+ onCameraDemandChangeRef.current,
2624
+ true,
2625
+ true,
2626
+ cameraMetadata,
2627
+ );
2628
+ // Kick off the per-second outbound stats sampler. We do
2629
+ // this *after* `setParameters` so the first sample
2630
+ // already sees the encoder under its final bitrate /
2631
+ // degradation policy, and *after* the host-side
2632
+ // attachInboundTrack will have wired up (the
2633
+ // cameraResult ACK is what triggers it on the host),
2634
+ // so framesEncoded starts climbing immediately.
2635
+ startCameraStatsPoller();
2141
2636
  break;
2142
2637
  }
2143
2638
  case 'terminateAppResult':
@@ -2227,6 +2722,41 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
2227
2722
  start();
2228
2723
  };
2229
2724
 
2725
+ // Re-apply the resolution cap on the currently-sending track
2726
+ // whenever the host app changes its preference. Skips when no
2727
+ // camera is active — the next `getUserMedia` will pick up the
2728
+ // new value via `cameraCapToConstraints` automatically.
2729
+ useEffect(() => {
2730
+ const stream = outboundLocalStreamRef.current;
2731
+ if (!stream) return;
2732
+ const track = stream.getVideoTracks()[0];
2733
+ if (!track) return;
2734
+ const constraints = cameraCapToConstraints(cameraResolutionCap);
2735
+ track.applyConstraints(constraints).catch((err) => {
2736
+ debugWarn('applyConstraints for new camera cap failed:', err);
2737
+ });
2738
+ // eslint-disable-next-line react-hooks/exhaustive-deps
2739
+ }, [cameraResolutionCap]);
2740
+
2741
+ // Push the aspect preference to the host whenever it changes.
2742
+ // The host rebuilds its IOSurface pool and bumps `pool_generation`
2743
+ // on its end; the dylib re-handshakes on next sem_wait. No
2744
+ // peer-connection renegotiation is needed — the aspect change is
2745
+ // purely about the pixel buffer dimensions iOS apps observe, not
2746
+ // about WebRTC track layout. The `cameraAspectRef` is mirrored
2747
+ // higher up so the WS `onopen` reconnect path can replay the
2748
+ // latest value on fresh connections.
2749
+ useEffect(() => {
2750
+ if (!cameraAspect) return;
2751
+ const ws = wsRef.current;
2752
+ if (!ws || ws.readyState !== WebSocket.OPEN) return;
2753
+ try {
2754
+ ws.send(JSON.stringify({ type: 'cameraAspect', aspect: cameraAspect }));
2755
+ } catch (err) {
2756
+ debugWarn('cameraAspect send failed:', err);
2757
+ }
2758
+ }, [cameraAspect]);
2759
+
2230
2760
  useEffect(() => {
2231
2761
  // Reset video loaded state when connection params change
2232
2762
  setVideoLoaded(false);