@limrun/ui 0.13.8 → 0.13.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/components/remote-control.d.ts +156 -0
- package/dist/index.cjs +1 -1
- package/dist/index.css +1 -1
- package/dist/index.js +1405 -1125
- package/package.json +1 -1
- package/src/components/remote-control.css +32 -0
- package/src/components/remote-control.tsx +865 -5
- package/src/device-install/README.md +1 -1
|
@@ -59,6 +59,19 @@ interface RemoteControlProps {
|
|
|
59
59
|
// surfacing the manual "Retry" button. Defaults to false.
|
|
60
60
|
autoReconnect?: boolean;
|
|
61
61
|
|
|
62
|
+
/**
|
|
63
|
+
* Fires once when the component concludes the instance is permanently
|
|
64
|
+
* gone rather than suffering a recoverable connection blip.
|
|
65
|
+
*
|
|
66
|
+
* After a drop, the component probes the instance endpoint over HTTP;
|
|
67
|
+
* a 404 means the instance no longer exists (terminated/deleted). When
|
|
68
|
+
* that happens the component stops reconnecting and renders a built-in
|
|
69
|
+
* "terminated" message in place of the "Retry" button, and invokes this
|
|
70
|
+
* callback so the host can react (navigate away, surface its own UI,
|
|
71
|
+
* etc.). It is not called for ordinary network drops.
|
|
72
|
+
*/
|
|
73
|
+
onTerminated?: () => void;
|
|
74
|
+
|
|
62
75
|
/**
|
|
63
76
|
* Enable the inspect overlay. When set, the component starts polling the
|
|
64
77
|
* accessibility tree and draws boxes over each element on top of the
|
|
@@ -140,6 +153,157 @@ interface RemoteControlProps {
|
|
|
140
153
|
* @default 2000
|
|
141
154
|
*/
|
|
142
155
|
axMaxBackoffMs?: number;
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Fires whenever the iOS simulator's camera demand state changes —
|
|
159
|
+
* i.e. an app inside the sim called
|
|
160
|
+
* `[AVCaptureSession startRunning]` or `[stopRunning]`. The
|
|
161
|
+
* component handles the `navigator.mediaDevices.getUserMedia` prompt
|
|
162
|
+
* and SDP plumbing internally; this callback is purely so the host
|
|
163
|
+
* UI can render a status indicator ("simulator is using your
|
|
164
|
+
* camera", etc.).
|
|
165
|
+
*
|
|
166
|
+
* `active` reflects whether the sim is currently asking for
|
|
167
|
+
* frames. `granted` is set only on the call that follows a
|
|
168
|
+
* `getUserMedia` attempt: `true` when the user accepted the
|
|
169
|
+
* browser prompt, `false` when they denied or the call failed
|
|
170
|
+
* (in which case the limulator side switches to a black-frame
|
|
171
|
+
* fallback so the app keeps ticking).
|
|
172
|
+
*
|
|
173
|
+
* `camera` (optional, only meaningful when `granted === true`)
|
|
174
|
+
* carries what `MediaStreamTrack.getSettings()` reported for the
|
|
175
|
+
* active capture: resolution, framerate, device label, facing
|
|
176
|
+
* mode. Use it to render a richer status indicator (e.g.
|
|
177
|
+
* "Camera · 1920×1080 · 30 fps · FaceTime HD").
|
|
178
|
+
*
|
|
179
|
+
* Only iOS instances ever fire this callback; Android instances
|
|
180
|
+
* have no camera-injector path and stay silent.
|
|
181
|
+
*/
|
|
182
|
+
onCameraDemandChange?: (active: boolean, granted?: boolean, camera?: CameraCaptureInfo) => void;
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Periodically (~1Hz) fires with a snapshot of the outbound
|
|
186
|
+
* camera stream's live WebRTC stats — codec, encoder
|
|
187
|
+
* implementation, hardware acceleration, encoded fps, bitrate,
|
|
188
|
+
* round-trip-time, and the encoder's
|
|
189
|
+
* `qualityLimitationReason` (which is what Meet/Zoom use to
|
|
190
|
+
* decide whether to show "Bandwidth limited" or "CPU limited"
|
|
191
|
+
* banners).
|
|
192
|
+
*
|
|
193
|
+
* Only fires while the simulator is actively pulling the
|
|
194
|
+
* camera AND `getUserMedia` was granted. `null` is emitted
|
|
195
|
+
* once when the stream goes back to idle so consumers can
|
|
196
|
+
* clear their UI without having to maintain their own
|
|
197
|
+
* timeout. The host app does not need to poll `getStats()`
|
|
198
|
+
* itself — this is the canonical place.
|
|
199
|
+
*/
|
|
200
|
+
onCameraStats?: (stats: CameraStreamStats | null) => void;
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Optional resolution cap for outbound camera capture.
|
|
204
|
+
*
|
|
205
|
+
* - `'auto'` (default): no extra constraint. The browser captures
|
|
206
|
+
* at its webcam's native max; WebRTC's quality scaler may step
|
|
207
|
+
* down resolution on the encoder side under bandwidth pressure
|
|
208
|
+
* while still feeding the simulator at the pool's native size.
|
|
209
|
+
* - `'1080p'` / `'720p'` / `'480p'`: hard cap applied via
|
|
210
|
+
* `getUserMedia` constraints (for new captures) and
|
|
211
|
+
* `track.applyConstraints` (for the currently-active track),
|
|
212
|
+
* matching the way Meet/Zoom expose a "Send resolution" picker.
|
|
213
|
+
*
|
|
214
|
+
* Bumping or lowering the cap mid-stream is supported; the
|
|
215
|
+
* change takes effect within a frame or two as the webcam
|
|
216
|
+
* re-negotiates.
|
|
217
|
+
*/
|
|
218
|
+
cameraResolutionCap?: CameraResolutionCap;
|
|
219
|
+
/**
|
|
220
|
+
* Aspect ratio the simulator's virtual camera should report to apps.
|
|
221
|
+
* Picking a value here triggers a `cameraAspect` WS message to the
|
|
222
|
+
* host, which rebuilds its IOSurface ring at the matching dimensions
|
|
223
|
+
* (16:9 → 1920×1080, 4:3 → 1440×1080, 1:1 → 1080×1080, 9:16 →
|
|
224
|
+
* 1080×1920) and signals the in-sim dylib to re-handshake. iOS apps
|
|
225
|
+
* see CMSampleBuffers at the new dimensions within a frame or two.
|
|
226
|
+
*
|
|
227
|
+
* The browser still captures whatever the webcam offers; the host
|
|
228
|
+
* aspect-fills (cover, center-crop) into the new pool. Switching
|
|
229
|
+
* aspect at runtime is intentionally cheap so users can A/B preview
|
|
230
|
+
* styles without restarting the simulator.
|
|
231
|
+
*
|
|
232
|
+
* `undefined` leaves the pool untouched (the host's boot default —
|
|
233
|
+
* 16:9 / 1920×1080 — applies).
|
|
234
|
+
*/
|
|
235
|
+
cameraAspect?: CameraAspect;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Resolution caps a host app can request on the outbound camera.
|
|
240
|
+
* `'auto'` is "let the browser decide" (no constraints beyond the
|
|
241
|
+
* 30 fps ceiling); the other options clamp width/height to the
|
|
242
|
+
* named target. Aspect ratio is preserved.
|
|
243
|
+
*/
|
|
244
|
+
export type CameraResolutionCap = 'auto' | '1080p' | '720p' | '480p';
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Aspect ratios exposed to the operator for the simulated camera.
|
|
248
|
+
* The host maps each label to concrete IOSurface dimensions; values
|
|
249
|
+
* the host doesn't recognise are silently ignored.
|
|
250
|
+
*/
|
|
251
|
+
export type CameraAspect = '16:9' | '4:3' | '1:1' | '9:16';
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Snapshot of the browser's webcam capture, mirrored from
|
|
255
|
+
* `MediaStreamTrack.getSettings()`. Forwarded to the host alongside
|
|
256
|
+
* `cameraResult` and exposed to the host app via
|
|
257
|
+
* `onCameraDemandChange` so it can render a status indicator without
|
|
258
|
+
* having to call `getStats()` itself.
|
|
259
|
+
*/
|
|
260
|
+
export interface CameraCaptureInfo {
|
|
261
|
+
width?: number;
|
|
262
|
+
height?: number;
|
|
263
|
+
frameRate?: number;
|
|
264
|
+
deviceId?: string;
|
|
265
|
+
label?: string;
|
|
266
|
+
facingMode?: string;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Live outbound-camera quality snapshot. Derived from
|
|
271
|
+
* `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
|
|
272
|
+
* once per second while the camera is sending. All fields optional:
|
|
273
|
+
* some browsers omit fields (Safari rarely reports
|
|
274
|
+
* `encoderImplementation`), and the first sample after camera start
|
|
275
|
+
* has no delta-derived numbers yet.
|
|
276
|
+
*/
|
|
277
|
+
export interface CameraStreamStats {
|
|
278
|
+
/** "H264", "HEVC"/"H265", "VP9", "VP8", "AV1", etc. (uppercased). */
|
|
279
|
+
codec?: string;
|
|
280
|
+
/** e.g. "VideoToolbox" (hw), "OpenH264" (sw), "ExternalEncoder". */
|
|
281
|
+
encoderImplementation?: string;
|
|
282
|
+
/**
|
|
283
|
+
* Browser-reported hardware-acceleration hint. Some Chromium
|
|
284
|
+
* versions expose this via `powerEfficientEncoder`; we mirror it
|
|
285
|
+
* here so consumers don't have to know the spec quirk.
|
|
286
|
+
*/
|
|
287
|
+
hardwareAccelerated?: boolean;
|
|
288
|
+
/** Outbound encoded fps over the last sample window. */
|
|
289
|
+
framesPerSecond?: number;
|
|
290
|
+
/** Cumulative encoded frame count. */
|
|
291
|
+
framesEncoded?: number;
|
|
292
|
+
/** Outbound encoded bitrate (bits/s) over the last window. */
|
|
293
|
+
bitrateBps?: number;
|
|
294
|
+
/** Width/height the encoder is currently producing. */
|
|
295
|
+
width?: number;
|
|
296
|
+
height?: number;
|
|
297
|
+
/**
|
|
298
|
+
* One of `'none' | 'cpu' | 'bandwidth' | 'other'`. Mirrors
|
|
299
|
+
* Meet/Zoom's "limited by …" banners. Anything other than
|
|
300
|
+
* `'none'` means the encoder dropped resolution to keep up.
|
|
301
|
+
*/
|
|
302
|
+
qualityLimitationReason?: string;
|
|
303
|
+
/** Round-trip time in milliseconds, from the matching RTCP. */
|
|
304
|
+
rttMs?: number;
|
|
305
|
+
/** Packet-loss percentage over the last sample window (0..100). */
|
|
306
|
+
packetsLostPct?: number;
|
|
143
307
|
}
|
|
144
308
|
|
|
145
309
|
interface ScreenshotData {
|
|
@@ -254,6 +418,9 @@ const MAX_CONNECTION_ATTEMPTS = 3;
|
|
|
254
418
|
const CONNECTION_RETRY_DELAY_MS = 1000;
|
|
255
419
|
const CONNECTION_SUCCESS_TIMEOUT_MS = 15000;
|
|
256
420
|
const ICE_DISCONNECTED_GRACE_MS = 3000;
|
|
421
|
+
// Upper bound for the one-shot HTTP probe that disambiguates a terminated
|
|
422
|
+
// instance (endpoint 404s) from a transient connection drop.
|
|
423
|
+
const TERMINATION_PROBE_TIMEOUT_MS = 4000;
|
|
257
424
|
|
|
258
425
|
const isAndroidTabletVideo = (width: number, height: number): boolean =>
|
|
259
426
|
(width === ANDROID_TABLET_VIDEO_WIDTH && height === ANDROID_TABLET_VIDEO_HEIGHT) ||
|
|
@@ -332,12 +499,17 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
332
499
|
openUrl,
|
|
333
500
|
showFrame = true,
|
|
334
501
|
autoReconnect = false,
|
|
502
|
+
onTerminated,
|
|
335
503
|
inspectMode,
|
|
336
504
|
onAxSnapshotChange,
|
|
337
505
|
onInspectSelectionChange,
|
|
338
506
|
onAxStatusChange,
|
|
339
507
|
axPollIntervalMs,
|
|
340
508
|
axMaxBackoffMs,
|
|
509
|
+
onCameraDemandChange,
|
|
510
|
+
onCameraStats,
|
|
511
|
+
cameraResolutionCap = 'auto',
|
|
512
|
+
cameraAspect,
|
|
341
513
|
}: RemoteControlProps,
|
|
342
514
|
ref,
|
|
343
515
|
) => {
|
|
@@ -346,6 +518,13 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
346
518
|
const frameRef = useRef<HTMLImageElement>(null);
|
|
347
519
|
const [videoLoaded, setVideoLoaded] = useState(false);
|
|
348
520
|
const [retryExhausted, setRetryExhausted] = useState(false);
|
|
521
|
+
// Set once we've concluded the instance is permanently gone (its
|
|
522
|
+
// endpoint 404s after a drop), as opposed to a recoverable blip. When
|
|
523
|
+
// set, reconnection stops and a terminated message replaces "Retry".
|
|
524
|
+
const [terminated, setTerminated] = useState(false);
|
|
525
|
+
const terminatedRef = useRef(false);
|
|
526
|
+
// Guards against firing overlapping termination probes.
|
|
527
|
+
const terminationProbeInFlightRef = useRef(false);
|
|
349
528
|
const [isLandscape, setIsLandscape] = useState(false);
|
|
350
529
|
const [useAndroidTabletFrame, setUseAndroidTabletFrame] = useState(false);
|
|
351
530
|
const [videoStyle, setVideoStyle] = useState<React.CSSProperties>({});
|
|
@@ -363,6 +542,58 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
363
542
|
// Mirrored to a ref so stale closures in event handlers see the latest value.
|
|
364
543
|
const autoReconnectRef = useRef(autoReconnect);
|
|
365
544
|
autoReconnectRef.current = autoReconnect;
|
|
545
|
+
// Demand-driven outbound camera state.
|
|
546
|
+
//
|
|
547
|
+
// The limulator side broadcasts a `cameraRequest` WS message whenever
|
|
548
|
+
// an app inside the simulator opens/closes an `AVCaptureSession`.
|
|
549
|
+
// We respond by calling `navigator.mediaDevices.getUserMedia(...)`
|
|
550
|
+
// and `replaceTrack`ing the result onto a pre-allocated sendonly
|
|
551
|
+
// video transceiver. Pre-allocating the transceiver lets us
|
|
552
|
+
// attach/detach the local camera without renegotiating the SDP
|
|
553
|
+
// (the slot is already in the answer's a=video block, just with
|
|
554
|
+
// `inactive`/empty until we install the track).
|
|
555
|
+
//
|
|
556
|
+
// Refs (not state) because all callers live inside event-handler
|
|
557
|
+
// closures and the ref read happens during message processing, not
|
|
558
|
+
// during render. We keep both the sender and the active local
|
|
559
|
+
// stream so teardown can stop tracks without leaking the camera
|
|
560
|
+
// green light.
|
|
561
|
+
const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
|
|
562
|
+
const outboundLocalStreamRef = useRef<MediaStream | null>(null);
|
|
563
|
+
// Bumped on every `cameraRequest` so a handler suspended on an
|
|
564
|
+
// await (e.g. the getUserMedia prompt) can detect it was superseded
|
|
565
|
+
// by a newer request and bail instead of re-attaching the camera.
|
|
566
|
+
const cameraRequestGenerationRef = useRef(0);
|
|
567
|
+
const cameraResolutionCapRef = useRef(cameraResolutionCap);
|
|
568
|
+
cameraResolutionCapRef.current = cameraResolutionCap;
|
|
569
|
+
// The aspect prop also rides a ref so the WS `onopen` reconnect
|
|
570
|
+
// path can replay the operator's last pick without depending on
|
|
571
|
+
// an in-flight render cycle. We mutate it inline (same render-
|
|
572
|
+
// time pattern as the cap ref) so a parent prop change is visible
|
|
573
|
+
// to closures captured during the next render; the useEffect
|
|
574
|
+
// below handles the actual "send to host" side-effect.
|
|
575
|
+
const cameraAspectRef = useRef<CameraAspect | undefined>(cameraAspect);
|
|
576
|
+
cameraAspectRef.current = cameraAspect;
|
|
577
|
+
// Mirror the demand-change callback into a ref so the WS message
|
|
578
|
+
// handler always sees the freshest customer callback even when the
|
|
579
|
+
// parent re-renders mid-session.
|
|
580
|
+
const onCameraDemandChangeRef = useRef(onCameraDemandChange);
|
|
581
|
+
onCameraDemandChangeRef.current = onCameraDemandChange;
|
|
582
|
+
const onCameraStatsRef = useRef(onCameraStats);
|
|
583
|
+
onCameraStatsRef.current = onCameraStats;
|
|
584
|
+
// Active outbound-camera stats poller. While the camera is
|
|
585
|
+
// sending we sample `RTCPeerConnection.getStats()` once per
|
|
586
|
+
// second, derive deltas (bitrate, fps, packet loss) from the
|
|
587
|
+
// previous sample, and push the result through `onCameraStats`.
|
|
588
|
+
// Cleared on teardown so we never leak the timer.
|
|
589
|
+
const cameraStatsTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
|
590
|
+
const cameraStatsPrevRef = useRef<{
|
|
591
|
+
timestamp: number;
|
|
592
|
+
framesEncoded?: number;
|
|
593
|
+
bytesSent?: number;
|
|
594
|
+
packetsSent?: number;
|
|
595
|
+
packetsLost?: number;
|
|
596
|
+
} | null>(null);
|
|
366
597
|
const firstFrameShownRef = useRef(false);
|
|
367
598
|
const pendingScreenshotResolversRef = useRef<
|
|
368
599
|
Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
|
|
@@ -463,6 +694,8 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
463
694
|
onInspectSelectionChangeRef.current = onInspectSelectionChange;
|
|
464
695
|
const onAxStatusChangeRef = useRef(onAxStatusChange);
|
|
465
696
|
onAxStatusChangeRef.current = onAxStatusChange;
|
|
697
|
+
const onTerminatedRef = useRef(onTerminated);
|
|
698
|
+
onTerminatedRef.current = onTerminated;
|
|
466
699
|
|
|
467
700
|
const inspectActive = inspectMode === true || inspectMode === 'hover-only';
|
|
468
701
|
const inspectModeResolved: InspectMode = inspectMode === 'hover-only' ? 'hover-only' : 'select';
|
|
@@ -1435,6 +1668,169 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1435
1668
|
setVideoLoaded(true);
|
|
1436
1669
|
};
|
|
1437
1670
|
|
|
1671
|
+
// Stop every track on a MediaStream and release the device handle.
|
|
1672
|
+
// The browser keeps the camera "on" indicator lit until at least
|
|
1673
|
+
// one track on the underlying source is stopped, so we have to
|
|
1674
|
+
// call stop() explicitly — closing the peer connection alone
|
|
1675
|
+
// won't do it.
|
|
1676
|
+
const stopMediaStream = (stream: MediaStream) => {
|
|
1677
|
+
for (const track of stream.getTracks()) {
|
|
1678
|
+
try {
|
|
1679
|
+
track.stop();
|
|
1680
|
+
} catch (err) {
|
|
1681
|
+
debugWarn('track.stop() failed:', err);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
};
|
|
1685
|
+
|
|
1686
|
+
// Stop and forget any currently-attached outbound camera stream.
|
|
1687
|
+
// Safe to call when nothing is attached. Does not touch the
|
|
1688
|
+
// sender — the caller is responsible for `replaceTrack(null)`
|
|
1689
|
+
// separately when it wants the SDP slot itself empty.
|
|
1690
|
+
const stopOutboundLocalStream = () => {
|
|
1691
|
+
const stream = outboundLocalStreamRef.current;
|
|
1692
|
+
if (!stream) return;
|
|
1693
|
+
outboundLocalStreamRef.current = null;
|
|
1694
|
+
stopMediaStream(stream);
|
|
1695
|
+
};
|
|
1696
|
+
|
|
1697
|
+
// Translate the user-facing resolution cap into a
|
|
1698
|
+
// MediaTrackConstraints fragment we can feed `getUserMedia`
|
|
1699
|
+
// or `applyConstraints`. Always returns a 30 fps ceiling so
|
|
1700
|
+
// a 60 fps webcam doesn't double our encoder cost for no
|
|
1701
|
+
// visible win (the simulator pool is paced at 30).
|
|
1702
|
+
//
|
|
1703
|
+
// We use `ideal` rather than `max` so that a webcam with a
|
|
1704
|
+
// native 720p mode still hands us 720p when the user picks
|
|
1705
|
+
// 1080p — instead of refusing the constraint outright. The
|
|
1706
|
+
// browser's NotReadableError on a too-strict `max` is the
|
|
1707
|
+
// most common camera-permission gotcha in the wild.
|
|
1708
|
+
const cameraCapToConstraints = (cap: CameraResolutionCap): MediaTrackConstraints => {
|
|
1709
|
+
const base: MediaTrackConstraints = {
|
|
1710
|
+
frameRate: { ideal: 30, max: 30 },
|
|
1711
|
+
};
|
|
1712
|
+
switch (cap) {
|
|
1713
|
+
case '1080p':
|
|
1714
|
+
return { ...base, width: { ideal: 1920 }, height: { ideal: 1080 } };
|
|
1715
|
+
case '720p':
|
|
1716
|
+
return { ...base, width: { ideal: 1280 }, height: { ideal: 720 } };
|
|
1717
|
+
case '480p':
|
|
1718
|
+
return { ...base, width: { ideal: 854 }, height: { ideal: 480 } };
|
|
1719
|
+
case 'auto':
|
|
1720
|
+
default:
|
|
1721
|
+
return base;
|
|
1722
|
+
}
|
|
1723
|
+
};
|
|
1724
|
+
|
|
1725
|
+
// Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
|
|
1726
|
+
// into a short uppercase label suitable for a status badge.
|
|
1727
|
+
const shortenCodecMime = (mime: string | undefined): string | undefined => {
|
|
1728
|
+
if (!mime) return undefined;
|
|
1729
|
+
const slash = mime.indexOf('/');
|
|
1730
|
+
const tail = slash >= 0 ? mime.slice(slash + 1) : mime;
|
|
1731
|
+
const upper = tail.toUpperCase();
|
|
1732
|
+
if (upper === 'HEVC') return 'H265';
|
|
1733
|
+
return upper;
|
|
1734
|
+
};
|
|
1735
|
+
|
|
1736
|
+
// Sample outbound-camera stats once and push a normalised
|
|
1737
|
+
// snapshot through `onCameraStats`. Skips silently if the sender
|
|
1738
|
+
// has been torn down between intervals.
|
|
1739
|
+
const sampleOutboundCameraStats = async () => {
|
|
1740
|
+
const sender = outboundCameraSenderRef.current;
|
|
1741
|
+
const handler = onCameraStatsRef.current;
|
|
1742
|
+
if (!sender || !handler) return;
|
|
1743
|
+
let report: RTCStatsReport;
|
|
1744
|
+
try {
|
|
1745
|
+
report = await sender.getStats();
|
|
1746
|
+
} catch {
|
|
1747
|
+
return;
|
|
1748
|
+
}
|
|
1749
|
+
let outbound: any | undefined;
|
|
1750
|
+
let codecMime: string | undefined;
|
|
1751
|
+
let remoteInbound: any | undefined;
|
|
1752
|
+
report.forEach((entry: any) => {
|
|
1753
|
+
if (entry.type === 'outbound-rtp' && entry.kind === 'video') {
|
|
1754
|
+
outbound = entry;
|
|
1755
|
+
} else if (entry.type === 'remote-inbound-rtp' && entry.kind === 'video') {
|
|
1756
|
+
remoteInbound = entry;
|
|
1757
|
+
}
|
|
1758
|
+
});
|
|
1759
|
+
if (outbound?.codecId) {
|
|
1760
|
+
const codec = report.get(outbound.codecId);
|
|
1761
|
+
if (codec) codecMime = (codec as any).mimeType;
|
|
1762
|
+
}
|
|
1763
|
+
if (!outbound) return;
|
|
1764
|
+
const now = (outbound.timestamp as number) ?? Date.now();
|
|
1765
|
+
const prev = cameraStatsPrevRef.current;
|
|
1766
|
+
let fps: number | undefined;
|
|
1767
|
+
let bitrate: number | undefined;
|
|
1768
|
+
let lossPct: number | undefined;
|
|
1769
|
+
if (prev && now > prev.timestamp) {
|
|
1770
|
+
const dt = (now - prev.timestamp) / 1000;
|
|
1771
|
+
if (typeof outbound.framesEncoded === 'number' && typeof prev.framesEncoded === 'number') {
|
|
1772
|
+
fps = Math.max(0, (outbound.framesEncoded - prev.framesEncoded) / dt);
|
|
1773
|
+
}
|
|
1774
|
+
if (typeof outbound.bytesSent === 'number' && typeof prev.bytesSent === 'number') {
|
|
1775
|
+
bitrate = Math.max(0, ((outbound.bytesSent - prev.bytesSent) * 8) / dt);
|
|
1776
|
+
}
|
|
1777
|
+
if (
|
|
1778
|
+
typeof outbound.packetsSent === 'number' &&
|
|
1779
|
+
typeof prev.packetsSent === 'number' &&
|
|
1780
|
+
remoteInbound &&
|
|
1781
|
+
typeof remoteInbound.packetsLost === 'number' &&
|
|
1782
|
+
typeof prev.packetsLost === 'number'
|
|
1783
|
+
) {
|
|
1784
|
+
const sent = outbound.packetsSent - prev.packetsSent;
|
|
1785
|
+
const lost = remoteInbound.packetsLost - prev.packetsLost;
|
|
1786
|
+
if (sent > 0) lossPct = Math.max(0, Math.min(100, (lost / sent) * 100));
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
cameraStatsPrevRef.current = {
|
|
1790
|
+
timestamp: now,
|
|
1791
|
+
framesEncoded: outbound.framesEncoded,
|
|
1792
|
+
bytesSent: outbound.bytesSent,
|
|
1793
|
+
packetsSent: outbound.packetsSent,
|
|
1794
|
+
packetsLost: remoteInbound?.packetsLost,
|
|
1795
|
+
};
|
|
1796
|
+
const stats: CameraStreamStats = {
|
|
1797
|
+
codec: shortenCodecMime(codecMime),
|
|
1798
|
+
encoderImplementation: outbound.encoderImplementation,
|
|
1799
|
+
hardwareAccelerated:
|
|
1800
|
+
typeof outbound.powerEfficientEncoder === 'boolean' ? outbound.powerEfficientEncoder : undefined,
|
|
1801
|
+
framesPerSecond: fps,
|
|
1802
|
+
framesEncoded: outbound.framesEncoded,
|
|
1803
|
+
bitrateBps: bitrate,
|
|
1804
|
+
width: outbound.frameWidth,
|
|
1805
|
+
height: outbound.frameHeight,
|
|
1806
|
+
qualityLimitationReason: outbound.qualityLimitationReason,
|
|
1807
|
+
rttMs:
|
|
1808
|
+
typeof remoteInbound?.roundTripTime === 'number' ? remoteInbound.roundTripTime * 1000 : undefined,
|
|
1809
|
+
packetsLostPct: lossPct,
|
|
1810
|
+
};
|
|
1811
|
+
safeInvoke('onCameraStats', onCameraStatsRef.current, stats);
|
|
1812
|
+
};
|
|
1813
|
+
|
|
1814
|
+
const startCameraStatsPoller = () => {
|
|
1815
|
+
if (cameraStatsTimerRef.current !== null) return;
|
|
1816
|
+
cameraStatsPrevRef.current = null;
|
|
1817
|
+
// First sample fires after one interval — the very first
|
|
1818
|
+
// sample has no deltas to compare against, which is fine: the
|
|
1819
|
+
// codec/encoder identity is still useful on its own.
|
|
1820
|
+
cameraStatsTimerRef.current = setInterval(() => {
|
|
1821
|
+
void sampleOutboundCameraStats();
|
|
1822
|
+
}, 1000);
|
|
1823
|
+
};
|
|
1824
|
+
|
|
1825
|
+
const stopCameraStatsPoller = () => {
|
|
1826
|
+
if (cameraStatsTimerRef.current !== null) {
|
|
1827
|
+
clearInterval(cameraStatsTimerRef.current);
|
|
1828
|
+
cameraStatsTimerRef.current = null;
|
|
1829
|
+
}
|
|
1830
|
+
cameraStatsPrevRef.current = null;
|
|
1831
|
+
safeInvoke('onCameraStats', onCameraStatsRef.current, null);
|
|
1832
|
+
};
|
|
1833
|
+
|
|
1438
1834
|
const teardownConnection = () => {
|
|
1439
1835
|
clearConnectionSuccessTimeout();
|
|
1440
1836
|
clearIceDisconnectedGrace();
|
|
@@ -1443,6 +1839,12 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1443
1839
|
axFetcherRef.current.stop();
|
|
1444
1840
|
axFetcherRef.current = null;
|
|
1445
1841
|
}
|
|
1842
|
+
// Drop any active outbound camera before the PC dies so the
|
|
1843
|
+
// browser doesn't leave the camera indicator lit between
|
|
1844
|
+
// reconnects.
|
|
1845
|
+
stopCameraStatsPoller();
|
|
1846
|
+
stopOutboundLocalStream();
|
|
1847
|
+
outboundCameraSenderRef.current = null;
|
|
1446
1848
|
// A scheduled cursor flush would otherwise call setState on a
|
|
1447
1849
|
// teardown component once the next frame runs.
|
|
1448
1850
|
if (cursorRafIdRef.current !== undefined) {
|
|
@@ -1485,11 +1887,92 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1485
1887
|
}
|
|
1486
1888
|
};
|
|
1487
1889
|
|
|
1890
|
+
// Probe the instance endpoint over HTTP to tell a permanent termination
|
|
1891
|
+
// apart from a recoverable connection drop. The router serves the same
|
|
1892
|
+
// path the WebSocket uses and returns 404 once the instance is gone
|
|
1893
|
+
// (deleted / terminated), while a live instance returns something else.
|
|
1894
|
+
//
|
|
1895
|
+
// Only an authoritative, freshly-fetched 404 counts as terminated. A
|
|
1896
|
+
// network outage, a non-404 status, or a CORS-blocked (unreadable)
|
|
1897
|
+
// response all resolve to `false` so we keep the normal retry/reconnect
|
|
1898
|
+
// behavior and never mistake a connectivity problem for a termination.
|
|
1899
|
+
const probeTerminated = async (): Promise<boolean> => {
|
|
1900
|
+
// The browser knows it has no network — this is a connectivity drop,
|
|
1901
|
+
// not a termination. Don't probe (and don't risk a cached 404).
|
|
1902
|
+
if (typeof navigator !== 'undefined' && navigator.onLine === false) {
|
|
1903
|
+
return false;
|
|
1904
|
+
}
|
|
1905
|
+
let probeUrl: string;
|
|
1906
|
+
try {
|
|
1907
|
+
const parsed = new URL(url);
|
|
1908
|
+
parsed.protocol =
|
|
1909
|
+
parsed.protocol === 'wss:' ? 'https:'
|
|
1910
|
+
: parsed.protocol === 'ws:' ? 'http:'
|
|
1911
|
+
: parsed.protocol;
|
|
1912
|
+
parsed.searchParams.set('token', token);
|
|
1913
|
+
probeUrl = parsed.toString();
|
|
1914
|
+
} catch {
|
|
1915
|
+
return false;
|
|
1916
|
+
}
|
|
1917
|
+
const controller = new AbortController();
|
|
1918
|
+
const timeoutId = window.setTimeout(() => controller.abort(), TERMINATION_PROBE_TIMEOUT_MS);
|
|
1919
|
+
try {
|
|
1920
|
+
// `no-store` so a prior response can't be served from the HTTP cache
|
|
1921
|
+
// (e.g. a stale 404 returned while offline would look like a
|
|
1922
|
+
// termination); the probe must always reach the live router.
|
|
1923
|
+
const res = await fetch(probeUrl, { method: 'GET', cache: 'no-store', signal: controller.signal });
|
|
1924
|
+
return res.status === 404;
|
|
1925
|
+
} catch {
|
|
1926
|
+
return false;
|
|
1927
|
+
} finally {
|
|
1928
|
+
window.clearTimeout(timeoutId);
|
|
1929
|
+
}
|
|
1930
|
+
};
|
|
1931
|
+
|
|
1932
|
+
const markTerminated = () => {
|
|
1933
|
+
if (terminatedRef.current) return;
|
|
1934
|
+
terminatedRef.current = true;
|
|
1935
|
+
setTerminated(true);
|
|
1936
|
+
setRetryExhausted(false);
|
|
1937
|
+
// Invalidate any in-flight/scheduled attempt and stop all activity so
|
|
1938
|
+
// we don't keep hammering an endpoint that will never come back.
|
|
1939
|
+
connectionGenerationRef.current += 1;
|
|
1940
|
+
clearScheduledRetry();
|
|
1941
|
+
clearConnectionSuccessTimeout();
|
|
1942
|
+
clearIceDisconnectedGrace();
|
|
1943
|
+
teardownConnection();
|
|
1944
|
+
updateStatus('Instance terminated');
|
|
1945
|
+
safeInvoke('onTerminated', onTerminatedRef.current);
|
|
1946
|
+
};
|
|
1947
|
+
|
|
1948
|
+
// Run a one-shot termination check in reaction to a drop. Event-driven
|
|
1949
|
+
// (no polling), deduped while one is already in flight, and a no-op once
|
|
1950
|
+
// we've concluded the instance is gone.
|
|
1951
|
+
const checkTerminated = () => {
|
|
1952
|
+
if (terminatedRef.current || terminationProbeInFlightRef.current) return;
|
|
1953
|
+
terminationProbeInFlightRef.current = true;
|
|
1954
|
+
void probeTerminated().then((isTerminated) => {
|
|
1955
|
+
terminationProbeInFlightRef.current = false;
|
|
1956
|
+
if (isTerminated) {
|
|
1957
|
+
markTerminated();
|
|
1958
|
+
}
|
|
1959
|
+
});
|
|
1960
|
+
};
|
|
1961
|
+
|
|
1488
1962
|
const scheduleRetry = (reason: string, generation: number) => {
|
|
1963
|
+
if (terminatedRef.current) {
|
|
1964
|
+
return;
|
|
1965
|
+
}
|
|
1489
1966
|
if (generation !== connectionGenerationRef.current) {
|
|
1490
1967
|
return;
|
|
1491
1968
|
}
|
|
1492
1969
|
|
|
1970
|
+
// A drop just happened. Find out whether the instance is gone for good
|
|
1971
|
+
// (404) rather than a recoverable blip; if so the probe stops retries
|
|
1972
|
+
// and surfaces the terminated message. Meanwhile the normal retry path
|
|
1973
|
+
// below proceeds and is short-circuited once the verdict lands.
|
|
1974
|
+
checkTerminated();
|
|
1975
|
+
|
|
1493
1976
|
if (controlChannelOpenedRef.current) {
|
|
1494
1977
|
if (!autoReconnectRef.current) {
|
|
1495
1978
|
updateStatus(`Connection failed after it was established: ${reason}`);
|
|
@@ -1525,6 +2008,9 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1525
2008
|
};
|
|
1526
2009
|
|
|
1527
2010
|
const startAttempt = async (attemptNumber = 0) => {
|
|
2011
|
+
// Never reconnect to an instance we've concluded is gone. `start()`
|
|
2012
|
+
// clears this when a deliberate fresh connection is requested.
|
|
2013
|
+
if (terminatedRef.current) return;
|
|
1528
2014
|
const generation = connectionGenerationRef.current + 1;
|
|
1529
2015
|
connectionGenerationRef.current = generation;
|
|
1530
2016
|
connectionAttemptRef.current = attemptNumber;
|
|
@@ -1568,6 +2054,21 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1568
2054
|
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
1569
2055
|
return;
|
|
1570
2056
|
}
|
|
2057
|
+
// Replay the saved camera aspect for fresh connections
|
|
2058
|
+
// (initial mount, autoreconnect, sim reboot). The host's
|
|
2059
|
+
// streamer starts at its boot default (16:9 / 1920×1080)
|
|
2060
|
+
// and a missing message would mean the operator's pick
|
|
2061
|
+
// silently reverts on reconnect. We always send something
|
|
2062
|
+
// when the prop is set, even if it matches the host
|
|
2063
|
+
// default — the host short-circuits no-op rebuilds.
|
|
2064
|
+
const initialAspect = cameraAspectRef.current;
|
|
2065
|
+
if (initialAspect) {
|
|
2066
|
+
try {
|
|
2067
|
+
ws.send(JSON.stringify({ type: 'cameraAspect', aspect: initialAspect }));
|
|
2068
|
+
} catch (err) {
|
|
2069
|
+
debugWarn('initial cameraAspect send failed:', err);
|
|
2070
|
+
}
|
|
2071
|
+
}
|
|
1571
2072
|
settle(resolve);
|
|
1572
2073
|
};
|
|
1573
2074
|
|
|
@@ -1644,6 +2145,23 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1644
2145
|
peerConnection.addTransceiver('audio', { direction: 'recvonly' });
|
|
1645
2146
|
const videoTransceiver = peerConnection.addTransceiver('video', { direction: 'recvonly' });
|
|
1646
2147
|
|
|
2148
|
+
// Pre-allocate a sendonly slot for the user's camera so the
|
|
2149
|
+
// sim can pull frames later without renegotiating SDP.
|
|
2150
|
+
// Allocating up-front means we don't have to renegotiate the
|
|
2151
|
+
// SDP when the simulator app actually opens its
|
|
2152
|
+
// `AVCaptureSession` — the codec/SSRC negotiation happens
|
|
2153
|
+
// once, here, and turning the camera on/off later is just a
|
|
2154
|
+
// `replaceTrack`.
|
|
2155
|
+
// iOS only: Android has no camera consumer, and the extra
|
|
2156
|
+
// m=video line crashes Android 14's scrcpy/libwebrtc.
|
|
2157
|
+
const outboundCameraTransceiver =
|
|
2158
|
+
platform === 'ios' ?
|
|
2159
|
+
peerConnection.addTransceiver('video', {
|
|
2160
|
+
direction: 'sendonly',
|
|
2161
|
+
})
|
|
2162
|
+
: null;
|
|
2163
|
+
outboundCameraSenderRef.current = outboundCameraTransceiver?.sender ?? null;
|
|
2164
|
+
|
|
1647
2165
|
// As hardware encoder, we use H265 for iOS and VP9 for Android.
|
|
1648
2166
|
// We make sure these two are the first ones in the list.
|
|
1649
2167
|
// If not, the fallback is H264 which is also hardware accelerated, although not as good,
|
|
@@ -1669,6 +2187,59 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1669
2187
|
}
|
|
1670
2188
|
}
|
|
1671
2189
|
|
|
2190
|
+
// Pin the outbound camera transceiver's codec order with
|
|
2191
|
+
// negotiation-time fallback. Preference order:
|
|
2192
|
+
// 1. H.265 / HEVC — VideoToolbox HW on M-series + recent
|
|
2193
|
+
// Chrome. ~30% less bitrate at equal quality. If the
|
|
2194
|
+
// answerer (limulator) supports HEVC, this wins.
|
|
2195
|
+
// 2. H.264 with VideoToolbox-friendly profile-level-ids
|
|
2196
|
+
// (42e01f / 42e028 / 640c1f). Universally HW-accelerated
|
|
2197
|
+
// on every Mac since 2009. SDP-time fallback when HEVC
|
|
2198
|
+
// isn't in the answer.
|
|
2199
|
+
// 3. Any other H.264 profile. Often resolves to OpenH264
|
|
2200
|
+
// (software) so we keep it explicitly last among H.264
|
|
2201
|
+
// to avoid Chrome's default pick.
|
|
2202
|
+
// 4. VP9 / VP8 (libvpx software on Mac). Last resort.
|
|
2203
|
+
//
|
|
2204
|
+
// Caveat: this is *negotiation* fallback. If HEVC HW encoder
|
|
2205
|
+
// init fails at session start or gets demoted mid-stream,
|
|
2206
|
+
// Chrome falls back to *software* HEVC, not H.264 — no spec
|
|
2207
|
+
// hook for runtime re-negotiation. The host-side stats
|
|
2208
|
+
// logger will surface this as decoder=ffmpeg / hw=false.
|
|
2209
|
+
if (outboundCameraTransceiver && RTCRtpSender.getCapabilities) {
|
|
2210
|
+
const senderCaps = RTCRtpSender.getCapabilities('video');
|
|
2211
|
+
if (senderCaps && senderCaps.codecs) {
|
|
2212
|
+
const vtH264Profiles = new Set(['42e01f', '42e028', '640c1f']);
|
|
2213
|
+
const score = (c: { mimeType: string; sdpFmtpLine?: string }): number => {
|
|
2214
|
+
const mime = c.mimeType.toLowerCase();
|
|
2215
|
+
const fmtp = (c.sdpFmtpLine || '').toLowerCase();
|
|
2216
|
+
const profileMatch = fmtp.match(/profile-level-id=([0-9a-f]{6})/);
|
|
2217
|
+
const profile = profileMatch ? profileMatch[1] : '';
|
|
2218
|
+
if (mime === 'video/h265' || mime === 'video/hevc') return 1;
|
|
2219
|
+
if (mime === 'video/h264' && vtH264Profiles.has(profile)) return 2;
|
|
2220
|
+
if (mime === 'video/h264') return 3;
|
|
2221
|
+
if (mime === 'video/vp9') return 4;
|
|
2222
|
+
if (mime === 'video/vp8') return 5;
|
|
2223
|
+
if (mime === 'video/rtx' || mime === 'video/red' || mime === 'video/ulpfec') {
|
|
2224
|
+
return 0; // Keep RTX/FEC available alongside everything else.
|
|
2225
|
+
}
|
|
2226
|
+
return 6;
|
|
2227
|
+
};
|
|
2228
|
+
const sortedSendCodecs = [...senderCaps.codecs].sort((a, b) => score(a) - score(b));
|
|
2229
|
+
try {
|
|
2230
|
+
outboundCameraTransceiver.setCodecPreferences(sortedSendCodecs);
|
|
2231
|
+
debugLog(
|
|
2232
|
+
'Outbound camera codec preferences:',
|
|
2233
|
+
sortedSendCodecs
|
|
2234
|
+
.map((c) => `${c.mimeType}${c.sdpFmtpLine ? `[${c.sdpFmtpLine}]` : ''}`)
|
|
2235
|
+
.join(', '),
|
|
2236
|
+
);
|
|
2237
|
+
} catch (err) {
|
|
2238
|
+
debugWarn('Failed to set outbound camera codec preferences:', err);
|
|
2239
|
+
}
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
|
|
1672
2243
|
const dataChannel = peerConnection.createDataChannel('control', {
|
|
1673
2244
|
ordered: true,
|
|
1674
2245
|
negotiated: true,
|
|
@@ -1972,6 +2543,243 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1972
2543
|
pendingScreenshotResolversRef.current.delete(message.id);
|
|
1973
2544
|
pendingScreenshotRejectersRef.current.delete(message.id);
|
|
1974
2545
|
break;
|
|
2546
|
+
case 'cameraRequest': {
|
|
2547
|
+
// Only iOS sessions allocate the sendonly outbound-camera
|
|
2548
|
+
// transceiver; without it there is nothing to feed the sim, so
|
|
2549
|
+
// ignore camera requests entirely — no getUserMedia prompt and
|
|
2550
|
+
// no onCameraDemandChange callbacks. Android intentionally stays
|
|
2551
|
+
// silent, as the API docs promise.
|
|
2552
|
+
if (platform !== 'ios' || !outboundCameraSenderRef.current) {
|
|
2553
|
+
break;
|
|
2554
|
+
}
|
|
2555
|
+
const active = message.active === true;
|
|
2556
|
+
// Bump up front so any earlier in-flight handler bails.
|
|
2557
|
+
const cameraGeneration = ++cameraRequestGenerationRef.current;
|
|
2558
|
+
const isCurrentCameraRequest = () =>
|
|
2559
|
+
isCurrentAttempt() &&
|
|
2560
|
+
wsRef.current === ws &&
|
|
2561
|
+
cameraGeneration === cameraRequestGenerationRef.current;
|
|
2562
|
+
// Log unconditionally — this is one of the few places we
|
|
2563
|
+
// hand control to the browser's permission UI, and a
|
|
2564
|
+
// silent failure here looks identical to "limulator
|
|
2565
|
+
// never asked" from the user's point of view.
|
|
2566
|
+
// eslint-disable-next-line no-console
|
|
2567
|
+
console.info('[RemoteControl] cameraRequest received, active=', active);
|
|
2568
|
+
if (!active) {
|
|
2569
|
+
// Sim no longer wants frames. Drop our local track and
|
|
2570
|
+
// shut the browser's camera green light off.
|
|
2571
|
+
const sender = outboundCameraSenderRef.current;
|
|
2572
|
+
if (sender) {
|
|
2573
|
+
try {
|
|
2574
|
+
await sender.replaceTrack(null);
|
|
2575
|
+
} catch (err) {
|
|
2576
|
+
debugWarn('replaceTrack(null) on camera detach failed:', err);
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
stopCameraStatsPoller();
|
|
2580
|
+
stopOutboundLocalStream();
|
|
2581
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
|
|
2582
|
+
break;
|
|
2583
|
+
}
|
|
2584
|
+
// Sim is asking for camera. Ask the browser; the user's
|
|
2585
|
+
// prompt response is reported back to the host via a
|
|
2586
|
+
// `cameraResult` message so it can swap to a
|
|
2587
|
+
// black-frame fallback on denial.
|
|
2588
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
|
|
2589
|
+
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
2590
|
+
// Likely an insecure context (http on a non-localhost
|
|
2591
|
+
// origin) — Chrome strips `mediaDevices` off
|
|
2592
|
+
// `navigator` in that case and the only signal is
|
|
2593
|
+
// this undefined check.
|
|
2594
|
+
// eslint-disable-next-line no-console
|
|
2595
|
+
console.warn(
|
|
2596
|
+
'[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
|
|
2597
|
+
'getUserMedia requires a secure context (https or http://localhost). ' +
|
|
2598
|
+
'Replying cameraResult granted=false so limulator falls back to black frames.',
|
|
2599
|
+
);
|
|
2600
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2601
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2602
|
+
break;
|
|
2603
|
+
}
|
|
2604
|
+
let stream: MediaStream | null = null;
|
|
2605
|
+
try {
|
|
2606
|
+
// Capture at the webcam's *native* resolution by
|
|
2607
|
+
// default (no width/height constraints). When the
|
|
2608
|
+
// host app has picked an explicit cap via
|
|
2609
|
+
// `cameraResolutionCap`, we honour it here. Frame
|
|
2610
|
+
// rate is always capped to 30 to match the
|
|
2611
|
+
// simulator pool's pacing.
|
|
2612
|
+
stream = await navigator.mediaDevices.getUserMedia({
|
|
2613
|
+
video: cameraCapToConstraints(cameraResolutionCapRef.current),
|
|
2614
|
+
audio: false,
|
|
2615
|
+
});
|
|
2616
|
+
} catch (err) {
|
|
2617
|
+
// Surface unconditionally — the user is the only one
|
|
2618
|
+
// who can fix this (permission denied, no device,
|
|
2619
|
+
// dismissed prompt, etc.). `debugWarn` alone would
|
|
2620
|
+
// hide it in normal builds.
|
|
2621
|
+
// eslint-disable-next-line no-console
|
|
2622
|
+
console.warn('[RemoteControl] getUserMedia denied/failed:', err);
|
|
2623
|
+
}
|
|
2624
|
+
if (!isCurrentCameraRequest()) {
|
|
2625
|
+
// Superseded during the prompt; drop the stream we got.
|
|
2626
|
+
if (stream) stopMediaStream(stream);
|
|
2627
|
+
return;
|
|
2628
|
+
}
|
|
2629
|
+
if (!stream) {
|
|
2630
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2631
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2632
|
+
break;
|
|
2633
|
+
}
|
|
2634
|
+
// Replace any previous local stream (e.g. an earlier
|
|
2635
|
+
// cameraRequest that resolved with a different device)
|
|
2636
|
+
// before we install the new tracks.
|
|
2637
|
+
stopOutboundLocalStream();
|
|
2638
|
+
outboundLocalStreamRef.current = stream;
|
|
2639
|
+
const sender = outboundCameraSenderRef.current;
|
|
2640
|
+
const videoTrack = stream.getVideoTracks()[0] ?? null;
|
|
2641
|
+
if (!sender || !videoTrack) {
|
|
2642
|
+
if (stream) stopMediaStream(stream);
|
|
2643
|
+
outboundLocalStreamRef.current = null;
|
|
2644
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2645
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2646
|
+
break;
|
|
2647
|
+
}
|
|
2648
|
+
try {
|
|
2649
|
+
await sender.replaceTrack(videoTrack);
|
|
2650
|
+
} catch (err) {
|
|
2651
|
+
debugWarn('replaceTrack(videoTrack) failed:', err);
|
|
2652
|
+
stopMediaStream(stream);
|
|
2653
|
+
outboundLocalStreamRef.current = null;
|
|
2654
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2655
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2656
|
+
break;
|
|
2657
|
+
}
|
|
2658
|
+
if (!isCurrentCameraRequest()) {
|
|
2659
|
+
// Superseded mid-attach; detach and skip the ACK/poller.
|
|
2660
|
+
try {
|
|
2661
|
+
await sender.replaceTrack(null);
|
|
2662
|
+
} catch (err) {
|
|
2663
|
+
debugWarn('replaceTrack(null) on stale camera attach failed:', err);
|
|
2664
|
+
}
|
|
2665
|
+
stopMediaStream(stream);
|
|
2666
|
+
if (outboundLocalStreamRef.current === stream) {
|
|
2667
|
+
outboundLocalStreamRef.current = null;
|
|
2668
|
+
}
|
|
2669
|
+
return;
|
|
2670
|
+
}
|
|
2671
|
+
// Tell the encoder this is real motion content (a
|
|
2672
|
+
// physical camera), not text/slides. With `'motion'`
|
|
2673
|
+
// VideoToolbox / libvpx pick latency-friendly tuning
|
|
2674
|
+
// (no extra B-frames, shorter GoP smoothing). Set
|
|
2675
|
+
// before the first frame so the encoder init reads it.
|
|
2676
|
+
try {
|
|
2677
|
+
videoTrack.contentHint = 'motion';
|
|
2678
|
+
} catch {
|
|
2679
|
+
/* older browsers don't support contentHint; ignore */
|
|
2680
|
+
}
|
|
2681
|
+
// Bound the outbound bitrate generously and let
|
|
2682
|
+
// WebRTC's congestion control (BWE) find the floor.
|
|
2683
|
+
// 8 Mbps is comfortable for native 1080p30 webcam
|
|
2684
|
+
// content over LAN — the encoder will use far less when
|
|
2685
|
+
// the scene is static, and BWE will throttle if a
|
|
2686
|
+
// hop is constrained. Pair with
|
|
2687
|
+
// `maintain-framerate` so the quality scaler steps
|
|
2688
|
+
// down resolution before dropping frames, matching how
|
|
2689
|
+
// Meet/Zoom degrade.
|
|
2690
|
+
try {
|
|
2691
|
+
const params = sender.getParameters();
|
|
2692
|
+
if (!params.encodings || params.encodings.length === 0) {
|
|
2693
|
+
params.encodings = [{}];
|
|
2694
|
+
}
|
|
2695
|
+
params.encodings[0].maxBitrate = 8_000_000;
|
|
2696
|
+
params.encodings[0].maxFramerate = 30;
|
|
2697
|
+
params.degradationPreference = 'maintain-framerate';
|
|
2698
|
+
await sender.setParameters(params);
|
|
2699
|
+
} catch (err) {
|
|
2700
|
+
debugWarn('setParameters on outbound camera failed:', err);
|
|
2701
|
+
}
|
|
2702
|
+
// Surface the actual captured geometry to the host so
|
|
2703
|
+
// it can size its IOSurface pool / picture-format
|
|
2704
|
+
// metadata to match. `getSettings()` returns what the
|
|
2705
|
+
// browser actually picked — which may differ from any
|
|
2706
|
+
// hints we sent in the constraints — including the
|
|
2707
|
+
// selected `deviceId` / `label` (useful when a user has
|
|
2708
|
+
// multiple cameras and we eventually expose a picker).
|
|
2709
|
+
let cameraMetadata: {
|
|
2710
|
+
width?: number;
|
|
2711
|
+
height?: number;
|
|
2712
|
+
frameRate?: number;
|
|
2713
|
+
deviceId?: string;
|
|
2714
|
+
label?: string;
|
|
2715
|
+
facingMode?: string;
|
|
2716
|
+
} = {};
|
|
2717
|
+
try {
|
|
2718
|
+
const settings = videoTrack.getSettings();
|
|
2719
|
+
cameraMetadata = {
|
|
2720
|
+
width: settings.width,
|
|
2721
|
+
height: settings.height,
|
|
2722
|
+
frameRate: settings.frameRate,
|
|
2723
|
+
deviceId: settings.deviceId,
|
|
2724
|
+
label: videoTrack.label || undefined,
|
|
2725
|
+
facingMode: settings.facingMode,
|
|
2726
|
+
};
|
|
2727
|
+
// eslint-disable-next-line no-console
|
|
2728
|
+
console.info(
|
|
2729
|
+
`[RemoteControl] camera capture: ${cameraMetadata.width}x${cameraMetadata.height}` +
|
|
2730
|
+
` @ ${cameraMetadata.frameRate ?? '?'}fps` +
|
|
2731
|
+
(cameraMetadata.label ? ` — ${cameraMetadata.label}` : ''),
|
|
2732
|
+
);
|
|
2733
|
+
} catch (err) {
|
|
2734
|
+
debugWarn('getSettings() on outbound camera track failed:', err);
|
|
2735
|
+
}
|
|
2736
|
+
// If the browser revokes the track later (extension,
|
|
2737
|
+
// user clicks Stop in the camera tab UI, etc.), notify
|
|
2738
|
+
// the host so it can switch to black frames.
|
|
2739
|
+
videoTrack.onended = () => {
|
|
2740
|
+
if (outboundLocalStreamRef.current !== stream) return;
|
|
2741
|
+
stopCameraStatsPoller();
|
|
2742
|
+
stopOutboundLocalStream();
|
|
2743
|
+
if (wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
|
|
2744
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2745
|
+
}
|
|
2746
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2747
|
+
};
|
|
2748
|
+
if (!isCurrentCameraRequest()) {
|
|
2749
|
+
// Superseded during setParameters; detach and bail.
|
|
2750
|
+
videoTrack.onended = null;
|
|
2751
|
+
try {
|
|
2752
|
+
await sender.replaceTrack(null);
|
|
2753
|
+
} catch (err) {
|
|
2754
|
+
debugWarn('replaceTrack(null) on stale camera finalize failed:', err);
|
|
2755
|
+
}
|
|
2756
|
+
stopMediaStream(stream);
|
|
2757
|
+
if (outboundLocalStreamRef.current === stream) {
|
|
2758
|
+
outboundLocalStreamRef.current = null;
|
|
2759
|
+
}
|
|
2760
|
+
return;
|
|
2761
|
+
}
|
|
2762
|
+
ws.send(
|
|
2763
|
+
JSON.stringify({
|
|
2764
|
+
type: 'cameraResult',
|
|
2765
|
+
granted: true,
|
|
2766
|
+
// Forward what the browser actually captured so the
|
|
2767
|
+
// host can size its IOSurface pool, log the device,
|
|
2768
|
+
// and (eventually) surface a status pill / picker.
|
|
2769
|
+
camera: cameraMetadata,
|
|
2770
|
+
}),
|
|
2771
|
+
);
|
|
2772
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, cameraMetadata);
|
|
2773
|
+
// Kick off the per-second outbound stats sampler. We do
|
|
2774
|
+
// this *after* `setParameters` so the first sample
|
|
2775
|
+
// already sees the encoder under its final bitrate /
|
|
2776
|
+
// degradation policy, and *after* the host-side
|
|
2777
|
+
// attachInboundTrack will have wired up (the
|
|
2778
|
+
// cameraResult ACK is what triggers it on the host),
|
|
2779
|
+
// so framesEncoded starts climbing immediately.
|
|
2780
|
+
startCameraStatsPoller();
|
|
2781
|
+
break;
|
|
2782
|
+
}
|
|
1975
2783
|
case 'terminateAppResult':
|
|
1976
2784
|
if (typeof message.id !== 'string') {
|
|
1977
2785
|
debugWarn('Received invalid terminateApp result message:', message);
|
|
@@ -2041,6 +2849,10 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2041
2849
|
};
|
|
2042
2850
|
|
|
2043
2851
|
const start = () => {
|
|
2852
|
+
// A fresh connection (mount, prop change, manual retry, reconnect())
|
|
2853
|
+
// clears any prior terminated verdict so we re-evaluate from scratch.
|
|
2854
|
+
terminatedRef.current = false;
|
|
2855
|
+
setTerminated(false);
|
|
2044
2856
|
void startAttempt(0);
|
|
2045
2857
|
};
|
|
2046
2858
|
|
|
@@ -2059,6 +2871,41 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2059
2871
|
start();
|
|
2060
2872
|
};
|
|
2061
2873
|
|
|
2874
|
+
// Re-apply the resolution cap on the currently-sending track
|
|
2875
|
+
// whenever the host app changes its preference. Skips when no
|
|
2876
|
+
// camera is active — the next `getUserMedia` will pick up the
|
|
2877
|
+
// new value via `cameraCapToConstraints` automatically.
|
|
2878
|
+
useEffect(() => {
|
|
2879
|
+
const stream = outboundLocalStreamRef.current;
|
|
2880
|
+
if (!stream) return;
|
|
2881
|
+
const track = stream.getVideoTracks()[0];
|
|
2882
|
+
if (!track) return;
|
|
2883
|
+
const constraints = cameraCapToConstraints(cameraResolutionCap);
|
|
2884
|
+
track.applyConstraints(constraints).catch((err) => {
|
|
2885
|
+
debugWarn('applyConstraints for new camera cap failed:', err);
|
|
2886
|
+
});
|
|
2887
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
2888
|
+
}, [cameraResolutionCap]);
|
|
2889
|
+
|
|
2890
|
+
// Push the aspect preference to the host whenever it changes.
|
|
2891
|
+
// The host rebuilds its IOSurface pool and bumps `pool_generation`
|
|
2892
|
+
// on its end; the dylib re-handshakes on next sem_wait. No
|
|
2893
|
+
// peer-connection renegotiation is needed — the aspect change is
|
|
2894
|
+
// purely about the pixel buffer dimensions iOS apps observe, not
|
|
2895
|
+
// about WebRTC track layout. The `cameraAspectRef` is mirrored
|
|
2896
|
+
// higher up so the WS `onopen` reconnect path can replay the
|
|
2897
|
+
// latest value on fresh connections.
|
|
2898
|
+
useEffect(() => {
|
|
2899
|
+
if (!cameraAspect) return;
|
|
2900
|
+
const ws = wsRef.current;
|
|
2901
|
+
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2902
|
+
try {
|
|
2903
|
+
ws.send(JSON.stringify({ type: 'cameraAspect', aspect: cameraAspect }));
|
|
2904
|
+
} catch (err) {
|
|
2905
|
+
debugWarn('cameraAspect send failed:', err);
|
|
2906
|
+
}
|
|
2907
|
+
}, [cameraAspect]);
|
|
2908
|
+
|
|
2062
2909
|
useEffect(() => {
|
|
2063
2910
|
// Reset video loaded state when connection params change
|
|
2064
2911
|
setVideoLoaded(false);
|
|
@@ -2080,6 +2927,11 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2080
2927
|
stop();
|
|
2081
2928
|
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
|
2082
2929
|
};
|
|
2930
|
+
// Camera attach/detach happens entirely inside the WS message
|
|
2931
|
+
// loop now (sendonly transceiver + `replaceTrack`), so the
|
|
2932
|
+
// connection effect doesn't need to bounce when the camera
|
|
2933
|
+
// turns on/off — no SDP-affecting change.
|
|
2934
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
2083
2935
|
}, [url, token, propSessionId]);
|
|
2084
2936
|
|
|
2085
2937
|
// Recompute the inspect-overlay geometry (container-local pixel rect of
|
|
@@ -2567,11 +3419,19 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2567
3419
|
}}
|
|
2568
3420
|
/>
|
|
2569
3421
|
)}
|
|
2570
|
-
{
|
|
2571
|
-
<
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
3422
|
+
{terminated ?
|
|
3423
|
+
<div className="rc-terminated" role="status">
|
|
3424
|
+
<div className="rc-terminated-title">Instance terminated</div>
|
|
3425
|
+
<div className="rc-terminated-message">
|
|
3426
|
+
This instance no longer exists and can't be reconnected.
|
|
3427
|
+
</div>
|
|
3428
|
+
</div>
|
|
3429
|
+
: retryExhausted && (
|
|
3430
|
+
<button type="button" className="rc-retry-button" onClick={handleManualRetry}>
|
|
3431
|
+
Retry
|
|
3432
|
+
</button>
|
|
3433
|
+
)
|
|
3434
|
+
}
|
|
2575
3435
|
</div>
|
|
2576
3436
|
);
|
|
2577
3437
|
},
|