@limrun/ui 0.14.0-rc.2 → 0.14.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app-store-relay/index.js +3 -3
- package/dist/app-store-relay/react.d.ts +1 -1
- package/dist/app-store-relay/react.js +6 -2
- package/dist/{client-CBX-hLnL.mjs → client-C8ZDEjtD.mjs} +1 -6
- package/dist/components/remote-control.d.ts +0 -156
- package/dist/core/device-install/operations/operations.d.ts +1 -1
- package/dist/core/play-publish/google.d.ts +13 -0
- package/dist/core/play-publish/publish.d.ts +34 -0
- package/dist/device-build/index.js +5 -5
- package/dist/device-install/index.js +7 -1
- package/dist/device-install/react.js +24 -21
- package/dist/index.cjs +1 -1
- package/dist/index.css +1 -1
- package/dist/index.js +1125 -1405
- package/dist/play-publish/index.cjs +1 -0
- package/dist/play-publish/index.d.ts +2 -0
- package/dist/play-publish/index.js +106 -0
- package/dist/play-publish/react.cjs +1 -0
- package/dist/play-publish/react.d.ts +21 -0
- package/dist/play-publish/react.js +81 -0
- package/dist/{provisioning-DKbkBO-p.mjs → provisioning-BmsEnF6X.mjs} +8 -8
- package/package.json +13 -2
- package/src/app-store-relay/index.test.ts +6 -6
- package/src/app-store-relay/react.ts +5 -1
- package/src/components/remote-control.css +0 -32
- package/src/components/remote-control.tsx +5 -865
- package/src/core/device-install/apple/client.ts +1 -6
- package/src/core/device-install/apple/relay.ts +1 -4
- package/src/core/device-install/operations/operations.ts +7 -1
- package/src/core/play-publish/google.ts +104 -0
- package/src/core/play-publish/publish.test.ts +68 -0
- package/src/core/play-publish/publish.ts +89 -0
- package/src/device-install/demo/demo.tsx +1 -4
- package/src/device-install/react.ts +30 -27
- package/src/play-publish/README.md +65 -0
- package/src/play-publish/index.ts +12 -0
- package/src/play-publish/react.ts +168 -0
- package/vite.config.ts +2 -0
|
@@ -59,19 +59,6 @@ interface RemoteControlProps {
|
|
|
59
59
|
// surfacing the manual "Retry" button. Defaults to false.
|
|
60
60
|
autoReconnect?: boolean;
|
|
61
61
|
|
|
62
|
-
/**
|
|
63
|
-
* Fires once when the component concludes the instance is permanently
|
|
64
|
-
* gone rather than suffering a recoverable connection blip.
|
|
65
|
-
*
|
|
66
|
-
* After a drop, the component probes the instance endpoint over HTTP;
|
|
67
|
-
* a 404 means the instance no longer exists (terminated/deleted). When
|
|
68
|
-
* that happens the component stops reconnecting and renders a built-in
|
|
69
|
-
* "terminated" message in place of the "Retry" button, and invokes this
|
|
70
|
-
* callback so the host can react (navigate away, surface its own UI,
|
|
71
|
-
* etc.). It is not called for ordinary network drops.
|
|
72
|
-
*/
|
|
73
|
-
onTerminated?: () => void;
|
|
74
|
-
|
|
75
62
|
/**
|
|
76
63
|
* Enable the inspect overlay. When set, the component starts polling the
|
|
77
64
|
* accessibility tree and draws boxes over each element on top of the
|
|
@@ -153,157 +140,6 @@ interface RemoteControlProps {
|
|
|
153
140
|
* @default 2000
|
|
154
141
|
*/
|
|
155
142
|
axMaxBackoffMs?: number;
|
|
156
|
-
|
|
157
|
-
/**
|
|
158
|
-
* Fires whenever the iOS simulator's camera demand state changes —
|
|
159
|
-
* i.e. an app inside the sim called
|
|
160
|
-
* `[AVCaptureSession startRunning]` or `[stopRunning]`. The
|
|
161
|
-
* component handles the `navigator.mediaDevices.getUserMedia` prompt
|
|
162
|
-
* and SDP plumbing internally; this callback is purely so the host
|
|
163
|
-
* UI can render a status indicator ("simulator is using your
|
|
164
|
-
* camera", etc.).
|
|
165
|
-
*
|
|
166
|
-
* `active` reflects whether the sim is currently asking for
|
|
167
|
-
* frames. `granted` is set only on the call that follows a
|
|
168
|
-
* `getUserMedia` attempt: `true` when the user accepted the
|
|
169
|
-
* browser prompt, `false` when they denied or the call failed
|
|
170
|
-
* (in which case the limulator side switches to a black-frame
|
|
171
|
-
* fallback so the app keeps ticking).
|
|
172
|
-
*
|
|
173
|
-
* `camera` (optional, only meaningful when `granted === true`)
|
|
174
|
-
* carries what `MediaStreamTrack.getSettings()` reported for the
|
|
175
|
-
* active capture: resolution, framerate, device label, facing
|
|
176
|
-
* mode. Use it to render a richer status indicator (e.g.
|
|
177
|
-
* "Camera · 1920×1080 · 30 fps · FaceTime HD").
|
|
178
|
-
*
|
|
179
|
-
* Only iOS instances ever fire this callback; Android instances
|
|
180
|
-
* have no camera-injector path and stay silent.
|
|
181
|
-
*/
|
|
182
|
-
onCameraDemandChange?: (active: boolean, granted?: boolean, camera?: CameraCaptureInfo) => void;
|
|
183
|
-
|
|
184
|
-
/**
|
|
185
|
-
* Periodically (~1Hz) fires with a snapshot of the outbound
|
|
186
|
-
* camera stream's live WebRTC stats — codec, encoder
|
|
187
|
-
* implementation, hardware acceleration, encoded fps, bitrate,
|
|
188
|
-
* round-trip-time, and the encoder's
|
|
189
|
-
* `qualityLimitationReason` (which is what Meet/Zoom use to
|
|
190
|
-
* decide whether to show "Bandwidth limited" or "CPU limited"
|
|
191
|
-
* banners).
|
|
192
|
-
*
|
|
193
|
-
* Only fires while the simulator is actively pulling the
|
|
194
|
-
* camera AND `getUserMedia` was granted. `null` is emitted
|
|
195
|
-
* once when the stream goes back to idle so consumers can
|
|
196
|
-
* clear their UI without having to maintain their own
|
|
197
|
-
* timeout. The host app does not need to poll `getStats()`
|
|
198
|
-
* itself — this is the canonical place.
|
|
199
|
-
*/
|
|
200
|
-
onCameraStats?: (stats: CameraStreamStats | null) => void;
|
|
201
|
-
|
|
202
|
-
/**
|
|
203
|
-
* Optional resolution cap for outbound camera capture.
|
|
204
|
-
*
|
|
205
|
-
* - `'auto'` (default): no extra constraint. The browser captures
|
|
206
|
-
* at its webcam's native max; WebRTC's quality scaler may step
|
|
207
|
-
* down resolution on the encoder side under bandwidth pressure
|
|
208
|
-
* while still feeding the simulator at the pool's native size.
|
|
209
|
-
* - `'1080p'` / `'720p'` / `'480p'`: hard cap applied via
|
|
210
|
-
* `getUserMedia` constraints (for new captures) and
|
|
211
|
-
* `track.applyConstraints` (for the currently-active track),
|
|
212
|
-
* matching the way Meet/Zoom expose a "Send resolution" picker.
|
|
213
|
-
*
|
|
214
|
-
* Bumping or lowering the cap mid-stream is supported; the
|
|
215
|
-
* change takes effect within a frame or two as the webcam
|
|
216
|
-
* re-negotiates.
|
|
217
|
-
*/
|
|
218
|
-
cameraResolutionCap?: CameraResolutionCap;
|
|
219
|
-
/**
|
|
220
|
-
* Aspect ratio the simulator's virtual camera should report to apps.
|
|
221
|
-
* Picking a value here triggers a `cameraAspect` WS message to the
|
|
222
|
-
* host, which rebuilds its IOSurface ring at the matching dimensions
|
|
223
|
-
* (16:9 → 1920×1080, 4:3 → 1440×1080, 1:1 → 1080×1080, 9:16 →
|
|
224
|
-
* 1080×1920) and signals the in-sim dylib to re-handshake. iOS apps
|
|
225
|
-
* see CMSampleBuffers at the new dimensions within a frame or two.
|
|
226
|
-
*
|
|
227
|
-
* The browser still captures whatever the webcam offers; the host
|
|
228
|
-
* aspect-fills (cover, center-crop) into the new pool. Switching
|
|
229
|
-
* aspect at runtime is intentionally cheap so users can A/B preview
|
|
230
|
-
* styles without restarting the simulator.
|
|
231
|
-
*
|
|
232
|
-
* `undefined` leaves the pool untouched (the host's boot default —
|
|
233
|
-
* 16:9 / 1920×1080 — applies).
|
|
234
|
-
*/
|
|
235
|
-
cameraAspect?: CameraAspect;
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
/**
|
|
239
|
-
* Resolution caps a host app can request on the outbound camera.
|
|
240
|
-
* `'auto'` is "let the browser decide" (no constraints beyond the
|
|
241
|
-
* 30 fps ceiling); the other options clamp width/height to the
|
|
242
|
-
* named target. Aspect ratio is preserved.
|
|
243
|
-
*/
|
|
244
|
-
export type CameraResolutionCap = 'auto' | '1080p' | '720p' | '480p';
|
|
245
|
-
|
|
246
|
-
/**
|
|
247
|
-
* Aspect ratios exposed to the operator for the simulated camera.
|
|
248
|
-
* The host maps each label to concrete IOSurface dimensions; values
|
|
249
|
-
* the host doesn't recognise are silently ignored.
|
|
250
|
-
*/
|
|
251
|
-
export type CameraAspect = '16:9' | '4:3' | '1:1' | '9:16';
|
|
252
|
-
|
|
253
|
-
/**
|
|
254
|
-
* Snapshot of the browser's webcam capture, mirrored from
|
|
255
|
-
* `MediaStreamTrack.getSettings()`. Forwarded to the host alongside
|
|
256
|
-
* `cameraResult` and exposed to the host app via
|
|
257
|
-
* `onCameraDemandChange` so it can render a status indicator without
|
|
258
|
-
* having to call `getStats()` itself.
|
|
259
|
-
*/
|
|
260
|
-
export interface CameraCaptureInfo {
|
|
261
|
-
width?: number;
|
|
262
|
-
height?: number;
|
|
263
|
-
frameRate?: number;
|
|
264
|
-
deviceId?: string;
|
|
265
|
-
label?: string;
|
|
266
|
-
facingMode?: string;
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
/**
|
|
270
|
-
* Live outbound-camera quality snapshot. Derived from
|
|
271
|
-
* `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
|
|
272
|
-
* once per second while the camera is sending. All fields optional:
|
|
273
|
-
* some browsers omit fields (Safari rarely reports
|
|
274
|
-
* `encoderImplementation`), and the first sample after camera start
|
|
275
|
-
* has no delta-derived numbers yet.
|
|
276
|
-
*/
|
|
277
|
-
export interface CameraStreamStats {
|
|
278
|
-
/** "H264", "HEVC"/"H265", "VP9", "VP8", "AV1", etc. (uppercased). */
|
|
279
|
-
codec?: string;
|
|
280
|
-
/** e.g. "VideoToolbox" (hw), "OpenH264" (sw), "ExternalEncoder". */
|
|
281
|
-
encoderImplementation?: string;
|
|
282
|
-
/**
|
|
283
|
-
* Browser-reported hardware-acceleration hint. Some Chromium
|
|
284
|
-
* versions expose this via `powerEfficientEncoder`; we mirror it
|
|
285
|
-
* here so consumers don't have to know the spec quirk.
|
|
286
|
-
*/
|
|
287
|
-
hardwareAccelerated?: boolean;
|
|
288
|
-
/** Outbound encoded fps over the last sample window. */
|
|
289
|
-
framesPerSecond?: number;
|
|
290
|
-
/** Cumulative encoded frame count. */
|
|
291
|
-
framesEncoded?: number;
|
|
292
|
-
/** Outbound encoded bitrate (bits/s) over the last window. */
|
|
293
|
-
bitrateBps?: number;
|
|
294
|
-
/** Width/height the encoder is currently producing. */
|
|
295
|
-
width?: number;
|
|
296
|
-
height?: number;
|
|
297
|
-
/**
|
|
298
|
-
* One of `'none' | 'cpu' | 'bandwidth' | 'other'`. Mirrors
|
|
299
|
-
* Meet/Zoom's "limited by …" banners. Anything other than
|
|
300
|
-
* `'none'` means the encoder dropped resolution to keep up.
|
|
301
|
-
*/
|
|
302
|
-
qualityLimitationReason?: string;
|
|
303
|
-
/** Round-trip time in milliseconds, from the matching RTCP. */
|
|
304
|
-
rttMs?: number;
|
|
305
|
-
/** Packet-loss percentage over the last sample window (0..100). */
|
|
306
|
-
packetsLostPct?: number;
|
|
307
143
|
}
|
|
308
144
|
|
|
309
145
|
interface ScreenshotData {
|
|
@@ -418,9 +254,6 @@ const MAX_CONNECTION_ATTEMPTS = 3;
|
|
|
418
254
|
const CONNECTION_RETRY_DELAY_MS = 1000;
|
|
419
255
|
const CONNECTION_SUCCESS_TIMEOUT_MS = 15000;
|
|
420
256
|
const ICE_DISCONNECTED_GRACE_MS = 3000;
|
|
421
|
-
// Upper bound for the one-shot HTTP probe that disambiguates a terminated
|
|
422
|
-
// instance (endpoint 404s) from a transient connection drop.
|
|
423
|
-
const TERMINATION_PROBE_TIMEOUT_MS = 4000;
|
|
424
257
|
|
|
425
258
|
const isAndroidTabletVideo = (width: number, height: number): boolean =>
|
|
426
259
|
(width === ANDROID_TABLET_VIDEO_WIDTH && height === ANDROID_TABLET_VIDEO_HEIGHT) ||
|
|
@@ -499,17 +332,12 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
499
332
|
openUrl,
|
|
500
333
|
showFrame = true,
|
|
501
334
|
autoReconnect = false,
|
|
502
|
-
onTerminated,
|
|
503
335
|
inspectMode,
|
|
504
336
|
onAxSnapshotChange,
|
|
505
337
|
onInspectSelectionChange,
|
|
506
338
|
onAxStatusChange,
|
|
507
339
|
axPollIntervalMs,
|
|
508
340
|
axMaxBackoffMs,
|
|
509
|
-
onCameraDemandChange,
|
|
510
|
-
onCameraStats,
|
|
511
|
-
cameraResolutionCap = 'auto',
|
|
512
|
-
cameraAspect,
|
|
513
341
|
}: RemoteControlProps,
|
|
514
342
|
ref,
|
|
515
343
|
) => {
|
|
@@ -518,13 +346,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
518
346
|
const frameRef = useRef<HTMLImageElement>(null);
|
|
519
347
|
const [videoLoaded, setVideoLoaded] = useState(false);
|
|
520
348
|
const [retryExhausted, setRetryExhausted] = useState(false);
|
|
521
|
-
// Set once we've concluded the instance is permanently gone (its
|
|
522
|
-
// endpoint 404s after a drop), as opposed to a recoverable blip. When
|
|
523
|
-
// set, reconnection stops and a terminated message replaces "Retry".
|
|
524
|
-
const [terminated, setTerminated] = useState(false);
|
|
525
|
-
const terminatedRef = useRef(false);
|
|
526
|
-
// Guards against firing overlapping termination probes.
|
|
527
|
-
const terminationProbeInFlightRef = useRef(false);
|
|
528
349
|
const [isLandscape, setIsLandscape] = useState(false);
|
|
529
350
|
const [useAndroidTabletFrame, setUseAndroidTabletFrame] = useState(false);
|
|
530
351
|
const [videoStyle, setVideoStyle] = useState<React.CSSProperties>({});
|
|
@@ -542,58 +363,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
542
363
|
// Mirrored to a ref so stale closures in event handlers see the latest value.
|
|
543
364
|
const autoReconnectRef = useRef(autoReconnect);
|
|
544
365
|
autoReconnectRef.current = autoReconnect;
|
|
545
|
-
// Demand-driven outbound camera state.
|
|
546
|
-
//
|
|
547
|
-
// The limulator side broadcasts a `cameraRequest` WS message whenever
|
|
548
|
-
// an app inside the simulator opens/closes an `AVCaptureSession`.
|
|
549
|
-
// We respond by calling `navigator.mediaDevices.getUserMedia(...)`
|
|
550
|
-
// and `replaceTrack`ing the result onto a pre-allocated sendonly
|
|
551
|
-
// video transceiver. Pre-allocating the transceiver lets us
|
|
552
|
-
// attach/detach the local camera without renegotiating the SDP
|
|
553
|
-
// (the slot is already in the answer's a=video block, just with
|
|
554
|
-
// `inactive`/empty until we install the track).
|
|
555
|
-
//
|
|
556
|
-
// Refs (not state) because all callers live inside event-handler
|
|
557
|
-
// closures and the ref read happens during message processing, not
|
|
558
|
-
// during render. We keep both the sender and the active local
|
|
559
|
-
// stream so teardown can stop tracks without leaking the camera
|
|
560
|
-
// green light.
|
|
561
|
-
const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
|
|
562
|
-
const outboundLocalStreamRef = useRef<MediaStream | null>(null);
|
|
563
|
-
// Bumped on every `cameraRequest` so a handler suspended on an
|
|
564
|
-
// await (e.g. the getUserMedia prompt) can detect it was superseded
|
|
565
|
-
// by a newer request and bail instead of re-attaching the camera.
|
|
566
|
-
const cameraRequestGenerationRef = useRef(0);
|
|
567
|
-
const cameraResolutionCapRef = useRef(cameraResolutionCap);
|
|
568
|
-
cameraResolutionCapRef.current = cameraResolutionCap;
|
|
569
|
-
// The aspect prop also rides a ref so the WS `onopen` reconnect
|
|
570
|
-
// path can replay the operator's last pick without depending on
|
|
571
|
-
// an in-flight render cycle. We mutate it inline (same render-
|
|
572
|
-
// time pattern as the cap ref) so a parent prop change is visible
|
|
573
|
-
// to closures captured during the next render; the useEffect
|
|
574
|
-
// below handles the actual "send to host" side-effect.
|
|
575
|
-
const cameraAspectRef = useRef<CameraAspect | undefined>(cameraAspect);
|
|
576
|
-
cameraAspectRef.current = cameraAspect;
|
|
577
|
-
// Mirror the demand-change callback into a ref so the WS message
|
|
578
|
-
// handler always sees the freshest customer callback even when the
|
|
579
|
-
// parent re-renders mid-session.
|
|
580
|
-
const onCameraDemandChangeRef = useRef(onCameraDemandChange);
|
|
581
|
-
onCameraDemandChangeRef.current = onCameraDemandChange;
|
|
582
|
-
const onCameraStatsRef = useRef(onCameraStats);
|
|
583
|
-
onCameraStatsRef.current = onCameraStats;
|
|
584
|
-
// Active outbound-camera stats poller. While the camera is
|
|
585
|
-
// sending we sample `RTCPeerConnection.getStats()` once per
|
|
586
|
-
// second, derive deltas (bitrate, fps, packet loss) from the
|
|
587
|
-
// previous sample, and push the result through `onCameraStats`.
|
|
588
|
-
// Cleared on teardown so we never leak the timer.
|
|
589
|
-
const cameraStatsTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
|
590
|
-
const cameraStatsPrevRef = useRef<{
|
|
591
|
-
timestamp: number;
|
|
592
|
-
framesEncoded?: number;
|
|
593
|
-
bytesSent?: number;
|
|
594
|
-
packetsSent?: number;
|
|
595
|
-
packetsLost?: number;
|
|
596
|
-
} | null>(null);
|
|
597
366
|
const firstFrameShownRef = useRef(false);
|
|
598
367
|
const pendingScreenshotResolversRef = useRef<
|
|
599
368
|
Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
|
|
@@ -694,8 +463,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
694
463
|
onInspectSelectionChangeRef.current = onInspectSelectionChange;
|
|
695
464
|
const onAxStatusChangeRef = useRef(onAxStatusChange);
|
|
696
465
|
onAxStatusChangeRef.current = onAxStatusChange;
|
|
697
|
-
const onTerminatedRef = useRef(onTerminated);
|
|
698
|
-
onTerminatedRef.current = onTerminated;
|
|
699
466
|
|
|
700
467
|
const inspectActive = inspectMode === true || inspectMode === 'hover-only';
|
|
701
468
|
const inspectModeResolved: InspectMode = inspectMode === 'hover-only' ? 'hover-only' : 'select';
|
|
@@ -1668,169 +1435,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1668
1435
|
setVideoLoaded(true);
|
|
1669
1436
|
};
|
|
1670
1437
|
|
|
1671
|
-
// Stop every track on a MediaStream and release the device handle.
|
|
1672
|
-
// The browser keeps the camera "on" indicator lit until at least
|
|
1673
|
-
// one track on the underlying source is stopped, so we have to
|
|
1674
|
-
// call stop() explicitly — closing the peer connection alone
|
|
1675
|
-
// won't do it.
|
|
1676
|
-
const stopMediaStream = (stream: MediaStream) => {
|
|
1677
|
-
for (const track of stream.getTracks()) {
|
|
1678
|
-
try {
|
|
1679
|
-
track.stop();
|
|
1680
|
-
} catch (err) {
|
|
1681
|
-
debugWarn('track.stop() failed:', err);
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
};
|
|
1685
|
-
|
|
1686
|
-
// Stop and forget any currently-attached outbound camera stream.
|
|
1687
|
-
// Safe to call when nothing is attached. Does not touch the
|
|
1688
|
-
// sender — the caller is responsible for `replaceTrack(null)`
|
|
1689
|
-
// separately when it wants the SDP slot itself empty.
|
|
1690
|
-
const stopOutboundLocalStream = () => {
|
|
1691
|
-
const stream = outboundLocalStreamRef.current;
|
|
1692
|
-
if (!stream) return;
|
|
1693
|
-
outboundLocalStreamRef.current = null;
|
|
1694
|
-
stopMediaStream(stream);
|
|
1695
|
-
};
|
|
1696
|
-
|
|
1697
|
-
// Translate the user-facing resolution cap into a
|
|
1698
|
-
// MediaTrackConstraints fragment we can feed `getUserMedia`
|
|
1699
|
-
// or `applyConstraints`. Always returns a 30 fps ceiling so
|
|
1700
|
-
// a 60 fps webcam doesn't double our encoder cost for no
|
|
1701
|
-
// visible win (the simulator pool is paced at 30).
|
|
1702
|
-
//
|
|
1703
|
-
// We use `ideal` rather than `max` so that a webcam with a
|
|
1704
|
-
// native 720p mode still hands us 720p when the user picks
|
|
1705
|
-
// 1080p — instead of refusing the constraint outright. The
|
|
1706
|
-
// browser's NotReadableError on a too-strict `max` is the
|
|
1707
|
-
// most common camera-permission gotcha in the wild.
|
|
1708
|
-
const cameraCapToConstraints = (cap: CameraResolutionCap): MediaTrackConstraints => {
|
|
1709
|
-
const base: MediaTrackConstraints = {
|
|
1710
|
-
frameRate: { ideal: 30, max: 30 },
|
|
1711
|
-
};
|
|
1712
|
-
switch (cap) {
|
|
1713
|
-
case '1080p':
|
|
1714
|
-
return { ...base, width: { ideal: 1920 }, height: { ideal: 1080 } };
|
|
1715
|
-
case '720p':
|
|
1716
|
-
return { ...base, width: { ideal: 1280 }, height: { ideal: 720 } };
|
|
1717
|
-
case '480p':
|
|
1718
|
-
return { ...base, width: { ideal: 854 }, height: { ideal: 480 } };
|
|
1719
|
-
case 'auto':
|
|
1720
|
-
default:
|
|
1721
|
-
return base;
|
|
1722
|
-
}
|
|
1723
|
-
};
|
|
1724
|
-
|
|
1725
|
-
// Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
|
|
1726
|
-
// into a short uppercase label suitable for a status badge.
|
|
1727
|
-
const shortenCodecMime = (mime: string | undefined): string | undefined => {
|
|
1728
|
-
if (!mime) return undefined;
|
|
1729
|
-
const slash = mime.indexOf('/');
|
|
1730
|
-
const tail = slash >= 0 ? mime.slice(slash + 1) : mime;
|
|
1731
|
-
const upper = tail.toUpperCase();
|
|
1732
|
-
if (upper === 'HEVC') return 'H265';
|
|
1733
|
-
return upper;
|
|
1734
|
-
};
|
|
1735
|
-
|
|
1736
|
-
// Sample outbound-camera stats once and push a normalised
|
|
1737
|
-
// snapshot through `onCameraStats`. Skips silently if the sender
|
|
1738
|
-
// has been torn down between intervals.
|
|
1739
|
-
const sampleOutboundCameraStats = async () => {
|
|
1740
|
-
const sender = outboundCameraSenderRef.current;
|
|
1741
|
-
const handler = onCameraStatsRef.current;
|
|
1742
|
-
if (!sender || !handler) return;
|
|
1743
|
-
let report: RTCStatsReport;
|
|
1744
|
-
try {
|
|
1745
|
-
report = await sender.getStats();
|
|
1746
|
-
} catch {
|
|
1747
|
-
return;
|
|
1748
|
-
}
|
|
1749
|
-
let outbound: any | undefined;
|
|
1750
|
-
let codecMime: string | undefined;
|
|
1751
|
-
let remoteInbound: any | undefined;
|
|
1752
|
-
report.forEach((entry: any) => {
|
|
1753
|
-
if (entry.type === 'outbound-rtp' && entry.kind === 'video') {
|
|
1754
|
-
outbound = entry;
|
|
1755
|
-
} else if (entry.type === 'remote-inbound-rtp' && entry.kind === 'video') {
|
|
1756
|
-
remoteInbound = entry;
|
|
1757
|
-
}
|
|
1758
|
-
});
|
|
1759
|
-
if (outbound?.codecId) {
|
|
1760
|
-
const codec = report.get(outbound.codecId);
|
|
1761
|
-
if (codec) codecMime = (codec as any).mimeType;
|
|
1762
|
-
}
|
|
1763
|
-
if (!outbound) return;
|
|
1764
|
-
const now = (outbound.timestamp as number) ?? Date.now();
|
|
1765
|
-
const prev = cameraStatsPrevRef.current;
|
|
1766
|
-
let fps: number | undefined;
|
|
1767
|
-
let bitrate: number | undefined;
|
|
1768
|
-
let lossPct: number | undefined;
|
|
1769
|
-
if (prev && now > prev.timestamp) {
|
|
1770
|
-
const dt = (now - prev.timestamp) / 1000;
|
|
1771
|
-
if (typeof outbound.framesEncoded === 'number' && typeof prev.framesEncoded === 'number') {
|
|
1772
|
-
fps = Math.max(0, (outbound.framesEncoded - prev.framesEncoded) / dt);
|
|
1773
|
-
}
|
|
1774
|
-
if (typeof outbound.bytesSent === 'number' && typeof prev.bytesSent === 'number') {
|
|
1775
|
-
bitrate = Math.max(0, ((outbound.bytesSent - prev.bytesSent) * 8) / dt);
|
|
1776
|
-
}
|
|
1777
|
-
if (
|
|
1778
|
-
typeof outbound.packetsSent === 'number' &&
|
|
1779
|
-
typeof prev.packetsSent === 'number' &&
|
|
1780
|
-
remoteInbound &&
|
|
1781
|
-
typeof remoteInbound.packetsLost === 'number' &&
|
|
1782
|
-
typeof prev.packetsLost === 'number'
|
|
1783
|
-
) {
|
|
1784
|
-
const sent = outbound.packetsSent - prev.packetsSent;
|
|
1785
|
-
const lost = remoteInbound.packetsLost - prev.packetsLost;
|
|
1786
|
-
if (sent > 0) lossPct = Math.max(0, Math.min(100, (lost / sent) * 100));
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
cameraStatsPrevRef.current = {
|
|
1790
|
-
timestamp: now,
|
|
1791
|
-
framesEncoded: outbound.framesEncoded,
|
|
1792
|
-
bytesSent: outbound.bytesSent,
|
|
1793
|
-
packetsSent: outbound.packetsSent,
|
|
1794
|
-
packetsLost: remoteInbound?.packetsLost,
|
|
1795
|
-
};
|
|
1796
|
-
const stats: CameraStreamStats = {
|
|
1797
|
-
codec: shortenCodecMime(codecMime),
|
|
1798
|
-
encoderImplementation: outbound.encoderImplementation,
|
|
1799
|
-
hardwareAccelerated:
|
|
1800
|
-
typeof outbound.powerEfficientEncoder === 'boolean' ? outbound.powerEfficientEncoder : undefined,
|
|
1801
|
-
framesPerSecond: fps,
|
|
1802
|
-
framesEncoded: outbound.framesEncoded,
|
|
1803
|
-
bitrateBps: bitrate,
|
|
1804
|
-
width: outbound.frameWidth,
|
|
1805
|
-
height: outbound.frameHeight,
|
|
1806
|
-
qualityLimitationReason: outbound.qualityLimitationReason,
|
|
1807
|
-
rttMs:
|
|
1808
|
-
typeof remoteInbound?.roundTripTime === 'number' ? remoteInbound.roundTripTime * 1000 : undefined,
|
|
1809
|
-
packetsLostPct: lossPct,
|
|
1810
|
-
};
|
|
1811
|
-
safeInvoke('onCameraStats', onCameraStatsRef.current, stats);
|
|
1812
|
-
};
|
|
1813
|
-
|
|
1814
|
-
const startCameraStatsPoller = () => {
|
|
1815
|
-
if (cameraStatsTimerRef.current !== null) return;
|
|
1816
|
-
cameraStatsPrevRef.current = null;
|
|
1817
|
-
// First sample fires after one interval — the very first
|
|
1818
|
-
// sample has no deltas to compare against, which is fine: the
|
|
1819
|
-
// codec/encoder identity is still useful on its own.
|
|
1820
|
-
cameraStatsTimerRef.current = setInterval(() => {
|
|
1821
|
-
void sampleOutboundCameraStats();
|
|
1822
|
-
}, 1000);
|
|
1823
|
-
};
|
|
1824
|
-
|
|
1825
|
-
const stopCameraStatsPoller = () => {
|
|
1826
|
-
if (cameraStatsTimerRef.current !== null) {
|
|
1827
|
-
clearInterval(cameraStatsTimerRef.current);
|
|
1828
|
-
cameraStatsTimerRef.current = null;
|
|
1829
|
-
}
|
|
1830
|
-
cameraStatsPrevRef.current = null;
|
|
1831
|
-
safeInvoke('onCameraStats', onCameraStatsRef.current, null);
|
|
1832
|
-
};
|
|
1833
|
-
|
|
1834
1438
|
const teardownConnection = () => {
|
|
1835
1439
|
clearConnectionSuccessTimeout();
|
|
1836
1440
|
clearIceDisconnectedGrace();
|
|
@@ -1839,12 +1443,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1839
1443
|
axFetcherRef.current.stop();
|
|
1840
1444
|
axFetcherRef.current = null;
|
|
1841
1445
|
}
|
|
1842
|
-
// Drop any active outbound camera before the PC dies so the
|
|
1843
|
-
// browser doesn't leave the camera indicator lit between
|
|
1844
|
-
// reconnects.
|
|
1845
|
-
stopCameraStatsPoller();
|
|
1846
|
-
stopOutboundLocalStream();
|
|
1847
|
-
outboundCameraSenderRef.current = null;
|
|
1848
1446
|
// A scheduled cursor flush would otherwise call setState on a
|
|
1849
1447
|
// teardown component once the next frame runs.
|
|
1850
1448
|
if (cursorRafIdRef.current !== undefined) {
|
|
@@ -1887,92 +1485,11 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1887
1485
|
}
|
|
1888
1486
|
};
|
|
1889
1487
|
|
|
1890
|
-
// Probe the instance endpoint over HTTP to tell a permanent termination
|
|
1891
|
-
// apart from a recoverable connection drop. The router serves the same
|
|
1892
|
-
// path the WebSocket uses and returns 404 once the instance is gone
|
|
1893
|
-
// (deleted / terminated), while a live instance returns something else.
|
|
1894
|
-
//
|
|
1895
|
-
// Only an authoritative, freshly-fetched 404 counts as terminated. A
|
|
1896
|
-
// network outage, a non-404 status, or a CORS-blocked (unreadable)
|
|
1897
|
-
// response all resolve to `false` so we keep the normal retry/reconnect
|
|
1898
|
-
// behavior and never mistake a connectivity problem for a termination.
|
|
1899
|
-
const probeTerminated = async (): Promise<boolean> => {
|
|
1900
|
-
// The browser knows it has no network — this is a connectivity drop,
|
|
1901
|
-
// not a termination. Don't probe (and don't risk a cached 404).
|
|
1902
|
-
if (typeof navigator !== 'undefined' && navigator.onLine === false) {
|
|
1903
|
-
return false;
|
|
1904
|
-
}
|
|
1905
|
-
let probeUrl: string;
|
|
1906
|
-
try {
|
|
1907
|
-
const parsed = new URL(url);
|
|
1908
|
-
parsed.protocol =
|
|
1909
|
-
parsed.protocol === 'wss:' ? 'https:'
|
|
1910
|
-
: parsed.protocol === 'ws:' ? 'http:'
|
|
1911
|
-
: parsed.protocol;
|
|
1912
|
-
parsed.searchParams.set('token', token);
|
|
1913
|
-
probeUrl = parsed.toString();
|
|
1914
|
-
} catch {
|
|
1915
|
-
return false;
|
|
1916
|
-
}
|
|
1917
|
-
const controller = new AbortController();
|
|
1918
|
-
const timeoutId = window.setTimeout(() => controller.abort(), TERMINATION_PROBE_TIMEOUT_MS);
|
|
1919
|
-
try {
|
|
1920
|
-
// `no-store` so a prior response can't be served from the HTTP cache
|
|
1921
|
-
// (e.g. a stale 404 returned while offline would look like a
|
|
1922
|
-
// termination); the probe must always reach the live router.
|
|
1923
|
-
const res = await fetch(probeUrl, { method: 'GET', cache: 'no-store', signal: controller.signal });
|
|
1924
|
-
return res.status === 404;
|
|
1925
|
-
} catch {
|
|
1926
|
-
return false;
|
|
1927
|
-
} finally {
|
|
1928
|
-
window.clearTimeout(timeoutId);
|
|
1929
|
-
}
|
|
1930
|
-
};
|
|
1931
|
-
|
|
1932
|
-
const markTerminated = () => {
|
|
1933
|
-
if (terminatedRef.current) return;
|
|
1934
|
-
terminatedRef.current = true;
|
|
1935
|
-
setTerminated(true);
|
|
1936
|
-
setRetryExhausted(false);
|
|
1937
|
-
// Invalidate any in-flight/scheduled attempt and stop all activity so
|
|
1938
|
-
// we don't keep hammering an endpoint that will never come back.
|
|
1939
|
-
connectionGenerationRef.current += 1;
|
|
1940
|
-
clearScheduledRetry();
|
|
1941
|
-
clearConnectionSuccessTimeout();
|
|
1942
|
-
clearIceDisconnectedGrace();
|
|
1943
|
-
teardownConnection();
|
|
1944
|
-
updateStatus('Instance terminated');
|
|
1945
|
-
safeInvoke('onTerminated', onTerminatedRef.current);
|
|
1946
|
-
};
|
|
1947
|
-
|
|
1948
|
-
// Run a one-shot termination check in reaction to a drop. Event-driven
|
|
1949
|
-
// (no polling), deduped while one is already in flight, and a no-op once
|
|
1950
|
-
// we've concluded the instance is gone.
|
|
1951
|
-
const checkTerminated = () => {
|
|
1952
|
-
if (terminatedRef.current || terminationProbeInFlightRef.current) return;
|
|
1953
|
-
terminationProbeInFlightRef.current = true;
|
|
1954
|
-
void probeTerminated().then((isTerminated) => {
|
|
1955
|
-
terminationProbeInFlightRef.current = false;
|
|
1956
|
-
if (isTerminated) {
|
|
1957
|
-
markTerminated();
|
|
1958
|
-
}
|
|
1959
|
-
});
|
|
1960
|
-
};
|
|
1961
|
-
|
|
1962
1488
|
const scheduleRetry = (reason: string, generation: number) => {
|
|
1963
|
-
if (terminatedRef.current) {
|
|
1964
|
-
return;
|
|
1965
|
-
}
|
|
1966
1489
|
if (generation !== connectionGenerationRef.current) {
|
|
1967
1490
|
return;
|
|
1968
1491
|
}
|
|
1969
1492
|
|
|
1970
|
-
// A drop just happened. Find out whether the instance is gone for good
|
|
1971
|
-
// (404) rather than a recoverable blip; if so the probe stops retries
|
|
1972
|
-
// and surfaces the terminated message. Meanwhile the normal retry path
|
|
1973
|
-
// below proceeds and is short-circuited once the verdict lands.
|
|
1974
|
-
checkTerminated();
|
|
1975
|
-
|
|
1976
1493
|
if (controlChannelOpenedRef.current) {
|
|
1977
1494
|
if (!autoReconnectRef.current) {
|
|
1978
1495
|
updateStatus(`Connection failed after it was established: ${reason}`);
|
|
@@ -2008,9 +1525,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2008
1525
|
};
|
|
2009
1526
|
|
|
2010
1527
|
const startAttempt = async (attemptNumber = 0) => {
|
|
2011
|
-
// Never reconnect to an instance we've concluded is gone. `start()`
|
|
2012
|
-
// clears this when a deliberate fresh connection is requested.
|
|
2013
|
-
if (terminatedRef.current) return;
|
|
2014
1528
|
const generation = connectionGenerationRef.current + 1;
|
|
2015
1529
|
connectionGenerationRef.current = generation;
|
|
2016
1530
|
connectionAttemptRef.current = attemptNumber;
|
|
@@ -2054,21 +1568,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2054
1568
|
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2055
1569
|
return;
|
|
2056
1570
|
}
|
|
2057
|
-
// Replay the saved camera aspect for fresh connections
|
|
2058
|
-
// (initial mount, autoreconnect, sim reboot). The host's
|
|
2059
|
-
// streamer starts at its boot default (16:9 / 1920×1080)
|
|
2060
|
-
// and a missing message would mean the operator's pick
|
|
2061
|
-
// silently reverts on reconnect. We always send something
|
|
2062
|
-
// when the prop is set, even if it matches the host
|
|
2063
|
-
// default — the host short-circuits no-op rebuilds.
|
|
2064
|
-
const initialAspect = cameraAspectRef.current;
|
|
2065
|
-
if (initialAspect) {
|
|
2066
|
-
try {
|
|
2067
|
-
ws.send(JSON.stringify({ type: 'cameraAspect', aspect: initialAspect }));
|
|
2068
|
-
} catch (err) {
|
|
2069
|
-
debugWarn('initial cameraAspect send failed:', err);
|
|
2070
|
-
}
|
|
2071
|
-
}
|
|
2072
1571
|
settle(resolve);
|
|
2073
1572
|
};
|
|
2074
1573
|
|
|
@@ -2145,23 +1644,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2145
1644
|
peerConnection.addTransceiver('audio', { direction: 'recvonly' });
|
|
2146
1645
|
const videoTransceiver = peerConnection.addTransceiver('video', { direction: 'recvonly' });
|
|
2147
1646
|
|
|
2148
|
-
// Pre-allocate a sendonly slot for the user's camera so the
|
|
2149
|
-
// sim can pull frames later without renegotiating SDP.
|
|
2150
|
-
// Allocating up-front means we don't have to renegotiate the
|
|
2151
|
-
// SDP when the simulator app actually opens its
|
|
2152
|
-
// `AVCaptureSession` — the codec/SSRC negotiation happens
|
|
2153
|
-
// once, here, and turning the camera on/off later is just a
|
|
2154
|
-
// `replaceTrack`.
|
|
2155
|
-
// iOS only: Android has no camera consumer, and the extra
|
|
2156
|
-
// m=video line crashes Android 14's scrcpy/libwebrtc.
|
|
2157
|
-
const outboundCameraTransceiver =
|
|
2158
|
-
platform === 'ios' ?
|
|
2159
|
-
peerConnection.addTransceiver('video', {
|
|
2160
|
-
direction: 'sendonly',
|
|
2161
|
-
})
|
|
2162
|
-
: null;
|
|
2163
|
-
outboundCameraSenderRef.current = outboundCameraTransceiver?.sender ?? null;
|
|
2164
|
-
|
|
2165
1647
|
// As hardware encoder, we use H265 for iOS and VP9 for Android.
|
|
2166
1648
|
// We make sure these two are the first ones in the list.
|
|
2167
1649
|
// If not, the fallback is H264 which is also hardware accelerated, although not as good,
|
|
@@ -2187,59 +1669,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2187
1669
|
}
|
|
2188
1670
|
}
|
|
2189
1671
|
|
|
2190
|
-
// Pin the outbound camera transceiver's codec order with
|
|
2191
|
-
// negotiation-time fallback. Preference order:
|
|
2192
|
-
// 1. H.265 / HEVC — VideoToolbox HW on M-series + recent
|
|
2193
|
-
// Chrome. ~30% less bitrate at equal quality. If the
|
|
2194
|
-
// answerer (limulator) supports HEVC, this wins.
|
|
2195
|
-
// 2. H.264 with VideoToolbox-friendly profile-level-ids
|
|
2196
|
-
// (42e01f / 42e028 / 640c1f). Universally HW-accelerated
|
|
2197
|
-
// on every Mac since 2009. SDP-time fallback when HEVC
|
|
2198
|
-
// isn't in the answer.
|
|
2199
|
-
// 3. Any other H.264 profile. Often resolves to OpenH264
|
|
2200
|
-
// (software) so we keep it explicitly last among H.264
|
|
2201
|
-
// to avoid Chrome's default pick.
|
|
2202
|
-
// 4. VP9 / VP8 (libvpx software on Mac). Last resort.
|
|
2203
|
-
//
|
|
2204
|
-
// Caveat: this is *negotiation* fallback. If HEVC HW encoder
|
|
2205
|
-
// init fails at session start or gets demoted mid-stream,
|
|
2206
|
-
// Chrome falls back to *software* HEVC, not H.264 — no spec
|
|
2207
|
-
// hook for runtime re-negotiation. The host-side stats
|
|
2208
|
-
// logger will surface this as decoder=ffmpeg / hw=false.
|
|
2209
|
-
if (outboundCameraTransceiver && RTCRtpSender.getCapabilities) {
|
|
2210
|
-
const senderCaps = RTCRtpSender.getCapabilities('video');
|
|
2211
|
-
if (senderCaps && senderCaps.codecs) {
|
|
2212
|
-
const vtH264Profiles = new Set(['42e01f', '42e028', '640c1f']);
|
|
2213
|
-
const score = (c: { mimeType: string; sdpFmtpLine?: string }): number => {
|
|
2214
|
-
const mime = c.mimeType.toLowerCase();
|
|
2215
|
-
const fmtp = (c.sdpFmtpLine || '').toLowerCase();
|
|
2216
|
-
const profileMatch = fmtp.match(/profile-level-id=([0-9a-f]{6})/);
|
|
2217
|
-
const profile = profileMatch ? profileMatch[1] : '';
|
|
2218
|
-
if (mime === 'video/h265' || mime === 'video/hevc') return 1;
|
|
2219
|
-
if (mime === 'video/h264' && vtH264Profiles.has(profile)) return 2;
|
|
2220
|
-
if (mime === 'video/h264') return 3;
|
|
2221
|
-
if (mime === 'video/vp9') return 4;
|
|
2222
|
-
if (mime === 'video/vp8') return 5;
|
|
2223
|
-
if (mime === 'video/rtx' || mime === 'video/red' || mime === 'video/ulpfec') {
|
|
2224
|
-
return 0; // Keep RTX/FEC available alongside everything else.
|
|
2225
|
-
}
|
|
2226
|
-
return 6;
|
|
2227
|
-
};
|
|
2228
|
-
const sortedSendCodecs = [...senderCaps.codecs].sort((a, b) => score(a) - score(b));
|
|
2229
|
-
try {
|
|
2230
|
-
outboundCameraTransceiver.setCodecPreferences(sortedSendCodecs);
|
|
2231
|
-
debugLog(
|
|
2232
|
-
'Outbound camera codec preferences:',
|
|
2233
|
-
sortedSendCodecs
|
|
2234
|
-
.map((c) => `${c.mimeType}${c.sdpFmtpLine ? `[${c.sdpFmtpLine}]` : ''}`)
|
|
2235
|
-
.join(', '),
|
|
2236
|
-
);
|
|
2237
|
-
} catch (err) {
|
|
2238
|
-
debugWarn('Failed to set outbound camera codec preferences:', err);
|
|
2239
|
-
}
|
|
2240
|
-
}
|
|
2241
|
-
}
|
|
2242
|
-
|
|
2243
1672
|
const dataChannel = peerConnection.createDataChannel('control', {
|
|
2244
1673
|
ordered: true,
|
|
2245
1674
|
negotiated: true,
|
|
@@ -2543,243 +1972,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2543
1972
|
pendingScreenshotResolversRef.current.delete(message.id);
|
|
2544
1973
|
pendingScreenshotRejectersRef.current.delete(message.id);
|
|
2545
1974
|
break;
|
|
2546
|
-
case 'cameraRequest': {
|
|
2547
|
-
// Only iOS sessions allocate the sendonly outbound-camera
|
|
2548
|
-
// transceiver; without it there is nothing to feed the sim, so
|
|
2549
|
-
// ignore camera requests entirely — no getUserMedia prompt and
|
|
2550
|
-
// no onCameraDemandChange callbacks. Android intentionally stays
|
|
2551
|
-
// silent, as the API docs promise.
|
|
2552
|
-
if (platform !== 'ios' || !outboundCameraSenderRef.current) {
|
|
2553
|
-
break;
|
|
2554
|
-
}
|
|
2555
|
-
const active = message.active === true;
|
|
2556
|
-
// Bump up front so any earlier in-flight handler bails.
|
|
2557
|
-
const cameraGeneration = ++cameraRequestGenerationRef.current;
|
|
2558
|
-
const isCurrentCameraRequest = () =>
|
|
2559
|
-
isCurrentAttempt() &&
|
|
2560
|
-
wsRef.current === ws &&
|
|
2561
|
-
cameraGeneration === cameraRequestGenerationRef.current;
|
|
2562
|
-
// Log unconditionally — this is one of the few places we
|
|
2563
|
-
// hand control to the browser's permission UI, and a
|
|
2564
|
-
// silent failure here looks identical to "limulator
|
|
2565
|
-
// never asked" from the user's point of view.
|
|
2566
|
-
// eslint-disable-next-line no-console
|
|
2567
|
-
console.info('[RemoteControl] cameraRequest received, active=', active);
|
|
2568
|
-
if (!active) {
|
|
2569
|
-
// Sim no longer wants frames. Drop our local track and
|
|
2570
|
-
// shut the browser's camera green light off.
|
|
2571
|
-
const sender = outboundCameraSenderRef.current;
|
|
2572
|
-
if (sender) {
|
|
2573
|
-
try {
|
|
2574
|
-
await sender.replaceTrack(null);
|
|
2575
|
-
} catch (err) {
|
|
2576
|
-
debugWarn('replaceTrack(null) on camera detach failed:', err);
|
|
2577
|
-
}
|
|
2578
|
-
}
|
|
2579
|
-
stopCameraStatsPoller();
|
|
2580
|
-
stopOutboundLocalStream();
|
|
2581
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
|
|
2582
|
-
break;
|
|
2583
|
-
}
|
|
2584
|
-
// Sim is asking for camera. Ask the browser; the user's
|
|
2585
|
-
// prompt response is reported back to the host via a
|
|
2586
|
-
// `cameraResult` message so it can swap to a
|
|
2587
|
-
// black-frame fallback on denial.
|
|
2588
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
|
|
2589
|
-
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
2590
|
-
// Likely an insecure context (http on a non-localhost
|
|
2591
|
-
// origin) — Chrome strips `mediaDevices` off
|
|
2592
|
-
// `navigator` in that case and the only signal is
|
|
2593
|
-
// this undefined check.
|
|
2594
|
-
// eslint-disable-next-line no-console
|
|
2595
|
-
console.warn(
|
|
2596
|
-
'[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
|
|
2597
|
-
'getUserMedia requires a secure context (https or http://localhost). ' +
|
|
2598
|
-
'Replying cameraResult granted=false so limulator falls back to black frames.',
|
|
2599
|
-
);
|
|
2600
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2601
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2602
|
-
break;
|
|
2603
|
-
}
|
|
2604
|
-
let stream: MediaStream | null = null;
|
|
2605
|
-
try {
|
|
2606
|
-
// Capture at the webcam's *native* resolution by
|
|
2607
|
-
// default (no width/height constraints). When the
|
|
2608
|
-
// host app has picked an explicit cap via
|
|
2609
|
-
// `cameraResolutionCap`, we honour it here. Frame
|
|
2610
|
-
// rate is always capped to 30 to match the
|
|
2611
|
-
// simulator pool's pacing.
|
|
2612
|
-
stream = await navigator.mediaDevices.getUserMedia({
|
|
2613
|
-
video: cameraCapToConstraints(cameraResolutionCapRef.current),
|
|
2614
|
-
audio: false,
|
|
2615
|
-
});
|
|
2616
|
-
} catch (err) {
|
|
2617
|
-
// Surface unconditionally — the user is the only one
|
|
2618
|
-
// who can fix this (permission denied, no device,
|
|
2619
|
-
// dismissed prompt, etc.). `debugWarn` alone would
|
|
2620
|
-
// hide it in normal builds.
|
|
2621
|
-
// eslint-disable-next-line no-console
|
|
2622
|
-
console.warn('[RemoteControl] getUserMedia denied/failed:', err);
|
|
2623
|
-
}
|
|
2624
|
-
if (!isCurrentCameraRequest()) {
|
|
2625
|
-
// Superseded during the prompt; drop the stream we got.
|
|
2626
|
-
if (stream) stopMediaStream(stream);
|
|
2627
|
-
return;
|
|
2628
|
-
}
|
|
2629
|
-
if (!stream) {
|
|
2630
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2631
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2632
|
-
break;
|
|
2633
|
-
}
|
|
2634
|
-
// Replace any previous local stream (e.g. an earlier
|
|
2635
|
-
// cameraRequest that resolved with a different device)
|
|
2636
|
-
// before we install the new tracks.
|
|
2637
|
-
stopOutboundLocalStream();
|
|
2638
|
-
outboundLocalStreamRef.current = stream;
|
|
2639
|
-
const sender = outboundCameraSenderRef.current;
|
|
2640
|
-
const videoTrack = stream.getVideoTracks()[0] ?? null;
|
|
2641
|
-
if (!sender || !videoTrack) {
|
|
2642
|
-
if (stream) stopMediaStream(stream);
|
|
2643
|
-
outboundLocalStreamRef.current = null;
|
|
2644
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2645
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2646
|
-
break;
|
|
2647
|
-
}
|
|
2648
|
-
try {
|
|
2649
|
-
await sender.replaceTrack(videoTrack);
|
|
2650
|
-
} catch (err) {
|
|
2651
|
-
debugWarn('replaceTrack(videoTrack) failed:', err);
|
|
2652
|
-
stopMediaStream(stream);
|
|
2653
|
-
outboundLocalStreamRef.current = null;
|
|
2654
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2655
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2656
|
-
break;
|
|
2657
|
-
}
|
|
2658
|
-
if (!isCurrentCameraRequest()) {
|
|
2659
|
-
// Superseded mid-attach; detach and skip the ACK/poller.
|
|
2660
|
-
try {
|
|
2661
|
-
await sender.replaceTrack(null);
|
|
2662
|
-
} catch (err) {
|
|
2663
|
-
debugWarn('replaceTrack(null) on stale camera attach failed:', err);
|
|
2664
|
-
}
|
|
2665
|
-
stopMediaStream(stream);
|
|
2666
|
-
if (outboundLocalStreamRef.current === stream) {
|
|
2667
|
-
outboundLocalStreamRef.current = null;
|
|
2668
|
-
}
|
|
2669
|
-
return;
|
|
2670
|
-
}
|
|
2671
|
-
// Tell the encoder this is real motion content (a
|
|
2672
|
-
// physical camera), not text/slides. With `'motion'`
|
|
2673
|
-
// VideoToolbox / libvpx pick latency-friendly tuning
|
|
2674
|
-
// (no extra B-frames, shorter GoP smoothing). Set
|
|
2675
|
-
// before the first frame so the encoder init reads it.
|
|
2676
|
-
try {
|
|
2677
|
-
videoTrack.contentHint = 'motion';
|
|
2678
|
-
} catch {
|
|
2679
|
-
/* older browsers don't support contentHint; ignore */
|
|
2680
|
-
}
|
|
2681
|
-
// Bound the outbound bitrate generously and let
|
|
2682
|
-
// WebRTC's congestion control (BWE) find the floor.
|
|
2683
|
-
// 8 Mbps is comfortable for native 1080p30 webcam
|
|
2684
|
-
// content over LAN — the encoder will use far less when
|
|
2685
|
-
// the scene is static, and BWE will throttle if a
|
|
2686
|
-
// hop is constrained. Pair with
|
|
2687
|
-
// `maintain-framerate` so the quality scaler steps
|
|
2688
|
-
// down resolution before dropping frames, matching how
|
|
2689
|
-
// Meet/Zoom degrade.
|
|
2690
|
-
try {
|
|
2691
|
-
const params = sender.getParameters();
|
|
2692
|
-
if (!params.encodings || params.encodings.length === 0) {
|
|
2693
|
-
params.encodings = [{}];
|
|
2694
|
-
}
|
|
2695
|
-
params.encodings[0].maxBitrate = 8_000_000;
|
|
2696
|
-
params.encodings[0].maxFramerate = 30;
|
|
2697
|
-
params.degradationPreference = 'maintain-framerate';
|
|
2698
|
-
await sender.setParameters(params);
|
|
2699
|
-
} catch (err) {
|
|
2700
|
-
debugWarn('setParameters on outbound camera failed:', err);
|
|
2701
|
-
}
|
|
2702
|
-
// Surface the actual captured geometry to the host so
|
|
2703
|
-
// it can size its IOSurface pool / picture-format
|
|
2704
|
-
// metadata to match. `getSettings()` returns what the
|
|
2705
|
-
// browser actually picked — which may differ from any
|
|
2706
|
-
// hints we sent in the constraints — including the
|
|
2707
|
-
// selected `deviceId` / `label` (useful when a user has
|
|
2708
|
-
// multiple cameras and we eventually expose a picker).
|
|
2709
|
-
let cameraMetadata: {
|
|
2710
|
-
width?: number;
|
|
2711
|
-
height?: number;
|
|
2712
|
-
frameRate?: number;
|
|
2713
|
-
deviceId?: string;
|
|
2714
|
-
label?: string;
|
|
2715
|
-
facingMode?: string;
|
|
2716
|
-
} = {};
|
|
2717
|
-
try {
|
|
2718
|
-
const settings = videoTrack.getSettings();
|
|
2719
|
-
cameraMetadata = {
|
|
2720
|
-
width: settings.width,
|
|
2721
|
-
height: settings.height,
|
|
2722
|
-
frameRate: settings.frameRate,
|
|
2723
|
-
deviceId: settings.deviceId,
|
|
2724
|
-
label: videoTrack.label || undefined,
|
|
2725
|
-
facingMode: settings.facingMode,
|
|
2726
|
-
};
|
|
2727
|
-
// eslint-disable-next-line no-console
|
|
2728
|
-
console.info(
|
|
2729
|
-
`[RemoteControl] camera capture: ${cameraMetadata.width}x${cameraMetadata.height}` +
|
|
2730
|
-
` @ ${cameraMetadata.frameRate ?? '?'}fps` +
|
|
2731
|
-
(cameraMetadata.label ? ` — ${cameraMetadata.label}` : ''),
|
|
2732
|
-
);
|
|
2733
|
-
} catch (err) {
|
|
2734
|
-
debugWarn('getSettings() on outbound camera track failed:', err);
|
|
2735
|
-
}
|
|
2736
|
-
// If the browser revokes the track later (extension,
|
|
2737
|
-
// user clicks Stop in the camera tab UI, etc.), notify
|
|
2738
|
-
// the host so it can switch to black frames.
|
|
2739
|
-
videoTrack.onended = () => {
|
|
2740
|
-
if (outboundLocalStreamRef.current !== stream) return;
|
|
2741
|
-
stopCameraStatsPoller();
|
|
2742
|
-
stopOutboundLocalStream();
|
|
2743
|
-
if (wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
|
|
2744
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2745
|
-
}
|
|
2746
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2747
|
-
};
|
|
2748
|
-
if (!isCurrentCameraRequest()) {
|
|
2749
|
-
// Superseded during setParameters; detach and bail.
|
|
2750
|
-
videoTrack.onended = null;
|
|
2751
|
-
try {
|
|
2752
|
-
await sender.replaceTrack(null);
|
|
2753
|
-
} catch (err) {
|
|
2754
|
-
debugWarn('replaceTrack(null) on stale camera finalize failed:', err);
|
|
2755
|
-
}
|
|
2756
|
-
stopMediaStream(stream);
|
|
2757
|
-
if (outboundLocalStreamRef.current === stream) {
|
|
2758
|
-
outboundLocalStreamRef.current = null;
|
|
2759
|
-
}
|
|
2760
|
-
return;
|
|
2761
|
-
}
|
|
2762
|
-
ws.send(
|
|
2763
|
-
JSON.stringify({
|
|
2764
|
-
type: 'cameraResult',
|
|
2765
|
-
granted: true,
|
|
2766
|
-
// Forward what the browser actually captured so the
|
|
2767
|
-
// host can size its IOSurface pool, log the device,
|
|
2768
|
-
// and (eventually) surface a status pill / picker.
|
|
2769
|
-
camera: cameraMetadata,
|
|
2770
|
-
}),
|
|
2771
|
-
);
|
|
2772
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, cameraMetadata);
|
|
2773
|
-
// Kick off the per-second outbound stats sampler. We do
|
|
2774
|
-
// this *after* `setParameters` so the first sample
|
|
2775
|
-
// already sees the encoder under its final bitrate /
|
|
2776
|
-
// degradation policy, and *after* the host-side
|
|
2777
|
-
// attachInboundTrack will have wired up (the
|
|
2778
|
-
// cameraResult ACK is what triggers it on the host),
|
|
2779
|
-
// so framesEncoded starts climbing immediately.
|
|
2780
|
-
startCameraStatsPoller();
|
|
2781
|
-
break;
|
|
2782
|
-
}
|
|
2783
1975
|
case 'terminateAppResult':
|
|
2784
1976
|
if (typeof message.id !== 'string') {
|
|
2785
1977
|
debugWarn('Received invalid terminateApp result message:', message);
|
|
@@ -2849,10 +2041,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2849
2041
|
};
|
|
2850
2042
|
|
|
2851
2043
|
const start = () => {
|
|
2852
|
-
// A fresh connection (mount, prop change, manual retry, reconnect())
|
|
2853
|
-
// clears any prior terminated verdict so we re-evaluate from scratch.
|
|
2854
|
-
terminatedRef.current = false;
|
|
2855
|
-
setTerminated(false);
|
|
2856
2044
|
void startAttempt(0);
|
|
2857
2045
|
};
|
|
2858
2046
|
|
|
@@ -2871,41 +2059,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2871
2059
|
start();
|
|
2872
2060
|
};
|
|
2873
2061
|
|
|
2874
|
-
// Re-apply the resolution cap on the currently-sending track
|
|
2875
|
-
// whenever the host app changes its preference. Skips when no
|
|
2876
|
-
// camera is active — the next `getUserMedia` will pick up the
|
|
2877
|
-
// new value via `cameraCapToConstraints` automatically.
|
|
2878
|
-
useEffect(() => {
|
|
2879
|
-
const stream = outboundLocalStreamRef.current;
|
|
2880
|
-
if (!stream) return;
|
|
2881
|
-
const track = stream.getVideoTracks()[0];
|
|
2882
|
-
if (!track) return;
|
|
2883
|
-
const constraints = cameraCapToConstraints(cameraResolutionCap);
|
|
2884
|
-
track.applyConstraints(constraints).catch((err) => {
|
|
2885
|
-
debugWarn('applyConstraints for new camera cap failed:', err);
|
|
2886
|
-
});
|
|
2887
|
-
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
2888
|
-
}, [cameraResolutionCap]);
|
|
2889
|
-
|
|
2890
|
-
// Push the aspect preference to the host whenever it changes.
|
|
2891
|
-
// The host rebuilds its IOSurface pool and bumps `pool_generation`
|
|
2892
|
-
// on its end; the dylib re-handshakes on next sem_wait. No
|
|
2893
|
-
// peer-connection renegotiation is needed — the aspect change is
|
|
2894
|
-
// purely about the pixel buffer dimensions iOS apps observe, not
|
|
2895
|
-
// about WebRTC track layout. The `cameraAspectRef` is mirrored
|
|
2896
|
-
// higher up so the WS `onopen` reconnect path can replay the
|
|
2897
|
-
// latest value on fresh connections.
|
|
2898
|
-
useEffect(() => {
|
|
2899
|
-
if (!cameraAspect) return;
|
|
2900
|
-
const ws = wsRef.current;
|
|
2901
|
-
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2902
|
-
try {
|
|
2903
|
-
ws.send(JSON.stringify({ type: 'cameraAspect', aspect: cameraAspect }));
|
|
2904
|
-
} catch (err) {
|
|
2905
|
-
debugWarn('cameraAspect send failed:', err);
|
|
2906
|
-
}
|
|
2907
|
-
}, [cameraAspect]);
|
|
2908
|
-
|
|
2909
2062
|
useEffect(() => {
|
|
2910
2063
|
// Reset video loaded state when connection params change
|
|
2911
2064
|
setVideoLoaded(false);
|
|
@@ -2927,11 +2080,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2927
2080
|
stop();
|
|
2928
2081
|
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
|
2929
2082
|
};
|
|
2930
|
-
// Camera attach/detach happens entirely inside the WS message
|
|
2931
|
-
// loop now (sendonly transceiver + `replaceTrack`), so the
|
|
2932
|
-
// connection effect doesn't need to bounce when the camera
|
|
2933
|
-
// turns on/off — no SDP-affecting change.
|
|
2934
|
-
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
2935
2083
|
}, [url, token, propSessionId]);
|
|
2936
2084
|
|
|
2937
2085
|
// Recompute the inspect-overlay geometry (container-local pixel rect of
|
|
@@ -3419,19 +2567,11 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
3419
2567
|
}}
|
|
3420
2568
|
/>
|
|
3421
2569
|
)}
|
|
3422
|
-
{
|
|
3423
|
-
<
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
</div>
|
|
3428
|
-
</div>
|
|
3429
|
-
: retryExhausted && (
|
|
3430
|
-
<button type="button" className="rc-retry-button" onClick={handleManualRetry}>
|
|
3431
|
-
Retry
|
|
3432
|
-
</button>
|
|
3433
|
-
)
|
|
3434
|
-
}
|
|
2570
|
+
{retryExhausted && (
|
|
2571
|
+
<button type="button" className="rc-retry-button" onClick={handleManualRetry}>
|
|
2572
|
+
Retry
|
|
2573
|
+
</button>
|
|
2574
|
+
)}
|
|
3435
2575
|
</div>
|
|
3436
2576
|
);
|
|
3437
2577
|
},
|