@limrun/ui 0.9.0-rc.14 → 0.9.0-rc.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/components/remote-control.d.ts +125 -1
- package/dist/core/device-install/apple/provisioning.d.ts +55 -4
- package/dist/core/device-install/operations/limbuild-client.d.ts +15 -1
- package/dist/core/device-install/storage/browser-storage.d.ts +9 -5
- package/dist/core/device-install/types.d.ts +4 -1
- package/dist/device-install/index.cjs +1 -1
- package/dist/device-install/index.js +70 -64
- package/dist/device-install/react.cjs +1 -1
- package/dist/device-install/react.js +1 -1
- package/dist/device-install-dialog-DY35un0b.js +9 -0
- package/dist/device-install-dialog-chNLeiiL.mjs +2000 -0
- package/dist/device-install-dialog.css +1 -1
- package/dist/hooks/use-device-install.d.ts +5 -1
- package/dist/index.cjs +1 -1
- package/dist/index.js +1286 -1116
- package/dist/use-device-install-BIrl0v-k.js +31 -0
- package/dist/{use-device-install-sDVvby1V.mjs → use-device-install-LGfEdqyM.mjs} +4388 -4271
- package/package.json +4 -2
- package/src/components/device-install/device-install-dialog.css +29 -0
- package/src/components/device-install/device-install-dialog.tsx +91 -30
- package/src/components/remote-control.tsx +535 -5
- package/src/core/device-install/apple/provisioning.test.ts +84 -0
- package/src/core/device-install/apple/provisioning.ts +91 -7
- package/src/core/device-install/operations/limbuild-client.ts +32 -2
- package/src/core/device-install/storage/browser-storage.ts +29 -14
- package/src/core/device-install/types.ts +5 -1
- package/src/hooks/use-device-install.ts +135 -59
- package/dist/device-install-dialog-CjH25hnN.js +0 -2
- package/dist/device-install-dialog-W5Xv9kWL.mjs +0 -443
- package/dist/use-device-install-Y1u6vIBB.js +0 -31
|
@@ -157,10 +157,144 @@ interface RemoteControlProps {
|
|
|
157
157
|
* (in which case the limulator side switches to a black-frame
|
|
158
158
|
* fallback so the app keeps ticking).
|
|
159
159
|
*
|
|
160
|
+
* `camera` (optional, only meaningful when `granted === true`)
|
|
161
|
+
* carries what `MediaStreamTrack.getSettings()` reported for the
|
|
162
|
+
* active capture: resolution, framerate, device label, facing
|
|
163
|
+
* mode. Use it to render a richer status indicator (e.g.
|
|
164
|
+
* "Camera · 1920×1080 · 30 fps · FaceTime HD").
|
|
165
|
+
*
|
|
160
166
|
* Only iOS instances ever fire this callback; Android instances
|
|
161
167
|
* have no camera-injector path and stay silent.
|
|
162
168
|
*/
|
|
163
|
-
onCameraDemandChange?: (
|
|
169
|
+
onCameraDemandChange?: (
|
|
170
|
+
active: boolean,
|
|
171
|
+
granted?: boolean,
|
|
172
|
+
camera?: CameraCaptureInfo,
|
|
173
|
+
) => void;
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Periodically (~1Hz) fires with a snapshot of the outbound
|
|
177
|
+
* camera stream's live WebRTC stats — codec, encoder
|
|
178
|
+
* implementation, hardware acceleration, encoded fps, bitrate,
|
|
179
|
+
* round-trip-time, and the encoder's
|
|
180
|
+
* `qualityLimitationReason` (which is what Meet/Zoom use to
|
|
181
|
+
* decide whether to show "Bandwidth limited" or "CPU limited"
|
|
182
|
+
* banners).
|
|
183
|
+
*
|
|
184
|
+
* Only fires while the simulator is actively pulling the
|
|
185
|
+
* camera AND `getUserMedia` was granted. `null` is emitted
|
|
186
|
+
* once when the stream goes back to idle so consumers can
|
|
187
|
+
* clear their UI without having to maintain their own
|
|
188
|
+
* timeout. The host app does not need to poll `getStats()`
|
|
189
|
+
* itself — this is the canonical place.
|
|
190
|
+
*/
|
|
191
|
+
onCameraStats?: (stats: CameraStreamStats | null) => void;
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Optional resolution cap for outbound camera capture.
|
|
195
|
+
*
|
|
196
|
+
* - `'auto'` (default): no extra constraint. The browser captures
|
|
197
|
+
* at its webcam's native max; WebRTC's quality scaler may step
|
|
198
|
+
* down resolution on the encoder side under bandwidth pressure
|
|
199
|
+
* while still feeding the simulator at the pool's native size.
|
|
200
|
+
* - `'1080p'` / `'720p'` / `'480p'`: hard cap applied via
|
|
201
|
+
* `getUserMedia` constraints (for new captures) and
|
|
202
|
+
* `track.applyConstraints` (for the currently-active track),
|
|
203
|
+
* matching the way Meet/Zoom expose a "Send resolution" picker.
|
|
204
|
+
*
|
|
205
|
+
* Bumping or lowering the cap mid-stream is supported; the
|
|
206
|
+
* change takes effect within a frame or two as the webcam
|
|
207
|
+
* re-negotiates.
|
|
208
|
+
*/
|
|
209
|
+
cameraResolutionCap?: CameraResolutionCap;
|
|
210
|
+
/**
|
|
211
|
+
* Aspect ratio the simulator's virtual camera should report to apps.
|
|
212
|
+
* Picking a value here triggers a `cameraAspect` WS message to the
|
|
213
|
+
* host, which rebuilds its IOSurface ring at the matching dimensions
|
|
214
|
+
* (16:9 → 1920×1080, 4:3 → 1440×1080, 1:1 → 1080×1080, 9:16 →
|
|
215
|
+
* 1080×1920) and signals the in-sim dylib to re-handshake. iOS apps
|
|
216
|
+
* see CMSampleBuffers at the new dimensions within a frame or two.
|
|
217
|
+
*
|
|
218
|
+
* The browser still captures whatever the webcam offers; the host
|
|
219
|
+
* aspect-fills (cover, center-crop) into the new pool. Switching
|
|
220
|
+
* aspect at runtime is intentionally cheap so users can A/B preview
|
|
221
|
+
* styles without restarting the simulator.
|
|
222
|
+
*
|
|
223
|
+
* `undefined` leaves the pool untouched (the host's boot default —
|
|
224
|
+
* 16:9 / 1920×1080 — applies).
|
|
225
|
+
*/
|
|
226
|
+
cameraAspect?: CameraAspect;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Resolution caps a host app can request on the outbound camera.
|
|
231
|
+
* `'auto'` is "let the browser decide" (no constraints beyond the
|
|
232
|
+
* 30 fps ceiling); the other options clamp width/height to the
|
|
233
|
+
* named target. Aspect ratio is preserved.
|
|
234
|
+
*/
|
|
235
|
+
export type CameraResolutionCap = 'auto' | '1080p' | '720p' | '480p';
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Aspect ratios exposed to the operator for the simulated camera.
|
|
239
|
+
* The host maps each label to concrete IOSurface dimensions; values
|
|
240
|
+
* the host doesn't recognise are silently ignored.
|
|
241
|
+
*/
|
|
242
|
+
export type CameraAspect = '16:9' | '4:3' | '1:1' | '9:16';
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Snapshot of the browser's webcam capture, mirrored from
|
|
246
|
+
* `MediaStreamTrack.getSettings()`. Forwarded to the host alongside
|
|
247
|
+
* `cameraResult` and exposed to the host app via
|
|
248
|
+
* `onCameraDemandChange` so it can render a status indicator without
|
|
249
|
+
* having to call `getStats()` itself.
|
|
250
|
+
*/
|
|
251
|
+
export interface CameraCaptureInfo {
|
|
252
|
+
width?: number;
|
|
253
|
+
height?: number;
|
|
254
|
+
frameRate?: number;
|
|
255
|
+
deviceId?: string;
|
|
256
|
+
label?: string;
|
|
257
|
+
facingMode?: string;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Live outbound-camera quality snapshot. Derived from
|
|
262
|
+
* `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
|
|
263
|
+
* once per second while the camera is sending. All fields optional:
|
|
264
|
+
* some browsers omit fields (Safari rarely reports
|
|
265
|
+
* `encoderImplementation`), and the first sample after camera start
|
|
266
|
+
* has no delta-derived numbers yet.
|
|
267
|
+
*/
|
|
268
|
+
export interface CameraStreamStats {
|
|
269
|
+
/** "H264", "HEVC"/"H265", "VP9", "VP8", "AV1", etc. (uppercased). */
|
|
270
|
+
codec?: string;
|
|
271
|
+
/** e.g. "VideoToolbox" (hw), "OpenH264" (sw), "ExternalEncoder". */
|
|
272
|
+
encoderImplementation?: string;
|
|
273
|
+
/**
|
|
274
|
+
* Browser-reported hardware-acceleration hint. Some Chromium
|
|
275
|
+
* versions expose this via `powerEfficientEncoder`; we mirror it
|
|
276
|
+
* here so consumers don't have to know the spec quirk.
|
|
277
|
+
*/
|
|
278
|
+
hardwareAccelerated?: boolean;
|
|
279
|
+
/** Outbound encoded fps over the last sample window. */
|
|
280
|
+
framesPerSecond?: number;
|
|
281
|
+
/** Cumulative encoded frame count. */
|
|
282
|
+
framesEncoded?: number;
|
|
283
|
+
/** Outbound encoded bitrate (bits/s) over the last window. */
|
|
284
|
+
bitrateBps?: number;
|
|
285
|
+
/** Width/height the encoder is currently producing. */
|
|
286
|
+
width?: number;
|
|
287
|
+
height?: number;
|
|
288
|
+
/**
|
|
289
|
+
* One of `'none' | 'cpu' | 'bandwidth' | 'other'`. Mirrors
|
|
290
|
+
* Meet/Zoom's "limited by …" banners. Anything other than
|
|
291
|
+
* `'none'` means the encoder dropped resolution to keep up.
|
|
292
|
+
*/
|
|
293
|
+
qualityLimitationReason?: string;
|
|
294
|
+
/** Round-trip time in milliseconds, from the matching RTCP. */
|
|
295
|
+
rttMs?: number;
|
|
296
|
+
/** Packet-loss percentage over the last sample window (0..100). */
|
|
297
|
+
packetsLostPct?: number;
|
|
164
298
|
}
|
|
165
299
|
|
|
166
300
|
interface ScreenshotData {
|
|
@@ -360,6 +494,9 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
360
494
|
axPollIntervalMs,
|
|
361
495
|
axMaxBackoffMs,
|
|
362
496
|
onCameraDemandChange,
|
|
497
|
+
onCameraStats,
|
|
498
|
+
cameraResolutionCap = 'auto',
|
|
499
|
+
cameraAspect,
|
|
363
500
|
}: RemoteControlProps,
|
|
364
501
|
ref,
|
|
365
502
|
) => {
|
|
@@ -403,11 +540,36 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
403
540
|
// green light.
|
|
404
541
|
const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
|
|
405
542
|
const outboundLocalStreamRef = useRef<MediaStream | null>(null);
|
|
543
|
+
const cameraResolutionCapRef = useRef(cameraResolutionCap);
|
|
544
|
+
cameraResolutionCapRef.current = cameraResolutionCap;
|
|
545
|
+
// The aspect prop also rides a ref so the WS `onopen` reconnect
|
|
546
|
+
// path can replay the operator's last pick without depending on
|
|
547
|
+
// an in-flight render cycle. We mutate it inline (same render-
|
|
548
|
+
// time pattern as the cap ref) so a parent prop change is visible
|
|
549
|
+
// to closures captured during the next render; the useEffect
|
|
550
|
+
// below handles the actual "send to host" side-effect.
|
|
551
|
+
const cameraAspectRef = useRef<CameraAspect | undefined>(cameraAspect);
|
|
552
|
+
cameraAspectRef.current = cameraAspect;
|
|
406
553
|
// Mirror the demand-change callback into a ref so the WS message
|
|
407
554
|
// handler always sees the freshest customer callback even when the
|
|
408
555
|
// parent re-renders mid-session.
|
|
409
556
|
const onCameraDemandChangeRef = useRef(onCameraDemandChange);
|
|
410
557
|
onCameraDemandChangeRef.current = onCameraDemandChange;
|
|
558
|
+
const onCameraStatsRef = useRef(onCameraStats);
|
|
559
|
+
onCameraStatsRef.current = onCameraStats;
|
|
560
|
+
// Active outbound-camera stats poller. While the camera is
|
|
561
|
+
// sending we sample `RTCPeerConnection.getStats()` once per
|
|
562
|
+
// second, derive deltas (bitrate, fps, packet loss) from the
|
|
563
|
+
// previous sample, and push the result through `onCameraStats`.
|
|
564
|
+
// Cleared on teardown so we never leak the timer.
|
|
565
|
+
const cameraStatsTimerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
|
566
|
+
const cameraStatsPrevRef = useRef<{
|
|
567
|
+
timestamp: number;
|
|
568
|
+
framesEncoded?: number;
|
|
569
|
+
bytesSent?: number;
|
|
570
|
+
packetsSent?: number;
|
|
571
|
+
packetsLost?: number;
|
|
572
|
+
} | null>(null);
|
|
411
573
|
const firstFrameShownRef = useRef(false);
|
|
412
574
|
const pendingScreenshotResolversRef = useRef<
|
|
413
575
|
Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
|
|
@@ -1506,6 +1668,150 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1506
1668
|
stopMediaStream(stream);
|
|
1507
1669
|
};
|
|
1508
1670
|
|
|
1671
|
+
// Translate the user-facing resolution cap into a
|
|
1672
|
+
// MediaTrackConstraints fragment we can feed `getUserMedia`
|
|
1673
|
+
// or `applyConstraints`. Always returns a 30 fps ceiling so
|
|
1674
|
+
// a 60 fps webcam doesn't double our encoder cost for no
|
|
1675
|
+
// visible win (the simulator pool is paced at 30).
|
|
1676
|
+
//
|
|
1677
|
+
// We use `ideal` rather than `max` so that a webcam with a
|
|
1678
|
+
// native 720p mode still hands us 720p when the user picks
|
|
1679
|
+
// 1080p — instead of refusing the constraint outright. The
|
|
1680
|
+
// browser's NotReadableError on a too-strict `max` is the
|
|
1681
|
+
// most common camera-permission gotcha in the wild.
|
|
1682
|
+
const cameraCapToConstraints = (cap: CameraResolutionCap): MediaTrackConstraints => {
|
|
1683
|
+
const base: MediaTrackConstraints = {
|
|
1684
|
+
frameRate: { ideal: 30, max: 30 },
|
|
1685
|
+
};
|
|
1686
|
+
switch (cap) {
|
|
1687
|
+
case '1080p':
|
|
1688
|
+
return { ...base, width: { ideal: 1920 }, height: { ideal: 1080 } };
|
|
1689
|
+
case '720p':
|
|
1690
|
+
return { ...base, width: { ideal: 1280 }, height: { ideal: 720 } };
|
|
1691
|
+
case '480p':
|
|
1692
|
+
return { ...base, width: { ideal: 854 }, height: { ideal: 480 } };
|
|
1693
|
+
case 'auto':
|
|
1694
|
+
default:
|
|
1695
|
+
return base;
|
|
1696
|
+
}
|
|
1697
|
+
};
|
|
1698
|
+
|
|
1699
|
+
// Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
|
|
1700
|
+
// into a short uppercase label suitable for a status badge.
|
|
1701
|
+
const shortenCodecMime = (mime: string | undefined): string | undefined => {
|
|
1702
|
+
if (!mime) return undefined;
|
|
1703
|
+
const slash = mime.indexOf('/');
|
|
1704
|
+
const tail = slash >= 0 ? mime.slice(slash + 1) : mime;
|
|
1705
|
+
const upper = tail.toUpperCase();
|
|
1706
|
+
if (upper === 'HEVC') return 'H265';
|
|
1707
|
+
return upper;
|
|
1708
|
+
};
|
|
1709
|
+
|
|
1710
|
+
// Sample outbound-camera stats once and push a normalised
|
|
1711
|
+
// snapshot through `onCameraStats`. Skips silently if the sender
|
|
1712
|
+
// has been torn down between intervals.
|
|
1713
|
+
const sampleOutboundCameraStats = async () => {
|
|
1714
|
+
const sender = outboundCameraSenderRef.current;
|
|
1715
|
+
const handler = onCameraStatsRef.current;
|
|
1716
|
+
if (!sender || !handler) return;
|
|
1717
|
+
let report: RTCStatsReport;
|
|
1718
|
+
try {
|
|
1719
|
+
report = await sender.getStats();
|
|
1720
|
+
} catch {
|
|
1721
|
+
return;
|
|
1722
|
+
}
|
|
1723
|
+
let outbound: any | undefined;
|
|
1724
|
+
let codecMime: string | undefined;
|
|
1725
|
+
let remoteInbound: any | undefined;
|
|
1726
|
+
report.forEach((entry: any) => {
|
|
1727
|
+
if (entry.type === 'outbound-rtp' && entry.kind === 'video') {
|
|
1728
|
+
outbound = entry;
|
|
1729
|
+
} else if (entry.type === 'remote-inbound-rtp' && entry.kind === 'video') {
|
|
1730
|
+
remoteInbound = entry;
|
|
1731
|
+
}
|
|
1732
|
+
});
|
|
1733
|
+
if (outbound?.codecId) {
|
|
1734
|
+
const codec = report.get(outbound.codecId);
|
|
1735
|
+
if (codec) codecMime = (codec as any).mimeType;
|
|
1736
|
+
}
|
|
1737
|
+
if (!outbound) return;
|
|
1738
|
+
const now = (outbound.timestamp as number) ?? Date.now();
|
|
1739
|
+
const prev = cameraStatsPrevRef.current;
|
|
1740
|
+
let fps: number | undefined;
|
|
1741
|
+
let bitrate: number | undefined;
|
|
1742
|
+
let lossPct: number | undefined;
|
|
1743
|
+
if (prev && now > prev.timestamp) {
|
|
1744
|
+
const dt = (now - prev.timestamp) / 1000;
|
|
1745
|
+
if (
|
|
1746
|
+
typeof outbound.framesEncoded === 'number' &&
|
|
1747
|
+
typeof prev.framesEncoded === 'number'
|
|
1748
|
+
) {
|
|
1749
|
+
fps = Math.max(0, (outbound.framesEncoded - prev.framesEncoded) / dt);
|
|
1750
|
+
}
|
|
1751
|
+
if (typeof outbound.bytesSent === 'number' && typeof prev.bytesSent === 'number') {
|
|
1752
|
+
bitrate = Math.max(0, ((outbound.bytesSent - prev.bytesSent) * 8) / dt);
|
|
1753
|
+
}
|
|
1754
|
+
if (
|
|
1755
|
+
typeof outbound.packetsSent === 'number' &&
|
|
1756
|
+
typeof prev.packetsSent === 'number' &&
|
|
1757
|
+
remoteInbound &&
|
|
1758
|
+
typeof remoteInbound.packetsLost === 'number' &&
|
|
1759
|
+
typeof prev.packetsLost === 'number'
|
|
1760
|
+
) {
|
|
1761
|
+
const sent = outbound.packetsSent - prev.packetsSent;
|
|
1762
|
+
const lost = remoteInbound.packetsLost - prev.packetsLost;
|
|
1763
|
+
if (sent > 0) lossPct = Math.max(0, Math.min(100, (lost / sent) * 100));
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
cameraStatsPrevRef.current = {
|
|
1767
|
+
timestamp: now,
|
|
1768
|
+
framesEncoded: outbound.framesEncoded,
|
|
1769
|
+
bytesSent: outbound.bytesSent,
|
|
1770
|
+
packetsSent: outbound.packetsSent,
|
|
1771
|
+
packetsLost: remoteInbound?.packetsLost,
|
|
1772
|
+
};
|
|
1773
|
+
const stats: CameraStreamStats = {
|
|
1774
|
+
codec: shortenCodecMime(codecMime),
|
|
1775
|
+
encoderImplementation: outbound.encoderImplementation,
|
|
1776
|
+
hardwareAccelerated:
|
|
1777
|
+
typeof outbound.powerEfficientEncoder === 'boolean'
|
|
1778
|
+
? outbound.powerEfficientEncoder
|
|
1779
|
+
: undefined,
|
|
1780
|
+
framesPerSecond: fps,
|
|
1781
|
+
framesEncoded: outbound.framesEncoded,
|
|
1782
|
+
bitrateBps: bitrate,
|
|
1783
|
+
width: outbound.frameWidth,
|
|
1784
|
+
height: outbound.frameHeight,
|
|
1785
|
+
qualityLimitationReason: outbound.qualityLimitationReason,
|
|
1786
|
+
rttMs:
|
|
1787
|
+
typeof remoteInbound?.roundTripTime === 'number'
|
|
1788
|
+
? remoteInbound.roundTripTime * 1000
|
|
1789
|
+
: undefined,
|
|
1790
|
+
packetsLostPct: lossPct,
|
|
1791
|
+
};
|
|
1792
|
+
safeInvoke('onCameraStats', onCameraStatsRef.current, stats);
|
|
1793
|
+
};
|
|
1794
|
+
|
|
1795
|
+
const startCameraStatsPoller = () => {
|
|
1796
|
+
if (cameraStatsTimerRef.current !== null) return;
|
|
1797
|
+
cameraStatsPrevRef.current = null;
|
|
1798
|
+
// First sample fires after one interval — the very first
|
|
1799
|
+
// sample has no deltas to compare against, which is fine: the
|
|
1800
|
+
// codec/encoder identity is still useful on its own.
|
|
1801
|
+
cameraStatsTimerRef.current = setInterval(() => {
|
|
1802
|
+
void sampleOutboundCameraStats();
|
|
1803
|
+
}, 1000);
|
|
1804
|
+
};
|
|
1805
|
+
|
|
1806
|
+
const stopCameraStatsPoller = () => {
|
|
1807
|
+
if (cameraStatsTimerRef.current !== null) {
|
|
1808
|
+
clearInterval(cameraStatsTimerRef.current);
|
|
1809
|
+
cameraStatsTimerRef.current = null;
|
|
1810
|
+
}
|
|
1811
|
+
cameraStatsPrevRef.current = null;
|
|
1812
|
+
safeInvoke('onCameraStats', onCameraStatsRef.current, null);
|
|
1813
|
+
};
|
|
1814
|
+
|
|
1509
1815
|
const teardownConnection = () => {
|
|
1510
1816
|
clearConnectionSuccessTimeout();
|
|
1511
1817
|
clearIceDisconnectedGrace();
|
|
@@ -1517,6 +1823,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1517
1823
|
// Drop any active outbound camera before the PC dies so the
|
|
1518
1824
|
// browser doesn't leave the camera indicator lit between
|
|
1519
1825
|
// reconnects.
|
|
1826
|
+
stopCameraStatsPoller();
|
|
1520
1827
|
stopOutboundLocalStream();
|
|
1521
1828
|
outboundCameraSenderRef.current = null;
|
|
1522
1829
|
// A scheduled cursor flush would otherwise call setState on a
|
|
@@ -1644,6 +1951,21 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1644
1951
|
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
1645
1952
|
return;
|
|
1646
1953
|
}
|
|
1954
|
+
// Replay the saved camera aspect for fresh connections
|
|
1955
|
+
// (initial mount, autoreconnect, sim reboot). The host's
|
|
1956
|
+
// streamer starts at its boot default (16:9 / 1920×1080)
|
|
1957
|
+
// and a missing message would mean the operator's pick
|
|
1958
|
+
// silently reverts on reconnect. We always send something
|
|
1959
|
+
// when the prop is set, even if it matches the host
|
|
1960
|
+
// default — the host short-circuits no-op rebuilds.
|
|
1961
|
+
const initialAspect = cameraAspectRef.current;
|
|
1962
|
+
if (initialAspect) {
|
|
1963
|
+
try {
|
|
1964
|
+
ws.send(JSON.stringify({ type: 'cameraAspect', aspect: initialAspect }));
|
|
1965
|
+
} catch (err) {
|
|
1966
|
+
debugWarn('initial cameraAspect send failed:', err);
|
|
1967
|
+
}
|
|
1968
|
+
}
|
|
1647
1969
|
settle(resolve);
|
|
1648
1970
|
};
|
|
1649
1971
|
|
|
@@ -1758,6 +2080,59 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1758
2080
|
}
|
|
1759
2081
|
}
|
|
1760
2082
|
|
|
2083
|
+
// Pin the outbound camera transceiver's codec order with
|
|
2084
|
+
// negotiation-time fallback. Preference order:
|
|
2085
|
+
// 1. H.265 / HEVC — VideoToolbox HW on M-series + recent
|
|
2086
|
+
// Chrome. ~30% less bitrate at equal quality. If the
|
|
2087
|
+
// answerer (limulator) supports HEVC, this wins.
|
|
2088
|
+
// 2. H.264 with VideoToolbox-friendly profile-level-ids
|
|
2089
|
+
// (42e01f / 42e028 / 640c1f). Universally HW-accelerated
|
|
2090
|
+
// on every Mac since 2009. SDP-time fallback when HEVC
|
|
2091
|
+
// isn't in the answer.
|
|
2092
|
+
// 3. Any other H.264 profile. Often resolves to OpenH264
|
|
2093
|
+
// (software) so we keep it explicitly last among H.264
|
|
2094
|
+
// to avoid Chrome's default pick.
|
|
2095
|
+
// 4. VP9 / VP8 (libvpx software on Mac). Last resort.
|
|
2096
|
+
//
|
|
2097
|
+
// Caveat: this is *negotiation* fallback. If HEVC HW encoder
|
|
2098
|
+
// init fails at session start or gets demoted mid-stream,
|
|
2099
|
+
// Chrome falls back to *software* HEVC, not H.264 — no spec
|
|
2100
|
+
// hook for runtime re-negotiation. The host-side stats
|
|
2101
|
+
// logger will surface this as decoder=ffmpeg / hw=false.
|
|
2102
|
+
if (RTCRtpSender.getCapabilities) {
|
|
2103
|
+
const senderCaps = RTCRtpSender.getCapabilities('video');
|
|
2104
|
+
if (senderCaps && senderCaps.codecs) {
|
|
2105
|
+
const vtH264Profiles = new Set(['42e01f', '42e028', '640c1f']);
|
|
2106
|
+
const score = (c: { mimeType: string; sdpFmtpLine?: string }): number => {
|
|
2107
|
+
const mime = c.mimeType.toLowerCase();
|
|
2108
|
+
const fmtp = (c.sdpFmtpLine || '').toLowerCase();
|
|
2109
|
+
const profileMatch = fmtp.match(/profile-level-id=([0-9a-f]{6})/);
|
|
2110
|
+
const profile = profileMatch ? profileMatch[1] : '';
|
|
2111
|
+
if (mime === 'video/h265' || mime === 'video/hevc') return 1;
|
|
2112
|
+
if (mime === 'video/h264' && vtH264Profiles.has(profile)) return 2;
|
|
2113
|
+
if (mime === 'video/h264') return 3;
|
|
2114
|
+
if (mime === 'video/vp9') return 4;
|
|
2115
|
+
if (mime === 'video/vp8') return 5;
|
|
2116
|
+
if (mime === 'video/rtx' || mime === 'video/red' || mime === 'video/ulpfec') {
|
|
2117
|
+
return 0; // Keep RTX/FEC available alongside everything else.
|
|
2118
|
+
}
|
|
2119
|
+
return 6;
|
|
2120
|
+
};
|
|
2121
|
+
const sortedSendCodecs = [...senderCaps.codecs].sort((a, b) => score(a) - score(b));
|
|
2122
|
+
try {
|
|
2123
|
+
outboundCameraTransceiver.setCodecPreferences(sortedSendCodecs);
|
|
2124
|
+
debugLog(
|
|
2125
|
+
'Outbound camera codec preferences:',
|
|
2126
|
+
sortedSendCodecs
|
|
2127
|
+
.map((c) => `${c.mimeType}${c.sdpFmtpLine ? `[${c.sdpFmtpLine}]` : ''}`)
|
|
2128
|
+
.join(', '),
|
|
2129
|
+
);
|
|
2130
|
+
} catch (err) {
|
|
2131
|
+
debugWarn('Failed to set outbound camera codec preferences:', err);
|
|
2132
|
+
}
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
|
|
1761
2136
|
const dataChannel = peerConnection.createDataChannel('control', {
|
|
1762
2137
|
ordered: true,
|
|
1763
2138
|
negotiated: true,
|
|
@@ -2063,6 +2438,12 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2063
2438
|
break;
|
|
2064
2439
|
case 'cameraRequest': {
|
|
2065
2440
|
const active = message.active === true;
|
|
2441
|
+
// Log unconditionally — this is one of the few places we
|
|
2442
|
+
// hand control to the browser's permission UI, and a
|
|
2443
|
+
// silent failure here looks identical to "limulator
|
|
2444
|
+
// never asked" from the user's point of view.
|
|
2445
|
+
// eslint-disable-next-line no-console
|
|
2446
|
+
console.info('[RemoteControl] cameraRequest received, active=', active);
|
|
2066
2447
|
if (!active) {
|
|
2067
2448
|
// Sim no longer wants frames. Drop our local track and
|
|
2068
2449
|
// shut the browser's camera green light off.
|
|
@@ -2074,6 +2455,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2074
2455
|
debugWarn('replaceTrack(null) on camera detach failed:', err);
|
|
2075
2456
|
}
|
|
2076
2457
|
}
|
|
2458
|
+
stopCameraStatsPoller();
|
|
2077
2459
|
stopOutboundLocalStream();
|
|
2078
2460
|
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
|
|
2079
2461
|
break;
|
|
@@ -2083,14 +2465,40 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2083
2465
|
// `cameraResult` message so it can swap to a
|
|
2084
2466
|
// black-frame fallback on denial.
|
|
2085
2467
|
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
|
|
2468
|
+
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
2469
|
+
// Likely an insecure context (http on a non-localhost
|
|
2470
|
+
// origin) — Chrome strips `mediaDevices` off
|
|
2471
|
+
// `navigator` in that case and the only signal is
|
|
2472
|
+
// this undefined check.
|
|
2473
|
+
// eslint-disable-next-line no-console
|
|
2474
|
+
console.warn(
|
|
2475
|
+
'[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
|
|
2476
|
+
'getUserMedia requires a secure context (https or http://localhost). ' +
|
|
2477
|
+
'Replying cameraResult granted=false so limulator falls back to black frames.',
|
|
2478
|
+
);
|
|
2479
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2480
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2481
|
+
break;
|
|
2482
|
+
}
|
|
2086
2483
|
let stream: MediaStream | null = null;
|
|
2087
2484
|
try {
|
|
2485
|
+
// Capture at the webcam's *native* resolution by
|
|
2486
|
+
// default (no width/height constraints). When the
|
|
2487
|
+
// host app has picked an explicit cap via
|
|
2488
|
+
// `cameraResolutionCap`, we honour it here. Frame
|
|
2489
|
+
// rate is always capped to 30 to match the
|
|
2490
|
+
// simulator pool's pacing.
|
|
2088
2491
|
stream = await navigator.mediaDevices.getUserMedia({
|
|
2089
|
-
video:
|
|
2492
|
+
video: cameraCapToConstraints(cameraResolutionCapRef.current),
|
|
2090
2493
|
audio: false,
|
|
2091
2494
|
});
|
|
2092
2495
|
} catch (err) {
|
|
2093
|
-
|
|
2496
|
+
// Surface unconditionally — the user is the only one
|
|
2497
|
+
// who can fix this (permission denied, no device,
|
|
2498
|
+
// dismissed prompt, etc.). `debugWarn` alone would
|
|
2499
|
+
// hide it in normal builds.
|
|
2500
|
+
// eslint-disable-next-line no-console
|
|
2501
|
+
console.warn('[RemoteControl] getUserMedia denied/failed:', err);
|
|
2094
2502
|
}
|
|
2095
2503
|
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2096
2504
|
if (stream) stopMediaStream(stream);
|
|
@@ -2125,19 +2533,106 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2125
2533
|
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2126
2534
|
break;
|
|
2127
2535
|
}
|
|
2536
|
+
// Tell the encoder this is real motion content (a
|
|
2537
|
+
// physical camera), not text/slides. With `'motion'`
|
|
2538
|
+
// VideoToolbox / libvpx pick latency-friendly tuning
|
|
2539
|
+
// (no extra B-frames, shorter GoP smoothing). Set
|
|
2540
|
+
// before the first frame so the encoder init reads it.
|
|
2541
|
+
try {
|
|
2542
|
+
videoTrack.contentHint = 'motion';
|
|
2543
|
+
} catch {
|
|
2544
|
+
/* older browsers don't support contentHint; ignore */
|
|
2545
|
+
}
|
|
2546
|
+
// Bound the outbound bitrate generously and let
|
|
2547
|
+
// WebRTC's congestion control (BWE) find the floor.
|
|
2548
|
+
// 8 Mbps is comfortable for native 1080p30 webcam
|
|
2549
|
+
// content over LAN — the encoder will use far less when
|
|
2550
|
+
// the scene is static, and BWE will throttle if a
|
|
2551
|
+
// hop is constrained. Pair with
|
|
2552
|
+
// `maintain-framerate` so the quality scaler steps
|
|
2553
|
+
// down resolution before dropping frames, matching how
|
|
2554
|
+
// Meet/Zoom degrade.
|
|
2555
|
+
try {
|
|
2556
|
+
const params = sender.getParameters();
|
|
2557
|
+
if (!params.encodings || params.encodings.length === 0) {
|
|
2558
|
+
params.encodings = [{}];
|
|
2559
|
+
}
|
|
2560
|
+
params.encodings[0].maxBitrate = 8_000_000;
|
|
2561
|
+
params.encodings[0].maxFramerate = 30;
|
|
2562
|
+
params.degradationPreference = 'maintain-framerate';
|
|
2563
|
+
await sender.setParameters(params);
|
|
2564
|
+
} catch (err) {
|
|
2565
|
+
debugWarn('setParameters on outbound camera failed:', err);
|
|
2566
|
+
}
|
|
2567
|
+
// Surface the actual captured geometry to the host so
|
|
2568
|
+
// it can size its IOSurface pool / picture-format
|
|
2569
|
+
// metadata to match. `getSettings()` returns what the
|
|
2570
|
+
// browser actually picked — which may differ from any
|
|
2571
|
+
// hints we sent in the constraints — including the
|
|
2572
|
+
// selected `deviceId` / `label` (useful when a user has
|
|
2573
|
+
// multiple cameras and we eventually expose a picker).
|
|
2574
|
+
let cameraMetadata: {
|
|
2575
|
+
width?: number;
|
|
2576
|
+
height?: number;
|
|
2577
|
+
frameRate?: number;
|
|
2578
|
+
deviceId?: string;
|
|
2579
|
+
label?: string;
|
|
2580
|
+
facingMode?: string;
|
|
2581
|
+
} = {};
|
|
2582
|
+
try {
|
|
2583
|
+
const settings = videoTrack.getSettings();
|
|
2584
|
+
cameraMetadata = {
|
|
2585
|
+
width: settings.width,
|
|
2586
|
+
height: settings.height,
|
|
2587
|
+
frameRate: settings.frameRate,
|
|
2588
|
+
deviceId: settings.deviceId,
|
|
2589
|
+
label: videoTrack.label || undefined,
|
|
2590
|
+
facingMode: settings.facingMode,
|
|
2591
|
+
};
|
|
2592
|
+
// eslint-disable-next-line no-console
|
|
2593
|
+
console.info(
|
|
2594
|
+
`[RemoteControl] camera capture: ${cameraMetadata.width}x${cameraMetadata.height}` +
|
|
2595
|
+
` @ ${cameraMetadata.frameRate ?? '?'}fps` +
|
|
2596
|
+
(cameraMetadata.label ? ` — ${cameraMetadata.label}` : ''),
|
|
2597
|
+
);
|
|
2598
|
+
} catch (err) {
|
|
2599
|
+
debugWarn('getSettings() on outbound camera track failed:', err);
|
|
2600
|
+
}
|
|
2128
2601
|
// If the browser revokes the track later (extension,
|
|
2129
2602
|
// user clicks Stop in the camera tab UI, etc.), notify
|
|
2130
2603
|
// the host so it can switch to black frames.
|
|
2131
2604
|
videoTrack.onended = () => {
|
|
2132
2605
|
if (outboundLocalStreamRef.current !== stream) return;
|
|
2606
|
+
stopCameraStatsPoller();
|
|
2133
2607
|
stopOutboundLocalStream();
|
|
2134
2608
|
if (wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
|
|
2135
2609
|
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2136
2610
|
}
|
|
2137
2611
|
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2138
2612
|
};
|
|
2139
|
-
ws.send(JSON.stringify({
|
|
2140
|
-
|
|
2613
|
+
ws.send(JSON.stringify({
|
|
2614
|
+
type: 'cameraResult',
|
|
2615
|
+
granted: true,
|
|
2616
|
+
// Forward what the browser actually captured so the
|
|
2617
|
+
// host can size its IOSurface pool, log the device,
|
|
2618
|
+
// and (eventually) surface a status pill / picker.
|
|
2619
|
+
camera: cameraMetadata,
|
|
2620
|
+
}));
|
|
2621
|
+
safeInvoke(
|
|
2622
|
+
'onCameraDemandChange',
|
|
2623
|
+
onCameraDemandChangeRef.current,
|
|
2624
|
+
true,
|
|
2625
|
+
true,
|
|
2626
|
+
cameraMetadata,
|
|
2627
|
+
);
|
|
2628
|
+
// Kick off the per-second outbound stats sampler. We do
|
|
2629
|
+
// this *after* `setParameters` so the first sample
|
|
2630
|
+
// already sees the encoder under its final bitrate /
|
|
2631
|
+
// degradation policy, and *after* the host-side
|
|
2632
|
+
// attachInboundTrack will have wired up (the
|
|
2633
|
+
// cameraResult ACK is what triggers it on the host),
|
|
2634
|
+
// so framesEncoded starts climbing immediately.
|
|
2635
|
+
startCameraStatsPoller();
|
|
2141
2636
|
break;
|
|
2142
2637
|
}
|
|
2143
2638
|
case 'terminateAppResult':
|
|
@@ -2227,6 +2722,41 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2227
2722
|
start();
|
|
2228
2723
|
};
|
|
2229
2724
|
|
|
2725
|
+
// Re-apply the resolution cap on the currently-sending track
|
|
2726
|
+
// whenever the host app changes its preference. Skips when no
|
|
2727
|
+
// camera is active — the next `getUserMedia` will pick up the
|
|
2728
|
+
// new value via `cameraCapToConstraints` automatically.
|
|
2729
|
+
useEffect(() => {
|
|
2730
|
+
const stream = outboundLocalStreamRef.current;
|
|
2731
|
+
if (!stream) return;
|
|
2732
|
+
const track = stream.getVideoTracks()[0];
|
|
2733
|
+
if (!track) return;
|
|
2734
|
+
const constraints = cameraCapToConstraints(cameraResolutionCap);
|
|
2735
|
+
track.applyConstraints(constraints).catch((err) => {
|
|
2736
|
+
debugWarn('applyConstraints for new camera cap failed:', err);
|
|
2737
|
+
});
|
|
2738
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
2739
|
+
}, [cameraResolutionCap]);
|
|
2740
|
+
|
|
2741
|
+
// Push the aspect preference to the host whenever it changes.
|
|
2742
|
+
// The host rebuilds its IOSurface pool and bumps `pool_generation`
|
|
2743
|
+
// on its end; the dylib re-handshakes on next sem_wait. No
|
|
2744
|
+
// peer-connection renegotiation is needed — the aspect change is
|
|
2745
|
+
// purely about the pixel buffer dimensions iOS apps observe, not
|
|
2746
|
+
// about WebRTC track layout. The `cameraAspectRef` is mirrored
|
|
2747
|
+
// higher up so the WS `onopen` reconnect path can replay the
|
|
2748
|
+
// latest value on fresh connections.
|
|
2749
|
+
useEffect(() => {
|
|
2750
|
+
if (!cameraAspect) return;
|
|
2751
|
+
const ws = wsRef.current;
|
|
2752
|
+
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
|
2753
|
+
try {
|
|
2754
|
+
ws.send(JSON.stringify({ type: 'cameraAspect', aspect: cameraAspect }));
|
|
2755
|
+
} catch (err) {
|
|
2756
|
+
debugWarn('cameraAspect send failed:', err);
|
|
2757
|
+
}
|
|
2758
|
+
}, [cameraAspect]);
|
|
2759
|
+
|
|
2230
2760
|
useEffect(() => {
|
|
2231
2761
|
// Reset video loaded state when connection params change
|
|
2232
2762
|
setVideoLoaded(false);
|