@limrun/ui 0.15.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/components/remote-control.d.ts +9 -71
- package/dist/core/ax-fetcher.d.ts +5 -0
- package/dist/core/camera.d.ts +58 -0
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +1384 -1336
- package/package.json +2 -3
- package/src/components/remote-control.tsx +259 -389
- package/src/core/ax-fetcher.ts +15 -5
- package/src/core/camera.test.ts +155 -0
- package/src/core/camera.ts +133 -0
- package/src/index.ts +8 -2
|
@@ -18,17 +18,28 @@ import {
|
|
|
18
18
|
createSetClipboardMessage,
|
|
19
19
|
createTwoFingerTouchControlMessage,
|
|
20
20
|
} from '../core/webrtc-messages';
|
|
21
|
-
import { AxFetcher, AxStatus } from '../core/ax-fetcher';
|
|
21
|
+
import { AxFetcher, AxStatus, type AndroidElementTreeOptions } from '../core/ax-fetcher';
|
|
22
22
|
import { AxElement, AxSnapshot, axElementAtPoint, axSnapshotsEqual } from '../core/ax-tree';
|
|
23
|
+
import {
|
|
24
|
+
CameraOperationCoordinator,
|
|
25
|
+
cameraCaptureConstraints,
|
|
26
|
+
createGrantedCameraResult,
|
|
27
|
+
parseCameraRequest,
|
|
28
|
+
shouldReacquireCamera,
|
|
29
|
+
type CameraCaptureInfo,
|
|
30
|
+
type CameraRequest,
|
|
31
|
+
} from '../core/camera';
|
|
23
32
|
import { InspectOverlay, InspectOverlayGeometry, InspectMode } from './inspect-overlay';
|
|
24
33
|
|
|
34
|
+
export type { CameraCaptureInfo, CameraFacingMode, CameraRequest, CameraResult } from '../core/camera';
|
|
35
|
+
|
|
25
36
|
declare global {
|
|
26
37
|
interface Window {
|
|
27
38
|
debugRemoteControl?: boolean;
|
|
28
39
|
}
|
|
29
40
|
}
|
|
30
41
|
|
|
31
|
-
interface RemoteControlProps {
|
|
42
|
+
export interface RemoteControlProps {
|
|
32
43
|
// url is the URL of the instance to connect to.
|
|
33
44
|
url: string;
|
|
34
45
|
|
|
@@ -154,6 +165,8 @@ interface RemoteControlProps {
|
|
|
154
165
|
*/
|
|
155
166
|
axMaxBackoffMs?: number;
|
|
156
167
|
|
|
168
|
+
androidElementTreeOptions?: AndroidElementTreeOptions;
|
|
169
|
+
|
|
157
170
|
/**
|
|
158
171
|
* Fires whenever the iOS simulator's camera demand state changes —
|
|
159
172
|
* i.e. an app inside the sim called
|
|
@@ -163,12 +176,10 @@ interface RemoteControlProps {
|
|
|
163
176
|
* UI can render a status indicator ("simulator is using your
|
|
164
177
|
* camera", etc.).
|
|
165
178
|
*
|
|
166
|
-
* `active` reflects whether the sim is currently asking for
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
* (in which case the limulator side switches to a black-frame
|
|
171
|
-
* fallback so the app keeps ticking).
|
|
179
|
+
* `active` reflects whether the sim is currently asking for frames.
|
|
180
|
+
* `granted` is set on capture acknowledgements: `true` after a new
|
|
181
|
+
* track, reused track, or constraint update; `false` when capture
|
|
182
|
+
* fails. The host switches to black frames after a failure.
|
|
172
183
|
*
|
|
173
184
|
* `camera` (optional, only meaningful when `granted === true`)
|
|
174
185
|
* carries what `MediaStreamTrack.getSettings()` reported for the
|
|
@@ -200,41 +211,6 @@ interface RemoteControlProps {
|
|
|
200
211
|
*/
|
|
201
212
|
onCameraStats?: (stats: CameraStreamStats | null) => void;
|
|
202
213
|
|
|
203
|
-
/**
|
|
204
|
-
* Optional resolution cap for outbound camera capture.
|
|
205
|
-
*
|
|
206
|
-
* - `'auto'` (default): no extra constraint. The browser captures
|
|
207
|
-
* at its webcam's native max; WebRTC's quality scaler may step
|
|
208
|
-
* down resolution on the encoder side under bandwidth pressure
|
|
209
|
-
* while still feeding the simulator at the pool's native size.
|
|
210
|
-
* - `'1080p'` / `'720p'` / `'480p'`: hard cap applied via
|
|
211
|
-
* `getUserMedia` constraints (for new captures) and
|
|
212
|
-
* `track.applyConstraints` (for the currently-active track),
|
|
213
|
-
* matching the way Meet/Zoom expose a "Send resolution" picker.
|
|
214
|
-
*
|
|
215
|
-
* Bumping or lowering the cap mid-stream is supported; the
|
|
216
|
-
* change takes effect within a frame or two as the webcam
|
|
217
|
-
* re-negotiates.
|
|
218
|
-
*/
|
|
219
|
-
cameraResolutionCap?: CameraResolutionCap;
|
|
220
|
-
/**
|
|
221
|
-
* Aspect ratio the simulator's virtual camera should report to apps.
|
|
222
|
-
* Picking a value here triggers a `cameraAspect` WS message to the
|
|
223
|
-
* host, which rebuilds its IOSurface ring at the matching dimensions
|
|
224
|
-
* (16:9 → 1920×1080, 4:3 → 1440×1080, 1:1 → 1080×1080, 9:16 →
|
|
225
|
-
* 1080×1920) and signals the in-sim dylib to re-handshake. iOS apps
|
|
226
|
-
* see CMSampleBuffers at the new dimensions within a frame or two.
|
|
227
|
-
*
|
|
228
|
-
* The browser still captures whatever the webcam offers; the host
|
|
229
|
-
* aspect-fills (cover, center-crop) into the new pool. Switching
|
|
230
|
-
* aspect at runtime is intentionally cheap so users can A/B preview
|
|
231
|
-
* styles without restarting the simulator.
|
|
232
|
-
*
|
|
233
|
-
* `undefined` leaves the pool untouched (the host's boot default —
|
|
234
|
-
* 16:9 / 1920×1080 — applies).
|
|
235
|
-
*/
|
|
236
|
-
cameraAspect?: CameraAspect;
|
|
237
|
-
|
|
238
214
|
/**
|
|
239
215
|
* When true, capture the user's microphone via
|
|
240
216
|
* `getUserMedia({ audio: true })` and stream it live into the
|
|
@@ -273,37 +249,6 @@ export interface MicrophoneState {
|
|
|
273
249
|
label?: string;
|
|
274
250
|
}
|
|
275
251
|
|
|
276
|
-
/**
|
|
277
|
-
* Resolution caps a host app can request on the outbound camera.
|
|
278
|
-
* `'auto'` is "let the browser decide" (no constraints beyond the
|
|
279
|
-
* 30 fps ceiling); the other options clamp width/height to the
|
|
280
|
-
* named target. Aspect ratio is preserved.
|
|
281
|
-
*/
|
|
282
|
-
export type CameraResolutionCap = 'auto' | '1080p' | '720p' | '480p';
|
|
283
|
-
|
|
284
|
-
/**
|
|
285
|
-
* Aspect ratios exposed to the operator for the simulated camera.
|
|
286
|
-
* The host maps each label to concrete IOSurface dimensions; values
|
|
287
|
-
* the host doesn't recognise are silently ignored.
|
|
288
|
-
*/
|
|
289
|
-
export type CameraAspect = '16:9' | '4:3' | '1:1' | '9:16';
|
|
290
|
-
|
|
291
|
-
/**
|
|
292
|
-
* Snapshot of the browser's webcam capture, mirrored from
|
|
293
|
-
* `MediaStreamTrack.getSettings()`. Forwarded to the host alongside
|
|
294
|
-
* `cameraResult` and exposed to the host app via
|
|
295
|
-
* `onCameraDemandChange` so it can render a status indicator without
|
|
296
|
-
* having to call `getStats()` itself.
|
|
297
|
-
*/
|
|
298
|
-
export interface CameraCaptureInfo {
|
|
299
|
-
width?: number;
|
|
300
|
-
height?: number;
|
|
301
|
-
frameRate?: number;
|
|
302
|
-
deviceId?: string;
|
|
303
|
-
label?: string;
|
|
304
|
-
facingMode?: string;
|
|
305
|
-
}
|
|
306
|
-
|
|
307
252
|
/**
|
|
308
253
|
* Live outbound-camera quality snapshot. Derived from
|
|
309
254
|
* `RTCPeerConnection.getStats()` and rate-derived deltas, sampled
|
|
@@ -551,10 +496,9 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
551
496
|
onAxStatusChange,
|
|
552
497
|
axPollIntervalMs,
|
|
553
498
|
axMaxBackoffMs,
|
|
499
|
+
androidElementTreeOptions,
|
|
554
500
|
onCameraDemandChange,
|
|
555
501
|
onCameraStats,
|
|
556
|
-
cameraResolutionCap = 'auto',
|
|
557
|
-
cameraAspect,
|
|
558
502
|
microphoneEnabled = false,
|
|
559
503
|
onMicrophoneStateChange,
|
|
560
504
|
}: RemoteControlProps,
|
|
@@ -607,20 +551,8 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
607
551
|
// green light.
|
|
608
552
|
const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
|
|
609
553
|
const outboundLocalStreamRef = useRef<MediaStream | null>(null);
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
// by a newer request and bail instead of re-attaching the camera.
|
|
613
|
-
const cameraRequestGenerationRef = useRef(0);
|
|
614
|
-
const cameraResolutionCapRef = useRef(cameraResolutionCap);
|
|
615
|
-
cameraResolutionCapRef.current = cameraResolutionCap;
|
|
616
|
-
// The aspect prop also rides a ref so the WS `onopen` reconnect
|
|
617
|
-
// path can replay the operator's last pick without depending on
|
|
618
|
-
// an in-flight render cycle. We mutate it inline (same render-
|
|
619
|
-
// time pattern as the cap ref) so a parent prop change is visible
|
|
620
|
-
// to closures captured during the next render; the useEffect
|
|
621
|
-
// below handles the actual "send to host" side-effect.
|
|
622
|
-
const cameraAspectRef = useRef<CameraAspect | undefined>(cameraAspect);
|
|
623
|
-
cameraAspectRef.current = cameraAspect;
|
|
554
|
+
const outboundCameraRequestRef = useRef<CameraRequest | undefined>(undefined);
|
|
555
|
+
const cameraOperations = useMemo(() => new CameraOperationCoordinator(), []);
|
|
624
556
|
// Mirror the demand-change callback into a ref so the WS message
|
|
625
557
|
// handler always sees the freshest customer callback even when the
|
|
626
558
|
// parent re-renders mid-session.
|
|
@@ -2055,32 +1987,30 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2055
1987
|
});
|
|
2056
1988
|
};
|
|
2057
1989
|
|
|
2058
|
-
//
|
|
2059
|
-
//
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
return { ...base, width: { ideal: 854 }, height: { ideal: 480 } };
|
|
2080
|
-
case 'auto':
|
|
2081
|
-
default:
|
|
2082
|
-
return base;
|
|
1990
|
+
// Publish a fresh settings snapshot after every successful capture
|
|
1991
|
+
// change. This is status metadata; host output sizing is frame-derived.
|
|
1992
|
+
const sendGrantedCameraResult = (ws: WebSocket, videoTrack: MediaStreamTrack) => {
|
|
1993
|
+
let result;
|
|
1994
|
+
try {
|
|
1995
|
+
result = createGrantedCameraResult(videoTrack);
|
|
1996
|
+
// eslint-disable-next-line no-console
|
|
1997
|
+
console.info(
|
|
1998
|
+
`[RemoteControl] camera capture: ${result.camera?.width}x${result.camera?.height}` +
|
|
1999
|
+
` @ ${result.camera?.frameRate ?? '?'}fps` +
|
|
2000
|
+
(result.camera?.label ? ` — ${result.camera.label}` : ''),
|
|
2001
|
+
);
|
|
2002
|
+
} catch (err) {
|
|
2003
|
+
debugWarn('getSettings() on outbound camera track failed:', err);
|
|
2004
|
+
result = {
|
|
2005
|
+
type: 'cameraResult' as const,
|
|
2006
|
+
granted: true,
|
|
2007
|
+
camera: {
|
|
2008
|
+
label: videoTrack.label || undefined,
|
|
2009
|
+
},
|
|
2010
|
+
};
|
|
2083
2011
|
}
|
|
2012
|
+
ws.send(JSON.stringify(result));
|
|
2013
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, result.camera);
|
|
2084
2014
|
};
|
|
2085
2015
|
|
|
2086
2016
|
// Convert the codec mime type (e.g. "video/H264" or "video/HEVC")
|
|
@@ -2192,6 +2122,210 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2192
2122
|
safeInvoke('onCameraStats', onCameraStatsRef.current, null);
|
|
2193
2123
|
};
|
|
2194
2124
|
|
|
2125
|
+
const handleCameraRequest = (
|
|
2126
|
+
request: CameraRequest,
|
|
2127
|
+
ws: WebSocket,
|
|
2128
|
+
sender: RTCRtpSender,
|
|
2129
|
+
isCurrentAttempt: () => boolean,
|
|
2130
|
+
): Promise<void> => {
|
|
2131
|
+
// `enqueue` increments its generation synchronously, so suspended
|
|
2132
|
+
// permission work becomes stale before the newer operation starts.
|
|
2133
|
+
return cameraOperations.enqueue(async ({ isCurrent }) => {
|
|
2134
|
+
const isCurrentCameraRequest = () =>
|
|
2135
|
+
isCurrent() &&
|
|
2136
|
+
isCurrentAttempt() &&
|
|
2137
|
+
wsRef.current === ws &&
|
|
2138
|
+
outboundCameraSenderRef.current === sender;
|
|
2139
|
+
if (!isCurrentCameraRequest()) return;
|
|
2140
|
+
|
|
2141
|
+
if (!request.active) {
|
|
2142
|
+
const stream = outboundLocalStreamRef.current;
|
|
2143
|
+
try {
|
|
2144
|
+
await sender.replaceTrack(null);
|
|
2145
|
+
} catch (err) {
|
|
2146
|
+
debugWarn('replaceTrack(null) on camera detach failed:', err);
|
|
2147
|
+
}
|
|
2148
|
+
if (outboundLocalStreamRef.current === stream) {
|
|
2149
|
+
outboundLocalStreamRef.current = null;
|
|
2150
|
+
outboundCameraRequestRef.current = undefined;
|
|
2151
|
+
if (stream) stopMediaStream(stream);
|
|
2152
|
+
}
|
|
2153
|
+
stopCameraStatsPoller();
|
|
2154
|
+
if (!isCurrentCameraRequest()) return;
|
|
2155
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
|
|
2156
|
+
return;
|
|
2157
|
+
}
|
|
2158
|
+
|
|
2159
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
|
|
2160
|
+
const currentStream = outboundLocalStreamRef.current;
|
|
2161
|
+
const currentTrack = currentStream?.getVideoTracks()[0];
|
|
2162
|
+
const currentRequest = outboundCameraRequestRef.current;
|
|
2163
|
+
|
|
2164
|
+
if (
|
|
2165
|
+
currentTrack?.readyState === 'live' &&
|
|
2166
|
+
currentRequest &&
|
|
2167
|
+
!shouldReacquireCamera(currentRequest, request)
|
|
2168
|
+
) {
|
|
2169
|
+
outboundCameraRequestRef.current = request;
|
|
2170
|
+
sendGrantedCameraResult(ws, currentTrack);
|
|
2171
|
+
startCameraStatsPoller();
|
|
2172
|
+
return;
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
// A facing change always restarts browser capture. Releasing first
|
|
2176
|
+
// lets mobile browsers open the opposite camera; one-camera laptops
|
|
2177
|
+
// naturally return the same available input for the new ideal.
|
|
2178
|
+
if (currentStream) {
|
|
2179
|
+
try {
|
|
2180
|
+
await sender.replaceTrack(null);
|
|
2181
|
+
} catch (err) {
|
|
2182
|
+
debugWarn('replaceTrack(null) before camera switch failed:', err);
|
|
2183
|
+
}
|
|
2184
|
+
if (outboundLocalStreamRef.current === currentStream) {
|
|
2185
|
+
outboundLocalStreamRef.current = null;
|
|
2186
|
+
outboundCameraRequestRef.current = undefined;
|
|
2187
|
+
stopMediaStream(currentStream);
|
|
2188
|
+
}
|
|
2189
|
+
stopCameraStatsPoller();
|
|
2190
|
+
if (!isCurrentCameraRequest()) return;
|
|
2191
|
+
}
|
|
2192
|
+
|
|
2193
|
+
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
2194
|
+
// eslint-disable-next-line no-console
|
|
2195
|
+
console.warn(
|
|
2196
|
+
'[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
|
|
2197
|
+
'getUserMedia requires a secure context (https or http://localhost). ' +
|
|
2198
|
+
'Replying cameraResult granted=false so limulator falls back to black frames.',
|
|
2199
|
+
);
|
|
2200
|
+
if (!isCurrentCameraRequest()) return;
|
|
2201
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2202
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2203
|
+
return;
|
|
2204
|
+
}
|
|
2205
|
+
|
|
2206
|
+
let stream: MediaStream | null = null;
|
|
2207
|
+
try {
|
|
2208
|
+
stream = await navigator.mediaDevices.getUserMedia({
|
|
2209
|
+
video: cameraCaptureConstraints(request),
|
|
2210
|
+
audio: false,
|
|
2211
|
+
});
|
|
2212
|
+
} catch (err) {
|
|
2213
|
+
// eslint-disable-next-line no-console
|
|
2214
|
+
console.warn('[RemoteControl] getUserMedia denied/failed:', err);
|
|
2215
|
+
}
|
|
2216
|
+
if (!isCurrentCameraRequest()) {
|
|
2217
|
+
if (stream) stopMediaStream(stream);
|
|
2218
|
+
return;
|
|
2219
|
+
}
|
|
2220
|
+
if (!stream) {
|
|
2221
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2222
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2223
|
+
return;
|
|
2224
|
+
}
|
|
2225
|
+
|
|
2226
|
+
const videoTrack = stream.getVideoTracks()[0] ?? null;
|
|
2227
|
+
if (!videoTrack) {
|
|
2228
|
+
stopMediaStream(stream);
|
|
2229
|
+
if (!isCurrentCameraRequest()) return;
|
|
2230
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2231
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2232
|
+
return;
|
|
2233
|
+
}
|
|
2234
|
+
|
|
2235
|
+
try {
|
|
2236
|
+
videoTrack.contentHint = 'motion';
|
|
2237
|
+
} catch {
|
|
2238
|
+
/* older browsers don't support contentHint; ignore */
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
try {
|
|
2242
|
+
await sender.replaceTrack(videoTrack);
|
|
2243
|
+
} catch (err) {
|
|
2244
|
+
debugWarn('replaceTrack(videoTrack) failed:', err);
|
|
2245
|
+
stopMediaStream(stream);
|
|
2246
|
+
if (!isCurrentCameraRequest()) return;
|
|
2247
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2248
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2249
|
+
return;
|
|
2250
|
+
}
|
|
2251
|
+
if (!isCurrentCameraRequest()) {
|
|
2252
|
+
try {
|
|
2253
|
+
await sender.replaceTrack(null);
|
|
2254
|
+
} catch (err) {
|
|
2255
|
+
debugWarn('replaceTrack(null) for stale camera attach failed:', err);
|
|
2256
|
+
}
|
|
2257
|
+
stopMediaStream(stream);
|
|
2258
|
+
return;
|
|
2259
|
+
}
|
|
2260
|
+
|
|
2261
|
+
try {
|
|
2262
|
+
const params = sender.getParameters();
|
|
2263
|
+
if (!params.encodings || params.encodings.length === 0) {
|
|
2264
|
+
params.encodings = [{}];
|
|
2265
|
+
}
|
|
2266
|
+
params.encodings[0].maxBitrate = 8_000_000;
|
|
2267
|
+
params.encodings[0].maxFramerate = 30;
|
|
2268
|
+
params.degradationPreference = 'maintain-framerate';
|
|
2269
|
+
await sender.setParameters(params);
|
|
2270
|
+
} catch (err) {
|
|
2271
|
+
debugWarn('setParameters on outbound camera failed:', err);
|
|
2272
|
+
}
|
|
2273
|
+
if (!isCurrentCameraRequest()) {
|
|
2274
|
+
try {
|
|
2275
|
+
await sender.replaceTrack(null);
|
|
2276
|
+
} catch (err) {
|
|
2277
|
+
debugWarn('replaceTrack(null) for stale camera finalize failed:', err);
|
|
2278
|
+
}
|
|
2279
|
+
stopMediaStream(stream);
|
|
2280
|
+
return;
|
|
2281
|
+
}
|
|
2282
|
+
|
|
2283
|
+
outboundLocalStreamRef.current = stream;
|
|
2284
|
+
outboundCameraRequestRef.current = request;
|
|
2285
|
+
|
|
2286
|
+
videoTrack.onended = () => {
|
|
2287
|
+
if (outboundLocalStreamRef.current !== stream) return;
|
|
2288
|
+
void cameraOperations
|
|
2289
|
+
.enqueueCurrent(async ({ isCurrent: isCurrentEndedCleanup }) => {
|
|
2290
|
+
if (
|
|
2291
|
+
!isCurrentEndedCleanup() ||
|
|
2292
|
+
outboundLocalStreamRef.current !== stream ||
|
|
2293
|
+
!isCurrentAttempt() ||
|
|
2294
|
+
wsRef.current !== ws ||
|
|
2295
|
+
outboundCameraSenderRef.current !== sender
|
|
2296
|
+
) {
|
|
2297
|
+
return;
|
|
2298
|
+
}
|
|
2299
|
+
try {
|
|
2300
|
+
await sender.replaceTrack(null);
|
|
2301
|
+
} catch (err) {
|
|
2302
|
+
debugWarn('replaceTrack(null) after camera ended failed:', err);
|
|
2303
|
+
}
|
|
2304
|
+
if (
|
|
2305
|
+
!isCurrentEndedCleanup() ||
|
|
2306
|
+
outboundLocalStreamRef.current !== stream ||
|
|
2307
|
+
!isCurrentAttempt() ||
|
|
2308
|
+
wsRef.current !== ws ||
|
|
2309
|
+
outboundCameraSenderRef.current !== sender
|
|
2310
|
+
) {
|
|
2311
|
+
return;
|
|
2312
|
+
}
|
|
2313
|
+
outboundLocalStreamRef.current = null;
|
|
2314
|
+
outboundCameraRequestRef.current = undefined;
|
|
2315
|
+
stopMediaStream(stream);
|
|
2316
|
+
stopCameraStatsPoller();
|
|
2317
|
+
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
2318
|
+
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
2319
|
+
})
|
|
2320
|
+
.catch((err) => debugWarn('camera ended cleanup failed:', err));
|
|
2321
|
+
};
|
|
2322
|
+
|
|
2323
|
+
if (!isCurrentCameraRequest()) return;
|
|
2324
|
+
sendGrantedCameraResult(ws, videoTrack);
|
|
2325
|
+
startCameraStatsPoller();
|
|
2326
|
+
});
|
|
2327
|
+
};
|
|
2328
|
+
|
|
2195
2329
|
const teardownConnection = () => {
|
|
2196
2330
|
clearConnectionSuccessTimeout();
|
|
2197
2331
|
clearIceDisconnectedGrace();
|
|
@@ -2203,8 +2337,10 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2203
2337
|
// Drop any active outbound camera before the PC dies so the
|
|
2204
2338
|
// browser doesn't leave the camera indicator lit between
|
|
2205
2339
|
// reconnects.
|
|
2340
|
+
cameraOperations.invalidate();
|
|
2206
2341
|
stopCameraStatsPoller();
|
|
2207
2342
|
stopOutboundLocalStream();
|
|
2343
|
+
outboundCameraRequestRef.current = undefined;
|
|
2208
2344
|
outboundCameraSenderRef.current = null;
|
|
2209
2345
|
// Same for the live microphone. Manual enablement survives reconnect;
|
|
2210
2346
|
// guest demand is replayed by the host after it assigns the new session.
|
|
@@ -2430,21 +2566,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2430
2566
|
if (!isCurrentAttempt() || wsRef.current !== ws) {
|
|
2431
2567
|
return;
|
|
2432
2568
|
}
|
|
2433
|
-
// Replay the saved camera aspect for fresh connections
|
|
2434
|
-
// (initial mount, autoreconnect, sim reboot). The host's
|
|
2435
|
-
// streamer starts at its boot default (16:9 / 1920×1080)
|
|
2436
|
-
// and a missing message would mean the operator's pick
|
|
2437
|
-
// silently reverts on reconnect. We always send something
|
|
2438
|
-
// when the prop is set, even if it matches the host
|
|
2439
|
-
// default — the host short-circuits no-op rebuilds.
|
|
2440
|
-
const initialAspect = cameraAspectRef.current;
|
|
2441
|
-
if (initialAspect) {
|
|
2442
|
-
try {
|
|
2443
|
-
ws.send(JSON.stringify({ type: 'cameraAspect', aspect: initialAspect }));
|
|
2444
|
-
} catch (err) {
|
|
2445
|
-
debugWarn('initial cameraAspect send failed:', err);
|
|
2446
|
-
}
|
|
2447
|
-
}
|
|
2448
2569
|
settle(resolve);
|
|
2449
2570
|
};
|
|
2450
2571
|
|
|
@@ -2663,6 +2784,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2663
2784
|
platform,
|
|
2664
2785
|
baseIntervalMs: axPollIntervalMs,
|
|
2665
2786
|
maxBackoffMs: axMaxBackoffMs,
|
|
2787
|
+
androidElementTreeOptions,
|
|
2666
2788
|
send: (payload) => {
|
|
2667
2789
|
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return false;
|
|
2668
2790
|
try {
|
|
@@ -2952,235 +3074,18 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2952
3074
|
// outbound-camera transceiver; without it there is nothing to
|
|
2953
3075
|
// feed the device, so ignore camera requests entirely — no
|
|
2954
3076
|
// getUserMedia prompt and no onCameraDemandChange callbacks.
|
|
2955
|
-
if (!outboundCameraSenderRef.current) {
|
|
2956
|
-
break;
|
|
2957
|
-
}
|
|
2958
|
-
const active = message.active === true;
|
|
2959
|
-
// Bump up front so any earlier in-flight handler bails.
|
|
2960
|
-
const cameraGeneration = ++cameraRequestGenerationRef.current;
|
|
2961
|
-
const isCurrentCameraRequest = () =>
|
|
2962
|
-
isCurrentAttempt() &&
|
|
2963
|
-
wsRef.current === ws &&
|
|
2964
|
-
cameraGeneration === cameraRequestGenerationRef.current;
|
|
2965
|
-
// Log unconditionally — this is one of the few places we
|
|
2966
|
-
// hand control to the browser's permission UI, and a
|
|
2967
|
-
// silent failure here looks identical to "limulator
|
|
2968
|
-
// never asked" from the user's point of view.
|
|
2969
|
-
// eslint-disable-next-line no-console
|
|
2970
|
-
console.info('[RemoteControl] cameraRequest received, active=', active);
|
|
2971
|
-
if (!active) {
|
|
2972
|
-
// Sim no longer wants frames. Drop our local track and
|
|
2973
|
-
// shut the browser's camera green light off.
|
|
2974
|
-
const sender = outboundCameraSenderRef.current;
|
|
2975
|
-
if (sender) {
|
|
2976
|
-
try {
|
|
2977
|
-
await sender.replaceTrack(null);
|
|
2978
|
-
} catch (err) {
|
|
2979
|
-
debugWarn('replaceTrack(null) on camera detach failed:', err);
|
|
2980
|
-
}
|
|
2981
|
-
}
|
|
2982
|
-
stopCameraStatsPoller();
|
|
2983
|
-
stopOutboundLocalStream();
|
|
2984
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
|
|
2985
|
-
break;
|
|
2986
|
-
}
|
|
2987
|
-
// Sim is asking for camera. Ask the browser; the user's
|
|
2988
|
-
// prompt response is reported back to the host via a
|
|
2989
|
-
// `cameraResult` message so it can swap to a
|
|
2990
|
-
// black-frame fallback on denial.
|
|
2991
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
|
|
2992
|
-
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
2993
|
-
// Likely an insecure context (http on a non-localhost
|
|
2994
|
-
// origin) — Chrome strips `mediaDevices` off
|
|
2995
|
-
// `navigator` in that case and the only signal is
|
|
2996
|
-
// this undefined check.
|
|
2997
|
-
// eslint-disable-next-line no-console
|
|
2998
|
-
console.warn(
|
|
2999
|
-
'[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
|
|
3000
|
-
'getUserMedia requires a secure context (https or http://localhost). ' +
|
|
3001
|
-
'Replying cameraResult granted=false so limulator falls back to black frames.',
|
|
3002
|
-
);
|
|
3003
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
3004
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
3005
|
-
break;
|
|
3006
|
-
}
|
|
3007
|
-
let stream: MediaStream | null = null;
|
|
3008
|
-
try {
|
|
3009
|
-
// Capture at the webcam's *native* resolution by
|
|
3010
|
-
// default (no width/height constraints). When the
|
|
3011
|
-
// host app has picked an explicit cap via
|
|
3012
|
-
// `cameraResolutionCap`, we honour it here. Frame
|
|
3013
|
-
// rate is always capped to 30 to match the
|
|
3014
|
-
// simulator pool's pacing.
|
|
3015
|
-
stream = await navigator.mediaDevices.getUserMedia({
|
|
3016
|
-
video: cameraCapToConstraints(cameraResolutionCapRef.current),
|
|
3017
|
-
audio: false,
|
|
3018
|
-
});
|
|
3019
|
-
} catch (err) {
|
|
3020
|
-
// Surface unconditionally — the user is the only one
|
|
3021
|
-
// who can fix this (permission denied, no device,
|
|
3022
|
-
// dismissed prompt, etc.). `debugWarn` alone would
|
|
3023
|
-
// hide it in normal builds.
|
|
3024
|
-
// eslint-disable-next-line no-console
|
|
3025
|
-
console.warn('[RemoteControl] getUserMedia denied/failed:', err);
|
|
3026
|
-
}
|
|
3027
|
-
if (!isCurrentCameraRequest()) {
|
|
3028
|
-
// Superseded during the prompt; drop the stream we got.
|
|
3029
|
-
if (stream) stopMediaStream(stream);
|
|
3030
|
-
return;
|
|
3031
|
-
}
|
|
3032
|
-
if (!stream) {
|
|
3033
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
3034
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
3035
|
-
break;
|
|
3036
|
-
}
|
|
3037
|
-
// Replace any previous local stream (e.g. an earlier
|
|
3038
|
-
// cameraRequest that resolved with a different device)
|
|
3039
|
-
// before we install the new tracks.
|
|
3040
|
-
stopOutboundLocalStream();
|
|
3041
|
-
outboundLocalStreamRef.current = stream;
|
|
3042
3077
|
const sender = outboundCameraSenderRef.current;
|
|
3043
|
-
|
|
3044
|
-
if (!sender || !videoTrack) {
|
|
3045
|
-
if (stream) stopMediaStream(stream);
|
|
3046
|
-
outboundLocalStreamRef.current = null;
|
|
3047
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
3048
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
3078
|
+
if (!sender) {
|
|
3049
3079
|
break;
|
|
3050
3080
|
}
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
debugWarn('replaceTrack(videoTrack) failed:', err);
|
|
3055
|
-
stopMediaStream(stream);
|
|
3056
|
-
outboundLocalStreamRef.current = null;
|
|
3057
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
3058
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
3081
|
+
const request = parseCameraRequest(message);
|
|
3082
|
+
if (!request) {
|
|
3083
|
+
debugWarn('Received invalid cameraRequest:', message);
|
|
3059
3084
|
break;
|
|
3060
3085
|
}
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
await sender.replaceTrack(null);
|
|
3065
|
-
} catch (err) {
|
|
3066
|
-
debugWarn('replaceTrack(null) on stale camera attach failed:', err);
|
|
3067
|
-
}
|
|
3068
|
-
stopMediaStream(stream);
|
|
3069
|
-
if (outboundLocalStreamRef.current === stream) {
|
|
3070
|
-
outboundLocalStreamRef.current = null;
|
|
3071
|
-
}
|
|
3072
|
-
return;
|
|
3073
|
-
}
|
|
3074
|
-
// Tell the encoder this is real motion content (a
|
|
3075
|
-
// physical camera), not text/slides. With `'motion'`
|
|
3076
|
-
// VideoToolbox / libvpx pick latency-friendly tuning
|
|
3077
|
-
// (no extra B-frames, shorter GoP smoothing). Set
|
|
3078
|
-
// before the first frame so the encoder init reads it.
|
|
3079
|
-
try {
|
|
3080
|
-
videoTrack.contentHint = 'motion';
|
|
3081
|
-
} catch {
|
|
3082
|
-
/* older browsers don't support contentHint; ignore */
|
|
3083
|
-
}
|
|
3084
|
-
// Bound the outbound bitrate generously and let
|
|
3085
|
-
// WebRTC's congestion control (BWE) find the floor.
|
|
3086
|
-
// 8 Mbps is comfortable for native 1080p30 webcam
|
|
3087
|
-
// content over LAN — the encoder will use far less when
|
|
3088
|
-
// the scene is static, and BWE will throttle if a
|
|
3089
|
-
// hop is constrained. Pair with
|
|
3090
|
-
// `maintain-framerate` so the quality scaler steps
|
|
3091
|
-
// down resolution before dropping frames, matching how
|
|
3092
|
-
// Meet/Zoom degrade.
|
|
3093
|
-
try {
|
|
3094
|
-
const params = sender.getParameters();
|
|
3095
|
-
if (!params.encodings || params.encodings.length === 0) {
|
|
3096
|
-
params.encodings = [{}];
|
|
3097
|
-
}
|
|
3098
|
-
params.encodings[0].maxBitrate = 8_000_000;
|
|
3099
|
-
params.encodings[0].maxFramerate = 30;
|
|
3100
|
-
params.degradationPreference = 'maintain-framerate';
|
|
3101
|
-
await sender.setParameters(params);
|
|
3102
|
-
} catch (err) {
|
|
3103
|
-
debugWarn('setParameters on outbound camera failed:', err);
|
|
3104
|
-
}
|
|
3105
|
-
// Surface the actual captured geometry to the host so
|
|
3106
|
-
// it can size its IOSurface pool / picture-format
|
|
3107
|
-
// metadata to match. `getSettings()` returns what the
|
|
3108
|
-
// browser actually picked — which may differ from any
|
|
3109
|
-
// hints we sent in the constraints — including the
|
|
3110
|
-
// selected `deviceId` / `label` (useful when a user has
|
|
3111
|
-
// multiple cameras and we eventually expose a picker).
|
|
3112
|
-
let cameraMetadata: {
|
|
3113
|
-
width?: number;
|
|
3114
|
-
height?: number;
|
|
3115
|
-
frameRate?: number;
|
|
3116
|
-
deviceId?: string;
|
|
3117
|
-
label?: string;
|
|
3118
|
-
facingMode?: string;
|
|
3119
|
-
} = {};
|
|
3120
|
-
try {
|
|
3121
|
-
const settings = videoTrack.getSettings();
|
|
3122
|
-
cameraMetadata = {
|
|
3123
|
-
width: settings.width,
|
|
3124
|
-
height: settings.height,
|
|
3125
|
-
frameRate: settings.frameRate,
|
|
3126
|
-
deviceId: settings.deviceId,
|
|
3127
|
-
label: videoTrack.label || undefined,
|
|
3128
|
-
facingMode: settings.facingMode,
|
|
3129
|
-
};
|
|
3130
|
-
// eslint-disable-next-line no-console
|
|
3131
|
-
console.info(
|
|
3132
|
-
`[RemoteControl] camera capture: ${cameraMetadata.width}x${cameraMetadata.height}` +
|
|
3133
|
-
` @ ${cameraMetadata.frameRate ?? '?'}fps` +
|
|
3134
|
-
(cameraMetadata.label ? ` — ${cameraMetadata.label}` : ''),
|
|
3135
|
-
);
|
|
3136
|
-
} catch (err) {
|
|
3137
|
-
debugWarn('getSettings() on outbound camera track failed:', err);
|
|
3138
|
-
}
|
|
3139
|
-
// If the browser revokes the track later (extension,
|
|
3140
|
-
// user clicks Stop in the camera tab UI, etc.), notify
|
|
3141
|
-
// the host so it can switch to black frames.
|
|
3142
|
-
videoTrack.onended = () => {
|
|
3143
|
-
if (outboundLocalStreamRef.current !== stream) return;
|
|
3144
|
-
stopCameraStatsPoller();
|
|
3145
|
-
stopOutboundLocalStream();
|
|
3146
|
-
if (wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
|
|
3147
|
-
ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
|
|
3148
|
-
}
|
|
3149
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
|
|
3150
|
-
};
|
|
3151
|
-
if (!isCurrentCameraRequest()) {
|
|
3152
|
-
// Superseded during setParameters; detach and bail.
|
|
3153
|
-
videoTrack.onended = null;
|
|
3154
|
-
try {
|
|
3155
|
-
await sender.replaceTrack(null);
|
|
3156
|
-
} catch (err) {
|
|
3157
|
-
debugWarn('replaceTrack(null) on stale camera finalize failed:', err);
|
|
3158
|
-
}
|
|
3159
|
-
stopMediaStream(stream);
|
|
3160
|
-
if (outboundLocalStreamRef.current === stream) {
|
|
3161
|
-
outboundLocalStreamRef.current = null;
|
|
3162
|
-
}
|
|
3163
|
-
return;
|
|
3164
|
-
}
|
|
3165
|
-
ws.send(
|
|
3166
|
-
JSON.stringify({
|
|
3167
|
-
type: 'cameraResult',
|
|
3168
|
-
granted: true,
|
|
3169
|
-
// Forward what the browser actually captured so the
|
|
3170
|
-
// host can size its IOSurface pool, log the device,
|
|
3171
|
-
// and (eventually) surface a status pill / picker.
|
|
3172
|
-
camera: cameraMetadata,
|
|
3173
|
-
}),
|
|
3174
|
-
);
|
|
3175
|
-
safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true, cameraMetadata);
|
|
3176
|
-
// Kick off the per-second outbound stats sampler. We do
|
|
3177
|
-
// this *after* `setParameters` so the first sample
|
|
3178
|
-
// already sees the encoder under its final bitrate /
|
|
3179
|
-
// degradation policy, and *after* the host-side
|
|
3180
|
-
// attachInboundTrack will have wired up (the
|
|
3181
|
-
// cameraResult ACK is what triggers it on the host),
|
|
3182
|
-
// so framesEncoded starts climbing immediately.
|
|
3183
|
-
startCameraStatsPoller();
|
|
3086
|
+
// eslint-disable-next-line no-console
|
|
3087
|
+
console.info('[RemoteControl] cameraRequest received:', request.active, request.facingMode);
|
|
3088
|
+
await handleCameraRequest(request, ws, sender, isCurrentAttempt);
|
|
3184
3089
|
break;
|
|
3185
3090
|
}
|
|
3186
3091
|
case 'microphoneRequest': {
|
|
@@ -3313,41 +3218,6 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
3313
3218
|
start();
|
|
3314
3219
|
};
|
|
3315
3220
|
|
|
3316
|
-
// Re-apply the resolution cap on the currently-sending track
|
|
3317
|
-
// whenever the host app changes its preference. Skips when no
|
|
3318
|
-
// camera is active — the next `getUserMedia` will pick up the
|
|
3319
|
-
// new value via `cameraCapToConstraints` automatically.
|
|
3320
|
-
useEffect(() => {
|
|
3321
|
-
const stream = outboundLocalStreamRef.current;
|
|
3322
|
-
if (!stream) return;
|
|
3323
|
-
const track = stream.getVideoTracks()[0];
|
|
3324
|
-
if (!track) return;
|
|
3325
|
-
const constraints = cameraCapToConstraints(cameraResolutionCap);
|
|
3326
|
-
track.applyConstraints(constraints).catch((err) => {
|
|
3327
|
-
debugWarn('applyConstraints for new camera cap failed:', err);
|
|
3328
|
-
});
|
|
3329
|
-
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
3330
|
-
}, [cameraResolutionCap]);
|
|
3331
|
-
|
|
3332
|
-
// Push the aspect preference to the host whenever it changes.
|
|
3333
|
-
// The host rebuilds its IOSurface pool and bumps `pool_generation`
|
|
3334
|
-
// on its end; the dylib re-handshakes on next sem_wait. No
|
|
3335
|
-
// peer-connection renegotiation is needed — the aspect change is
|
|
3336
|
-
// purely about the pixel buffer dimensions iOS apps observe, not
|
|
3337
|
-
// about WebRTC track layout. The `cameraAspectRef` is mirrored
|
|
3338
|
-
// higher up so the WS `onopen` reconnect path can replay the
|
|
3339
|
-
// latest value on fresh connections.
|
|
3340
|
-
useEffect(() => {
|
|
3341
|
-
if (!cameraAspect) return;
|
|
3342
|
-
const ws = wsRef.current;
|
|
3343
|
-
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
|
3344
|
-
try {
|
|
3345
|
-
ws.send(JSON.stringify({ type: 'cameraAspect', aspect: cameraAspect }));
|
|
3346
|
-
} catch (err) {
|
|
3347
|
-
debugWarn('cameraAspect send failed:', err);
|
|
3348
|
-
}
|
|
3349
|
-
}, [cameraAspect]);
|
|
3350
|
-
|
|
3351
3221
|
// Drive the manual override without interrupting guest-driven demand.
|
|
3352
3222
|
useEffect(() => {
|
|
3353
3223
|
if (isMicrophoneWanted()) {
|