@limrun/ui 0.9.0-rc.13 → 0.9.0-rc.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@limrun/ui",
-  "version": "0.9.0-rc.13",
+  "version": "0.9.0-rc.14",
   "publishConfig": {
     "access": "public"
   },

package/src/components/remote-control.tsx

@@ -142,33 +142,25 @@ interface RemoteControlProps {
   axMaxBackoffMs?: number;
 
   /**
-   * Optional outbound camera input.
+   * Fires whenever the iOS simulator's camera demand state changes —
+   * i.e. an app inside the sim called
+   * `[AVCaptureSession startRunning]` or `[stopRunning]`. The
+   * component handles the `navigator.mediaDevices.getUserMedia` prompt
+   * and SDP plumbing internally; this callback is purely so the host
+   * UI can render a status indicator ("simulator is using your
+   * camera", etc.).
    *
-   * When provided, every video track on this `MediaStream` is forwarded
-   * up the same `RTCPeerConnection` that already carries the device
-   * screen downstream. On iOS instances, the limulator-side injector
-   * splices those frames into apps that use AVFoundation, so the user's
-   * browser camera becomes the iOS camera (`AVCaptureDevice.default(for:
-   * .video)`).
+   * `active` reflects whether the sim is currently asking for
+   * frames. `granted` is set only on the call that follows a
+   * `getUserMedia` attempt: `true` when the user accepted the
+   * browser prompt, `false` when they denied or the call failed
+   * (in which case the limulator side switches to a black-frame
+   * fallback so the app keeps ticking).
    *
-   * Today this is only honored by iOS instances launched with the
-   * camera runtime; Android instances ignore it.
-   *
-   * Lifecycle expectations:
-   * - Pass `null`/`undefined` to skip outbound camera (default).
-   * - Pass a `MediaStream` obtained from
-   *   `navigator.mediaDevices.getUserMedia({ video: true })` (or any
-   *   other source — screen capture, virtual camera, etc.).
-   * - Changing the `MediaStream.id` triggers a connection restart so
-   *   the new track is included in the SDP offer. The reference itself
-   *   is allowed to be unstable across renders — we key the reconnect
-   *   on `id`, not object identity, so memoization isn't required.
-   * - Stopping the tracks (e.g. `track.stop()`) on the parent side is
-   *   sufficient to cut the outbound feed without restarting; the peer
-   *   connection just stops getting frames on that track. To fully
-   *   detach the track from the SDP, swap the prop to `null`.
+   * Only iOS instances ever fire this callback; Android instances
+   * have no camera-injector path and stay silent.
    */
-  cameraStream?: MediaStream | null;
+  onCameraDemandChange?: (active: boolean, granted?: boolean) => void;
 }
 
 interface ScreenshotData {
@@ -367,7 +359,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
       onAxStatusChange,
       axPollIntervalMs,
       axMaxBackoffMs,
-      cameraStream,
+      onCameraDemandChange,
     }: RemoteControlProps,
     ref,
   ) => {
@@ -393,13 +385,29 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
     // Mirrored to a ref so stale closures in event handlers see the latest value.
     const autoReconnectRef = useRef(autoReconnect);
     autoReconnectRef.current = autoReconnect;
-    // Outbound camera input. Held in a ref so the deferred WebRTC setup
-    // path picks up the latest stream even if React re-renders between
-    // the connection effect firing and `startAttempt` reaching the
-    // `addTrack` step. Reconnects are keyed on `MediaStream.id`, not
-    // reference identity (see the connection effect's deps).
-    const cameraStreamRef = useRef<MediaStream | null>(cameraStream ?? null);
-    cameraStreamRef.current = cameraStream ?? null;
+    // Demand-driven outbound camera state.
+    //
+    // The limulator side broadcasts a `cameraRequest` WS message whenever
+    // an app inside the simulator opens/closes an `AVCaptureSession`.
+    // We respond by calling `navigator.mediaDevices.getUserMedia(...)`
+    // and `replaceTrack`ing the result onto a pre-allocated sendonly
+    // video transceiver. Pre-allocating the transceiver lets us
+    // attach/detach the local camera without renegotiating the SDP
+    // (the slot is already in the answer's a=video block, just with
+    // `inactive`/empty until we install the track).
+    //
+    // Refs (not state) because all callers live inside event-handler
+    // closures and the ref read happens during message processing, not
+    // during render. We keep both the sender and the active local
+    // stream so teardown can stop tracks without leaking the camera
+    // green light.
+    const outboundCameraSenderRef = useRef<RTCRtpSender | null>(null);
+    const outboundLocalStreamRef = useRef<MediaStream | null>(null);
+    // Mirror the demand-change callback into a ref so the WS message
+    // handler always sees the freshest customer callback even when the
+    // parent re-renders mid-session.
+    const onCameraDemandChangeRef = useRef(onCameraDemandChange);
+    onCameraDemandChangeRef.current = onCameraDemandChange;
     const firstFrameShownRef = useRef(false);
     const pendingScreenshotResolversRef = useRef<
       Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
@@ -1472,6 +1480,32 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
       setVideoLoaded(true);
     };
 
+    // Stop every track on a MediaStream and release the device handle.
+    // The browser keeps the camera "on" indicator lit until at least
+    // one track on the underlying source is stopped, so we have to
+    // call stop() explicitly — closing the peer connection alone
+    // won't do it.
+    const stopMediaStream = (stream: MediaStream) => {
+      for (const track of stream.getTracks()) {
+        try {
+          track.stop();
+        } catch (err) {
+          debugWarn('track.stop() failed:', err);
+        }
+      }
+    };
+
+    // Stop and forget any currently-attached outbound camera stream.
+    // Safe to call when nothing is attached. Does not touch the
+    // sender — the caller is responsible for `replaceTrack(null)`
+    // separately when it wants the SDP slot itself empty.
+    const stopOutboundLocalStream = () => {
+      const stream = outboundLocalStreamRef.current;
+      if (!stream) return;
+      outboundLocalStreamRef.current = null;
+      stopMediaStream(stream);
+    };
+
     const teardownConnection = () => {
       clearConnectionSuccessTimeout();
       clearIceDisconnectedGrace();
@@ -1480,6 +1514,11 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
         axFetcherRef.current.stop();
         axFetcherRef.current = null;
       }
+      // Drop any active outbound camera before the PC dies so the
+      // browser doesn't leave the camera indicator lit between
+      // reconnects.
+      stopOutboundLocalStream();
+      outboundCameraSenderRef.current = null;
       // A scheduled cursor flush would otherwise call setState on a
       // teardown component once the next frame runs.
       if (cursorRafIdRef.current !== undefined) {
@@ -1681,26 +1720,18 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
         peerConnection.addTransceiver('audio', { direction: 'recvonly' });
         const videoTransceiver = peerConnection.addTransceiver('video', { direction: 'recvonly' });
 
-        // Outbound camera input — splice the parent-provided MediaStream's
-        // video tracks into the same PC. `addTrack` creates an implicit
-        // sendonly transceiver and lands the track in the SDP offer; the
-        // limulator side picks it up via `peerConnection(_:didAdd:)` and
-        // forwards every frame into the camera-injector IOSurface ring.
-        // We pass the original MediaStream so the answerer can recover
-        // the parent stream id if it cares; today nothing does, but it
-        // costs nothing to be correct.
-        const outboundCamera = cameraStreamRef.current;
-        if (outboundCamera) {
-          for (const track of outboundCamera.getVideoTracks()) {
-            peerConnection.addTrack(track, outboundCamera);
-          }
-          debugLog(
-            'Attached outbound camera tracks:',
-            outboundCamera.getVideoTracks().length,
-            'streamId:',
-            outboundCamera.id,
-          );
-        }
+        // Pre-allocate a sendonly video slot for the user's camera.
+        // The track stays `null` until the limulator side asks for one
+        // (via the `cameraRequest` WS message), at which point we call
+        // `replaceTrack` with a `getUserMedia` result. Allocating
+        // up-front means we don't have to renegotiate the SDP when the
+        // simulator app actually opens its `AVCaptureSession` — the
+        // codec/SSRC negotiation happens once, here, and turning the
+        // camera on/off later is just a track-replace.
+        const outboundCameraTransceiver = peerConnection.addTransceiver('video', {
+          direction: 'sendonly',
+        });
+        outboundCameraSenderRef.current = outboundCameraTransceiver.sender;
 
         // As hardware encoder, we use H265 for iOS and VP9 for Android.
         // We make sure these two are the first ones in the list.
@@ -2030,6 +2061,85 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
               pendingScreenshotResolversRef.current.delete(message.id);
               pendingScreenshotRejectersRef.current.delete(message.id);
               break;
+            case 'cameraRequest': {
+              const active = message.active === true;
+              if (!active) {
+                // Sim no longer wants frames. Drop our local track and
+                // shut the browser's camera green light off.
+                const sender = outboundCameraSenderRef.current;
+                if (sender) {
+                  try {
+                    await sender.replaceTrack(null);
+                  } catch (err) {
+                    debugWarn('replaceTrack(null) on camera detach failed:', err);
+                  }
+                }
+                stopOutboundLocalStream();
+                safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, false);
+                break;
+              }
+              // Sim is asking for camera. Ask the browser; the user's
+              // prompt response is reported back to the host via a
+              // `cameraResult` message so it can swap to a
+              // black-frame fallback on denial.
+              safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true);
+              let stream: MediaStream | null = null;
+              try {
+                stream = await navigator.mediaDevices.getUserMedia({
+                  video: true,
+                  audio: false,
+                });
+              } catch (err) {
+                debugWarn('getUserMedia denied/failed:', err);
+              }
+              if (!isCurrentAttempt() || wsRef.current !== ws) {
+                if (stream) stopMediaStream(stream);
+                return;
+              }
+              if (!stream) {
+                ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
+                safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
+                break;
+              }
+              // Replace any previous local stream (e.g. an earlier
+              // cameraRequest that resolved with a different device)
+              // before we install the new tracks.
+              stopOutboundLocalStream();
+              outboundLocalStreamRef.current = stream;
+              const sender = outboundCameraSenderRef.current;
+              const videoTrack = stream.getVideoTracks()[0] ?? null;
+              if (!sender || !videoTrack) {
+                if (stream) stopMediaStream(stream);
+                outboundLocalStreamRef.current = null;
+                ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
+                safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
+                break;
+              }
+              try {
+                await sender.replaceTrack(videoTrack);
+              } catch (err) {
+                debugWarn('replaceTrack(videoTrack) failed:', err);
+                stopMediaStream(stream);
+                outboundLocalStreamRef.current = null;
+                ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
+                safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
+                break;
+              }
+              // If the browser revokes the track later (extension,
+              // user clicks Stop in the camera tab UI, etc.), notify
+              // the host so it can switch to black frames.
+              videoTrack.onended = () => {
+                if (outboundLocalStreamRef.current !== stream) return;
+                stopOutboundLocalStream();
+                if (wsRef.current === ws && ws.readyState === WebSocket.OPEN) {
+                  ws.send(JSON.stringify({ type: 'cameraResult', granted: false }));
+                }
+                safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, false);
+              };
+              ws.send(JSON.stringify({ type: 'cameraResult', granted: true }));
+              safeInvoke('onCameraDemandChange', onCameraDemandChangeRef.current, true, true);
+              break;
+            }
             case 'terminateAppResult':
               if (typeof message.id !== 'string') {
                 debugWarn('Received invalid terminateApp result message:', message);
@@ -2138,13 +2248,12 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
         stop();
         document.removeEventListener('visibilitychange', handleVisibilityChange);
       };
-      // We key reconnects on `MediaStream.id` rather than the prop's
-      // object identity so callers don't have to memoize the stream —
-      // turning the camera on/off swaps the underlying MediaStream and
-      // its id, but rerenders that happen to re-create the parent's
-      // wrapping object don't trigger churn.
+      // Camera attach/detach happens entirely inside the WS message
+      // loop now (sendonly transceiver + `replaceTrack`), so the
+      // connection effect doesn't need to bounce when the camera
+      // turns on/off — no SDP-affecting change.
       // eslint-disable-next-line react-hooks/exhaustive-deps
-    }, [url, token, propSessionId, cameraStream?.id]);
+    }, [url, token, propSessionId]);
 
     // Recompute the inspect-overlay geometry (container-local pixel rect of
     // the actually-rendered video content) from the current mapping context.