@limrun/ui 0.16.0 → 0.17.0-rc.1

package/src/components/remote-control.css

@@ -129,3 +129,313 @@
   line-height: 1.3;
   color: #6b7280;
 }
+
+.rc-duo-view {
+  position: absolute;
+  inset: 0;
+}
+.rc-duo-mode {
+  position: absolute;
+  top: 12px;
+  right: 16px;
+  z-index: 3;
+  display: flex;
+  gap: 2px;
+  padding: 3px;
+  background: #fff;
+  border: 1px solid #d9dde4;
+  border-radius: 10px;
+}
+.rc-duo-mode button {
+  border: 0;
+  border-radius: 7px;
+  padding: 7px 13px;
+  font: 12px system-ui;
+  color: #29313e;
+  background: transparent;
+  cursor: pointer;
+}
+.rc-duo-mode button[aria-pressed='true'] {
+  background: #202b3c;
+  color: white;
+}
+.rc-duo-flat-stage {
+  position: relative;
+  flex: 1;
+  min-height: 160px;
+  margin-top: 48px;
+}
+.rc-duo-flat-device {
+  position: absolute;
+  left: 50%;
+  top: 50%;
+  flex: none;
+  transform-origin: center;
+}
+.rc-duo-flat-frame {
+  position: absolute;
+  inset: 0;
+  width: 100%;
+  height: 100%;
+  overflow: visible;
+  pointer-events: none;
+  filter: drop-shadow(0 5px 6px #17202e1c);
+}
+.rc-duo-flat-screen {
+  position: absolute;
+  overflow: hidden;
+  background: #090b0d;
+  touch-action: none;
+  outline: none;
+}
+.rc-duo-flat-screen video,
+.rc-duo-flat-screen canvas {
+  position: absolute;
+  left: 50%;
+  top: 50%;
+  max-width: none;
+  pointer-events: none;
+}
+.rc-duo-flat-stage[data-folding='true'] > .rc-duo-flat-device,
+.rc-duo-flat-stage[data-folding='true'] > .rc-duo-hardware {
+  visibility: hidden;
+  pointer-events: none;
+}
+.rc-duo-fold-hold {
+  position: absolute;
+  left: 50%;
+  top: 50%;
+  pointer-events: none;
+}
+.rc-duo-fold-still {
+  position: relative;
+}
+.rc-duo-fold-canvas {
+  position: absolute;
+  inset: 0;
+  width: 100%;
+  height: 100%;
+  pointer-events: none;
+}
+@media (prefers-reduced-motion: reduce) {
+  .rc-duo-fold-canvas,
+  .rc-duo-fold-hold {
+    display: none;
+  }
+  .rc-duo-flat-stage[data-folding='true'] > .rc-duo-flat-device {
+    visibility: visible;
+  }
+}
+.rc-duo .rc-duo-physical {
+  position: absolute;
+  z-index: 2;
+  padding: 0;
+  border-radius: 0;
+  touch-action: none;
+}
+.rc-duo-physical[data-edge='top'] {
+  transform: translate(-50%, -100%);
+}
+.rc-duo-physical[data-edge='right'] {
+  transform: translate(0, -50%);
+}
+.rc-duo-physical span {
+  position: absolute;
+  display: block;
+  border: 1px solid #7d8387;
+  border-radius: 2px;
+  box-sizing: border-box;
+  background: linear-gradient(180deg, #747d83, #fafcfb 32%, #afb5b9 64%, #636b70);
+  box-shadow: 0 0.5px 1px #0004;
+  transition: transform 110ms ease;
+  pointer-events: none;
+}
+.rc-duo-physical[data-edge='top'] span {
+  bottom: 0;
+  left: 50%;
+  transform: translateX(-50%);
+}
+.rc-duo-physical[data-edge='right'] span {
+  top: 50%;
+  left: 0;
+  transform: translateY(-50%);
+  background: linear-gradient(90deg, #747d83, #fafcfb 32%, #afb5b9 64%, #636b70);
+}
+.rc-duo-physical[data-edge='top'][data-highlighted='true'] span,
+.rc-duo-physical[data-edge='top']:focus-visible span {
+  transform: translate(-50%, -2px);
+}
+.rc-duo-physical[data-edge='right'][data-highlighted='true'] span,
+.rc-duo-physical[data-edge='right']:focus-visible span {
+  transform: translate(2px, -50%);
+}
+.rc-duo-physical[data-edge='top'][data-pressed='true'] span {
+  transform: translate(-50%, 0.5px);
+}
+.rc-duo-physical[data-edge='right'][data-pressed='true'] span {
+  transform: translate(-0.5px, -50%);
+}
+.rc-duo {
+  position: absolute;
+  inset: 0;
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+  background: radial-gradient(ellipse at 50% 40%, #ffffff, #e8ebee);
+  border-radius: 16px;
+  color: #29313e;
+}
+.rc-duo-stage {
+  position: relative;
+  margin-top: 48px;
+  flex: 1;
+  min-height: 160px;
+  width: 100%;
+  outline: none;
+  touch-action: none;
+}
+.rc-duo-stage canvas {
+  display: block;
+  width: 100%;
+  height: 100%;
+  touch-action: none;
+}
+.rc-duo .rc-duo-hardware {
+  position: absolute;
+  display: grid;
+  place-items: center;
+  z-index: 1;
+  width: 44px;
+  height: 44px;
+  padding: 0;
+  transform: translate(-50%, -50%);
+  color: #73777c;
+  opacity: 0;
+  pointer-events: none;
+  transition:
+    opacity 120ms ease,
+    color 100ms ease;
+}
+.rc-duo .rc-duo-hardware[data-visible='true'],
+.rc-duo .rc-duo-hardware:focus-visible {
+  opacity: 1;
+  pointer-events: auto;
+}
+.rc-duo .rc-duo-hardware svg {
+  width: var(--duo-hardware-icon-size, 24px);
+  height: var(--duo-hardware-icon-size, 24px);
+  transform: translate(var(--duo-hardware-icon-x, 0px), var(--duo-hardware-icon-y, 0px));
+}
+.rc-duo .rc-duo-hardware:hover,
+.rc-duo .rc-duo-hardware:focus-visible,
+.rc-duo .rc-duo-hardware[data-pressed='true'] {
+  color: #007aff;
+}
+.rc-duo-controls {
+  flex: none;
+  align-self: center;
+  width: min(540px, calc(100% - 40px));
+  box-sizing: border-box;
+  margin: 0 20px 20px;
+  padding: 12px;
+  background: rgba(255, 255, 255, 0.88);
+  border: 1px solid #d9dde4;
+  border-radius: 16px;
+  box-shadow: 0 6px 25px #13203610;
+  font:
+    12px system-ui,
+    sans-serif;
+}
+.rc-duo-hinge,
+.rc-duo-views {
+  display: flex;
+  align-items: center;
+  gap: 10px;
+  flex-wrap: wrap;
+}
+.rc-duo-hinge {
+  flex-wrap: nowrap;
+  gap: 8px;
+}
+.rc-duo-hinge input {
+  flex: 1;
+  min-width: 30px;
+  width: 100%;
+  accent-color: #34435b;
+}
+.rc-duo-hinge output {
+  min-width: 32px;
+  font-variant-numeric: tabular-nums;
+}
+.rc-duo button {
+  border: 0;
+  padding: 7px 10px;
+  border-radius: 8px;
+  background: transparent;
+  color: inherit;
+  cursor: pointer;
+  font: inherit;
+}
+.rc-duo-hinge .rc-duo-fold {
+  flex: none;
+  background: #202b3c;
+  color: white;
+  min-width: 65px;
+}
+.rc-duo-views {
+  min-height: 29px;
+  margin-top: 8px;
+  gap: 2px;
+  font-size: 11px;
+}
+.rc-duo-views button {
+  padding: 6px 8px;
+}
+.rc-duo-view-label {
+  padding: 6px 8px;
+  color: #77808d;
+}
+.rc-duo .rc-duo-rotate {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  padding: 7px;
+  border: 1px solid #d9dde4;
+}
+.rc-duo button:disabled {
+  cursor: default;
+  opacity: 0.5;
+}
+.rc-duo-views button[aria-pressed='true'] {
+  background: #e5eaf0;
+}
+.rc-duo button:focus-visible,
+.rc-duo input:focus-visible {
+  outline: 2px solid #4275c3;
+  outline-offset: 2px;
+}
+.rc-duo-error {
+  position: absolute;
+  top: 16px;
+  left: 16px;
+  right: 16px;
+  padding: 10px;
+  border-radius: 8px;
+  background: #fff0ee;
+  color: #9c2e26;
+  font: 13px system-ui;
+}
+
+.rc-duo .rc-duo-lock {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  margin-left: auto;
+  border: 1px solid #d3dae3;
+}
+.rc-duo-stage[data-position-locked='false'] canvas {
+  cursor: grab;
+}
+.rc-duo-stage[data-position-locked='false'] canvas:active {
+  cursor: grabbing;
+}

package/src/components/remote-control.tsx

@@ -1,7 +1,19 @@
-import React, { useEffect, useRef, useMemo, useState, forwardRef, useImperativeHandle } from 'react';
+import React, {
+  useEffect,
+  useRef,
+  useMemo,
+  useState,
+  forwardRef,
+  useImperativeHandle,
+  lazy,
+  Suspense,
+} from 'react';
 import { clsx } from 'clsx';
 import './remote-control.css';
 
+import { createDisplayTouchMessage, type DuoState, type DuoOrientation } from '../core/duo';
+const DuoView = lazy(() => import('./duo-view'));
+
 import { ANDROID_KEYS, AMOTION_EVENT, codeMap } from '../core/constants';
 
 import {
@@ -58,6 +70,12 @@ export interface RemoteControlProps {
   // around the video. Defaults to true.
   showFrame?: boolean;
 
+  /** Enables the official iPhone Duo's native displays and interactive folding frame. */
+  deviceModel?: 'iphone-duo';
+
+  /** Optional prepared Duo GLB, supplied and licensed by the host application. */
+  duoModelUrl?: string;
+
   // When true, drops after a working session auto-reconnect instead of
   // surfacing the manual "Retry" button. Defaults to false.
   autoReconnect?: boolean;
@@ -306,7 +324,7 @@ export interface ImperativeKeyboardEvent {
 export interface RemoteControlHandle {
   openUrl: (url: string) => void;
   sendKeyEvent: (event: ImperativeKeyboardEvent) => void;
-  screenshot: () => Promise<ScreenshotData>;
+  screenshot: (display?: 'outer' | 'inner') => Promise<ScreenshotData>;
   terminateApp: (bundleId: string) => Promise<void>;
   reconnect: () => void;
 
@@ -491,6 +509,8 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
       sessionId: propSessionId,
       openUrl,
       showFrame = true,
+      deviceModel,
+      duoModelUrl,
       autoReconnect = false,
       onTerminated,
       assets,
@@ -512,6 +532,32 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
     const containerRef = useRef<HTMLDivElement>(null);
     const videoRef = useRef<HTMLVideoElement>(null);
     const frameRef = useRef<HTMLImageElement>(null);
+    const [detectedDuo, setDetectedDuo] = useState(false);
+    const isDuo = deviceModel === 'iphone-duo' || detectedDuo;
+    const innerVideoRef = useRef<HTMLVideoElement>(null);
+    const [duoState, setDuoState] = useState<DuoState | null>(null);
+    const [duoError, setDuoError] = useState<string>();
+    const duoRequests = useRef(
+      new Map<
+        string,
+        { resolve: () => void; reject: (error: Error) => void; timer: ReturnType<typeof setTimeout> }
+      >(),
+    );
+    const sendDuoControl = (type: string, parameters: object): Promise<void> =>
+      new Promise((resolve, reject) => {
+        const socket = wsRef.current;
+        if (!socket || socket.readyState !== WebSocket.OPEN) {
+          reject(new Error('Simulator disconnected'));
+          return;
+        }
+        const id = crypto.randomUUID();
+        const timer = setTimeout(() => {
+          duoRequests.current.delete(id);
+          reject(new Error('Device control timed out'));
+        }, 5000);
+        duoRequests.current.set(id, { resolve, reject, timer });
+        socket.send(JSON.stringify({ type, id, ...parameters }));
+      });
     const [videoLoaded, setVideoLoaded] = useState(false);
     const [retryExhausted, setRetryExhausted] = useState(false);
     // Set once we've concluded the instance is permanently gone (its
@@ -740,7 +786,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
     const loadingLogo = platformAssets?.loadingLogo;
     // Without a frame image (the lite entry) the video lays out frameless
     // whatever showFrame says.
-    const frameVisible = showFrame && !!frameImageSrc;
+    const frameVisible = !isDuo && showFrame && !!frameImageSrc;
 
     const updateStatus = (message: string) => {
       // Use the wrapper for conditional logging
@@ -1239,6 +1285,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
 
     // Unified handler for both mouse and touch interactions
     const handleInteraction = (event: React.MouseEvent | React.TouchEvent) => {
+      if (isDuo) return;
       event.preventDefault();
       event.stopPropagation();
 
@@ -1688,6 +1735,9 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
     const handleKeyboard = (event: React.KeyboardEvent) => {
       event.preventDefault();
       event.stopPropagation();
+      const inputFocused =
+        document.activeElement === videoRef.current ||
+        (isDuo && document.activeElement === event.currentTarget);
       // Use the wrapper for conditional logging
       debugLog('Keyboard event:', {
         type: event.type,
@@ -1695,12 +1745,12 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
         keyCode: event.keyCode,
         code: event.code,
         target: (event.target as HTMLElement).tagName,
-        focused: document.activeElement === videoRef.current,
+        focused: inputFocused,
       });
 
-      if (document.activeElement !== videoRef.current) {
+      if (!inputFocused) {
         // Use the wrapper for conditional warning
-        debugWarn('Video element not focused, skipping keyboard event');
+        debugWarn('Simulator input not focused, skipping keyboard event');
         return;
       }
 
@@ -2354,6 +2404,13 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
     };
 
     const teardownConnection = () => {
+      for (const request of duoRequests.current.values()) {
+        clearTimeout(request.timer);
+        request.reject(new Error('Simulator disconnected'));
+      }
+      duoRequests.current.clear();
+      setDuoState(null);
+      if (innerVideoRef.current) innerVideoRef.current.srcObject = null;
       clearConnectionSuccessTimeout();
       clearIceDisconnectedGrace();
       stopRequestFrameLoop();
@@ -2644,6 +2701,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
         const rtcConfigPromise = new Promise<{
           rtcConfiguration: RTCConfiguration;
           cameraSupported: boolean;
+          foldSupported: boolean;
         }>((resolve, reject) => {
           const settle = (callback: () => void) => {
             window.clearTimeout(timeoutId);
@@ -2676,6 +2734,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
                   resolve({
                     rtcConfiguration: message.rtcConfiguration,
                     cameraSupported: message.cameraSupported === true,
+                    foldSupported: message.foldSupported === true,
                   }),
                 );
               }
@@ -2694,16 +2753,23 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
           );
         });
 
-        const { rtcConfiguration: rtcConfig, cameraSupported: serverCameraSupported } =
-          await rtcConfigPromise;
+        const {
+          rtcConfiguration: rtcConfig,
+          cameraSupported: serverCameraSupported,
+          foldSupported,
+        } = await rtcConfigPromise;
         if (!isCurrentAttempt() || wsRef.current !== ws) {
           return;
         }
 
+        const isDuo = deviceModel === 'iphone-duo' || foldSupported;
+        setDetectedDuo(isDuo);
         const peerConnection = new RTCPeerConnection(rtcConfig);
         peerConnectionRef.current = peerConnection;
         peerConnection.addTransceiver('audio', { direction: 'recvonly' });
         const videoTransceiver = peerConnection.addTransceiver('video', { direction: 'recvonly' });
+        const duoTransceiver =
+          isDuo ? peerConnection.addTransceiver('video', { direction: 'recvonly' }) : null;
 
         // iOS always supports camera/mic injection; Android servers
         // advertise it through `cameraSupported` (Android 15 and newer).
@@ -2757,6 +2823,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
               return getCodecPriority(a) - getCodecPriority(b);
             });
             videoTransceiver.setCodecPreferences(sortedCodecs);
+            duoTransceiver?.setCodecPreferences(sortedCodecs);
             debugLog('Set codec preferences:', sortedCodecs.map((c) => c.mimeType).join(', '));
           }
         }
@@ -2828,6 +2895,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
           controlChannelOpenedRef.current = true;
           clearConnectionSuccessTimeout();
           updateStatus('Control channel opened');
+          if (isDuo) ws.send(JSON.stringify({ type: 'getFoldState', id: 'duo-capabilities' }));
 
           // Spin up the AX fetcher now that we have a stable WS + control
           // channel. The fetcher's send function reuses this WS; it stops
@@ -2996,7 +3064,14 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
           updateStatus('Received remote track: ' + event.track.kind);
           if (event.track.kind === 'video' && videoRef.current) {
             debugLog(`[${new Date().toISOString()}] Video track received:`, event.track);
-            videoRef.current.srcObject = event.streams[0];
+            const inner =
+              event.track.id === 'duo-inner' || event.streams.some((stream) => stream.id === 'duo-inner');
+            const target = inner ? innerVideoRef.current : videoRef.current;
+            if (target) {
+              target.srcObject =
+                isDuo ? new MediaStream([event.track]) : event.streams[0] ?? new MediaStream([event.track]);
+              void target.play().catch(() => undefined);
+            }
           }
         };
 
@@ -3040,6 +3115,24 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
           }
           updateStatus('Received: ' + message.type);
           switch (message.type) {
+            case 'performActionsResult':
+            case 'foldStateResult':
+            case 'foldStateChanged': {
+              if (!isDuo) break;
+              if (message.state) {
+                setDuoState(message.state);
+                setDuoError(undefined);
+              } else if (message.id === 'duo-capabilities')
+                setDuoError(message.error || 'This instance does not support native iPhone Duo displays.');
+              const request = duoRequests.current.get(message.id);
+              if (request) {
+                clearTimeout(request.timer);
+                duoRequests.current.delete(message.id);
+                if (message.error) request.reject(new Error(message.error));
+                else request.resolve();
+              }
+              break;
+            }
             case 'signalingError': {
               if (message.requestType !== 'offer' || message.sessionId !== sessionId) {
                 break;
@@ -3244,6 +3337,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
             ws.send(
               JSON.stringify({
                 type: 'offer',
+                ...(isDuo ? { displayStreams: true } : {}),
                 sdp: offer.sdp,
                 sessionId: sessionId,
               }),
@@ -3320,7 +3414,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
       // connection effect doesn't need to bounce when the camera
       // turns on/off — no SDP-affecting change.
       // eslint-disable-next-line react-hooks/exhaustive-deps
-    }, [url, token, propSessionId]);
+    }, [url, token, propSessionId, deviceModel]);
 
     // Recompute the inspect-overlay geometry (container-local pixel rect of
     // the actually-rendered video content) from the current mapping context.
@@ -3556,7 +3650,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
           sendBinaryControlMessage(message);
         }
       },
-      screenshot: (): Promise<ScreenshotData> => {
+      screenshot: (display?: 'outer' | 'inner'): Promise<ScreenshotData> => {
         return new Promise<ScreenshotData>((resolve, reject) => {
           if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
             debugWarn('WebSocket not open, cannot send screenshot command.');
@@ -3565,7 +3659,8 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
 
           const id = `ui-ss-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
           const request = {
-            type: 'screenshot', // Matches the type expected by instance API
+            type: display ? 'screenshotDisplay' : 'screenshot',
+            ...(display ? { display } : {}),
             id: id,
           };
 
@@ -3732,6 +3827,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
           )}
           style={{
             ...videoStyle,
+            ...(isDuo ? { opacity: 0, width: 1, height: 1, pointerEvents: 'none' as const } : {}),
             ...(loadingLogo ?
               {
                 backgroundImage: `url("${loadingLogo}")`,
@@ -3744,7 +3840,7 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
           autoPlay
           playsInline
           muted
-          tabIndex={0}
+          tabIndex={isDuo ? -1 : 0}
           onKeyDown={handleKeyboard}
           onKeyUp={handleKeyboard}
           onClick={handleVideoClick}
@@ -3760,7 +3856,47 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
             }
           }}
         />
-        {inspectActive && (
+        {isDuo && (
+          <video
+            ref={innerVideoRef}
+            autoPlay
+            playsInline
+            muted
+            style={{ position: 'absolute', width: 1, height: 1, opacity: 0, pointerEvents: 'none' }}
+          />
+        )}
+        {isDuo && duoState && (
+          <Suspense fallback={<div>Loading iPhone Duo…</div>}>
+            <DuoView
+              modelUrl={duoModelUrl}
+              outer={videoRef.current}
+              inner={innerVideoRef.current}
+              state={duoState}
+              setAngle={(angleDegrees) => sendDuoControl('setHingeAngle', { angleDegrees })}
+              setOrientation={(orientation: DuoOrientation) =>
+                sendDuoControl('setDuoOrientation', { orientation })
+              }
+              button={(button, down) =>
+                sendDuoControl('performActions', {
+                  actions: [{ type: down ? 'buttonDown' : 'buttonUp', button }],
+                })
+              }
+              touch={(action, screenId, x, y) => {
+                const channel = dataChannelRef.current;
+                if (channel?.readyState === 'open')
+                  channel.send(createDisplayTouchMessage(action, screenId, x, y));
+              }}
+              onKeyDown={handleKeyboard}
+              onKeyUp={handleKeyboard}
+            />
+          </Suspense>
+        )}
+        {isDuo && duoError && (
+          <div role="alert" className="rc-duo-error">
+            {duoError}
+          </div>
+        )}
+        {!isDuo && inspectActive && (
           <InspectOverlay
             snapshot={axSnapshot}
             geometry={overlayGeometry}