@limrun/ui 0.14.0-rc.5 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -26
- package/dist/components/remote-control.d.ts +36 -0
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1351 -1196
- package/package.json +10 -51
- package/src/components/remote-control.tsx +414 -2
- package/src/index.ts +1 -1
- package/vite.config.ts +0 -9
- package/dist/app-store-relay/index.cjs +0 -9
- package/dist/app-store-relay/index.d.ts +0 -204
- package/dist/app-store-relay/index.js +0 -677
- package/dist/app-store-relay/react.cjs +0 -1
- package/dist/app-store-relay/react.d.ts +0 -15
- package/dist/app-store-relay/react.js +0 -69
- package/dist/apple-credentials/index.cjs +0 -1
- package/dist/apple-credentials/index.d.ts +0 -110
- package/dist/apple-credentials/index.js +0 -291
- package/dist/browser-storage-BnO-Ma-e.mjs +0 -219
- package/dist/browser-storage-DLxIqZk5.js +0 -1
- package/dist/client-C9z_sgOf.js +0 -1
- package/dist/client-CFDB4mX5.mjs +0 -191
- package/dist/core/device-install/apple/client.d.ts +0 -18
- package/dist/core/device-install/apple/crypto.d.ts +0 -20
- package/dist/core/device-install/apple/gsa-srp.d.ts +0 -26
- package/dist/core/device-install/apple/index.d.ts +0 -5
- package/dist/core/device-install/apple/provisioning.d.ts +0 -237
- package/dist/core/device-install/apple/relay.d.ts +0 -53
- package/dist/core/device-install/index.d.ts +0 -4
- package/dist/core/device-install/operations/index.d.ts +0 -6
- package/dist/core/device-install/operations/limbuild-client.d.ts +0 -43
- package/dist/core/device-install/operations/operations.d.ts +0 -47
- package/dist/core/device-install/operations/relay-client.d.ts +0 -26
- package/dist/core/device-install/operations/relay-protocol.d.ts +0 -27
- package/dist/core/device-install/operations/usbmux.d.ts +0 -32
- package/dist/core/device-install/operations/webusb.d.ts +0 -21
- package/dist/core/device-install/storage/browser-storage.d.ts +0 -56
- package/dist/core/device-install/storage/index.d.ts +0 -2
- package/dist/core/device-install/storage/secret-store.d.ts +0 -124
- package/dist/core/device-install/types.d.ts +0 -57
- package/dist/core/errors.d.ts +0 -1
- package/dist/core/play-publish/google.d.ts +0 -13
- package/dist/core/play-publish/publish.d.ts +0 -34
- package/dist/device-build/index.cjs +0 -1
- package/dist/device-build/index.d.ts +0 -4
- package/dist/device-build/index.js +0 -85
- package/dist/device-build/react.cjs +0 -1
- package/dist/device-build/react.d.ts +0 -21
- package/dist/device-build/react.js +0 -68
- package/dist/device-build/signing.d.ts +0 -20
- package/dist/device-install/demo/demo.d.ts +0 -0
- package/dist/device-install/index.cjs +0 -1
- package/dist/device-install/index.d.ts +0 -18
- package/dist/device-install/index.js +0 -594
- package/dist/device-install/react.cjs +0 -1
- package/dist/device-install/react.d.ts +0 -24
- package/dist/device-install/react.js +0 -105
- package/dist/errors-BSkqS7U-.js +0 -1
- package/dist/errors-gy8yQJos.mjs +0 -6
- package/dist/gsa-srp-Bme3JLEi.mjs +0 -266
- package/dist/gsa-srp-CCIfUz7Y.js +0 -1
- package/dist/index-ChhtuQ7e.js +0 -22
- package/dist/index-YDu8K9a4.mjs +0 -11279
- package/dist/limbuild-client-BagK_FWE.mjs +0 -83
- package/dist/limbuild-client-CVDwq54z.js +0 -1
- package/dist/play-publish/index.cjs +0 -1
- package/dist/play-publish/index.d.ts +0 -2
- package/dist/play-publish/index.js +0 -109
- package/dist/play-publish/react.cjs +0 -1
- package/dist/play-publish/react.d.ts +0 -31
- package/dist/play-publish/react.js +0 -91
- package/dist/provisioning-CDPvxd1h.mjs +0 -270
- package/dist/provisioning-CbC4h--7.js +0 -1
- package/src/app-store-relay/index.test.ts +0 -60
- package/src/app-store-relay/index.ts +0 -846
- package/src/app-store-relay/react.ts +0 -131
- package/src/apple-credentials/index.test.ts +0 -180
- package/src/apple-credentials/index.ts +0 -388
- package/src/core/device-install/apple/client.ts +0 -150
- package/src/core/device-install/apple/crypto.ts +0 -205
- package/src/core/device-install/apple/gsa-srp.ts +0 -123
- package/src/core/device-install/apple/index.ts +0 -5
- package/src/core/device-install/apple/provisioning.test.ts +0 -114
- package/src/core/device-install/apple/provisioning.ts +0 -419
- package/src/core/device-install/apple/relay.ts +0 -202
- package/src/core/device-install/index.ts +0 -4
- package/src/core/device-install/operations/index.ts +0 -6
- package/src/core/device-install/operations/limbuild-client.ts +0 -144
- package/src/core/device-install/operations/operations.ts +0 -239
- package/src/core/device-install/operations/relay-client.ts +0 -286
- package/src/core/device-install/operations/relay-protocol.ts +0 -71
- package/src/core/device-install/operations/usbmux.ts +0 -270
- package/src/core/device-install/operations/webusb-dom.d.ts +0 -54
- package/src/core/device-install/operations/webusb.ts +0 -110
- package/src/core/device-install/storage/browser-storage.test.ts +0 -65
- package/src/core/device-install/storage/browser-storage.ts +0 -325
- package/src/core/device-install/storage/index.ts +0 -2
- package/src/core/device-install/storage/secret-store.ts +0 -332
- package/src/core/device-install/types.ts +0 -69
- package/src/core/errors.ts +0 -3
- package/src/core/play-publish/google.ts +0 -113
- package/src/core/play-publish/publish.test.ts +0 -68
- package/src/core/play-publish/publish.ts +0 -89
- package/src/device-build/index.ts +0 -42
- package/src/device-build/react.ts +0 -130
- package/src/device-build/signing.ts +0 -94
- package/src/device-install/README.md +0 -423
- package/src/device-install/demo/demo.css +0 -493
- package/src/device-install/demo/demo.tsx +0 -1061
- package/src/device-install/demo/index.html +0 -12
- package/src/device-install/index.ts +0 -50
- package/src/device-install/react.ts +0 -195
- package/src/play-publish/README.md +0 -66
- package/src/play-publish/index.ts +0 -12
- package/src/play-publish/react.ts +0 -208
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@limrun/ui",
|
|
3
|
-
"version": "0.14.
|
|
3
|
+
"version": "0.14.1",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public",
|
|
6
6
|
"tag": "next"
|
|
@@ -14,51 +14,6 @@
|
|
|
14
14
|
"import": "./dist/index.js",
|
|
15
15
|
"require": "./dist/index.cjs"
|
|
16
16
|
},
|
|
17
|
-
"./app-store-relay": {
|
|
18
|
-
"types": "./dist/app-store-relay/index.d.ts",
|
|
19
|
-
"import": "./dist/app-store-relay/index.js",
|
|
20
|
-
"require": "./dist/app-store-relay/index.cjs"
|
|
21
|
-
},
|
|
22
|
-
"./app-store-relay/react": {
|
|
23
|
-
"types": "./dist/app-store-relay/react.d.ts",
|
|
24
|
-
"import": "./dist/app-store-relay/react.js",
|
|
25
|
-
"require": "./dist/app-store-relay/react.cjs"
|
|
26
|
-
},
|
|
27
|
-
"./apple-credentials": {
|
|
28
|
-
"types": "./dist/apple-credentials/index.d.ts",
|
|
29
|
-
"import": "./dist/apple-credentials/index.js",
|
|
30
|
-
"require": "./dist/apple-credentials/index.cjs"
|
|
31
|
-
},
|
|
32
|
-
"./device-build": {
|
|
33
|
-
"types": "./dist/device-build/index.d.ts",
|
|
34
|
-
"import": "./dist/device-build/index.js",
|
|
35
|
-
"require": "./dist/device-build/index.cjs"
|
|
36
|
-
},
|
|
37
|
-
"./device-build/react": {
|
|
38
|
-
"types": "./dist/device-build/react.d.ts",
|
|
39
|
-
"import": "./dist/device-build/react.js",
|
|
40
|
-
"require": "./dist/device-build/react.cjs"
|
|
41
|
-
},
|
|
42
|
-
"./device-install": {
|
|
43
|
-
"types": "./dist/device-install/index.d.ts",
|
|
44
|
-
"import": "./dist/device-install/index.js",
|
|
45
|
-
"require": "./dist/device-install/index.cjs"
|
|
46
|
-
},
|
|
47
|
-
"./device-install/react": {
|
|
48
|
-
"types": "./dist/device-install/react.d.ts",
|
|
49
|
-
"import": "./dist/device-install/react.js",
|
|
50
|
-
"require": "./dist/device-install/react.cjs"
|
|
51
|
-
},
|
|
52
|
-
"./play-publish": {
|
|
53
|
-
"types": "./dist/play-publish/index.d.ts",
|
|
54
|
-
"import": "./dist/play-publish/index.js",
|
|
55
|
-
"require": "./dist/play-publish/index.cjs"
|
|
56
|
-
},
|
|
57
|
-
"./play-publish/react": {
|
|
58
|
-
"types": "./dist/play-publish/react.d.ts",
|
|
59
|
-
"import": "./dist/play-publish/react.js",
|
|
60
|
-
"require": "./dist/play-publish/react.cjs"
|
|
61
|
-
},
|
|
62
17
|
"./package.json": "./package.json"
|
|
63
18
|
},
|
|
64
19
|
"scripts": {
|
|
@@ -76,23 +31,27 @@
|
|
|
76
31
|
"directory": "packages/ui"
|
|
77
32
|
},
|
|
78
33
|
"license": "MIT",
|
|
34
|
+
"peerDependencies": {
|
|
35
|
+
"react": "^18.0.0 || ^19.0.0",
|
|
36
|
+
"react-dom": "^18.0.0 || ^19.0.0"
|
|
37
|
+
},
|
|
79
38
|
"devDependencies": {
|
|
80
|
-
"@types/node-forge": "^1.3.14",
|
|
81
39
|
"@types/react": "^19.1.12",
|
|
82
40
|
"@types/react-dom": "^19.1.9",
|
|
83
41
|
"@vitejs/plugin-react-swc": "^4.0.1",
|
|
42
|
+
"eslint": "^9.39.1",
|
|
84
43
|
"jsdom": "^26.0.0",
|
|
85
44
|
"path": "^0.12.7",
|
|
86
45
|
"react": "^19.1.1",
|
|
87
46
|
"react-dom": "^19.1.1",
|
|
47
|
+
"typescript": "5.8.3",
|
|
48
|
+
"typescript-eslint": "8.31.1",
|
|
88
49
|
"vite": "^7.1.4",
|
|
89
50
|
"vite-plugin-dts": "^4.5.4",
|
|
90
51
|
"vite-plugin-lib-inject-css": "^2.2.2",
|
|
91
|
-
"vitest": "^2.
|
|
52
|
+
"vitest": "^3.2.6"
|
|
92
53
|
},
|
|
93
54
|
"dependencies": {
|
|
94
|
-
"
|
|
95
|
-
"clsx": "^2.1.1",
|
|
96
|
-
"node-forge": "^1.4.0"
|
|
55
|
+
"clsx": "^2.1.1"
|
|
97
56
|
}
|
|
98
57
|
}
|
|
@@ -233,6 +233,45 @@ interface RemoteControlProps {
|
|
|
233
233
|
* 16:9 / 1920×1080 — applies).
|
|
234
234
|
*/
|
|
235
235
|
cameraAspect?: CameraAspect;
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* iOS only: when true, capture the user's microphone via
|
|
239
|
+
* `getUserMedia({ audio: true })` and stream it live into the
|
|
240
|
+
* simulator's mock microphone. Unlike the camera (which is
|
|
241
|
+
* demand-driven by apps inside the sim), the microphone is
|
|
242
|
+
* user-driven: flip this prop from a button in your UI. The
|
|
243
|
+
* component handles the browser permission prompt, the WebRTC
|
|
244
|
+
* track attach (no SDP renegotiation — the audio slot is
|
|
245
|
+
* pre-allocated) and the `microphoneStart`/`microphoneStop`
|
|
246
|
+
* signaling to the host. Progress and failures are reported via
|
|
247
|
+
* `onMicrophoneStateChange`. Ignored on Android instances.
|
|
248
|
+
*/
|
|
249
|
+
microphoneEnabled?: boolean;
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Fires whenever the live-microphone pipeline changes state:
|
|
253
|
+
* after an enable attempt resolves (granted or denied), when the
|
|
254
|
+
* host acknowledges or rejects the stream, when the browser
|
|
255
|
+
* revokes the track mid-stream, and on disable. Use it to render
|
|
256
|
+
* the mic button's on/off/error state.
|
|
257
|
+
*/
|
|
258
|
+
onMicrophoneStateChange?: (state: MicrophoneState) => void;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Snapshot of the live-microphone pipeline, reported through
|
|
263
|
+
* `onMicrophoneStateChange`.
|
|
264
|
+
*/
|
|
265
|
+
export interface MicrophoneState {
|
|
266
|
+
/** True while the browser mic is captured and streaming to the sim. */
|
|
267
|
+
active: boolean;
|
|
268
|
+
/**
|
|
269
|
+
* Human-readable failure from the last attempt, if any (permission
|
|
270
|
+
* denied, no device, host rejection, ...).
|
|
271
|
+
*/
|
|
272
|
+
error?: string;
|
|
273
|
+
/** Device label of the captured microphone, when available. */
|
|
274
|
+
label?: string;
|
|
236
275
|
}
|
|
237
276
|
|
|
238
277
|
/**
|
|
@@ -510,6 +549,8 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
510
549
|
onCameraStats,
|
|
511
550
|
cameraResolutionCap = 'auto',
|
|
512
551
|
cameraAspect,
|
|
552
|
+
microphoneEnabled = false,
|
|
553
|
+
onMicrophoneStateChange,
|
|
513
554
|
}: RemoteControlProps,
|
|
514
555
|
ref,
|
|
515
556
|
) => {
|
|
@@ -594,6 +635,28 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
594
635
|
packetsSent?: number;
|
|
595
636
|
packetsLost?: number;
|
|
596
637
|
} | null>(null);
|
|
638
|
+
// User-driven outbound microphone state (iOS only). Unlike the
|
|
639
|
+
// camera, the mic is toggled by the `microphoneEnabled` prop; the
|
|
640
|
+
// track rides a pre-allocated sendonly audio transceiver so
|
|
641
|
+
// enabling/disabling is just a `replaceTrack`, plus a
|
|
642
|
+
// `microphoneStart`/`microphoneStop` WS signal so the host's
|
|
643
|
+
// audio state machine switches its source.
|
|
644
|
+
const microphoneEnabledRef = useRef(microphoneEnabled);
|
|
645
|
+
microphoneEnabledRef.current = microphoneEnabled;
|
|
646
|
+
const onMicrophoneStateChangeRef = useRef(onMicrophoneStateChange);
|
|
647
|
+
onMicrophoneStateChangeRef.current = onMicrophoneStateChange;
|
|
648
|
+
const outboundMicSenderRef = useRef<RTCRtpSender | null>(null);
|
|
649
|
+
const outboundMicStreamRef = useRef<MediaStream | null>(null);
|
|
650
|
+
// Bumped by detach and teardown so an attach suspended on an await
|
|
651
|
+
// (the getUserMedia prompt) detects it was superseded and bails.
|
|
652
|
+
const micGenerationRef = useRef(0);
|
|
653
|
+
// Serializes attach/detach work so their awaits (replaceTrack on the
|
|
654
|
+
// shared sender) can never interleave and undo each other's effects.
|
|
655
|
+
const micOpQueueRef = useRef<Promise<void>>(Promise.resolve());
|
|
656
|
+
// Correlates the in-flight `microphoneStart` with its result so a
|
|
657
|
+
// host-side rejection (e.g. no loopback device on the node) can
|
|
658
|
+
// tear the capture back down and surface the error.
|
|
659
|
+
const micStartPendingIdRef = useRef<string | null>(null);
|
|
597
660
|
const firstFrameShownRef = useRef(false);
|
|
598
661
|
const pendingScreenshotResolversRef = useRef<
|
|
599
662
|
Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
|
|
@@ -625,6 +688,14 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
625
688
|
pointerId1: number;
|
|
626
689
|
};
|
|
627
690
|
const twoFingerStateRef = useRef<TwoFingerState | null>(null);
|
|
691
|
+
// Wheel-scroll drag state; lives for a single synthesized gesture.
|
|
692
|
+
const wheelStateRef = useRef<{ pos: PointerGeometry; endTimer: number } | null>(null);
|
|
693
|
+
// Latest wheel handlers, so the once-attached native listener and the
|
|
694
|
+
// pointer machinery always call the current render's closures.
|
|
695
|
+
const wheelCallbacksRef = useRef<{ handle: (event: WheelEvent) => void; end: () => void }>({
|
|
696
|
+
handle: () => {},
|
|
697
|
+
end: () => {},
|
|
698
|
+
});
|
|
628
699
|
|
|
629
700
|
// Hover point for rendering two-finger indicators when Alt is held.
|
|
630
701
|
// Only computed/set when Alt is held to avoid unnecessary re-renders.
|
|
@@ -726,8 +797,8 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
726
797
|
};
|
|
727
798
|
|
|
728
799
|
// Pointer ID used by inspect-driven taps. Distinct from human pointers
|
|
729
|
-
// (-1 mouse, -2 alt-mirror)
|
|
730
|
-
// interfere with an in-progress drag.
|
|
800
|
+
// (-1 mouse, -2 alt-mirror), the synthesized wheel finger (-3), and our
|
|
801
|
+
// touch identifiers so they never interfere with an in-progress drag.
|
|
731
802
|
const AX_TAP_POINTER_ID = -10;
|
|
732
803
|
|
|
733
804
|
// Send a down+up tap at a viewport-space (clientX/Y) position. The point
|
|
@@ -791,6 +862,8 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
791
862
|
// Fixed pointer IDs for Alt-simulated two-finger gestures
|
|
792
863
|
const ALT_POINTER_ID_PRIMARY = -1;
|
|
793
864
|
const ALT_POINTER_ID_MIRROR = -2;
|
|
865
|
+
// Synthesized finger for wheel-scroll drags.
|
|
866
|
+
const WHEEL_POINTER_ID = -3;
|
|
794
867
|
|
|
795
868
|
// Helper to send a single-touch control message (used by both single-finger and Android two-finger paths)
|
|
796
869
|
const sendSingleTouch = (
|
|
@@ -848,6 +921,11 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
848
921
|
|
|
849
922
|
switch (eventType) {
|
|
850
923
|
case 'down':
|
|
924
|
+
// A real pointer going down takes over from any synthesized
|
|
925
|
+
// wheel-scroll finger, so the tap isn't sent as a second finger.
|
|
926
|
+
if (pointerId !== WHEEL_POINTER_ID) {
|
|
927
|
+
wheelCallbacksRef.current.end();
|
|
928
|
+
}
|
|
851
929
|
// For multi-touch: use ACTION_DOWN for first pointer, ACTION_POINTER_DOWN for additional pointers
|
|
852
930
|
const currentPointerCount = activePointers.current.size;
|
|
853
931
|
action = currentPointerCount === 0 ? AMOTION_EVENT.ACTION_DOWN : AMOTION_EVENT.ACTION_POINTER_DOWN;
|
|
@@ -1136,6 +1214,12 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1136
1214
|
p1: { x: x1, y: y1, id: pointerId1 },
|
|
1137
1215
|
});
|
|
1138
1216
|
|
|
1217
|
+
if (eventType === 'down') {
|
|
1218
|
+
// A real two-finger gesture takes over from any synthesized
|
|
1219
|
+
// wheel-scroll finger.
|
|
1220
|
+
wheelCallbacksRef.current.end();
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1139
1223
|
if (platform === 'ios') {
|
|
1140
1224
|
// iOS: use special two-finger message (type=18)
|
|
1141
1225
|
const action =
|
|
@@ -1491,6 +1575,130 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1491
1575
|
}
|
|
1492
1576
|
};
|
|
1493
1577
|
|
|
1578
|
+
// Mouse wheel / trackpad scrolling over the stream scrolls inside the
|
|
1579
|
+
// simulator. iOS has no scroll wheel, so we fake a finger: press down
|
|
1580
|
+
// where the cursor is, drag it as wheel events come in, lift when they
|
|
1581
|
+
// stop. If the fake finger reaches the screen edge before the user stops
|
|
1582
|
+
// scrolling, lift and press again to keep going.
|
|
1583
|
+
const WHEEL_END_QUIET_MS = 150;
|
|
1584
|
+
const endWheelGesture = () => {
|
|
1585
|
+
const state = wheelStateRef.current;
|
|
1586
|
+
if (!state) return;
|
|
1587
|
+
window.clearTimeout(state.endTimer);
|
|
1588
|
+
applyPointerEvent(WHEEL_POINTER_ID, 'up', state.pos);
|
|
1589
|
+
wheelStateRef.current = null;
|
|
1590
|
+
};
|
|
1591
|
+
const handleWheel = (event: WheelEvent) => {
|
|
1592
|
+
// Trackpad pinches arrive as ctrl+wheel; don't turn them into scrolls.
|
|
1593
|
+
if (event.ctrlKey) return;
|
|
1594
|
+
// If the user is already touching/dragging (or holding Alt for the
|
|
1595
|
+
// two-finger gesture), let that win - don't add a second fake finger.
|
|
1596
|
+
if (isAltHeldRef.current || twoFingerStateRef.current) return;
|
|
1597
|
+
const pointerCount = activePointers.current.size;
|
|
1598
|
+
if (pointerCount > 1 || (pointerCount === 1 && !activePointers.current.has(WHEEL_POINTER_ID))) return;
|
|
1599
|
+
|
|
1600
|
+
const ctx = computeVideoMappingContext();
|
|
1601
|
+
if (!ctx) return;
|
|
1602
|
+
// Outside the rendered video (frame, letterbox) let the page scroll.
|
|
1603
|
+
const relativeX = event.clientX - ctx.videoRect.left - ctx.offsetX;
|
|
1604
|
+
const relativeY = event.clientY - ctx.videoRect.top - ctx.offsetY;
|
|
1605
|
+
if (relativeX < 0 || relativeX > ctx.actualWidth || relativeY < 0 || relativeY > ctx.actualHeight) {
|
|
1606
|
+
return;
|
|
1607
|
+
}
|
|
1608
|
+
const anchor = mapClientPointToVideo(ctx, event.clientX, event.clientY);
|
|
1609
|
+
if (!anchor) return;
|
|
1610
|
+
event.preventDefault();
|
|
1611
|
+
|
|
1612
|
+
// deltaMode: 0=pixels, 1=lines, 2=pages. Deltas are CSS pixels; convert
|
|
1613
|
+
// to stream pixels so a tick scrolls the same visual distance at any
|
|
1614
|
+
// window size.
|
|
1615
|
+
const scaleX =
|
|
1616
|
+
(event.deltaMode === 1 ? 16
|
|
1617
|
+
: event.deltaMode === 2 ? ctx.actualWidth
|
|
1618
|
+
: 1) *
|
|
1619
|
+
(ctx.videoWidth / ctx.actualWidth);
|
|
1620
|
+
const scaleY =
|
|
1621
|
+
(event.deltaMode === 1 ? 16
|
|
1622
|
+
: event.deltaMode === 2 ? ctx.actualHeight
|
|
1623
|
+
: 1) *
|
|
1624
|
+
(ctx.videoHeight / ctx.actualHeight);
|
|
1625
|
+
const dx = -event.deltaX * scaleX;
|
|
1626
|
+
const dy = -event.deltaY * scaleY;
|
|
1627
|
+
const armEnd = (s: { endTimer: number }) => {
|
|
1628
|
+
window.clearTimeout(s.endTimer);
|
|
1629
|
+
s.endTimer = window.setTimeout(endWheelGesture, WHEEL_END_QUIET_MS);
|
|
1630
|
+
};
|
|
1631
|
+
if (dx === 0 && dy === 0) {
|
|
1632
|
+
// Some wheel events carry no movement (momentum tails); just keep
|
|
1633
|
+
// the gesture alive.
|
|
1634
|
+
if (wheelStateRef.current) armEnd(wheelStateRef.current);
|
|
1635
|
+
return;
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
// Where to press the fake finger down: at the cursor, but nudged away
|
|
1639
|
+
// from the screen edges so the drag has room to travel.
|
|
1640
|
+
const pressPoint = (): PointerGeometry => ({
|
|
1641
|
+
videoWidth: anchor.videoWidth,
|
|
1642
|
+
videoHeight: anchor.videoHeight,
|
|
1643
|
+
videoX:
|
|
1644
|
+
Math.abs(dx) > Math.abs(dy) ?
|
|
1645
|
+
Math.min(Math.max(anchor.videoX, anchor.videoWidth * 0.2), anchor.videoWidth * 0.8)
|
|
1646
|
+
: anchor.videoX,
|
|
1647
|
+
videoY:
|
|
1648
|
+
Math.abs(dy) >= Math.abs(dx) ?
|
|
1649
|
+
Math.min(Math.max(anchor.videoY, anchor.videoHeight * 0.2), anchor.videoHeight * 0.8)
|
|
1650
|
+
: anchor.videoY,
|
|
1651
|
+
});
|
|
1652
|
+
|
|
1653
|
+
const advance = (p: PointerGeometry): PointerGeometry => ({
|
|
1654
|
+
...p,
|
|
1655
|
+
videoX: Math.min(Math.max(p.videoX + dx, 0), p.videoWidth),
|
|
1656
|
+
videoY: Math.min(Math.max(p.videoY + dy, 0), p.videoHeight),
|
|
1657
|
+
});
|
|
1658
|
+
|
|
1659
|
+
let state = wheelStateRef.current;
|
|
1660
|
+
if (!state) {
|
|
1661
|
+
const start = pressPoint();
|
|
1662
|
+
applyPointerEvent(WHEEL_POINTER_ID, 'down', start);
|
|
1663
|
+
state = { pos: start, endTimer: 0 };
|
|
1664
|
+
wheelStateRef.current = state;
|
|
1665
|
+
} else {
|
|
1666
|
+
window.clearTimeout(state.endTimer);
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
let pos = state.pos;
|
|
1670
|
+
let next = advance(pos);
|
|
1671
|
+
if (next.videoX === pos.videoX && next.videoY === pos.videoY) {
|
|
1672
|
+
// Finger reached the screen edge: lift it, press again with room to
|
|
1673
|
+
// travel, and move right away (a press that never moves would count
|
|
1674
|
+
// as a tap).
|
|
1675
|
+
applyPointerEvent(WHEEL_POINTER_ID, 'up', pos);
|
|
1676
|
+
pos = pressPoint();
|
|
1677
|
+
applyPointerEvent(WHEEL_POINTER_ID, 'down', pos);
|
|
1678
|
+
next = advance(pos);
|
|
1679
|
+
}
|
|
1680
|
+
applyPointerEvent(WHEEL_POINTER_ID, 'move', next);
|
|
1681
|
+
state.pos = next;
|
|
1682
|
+
armEnd(state);
|
|
1683
|
+
};
|
|
1684
|
+
useEffect(() => {
|
|
1685
|
+
wheelCallbacksRef.current = { handle: handleWheel, end: endWheelGesture };
|
|
1686
|
+
});
|
|
1687
|
+
|
|
1688
|
+
// React attaches onWheel passively, so preventDefault (needed to keep the
|
|
1689
|
+
// page from scrolling) requires a native non-passive listener. Cleanup
|
|
1690
|
+
// lifts an in-flight synthesized finger before the channel goes away.
|
|
1691
|
+
useEffect(() => {
|
|
1692
|
+
const container = containerRef.current;
|
|
1693
|
+
if (!container) return;
|
|
1694
|
+
const listener = (event: WheelEvent) => wheelCallbacksRef.current.handle(event);
|
|
1695
|
+
container.addEventListener('wheel', listener, { passive: false });
|
|
1696
|
+
return () => {
|
|
1697
|
+
wheelCallbacksRef.current.end();
|
|
1698
|
+
container.removeEventListener('wheel', listener);
|
|
1699
|
+
};
|
|
1700
|
+
}, []);
|
|
1701
|
+
|
|
1494
1702
|
useEffect(() => {
|
|
1495
1703
|
const handleKeyDown = (event: KeyboardEvent) => {
|
|
1496
1704
|
if (event.key === 'Alt') {
|
|
@@ -1694,6 +1902,149 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1694
1902
|
stopMediaStream(stream);
|
|
1695
1903
|
};
|
|
1696
1904
|
|
|
1905
|
+
// Runs one attach/detach operation after all previously queued ones,
|
|
1906
|
+
// so two operations' awaits never interleave on the shared sender (a
|
|
1907
|
+
// stale detach could otherwise clear a fresh attach's track, or send
|
|
1908
|
+
// `microphoneStop` after its `microphoneStart`).
|
|
1909
|
+
const runMicOp = (op: () => Promise<void>): Promise<void> => {
|
|
1910
|
+
const next = micOpQueueRef.current.then(op).catch((err) => {
|
|
1911
|
+
debugWarn('microphone operation failed:', err);
|
|
1912
|
+
});
|
|
1913
|
+
micOpQueueRef.current = next;
|
|
1914
|
+
return next;
|
|
1915
|
+
};
|
|
1916
|
+
|
|
1917
|
+
// Capture the user's microphone and stream it to the simulator's
|
|
1918
|
+
// mock microphone. Safe to call redundantly: bails when already
|
|
1919
|
+
// attached, when the connection isn't up yet (the ICE-connected
|
|
1920
|
+
// handler re-invokes it), or when the prop flipped off mid-prompt.
|
|
1921
|
+
// Does NOT bump the generation: a redundant call (e.g. the ICE
|
|
1922
|
+
// handler firing on both `connected` and `completed`) must not
|
|
1923
|
+
// cancel an attach already in flight; only detach/teardown cancel.
|
|
1924
|
+
const attachMicrophone = (): Promise<void> => {
|
|
1925
|
+
if (platform !== 'ios') return Promise.resolve();
|
|
1926
|
+
return runMicOp(() => attachMicrophoneOp());
|
|
1927
|
+
};
|
|
1928
|
+
|
|
1929
|
+
const attachMicrophoneOp = async () => {
|
|
1930
|
+
const generation = micGenerationRef.current;
|
|
1931
|
+
const isCurrent = () => generation === micGenerationRef.current && microphoneEnabledRef.current;
|
|
1932
|
+
const reportMicError = (error: string) => {
|
|
1933
|
+
safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
|
|
1934
|
+
active: false,
|
|
1935
|
+
error,
|
|
1936
|
+
});
|
|
1937
|
+
};
|
|
1938
|
+
if (!isCurrent()) return;
|
|
1939
|
+
const ws = wsRef.current;
|
|
1940
|
+
const sender = outboundMicSenderRef.current;
|
|
1941
|
+
if (!ws || ws.readyState !== WebSocket.OPEN || !sender) {
|
|
1942
|
+
// Not connected yet — the post-connect hook retries while the
|
|
1943
|
+
// prop is still on.
|
|
1944
|
+
return;
|
|
1945
|
+
}
|
|
1946
|
+
if (outboundMicStreamRef.current) return;
|
|
1947
|
+
if (!navigator.mediaDevices || typeof navigator.mediaDevices.getUserMedia !== 'function') {
|
|
1948
|
+
// eslint-disable-next-line no-console
|
|
1949
|
+
console.warn(
|
|
1950
|
+
'[RemoteControl] navigator.mediaDevices.getUserMedia unavailable. ' +
|
|
1951
|
+
'getUserMedia requires a secure context (https or http://localhost).',
|
|
1952
|
+
);
|
|
1953
|
+
reportMicError('Microphone capture requires a secure context (https)');
|
|
1954
|
+
return;
|
|
1955
|
+
}
|
|
1956
|
+
let stream: MediaStream | null = null;
|
|
1957
|
+
try {
|
|
1958
|
+
stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
|
1959
|
+
} catch (err) {
|
|
1960
|
+
// eslint-disable-next-line no-console
|
|
1961
|
+
console.warn('[RemoteControl] microphone getUserMedia denied/failed:', err);
|
|
1962
|
+
}
|
|
1963
|
+
if (!isCurrent()) {
|
|
1964
|
+
if (stream) stopMediaStream(stream);
|
|
1965
|
+
return;
|
|
1966
|
+
}
|
|
1967
|
+
if (!stream) {
|
|
1968
|
+
reportMicError('Microphone permission denied or no device available');
|
|
1969
|
+
return;
|
|
1970
|
+
}
|
|
1971
|
+
const audioTrack = stream.getAudioTracks()[0] ?? null;
|
|
1972
|
+
if (!audioTrack) {
|
|
1973
|
+
stopMediaStream(stream);
|
|
1974
|
+
reportMicError('Captured stream has no audio track');
|
|
1975
|
+
return;
|
|
1976
|
+
}
|
|
1977
|
+
try {
|
|
1978
|
+
await sender.replaceTrack(audioTrack);
|
|
1979
|
+
} catch (err) {
|
|
1980
|
+
debugWarn('replaceTrack(audioTrack) failed:', err);
|
|
1981
|
+
stopMediaStream(stream);
|
|
1982
|
+
reportMicError('Failed to attach the microphone track to the connection');
|
|
1983
|
+
return;
|
|
1984
|
+
}
|
|
1985
|
+
if (!isCurrent()) {
|
|
1986
|
+
try {
|
|
1987
|
+
await sender.replaceTrack(null);
|
|
1988
|
+
} catch (err) {
|
|
1989
|
+
debugWarn('replaceTrack(null) on stale mic attach failed:', err);
|
|
1990
|
+
}
|
|
1991
|
+
stopMediaStream(stream);
|
|
1992
|
+
return;
|
|
1993
|
+
}
|
|
1994
|
+
outboundMicStreamRef.current = stream;
|
|
1995
|
+
// If the browser revokes the track later (user clicks Stop in
|
|
1996
|
+
// the tab's mic indicator, device unplugged, ...) tear down and
|
|
1997
|
+
// tell the host, so the sim's mic goes back to silence instead
|
|
1998
|
+
// of freezing on the last buffer.
|
|
1999
|
+
audioTrack.onended = () => {
|
|
2000
|
+
if (outboundMicStreamRef.current !== stream) return;
|
|
2001
|
+
void detachMicrophone('microphone capture ended by the browser');
|
|
2002
|
+
};
|
|
2003
|
+
const id = `ui-mic-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
|
|
2004
|
+
micStartPendingIdRef.current = id;
|
|
2005
|
+
ws.send(JSON.stringify({ type: 'microphoneStart', id }));
|
|
2006
|
+
safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
|
|
2007
|
+
active: true,
|
|
2008
|
+
label: audioTrack.label || undefined,
|
|
2009
|
+
});
|
|
2010
|
+
};
|
|
2011
|
+
|
|
2012
|
+
// Stop the live microphone: drop the track from the sender, stop
|
|
2013
|
+
// the capture (turning the browser's mic indicator off) and tell
|
|
2014
|
+
// the host to switch its audio source back to silence. Safe to
|
|
2015
|
+
// call when nothing is attached.
|
|
2016
|
+
const detachMicrophone = (error?: string): Promise<void> => {
|
|
2017
|
+
// Bump synchronously (not on the queue) so an attach that is
|
|
2018
|
+
// suspended on an await bails at its next checkpoint even while
|
|
2019
|
+
// it still owns the queue.
|
|
2020
|
+
++micGenerationRef.current;
|
|
2021
|
+
micStartPendingIdRef.current = null;
|
|
2022
|
+
return runMicOp(async () => {
|
|
2023
|
+
const sender = outboundMicSenderRef.current;
|
|
2024
|
+
const stream = outboundMicStreamRef.current;
|
|
2025
|
+
outboundMicStreamRef.current = null;
|
|
2026
|
+
if (sender && stream) {
|
|
2027
|
+
try {
|
|
2028
|
+
await sender.replaceTrack(null);
|
|
2029
|
+
} catch (err) {
|
|
2030
|
+
debugWarn('replaceTrack(null) on mic detach failed:', err);
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
if (stream) stopMediaStream(stream);
|
|
2034
|
+
const ws = wsRef.current;
|
|
2035
|
+
if (stream && ws && ws.readyState === WebSocket.OPEN) {
|
|
2036
|
+
const id = `ui-mic-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
|
|
2037
|
+
ws.send(JSON.stringify({ type: 'microphoneStop', id }));
|
|
2038
|
+
}
|
|
2039
|
+
if (stream || error) {
|
|
2040
|
+
safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
|
|
2041
|
+
active: false,
|
|
2042
|
+
...(error !== undefined && { error }),
|
|
2043
|
+
});
|
|
2044
|
+
}
|
|
2045
|
+
});
|
|
2046
|
+
};
|
|
2047
|
+
|
|
1697
2048
|
// Translate the user-facing resolution cap into a
|
|
1698
2049
|
// MediaTrackConstraints fragment we can feed `getUserMedia`
|
|
1699
2050
|
// or `applyConstraints`. Always returns a 30 fps ceiling so
|
|
@@ -1845,6 +2196,22 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
1845
2196
|
stopCameraStatsPoller();
|
|
1846
2197
|
stopOutboundLocalStream();
|
|
1847
2198
|
outboundCameraSenderRef.current = null;
|
|
2199
|
+
// Same for the live microphone. No `microphoneStop` signal here —
|
|
2200
|
+
// the WS is going away with the connection; if `microphoneEnabled`
|
|
2201
|
+
// is still on, the post-reconnect hook re-attaches.
|
|
2202
|
+
++micGenerationRef.current;
|
|
2203
|
+
micStartPendingIdRef.current = null;
|
|
2204
|
+
if (outboundMicStreamRef.current) {
|
|
2205
|
+
stopMediaStream(outboundMicStreamRef.current);
|
|
2206
|
+
outboundMicStreamRef.current = null;
|
|
2207
|
+
// The capture is gone; report it so consumers don't keep a stale
|
|
2208
|
+
// `active: true` across drops (a re-attach after reconnect
|
|
2209
|
+
// reports again).
|
|
2210
|
+
safeInvoke('onMicrophoneStateChange', onMicrophoneStateChangeRef.current, {
|
|
2211
|
+
active: false,
|
|
2212
|
+
});
|
|
2213
|
+
}
|
|
2214
|
+
outboundMicSenderRef.current = null;
|
|
1848
2215
|
// A scheduled cursor flush would otherwise call setState on a
|
|
1849
2216
|
// teardown component once the next frame runs.
|
|
1850
2217
|
if (cursorRafIdRef.current !== undefined) {
|
|
@@ -2162,6 +2529,18 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2162
2529
|
: null;
|
|
2163
2530
|
outboundCameraSenderRef.current = outboundCameraTransceiver?.sender ?? null;
|
|
2164
2531
|
|
|
2532
|
+
// Same pre-allocation for the live microphone: a sendonly audio
|
|
2533
|
+
// slot negotiated once at connection setup, so toggling the mic
|
|
2534
|
+
// later is just `replaceTrack` — no SDP renegotiation. iOS only,
|
|
2535
|
+
// like the camera.
|
|
2536
|
+
const outboundMicTransceiver =
|
|
2537
|
+
platform === 'ios' ?
|
|
2538
|
+
peerConnection.addTransceiver('audio', {
|
|
2539
|
+
direction: 'sendonly',
|
|
2540
|
+
})
|
|
2541
|
+
: null;
|
|
2542
|
+
outboundMicSenderRef.current = outboundMicTransceiver?.sender ?? null;
|
|
2543
|
+
|
|
2165
2544
|
// As hardware encoder, we use H265 for iOS and VP9 for Android.
|
|
2166
2545
|
// We make sure these two are the first ones in the list.
|
|
2167
2546
|
// If not, the fallback is H264 which is also hardware accelerated, although not as good,
|
|
@@ -2383,6 +2762,11 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2383
2762
|
updateStatus('ICE state: ' + iceState);
|
|
2384
2763
|
if (iceState === 'connected' || iceState === 'completed') {
|
|
2385
2764
|
clearIceDisconnectedGrace();
|
|
2765
|
+
// Re-attach the live microphone across reconnects (or apply a
|
|
2766
|
+
// prop that was flipped on before the connection was up).
|
|
2767
|
+
if (microphoneEnabledRef.current && !outboundMicStreamRef.current) {
|
|
2768
|
+
void attachMicrophone();
|
|
2769
|
+
}
|
|
2386
2770
|
return;
|
|
2387
2771
|
}
|
|
2388
2772
|
if (iceState === 'failed') {
|
|
@@ -2780,6 +3164,22 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2780
3164
|
startCameraStatsPoller();
|
|
2781
3165
|
break;
|
|
2782
3166
|
}
|
|
3167
|
+
case 'microphoneStartResult': {
|
|
3168
|
+
if (micStartPendingIdRef.current === null || message.id !== micStartPendingIdRef.current) {
|
|
3169
|
+
break;
|
|
3170
|
+
}
|
|
3171
|
+
micStartPendingIdRef.current = null;
|
|
3172
|
+
if (typeof message.error === 'string') {
|
|
3173
|
+
// Host rejected the stream (e.g. no loopback device
|
|
3174
|
+
// assigned on the node). Tear the capture back down so
|
|
3175
|
+
// the browser's mic indicator doesn't stay lit for a
|
|
3176
|
+
// stream nobody hears.
|
|
3177
|
+
// eslint-disable-next-line no-console
|
|
3178
|
+
console.warn('[RemoteControl] microphoneStart rejected by host:', message.error);
|
|
3179
|
+
void detachMicrophone(message.error);
|
|
3180
|
+
}
|
|
3181
|
+
break;
|
|
3182
|
+
}
|
|
2783
3183
|
case 'terminateAppResult':
|
|
2784
3184
|
if (typeof message.id !== 'string') {
|
|
2785
3185
|
debugWarn('Received invalid terminateApp result message:', message);
|
|
@@ -2906,6 +3306,18 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
|
|
|
2906
3306
|
}
|
|
2907
3307
|
}, [cameraAspect]);
|
|
2908
3308
|
|
|
3309
|
+
// Drive the live microphone from the `microphoneEnabled` prop.
|
|
3310
|
+
// When flipped on before the connection is up, attachMicrophone
|
|
3311
|
+
// bails and the ICE-connected handler picks it up instead.
|
|
3312
|
+
useEffect(() => {
|
|
3313
|
+
if (microphoneEnabled) {
|
|
3314
|
+
void attachMicrophone();
|
|
3315
|
+
} else {
|
|
3316
|
+
void detachMicrophone();
|
|
3317
|
+
}
|
|
3318
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
3319
|
+
}, [microphoneEnabled]);
|
|
3320
|
+
|
|
2909
3321
|
useEffect(() => {
|
|
2910
3322
|
// Reset video loaded state when connection params change
|
|
2911
3323
|
setVideoLoaded(false);
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { RemoteControl } from './components/remote-control';
|
|
2
|
-
export type { RemoteControlHandle } from './components/remote-control';
|
|
2
|
+
export type { RemoteControlHandle, MicrophoneState } from './components/remote-control';
|
|
3
3
|
|
|
4
4
|
// Accessibility / inspect-mode types and helpers. Exported so customers can
|
|
5
5
|
// build their own side panels, search UIs, or agent-driven inspectors on top
|
package/vite.config.ts
CHANGED
|
@@ -11,15 +11,6 @@ export default defineConfig({
|
|
|
11
11
|
lib: {
|
|
12
12
|
entry: {
|
|
13
13
|
index: resolve(__dirname, 'src/index.ts'),
|
|
14
|
-
'app-store-relay/index': resolve(__dirname, 'src/app-store-relay/index.ts'),
|
|
15
|
-
'app-store-relay/react': resolve(__dirname, 'src/app-store-relay/react.ts'),
|
|
16
|
-
'apple-credentials/index': resolve(__dirname, 'src/apple-credentials/index.ts'),
|
|
17
|
-
'device-build/index': resolve(__dirname, 'src/device-build/index.ts'),
|
|
18
|
-
'device-build/react': resolve(__dirname, 'src/device-build/react.ts'),
|
|
19
|
-
'device-install/index': resolve(__dirname, 'src/device-install/index.ts'),
|
|
20
|
-
'device-install/react': resolve(__dirname, 'src/device-install/react.ts'),
|
|
21
|
-
'play-publish/index': resolve(__dirname, 'src/play-publish/index.ts'),
|
|
22
|
-
'play-publish/react': resolve(__dirname, 'src/play-publish/react.ts'),
|
|
23
14
|
},
|
|
24
15
|
name: 'LimrunUI',
|
|
25
16
|
formats: ['es', 'cjs'],
|