@limrun/ui 0.4.3 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,6 +14,7 @@ import {
14
14
  createTouchControlMessage,
15
15
  createInjectKeycodeMessage,
16
16
  createSetClipboardMessage,
17
+ createTwoFingerTouchControlMessage,
17
18
  } from '../core/webrtc-messages';
18
19
 
19
20
  declare global {
@@ -67,6 +68,7 @@ export interface RemoteControlHandle {
67
68
  openUrl: (url: string) => void;
68
69
  sendKeyEvent: (event: ImperativeKeyboardEvent) => void;
69
70
  screenshot: () => Promise<ScreenshotData>;
71
+ terminateApp: (bundleId: string) => Promise<void>;
70
72
  }
71
73
 
72
74
  const debugLog = (...args: any[]) => {
@@ -81,6 +83,14 @@ const debugWarn = (...args: any[]) => {
81
83
  }
82
84
  };
83
85
 
86
+ const motionActionToString = (action: number): string => {
87
+ // AMOTION_EVENT is a constants object; find the matching ACTION_* key if present
88
+ const match = Object.entries(AMOTION_EVENT).find(
89
+ ([key, value]) => key.startsWith('ACTION_') && value === action,
90
+ );
91
+ return match?.[0] ?? String(action);
92
+ };
93
+
84
94
  type DevicePlatform = 'ios' | 'android';
85
95
 
86
96
  const detectPlatform = (url: string): DevicePlatform => {
@@ -170,6 +180,7 @@ function getAndroidKeycodeAndMeta(event: React.KeyboardEvent): { keycode: number
170
180
 
171
181
  export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>(
172
182
  ({ className, url, token, sessionId: propSessionId, openUrl, showFrame = true }: RemoteControlProps, ref) => {
183
+ const containerRef = useRef<HTMLDivElement>(null);
173
184
  const videoRef = useRef<HTMLVideoElement>(null);
174
185
  const frameRef = useRef<HTMLImageElement>(null);
175
186
  const [videoLoaded, setVideoLoaded] = useState(false);
@@ -183,11 +194,49 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
183
194
  Map<string, (value: ScreenshotData | PromiseLike<ScreenshotData>) => void>
184
195
  >(new Map());
185
196
  const pendingScreenshotRejectersRef = useRef<Map<string, (reason?: any) => void>>(new Map());
197
+ const pendingTerminateAppResolversRef = useRef<Map<string, () => void>>(new Map());
198
+ const pendingTerminateAppRejectersRef = useRef<Map<string, (reason?: any) => void>>(new Map());
186
199
 
187
- // Map to track active pointers (mouse or touch) and their last known position inside the video
200
+ // Map to track active pointers for real touch/mouse single-finger events.
188
201
  // Key: pointerId (-1 for mouse, touch.identifier for touch), Value: { x: number, y: number }
189
202
  const activePointers = useRef<Map<number, { x: number; y: number }>>(new Map());
190
203
 
204
+ // Alt/Option modifier state for pinch emulation.
205
+ // We use a ref as the source of truth (for synchronous event handler access)
206
+ // and state only to trigger re-renders for the visual indicators.
207
+ const isAltHeldRef = useRef(false);
208
+ const [isAltHeld, setIsAltHeld] = useState(false);
209
+
210
+ // State for any two-finger gesture (Alt+mouse simulated or real two-finger touch).
211
+ // Tracks positions, video size, source, and pointer IDs (for Android protocol).
212
+ type TwoFingerState = {
213
+ finger0: { x: number; y: number };
214
+ finger1: { x: number; y: number };
215
+ videoSize: { width: number; height: number };
216
+ // Track source so we know when to clear (Alt release vs touch end)
217
+ source: 'alt-mouse' | 'real-touch';
218
+ // Pointer IDs for Android (real touch.identifier or simulated -1/-2)
219
+ pointerId0: number;
220
+ pointerId1: number;
221
+ };
222
+ const twoFingerStateRef = useRef<TwoFingerState | null>(null);
223
+
224
+ // Hover point for rendering two-finger indicators when Alt is held.
225
+ // Only computed/set when Alt is held to avoid unnecessary re-renders.
226
+ type HoverPoint = {
227
+ containerX: number;
228
+ containerY: number;
229
+ mirrorContainerX: number;
230
+ mirrorContainerY: number;
231
+ videoX: number;
232
+ videoY: number;
233
+ mirrorVideoX: number;
234
+ mirrorVideoY: number;
235
+ videoWidth: number;
236
+ videoHeight: number;
237
+ };
238
+ const [hoverPoint, setHoverPoint] = useState<HoverPoint | null>(null);
239
+
191
240
  const sessionId = useMemo(
192
241
  () =>
193
242
  propSessionId ||
@@ -210,130 +259,449 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
210
259
  dataChannelRef.current.send(data);
211
260
  };
212
261
 
213
- // Unified handler for both mouse and touch interactions
214
- const handleInteraction = (event: React.MouseEvent | React.TouchEvent) => {
215
- event.preventDefault();
216
- event.stopPropagation();
217
-
218
- if (!dataChannelRef.current || dataChannelRef.current.readyState !== 'open' || !videoRef.current) {
219
- return;
262
+ // Fixed pointer IDs for Alt-simulated two-finger gestures
263
+ const ALT_POINTER_ID_PRIMARY = -1;
264
+ const ALT_POINTER_ID_MIRROR = -2;
265
+
266
+ // Helper to send a single-touch control message (used by both single-finger and Android two-finger paths)
267
+ const sendSingleTouch = (
268
+ action: number,
269
+ pointerId: number,
270
+ videoWidth: number,
271
+ videoHeight: number,
272
+ x: number,
273
+ y: number,
274
+ ) => {
275
+ const message = createTouchControlMessage(
276
+ action,
277
+ pointerId,
278
+ videoWidth,
279
+ videoHeight,
280
+ x,
281
+ y,
282
+ 1.0, // pressure
283
+ AMOTION_EVENT.BUTTON_PRIMARY,
284
+ AMOTION_EVENT.BUTTON_PRIMARY,
285
+ );
286
+ if (message) {
287
+ debugLog('[rc-touch] sendSingleTouch', {
288
+ action,
289
+ actionName: motionActionToString(action),
290
+ pointerId,
291
+ x,
292
+ y,
293
+ video: { width: videoWidth, height: videoHeight },
294
+ });
295
+ sendBinaryControlMessage(message);
220
296
  }
297
+ };
221
298
 
222
- const video = videoRef.current;
223
- const rect = video.getBoundingClientRect();
224
- const videoWidth = video.videoWidth;
225
- const videoHeight = video.videoHeight;
226
-
227
- if (!videoWidth || !videoHeight) return; // Video dimensions not ready
228
-
229
- // Helper to process a single pointer event (either mouse or a single touch point)
230
- const processPointer = (
231
- pointerId: number,
232
- clientX: number,
233
- clientY: number,
234
- eventType: 'down' | 'move' | 'up' | 'cancel',
235
- ) => {
236
- // --- Start: Coordinate Calculation ---
237
- const displayWidth = rect.width;
238
- const displayHeight = rect.height;
239
- const videoAspectRatio = videoWidth / videoHeight;
240
- const containerAspectRatio = displayWidth / displayHeight;
241
- let actualWidth = displayWidth;
242
- let actualHeight = displayHeight;
243
- if (videoAspectRatio > containerAspectRatio) {
244
- actualHeight = displayWidth / videoAspectRatio;
245
- } else {
246
- actualWidth = displayHeight * videoAspectRatio;
247
- }
248
- const offsetX = (displayWidth - actualWidth) / 2;
249
- const offsetY = (displayHeight - actualHeight) / 2;
250
- const relativeX = clientX - rect.left - offsetX;
251
- const relativeY = clientY - rect.top - offsetY;
252
- const isInside =
253
- relativeX >= 0 && relativeX <= actualWidth && relativeY >= 0 && relativeY <= actualHeight;
254
-
255
- let videoX = 0;
256
- let videoY = 0;
257
- if (isInside) {
258
- videoX = Math.max(0, Math.min(videoWidth, (relativeX / actualWidth) * videoWidth));
259
- videoY = Math.max(0, Math.min(videoHeight, (relativeY / actualHeight) * videoHeight));
260
- }
261
- // --- End: Coordinate Calculation ---
299
+ // Minimal geometry for single-finger touch events (no mirror/container coords needed).
300
+ type PointerGeometry = {
301
+ inside: boolean;
302
+ videoX: number;
303
+ videoY: number;
304
+ videoWidth: number;
305
+ videoHeight: number;
306
+ };
262
307
 
263
- let action: number | null = null;
264
- let positionToSend: { x: number; y: number } | null = null;
265
- let pressure = 1.0; // Default pressure
266
- const buttons = AMOTION_EVENT.BUTTON_PRIMARY; // Assume primary button
308
+ const applyPointerEvent = (
309
+ pointerId: number,
310
+ eventType: 'down' | 'move' | 'up' | 'cancel',
311
+ geometry: PointerGeometry | null,
312
+ ) => {
313
+ if (!geometry) return;
314
+ const { inside: isInside, videoX, videoY, videoWidth, videoHeight } = geometry;
315
+
316
+ let action: number | null = null;
317
+ let positionToSend: { x: number; y: number } | null = null;
318
+ let pressure = 1.0; // Default pressure
319
+ const buttons = AMOTION_EVENT.BUTTON_PRIMARY; // Assume primary button
320
+
321
+ switch (eventType) {
322
+ case 'down':
323
+ if (isInside) {
324
+ // For multi-touch: use ACTION_DOWN for first pointer, ACTION_POINTER_DOWN for additional pointers
325
+ const currentPointerCount = activePointers.current.size;
326
+ action =
327
+ currentPointerCount === 0
328
+ ? AMOTION_EVENT.ACTION_DOWN
329
+ : AMOTION_EVENT.ACTION_POINTER_DOWN;
330
+ positionToSend = { x: videoX, y: videoY };
331
+ activePointers.current.set(pointerId, positionToSend);
332
+ if (pointerId === -1) {
333
+ // Focus on mouse down
334
+ videoRef.current?.focus();
335
+ }
336
+ } else {
337
+ // If the initial down event is outside, ignore it for this pointer
338
+ activePointers.current.delete(pointerId);
339
+ }
340
+ break;
267
341
 
268
- switch (eventType) {
269
- case 'down':
342
+ case 'move':
343
+ if (activePointers.current.has(pointerId)) {
270
344
  if (isInside) {
271
- action = AMOTION_EVENT.ACTION_DOWN;
345
+ action = AMOTION_EVENT.ACTION_MOVE;
272
346
  positionToSend = { x: videoX, y: videoY };
347
+ // Update the last known position for this active pointer
273
348
  activePointers.current.set(pointerId, positionToSend);
274
- if (pointerId === -1) {
275
- // Focus on mouse down
276
- videoRef.current?.focus();
277
- }
278
349
  } else {
279
- // If the initial down event is outside, ignore it for this pointer
280
- activePointers.current.delete(pointerId);
350
+ // Moved outside while active - do nothing, UP/CANCEL will use last known pos
281
351
  }
282
- break;
352
+ }
353
+ break;
283
354
 
284
- case 'move':
285
- if (activePointers.current.has(pointerId)) {
286
- if (isInside) {
287
- action = AMOTION_EVENT.ACTION_MOVE;
288
- positionToSend = { x: videoX, y: videoY };
289
- // Update the last known position for this active pointer
290
- activePointers.current.set(pointerId, positionToSend);
291
- } else {
292
- // Moved outside while active - do nothing, UP/CANCEL will use last known pos
293
- }
294
- }
295
- break;
355
+ case 'up':
356
+ case 'cancel': // Treat cancel like up, but use ACTION_CANCEL
357
+ if (activePointers.current.has(pointerId)) {
358
+ // IMPORTANT: Send the UP/CANCEL at the *last known position* inside the video
359
+ positionToSend = activePointers.current.get(pointerId)!;
360
+ activePointers.current.delete(pointerId); // Remove pointer as it's no longer active
296
361
 
297
- case 'up':
298
- case 'cancel': // Treat cancel like up, but use ACTION_CANCEL
299
- if (activePointers.current.has(pointerId)) {
300
- action = eventType === 'cancel' ? AMOTION_EVENT.ACTION_CANCEL : AMOTION_EVENT.ACTION_UP;
301
- // IMPORTANT: Send the UP/CANCEL at the *last known position* inside the video
302
- positionToSend = activePointers.current.get(pointerId)!;
303
- activePointers.current.delete(pointerId); // Remove pointer as it's no longer active
362
+ if (eventType === 'cancel') {
363
+ action = AMOTION_EVENT.ACTION_CANCEL;
364
+ } else {
365
+ // For multi-touch: use ACTION_UP for last pointer, ACTION_POINTER_UP for non-last pointers
366
+ const remainingPointerCount = activePointers.current.size;
367
+ action =
368
+ remainingPointerCount === 0
369
+ ? AMOTION_EVENT.ACTION_UP
370
+ : AMOTION_EVENT.ACTION_POINTER_UP;
304
371
  }
305
- break;
306
- }
372
+ }
373
+ break;
374
+ }
307
375
 
308
- // Send message if action and position determined
309
- if (action !== null && positionToSend !== null) {
310
- const message = createTouchControlMessage(
311
- action,
376
+ // Send message if action and position determined
377
+ if (action !== null && positionToSend !== null) {
378
+ debugLog('[rc-touch][mouse->touch] sending', {
312
379
  pointerId,
313
- videoWidth,
314
- videoHeight,
315
- positionToSend.x,
316
- positionToSend.y,
317
- pressure,
318
- buttons,
319
- buttons,
320
- );
321
- if (message) {
322
- sendBinaryControlMessage(message);
323
- }
324
- } else if (eventType === 'up' || eventType === 'cancel') {
325
- // Clean up map just in case if 'down' was outside and 'up'/'cancel' is triggered
326
- activePointers.current.delete(pointerId);
380
+ eventType,
381
+ action,
382
+ actionName: motionActionToString(action),
383
+ isInside,
384
+ positionToSend,
385
+ video: { width: videoWidth, height: videoHeight },
386
+ altHeld: isAltHeldRef.current,
387
+ activePointersAfter: Array.from(activePointers.current.entries()).map(([id, pos]) => ({
388
+ id,
389
+ x: pos.x,
390
+ y: pos.y,
391
+ })),
392
+ });
393
+ const message = createTouchControlMessage(
394
+ action,
395
+ pointerId,
396
+ videoWidth,
397
+ videoHeight,
398
+ positionToSend.x,
399
+ positionToSend.y,
400
+ pressure,
401
+ buttons,
402
+ buttons,
403
+ );
404
+ if (message) {
405
+ debugLog('[rc-touch][mouse->touch] buffer', {
406
+ pointerId,
407
+ actionName: motionActionToString(action),
408
+ byteLength: message.byteLength,
409
+ });
410
+ sendBinaryControlMessage(message);
327
411
  }
412
+ } else if (eventType === 'up' || eventType === 'cancel') {
413
+ // Clean up map just in case if 'down' was outside and 'up'/'cancel' is triggered
414
+ activePointers.current.delete(pointerId);
415
+ }
416
+ };
417
+
418
+ // Update Alt modifier state. Only iOS Simulator uses Indigo modifier injection.
419
+ const updateAltHeld = (nextHeld: boolean) => {
420
+ if (isAltHeldRef.current === nextHeld) {
421
+ return;
422
+ }
423
+ isAltHeldRef.current = nextHeld;
424
+ setIsAltHeld(nextHeld);
425
+
426
+ // Clear hover point when Alt is released to hide indicators immediately.
427
+ if (!nextHeld) {
428
+ setHoverPoint(null);
429
+ }
430
+
431
+ debugLog('[rc-touch][alt] updateAltHeld', {
432
+ nextHeld,
433
+ activePointerIds: Array.from(activePointers.current.keys()),
434
+ });
435
+
436
+ // iOS Simulator pinch (Option/Alt+drag) behavior depends on the Option modifier being
437
+ // active on the Indigo HID side. Send Alt key down/up immediately on toggle so the
438
+ // sequence matches Simulator.app (Alt down -> mouse down/drag -> mouse up -> Alt up).
439
+ // This is iOS-specific; Android doesn't use this modifier injection.
440
+ if (platform === 'ios' && dataChannelRef.current && dataChannelRef.current.readyState === 'open') {
441
+ const action = nextHeld ? ANDROID_KEYS.ACTION_DOWN : ANDROID_KEYS.ACTION_UP;
442
+ const message = createInjectKeycodeMessage(action, ANDROID_KEYS.KEYCODE_ALT_LEFT, 0, ANDROID_KEYS.META_NONE);
443
+ debugLog('[rc-touch][alt] sending Indigo modifier keycode', {
444
+ action,
445
+ keycode: ANDROID_KEYS.KEYCODE_ALT_LEFT,
446
+ });
447
+ if (message) {
448
+ sendBinaryControlMessage(message);
449
+ }
450
+ }
451
+ };
452
+
453
+ // Mapping context computed once per DOM event, then reused for each pointer.
454
+ type VideoMappingContext = {
455
+ videoWidth: number;
456
+ videoHeight: number;
457
+ videoRect: DOMRect;
458
+ containerRect: DOMRect;
459
+ actualWidth: number;
460
+ actualHeight: number;
461
+ offsetX: number;
462
+ offsetY: number;
463
+ };
464
+
465
+ // Compute mapping context from current video/container state (once per event).
466
+ const computeVideoMappingContext = (): VideoMappingContext | null => {
467
+ const video = videoRef.current;
468
+ const container = containerRef.current;
469
+ if (!video || !container) return null;
470
+
471
+ const videoWidth = video.videoWidth;
472
+ const videoHeight = video.videoHeight;
473
+ if (!videoWidth || !videoHeight) return null;
474
+
475
+ const videoRect = video.getBoundingClientRect();
476
+ const containerRect = container.getBoundingClientRect();
477
+
478
+ const displayWidth = videoRect.width;
479
+ const displayHeight = videoRect.height;
480
+ const videoAspectRatio = videoWidth / videoHeight;
481
+ const containerAspectRatio = displayWidth / displayHeight;
482
+
483
+ let actualWidth = displayWidth;
484
+ let actualHeight = displayHeight;
485
+ if (videoAspectRatio > containerAspectRatio) {
486
+ actualHeight = displayWidth / videoAspectRatio;
487
+ } else {
488
+ actualWidth = displayHeight * videoAspectRatio;
489
+ }
490
+
491
+ const offsetX = (displayWidth - actualWidth) / 2;
492
+ const offsetY = (displayHeight - actualHeight) / 2;
493
+
494
+ return {
495
+ videoWidth,
496
+ videoHeight,
497
+ videoRect,
498
+ containerRect,
499
+ actualWidth,
500
+ actualHeight,
501
+ offsetX,
502
+ offsetY,
328
503
  };
504
+ };
505
+
506
+ // Map a client point to video coordinates using a pre-computed context.
507
+ // Returns null if outside the video content area or context is missing.
508
+ const mapClientPointToVideo = (
509
+ ctx: VideoMappingContext,
510
+ clientX: number,
511
+ clientY: number,
512
+ ): PointerGeometry | null => {
513
+ const relativeX = clientX - ctx.videoRect.left - ctx.offsetX;
514
+ const relativeY = clientY - ctx.videoRect.top - ctx.offsetY;
515
+
516
+ const isInside =
517
+ relativeX >= 0 && relativeX <= ctx.actualWidth &&
518
+ relativeY >= 0 && relativeY <= ctx.actualHeight;
519
+
520
+ if (!isInside) {
521
+ return {
522
+ inside: false,
523
+ videoX: 0,
524
+ videoY: 0,
525
+ videoWidth: ctx.videoWidth,
526
+ videoHeight: ctx.videoHeight,
527
+ };
528
+ }
529
+
530
+ const videoX = Math.max(0, Math.min(ctx.videoWidth, (relativeX / ctx.actualWidth) * ctx.videoWidth));
531
+ const videoY = Math.max(0, Math.min(ctx.videoHeight, (relativeY / ctx.actualHeight) * ctx.videoHeight));
532
+
533
+ return {
534
+ inside: true,
535
+ videoX,
536
+ videoY,
537
+ videoWidth: ctx.videoWidth,
538
+ videoHeight: ctx.videoHeight,
539
+ };
540
+ };
541
+
542
+ // Compute full hover point with mirror/container coordinates (for Alt indicator rendering).
543
+ const computeFullHoverPoint = (
544
+ ctx: VideoMappingContext,
545
+ clientX: number,
546
+ clientY: number,
547
+ ): HoverPoint | null => {
548
+ const relativeX = clientX - ctx.videoRect.left - ctx.offsetX;
549
+ const relativeY = clientY - ctx.videoRect.top - ctx.offsetY;
550
+
551
+ const isInside =
552
+ relativeX >= 0 && relativeX <= ctx.actualWidth &&
553
+ relativeY >= 0 && relativeY <= ctx.actualHeight;
554
+
555
+ if (!isInside) {
556
+ return null;
557
+ }
558
+
559
+ const videoX = Math.max(0, Math.min(ctx.videoWidth, (relativeX / ctx.actualWidth) * ctx.videoWidth));
560
+ const videoY = Math.max(0, Math.min(ctx.videoHeight, (relativeY / ctx.actualHeight) * ctx.videoHeight));
561
+ const mirrorVideoX = ctx.videoWidth - videoX;
562
+ const mirrorVideoY = ctx.videoHeight - videoY;
563
+
564
+ const contentLeft = ctx.videoRect.left + ctx.offsetX;
565
+ const contentTop = ctx.videoRect.top + ctx.offsetY;
566
+ const containerX = contentLeft - ctx.containerRect.left + relativeX;
567
+ const containerY = contentTop - ctx.containerRect.top + relativeY;
568
+ const mirrorContainerX = contentLeft - ctx.containerRect.left + (ctx.actualWidth - relativeX);
569
+ const mirrorContainerY = contentTop - ctx.containerRect.top + (ctx.actualHeight - relativeY);
570
+
571
+ return {
572
+ containerX,
573
+ containerY,
574
+ mirrorContainerX,
575
+ mirrorContainerY,
576
+ videoX,
577
+ videoY,
578
+ mirrorVideoX,
579
+ mirrorVideoY,
580
+ videoWidth: ctx.videoWidth,
581
+ videoHeight: ctx.videoHeight,
582
+ };
583
+ };
584
+
585
+ // Helper to send a two-finger touch message (iOS-specific type=18 message).
586
+ const sendTwoFingerMessage = (
587
+ action: number,
588
+ videoWidth: number,
589
+ videoHeight: number,
590
+ x0: number,
591
+ y0: number,
592
+ x1: number,
593
+ y1: number,
594
+ ) => {
595
+ const msg = createTwoFingerTouchControlMessage(
596
+ action,
597
+ videoWidth,
598
+ videoHeight,
599
+ x0,
600
+ y0,
601
+ x1,
602
+ y1,
603
+ );
604
+ debugLog('[rc-touch2] sendTwoFingerMessage (iOS)', {
605
+ actionName: motionActionToString(action),
606
+ video: { width: videoWidth, height: videoHeight },
607
+ p0: { x: x0, y: y0 },
608
+ p1: { x: x1, y: y1 },
609
+ byteLength: msg.byteLength,
610
+ });
611
+ sendBinaryControlMessage(msg);
612
+ };
613
+
614
+ // Generic two-finger event handler - sends platform-appropriate messages.
615
+ // iOS: uses special two-finger message (type=18)
616
+ // Android: sends two separate single-touch messages with proper action sequencing
617
+ const applyTwoFingerEvent = (
618
+ eventType: 'down' | 'move' | 'up',
619
+ videoWidth: number,
620
+ videoHeight: number,
621
+ x0: number,
622
+ y0: number,
623
+ x1: number,
624
+ y1: number,
625
+ pointerId0: number,
626
+ pointerId1: number,
627
+ ) => {
628
+ debugLog('[rc-touch2] applyTwoFingerEvent', {
629
+ platform,
630
+ eventType,
631
+ video: { width: videoWidth, height: videoHeight },
632
+ p0: { x: x0, y: y0, id: pointerId0 },
633
+ p1: { x: x1, y: y1, id: pointerId1 },
634
+ });
635
+
636
+ if (platform === 'ios') {
637
+ // iOS: use special two-finger message (type=18)
638
+ const action = eventType === 'down' ? AMOTION_EVENT.ACTION_DOWN
639
+ : eventType === 'move' ? AMOTION_EVENT.ACTION_MOVE
640
+ : AMOTION_EVENT.ACTION_UP;
641
+ sendTwoFingerMessage(action, videoWidth, videoHeight, x0, y0, x1, y1);
642
+ } else {
643
+ // Android: send two separate single-touch messages with proper action codes
644
+ // Per scrcpy protocol, each finger is a separate INJECT_TOUCH_EVENT with unique pointerId
645
+ if (eventType === 'down') {
646
+ // First finger down (ACTION_DOWN), then second finger down (ACTION_POINTER_DOWN)
647
+ sendSingleTouch(AMOTION_EVENT.ACTION_DOWN, pointerId0, videoWidth, videoHeight, x0, y0);
648
+ sendSingleTouch(AMOTION_EVENT.ACTION_POINTER_DOWN, pointerId1, videoWidth, videoHeight, x1, y1);
649
+ } else if (eventType === 'move') {
650
+ // Both fingers move (ACTION_MOVE for each)
651
+ sendSingleTouch(AMOTION_EVENT.ACTION_MOVE, pointerId0, videoWidth, videoHeight, x0, y0);
652
+ sendSingleTouch(AMOTION_EVENT.ACTION_MOVE, pointerId1, videoWidth, videoHeight, x1, y1);
653
+ } else {
654
+ // Second finger up (ACTION_POINTER_UP), then first finger up (ACTION_UP)
655
+ sendSingleTouch(AMOTION_EVENT.ACTION_POINTER_UP, pointerId1, videoWidth, videoHeight, x1, y1);
656
+ sendSingleTouch(AMOTION_EVENT.ACTION_UP, pointerId0, videoWidth, videoHeight, x0, y0);
657
+ }
658
+ }
659
+ };
660
+
661
+ // Update hover point only when Alt is held (to avoid re-renders in normal path).
662
+ const updateHoverPoint = (ctx: VideoMappingContext, clientX: number, clientY: number) => {
663
+ if (!isAltHeldRef.current) {
664
+ // Don't compute or update when Alt isn't held
665
+ if (hoverPoint !== null) {
666
+ setHoverPoint(null);
667
+ }
668
+ return;
669
+ }
670
+ const fullPoint = computeFullHoverPoint(ctx, clientX, clientY);
671
+ setHoverPoint(fullPoint);
672
+ };
673
+
674
+ // Unified handler for both mouse and touch interactions
675
+ const handleInteraction = (event: React.MouseEvent | React.TouchEvent) => {
676
+ event.preventDefault();
677
+ event.stopPropagation();
678
+
679
+ // Compute mapping context once per event (reused for all pointers)
680
+ const ctx = computeVideoMappingContext();
681
+
682
+ // Handle hover point updates for mouse events (only when Alt is held)
683
+ if (!('touches' in event) && ctx) {
684
+ if (event.type === 'mousemove') {
685
+ updateHoverPoint(ctx, event.clientX, event.clientY);
686
+ } else if (event.type === 'mouseleave') {
687
+ setHoverPoint(null);
688
+ }
689
+ // Note: Alt state is tracked via global keydown/keyup listeners, not event.altKey,
690
+ // to ensure consistent behavior across focus transitions.
691
+ }
692
+
693
+ if (!dataChannelRef.current || dataChannelRef.current.readyState !== 'open' || !videoRef.current || !ctx) {
694
+ return;
695
+ }
329
696
 
330
697
  // --- Event Type Handling ---
331
698
 
332
699
  if ('touches' in event) {
333
- // Touch Events
334
- const touches = event.changedTouches; // Use changedTouches for start/end/cancel
335
- let eventType: 'down' | 'move' | 'up' | 'cancel';
700
+ // Touch Events - handle both single-finger and two-finger gestures
701
+ const allTouches = event.touches; // All currently active touches
702
+ const changedTouches = event.changedTouches;
336
703
 
704
+ let eventType: 'down' | 'move' | 'up' | 'cancel';
337
705
  switch (event.type) {
338
706
  case 'touchstart':
339
707
  eventType = 'down';
@@ -348,45 +716,210 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
348
716
  eventType = 'cancel';
349
717
  break;
350
718
  default:
351
- return; // Should not happen
719
+ return;
352
720
  }
353
721
 
354
- for (let i = 0; i < touches.length; i++) {
355
- const touch = touches[i];
356
- processPointer(touch.identifier, touch.clientX, touch.clientY, eventType);
722
+ // Check if we have exactly 2 active touches - route to two-finger logic
723
+ if (allTouches.length === 2) {
724
+ const t0 = allTouches[0];
725
+ const t1 = allTouches[1];
726
+ const g0 = mapClientPointToVideo(ctx, t0.clientX, t0.clientY);
727
+ const g1 = mapClientPointToVideo(ctx, t1.clientX, t1.clientY);
728
+
729
+ if (!g0 || !g1) return;
730
+
731
+ if (!twoFingerStateRef.current) {
732
+ // Starting a new two-finger gesture
733
+ if (g0.inside && g1.inside) {
734
+ twoFingerStateRef.current = {
735
+ finger0: { x: g0.videoX, y: g0.videoY },
736
+ finger1: { x: g1.videoX, y: g1.videoY },
737
+ videoSize: { width: g0.videoWidth, height: g0.videoHeight },
738
+ source: 'real-touch',
739
+ pointerId0: t0.identifier,
740
+ pointerId1: t1.identifier,
741
+ };
742
+ applyTwoFingerEvent('down', g0.videoWidth, g0.videoHeight,
743
+ g0.videoX, g0.videoY, g1.videoX, g1.videoY,
744
+ t0.identifier, t1.identifier);
745
+ }
746
+ } else if (twoFingerStateRef.current.source === 'real-touch') {
747
+ // Continuing two-finger gesture (move)
748
+ if (g0.inside && g1.inside) {
749
+ twoFingerStateRef.current.finger0 = { x: g0.videoX, y: g0.videoY };
750
+ twoFingerStateRef.current.finger1 = { x: g1.videoX, y: g1.videoY };
751
+ applyTwoFingerEvent('move', g0.videoWidth, g0.videoHeight,
752
+ g0.videoX, g0.videoY, g1.videoX, g1.videoY,
753
+ twoFingerStateRef.current.pointerId0,
754
+ twoFingerStateRef.current.pointerId1);
755
+ }
756
+ }
757
+ } else if (allTouches.length < 2 && twoFingerStateRef.current?.source === 'real-touch') {
758
+ // Finger lifted - end two-finger gesture using last known state
759
+ const state = twoFingerStateRef.current;
760
+ applyTwoFingerEvent('up', state.videoSize.width, state.videoSize.height,
761
+ state.finger0.x, state.finger0.y,
762
+ state.finger1.x, state.finger1.y,
763
+ state.pointerId0, state.pointerId1);
764
+ twoFingerStateRef.current = null;
765
+ // Don't process remaining finger - gesture ended
766
+ return;
767
+ } else if (allTouches.length > 2) {
768
+ // 3+ fingers - not supported, ignore
769
+ return;
770
+ } else {
771
+ // Single finger touch (allTouches is 0 or 1)
772
+ // Note: allTouches=0 happens on touchend when last finger lifts
773
+ const touch = changedTouches[0];
774
+ if (touch) {
775
+ const geometry = mapClientPointToVideo(ctx, touch.clientX, touch.clientY);
776
+ applyPointerEvent(touch.identifier, eventType, geometry);
777
+ }
357
778
  }
358
779
  } else {
359
780
  // Mouse Events
360
- const pointerId = -1; // Use -1 for mouse pointer
781
+ const pointerId = -1; // Primary mouse pointer
361
782
  let eventType: 'down' | 'move' | 'up' | 'cancel' | null = null;
362
783
 
784
+ // Determine if we're in two-finger mode (Alt+mouse drag)
785
+ const inTwoFingerMode = twoFingerStateRef.current?.source === 'alt-mouse';
786
+
363
787
  switch (event.type) {
364
788
  case 'mousedown':
365
- if (event.button === 0) eventType = 'down'; // Only primary button
789
+ if (event.button === 0) eventType = 'down';
366
790
  break;
367
791
  case 'mousemove':
368
- // Only process move if primary button is down (check map)
369
- if (activePointers.current.has(pointerId)) {
792
+ // Process move if either in two-finger mode or has active pointer (normal drag)
793
+ if (inTwoFingerMode || activePointers.current.has(pointerId)) {
370
794
  eventType = 'move';
371
795
  }
372
796
  break;
373
797
  case 'mouseup':
374
- if (event.button === 0) eventType = 'up'; // Only primary button
798
+ if (event.button === 0) eventType = 'up';
375
799
  break;
376
800
  case 'mouseleave':
377
- // Treat leave like up only if button was down
378
- if (activePointers.current.has(pointerId)) {
801
+ // Treat leave like up only if in drag/two-finger mode
802
+ if (inTwoFingerMode || activePointers.current.has(pointerId)) {
379
803
  eventType = 'up';
380
804
  }
381
805
  break;
382
806
  }
383
807
 
384
808
  if (eventType) {
385
- processPointer(pointerId, event.clientX, event.clientY, eventType);
809
+ const geometry = mapClientPointToVideo(ctx, event.clientX, event.clientY);
810
+ if (!geometry) {
811
+ return;
812
+ }
813
+
814
+ debugLog('[rc-touch][mouse] event', {
815
+ domType: event.type,
816
+ eventType,
817
+ button: event.button,
818
+ buttons: (event as React.MouseEvent).buttons,
819
+ client: { x: event.clientX, y: event.clientY },
820
+ altHeldRef: isAltHeldRef.current,
821
+ inTwoFingerMode,
822
+ geometry: {
823
+ inside: geometry.inside,
824
+ videoX: geometry.videoX,
825
+ videoY: geometry.videoY,
826
+ videoWidth: geometry.videoWidth,
827
+ videoHeight: geometry.videoHeight,
828
+ },
829
+ activePointerIds: Array.from(activePointers.current.keys()),
830
+ });
831
+
832
+ // Route to two-finger (Alt+mouse) or single-finger path
833
+ if (isAltHeldRef.current || inTwoFingerMode) {
834
+ // Two-finger mode - Alt simulates second finger at mirror position
835
+ handleAltMouseGesture(eventType, geometry);
836
+ } else {
837
+ // Normal single-finger touch
838
+ applyPointerEvent(pointerId, eventType, geometry);
839
+ }
840
+ }
841
+ }
842
+ };
843
+
844
+ // Handle Alt+mouse gestures (simulated two-finger with mirror position).
845
+ // Works on both iOS and Android - applyTwoFingerEvent handles platform differences.
846
+ const handleAltMouseGesture = (
847
+ eventType: 'down' | 'move' | 'up' | 'cancel',
848
+ geometry: PointerGeometry,
849
+ ) => {
850
+ const { inside, videoX, videoY, videoWidth, videoHeight } = geometry;
851
+ const mirrorX = videoWidth - videoX;
852
+ const mirrorY = videoHeight - videoY;
853
+
854
+ if (eventType === 'down') {
855
+ if (inside) {
856
+ // Start two-finger gesture
857
+ twoFingerStateRef.current = {
858
+ finger0: { x: videoX, y: videoY },
859
+ finger1: { x: mirrorX, y: mirrorY },
860
+ videoSize: { width: videoWidth, height: videoHeight },
861
+ source: 'alt-mouse',
862
+ pointerId0: ALT_POINTER_ID_PRIMARY,
863
+ pointerId1: ALT_POINTER_ID_MIRROR,
864
+ };
865
+ videoRef.current?.focus();
866
+ applyTwoFingerEvent('down', videoWidth, videoHeight, videoX, videoY, mirrorX, mirrorY,
867
+ ALT_POINTER_ID_PRIMARY, ALT_POINTER_ID_MIRROR);
868
+ }
869
+ return;
870
+ }
871
+
872
+ if (eventType === 'move') {
873
+ if (twoFingerStateRef.current?.source === 'alt-mouse' && inside) {
874
+ // Update positions
875
+ twoFingerStateRef.current.finger0 = { x: videoX, y: videoY };
876
+ twoFingerStateRef.current.finger1 = { x: mirrorX, y: mirrorY };
877
+ applyTwoFingerEvent('move', videoWidth, videoHeight, videoX, videoY, mirrorX, mirrorY,
878
+ ALT_POINTER_ID_PRIMARY, ALT_POINTER_ID_MIRROR);
879
+ }
880
+ // If outside, we just don't send a move - UP will use last known position
881
+ return;
882
+ }
883
+
884
+ if (eventType === 'up' || eventType === 'cancel') {
885
+ const state = twoFingerStateRef.current;
886
+ if (state?.source === 'alt-mouse') {
887
+ // End gesture at last known inside positions
888
+ const { finger0, finger1, videoSize } = state;
889
+ applyTwoFingerEvent('up', videoSize.width, videoSize.height,
890
+ finger0.x, finger0.y, finger1.x, finger1.y,
891
+ ALT_POINTER_ID_PRIMARY, ALT_POINTER_ID_MIRROR);
892
+ twoFingerStateRef.current = null;
386
893
  }
894
+ return;
387
895
  }
388
896
  };
389
897
 
898
+ useEffect(() => {
899
+ const handleKeyDown = (event: KeyboardEvent) => {
900
+ if (event.key === 'Alt') {
901
+ updateAltHeld(true);
902
+ }
903
+ };
904
+ const handleKeyUp = (event: KeyboardEvent) => {
905
+ if (event.key === 'Alt') {
906
+ updateAltHeld(false);
907
+ }
908
+ };
909
+ const handleWindowBlur = () => {
910
+ updateAltHeld(false);
911
+ };
912
+ // Use capture phase so these fire before handleKeyboard's stopPropagation
913
+ window.addEventListener('keydown', handleKeyDown, true);
914
+ window.addEventListener('keyup', handleKeyUp, true);
915
+ window.addEventListener('blur', handleWindowBlur);
916
+ return () => {
917
+ window.removeEventListener('keydown', handleKeyDown, true);
918
+ window.removeEventListener('keyup', handleKeyUp, true);
919
+ window.removeEventListener('blur', handleWindowBlur);
920
+ };
921
+ }, []);
922
+
390
923
  const handleKeyboard = (event: React.KeyboardEvent) => {
391
924
  event.preventDefault();
392
925
  event.stopPropagation();
@@ -743,6 +1276,33 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
743
1276
  pendingScreenshotResolversRef.current.delete(message.id);
744
1277
  pendingScreenshotRejectersRef.current.delete(message.id);
745
1278
  break;
1279
+ case 'terminateAppResult':
1280
+ if (typeof message.id !== 'string') {
1281
+ debugWarn('Received invalid terminateApp result message:', message);
1282
+ break;
1283
+ }
1284
+ if (typeof message.error === 'string') {
1285
+ const terminateRejecter = pendingTerminateAppRejectersRef.current.get(message.id);
1286
+ if (!terminateRejecter) {
1287
+ debugWarn(`Received terminateApp error for unknown or handled id: ${message.id}`);
1288
+ break;
1289
+ }
1290
+ debugWarn(`Received terminateApp error for id ${message.id}: ${message.error}`);
1291
+ terminateRejecter(new Error(message.error));
1292
+ pendingTerminateAppResolversRef.current.delete(message.id);
1293
+ pendingTerminateAppRejectersRef.current.delete(message.id);
1294
+ break;
1295
+ }
1296
+ const terminateResolver = pendingTerminateAppResolversRef.current.get(message.id);
1297
+ if (!terminateResolver) {
1298
+ debugWarn(`Received terminateApp result for unknown or handled id: ${message.id}`);
1299
+ break;
1300
+ }
1301
+ debugLog(`Received terminateApp success for id ${message.id}`);
1302
+ terminateResolver();
1303
+ pendingTerminateAppResolversRef.current.delete(message.id);
1304
+ pendingTerminateAppRejectersRef.current.delete(message.id);
1305
+ break;
746
1306
  default:
747
1307
  debugWarn(`Received unhandled message type: ${message.type}`, message);
748
1308
  break;
@@ -988,10 +1548,51 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
988
1548
  }, 30000); // 30-second timeout
989
1549
  });
990
1550
  },
1551
+ terminateApp: (bundleId: string): Promise<void> => {
1552
+ return new Promise<void>((resolve, reject) => {
1553
+ if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
1554
+ debugWarn('WebSocket not open, cannot send terminateApp command.');
1555
+ return reject(new Error('WebSocket is not connected or connection is not open.'));
1556
+ }
1557
+ const id = `ui-term-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
1558
+ const request = {
1559
+ type: 'terminateApp',
1560
+ id,
1561
+ bundleId,
1562
+ };
1563
+
1564
+ pendingTerminateAppResolversRef.current.set(id, resolve);
1565
+ pendingTerminateAppRejectersRef.current.set(id, reject);
1566
+
1567
+ debugLog('Sending terminateApp request:', request);
1568
+ try {
1569
+ wsRef.current.send(JSON.stringify(request));
1570
+ } catch (err) {
1571
+ debugWarn('Failed to send terminateApp request immediately:', err);
1572
+ pendingTerminateAppResolversRef.current.delete(id);
1573
+ pendingTerminateAppRejectersRef.current.delete(id);
1574
+ reject(err);
1575
+ return;
1576
+ }
1577
+
1578
+ setTimeout(() => {
1579
+ if (pendingTerminateAppResolversRef.current.has(id)) {
1580
+ debugWarn(`terminateApp request timed out for id ${id}`);
1581
+ pendingTerminateAppRejectersRef.current.get(id)?.(new Error('terminateApp request timed out'));
1582
+ pendingTerminateAppResolversRef.current.delete(id);
1583
+ pendingTerminateAppRejectersRef.current.delete(id);
1584
+ }
1585
+ }, 30000);
1586
+ });
1587
+ },
991
1588
  }));
992
1589
 
1590
+ // Show indicators when Alt is held and we have a valid hover point (null when outside)
1591
+ const showAltIndicators = isAltHeld && hoverPoint !== null;
1592
+
993
1593
  return (
994
1594
  <div
1595
+ ref={containerRef}
995
1596
  className={clsx(
996
1597
  'rc-container',
997
1598
  className,
@@ -1008,6 +1609,24 @@ export const RemoteControl = forwardRef<RemoteControlHandle, RemoteControlProps>
1008
1609
  onTouchEnd={handleInteraction}
1009
1610
  onTouchCancel={handleInteraction}
1010
1611
  >
1612
+ {showAltIndicators && hoverPoint && (
1613
+ <>
1614
+ <div
1615
+ className="rc-touch-indicator"
1616
+ style={{
1617
+ left: `${hoverPoint.containerX}px`,
1618
+ top: `${hoverPoint.containerY}px`,
1619
+ }}
1620
+ />
1621
+ <div
1622
+ className="rc-touch-indicator"
1623
+ style={{
1624
+ left: `${hoverPoint.mirrorContainerX}px`,
1625
+ top: `${hoverPoint.mirrorContainerY}px`,
1626
+ }}
1627
+ />
1628
+ </>
1629
+ )}
1011
1630
  {showFrame && (
1012
1631
  <img
1013
1632
  ref={frameRef}