@tapflowio/ios-agent 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/IOSAgent.d.ts +18 -0
- package/dist/IOSAgent.d.ts.map +1 -1
- package/dist/IOSAgent.js +395 -48
- package/dist/IOSAgent.js.map +1 -1
- package/dist/MjpegStreamer.d.ts +2 -1
- package/dist/MjpegStreamer.d.ts.map +1 -1
- package/dist/MjpegStreamer.js +4 -2
- package/dist/MjpegStreamer.js.map +1 -1
- package/dist/ScreenCaptureStreamer.d.ts +2 -2
- package/dist/ScreenCaptureStreamer.d.ts.map +1 -1
- package/dist/ScreenCaptureStreamer.js +6 -3
- package/dist/ScreenCaptureStreamer.js.map +1 -1
- package/dist/SimctlWrapper.d.ts +11 -5
- package/dist/SimctlWrapper.d.ts.map +1 -1
- package/dist/SimctlWrapper.js +70 -33
- package/dist/SimctlWrapper.js.map +1 -1
- package/dist/simctl.d.ts +18 -0
- package/dist/simctl.d.ts.map +1 -1
- package/dist/simctl.js +110 -5
- package/dist/simctl.js.map +1 -1
- package/package.json +5 -4
package/dist/IOSAgent.js
CHANGED
|
@@ -7,6 +7,8 @@ import { spawnSync } from 'child_process';
|
|
|
7
7
|
import { WebSocket } from 'ws';
|
|
8
8
|
import { createLogger, PlatformError, ValidationError } from '@tapflowio/agent-core';
|
|
9
9
|
const logger = createLogger('ios-agent');
|
|
10
|
+
// Typed so a typo cannot ship silently — the viewer gates the whole clipboard bridge on this.
|
|
11
|
+
const AGENT_CAPABILITIES = ['clipboard'];
|
|
10
12
|
// Cross-platform button name → iOS device-chrome button name. Chrome uses
|
|
11
13
|
// hyphens and "power" (not "lock"); MCP's vocabulary uses underscores. Names
|
|
12
14
|
// not listed here (incl. the raw chrome names the dashboard sends) pass through.
|
|
@@ -16,7 +18,8 @@ export const IOS_BUTTON_ALIASES = {
|
|
|
16
18
|
volume_down: 'volume-down',
|
|
17
19
|
};
|
|
18
20
|
import { createResourceSampler, registerStreamWs, disableNagle, createKeyframeAwareSender, pickMaxSize, createRateLimitedDropWarn, createThroughputSampler, createSleepBlocker, getMachineId, isLocalhostWss, DEFAULT_BACKPRESSURE_BYTES, writeEnvelopeHeader, rewriteLowLatencySpsInFrame, CODEC_JPEG, CODEC_H264, CODEC_AUDIO, sendAudioYieldingToVideo, } from '@tapflowio/agent-core/utils';
|
|
19
|
-
import { SimctlWrapper, isDeviceMissingError } from './SimctlWrapper.js';
|
|
21
|
+
import { SimctlWrapper, isDeviceMissingError, ClipboardTooLargeError } from './SimctlWrapper.js';
|
|
22
|
+
import { MAX_CLIPBOARD_BYTES, clipboardByteLength, CLIPBOARD_SENTINEL_PREFIX as SENTINEL_PREFIX, isClipboardSentinel as isSentinel, CLIPBOARD_COPY_DEADLINE_MS, CLIPBOARD_WRITE_DEADLINE_MS, CLIPBOARD_RESTORE_DEADLINE_MS, CLIPBOARD_POLL_MS, createKeyedSerialQueue, } from '@tapflowio/agent-core';
|
|
20
23
|
import { ScreenCaptureStreamer } from './ScreenCaptureStreamer.js';
|
|
21
24
|
import { AudioCaptureStreamer, readSimVolume, applyGain } from './AudioCaptureStreamer.js';
|
|
22
25
|
import { ensureHelperApp, launchAudioHelper, isAudioSupported } from '@tapflowio/audiotap-helper';
|
|
@@ -103,6 +106,9 @@ export class IOSAgent {
|
|
|
103
106
|
ws.send(JSON.stringify({
|
|
104
107
|
type: 'agent:register',
|
|
105
108
|
platform: 'ios',
|
|
109
|
+
// Lets a viewer tell a clipboard-capable agent from one that predates the
|
|
110
|
+
// feature, instead of inferring it from silence. See agent-core AgentCapability.
|
|
111
|
+
capabilities: AGENT_CAPABILITIES,
|
|
106
112
|
agentId: getMachineId(),
|
|
107
113
|
agentName: os.hostname(),
|
|
108
114
|
devices: devices.map((d) => ({
|
|
@@ -172,6 +178,7 @@ export class IOSAgent {
|
|
|
172
178
|
sessionId,
|
|
173
179
|
deviceId,
|
|
174
180
|
touchHelper: null,
|
|
181
|
+
booted: false,
|
|
175
182
|
streamWs: null,
|
|
176
183
|
streamReader: null,
|
|
177
184
|
captureStreamer: null,
|
|
@@ -273,6 +280,7 @@ export class IOSAgent {
|
|
|
273
280
|
state.audioPids = null;
|
|
274
281
|
state.touchHelper?.stop();
|
|
275
282
|
state.touchHelper = null;
|
|
283
|
+
state.booted = false;
|
|
276
284
|
state.streamWs?.close();
|
|
277
285
|
state.streamWs = null;
|
|
278
286
|
}
|
|
@@ -308,7 +316,7 @@ export class IOSAgent {
|
|
|
308
316
|
let stream;
|
|
309
317
|
if (this.intervalMs !== undefined) {
|
|
310
318
|
state.captureStreamer = null;
|
|
311
|
-
stream = new MjpegStreamer(this.simctl, this.intervalMs).start();
|
|
319
|
+
stream = new MjpegStreamer(this.simctl, state.deviceId, this.intervalMs).start();
|
|
312
320
|
}
|
|
313
321
|
else {
|
|
314
322
|
const maxSize = pickMaxSize({
|
|
@@ -316,7 +324,7 @@ export class IOSAgent {
|
|
|
316
324
|
external: state.external,
|
|
317
325
|
override: process.env.TAPFLOW_IOS_MAX_SIZE ?? process.env.TAPFLOW_MAX_SIZE,
|
|
318
326
|
});
|
|
319
|
-
const capture = new ScreenCaptureStreamer(
|
|
327
|
+
const capture = new ScreenCaptureStreamer(state.deviceId, this.fps, useH264 ? 'h264' : 'jpeg', maxSize);
|
|
320
328
|
state.captureStreamer = capture;
|
|
321
329
|
stream = capture.start();
|
|
322
330
|
}
|
|
@@ -405,6 +413,7 @@ export class IOSAgent {
|
|
|
405
413
|
state.captureStreamer = null; // reader.cancel() kills the helper proc; drop the ref so a stale requestKeyframe() no-ops
|
|
406
414
|
state.touchHelper?.stop();
|
|
407
415
|
state.touchHelper = null;
|
|
416
|
+
state.booted = false;
|
|
408
417
|
state.streamWs?.close();
|
|
409
418
|
state.streamWs = null;
|
|
410
419
|
this.ws.send(JSON.stringify({ type: 'device:booting', sessionId }));
|
|
@@ -416,7 +425,48 @@ export class IOSAgent {
|
|
|
416
425
|
if (!target)
|
|
417
426
|
throw new PlatformError(`Device not found: ${deviceId}`);
|
|
418
427
|
if (fullErase) {
|
|
419
|
-
|
|
428
|
+
// `simctl erase` only accepts a shut-down device, and a device is often already running —
|
|
429
|
+
// it survives agent restarts and sessions that ended without a clean shutdown. Without this
|
|
430
|
+
// the boot fails outright (#439).
|
|
431
|
+
//
|
|
432
|
+
// Anything that is not `shutdown` gets the shutdown, not just `booted`: `toDeviceStatus`
|
|
433
|
+
// collapses `Booting` / `Shutting Down` / `Creating` into `unknown`, and `erase` refuses
|
|
434
|
+
// every one of them. Re-picking a device while its `device:shutdown` is still draining
|
|
435
|
+
// lands exactly there. `SimctlWrapper.shutdown` already tolerates an off device, so
|
|
436
|
+
// widening the condition costs nothing.
|
|
437
|
+
//
|
|
438
|
+
// This makes an unwanted reset *succeed* rather than fail, so it is only safe alongside the
|
|
439
|
+
// dashboard change that consumes the toggle after one use. The erase failing is what used
|
|
440
|
+
// to stand between a stale toggle and a wiped device.
|
|
441
|
+
if (target.status !== 'shutdown')
|
|
442
|
+
await this.simctl.shutdown(deviceId);
|
|
443
|
+
// The shutdown is an await, so a newer boot for this device may have overtaken us while it
|
|
444
|
+
// ran. Without this check the superseded boot goes on to erase a device the tester has
|
|
445
|
+
// since re-picked with the toggle off — the exact wipe-with-no-click this issue is about.
|
|
446
|
+
if (seq !== state.bootSeq)
|
|
447
|
+
return;
|
|
448
|
+
try {
|
|
449
|
+
await this.simctl.erase(deviceId);
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
// We powered the device off to make the erase possible. If the erase then fails, leaving
|
|
453
|
+
// it off strands the tester with a dead device on top of the error message, so put it
|
|
454
|
+
// back the way we found it and report the original failure.
|
|
455
|
+
//
|
|
456
|
+
// `=== 'booted'`, deliberately narrower than the `!== 'shutdown'` above: that one asks
|
|
457
|
+
// "does this need shutting down", this one asks "did we take it down". `unknown` covers
|
|
458
|
+
// `Shutting Down` (someone else was already stopping it) and `Creating` (never ran) —
|
|
459
|
+
// booting either would not be putting it back.
|
|
460
|
+
//
|
|
461
|
+
// The seq check matters as much: erase takes seconds, and a shutdown arriving mid-flight
|
|
462
|
+
// bumps bootSeq. Recovering then would power a device back up right after the tester
|
|
463
|
+
// asked for it to stop — and the throw below is swallowed by the outer catch on the same
|
|
464
|
+
// seq mismatch, so nothing would say why.
|
|
465
|
+
if (seq === state.bootSeq && target.status === 'booted') {
|
|
466
|
+
await this.simctl.boot(deviceId).catch(() => { });
|
|
467
|
+
}
|
|
468
|
+
throw err;
|
|
469
|
+
}
|
|
420
470
|
await this.simctl.boot(deviceId);
|
|
421
471
|
}
|
|
422
472
|
else if (target.status !== 'booted') {
|
|
@@ -440,6 +490,7 @@ export class IOSAgent {
|
|
|
440
490
|
// — never blocks/affects the video path.
|
|
441
491
|
if (this.audioEnabled())
|
|
442
492
|
this.startAudioCapture(state, streamWs, deviceId);
|
|
493
|
+
state.booted = true;
|
|
443
494
|
this.ws?.send(JSON.stringify({ type: 'device:ready', sessionId, payload: { deviceId } }));
|
|
444
495
|
// Sync AppleKeyboards after ready — fire-and-forget so streaming isn't delayed.
|
|
445
496
|
// hw=Automatic lets the hardware layout follow the active input source on LANG1/CapsLock.
|
|
@@ -480,6 +531,7 @@ export class IOSAgent {
|
|
|
480
531
|
state.captureStreamer = null; // reader.cancel() kills the helper proc; drop the ref so a stale requestKeyframe() no-ops
|
|
481
532
|
state.touchHelper?.stop();
|
|
482
533
|
state.touchHelper = null;
|
|
534
|
+
state.booted = false;
|
|
483
535
|
state.streamWs?.close();
|
|
484
536
|
state.streamWs = null;
|
|
485
537
|
this.lastBundleIds.delete(deviceId);
|
|
@@ -499,6 +551,51 @@ export class IOSAgent {
|
|
|
499
551
|
logger.error('shutdown failed:', message);
|
|
500
552
|
}
|
|
501
553
|
}
|
|
554
|
+
// Lazily set up the touch channel: a reconnect re-issues the session with touchHelper=null while the sim stays booted, so input self-heals without a fresh device:boot. Sync so a tap's start+end stay paired (async would drop touchEnd → stuck finger).
|
|
555
|
+
ensureTouchHelper(state) {
|
|
556
|
+
if (state.touchHelper)
|
|
557
|
+
return;
|
|
558
|
+
state.touchHelper = new TouchHelper(state.deviceId);
|
|
559
|
+
state.touchHelper.start();
|
|
560
|
+
}
|
|
561
|
+
// Clipboard work parks a sentinel on the device while it waits, so two operations on the same
|
|
562
|
+
// device must not interleave — each would read the other's marker instead of the real
|
|
563
|
+
// clipboard. Keyed by device, not session: several sessions (and MCP) can address one device.
|
|
564
|
+
clipboardQueue = createKeyedSerialQueue();
|
|
565
|
+
// Device-scoped, not operation-scoped. Several sessions (and MCP) can address one simulator,
|
|
566
|
+
// so an operation that fails before parking anything may still be answering while ANOTHER
|
|
567
|
+
// holds a marker down. The viewer decides from this whether pressing the plain chord is safe,
|
|
568
|
+
// and that chord travels as `input:key` — outside this queue — so the answer has to describe
|
|
569
|
+
// the device rather than the caller.
|
|
570
|
+
parkedSentinels = new Map();
|
|
571
|
+
markSentinel(deviceId, delta) {
|
|
572
|
+
const n = (this.parkedSentinels.get(deviceId) ?? 0) + delta;
|
|
573
|
+
if (n > 0)
|
|
574
|
+
this.parkedSentinels.set(deviceId, n);
|
|
575
|
+
else
|
|
576
|
+
this.parkedSentinels.delete(deviceId);
|
|
577
|
+
}
|
|
578
|
+
sentinelParked(deviceId) {
|
|
579
|
+
return (this.parkedSentinels.get(deviceId) ?? 0) > 0;
|
|
580
|
+
}
|
|
581
|
+
// Ack a terminal input: input:done = dispatched to a booted device (not a landing guarantee — HID is fire-and-forget); input:error = no live channel / not booted. Off the sync inject path, so start/end pairing is unaffected.
|
|
582
|
+
async ackInput(state, dispatched) {
|
|
583
|
+
const booted = dispatched && (state.booted || (await this.isBooted(state.deviceId)));
|
|
584
|
+
if (booted)
|
|
585
|
+
state.booted = true; // cache the post-reconnect verify so later inputs skip simctl
|
|
586
|
+
this.ws?.send(JSON.stringify(booted
|
|
587
|
+
? { type: 'input:done', sessionId: state.sessionId }
|
|
588
|
+
: { type: 'input:error', sessionId: state.sessionId, message: dispatched ? 'device not booted' : 'input channel not ready' }));
|
|
589
|
+
}
|
|
590
|
+
async isBooted(deviceId) {
|
|
591
|
+
try {
|
|
592
|
+
const devices = await this.simctl.listDevices();
|
|
593
|
+
return devices.find((d) => d.id === deviceId)?.status === 'booted';
|
|
594
|
+
}
|
|
595
|
+
catch {
|
|
596
|
+
return false;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
502
599
|
handleRelayMessage(msg) {
|
|
503
600
|
switch (msg.type) {
|
|
504
601
|
case 'device:boot': {
|
|
@@ -518,7 +615,12 @@ export class IOSAgent {
|
|
|
518
615
|
case 'app:install': {
|
|
519
616
|
const { filePath, bundleId } = msg.payload;
|
|
520
617
|
const sessionId = msg.sessionId;
|
|
521
|
-
this.
|
|
618
|
+
const installState = this.deviceStates.get(sessionId);
|
|
619
|
+
if (!installState) {
|
|
620
|
+
this.ws?.send(JSON.stringify({ type: 'app:install-error', sessionId, message: 'No booted device' }));
|
|
621
|
+
break;
|
|
622
|
+
}
|
|
623
|
+
this.installBuild(installState.deviceId, filePath, bundleId)
|
|
522
624
|
.then(() => this.ws?.send(JSON.stringify({ type: 'app:install-done', sessionId })))
|
|
523
625
|
.catch((e) => {
|
|
524
626
|
const message = e instanceof Error ? e.message : String(e);
|
|
@@ -530,11 +632,14 @@ export class IOSAgent {
|
|
|
530
632
|
const { bundleId } = msg.payload;
|
|
531
633
|
const sessionId = msg.sessionId;
|
|
532
634
|
const launchState = this.deviceStates.get(sessionId);
|
|
533
|
-
|
|
635
|
+
if (!launchState) {
|
|
636
|
+
this.ws?.send(JSON.stringify({ type: 'app:launch-error', sessionId, message: 'No booted device' }));
|
|
637
|
+
break;
|
|
638
|
+
}
|
|
639
|
+
this.simctl.launchApp(launchState.deviceId, bundleId)
|
|
534
640
|
.then(() => {
|
|
535
641
|
// Track the foreground app so ui:tree:request can query it via XCUITest.
|
|
536
|
-
|
|
537
|
-
this.lastBundleIds.set(launchState.deviceId, bundleId);
|
|
642
|
+
this.lastBundleIds.set(launchState.deviceId, bundleId);
|
|
538
643
|
// Audio: the whole-sim tap's poll picks up the launched app process within one interval;
|
|
539
644
|
// no per-launch helper needed.
|
|
540
645
|
this.ws?.send(JSON.stringify({ type: 'app:launch-done', sessionId }));
|
|
@@ -547,12 +652,11 @@ export class IOSAgent {
|
|
|
547
652
|
}
|
|
548
653
|
case 'input:touch:start': {
|
|
549
654
|
const state = this.deviceStates.get(msg.sessionId);
|
|
550
|
-
if (!state
|
|
551
|
-
logger.error('touch:start — touchHelper not ready');
|
|
655
|
+
if (!state)
|
|
552
656
|
break;
|
|
553
|
-
|
|
657
|
+
this.ensureTouchHelper(state);
|
|
554
658
|
const { x, y } = msg.payload;
|
|
555
|
-
state.touchHelper
|
|
659
|
+
state.touchHelper?.touchStart(x, y);
|
|
556
660
|
break;
|
|
557
661
|
}
|
|
558
662
|
case 'input:touch:move': {
|
|
@@ -565,15 +669,19 @@ export class IOSAgent {
|
|
|
565
669
|
}
|
|
566
670
|
case 'input:touch:end': {
|
|
567
671
|
const state = this.deviceStates.get(msg.sessionId);
|
|
568
|
-
state
|
|
672
|
+
if (!state)
|
|
673
|
+
break;
|
|
674
|
+
state.touchHelper?.touchEnd();
|
|
675
|
+
void this.ackInput(state, state.touchHelper !== null); // terminal of a tap/swipe → ack the gesture
|
|
569
676
|
break;
|
|
570
677
|
}
|
|
571
678
|
case 'input:pinch:start': {
|
|
572
679
|
const state = this.deviceStates.get(msg.sessionId);
|
|
573
|
-
if (!state
|
|
680
|
+
if (!state)
|
|
574
681
|
break;
|
|
682
|
+
this.ensureTouchHelper(state);
|
|
575
683
|
const { f0, f1 } = msg.payload;
|
|
576
|
-
state.touchHelper
|
|
684
|
+
state.touchHelper?.pinchStart(f0.x, f0.y, f1.x, f1.y);
|
|
577
685
|
break;
|
|
578
686
|
}
|
|
579
687
|
case 'input:pinch:move': {
|
|
@@ -586,9 +694,10 @@ export class IOSAgent {
|
|
|
586
694
|
}
|
|
587
695
|
case 'input:pinch:end': {
|
|
588
696
|
const state = this.deviceStates.get(msg.sessionId);
|
|
589
|
-
if (!state
|
|
697
|
+
if (!state)
|
|
590
698
|
break;
|
|
591
|
-
state.touchHelper
|
|
699
|
+
state.touchHelper?.pinchEnd();
|
|
700
|
+
void this.ackInput(state, state.touchHelper !== null);
|
|
592
701
|
break;
|
|
593
702
|
}
|
|
594
703
|
case 'input:rotate': {
|
|
@@ -628,6 +737,8 @@ export class IOSAgent {
|
|
|
628
737
|
case 'input:type': {
|
|
629
738
|
const sessionId = msg.sessionId;
|
|
630
739
|
const state = this.deviceStates.get(sessionId);
|
|
740
|
+
if (state)
|
|
741
|
+
this.ensureTouchHelper(state);
|
|
631
742
|
const { text } = (msg.payload ?? {});
|
|
632
743
|
if (!state?.touchHelper) {
|
|
633
744
|
this.ws?.send(JSON.stringify({ type: 'input:type-error', sessionId, message: 'No booted device' }));
|
|
@@ -651,7 +762,9 @@ export class IOSAgent {
|
|
|
651
762
|
};
|
|
652
763
|
// Ack on completion so a following input step (e.g. pressKey Enter) is
|
|
653
764
|
// only sent after the paste has actually landed.
|
|
654
|
-
|
|
765
|
+
// Shares the clipboard queue: this writes the pasteboard, so running it alongside a
|
|
766
|
+
// clipboard:read would overwrite that read's sentinel and be returned as "copied".
|
|
767
|
+
this.clipboardQueue(state.deviceId, doType)
|
|
655
768
|
.then(() => this.ws?.send(JSON.stringify({ type: 'input:type-done', sessionId })))
|
|
656
769
|
.catch((e) => {
|
|
657
770
|
const message = e instanceof Error ? e.message : String(e);
|
|
@@ -662,12 +775,15 @@ export class IOSAgent {
|
|
|
662
775
|
}
|
|
663
776
|
case 'input:key': {
|
|
664
777
|
const state = this.deviceStates.get(msg.sessionId);
|
|
665
|
-
if (!state
|
|
778
|
+
if (!state)
|
|
666
779
|
break;
|
|
780
|
+
this.ensureTouchHelper(state);
|
|
667
781
|
const { code, modifiers } = msg.payload;
|
|
668
782
|
const usage = KEY_CODE_MAP[code];
|
|
669
|
-
if (usage === undefined)
|
|
783
|
+
if (usage === undefined) {
|
|
784
|
+
this.ws?.send(JSON.stringify({ type: 'input:error', sessionId: msg.sessionId, message: `unknown key code: ${code}` }));
|
|
670
785
|
break;
|
|
786
|
+
}
|
|
671
787
|
if (state.softKeyboardVisible) {
|
|
672
788
|
// Hide the SW keyboard first so iOS re-initialises the HW keyboard
|
|
673
789
|
// context. Skipping this causes input-source desync (qks / ㅂㅏㄴ symptoms).
|
|
@@ -680,14 +796,16 @@ export class IOSAgent {
|
|
|
680
796
|
});
|
|
681
797
|
}
|
|
682
798
|
else {
|
|
683
|
-
state.touchHelper
|
|
799
|
+
state.touchHelper?.sendKey(usage, modifiers ?? 0);
|
|
684
800
|
}
|
|
801
|
+
void this.ackInput(state, state.touchHelper !== null);
|
|
685
802
|
break;
|
|
686
803
|
}
|
|
687
804
|
case 'input:button': {
|
|
688
805
|
const state = this.deviceStates.get(msg.sessionId);
|
|
689
|
-
if (!state
|
|
806
|
+
if (!state)
|
|
690
807
|
break;
|
|
808
|
+
this.ensureTouchHelper(state);
|
|
691
809
|
const { name, phase } = msg.payload;
|
|
692
810
|
// Map the cross-platform button vocabulary (used by MCP) onto this
|
|
693
811
|
// device's actual chrome button names. Dashboard already sends the raw
|
|
@@ -697,19 +815,20 @@ export class IOSAgent {
|
|
|
697
815
|
// Home has no HID down/up split — always a single legacy press. Send once on release
|
|
698
816
|
// (or on a phase-less legacy message) so a down+up pair doesn't fire it twice.
|
|
699
817
|
if (phase !== 'down')
|
|
700
|
-
state.touchHelper
|
|
818
|
+
state.touchHelper?.pressLegacyButton(0);
|
|
701
819
|
}
|
|
702
820
|
else {
|
|
703
821
|
const btn = state.loadedChrome?.buttons.find((b) => b.name === chromeName);
|
|
704
822
|
if (btn && btn.usagePage > 0 && btn.usage > 0) {
|
|
705
823
|
if (phase === 'down')
|
|
706
|
-
state.touchHelper
|
|
824
|
+
state.touchHelper?.pressButtonDown(btn.usagePage, btn.usage);
|
|
707
825
|
else if (phase === 'up')
|
|
708
|
-
state.touchHelper
|
|
826
|
+
state.touchHelper?.pressButtonUp(btn.usagePage, btn.usage);
|
|
709
827
|
else
|
|
710
|
-
state.touchHelper
|
|
828
|
+
state.touchHelper?.pressButton(btn.usagePage, btn.usage);
|
|
711
829
|
}
|
|
712
830
|
}
|
|
831
|
+
void this.ackInput(state, state.touchHelper !== null);
|
|
713
832
|
break;
|
|
714
833
|
}
|
|
715
834
|
case 'open-url': {
|
|
@@ -732,7 +851,16 @@ export class IOSAgent {
|
|
|
732
851
|
const raw = msg;
|
|
733
852
|
const { requestId, format } = raw;
|
|
734
853
|
const sessionId = msg.sessionId;
|
|
735
|
-
this.
|
|
854
|
+
const shotState = this.deviceStates.get(sessionId);
|
|
855
|
+
if (!shotState) {
|
|
856
|
+
this.ws?.send(JSON.stringify({ type: 'screenshot:error', sessionId, requestId, message: 'No booted device' }));
|
|
857
|
+
break;
|
|
858
|
+
}
|
|
859
|
+
// Note for anyone widening this signature again: before the udid parameter existed this
|
|
860
|
+
// read `screenshot(format ?? 'png')`, and adding a leading `udid: string` did NOT make it a
|
|
861
|
+
// type error — the format string simply became the device id. The compiler cannot help
|
|
862
|
+
// here; the test that scans exec arguments is what does.
|
|
863
|
+
this.simctl.screenshot(shotState.deviceId, format ?? 'png')
|
|
736
864
|
.then((buf) => this.ws?.send(JSON.stringify({
|
|
737
865
|
type: 'screenshot:done',
|
|
738
866
|
sessionId,
|
|
@@ -754,7 +882,7 @@ export class IOSAgent {
|
|
|
754
882
|
this.ws?.send(JSON.stringify({ type: 'app:clear-state-error', sessionId, message: !state ? 'No booted device' : 'bundleId missing' }));
|
|
755
883
|
break;
|
|
756
884
|
}
|
|
757
|
-
this.simctl.clearAppData(bundleId)
|
|
885
|
+
this.simctl.clearAppData(state.deviceId, bundleId)
|
|
758
886
|
.then(() => this.ws?.send(JSON.stringify({ type: 'app:clear-state-done', sessionId })))
|
|
759
887
|
.catch((e) => {
|
|
760
888
|
const message = e instanceof Error ? e.message : String(e);
|
|
@@ -762,6 +890,194 @@ export class IOSAgent {
|
|
|
762
890
|
});
|
|
763
891
|
break;
|
|
764
892
|
}
|
|
893
|
+
// Clipboard bridge. The chord is pressed HERE rather than by the viewer: the browser
|
|
894
|
+
// cannot know when the key actually lands (a visible software keyboard makes this path
|
|
895
|
+
// await hideSoftwareKeyboard first), and reading too early returns the PREVIOUS
|
|
896
|
+
// pasteboard — a stale value the user would never notice.
|
|
897
|
+
case 'clipboard:read': {
|
|
898
|
+
const { requestId } = msg;
|
|
899
|
+
const sessionId = msg.sessionId;
|
|
900
|
+
const state = this.deviceStates.get(sessionId);
|
|
901
|
+
if (!state) {
|
|
902
|
+
this.ws?.send(JSON.stringify({
|
|
903
|
+
type: 'clipboard:error', sessionId, requestId, message: 'No booted device',
|
|
904
|
+
payload: { sentinelParked: false },
|
|
905
|
+
}));
|
|
906
|
+
break;
|
|
907
|
+
}
|
|
908
|
+
const { press } = (msg.payload ?? {});
|
|
909
|
+
// Answering is separate from cleaning up. The restore below runs in a `finally`, which
|
|
910
|
+
// executes AFTER this — so the viewer hears back as soon as the outcome is known and
|
|
911
|
+
// does not wait out the restore window. The queue is still held until the restore
|
|
912
|
+
// finishes, which is what actually matters: the next operation must not see a sentinel.
|
|
913
|
+
const respond = (body) => this.ws?.send(JSON.stringify({ sessionId, requestId, ...body }));
|
|
914
|
+
const read = async () => {
|
|
915
|
+
if (!press) {
|
|
916
|
+
respond({ type: 'clipboard:data', payload: { text: await this.simctl.getPasteboard(state.deviceId) } });
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
919
|
+
this.ensureTouchHelper(state);
|
|
920
|
+
if (!state.touchHelper)
|
|
921
|
+
throw new PlatformError('Cannot press copy — no input channel to the device');
|
|
922
|
+
if (state.softKeyboardVisible) {
|
|
923
|
+
state.softKeyboardVisible = false;
|
|
924
|
+
await this.simctl.hideSoftwareKeyboard(state.deviceId).catch(() => { });
|
|
925
|
+
}
|
|
926
|
+
// Overwrite the pasteboard with a value only we could have written, then press the
|
|
927
|
+
// chord and wait for it to change. Without this there is no way to tell "the app
|
|
928
|
+
// copied" from "the app has not copied yet" — a fixed delay guesses, and guessing
|
|
929
|
+
// wrong hands the PREVIOUS clipboard to the user with no error. The sentinel also
|
|
930
|
+
// covers re-copying the identical text, where a plain value-change watch never fires.
|
|
931
|
+
// Read the original first. If we cannot (a hung simctl, or a pasteboard past
|
|
932
|
+
// maxBuffer), do NOT continue: parking a sentinel we are unable to undo would
|
|
933
|
+
// destroy whatever the user had on the device clipboard.
|
|
934
|
+
const raw = await this.simctl.getPasteboard(state.deviceId);
|
|
935
|
+
// Never restore someone else's marker as if it were the user's text.
|
|
936
|
+
const before = isSentinel(raw) ? '' : raw;
|
|
937
|
+
const sentinel = `${SENTINEL_PREFIX}${randomUUID()}`;
|
|
938
|
+
let copied = null;
|
|
939
|
+
// Counted before the call: setPasteboard can reject after the device took the value,
|
|
940
|
+
// so anything from here until the restore has to be treated as parked.
|
|
941
|
+
this.markSentinel(state.deviceId, 1);
|
|
942
|
+
try {
|
|
943
|
+
// Inside the try: if this rejects *after* the device applied it, the restore below
|
|
944
|
+
// still runs. Outside, a leaked sentinel would sit on the device permanently.
|
|
945
|
+
await this.simctl.setPasteboard(state.deviceId, sentinel);
|
|
946
|
+
// `pbcopy` exiting means the write was accepted, not that the pasteboard shows it
|
|
947
|
+
// yet. Pressing before it does lets the first poll read the PRE-sentinel value,
|
|
948
|
+
// decide it is not a sentinel, and return it as "what the app copied" — the exact
|
|
949
|
+
// staleness this whole mechanism exists to prevent. Mirrors the Android read path.
|
|
950
|
+
const applied = Date.now() + CLIPBOARD_WRITE_DEADLINE_MS;
|
|
951
|
+
while ((await this.simctl.getPasteboard(state.deviceId)) !== sentinel) {
|
|
952
|
+
if (Date.now() >= applied)
|
|
953
|
+
throw new PlatformError('The device clipboard did not respond');
|
|
954
|
+
await new Promise((r) => setTimeout(r, CLIPBOARD_POLL_MS));
|
|
955
|
+
}
|
|
956
|
+
state.touchHelper.sendKey(KEY_CODE_MAP[press === 'cut' ? 'KeyX' : 'KeyC'], MODIFIER_BITS['MetaLeft']);
|
|
957
|
+
const deadline = Date.now() + CLIPBOARD_COPY_DEADLINE_MS;
|
|
958
|
+
do {
|
|
959
|
+
let now;
|
|
960
|
+
try {
|
|
961
|
+
now = await this.simctl.getPasteboard(state.deviceId);
|
|
962
|
+
}
|
|
963
|
+
catch (e) {
|
|
964
|
+
// ONLY the size ceiling means the app copied — we just cannot carry it — so the
|
|
965
|
+
// restore below must be skipped or it would overwrite the very text the user
|
|
966
|
+
// copied. Every other failure here (a hung or timed-out pbpaste) says nothing
|
|
967
|
+
// about the device, and skipping the restore for those left the sentinel parked
|
|
968
|
+
// for good: the user's clipboard destroyed, and a zero-width marker pasted into
|
|
969
|
+
// the app under test. Android reaches the same two states, by assigning `copied`
|
|
970
|
+
// before it throws for the over-sized case and not otherwise.
|
|
971
|
+
if (e instanceof ClipboardTooLargeError)
|
|
972
|
+
copied = '';
|
|
973
|
+
throw e;
|
|
974
|
+
}
|
|
975
|
+
// A sentinel is never a copy result — ours means "not yet", any other means a
|
|
976
|
+
// concurrent operation slipped in and its marker must not be handed to the user.
|
|
977
|
+
if (!isSentinel(now)) {
|
|
978
|
+
copied = now;
|
|
979
|
+
respond({ type: 'clipboard:data', payload: { text: now } });
|
|
980
|
+
return;
|
|
981
|
+
}
|
|
982
|
+
await new Promise((r) => setTimeout(r, CLIPBOARD_POLL_MS));
|
|
983
|
+
} while (Date.now() < deadline);
|
|
984
|
+
throw new PlatformError('The device did not copy anything — is something selected?');
|
|
985
|
+
}
|
|
986
|
+
catch (e) {
|
|
987
|
+
// Reply here rather than letting this propagate: a rejection would surface only
|
|
988
|
+
// after `finally` had finished restoring, putting that window inside the round trip.
|
|
989
|
+
respond({
|
|
990
|
+
type: 'clipboard:error', message: e instanceof Error ? e.message : String(e),
|
|
991
|
+
// A sentinel may be on the device: setPasteboard can reject after the device took
|
|
992
|
+
// it. The viewer must not press the chord — the restore would overwrite the copy.
|
|
993
|
+
payload: { sentinelParked: this.sentinelParked(state.deviceId) },
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
finally {
|
|
997
|
+
// Only restore when the copy never happened; otherwise this would overwrite the
|
|
998
|
+
// value we just captured. A leaked sentinel would be worse than the original bug.
|
|
999
|
+
// Wait for the restore to be visible, not merely accepted: releasing the queue early
|
|
1000
|
+
// lets the next operation read the sentinel as "the original", which then becomes ''
|
|
1001
|
+
// and wipes the user's device clipboard. Mirrors the Android read path.
|
|
1002
|
+
if (copied === null) {
|
|
1003
|
+
await this.simctl.setPasteboard(state.deviceId, before).catch(() => { });
|
|
1004
|
+
const restored = Date.now() + CLIPBOARD_RESTORE_DEADLINE_MS;
|
|
1005
|
+
while ((await this.simctl.getPasteboard(state.deviceId).catch(() => before)) !== before) {
|
|
1006
|
+
if (Date.now() >= restored)
|
|
1007
|
+
break; // best effort; the error is already going out
|
|
1008
|
+
await new Promise((r) => setTimeout(r, CLIPBOARD_POLL_MS));
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
this.markSentinel(state.deviceId, -1);
|
|
1012
|
+
}
|
|
1013
|
+
};
|
|
1014
|
+
// `read` answers for itself once the sentinel is in play. This catches only what can
|
|
1015
|
+
// throw BEFORE that — the press-less branch, a missing input channel, and the failed
|
|
1016
|
+
// read of the original — so nothing is parked and the viewer's chord fallback is safe.
|
|
1017
|
+
this.clipboardQueue(state.deviceId, read).catch((e) => {
|
|
1018
|
+
respond({
|
|
1019
|
+
type: 'clipboard:error', message: e instanceof Error ? e.message : String(e),
|
|
1020
|
+
payload: { sentinelParked: this.sentinelParked(state.deviceId) },
|
|
1021
|
+
});
|
|
1022
|
+
});
|
|
1023
|
+
break;
|
|
1024
|
+
}
|
|
1025
|
+
case 'clipboard:write': {
|
|
1026
|
+
const { requestId } = msg;
|
|
1027
|
+
const sessionId = msg.sessionId;
|
|
1028
|
+
const state = this.deviceStates.get(sessionId);
|
|
1029
|
+
if (!state) {
|
|
1030
|
+
this.ws?.send(JSON.stringify({
|
|
1031
|
+
type: 'clipboard:error', sessionId, requestId, message: 'No booted device',
|
|
1032
|
+
payload: { sentinelParked: false },
|
|
1033
|
+
}));
|
|
1034
|
+
break;
|
|
1035
|
+
}
|
|
1036
|
+
const { text, pasteAfter } = (msg.payload ?? {});
|
|
1037
|
+
if (clipboardByteLength(text ?? '') > MAX_CLIPBOARD_BYTES) {
|
|
1038
|
+
this.ws?.send(JSON.stringify({
|
|
1039
|
+
type: 'clipboard:error', sessionId, requestId,
|
|
1040
|
+
message: `Clipboard is too large (max ${Math.floor(MAX_CLIPBOARD_BYTES / 1024)} KB)`,
|
|
1041
|
+
payload: { sentinelParked: this.sentinelParked(state.deviceId) },
|
|
1042
|
+
}));
|
|
1043
|
+
break;
|
|
1044
|
+
}
|
|
1045
|
+
const write = async () => {
|
|
1046
|
+
const wanted = text ?? '';
|
|
1047
|
+
await this.simctl.setPasteboard(state.deviceId, wanted);
|
|
1048
|
+
if (!pasteAfter)
|
|
1049
|
+
return;
|
|
1050
|
+
this.ensureTouchHelper(state);
|
|
1051
|
+
if (!state.touchHelper)
|
|
1052
|
+
throw new PlatformError('Cannot press paste — no input channel to the device');
|
|
1053
|
+
// Confirm the pasteboard really holds it before pressing paste; otherwise the device
|
|
1054
|
+
// could paste whatever was there before. (On Android the same call is documented as
|
|
1055
|
+
// asynchronous, so this is not iOS paranoia — it is the same failure on both sides.)
|
|
1056
|
+
const deadline = Date.now() + CLIPBOARD_WRITE_DEADLINE_MS;
|
|
1057
|
+
while ((await this.simctl.getPasteboard(state.deviceId).catch(() => null)) !== wanted) {
|
|
1058
|
+
if (Date.now() >= deadline)
|
|
1059
|
+
throw new PlatformError('The device clipboard did not accept the text');
|
|
1060
|
+
await new Promise((r) => setTimeout(r, CLIPBOARD_POLL_MS));
|
|
1061
|
+
}
|
|
1062
|
+
// Same guard as input:key — skipping it desyncs the hardware keyboard context.
|
|
1063
|
+
if (state.softKeyboardVisible) {
|
|
1064
|
+
state.softKeyboardVisible = false;
|
|
1065
|
+
await this.simctl.hideSoftwareKeyboard(state.deviceId).catch(() => { });
|
|
1066
|
+
}
|
|
1067
|
+
state.touchHelper.sendKey(KEY_CODE_MAP['KeyV'], MODIFIER_BITS['MetaLeft']);
|
|
1068
|
+
};
|
|
1069
|
+
// Ack only once the write (and the paste, when asked for) actually landed.
|
|
1070
|
+
this.clipboardQueue(state.deviceId, write)
|
|
1071
|
+
.then(() => this.ws?.send(JSON.stringify({ type: 'clipboard:write-done', sessionId, requestId })))
|
|
1072
|
+
.catch((e) => {
|
|
1073
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
1074
|
+
this.ws?.send(JSON.stringify({
|
|
1075
|
+
type: 'clipboard:error', sessionId, requestId, message,
|
|
1076
|
+
payload: { sentinelParked: this.sentinelParked(state.deviceId) },
|
|
1077
|
+
}));
|
|
1078
|
+
});
|
|
1079
|
+
break;
|
|
1080
|
+
}
|
|
765
1081
|
case 'ui:tree:request': {
|
|
766
1082
|
const raw = msg;
|
|
767
1083
|
const { requestId } = raw;
|
|
@@ -796,15 +1112,15 @@ export class IOSAgent {
|
|
|
796
1112
|
* 직접 설치. tar 추출은 실행 비트·심볼릭 링크를 보존하고(재압축이 아니라 네이티브 보관),
|
|
797
1113
|
* macOS tar(libarchive)가 path traversal·symlink 탈출을 기본 차단한다. 완료 후 임시 정리.
|
|
798
1114
|
*/
|
|
799
|
-
async installBuild(filePath, bundleId) {
|
|
1115
|
+
async installBuild(udid, filePath, bundleId) {
|
|
800
1116
|
if (bundleId) {
|
|
801
|
-
await this.simctl.uninstallApp(bundleId).catch(() => { });
|
|
1117
|
+
await this.simctl.uninstallApp(udid, bundleId).catch(() => { });
|
|
802
1118
|
}
|
|
803
1119
|
const lower = filePath.toLowerCase();
|
|
804
1120
|
const isTar = lower.endsWith('.tar.gz') || lower.endsWith('.tgz');
|
|
805
1121
|
const isZip = lower.endsWith('.zip');
|
|
806
1122
|
if (!isTar && !isZip) {
|
|
807
|
-
return this.simctl.installApp(filePath);
|
|
1123
|
+
return this.simctl.installApp(udid, filePath);
|
|
808
1124
|
}
|
|
809
1125
|
const tmpDir = path.join(tmpdir(), `tapflow-install-${randomUUID()}`);
|
|
810
1126
|
fs.mkdirSync(tmpDir, { recursive: true });
|
|
@@ -829,7 +1145,7 @@ export class IOSAgent {
|
|
|
829
1145
|
if (!appDir) {
|
|
830
1146
|
throw new ValidationError('.app 디렉토리를 찾을 수 없습니다. iphonesimulator 로 빌드한 .app 을 .app.zip 또는 .tar.gz 로 업로드하세요.');
|
|
831
1147
|
}
|
|
832
|
-
await this.simctl.installApp(path.join(tmpDir, appDir));
|
|
1148
|
+
await this.simctl.installApp(udid, path.join(tmpDir, appDir));
|
|
833
1149
|
}
|
|
834
1150
|
finally {
|
|
835
1151
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
@@ -839,12 +1155,45 @@ export class IOSAgent {
|
|
|
839
1155
|
listDevices() { return this.simctl.listDevices(); }
|
|
840
1156
|
boot(deviceId) { return this.simctl.boot(deviceId); }
|
|
841
1157
|
shutdown(deviceId) { return this.simctl.shutdown(deviceId); }
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1158
|
+
/** The `DeviceAgent` interface has no device parameter — it predates multi-session agents and is
|
|
1159
|
+
* shared with Android, so widening it would be a breaking change for every implementation and
|
|
1160
|
+
* caller. `AndroidAgent` resolves the single live session internally instead
|
|
1161
|
+
* (`AndroidAgent.ts:1429`); these do the same. Throwing beats falling back to simctl's `booted`
|
|
1162
|
+
* alias, which would quietly act on whichever simulator happened to be up. */
|
|
1163
|
+
async installApp(appPath) {
|
|
1164
|
+
await this.simctl.installApp(this.soleDeviceId(), appPath);
|
|
1165
|
+
}
|
|
1166
|
+
async launchApp(bundleId) {
|
|
1167
|
+
await this.simctl.launchApp(this.soleDeviceId(), bundleId);
|
|
1168
|
+
}
|
|
1169
|
+
/** The one booted session, or undefined when there is no unambiguous answer. Callers that can
|
|
1170
|
+
* reasonably do nothing (input) use this; callers that owe the user a result use
|
|
1171
|
+
* `soleDeviceState`, which throws instead. */
|
|
1172
|
+
liveDeviceState() {
|
|
1173
|
+
const live = [...this.deviceStates.values()].filter((s) => s.booted);
|
|
1174
|
+
return live.length === 1 ? live[0] : undefined;
|
|
1175
|
+
}
|
|
1176
|
+
soleDeviceState() {
|
|
1177
|
+
// `deviceStates` holds one entry per *registered* simulator, not per running one — the relay
|
|
1178
|
+
// opens a session for every device in `agent:register` and this Mac reports dozens. Taking the
|
|
1179
|
+
// first entry would pick whichever simulator simctl listed first, almost always a shut-down
|
|
1180
|
+
// one, which is worse than the `booted` alias it replaced: the alias at least found the device
|
|
1181
|
+
// that was actually up. `booted` is set by handleDeviceBoot, so filtering on it is the liveness
|
|
1182
|
+
// check `AndroidAgent` gets for free from `getSerial` (its serial map is only populated on
|
|
1183
|
+
// launch).
|
|
1184
|
+
const live = [...this.deviceStates.values()].filter((s) => s.booted);
|
|
1185
|
+
if (live.length === 0)
|
|
847
1186
|
throw new ValidationError('no booted device — call connect() first');
|
|
1187
|
+
// Refusing beats guessing: this interface has no way to say which device is meant, and picking
|
|
1188
|
+
// one silently is the whole defect being fixed here.
|
|
1189
|
+
if (live.length > 1) {
|
|
1190
|
+
throw new ValidationError(`${live.length} booted devices — this entry point cannot choose between them`);
|
|
1191
|
+
}
|
|
1192
|
+
return live[0];
|
|
1193
|
+
}
|
|
1194
|
+
soleDeviceId() { return this.soleDeviceState().deviceId; }
|
|
1195
|
+
async queryUITree() {
|
|
1196
|
+
const state = this.soleDeviceState();
|
|
848
1197
|
return this.readUITree(state);
|
|
849
1198
|
}
|
|
850
1199
|
// ── Audio output (opt-in, macOS 14.2+ Core Audio process taps) ───────────────────────────────
|
|
@@ -938,35 +1287,33 @@ export class IOSAgent {
|
|
|
938
1287
|
// stream cancelled or ws closed — expected on teardown/restart
|
|
939
1288
|
}
|
|
940
1289
|
}
|
|
941
|
-
|
|
1290
|
+
// async, not a bare `return`: `soleDeviceId` throws, and a synchronous throw out of a method
|
|
1291
|
+
// typed `Promise<T>` skips every caller's `.catch`.
|
|
1292
|
+
async screenshot() { return this.simctl.screenshot(this.soleDeviceId()); }
|
|
942
1293
|
stream() {
|
|
943
|
-
const first = this.
|
|
944
|
-
if (!first)
|
|
945
|
-
throw new ValidationError('no booted device — call connect() first');
|
|
1294
|
+
const first = this.soleDeviceState();
|
|
946
1295
|
// DeviceAgent.stream() is the platform-neutral Buffer contract; unwrap StreamFrame payloads.
|
|
947
|
-
return new ScreenCaptureStreamer(
|
|
1296
|
+
return new ScreenCaptureStreamer(first.deviceId, this.fps).start()
|
|
948
1297
|
.pipeThrough(new TransformStream({
|
|
949
1298
|
transform(frame, controller) { controller.enqueue(frame.payload); },
|
|
950
1299
|
}));
|
|
951
1300
|
}
|
|
952
1301
|
touchStart(x, y) {
|
|
953
|
-
const first = this.
|
|
1302
|
+
const first = this.liveDeviceState();
|
|
954
1303
|
first?.touchHelper?.touchStart(x, y);
|
|
955
1304
|
}
|
|
956
1305
|
touchMove(x, y) {
|
|
957
|
-
const first = this.
|
|
1306
|
+
const first = this.liveDeviceState();
|
|
958
1307
|
first?.touchHelper?.touchMove(x, y);
|
|
959
1308
|
return Promise.resolve();
|
|
960
1309
|
}
|
|
961
1310
|
touchEnd() {
|
|
962
|
-
const first = this.
|
|
1311
|
+
const first = this.liveDeviceState();
|
|
963
1312
|
first?.touchHelper?.touchEnd();
|
|
964
1313
|
return Promise.resolve();
|
|
965
1314
|
}
|
|
966
1315
|
openUrl(url) {
|
|
967
|
-
const first = this.
|
|
968
|
-
if (!first)
|
|
969
|
-
throw new ValidationError('no booted device — call connect() first');
|
|
1316
|
+
const first = this.soleDeviceState();
|
|
970
1317
|
return this.simctl.openUrl(first.deviceId, url);
|
|
971
1318
|
}
|
|
972
1319
|
}
|