@tapflowio/ios-agent 0.29.0 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/bin/guest-helper +0 -0
  2. package/bin/screencapture-helper +0 -0
  3. package/dist/AudioCaptureStreamer.d.ts +3 -2
  4. package/dist/AudioCaptureStreamer.d.ts.map +1 -1
  5. package/dist/AudioCaptureStreamer.js +16 -10
  6. package/dist/AudioCaptureStreamer.js.map +1 -1
  7. package/dist/DeviceChromeLoader.d.ts +28 -1
  8. package/dist/DeviceChromeLoader.d.ts.map +1 -1
  9. package/dist/DeviceChromeLoader.js +118 -37
  10. package/dist/DeviceChromeLoader.js.map +1 -1
  11. package/dist/FoldableInput.d.ts +58 -0
  12. package/dist/FoldableInput.d.ts.map +1 -0
  13. package/dist/FoldableInput.js +166 -0
  14. package/dist/FoldableInput.js.map +1 -0
  15. package/dist/GuestHelper.d.ts +109 -0
  16. package/dist/GuestHelper.d.ts.map +1 -0
  17. package/dist/GuestHelper.js +437 -0
  18. package/dist/GuestHelper.js.map +1 -0
  19. package/dist/IOSAgent.d.ts +126 -4
  20. package/dist/IOSAgent.d.ts.map +1 -1
  21. package/dist/IOSAgent.js +673 -62
  22. package/dist/IOSAgent.js.map +1 -1
  23. package/dist/InputChannel.d.ts +38 -0
  24. package/dist/InputChannel.d.ts.map +1 -0
  25. package/dist/InputChannel.js +7 -0
  26. package/dist/InputChannel.js.map +1 -0
  27. package/dist/PanelState.d.ts +72 -0
  28. package/dist/PanelState.d.ts.map +1 -0
  29. package/dist/PanelState.js +192 -0
  30. package/dist/PanelState.js.map +1 -0
  31. package/dist/PhysicalOrientation.d.ts +58 -0
  32. package/dist/PhysicalOrientation.d.ts.map +1 -0
  33. package/dist/PhysicalOrientation.js +155 -0
  34. package/dist/PhysicalOrientation.js.map +1 -0
  35. package/dist/ScreenCaptureStreamer.d.ts +12 -1
  36. package/dist/ScreenCaptureStreamer.d.ts.map +1 -1
  37. package/dist/ScreenCaptureStreamer.js +28 -3
  38. package/dist/ScreenCaptureStreamer.js.map +1 -1
  39. package/dist/SimProcessTree.d.ts +3 -1
  40. package/dist/SimProcessTree.d.ts.map +1 -1
  41. package/dist/SimProcessTree.js +14 -4
  42. package/dist/SimProcessTree.js.map +1 -1
  43. package/dist/SimctlWrapper.d.ts +20 -2
  44. package/dist/SimctlWrapper.d.ts.map +1 -1
  45. package/dist/SimctlWrapper.js +53 -3
  46. package/dist/SimctlWrapper.js.map +1 -1
  47. package/dist/SimulatorNetwork.d.ts +3 -2
  48. package/dist/SimulatorNetwork.d.ts.map +1 -1
  49. package/dist/SimulatorNetwork.js +3 -2
  50. package/dist/SimulatorNetwork.js.map +1 -1
  51. package/dist/TouchHelper.d.ts +3 -2
  52. package/dist/TouchHelper.d.ts.map +1 -1
  53. package/dist/TouchHelper.js +4 -6
  54. package/dist/TouchHelper.js.map +1 -1
  55. package/dist/TypePanels.d.ts +33 -0
  56. package/dist/TypePanels.d.ts.map +1 -0
  57. package/dist/TypePanels.js +68 -0
  58. package/dist/TypePanels.js.map +1 -0
  59. package/dist/launchArgs.d.ts +6 -0
  60. package/dist/launchArgs.d.ts.map +1 -0
  61. package/dist/launchArgs.js +21 -0
  62. package/dist/launchArgs.js.map +1 -0
  63. package/dist/panels.d.ts +52 -0
  64. package/dist/panels.d.ts.map +1 -0
  65. package/dist/panels.js +68 -0
  66. package/dist/panels.js.map +1 -0
  67. package/dist/simctl.d.ts +3 -0
  68. package/dist/simctl.d.ts.map +1 -1
  69. package/dist/simctl.js +1 -1
  70. package/dist/simctl.js.map +1 -1
  71. package/package.json +6 -5
package/dist/IOSAgent.js CHANGED
@@ -6,6 +6,7 @@ import { randomUUID } from 'crypto';
6
6
  import { spawn } from 'child_process';
7
7
  import { WebSocket } from 'ws';
8
8
  import { createLogger, PlatformError, ValidationError, bootAbandonMessage, BOOT_NO_SESSION_STATE, SHUTDOWN_NO_SESSION_STATE, downloadBuild } from '@tapflowio/agent-core';
9
+ import { appliedKeys, compactLaunchOptions, launchOptionsProblem } from '@tapflowio/protocol/launch';
9
10
  const logger = createLogger('ios-agent');
10
11
  /** The run reset's pasteboard step, the wait for the clipboard queue included; one step's bound like the others. */
11
12
  const RUN_RESET_PASTEBOARD_TIMEOUT_MS = 10_000;
@@ -16,7 +17,7 @@ const RUN_RESET_PASTEBOARD_TIMEOUT_MS = 10_000;
16
17
  // actually took is per device and per app, and `network:state.available` carries that — the split the
17
18
  // protocol documents. Added last, after the handler and the boot-time arming, because the string on
18
19
  // its own is what puts a control on screen.
19
- const AGENT_CAPABILITIES = ['clipboard', 'full-reset', 'network-control', 'build-download'];
20
+ const AGENT_CAPABILITIES = ['clipboard', 'full-reset', 'network-control', 'build-download', 'launch-options', 'launch-env'];
20
21
  // Human prose for each reason. Not in `@tapflowio/protocol`: that package's main entry must stay
21
22
  // runtime-free so it erases under `import type` and never reaches the dashboard bundle — a lookup
22
23
  // table is a runtime value. The `reason` on the wire is the machine-readable half; this is only what
@@ -53,8 +54,13 @@ import { ensureHelperApp, launchAudioHelper, isAudioSupported } from '@tapflowio
53
54
  import { enumerateSimPids } from './SimProcessTree.js';
54
55
  import { MjpegStreamer } from './MjpegStreamer.js';
55
56
  import { TouchHelper } from './TouchHelper.js';
57
+ import { FoldableInput } from './FoldableInput.js';
58
+ import { GuestHelper, defaultGuestHelperNative } from './GuestHelper.js';
59
+ import { PanelState, FOLD_SETTLE_MS } from './PanelState.js';
60
+ import { PhysicalOrientation, ORIENTATION_DEGREES, turned } from './PhysicalOrientation.js';
56
61
  import { XCUITreeReader } from './XCUITreeReader.js';
57
62
  import { DeviceChromeLoader } from './DeviceChromeLoader.js';
63
+ import { TypePanels, panelFrame } from './TypePanels.js';
58
64
  import { KEY_CODE_MAP, MODIFIER_BITS } from './KeyCodeMap.js';
59
65
  // whole-sim audio: how often to re-enumerate the simulator's process tree for new audio-producing
60
66
  // processes (launched apps, WebKit WebContent). Short enough that a tab's audio starts promptly,
@@ -71,6 +77,39 @@ export function leanSupported(osVersion) {
71
77
  const [major, minor] = [Number(m[1]), Number(m[2])];
72
78
  return major > 18 || (major === 18 && minor >= 5);
73
79
  }
80
+ // The postures of a multi-panel device (iPhone Duo), most closed first as the protocol requires. Two, not
81
+ // DeviceHub's three: its `flat` (180°) lights the same panel as `open`, and a step between them would read
82
+ // as a press that did nothing — the reason Android's list leaves out HALF_OPENED's twin. Labels match Android's.
83
+ const FOLDABLE_POSTURES = [{ id: 'closed', label: 'Folded' }, { id: 'open', label: 'Unfolded' }];
84
+ // The angles DeviceHub's pose picker sweeps to (closed reads ≈3° after it). Unfolded is its flat pose, 180°, not
85
+ // its half-open 130°: both light the same panel, and unfolded means opened all the way — 130 left DeviceHub
86
+ // showing the device half open after tapflow unfolded it.
87
+ const POSTURE_DEGREES = { closed: 0, open: 180 };
88
+ // devicectl costs ~0.23 s a read, so this is a fold made outside tapflow showing up within a few seconds,
89
+ // not a frame-rate watch. And only while something is moving: each read also costs ~0.2 s of CPU, so a 2 s
90
+ // poll held ~18% of a core for as long as a session was open (measured, 8 GB Mac mini). Once nothing has
91
+ // changed for PANEL_QUIET_MS it reads every PANEL_IDLE_POLL_MS — a fold in DeviceHub on a device left alone
92
+ // shows within 6 s rather than 2 — and the first change it sees, or a fold or turn sent from tapflow, brings
93
+ // it back.
94
+ const PANEL_POLL_MS = 2_000;
95
+ const PANEL_IDLE_POLL_MS = 6_000;
96
+ const PANEL_QUIET_MS = 30_000;
97
+ // How long a boot waits for a multi-panel device's helper. A helper that dies without saying `fatal` is
98
+ // relaunched on a rolling budget for ever, which is right for a session already on the device and wrong for a
99
+ // boot, which would otherwise never answer. Four of the helper's own 15 s start-up deadlines.
100
+ const HELPER_BOOT_READY_MS = 60_000;
101
+ // How long a turn sent through the helper has to show up in `devicectl device orientation get`. M1 read each
102
+ // one back 2.5 s after sending and every one had landed; the wait ends as soon as it does.
103
+ const ROTATE_SETTLE_MS = 3_000;
104
+ // `rotation-helper` takes UIInterfaceOrientation's names, whose landscapes are the device's the other way round:
105
+ // what it calls `landscapeRight` devicectl reads back as `landscapeLeft` (measured, Xcode 27 host, iOS 26.5).
106
+ const PURPLE_ARGS = {
107
+ portrait: 'portrait', landscapeRight: 'landscapeLeft', portraitUpsideDown: 'portraitUpsideDown', landscapeLeft: 'landscapeRight',
108
+ };
109
+ // The helper's labels are the physical orientations' own names, on both of the Duo's panels (measured).
110
+ const ORIENT_LABELS = {
111
+ portrait: 'portrait', landscapeRight: 'landscape-right', portraitUpsideDown: 'pud', landscapeLeft: 'landscape-left',
112
+ };
74
113
  export class IOSAgent {
75
114
  simctl;
76
115
  network;
@@ -144,6 +183,37 @@ export class IOSAgent {
144
183
  // queries by bundleId; kept outside DeviceState so it survives a relay reconnect
145
184
  // (which clears deviceStates) while the app keeps running in the simulator.
146
185
  lastBundleIds = new Map();
186
+ // **A multi-panel device (iPhone Duo) is known per device id, from the connect, not from a boot.** A
187
+ // reconnect re-seeds sessions with no `device:boot`, and `ensureTouchHelper` then builds the channel with
188
+ // nothing else to go on — so the answer must already be here, or an agent restart would hand the Duo an
189
+ // Indigo channel that answers success and lands nowhere.
190
+ multiPanelDevices = new Set();
191
+ // Each device's type, from the connect: re-sending a panel's frame after a fold has no boot to read it from.
192
+ deviceTypeIds = new Map();
193
+ // The guest helper and the panels' display UUIDs are keyed by device and live outside `DeviceState`:
194
+ // every relay reconnect rebuilds that map, and stopping the helper there would hide the fold control
195
+ // (no helper, no postures) until something respawned it. They are stopped by the device's own teardown
196
+ // — `stopGuest` — and nowhere else.
197
+ guests = new Map();
198
+ guestScreens = new Map();
199
+ panelState;
200
+ physical;
201
+ // A multi-panel device type's panels: each one's native turn, and the scale every panel is drawn at.
202
+ typePanels;
203
+ guestHelperNative;
204
+ foldSettleMs;
205
+ panelPollMs;
206
+ panelIdlePollMs;
207
+ panelQuietMs;
208
+ helperBootReadyMs;
209
+ panelPolls = new Map();
210
+ // When each device's panel or orientation last moved, or tapflow last moved it: the poll is fast after that.
211
+ panelActivity = new Map();
212
+ panelRechecks = new Map();
213
+ // A posture change in flight per device: a second press during a fold is a double-press, dropped like Android's.
214
+ posturing = new Set();
215
+ // The posture last reported per device, so the poll reports only a change.
216
+ reportedPosture = new Map();
147
217
  uiTreeReader = new XCUITreeReader();
148
218
  // Holds a macOS power assertion while connected so the host doesn't idle-throttle the
149
219
  // simulator capture/encode when the Mac is unattended. No-op off macOS.
@@ -158,6 +228,7 @@ export class IOSAgent {
158
228
  * falling back to nothing would blank a form factor it already had until the next reconnect. */
159
229
  formFactors = new Map();
160
230
  resourcesTimer = null;
231
+ resourcesInFlight = null;
161
232
  resources = createResourceSampler();
162
233
  _stopping = false;
163
234
  _reconnectTimer = null;
@@ -184,6 +255,18 @@ export class IOSAgent {
184
255
  // reason, as `sleepBlocker` below.
185
256
  this.lean = options.lean ?? false;
186
257
  // Never the host's real store under vitest, for the reason given for `network` just below.
258
+ this.panelState = options.panelState ?? new PanelState();
259
+ // Never the host's devicectl under vitest: every single-panel boot reads the orientation, and on an Xcode 27
260
+ // host a real call would read whatever simulator the developer has under that udid. A suite gets the
261
+ // Xcode 26 answer — unreadable, so tapflow remembers — unless it injects one.
262
+ this.physical = options.physicalOrientation ?? new PhysicalOrientation(process.env.VITEST
263
+ ? { readOrientation: () => Promise.reject(new Error('no devicectl under vitest')) } : undefined);
264
+ this.guestHelperNative = options.guestHelperNative ?? defaultGuestHelperNative;
265
+ this.foldSettleMs = options.foldSettleMs ?? FOLD_SETTLE_MS;
266
+ this.panelPollMs = options.panelPollMs ?? PANEL_POLL_MS;
267
+ this.panelIdlePollMs = options.panelIdlePollMs ?? PANEL_IDLE_POLL_MS;
268
+ this.panelQuietMs = options.panelQuietMs ?? PANEL_QUIET_MS;
269
+ this.helperBootReadyMs = options.helperBootReadyMs ?? HELPER_BOOT_READY_MS;
187
270
  this.leanStore = options.leanStore ?? new LeanStore(process.env.VITEST ? path.join(tmpdir(), 'tapflow-no-lean-store') : undefined, MANAGED_LABELS);
188
271
  this.network = options.network ?? new SimulatorNetwork(this.simctl, process.env.VITEST ? { filterHostBinary: path.join(tmpdir(), 'tapflow-no-filter-host') } : {});
189
272
  // **Wired here rather than in the constructor call above, and the difference is coverage.** An
@@ -197,6 +280,7 @@ export class IOSAgent {
197
280
  this.chromeLoader = options.chromeLoader ?? new DeviceChromeLoader(process.env.VITEST
198
281
  ? { run: () => Promise.reject(new Error('no chrome under vitest')) }
199
282
  : {});
283
+ this.typePanels = new TypePanels(this.simctl, this.chromeLoader);
200
284
  this.deviceFilter = options.deviceFilter;
201
285
  this.token = options.token;
202
286
  this.handshakeTimeoutMs = options.handshakeTimeoutMs ?? 10_000;
@@ -241,6 +325,25 @@ export class IOSAgent {
241
325
  logger.warn(`device types unavailable, registering with the last known form factors: ${e instanceof Error ? e.message : String(e)}`);
242
326
  return this.formFactors;
243
327
  });
328
+ // Whether each device has more than one integrated panel. A failed lookup keeps the last answer, like the
329
+ // form factors above; with no answer at all every device stays on the Indigo path, which is how a
330
+ // multi-panel device behaved before this existed. Wrapped so a simctl without the method cannot throw here.
331
+ await Promise.resolve()
332
+ .then(() => this.simctl.integratedPanelCounts([...new Set(allDevices.flatMap((d) => (d.typeId ? [d.typeId] : [])))]))
333
+ .then((counts) => {
334
+ this.multiPanelDevices = new Set(allDevices.filter((d) => d.typeId && (counts.get(d.typeId) ?? 1) > 1).map((d) => d.id));
335
+ this.deviceTypeIds = new Map(allDevices.flatMap((d) => (d.typeId ? [[d.id, d.typeId]] : [])));
336
+ }, (e) => logger.warn(`could not read the devices' panels, keeping the last answer: ${e instanceof Error ? e.message : String(e)}`));
337
+ // **A multi-panel device this Mac cannot drive is not offered.** Its input has no other path, so listing it
338
+ // would let a tester — or an MCP agent's `list_devices` — pick a device that boots and then refuses.
339
+ let offered = devices;
340
+ if (devices.some((d) => this.multiPanelDevices.has(d.id))) {
341
+ const reason = await this.guestHelperNative.unavailableReason().catch((e) => String(e));
342
+ if (reason) {
343
+ logger.warn(`not offering ${devices.filter((d) => this.multiPanelDevices.has(d.id)).map((d) => d.name).join(', ')}: ${reason}`);
344
+ offered = devices.filter((d) => !this.multiPanelDevices.has(d.id));
345
+ }
346
+ }
244
347
  return new Promise((resolve, reject) => {
245
348
  const ws = new WebSocket(relayUrl, this.wsClientOptions());
246
349
  let registered = false;
@@ -259,13 +362,14 @@ export class IOSAgent {
259
362
  capabilities: AGENT_CAPABILITIES,
260
363
  agentId: getMachineId(),
261
364
  agentName: os.hostname(),
262
- devices: devices.map((d) => ({
365
+ devices: offered.map((d) => ({
263
366
  id: d.id,
264
367
  name: d.name,
265
368
  platform: d.platform,
266
369
  status: d.status,
267
370
  osVersion: d.osVersion,
268
- formFactor: d.typeId ? formFactors.get(d.typeId) : undefined,
371
+ // A device with more than one panel folds, whatever its product family says (the Duo is an iPhone).
372
+ formFactor: this.multiPanelDevices.has(d.id) ? 'foldable' : d.typeId ? formFactors.get(d.typeId) : undefined,
269
373
  serviceTier: this.reportedTier(d),
270
374
  })),
271
375
  });
@@ -301,8 +405,8 @@ export class IOSAgent {
301
405
  }
302
406
  catch { /* ignore malformed */ }
303
407
  });
304
- this.reportResources();
305
- this.resourcesTimer = setInterval(() => this.reportResources(), 5000);
408
+ void this.reportResources();
409
+ this.resourcesTimer = setInterval(() => { void this.reportResources(); }, 5000);
306
410
  this.lastPingAt = null;
307
411
  this.lastSocketError = undefined;
308
412
  ws.on('ping', () => { this.lastPingAt = Date.now(); });
@@ -353,8 +457,10 @@ export class IOSAgent {
353
457
  bootSeq: 0,
354
458
  bootAbandon: new Map(),
355
459
  bootsInFlight: new Set(),
356
- orientation: 'portrait',
357
460
  loadedChrome: null,
461
+ chromeSettled: false,
462
+ sentRotation: undefined,
463
+ sentBasis: undefined,
358
464
  softKeyboardVisible: false,
359
465
  acceptH264: false,
360
466
  secureContext: false,
@@ -382,6 +488,8 @@ export class IOSAgent {
382
488
  this.cleanupDeviceState(state);
383
489
  }
384
490
  this.deviceStates.clear();
491
+ for (const deviceId of [...this.guests.keys()])
492
+ this.stopGuest(deviceId);
385
493
  this.uiTreeReader.stop();
386
494
  // The liveness watcher outlives every device it was started for, so it is stopped here with the
387
495
  // other things that tick. It is `unref`'d, which is why nobody noticed it had no caller.
@@ -434,24 +542,41 @@ export class IOSAgent {
434
542
  });
435
543
  }, delay);
436
544
  }
437
- reportResources() {
438
- if (!this.ws || this.ws.readyState !== WebSocket.OPEN)
545
+ /** Samples off the event loop, then reports. One sample at a time per socket, so a slow one (bounded by its
546
+ * 1 s timeout) is not stacked by the next tick; keyed by socket, so a reconnect is not skipped behind the
547
+ * old socket's sample. Slots and the clock are read after the sample, when the report leaves. */
548
+ async reportResources() {
549
+ const ws = this.ws;
550
+ if (!ws || ws.readyState !== WebSocket.OPEN || this.resourcesInFlight === ws)
439
551
  return;
440
- const bootedCount = Array.from(this.deviceStates.values()).filter((s) => s.streamReader !== null).length;
441
- const slotsTotal = this.deviceStates.size;
442
- const { memUsedMB, memTotalMB } = this.resources.getMemoryUsage();
443
- this.sendOn(this.ws, {
444
- type: 'agent:resources',
445
- resources: {
446
- cpuPercent: this.resources.getCpuPercent(),
447
- memUsedMB,
448
- memTotalMB,
449
- slotsAvailable: Math.max(0, slotsTotal - bootedCount),
450
- slotsTotal,
451
- reportedAt: Date.now(),
452
- memoryPressure: this.resources.getMemoryPressure(),
453
- },
454
- });
552
+ this.resourcesInFlight = ws;
553
+ try {
554
+ const [{ memUsedMB, memTotalMB }, memoryPressure] = await Promise.all([
555
+ this.resources.getMemoryUsage(),
556
+ this.resources.getMemoryPressure(),
557
+ ]);
558
+ // A reconnect while sampling replaced the socket: this report has nowhere to go.
559
+ if (this.ws !== ws || ws.readyState !== WebSocket.OPEN)
560
+ return;
561
+ const bootedCount = Array.from(this.deviceStates.values()).filter((s) => s.streamReader !== null).length;
562
+ const slotsTotal = this.deviceStates.size;
563
+ this.sendOn(ws, {
564
+ type: 'agent:resources',
565
+ resources: {
566
+ cpuPercent: this.resources.getCpuPercent(),
567
+ memUsedMB,
568
+ memTotalMB,
569
+ slotsAvailable: Math.max(0, slotsTotal - bootedCount),
570
+ slotsTotal,
571
+ reportedAt: Date.now(),
572
+ memoryPressure,
573
+ },
574
+ });
575
+ }
576
+ finally {
577
+ if (this.resourcesInFlight === ws)
578
+ this.resourcesInFlight = null;
579
+ }
455
580
  }
456
581
  cleanupDeviceState(state) {
457
582
  // Drop this device out of the filter rule. The rule lives on the host and is keyed only by udid,
@@ -494,7 +619,7 @@ export class IOSAgent {
494
619
  // now that a helper revives itself on death, an orphan would also keep respawning with
495
620
  // nobody left holding a reference to stop it.
496
621
  state.touchHelper?.stop();
497
- state.touchHelper = new TouchHelper(device.id);
622
+ state.touchHelper = this.createInputChannel(device.id);
498
623
  state.touchHelper.start();
499
624
  this.sendOn(this.ws, {
500
625
  type: 'session:deviceInfo',
@@ -507,20 +632,39 @@ export class IOSAgent {
507
632
  // Awaited where it used to block: a cold load is seconds of swift renders. A newer boot or a shutdown can
508
633
  // land meanwhile, so the result is stored only after the seq is checked — otherwise a slow stale load
509
634
  // would overwrite the chrome a faster newer boot already set, and the button path reads it.
510
- const chrome = await this.chromeLoader.load(device.typeId ?? device.name);
635
+ // A multi-panel device shows its lit panel's frame; until the panels are known it gets the model's, and
636
+ // `applyLitPanel` replaces it once they are.
637
+ // A multi-panel device's scale comes with its first frame, from its type, so a viewer sizes it once.
638
+ const multiPanel = this.multiPanelDevices.has(device.id) && device.typeId;
639
+ const [chrome, basis] = await Promise.all([
640
+ this.chromeLoader.load(device.typeId ?? device.name, this.chromePanel(device.id)),
641
+ multiPanel ? this.typePanels.scaleBasis(multiPanel) : undefined,
642
+ ]);
511
643
  if (seq !== state.bootSeq)
512
644
  return false;
513
645
  const ws = this.ws;
514
646
  if (ws?.readyState !== WebSocket.OPEN)
515
647
  return true;
516
648
  state.loadedChrome = chrome;
517
- if (!chrome)
649
+ state.chromeSettled = true;
650
+ if (!chrome) {
651
+ // The model's frame did not load; the lit panel's has its own and may.
652
+ if (this.multiPanelDevices.has(device.id))
653
+ void this.resendChrome(device.id);
518
654
  return true;
655
+ }
656
+ state.sentRotation = this.chromeRotation(device.id);
657
+ state.sentBasis = basis;
519
658
  this.sendOn(ws, {
520
659
  type: 'session:chrome',
521
660
  sessionId: state.sessionId,
522
- payload: chrome,
661
+ payload: this.forViewer(chrome, state.sentRotation, state.sentBasis),
523
662
  });
663
+ // A multi-panel device's panels can become known while the load above was running — the helper reads them
664
+ // in parallel with the boot — and the frame just sent may be the model's. Settled here, from whichever
665
+ // side finishes last: `resendChrome` replaces it only if the lit panel's frame differs.
666
+ if (this.multiPanelDevices.has(device.id))
667
+ void this.resendChrome(device.id);
524
668
  return true;
525
669
  }
526
670
  /** `failures` counts the frameless runs before this one; only the restart in `pump` passes it, so a new
@@ -544,7 +688,10 @@ export class IOSAgent {
544
688
  external: state.external,
545
689
  override: process.env.TAPFLOW_IOS_MAX_SIZE ?? process.env.TAPFLOW_MAX_SIZE,
546
690
  });
547
- const capture = new ScreenCaptureStreamer(state.deviceId, this.fps, useH264 ? 'h264' : 'jpeg', maxSize);
691
+ // A multi-panel device is captured at its lit panel; a restart re-applies it here, since a stdin switch
692
+ // lives only as long as the helper it was written to.
693
+ const panel = this.multiPanelDevices.has(state.deviceId) ? this.litPanel(state.deviceId) : null;
694
+ const capture = new ScreenCaptureStreamer(state.deviceId, this.fps, useH264 ? 'h264' : 'jpeg', maxSize, panel ? { width: panel.width, height: panel.height } : null);
548
695
  state.captureStreamer = capture;
549
696
  stream = capture.start();
550
697
  }
@@ -732,6 +879,11 @@ export class IOSAgent {
732
879
  // Read before anything below changes it: a Full reset shuts the device down itself, and `unknown`
733
880
  // (booting, shutting down) is a device paying a boot's cost now.
734
881
  const cold = fullErase || retier || target.status !== 'booted';
882
+ // A device this boot brings up gets a new helper and fresh readings. One left from before was serving a
883
+ // simulator that is gone — shut down outside tapflow (DeviceHub quitting takes every one), or about to be
884
+ // — and kept, the boot waited out its respawn cooldown and started from the panel lit before.
885
+ if (cold)
886
+ this.stopGuest(deviceId);
735
887
  if (fullErase) {
736
888
  // `simctl erase` only accepts a shut-down device, and a device is often already running —
737
889
  // it survives agent restarts and sessions that ended without a clean shutdown. Without this
@@ -841,15 +993,18 @@ export class IOSAgent {
841
993
  await this.network.arm(deviceId).catch((e) => {
842
994
  logger.warn('could not arm the network injection:', e.message);
843
995
  });
844
- // **Stand it up (#910).** A fresh viewer always starts portrait, and a simulator left turned —
845
- // by a session whose undo never arrived, or one an agent restart reset the memory of — would
846
- // otherwise stream sideways under an upright frame, with every later press inverted. Queued with
847
- // the rotations, so one still in flight from before a re-register lands first and is overridden.
848
- // Best-effort: a failure leaves the simulator where it was, and the boot goes on.
849
- await this.enqueueRotation(deviceId, async () => {
850
- await this.simctl.rotate(deviceId, 'portrait');
851
- state.orientation = 'portrait';
852
- });
996
+ // **Where it is held, before the first frame says so** (#910). The orientation is the device's, shared with
997
+ // DeviceHub and every viewer, so it is read and reported as it is, and a reload or a second viewer joins it
998
+ // there rather than turning it under the first. It is stood up only when there is nothing to report — no
999
+ // reading and nothing remembered — and **for a run**: a flow replays from the same orientation every time,
1000
+ // whatever a person left the device in (`runReset` is a run's boot; review finding). Queued with the
1001
+ // rotations, so one still in flight lands first. A multi-panel device reads its own through its helper.
1002
+ if (!this.multiPanelDevices.has(deviceId)) {
1003
+ await this.enqueueRotation(deviceId, async () => {
1004
+ if (runReset || !(await this.heldSingle(deviceId)))
1005
+ await this.standUp(deviceId);
1006
+ });
1007
+ }
853
1008
  // Another await — a multi-second one — and `sendChromeData` below starts a helper process. A
854
1009
  // shutdown or a newer boot arriving in this gap would otherwise get a helper installed for the
855
1010
  // device it is taking down — and one that revives itself, so the stale reference outlives the
@@ -862,6 +1017,54 @@ export class IOSAgent {
862
1017
  this.abandonBoot(state, seq, sessionId, requestId);
863
1018
  return;
864
1019
  }
1020
+ // **A session with no frame cannot be told the turn**: no `session:chrome` goes out, and its viewer draws
1021
+ // the device upright, as one always started. So that device is stood up, the old guarantee for the one
1022
+ // viewer that still relies on it (review finding).
1023
+ if (!this.multiPanelDevices.has(deviceId) && !state.loadedChrome && this.physical.last(deviceId) !== 'portrait') {
1024
+ await this.enqueueRotation(deviceId, async () => { await this.standUp(deviceId); });
1025
+ }
1026
+ // **A multi-panel device is usable only through its helper**, so the boot is not ready until the helper
1027
+ // is, and a runtime that refuses it fails the boot with the reason. The device is shut down only if this
1028
+ // boot brought it up: one that was already running (DeviceHub, another session) is left as it was found.
1029
+ //
1030
+ // No helper at all is a socket that closed before `sendChromeData` built the channel: the boot carries on
1031
+ // as a single-panel one does, and the relay loss that follows retires it — not a refusal to shut down for.
1032
+ const guest = this.multiPanelDevices.has(deviceId) ? this.guests.get(deviceId) : undefined;
1033
+ if (guest) {
1034
+ let limit;
1035
+ const ready = await Promise.race([
1036
+ guest.whenReady(),
1037
+ new Promise((resolve) => { limit = setTimeout(() => resolve(false), this.helperBootReadyMs); }),
1038
+ ]).finally(() => clearTimeout(limit));
1039
+ if (seq !== state.bootSeq) {
1040
+ this.abandonBoot(state, seq, sessionId, requestId);
1041
+ return;
1042
+ }
1043
+ if (!ready) {
1044
+ const reason = guest.unavailableReason() ?? `the input helper was not ready within ${this.helperBootReadyMs / 1000}s`;
1045
+ this.stopGuest(deviceId);
1046
+ if (cold)
1047
+ await this.simctl.shutdown(deviceId).catch(() => { });
1048
+ throw new PlatformError(`${bootedDevice.name} cannot be used on this Mac: ${reason}`);
1049
+ }
1050
+ // A run replays from the same orientation, as a single-panel device's does (the stand-up above); the Duo
1051
+ // turns through its helper, so only once the helper is up.
1052
+ // Re-described afterwards, as `input:rotate` does: the chrome sent while the helper came up carried the old
1053
+ // turn. The panel poll the turn re-arms would correct it too, but up to its interval later.
1054
+ // A newer boot can arrive while the turn waits in the queue: it neither turns for a boot already superseded
1055
+ // nor goes on to open a stream over the newer one's.
1056
+ if (runReset) {
1057
+ await this.enqueueRotation(deviceId, async () => {
1058
+ if (seq === state.bootSeq)
1059
+ await this.turnHeld(deviceId, undefined, 'portrait', () => seq !== state.bootSeq);
1060
+ });
1061
+ if (seq !== state.bootSeq) {
1062
+ this.abandonBoot(state, seq, sessionId, requestId);
1063
+ return;
1064
+ }
1065
+ this.applyLitPanel(deviceId);
1066
+ }
1067
+ }
865
1068
  const streamWs = await this.openStreamWs(state);
866
1069
  if (seq !== state.bootSeq) {
867
1070
  streamWs.close();
@@ -884,6 +1087,8 @@ export class IOSAgent {
884
1087
  // riding inside it: a tester whose device just came up can arm the toggle in the same breath, and
885
1088
  // a boot that armed nothing still has to say so rather than leaving the control blank.
886
1089
  void this.reportNetworkState(sessionId);
1090
+ if (this.multiPanelDevices.has(deviceId))
1091
+ void this.sendPostures(state);
887
1092
  // An automation client's first selector step would otherwise spend the runner's cold start inside its timeout.
888
1093
  if (automation)
889
1094
  this.uiTreeReader.warm(deviceId);
@@ -1019,6 +1224,7 @@ export class IOSAgent {
1019
1224
  state.captureStreamer = null; // reader.cancel() kills the helper proc; drop the ref so a stale requestKeyframe() no-ops
1020
1225
  state.touchHelper?.stop();
1021
1226
  state.touchHelper = null;
1227
+ this.stopGuest(deviceId);
1022
1228
  state.booted = false;
1023
1229
  state.streamWs?.close();
1024
1230
  state.streamWs = null;
@@ -1103,9 +1309,363 @@ export class IOSAgent {
1103
1309
  ensureTouchHelper(state) {
1104
1310
  if (state.touchHelper)
1105
1311
  return;
1106
- state.touchHelper = new TouchHelper(state.deviceId);
1312
+ state.touchHelper = this.createInputChannel(state.deviceId);
1107
1313
  state.touchHelper.start();
1108
1314
  }
1315
+ /**
1316
+ * **The one place a device's input path is chosen** (see `InputChannel`). A device with more than one
1317
+ * integrated panel gets `FoldableInput` over its in-simulator helper; every other device gets the Indigo
1318
+ * `TouchHelper` it always had. A multi-panel device is never given a `TouchHelper`: its writes would
1319
+ * answer success and land on a dark panel or nowhere.
1320
+ */
1321
+ createInputChannel(deviceId) {
1322
+ if (!this.multiPanelDevices.has(deviceId))
1323
+ return new TouchHelper(deviceId);
1324
+ this.ensureGuest(deviceId);
1325
+ return new FoldableInput({
1326
+ guest: () => this.guests.get(deviceId) ?? null,
1327
+ litScreen: () => this.litScreen(deviceId),
1328
+ });
1329
+ }
1330
+ ensureGuest(deviceId) {
1331
+ if (this.guests.has(deviceId))
1332
+ return;
1333
+ const guest = new GuestHelper(deviceId, async () => {
1334
+ const screens = await this.simctl.integratedScreens(deviceId);
1335
+ this.guestScreens.set(deviceId, new Map(screens.map((s) => [s.screenId, s])));
1336
+ const typeId = this.deviceTypeIds.get(deviceId);
1337
+ if (typeId)
1338
+ await this.typePanels.load(typeId);
1339
+ return screens.map((s) => s.uuid);
1340
+ }, this.guestHelperNative, {
1341
+ onUnavailable: (reason) => {
1342
+ logger.error(`input for ${deviceId} is unavailable: ${reason}`);
1343
+ // Hide the fold control everywhere, and in the relay's replay cache: a control left up would do nothing.
1344
+ for (const state of this.deviceStates.values())
1345
+ if (state.deviceId === deviceId)
1346
+ void this.sendPostures(state);
1347
+ },
1348
+ // Back after an outage (the respawn window cleared): show it again, from the device's real posture.
1349
+ onAvailable: () => {
1350
+ for (const state of this.deviceStates.values())
1351
+ if (state.deviceId === deviceId)
1352
+ void this.sendPostures(state);
1353
+ },
1354
+ });
1355
+ this.guests.set(deviceId, guest);
1356
+ void guest.start();
1357
+ this.startPanelPoll(deviceId);
1358
+ // Which panel is lit decides where every touch goes and which panel is captured; until a reading arrives
1359
+ // the channel says `starting`. The panels are known once the helper has read them (`whenReady`), and a
1360
+ // stream that started before that captured the largest panel — dark while folded — so it is corrected here.
1361
+ // The way it is held is read alongside, so the first frame a viewer gets after the helper is up is
1362
+ // already turned rather than corrected a poll later.
1363
+ void this.panelState.read(deviceId);
1364
+ void this.physical.read(deviceId);
1365
+ void guest.whenReady()
1366
+ .then((ready) => (ready ? Promise.all([this.panelState.read(deviceId), this.physical.read(deviceId)]) : null))
1367
+ .then(() => this.applyLitPanel(deviceId));
1368
+ }
1369
+ /** The lit panel of a multi-panel device, from the last devicectl reading and this boot's panels. */
1370
+ litPanel(deviceId) {
1371
+ const lit = this.panelState.last(deviceId)?.lit;
1372
+ return lit ? this.guestScreens.get(deviceId)?.get(lit.displayId) ?? null : null;
1373
+ }
1374
+ /**
1375
+ * How far the viewer turns a multi-panel device (`ChromeData.streamRotation`): the way it is held, plus the
1376
+ * lit panel's own native turn — the Duo's unfolded panel is drawn turned 270, so a device held upright
1377
+ * shows it turned 90 (measured: portrait → rot90 … landscapeLeft → rot0). Undefined for a single-panel
1378
+ * device, and until both the orientation and the lit panel have been read.
1379
+ */
1380
+ chromeRotation(deviceId) {
1381
+ if (!this.multiPanelDevices.has(deviceId)) {
1382
+ const single = this.physical.last(deviceId);
1383
+ return single ? ORIENTATION_DEGREES[single] : undefined;
1384
+ }
1385
+ const held = this.physical.last(deviceId);
1386
+ const panel = this.litPanel(deviceId);
1387
+ if (!held || !panel)
1388
+ return undefined;
1389
+ const typeId = this.deviceTypeIds.get(deviceId);
1390
+ const native = typeId ? this.typePanels.nativeOrientation(typeId, panel.deviceName) : 0;
1391
+ return ((ORIENTATION_DEGREES[held] + 360 - native) % 360);
1392
+ }
1393
+ /** The cached frame with this send's turn and scale — never written into the loader's shared object. */
1394
+ forViewer(chrome, streamRotation, scaleBasis) {
1395
+ if (streamRotation === undefined && scaleBasis === undefined)
1396
+ return chrome;
1397
+ return {
1398
+ ...chrome,
1399
+ ...(streamRotation === undefined ? {} : { streamRotation }),
1400
+ ...(scaleBasis === undefined ? {} : { scaleBasis }),
1401
+ };
1402
+ }
1403
+ /** The lit panel's frame for `DeviceChromeLoader`, or undefined for a single-panel device or an unknown panel. */
1404
+ chromePanel(deviceId) {
1405
+ if (!this.multiPanelDevices.has(deviceId))
1406
+ return undefined;
1407
+ const panel = this.litPanel(deviceId);
1408
+ return panel ? panelFrame(panel) : undefined;
1409
+ }
1410
+ /** Point every stream of this device at its lit panel, and give its viewers that panel's frame. */
1411
+ applyLitPanel(deviceId) {
1412
+ const panel = this.litPanel(deviceId);
1413
+ if (!panel)
1414
+ return;
1415
+ for (const state of this.deviceStates.values()) {
1416
+ if (state.deviceId === deviceId)
1417
+ state.captureStreamer?.selectPanel({ width: panel.width, height: panel.height });
1418
+ }
1419
+ void this.resendChrome(deviceId);
1420
+ }
1421
+ /**
1422
+ * Load the lit panel's frame and send it as `session:chrome` to every session on this device whose frame
1423
+ * it is not already. `session:chrome` is re-sendable by contract (protocol `InputPosture`), and the relay
1424
+ * caches the last one for a re-joining viewer.
1425
+ */
1426
+ async resendChrome(deviceId) {
1427
+ const typeId = this.deviceTypeIds.get(deviceId);
1428
+ const panel = this.chromePanel(deviceId);
1429
+ if (!typeId || !panel)
1430
+ return;
1431
+ const [chrome, basis] = await Promise.all([this.chromeLoader.load(typeId, panel), this.typePanels.scaleBasis(typeId)]);
1432
+ const ws = this.ws;
1433
+ if (!chrome || ws?.readyState !== WebSocket.OPEN)
1434
+ return;
1435
+ for (const state of this.deviceStates.values()) {
1436
+ // Only a session whose boot has finished loading its first frame: one still loading gets the lit panel's
1437
+ // frame from the tail of `sendChromeData`, and sending it here first would be overwritten by the model's.
1438
+ const rotation = this.chromeRotation(deviceId);
1439
+ if (state.deviceId !== deviceId || !state.chromeSettled)
1440
+ continue;
1441
+ if (state.loadedChrome === chrome && state.sentRotation === rotation && state.sentBasis === basis)
1442
+ continue;
1443
+ state.loadedChrome = chrome;
1444
+ state.sentRotation = rotation;
1445
+ state.sentBasis = basis;
1446
+ this.sendOn(ws, { type: 'session:chrome', sessionId: state.sessionId, payload: this.forViewer(chrome, rotation, basis) });
1447
+ }
1448
+ }
1449
+ /** The display UUID of the panel a multi-panel device is showing, or null if not known yet. */
1450
+ litScreen(deviceId) {
1451
+ return this.litPanel(deviceId)?.uuid ?? null;
1452
+ }
1453
+ /**
1454
+ * Follow a fold made outside tapflow (DeviceHub's picker) while a session is on the device: read the lit
1455
+ * panel, re-point the capture and the frame, and report the posture if it moved. Skipped while tapflow's own
1456
+ * fold is in flight, which reports for itself.
1457
+ */
1458
+ startPanelPoll(deviceId) {
1459
+ if (this.panelPolls.has(deviceId))
1460
+ return;
1461
+ this.panelActivity.set(deviceId, Date.now());
1462
+ this.schedulePanelPoll(deviceId);
1463
+ }
1464
+ schedulePanelPoll(deviceId) {
1465
+ const quiet = Date.now() - (this.panelActivity.get(deviceId) ?? 0) >= this.panelQuietMs;
1466
+ const timer = setTimeout(() => {
1467
+ this.followPanel(deviceId);
1468
+ // Stopped while it ran (`stopGuest`) is stopped for good.
1469
+ if (this.panelPolls.get(deviceId) === timer)
1470
+ this.schedulePanelPoll(deviceId);
1471
+ }, quiet ? this.panelIdlePollMs : this.panelPollMs);
1472
+ timer.unref?.();
1473
+ this.panelPolls.set(deviceId, timer);
1474
+ }
1475
+ /** The device's panel or orientation is moving, or about to: the poll goes back to its fast rate now, not
1476
+ * after the slow wait already scheduled. */
1477
+ panelMoved(deviceId) {
1478
+ if (!this.panelPolls.has(deviceId))
1479
+ return;
1480
+ this.panelActivity.set(deviceId, Date.now());
1481
+ clearTimeout(this.panelPolls.get(deviceId));
1482
+ this.schedulePanelPoll(deviceId);
1483
+ }
1484
+ followPanel(deviceId) {
1485
+ if (this.posturing.has(deviceId) || this.guests.get(deviceId)?.inputState() !== 'ready')
1486
+ return;
1487
+ if (![...this.deviceStates.values()].some((s) => s.deviceId === deviceId && s.booted))
1488
+ return;
1489
+ const pose = () => `${this.panelState.last(deviceId)?.lit?.displayId}:${this.physical.last(deviceId)}`;
1490
+ const before = pose();
1491
+ void Promise.all([this.panelState.read(deviceId), this.physical.read(deviceId)]).then(() => {
1492
+ if (pose() !== before)
1493
+ this.panelMoved(deviceId);
1494
+ // The panel it has settled on, not the one just read: one lit only for a moment is not followed.
1495
+ const settled = this.panelState.last(deviceId);
1496
+ if (!settled?.lit || this.posturing.has(deviceId))
1497
+ return;
1498
+ // A new panel is read again once it has had time to hold, rather than at the next poll or two.
1499
+ if (this.panelState.changing(deviceId) && !this.panelRechecks.has(deviceId)) {
1500
+ const recheck = setTimeout(() => { this.panelRechecks.delete(deviceId); this.followPanel(deviceId); }, this.panelState.holdMs);
1501
+ recheck.unref?.();
1502
+ this.panelRechecks.set(deviceId, recheck);
1503
+ }
1504
+ this.applyLitPanel(deviceId);
1505
+ const posture = this.postureOf(deviceId, settled);
1506
+ if (posture !== this.reportedPosture.get(deviceId)) {
1507
+ for (const state of this.deviceStates.values())
1508
+ if (state.deviceId === deviceId)
1509
+ void this.sendPostures(state);
1510
+ }
1511
+ });
1512
+ }
1513
+ /** The device's teardown: shut down, erased, or the agent leaving. Not a relay reconnect — see `guests`. */
1514
+ stopGuest(deviceId) {
1515
+ clearTimeout(this.panelPolls.get(deviceId));
1516
+ this.panelPolls.delete(deviceId);
1517
+ this.panelActivity.delete(deviceId);
1518
+ clearTimeout(this.panelRechecks.get(deviceId));
1519
+ this.panelRechecks.delete(deviceId);
1520
+ this.reportedPosture.delete(deviceId);
1521
+ this.guests.get(deviceId)?.stop();
1522
+ this.guests.delete(deviceId);
1523
+ this.guestScreens.delete(deviceId);
1524
+ this.panelState.forget(deviceId);
1525
+ this.physical.forget(deviceId);
1526
+ }
1527
+ // ── PosturableAgent ────────────────────────────────────────────────────────
1528
+ // A multi-panel device (iPhone Duo) folds through its in-simulator helper's hinge, and which panel is lit
1529
+ // is read back from devicectl — never inferred from the angle sent.
1530
+ async listPostures(deviceId) {
1531
+ // The list's length is the capability's real gate: a device that cannot fold, or whose helper is not up
1532
+ // to fold it, answers none, and the viewer shows no control.
1533
+ return this.multiPanelDevices.has(deviceId) && this.guests.get(deviceId)?.inputState() === 'ready' ? FOLDABLE_POSTURES : [];
1534
+ }
1535
+ async getPosture(deviceId) {
1536
+ if ((await this.listPostures(deviceId)).length === 0)
1537
+ return null;
1538
+ const reading = (await this.panelState.read(deviceId)) ?? this.panelState.last(deviceId);
1539
+ const id = reading ? this.postureOf(deviceId, reading) : null;
1540
+ return FOLDABLE_POSTURES.find((p) => p.id === id) ?? null;
1541
+ }
1542
+ async setPosture(deviceId, postureId) {
1543
+ if (!(await this.listPostures(deviceId)).some((p) => p.id === postureId)) {
1544
+ throw new PlatformError(`Unknown posture "${postureId}" for ${deviceId}`);
1545
+ }
1546
+ const guest = this.guests.get(deviceId);
1547
+ this.panelMoved(deviceId);
1548
+ const target = [...(this.guestScreens.get(deviceId)?.values() ?? [])]
1549
+ .find((s) => s.deviceName === (postureId === 'closed' ? 'primary' : 'primary-1'));
1550
+ if (!target)
1551
+ throw new PlatformError(`${deviceId} has no ${postureId === 'closed' ? 'cover' : 'unfolded'} panel`);
1552
+ const from = POSTURE_DEGREES[(await this.getPosture(deviceId))?.id ?? (postureId === 'closed' ? 'open' : 'closed')];
1553
+ // **The panel is the answer, not the helper's.** The sweep ending is not the panel changing — measured, the
1554
+ // new panel lights 4–6 s after the command — and on a loaded Mac the helper's answer came after the panel
1555
+ // had lit. So both are watched at once, and a lit panel is success however late the answer.
1556
+ const swept = guest.hinge(from, POSTURE_DEGREES[postureId], 800);
1557
+ const settled = await this.panelState.awaitLit(deviceId, (r) => r.lit?.displayId === target.screenId, this.foldSettleMs);
1558
+ if (!settled) {
1559
+ if (!(await swept))
1560
+ throw new PlatformError('the hinge did not move');
1561
+ throw new PlatformError(`the ${postureId === 'closed' ? 'cover' : 'unfolded panel'} did not light within ${this.foldSettleMs}ms`);
1562
+ }
1563
+ // Nothing is turned: a fold does not change the way the device is held (measured), so only the frame and
1564
+ // the lit panel's native turn change — which is what DeviceHub shows.
1565
+ this.applyLitPanel(deviceId);
1566
+ }
1567
+ /**
1568
+ * Turn a multi-panel device the way it is held, through the helper (Purple and `devicectl orientation set`
1569
+ * do nothing on the Duo — measured). From the orientation read back now, never one remembered: a quarter
1570
+ * turn `direction`, or an older viewer's `target` — where either landscape meets landscape, so asking for
1571
+ * it never flips one into the other, and portrait means upright. Then waits for devicectl to say so, which
1572
+ * is what the re-sent chrome reports; a turn that does not land is logged and the chrome says where the
1573
+ * device really is.
1574
+ */
1575
+ async turnHeld(deviceId, direction, target, isStale) {
1576
+ const guest = this.guests.get(deviceId);
1577
+ this.panelMoved(deviceId);
1578
+ const held = (await this.physical.read(deviceId)) ?? this.physical.last(deviceId);
1579
+ // The read takes a moment, and a caller whose boot was superseded in it must not turn the shared device.
1580
+ if (isStale?.())
1581
+ return;
1582
+ if (!guest || !held) {
1583
+ logger.warn(`rotate ${deviceId}: ${guest ? 'its orientation could not be read' : 'no helper'}, not turning`);
1584
+ return;
1585
+ }
1586
+ const landscape = held === 'landscapeLeft' || held === 'landscapeRight';
1587
+ const want = direction ? turned(held, direction)
1588
+ : target === 'portrait' ? (held === 'portrait' ? null : 'portrait')
1589
+ : target === 'landscape' ? (landscape ? null : 'landscapeRight')
1590
+ : (landscape ? 'portrait' : 'landscapeRight');
1591
+ if (!want)
1592
+ return;
1593
+ if (!(await guest.orient(ORIENT_LABELS[want]))) {
1594
+ logger.warn(`rotate ${deviceId}: the helper did not take ${want}`);
1595
+ return;
1596
+ }
1597
+ if (!(await this.physical.await(deviceId, (o) => o === want, ROTATE_SETTLE_MS))) {
1598
+ logger.warn(`rotate ${deviceId}: asked for ${want}, still ${this.physical.last(deviceId) ?? 'unread'} after ${ROTATE_SETTLE_MS}ms`);
1599
+ }
1600
+ }
1601
+ /**
1602
+ * Where a single-panel device is held: read (devicectl on Xcode 27, the guest helper before — see
1603
+ * `defaultPhysicalOrientationNative`), else the last known — the last reading, or what tapflow last set on a
1604
+ * host where neither source answers. Read every time, never marked unreadable: one failure right after a
1605
+ * boot would otherwise have left the device on memory for the life of the agent (review finding).
1606
+ */
1607
+ async heldSingle(deviceId) {
1608
+ return (await this.physical.read(deviceId)) ?? this.physical.last(deviceId);
1609
+ }
1610
+ /** Turn a device nobody can say the orientation of upright, so there is one to turn from. */
1611
+ async standUp(deviceId) {
1612
+ await this.simctl.rotate(deviceId, PURPLE_ARGS.portrait);
1613
+ this.physical.note(deviceId, 'portrait');
1614
+ return 'portrait';
1615
+ }
1616
+ /**
1617
+ * Turn a single-panel device, through Purple: a quarter turn `direction` from where it is held, or an older
1618
+ * viewer's `target`. **That viewer draws landscape as 270**, so `landscape` means exactly `landscapeLeft` —
1619
+ * the other landscape would show it upside down — and is met only there; `portrait` means upright. A device
1620
+ * whose orientation cannot be read or remembered (an agent restarted under it) is stood up first.
1621
+ */
1622
+ async turnSingle(deviceId, direction, target) {
1623
+ const held = (await this.heldSingle(deviceId)) ?? (await this.standUp(deviceId));
1624
+ const want = direction ? turned(held, direction)
1625
+ : target === 'portrait' ? (held === 'portrait' ? null : 'portrait')
1626
+ : target === 'landscape' ? (held === 'landscapeLeft' ? null : 'landscapeLeft')
1627
+ : (held === 'portrait' ? 'landscapeLeft' : 'portrait');
1628
+ if (!want)
1629
+ return;
1630
+ await this.simctl.rotate(deviceId, PURPLE_ARGS[want]);
1631
+ // Read back once; a host that cannot read at all remembers what it set instead of waiting out the settle.
1632
+ const back = await this.physical.read(deviceId);
1633
+ if (!back) {
1634
+ this.physical.note(deviceId, want);
1635
+ return;
1636
+ }
1637
+ if (back !== want && !(await this.physical.await(deviceId, (o) => o === want, ROTATE_SETTLE_MS))) {
1638
+ logger.warn(`rotate ${deviceId}: asked for ${want}, still ${this.physical.last(deviceId) ?? 'unread'} after ${ROTATE_SETTLE_MS}ms`);
1639
+ }
1640
+ }
1641
+ /** Re-send the frame each session on this device already has, with the turn it is held at now. */
1642
+ resendTurn(deviceId) {
1643
+ const ws = this.ws;
1644
+ if (ws?.readyState !== WebSocket.OPEN)
1645
+ return;
1646
+ const rotation = this.chromeRotation(deviceId);
1647
+ for (const state of this.deviceStates.values()) {
1648
+ // Not to a session whose boot is still running: its frame is the last boot's, and that boot sends its own.
1649
+ if (state.deviceId !== deviceId || !state.loadedChrome || state.bootsInFlight.size > 0 || state.sentRotation === rotation)
1650
+ continue;
1651
+ state.sentRotation = rotation;
1652
+ this.sendOn(ws, { type: 'session:chrome', sessionId: state.sessionId, payload: this.forViewer(state.loadedChrome, rotation, state.sentBasis) });
1653
+ }
1654
+ }
1655
+ /** `closed` while the cover is lit, `open` while the unfolded panel is. */
1656
+ postureOf(deviceId, reading) {
1657
+ const lit = reading.lit && this.guestScreens.get(deviceId)?.get(reading.lit.displayId);
1658
+ if (!lit)
1659
+ return null;
1660
+ return lit.deviceName === 'primary' ? 'closed' : 'open';
1661
+ }
1662
+ /** Tell the viewer which postures exist and which one the device is in — read from the device, so a fold
1663
+ * made outside tapflow shows too. Sent at boot, after a change (failed or not), and on the helper dying. */
1664
+ async sendPostures(state) {
1665
+ const [postures, current] = await Promise.all([this.listPostures(state.deviceId), this.getPosture(state.deviceId)]);
1666
+ this.reportedPosture.set(state.deviceId, current?.id ?? null);
1667
+ this.sendMsg({ type: 'device:postures', sessionId: state.sessionId, payload: { postures, currentId: current?.id ?? null } });
1668
+ }
1109
1669
  // Clipboard work parks a sentinel on the device while it waits, so two operations on the same
1110
1670
  // device must not interleave — each would read the other's marker instead of the real
1111
1671
  // clipboard. Keyed by device, not session: several sessions (and MCP) can address one device.
@@ -1272,7 +1832,7 @@ export class IOSAgent {
1272
1832
  break;
1273
1833
  }
1274
1834
  case 'app:launch': {
1275
- const { bundleId } = msg.payload;
1835
+ const { bundleId, arguments: args, env } = msg.payload;
1276
1836
  const sessionId = msg.sessionId;
1277
1837
  const { requestId } = msg;
1278
1838
  if (typeof requestId !== 'string' || requestId === '') {
@@ -1287,12 +1847,20 @@ export class IOSAgent {
1287
1847
  respond({ type: 'app:launch-error', message: 'No booted device' });
1288
1848
  break;
1289
1849
  }
1850
+ // Checked again here rather than trusted: the relay checks with the same function, but an agent is the last
1851
+ // door before a command line, and a relay of another version may check less.
1852
+ const problem = launchOptionsProblem({ arguments: args, env });
1853
+ if (problem) {
1854
+ respond({ type: 'app:launch-error', message: problem });
1855
+ break;
1856
+ }
1857
+ const options = compactLaunchOptions({ arguments: args, env });
1290
1858
  // The target is named BEFORE the launch, and the order is the whole point: dyld reads the
1291
1859
  // environment when the process starts, so naming it afterwards arms the next launch and leaves
1292
1860
  // this one unhooked while `available` would still say true.
1293
1861
  this.network.target(launchState.deviceId, bundleId)
1294
1862
  .catch((e) => logger.warn('could not name the network target:', e.message))
1295
- .then(() => this.simctl.launchApp(launchState.deviceId, bundleId))
1863
+ .then(() => this.simctl.launchApp(launchState.deviceId, bundleId, options))
1296
1864
  .then((pid) => {
1297
1865
  // **After the launch, with the pid it returned.** The network control measures how long a
1298
1866
  // started process has gone without reporting its hooks; opening that window before
@@ -1303,7 +1871,7 @@ export class IOSAgent {
1303
1871
  this.lastBundleIds.set(launchState.deviceId, bundleId);
1304
1872
  // Audio: the whole-sim tap's poll picks up the launched app process within one interval;
1305
1873
  // no per-launch helper needed.
1306
- respond({ type: 'app:launch-done' });
1874
+ respond(options ? { type: 'app:launch-done', applied: appliedKeys(options) } : { type: 'app:launch-done' });
1307
1875
  })
1308
1876
  .catch((e) => {
1309
1877
  // The toast gets the reason alone; the full simctl output stays in this log.
@@ -1387,17 +1955,38 @@ export class IOSAgent {
1387
1955
  // dashboard's. The memory is read when the step runs, not when it was queued, and written only
1388
1956
  // once the simulator took it: a step queued behind another toggles from what that one applied.
1389
1957
  const sessionId = msg.sessionId;
1390
- const target = msg.payload?.orientation;
1958
+ const payload = msg.payload;
1959
+ const target = payload?.orientation;
1391
1960
  void this.enqueueRotation(state.deviceId, async () => {
1392
1961
  const live = this.deviceStates.get(sessionId) ?? state;
1393
- const next = target
1394
- ? (target === 'landscape' ? 'landscapeRight' : 'portrait')
1395
- : (live.orientation === 'portrait' ? 'landscapeRight' : 'portrait');
1396
- await this.simctl.rotate(live.deviceId, next);
1397
- live.orientation = next;
1962
+ if (this.multiPanelDevices.has(live.deviceId)) {
1963
+ await this.turnHeld(live.deviceId, payload?.direction, target);
1964
+ this.applyLitPanel(live.deviceId);
1965
+ return;
1966
+ }
1967
+ await this.turnSingle(live.deviceId, payload?.direction, target);
1968
+ this.resendTurn(live.deviceId);
1398
1969
  });
1399
1970
  break;
1400
1971
  }
1972
+ case 'input:posture': {
1973
+ const state = this.deviceStates.get(msg.sessionId);
1974
+ if (!state)
1975
+ break;
1976
+ const { postureId } = msg.payload;
1977
+ // No ack by declaration (like `input:rotate`): a fold that fails is reported by the posture list that
1978
+ // follows saying where the device really is. One at a time per device, like Android's.
1979
+ if (this.posturing.has(state.deviceId)) {
1980
+ logger.info(`posture ${postureId}: a change is already in flight, ignoring`);
1981
+ break;
1982
+ }
1983
+ this.posturing.add(state.deviceId);
1984
+ void this.setPosture(state.deviceId, postureId)
1985
+ .catch((e) => logger.warn(`setPosture(${postureId}): ${e.message}`))
1986
+ .then(() => this.sendPostures(state))
1987
+ .finally(() => this.posturing.delete(state.deviceId));
1988
+ break;
1989
+ }
1401
1990
  case 'stream:request-idr': {
1402
1991
  // Relay drop-to-keyframe recovery: force an IDR so the stream resyncs fast.
1403
1992
  this.deviceStates.get(msg.sessionId)?.captureStreamer?.requestKeyframe();
@@ -2119,8 +2708,11 @@ export class IOSAgent {
2119
2708
  async networkState() {
2120
2709
  return this.network.state(await this.soleLiveDeviceId());
2121
2710
  }
2122
- async launchApp(bundleId) {
2123
- await this.simctl.launchApp(await this.soleLiveDeviceId(), bundleId);
2711
+ async launchApp(bundleId, options) {
2712
+ const problem = launchOptionsProblem(options ?? {});
2713
+ if (problem)
2714
+ throw new ValidationError(problem);
2715
+ await this.simctl.launchApp(await this.soleLiveDeviceId(), bundleId, compactLaunchOptions(options));
2124
2716
  }
2125
2717
  /** The one booted session, or undefined when there is no unambiguous answer. Callers that can
2126
2718
  * reasonably do nothing (input) use this; callers that owe the user a result use
@@ -2228,7 +2820,7 @@ export class IOSAgent {
2228
2820
  const seq = state.bootSeq;
2229
2821
  const streamer = new AudioCaptureStreamer();
2230
2822
  streamer.listen()
2231
- .then((port) => {
2823
+ .then(async (port) => {
2232
2824
  // A reboot/shutdown/disconnect may have superseded this boot while listen() was binding —
2233
2825
  // discard so we don't leave an orphan helper/poll/server behind the current lifecycle.
2234
2826
  if (seq !== state.bootSeq || streamWs.readyState !== WebSocket.OPEN) {
@@ -2237,21 +2829,37 @@ export class IOSAgent {
2237
2829
  }
2238
2830
  state.audioStreamer = streamer;
2239
2831
  state.audioPort = port;
2240
- state.audioVolume = readSimVolume(udid);
2832
+ // Read before launching, so the first frames carry the sim's volume rather than full.
2833
+ const [pids, volume] = await Promise.all([enumerateSimPids(udid), readSimVolume(udid)]);
2834
+ // A stop or reboot while reading retired this capture: `stopAudioCapture` already stopped `streamer`,
2835
+ // and launching the helper now would tap a device nobody is streaming.
2836
+ if (state.audioStreamer !== streamer)
2837
+ return;
2838
+ // Only the control socket's close tears audio down; a stream socket that closed while reading would
2839
+ // leave a helper and a poll running for nothing until the next reboot.
2840
+ if (streamWs.readyState !== WebSocket.OPEN) {
2841
+ this.stopAudioCapture(state);
2842
+ return;
2843
+ }
2844
+ state.audioVolume = volume;
2241
2845
  void this.pumpAudio(streamWs, streamer.frames(), state);
2242
- this.launchWholeSimTap(state, udid);
2846
+ this.launchWholeSimTap(state, pids);
2847
+ // Started after the launch, so the first poll never races the launch's own pid set. One poll at a
2848
+ // time: `ps` is bounded by its timeout, and a slow one is not stacked by the next tick.
2849
+ let polling = false;
2243
2850
  state.audioPoll = setInterval(() => {
2244
- this.refreshAudioPids(state, udid);
2245
- state.audioVolume = readSimVolume(udid); // track live sim-volume changes
2851
+ if (polling)
2852
+ return;
2853
+ polling = true;
2854
+ void this.refreshAudio(state, udid, streamer).finally(() => { polling = false; });
2246
2855
  }, AUDIO_POLL_MS);
2247
2856
  })
2248
2857
  .catch((e) => logger.warn(`audio capture server failed to start: ${e instanceof Error ? e.message : String(e)}`));
2249
2858
  }
2250
2859
  // Launch the tap helper for the simulator's whole process tree. Swallows errors — audio must never
2251
2860
  // break the launch/video path (e.g. helper build fails, or the user hasn't granted the audio
2252
- // permission). The helper holds the socket open; refreshAudioPids() pushes later updates to it.
2253
- launchWholeSimTap(state, udid) {
2254
- const pids = enumerateSimPids(udid);
2861
+ // permission). The helper holds the socket open; refreshAudio() pushes later updates to it.
2862
+ launchWholeSimTap(state, pids) {
2255
2863
  if (!pids.length) {
2256
2864
  logger.warn('no simulator processes to tap (audio idle until first poll)');
2257
2865
  return;
@@ -2268,18 +2876,21 @@ export class IOSAgent {
2268
2876
  // freshly spawned WebKit WebContent), push the updated set so the helper rebuilds its tap. Dead pids
2269
2877
  // need no rebuild — they resolve to no audio object, so the helper just stops mixing them; we still
2270
2878
  // refresh the baseline so a later reappearance is detected. Rebuilding only on additions keeps
2271
- // short-lived daemon churn from causing constant tap teardown (audio glitches).
2272
- refreshAudioPids(state, udid) {
2273
- if (!state.audioStreamer)
2879
+ // short-lived daemon churn from causing constant tap teardown (audio glitches). Also tracks live sim-volume
2880
+ // changes. Both reads are asynchronous, so the capture may have been retired by the time they land; `streamer`
2881
+ // is the capture this poll belongs to.
2882
+ async refreshAudio(state, udid, streamer) {
2883
+ const [pids, volume] = await Promise.all([enumerateSimPids(udid), readSimVolume(udid)]);
2884
+ if (state.audioStreamer !== streamer)
2274
2885
  return;
2275
- const pids = enumerateSimPids(udid);
2886
+ state.audioVolume = volume;
2276
2887
  if (!pids.length)
2277
2888
  return;
2278
2889
  const prev = state.audioPids;
2279
2890
  const hasNew = !prev || pids.some((p) => !prev.has(p));
2280
2891
  state.audioPids = new Set(pids);
2281
2892
  if (hasNew)
2282
- state.audioStreamer.updatePids(pids);
2893
+ streamer.updatePids(pids);
2283
2894
  }
2284
2895
  // Forward captured PCM to the relay on the shared stream socket via the yielding sender (audio must
2285
2896
  // never inflate the socket buffer enough to trip video's backpressure). Mirrors android-agent.