@e2e-dev/mobile 0.12.1 → 0.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/.build.json CHANGED
@@ -1 +1 @@
1
- {"version":"0.12.1"}
1
+ {"version":"0.12.2"}
package/dist/device.js CHANGED
@@ -25,7 +25,7 @@ export function createDeviceFixture(surface, context) {
25
25
  throw unsupported(`device.setNetwork('offline') turns off an Android device's Wi-Fi; ${IOS_OFFLINE}`);
26
26
  return;
27
27
  }
28
- await surface.command('device.setNetwork', (client) => client.settings.update({ setting: 'wifi', state: state === 'offline' ? 'off' : 'on' }), context.signal);
28
+ await surface.command('device.setNetwork', (client) => client.settings.update({ ...surface.selection(), setting: 'wifi', state: state === 'offline' ? 'off' : 'on' }), context.signal);
29
29
  },
30
30
  async setAirplaneMode(enabled) {
31
31
  if (surface.options.platform === 'ios') {
@@ -33,7 +33,7 @@ export function createDeviceFixture(surface, context) {
33
33
  throw unsupported(`device.setAirplaneMode(true) cuts an Android device's radios; ${IOS_OFFLINE}`);
34
34
  return;
35
35
  }
36
- await surface.command('device.setAirplaneMode', (client) => client.settings.update({ setting: 'airplane', state: enabled ? 'on' : 'off' }), context.signal);
36
+ await surface.command('device.setAirplaneMode', (client) => client.settings.update({ ...surface.selection(), setting: 'airplane', state: enabled ? 'on' : 'off' }), context.signal);
37
37
  },
38
38
  async setPermission(permission, state) {
39
39
  await surface.setPermission(permission, state, context.signal);
@@ -43,29 +43,29 @@ export function createDeviceFixture(surface, context) {
43
43
  // `clearLocation` switched them off for good on the emulator (the
44
44
  // setting outlives the app); iOS has no such switch.
45
45
  if (surface.options.platform === 'android') {
46
- await surface.command('device.setLocation', (client) => client.settings.update({ setting: 'location', state: 'on' }), context.signal);
46
+ await surface.command('device.setLocation', (client) => client.settings.update({ ...surface.selection(), setting: 'location', state: 'on' }), context.signal);
47
47
  }
48
- await surface.command('device.setLocation', (client) => client.settings.update({ setting: 'location', state: 'set', latitude, longitude }), context.signal);
48
+ await surface.command('device.setLocation', (client) => client.settings.update({ ...surface.selection(), setting: 'location', state: 'set', latitude, longitude }), context.signal);
49
49
  },
50
50
  async clearLocation() {
51
- await surface.command('device.clearLocation', (client) => client.settings.update({ setting: 'location', state: 'off' }), context.signal);
51
+ await surface.command('device.clearLocation', (client) => client.settings.update({ ...surface.selection(), setting: 'location', state: 'off' }), context.signal);
52
52
  },
53
53
  async setAppearance(mode) {
54
- await surface.command('device.setAppearance', (client) => client.settings.update({ setting: 'appearance', state: mode }), context.signal);
54
+ await surface.command('device.setAppearance', (client) => client.settings.update({ ...surface.selection(), setting: 'appearance', state: mode }), context.signal);
55
55
  },
56
56
  async setOrientation(orientation) {
57
- await surface.screenCommand('device.setOrientation', (client) => client.command.orientation({ orientation }), context.signal);
57
+ await surface.screenCommand('device.setOrientation', (client) => client.command.orientation({ ...surface.selection(), orientation }), context.signal);
58
58
  },
59
59
  async fold(pose) {
60
60
  await surface.fold(pose, context.signal);
61
61
  },
62
62
  async setBiometrics(sensor, result) {
63
63
  await surface.command('device.setBiometrics', (client) => sensor === 'fingerprint'
64
- ? client.settings.update({ setting: 'fingerprint', state: result })
65
- : client.settings.update({ setting: sensor, state: result }), context.signal);
64
+ ? client.settings.update({ ...surface.selection(), setting: 'fingerprint', state: result })
65
+ : client.settings.update({ ...surface.selection(), setting: sensor, state: result }), context.signal);
66
66
  },
67
67
  async enrollBiometrics(sensor, enrolled) {
68
- await surface.command('device.enrollBiometrics', (client) => client.settings.update({ setting: sensor, state: enrolled ? 'enroll' : 'unenroll' }), context.signal);
68
+ await surface.command('device.enrollBiometrics', (client) => client.settings.update({ ...surface.selection(), setting: sensor, state: enrolled ? 'enroll' : 'unenroll' }), context.signal);
69
69
  },
70
70
  async installApp(appPath, options) {
71
71
  return surface.installApp(appPath, options ?? {}, context.signal);
@@ -92,23 +92,23 @@ export function createDeviceFixture(surface, context) {
92
92
  };
93
93
  },
94
94
  async home() {
95
- await surface.screenCommand('device.home', (client) => client.command.home({}), context.signal);
95
+ await surface.screenCommand('device.home', (client) => client.command.home(surface.selection()), context.signal);
96
96
  },
97
97
  async back() {
98
98
  await surface.screenCommand('device.back', (client) => client.command.back({ ...surface.settleOptions }), context.signal);
99
99
  },
100
100
  async alert(action) {
101
- await surface.screenCommand('device.alert', (client) => client.command.alert({ action }), context.signal);
101
+ await surface.screenCommand('device.alert', (client) => client.command.alert({ ...surface.selection(), action }), context.signal);
102
102
  },
103
103
  async dismissKeyboard() {
104
104
  await surface.dismissKeyboard(context.signal);
105
105
  },
106
106
  async clipboard() {
107
- const result = await surface.command('device.clipboard', (client) => client.command.clipboard({ action: 'read' }), context.signal);
107
+ const result = await surface.command('device.clipboard', (client) => client.command.clipboard({ ...surface.selection(), action: 'read' }), context.signal);
108
108
  return result.action === 'read' ? result.text : '';
109
109
  },
110
110
  async setClipboard(text) {
111
- await surface.command('device.setClipboard', (client) => client.command.clipboard({ action: 'write', text }), context.signal);
111
+ await surface.command('device.setClipboard', (client) => client.command.clipboard({ ...surface.selection(), action: 'write', text }), context.signal);
112
112
  },
113
113
  };
114
114
  const action = { kind: 'resource' };
package/dist/nodes.d.ts CHANGED
@@ -32,6 +32,8 @@ export interface ProjectedSnapshot {
32
32
  readonly index: readonly ProjectedNode[];
33
33
  readonly viewport: ViewportSize | undefined;
34
34
  }
35
+ /** One element-type spelling for `NavigationBar`, `XCUIElementTypeNavigationBar`, `navigation-bar`, and `android.widget.NavigationBar` alike. */
36
+ export declare function normalizeKind(type: string): string;
35
37
  /**
36
38
  * Projects one snapshot. `mintId` is called once per node in document order,
37
39
  * so the surface's id space stays unique across observations. `viewport` is
package/dist/nodes.js CHANGED
@@ -187,9 +187,10 @@ const ANDROID_ROLE_MAP = {
187
187
  const ANDROID_TITLE_IDS = [':id/collapsing_toolbar', ':id/action_bar', ':id/toolbar'];
188
188
  /**
189
189
  * Platform element type of one raw node as a kebab-case token: XCTest sends
190
- * `NavigationBar` and `StaticText`, Android sends `android.widget.TextView`;
191
- * both read as one vocabulary here, the Android package prefix dropped.
192
- * `role` is the fallback some platforms send instead.
190
+ * `NavigationBar` and `StaticText` (`XCUIElementTypeNavigationBar` over
191
+ * WebDriver), Android sends `android.widget.TextView`; all read as one
192
+ * vocabulary here, the class and package prefixes dropped. `role` is the
193
+ * fallback some platforms send instead.
193
194
  */
194
195
  function kindOf(raw) {
195
196
  return normalizeKind(raw.type ?? raw.role ?? '');
@@ -198,9 +199,9 @@ function kindOf(raw) {
198
199
  function isAndroidClass(type) {
199
200
  return type !== undefined && type.includes('.');
200
201
  }
201
- /** One element-type spelling for `NavigationBar`, `navigation-bar`, and `android.widget.NavigationBar` alike. */
202
- function normalizeKind(type) {
203
- const simple = type.slice(type.lastIndexOf('.') + 1);
202
+ /** One element-type spelling for `NavigationBar`, `XCUIElementTypeNavigationBar`, `navigation-bar`, and `android.widget.NavigationBar` alike. */
203
+ export function normalizeKind(type) {
204
+ const simple = type.slice(type.lastIndexOf('.') + 1).replace(/^XCUIElementType(?=[A-Z])/, '');
204
205
  return simple
205
206
  .replaceAll(/([a-z0-9])([A-Z])/g, '$1-$2')
206
207
  .replaceAll(/[\s_]+/g, '-')
package/dist/surface.d.ts CHANGED
@@ -13,7 +13,7 @@
13
13
  */
14
14
  import { type EngineAttemptContext, type EngineCleanupContext, type EngineInitInfo, type EngineObserveOptions, type EngineSnapshot, type LocatorAction, type LocatorExpression, type NodeRef, type OperationContext, type PointerAction, type VideoSegment, type SemanticNode, type ViewportPoint } from 'e2e/engine';
15
15
  import { type AgentDeviceClient, type ClientFactory, type DevicePermission, type LaunchPermissions, type MobileOptions, type PermissionState } from './options.ts';
16
- import { DevicePool } from './pool.ts';
16
+ import { DevicePool, type DeviceSelection } from './pool.ts';
17
17
  import { InputBudget } from './support.ts';
18
18
  /** Hinge poses `device.fold` accepts: `closed` lights the outer display, `half-open` and `open` the inner one. */
19
19
  export type FoldPose = 'closed' | 'half-open' | 'open';
@@ -149,8 +149,14 @@ export declare class AgentDeviceSurface {
149
149
  * racing it: the launch timeout is the honest bound on a stuck device.
150
150
  */
151
151
  private settleInflight;
152
- /** The device selection this worker's commands take. */
153
- private selection;
152
+ /**
153
+ * The device selection this worker's commands take. A command that can
154
+ * run with no app open (before `app.open()`, after `closeApp()` ends the
155
+ * session) carries it: the daemon then resolves the device from it and
156
+ * not from every booted one. With a session open it names the session's
157
+ * own device, which agent-device accepts.
158
+ */
159
+ selection(): DeviceSelection;
154
160
  init(info: EngineInitInfo): Promise<void>;
155
161
  startAttempt(context: EngineAttemptContext): Promise<void>;
156
162
  endAttempt(context: EngineCleanupContext): Promise<void>;
package/dist/surface.js CHANGED
@@ -18,7 +18,7 @@ import { EngineError, KEY_NAMES, parseKey, raceAbort, rejectUnknownKeys, stopPro
18
18
  import { isSnapshotPresentationFailure, runCommand, staleOr } from './errors.js';
19
19
  import { pointerInteraction, DEFAULT_LONG_PRESS_MS } from './actions.js';
20
20
  import { resolveExpression } from './locate.js';
21
- import { isWithin, projectSnapshot, ROOT_ID, screenRoot, screenTitle, } from './nodes.js';
21
+ import { isWithin, normalizeKind, projectSnapshot, ROOT_ID, screenRoot, screenTitle, } from './nodes.js';
22
22
  import { DEVICE_PERMISSIONS } from './options.js';
23
23
  import { maskPng } from './png.js';
24
24
  import { deviceLabel, pinnedApp } from './bindings.js';
@@ -37,6 +37,17 @@ const KEYBOARD_TYPES = new Set(['keyboard', 'key']);
37
37
  * Maestro's `hideKeyboard` uses.
38
38
  */
39
39
  const KEYBOARD_DRAG_RATIO = 0.03;
40
+ /**
41
+ * Whether the capture's producer measured the soft keyboard on screen:
42
+ * undefined when it did not look or could not.
43
+ */
44
+ function measuredKeyboard(raw) {
45
+ if (raw.keyboard?.kind === 'visible')
46
+ return true;
47
+ if (raw.keyboard?.kind === 'absent')
48
+ return false;
49
+ return undefined;
50
+ }
40
51
  /**
41
52
  * Whether a capture shows the soft keyboard: the band its producer measured
42
53
  * when it measured one, else a keyboard or key element in the tree, the probe
@@ -45,11 +56,7 @@ const KEYBOARD_DRAG_RATIO = 0.03;
45
56
  * tap refuses against by name.
46
57
  */
47
58
  function keyboardShowing(raw) {
48
- if (raw.keyboard?.kind === 'visible')
49
- return true;
50
- if (raw.keyboard?.kind === 'absent')
51
- return false;
52
- return (raw.nodes ?? []).some((node) => KEYBOARD_TYPES.has((node.type ?? '').toLowerCase()));
59
+ return measuredKeyboard(raw) ?? (raw.nodes ?? []).some((node) => KEYBOARD_TYPES.has(normalizeKind(node.type ?? '')));
53
60
  }
54
61
  /**
55
62
  * Whether a snapshot leaves out nodes that are on screen: agent-device says
@@ -300,7 +307,13 @@ export class AgentDeviceSurface {
300
307
  await raceAbort(Promise.allSettled(this.inflight), signal, 'settling in-flight device commands');
301
308
  }
302
309
  }
303
- /** The device selection this worker's commands take. */
310
+ /**
311
+ * The device selection this worker's commands take. A command that can
312
+ * run with no app open (before `app.open()`, after `closeApp()` ends the
313
+ * session) carries it: the daemon then resolves the device from it and
314
+ * not from every booted one. With a session open it names the session's
315
+ * own device, which agent-device accepts.
316
+ */
304
317
  selection() {
305
318
  return deviceSelection(this.options.platform, this.device);
306
319
  }
@@ -389,6 +402,7 @@ export class AgentDeviceSurface {
389
402
  await this.command('start video recording',
390
403
  // The whole screen: an app scope refuses to start while no app is open, before `app.open()` and after `closeApp()`.
391
404
  (client) => client.recording.record({
405
+ ...this.selection(),
392
406
  action: 'start',
393
407
  path: absolute,
394
408
  quality: 'medium',
@@ -431,7 +445,7 @@ export class AgentDeviceSurface {
431
445
  * `closeApp` leaves it.
432
446
  */
433
447
  async stopScreenRecording(signal) {
434
- const result = await this.command('stop video recording', (client) => client.recording.record({ action: 'stop' }), signal);
448
+ const result = await this.command('stop video recording', (client) => client.recording.record({ ...this.selection(), action: 'stop' }), signal);
435
449
  if (result.recordOnlySession === true) {
436
450
  this.screenReplaced();
437
451
  }
@@ -580,7 +594,7 @@ export class AgentDeviceSurface {
580
594
  */
581
595
  async closeApp(signal) {
582
596
  const app = this.appIdentity ?? this.pinnedApp;
583
- await this.command('device.closeApp', (client) => client.apps.close(app === undefined ? {} : { app }), signal);
597
+ await this.command('device.closeApp', (client) => client.apps.close({ ...this.selection(), ...(app === undefined ? {} : { app }) }), signal);
584
598
  this.screenReplaced();
585
599
  }
586
600
  /**
@@ -704,10 +718,12 @@ export class AgentDeviceSurface {
704
718
  const viewport = await this.viewportFor(projected, operation.signal);
705
719
  const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected));
706
720
  const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined;
721
+ const keyboardVisible = measuredKeyboard(raw);
707
722
  return {
708
723
  root: screenRoot(projected.roots, viewport),
709
724
  viewport,
710
725
  ...(isTruncated(raw) ? { truncated: true } : {}),
726
+ ...(keyboardVisible === undefined ? {} : { keyboardVisible }),
711
727
  ...(location === undefined ? {} : { location }),
712
728
  ...(capture === undefined ? {} : { pixels: capture.pixels, maskedRegionCount: capture.masked }),
713
729
  };
@@ -1131,7 +1147,7 @@ export class AgentDeviceSurface {
1131
1147
  const app = this.pinnedApp;
1132
1148
  if (app === undefined)
1133
1149
  throw unsupported(`app.clearState needs the target's \`app.bundleId\`, or ${UNINSTALLED_BUILD}`);
1134
- await this.command('clear app state', (client) => client.settings.update({ setting: 'clear-app-state', state: 'clear', app }), operation.signal);
1150
+ await this.command('clear app state', (client) => client.settings.update({ ...this.selection(), setting: 'clear-app-state', state: 'clear', app }), operation.signal);
1135
1151
  await this.openApp(app, { relaunch: true }, operation.signal);
1136
1152
  }
1137
1153
  /**
@@ -1142,7 +1158,7 @@ export class AgentDeviceSurface {
1142
1158
  * without a hinge comes back from agent-device as `UNSUPPORTED_OPERATION`.
1143
1159
  */
1144
1160
  async fold(pose, signal) {
1145
- await this.screenCommand('device.fold', (client) => client.command.fold({ pose }), signal);
1161
+ await this.screenCommand('device.fold', (client) => client.command.fold({ ...this.selection(), pose }), signal);
1146
1162
  }
1147
1163
  /**
1148
1164
  * Resets the simulator's keychain through agent-device. simctl resets the
@@ -1153,7 +1169,7 @@ export class AgentDeviceSurface {
1153
1169
  if (this.options.platform !== 'ios') {
1154
1170
  throw unsupported('device.clearKeychain resets an iOS simulator keychain; Android has none to reset');
1155
1171
  }
1156
- await this.command('device.clearKeychain', (client) => client.settings.update({ setting: 'reset-keychain', state: 'clear' }), signal);
1172
+ await this.command('device.clearKeychain', (client) => client.settings.update({ ...this.selection(), setting: 'reset-keychain', state: 'clear' }), signal);
1157
1173
  }
1158
1174
  /**
1159
1175
  * A redacted screenshot artifact. The device paints secure fields as dots,
package/dist/tools.js CHANGED
@@ -81,7 +81,8 @@ export function mobileTools(...engines) {
81
81
  description: 'Accept or dismiss a visible system alert or permission prompt. Use it when the alert\'s buttons are not listed on screen; a listed button tapped by id is the same tap and replays from the trace cache, this tool does not.',
82
82
  inputSchema: z.object({ action: z.enum(['accept', 'dismiss']) }),
83
83
  execute: async ({ action }, options) => {
84
- await active().command('alert', (client) => client.command.alert({ action }), abort(options));
84
+ const surface = active();
85
+ await surface.command('alert', (client) => client.command.alert({ ...surface.selection(), action }), abort(options));
85
86
  return `Alert ${action}ed.`;
86
87
  },
87
88
  }), annotate(true)),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@e2e-dev/mobile",
3
- "version": "0.12.1",
3
+ "version": "0.12.2",
4
4
  "description": "Mobile engine (iOS and Android) for e2e agentic end-to-end testing, built on agent-device",
5
5
  "keywords": [
6
6
  "e2e",
@@ -68,7 +68,7 @@
68
68
  "@types/node": "26.6.2",
69
69
  "@types/pngjs": "6.0.5",
70
70
  "ai": "7.0.107",
71
- "e2e": "0.21.0",
71
+ "e2e": "0.21.1",
72
72
  "typescript": "7.0.2",
73
73
  "vitest": "5.0.1"
74
74
  },