@e2e-dev/mobile 0.12.4-nightly-20261021080401 → 0.12.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/.build.json +1 -1
- package/dist/keyboard.d.ts +19 -0
- package/dist/keyboard.js +105 -0
- package/dist/locate.js +7 -0
- package/dist/nodes.d.ts +2 -0
- package/dist/nodes.js +2 -1
- package/dist/surface.d.ts +37 -17
- package/dist/surface.js +86 -34
- package/package.json +4 -4
package/README.md
CHANGED
package/dist/.build.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":"0.12.4
|
|
1
|
+
{"version":"0.12.4"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Where the keyboard-dismissing drag may start on a screen without pressing anything. */
|
|
2
|
+
import type { ViewportSize } from 'e2e/engine';
|
|
3
|
+
import { type ProjectedSnapshot } from './nodes.ts';
|
|
4
|
+
import type { Point } from './support.ts';
|
|
5
|
+
/**
|
|
6
|
+
* Where a keyboard-dismissing drag starts: the point nearest the middle of
|
|
7
|
+
* the screen that is above the keyboard and clear of every control. When the
|
|
8
|
+
* screen has a scroll view the point must be inside one, since a scroll view
|
|
9
|
+
* is what turns the drag into a dismissal and a drag anywhere else does
|
|
10
|
+
* nothing. A control is a node a touch acts on: one with a control role, or
|
|
11
|
+
* a leaf that is not plain content (text, an image), since a React Native
|
|
12
|
+
* `Pressable` with no role is an unnamed leaf like any other. A `Text` with
|
|
13
|
+
* its own `onPress` reads as plain text and is one this misses; a control
|
|
14
|
+
* inside a flattened tree, which shows as one leaf over most of the screen,
|
|
15
|
+
* is the other.
|
|
16
|
+
* Undefined when no point qualifies: a drag there would press a control or
|
|
17
|
+
* do nothing.
|
|
18
|
+
*/
|
|
19
|
+
export declare function keyboardDragStart(projected: ProjectedSnapshot, viewport: ViewportSize, keyboardTop: number): Point | undefined;
|
package/dist/keyboard.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/** Where the keyboard-dismissing drag may start on a screen without pressing anything. */
|
|
2
|
+
import { SCREEN_KINDS } from './nodes.js';
|
|
3
|
+
/**
|
|
4
|
+
* Roles no drag starts on, whether or not their children are listed: the
|
|
5
|
+
* ones a touch presses, toggles, or focuses, and the bars, where a drag
|
|
6
|
+
* never reaches the content.
|
|
7
|
+
*/
|
|
8
|
+
const CONTROL_ROLES = new Set([
|
|
9
|
+
'button',
|
|
10
|
+
'link',
|
|
11
|
+
'textbox',
|
|
12
|
+
'searchbox',
|
|
13
|
+
'combobox',
|
|
14
|
+
'checkbox',
|
|
15
|
+
'radio',
|
|
16
|
+
'switch',
|
|
17
|
+
'tab',
|
|
18
|
+
'menuitem',
|
|
19
|
+
'option',
|
|
20
|
+
'slider',
|
|
21
|
+
'spinbutton',
|
|
22
|
+
'listitem',
|
|
23
|
+
'navigation',
|
|
24
|
+
'toolbar',
|
|
25
|
+
'tablist',
|
|
26
|
+
]);
|
|
27
|
+
/** Platform kinds that scroll, so a short drag on them reads as the touch that sends the keyboard away. */
|
|
28
|
+
const SCROLL_KINDS = new Set(['scroll-view', 'table', 'collection-view', 'recycler-view', 'list-view']);
|
|
29
|
+
/** Roles a touch does nothing to: a leaf with one of these is content, not a control. */
|
|
30
|
+
const STATIC_ROLES = new Set(['text', 'image', 'heading', 'status', 'progressbar']);
|
|
31
|
+
/**
|
|
32
|
+
* How far a drag starts from any control, in points: a control's touch area
|
|
33
|
+
* runs past its frame (React Native's `hitSlop`, UIKit's own padding), and
|
|
34
|
+
* the drag's first few points are inside touch slop, where it still counts
|
|
35
|
+
* as a tap on whatever it started on.
|
|
36
|
+
*/
|
|
37
|
+
const CONTROL_CLEARANCE = 8;
|
|
38
|
+
/**
|
|
39
|
+
* The share of the screen a leaf covers past which it is content merged into
|
|
40
|
+
* one element (a flattened tree), not a control: the tree shows nothing
|
|
41
|
+
* inside it to steer clear of.
|
|
42
|
+
*/
|
|
43
|
+
const MERGED_SHARE = 0.5;
|
|
44
|
+
/** The share of each screen edge no drag starts in: the status bar, the home indicator, and the system edge gestures live there. */
|
|
45
|
+
const EDGE_SHARE = 0.1;
|
|
46
|
+
/** How many candidate points the search lays across the screen's shorter side. */
|
|
47
|
+
const GRID_STEPS = 40;
|
|
48
|
+
/**
|
|
49
|
+
* Where a keyboard-dismissing drag starts: the point nearest the middle of
|
|
50
|
+
* the screen that is above the keyboard and clear of every control. When the
|
|
51
|
+
* screen has a scroll view the point must be inside one, since a scroll view
|
|
52
|
+
* is what turns the drag into a dismissal and a drag anywhere else does
|
|
53
|
+
* nothing. A control is a node a touch acts on: one with a control role, or
|
|
54
|
+
* a leaf that is not plain content (text, an image), since a React Native
|
|
55
|
+
* `Pressable` with no role is an unnamed leaf like any other. A `Text` with
|
|
56
|
+
* its own `onPress` reads as plain text and is one this misses; a control
|
|
57
|
+
* inside a flattened tree, which shows as one leaf over most of the screen,
|
|
58
|
+
* is the other.
|
|
59
|
+
* Undefined when no point qualifies: a drag there would press a control or
|
|
60
|
+
* do nothing.
|
|
61
|
+
*/
|
|
62
|
+
export function keyboardDragStart(projected, viewport, keyboardTop) {
|
|
63
|
+
const controls = [];
|
|
64
|
+
const scrollViews = [];
|
|
65
|
+
const merged = viewport.width * viewport.height * MERGED_SHARE;
|
|
66
|
+
for (const entry of projected.index) {
|
|
67
|
+
const { rect, role, children, states } = entry.node;
|
|
68
|
+
if (rect === undefined || states?.hidden === true || SCREEN_KINDS.has(entry.kind))
|
|
69
|
+
continue;
|
|
70
|
+
if (SCROLL_KINDS.has(entry.kind))
|
|
71
|
+
scrollViews.push(rect);
|
|
72
|
+
else if ((role !== undefined && CONTROL_ROLES.has(role)) || (children === undefined && !STATIC_ROLES.has(role ?? '') && rect.width * rect.height <= merged)) {
|
|
73
|
+
controls.push(grow(rect, CONTROL_CLEARANCE));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const left = viewport.width * EDGE_SHARE;
|
|
77
|
+
const right = viewport.width * (1 - EDGE_SHARE);
|
|
78
|
+
const top = viewport.height * EDGE_SHARE;
|
|
79
|
+
const bottom = Math.min(viewport.height * (1 - EDGE_SHARE), keyboardTop - CONTROL_CLEARANCE);
|
|
80
|
+
const centre = { x: viewport.width / 2, y: viewport.height / 2 };
|
|
81
|
+
// The grid is anchored on the centre, so a screen with nothing there starts where a user's thumb would.
|
|
82
|
+
const step = Math.min(viewport.width, viewport.height) / GRID_STEPS;
|
|
83
|
+
let best;
|
|
84
|
+
for (let y = centre.y - Math.floor((centre.y - top) / step) * step; y <= bottom; y += step) {
|
|
85
|
+
for (let x = centre.x - Math.floor((centre.x - left) / step) * step; x <= right; x += step) {
|
|
86
|
+
const point = { x, y };
|
|
87
|
+
if (scrollViews.length > 0 && !scrollViews.some((view) => contains(view, point)))
|
|
88
|
+
continue;
|
|
89
|
+
if (controls.some((control) => contains(control, point)))
|
|
90
|
+
continue;
|
|
91
|
+
const distance = Math.hypot(x - centre.x, y - centre.y);
|
|
92
|
+
if (best === undefined || distance < best.distance)
|
|
93
|
+
best = { point, distance };
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return best?.point;
|
|
97
|
+
}
|
|
98
|
+
/** A rect grown by `margin` on every side. */
|
|
99
|
+
function grow(rect, margin) {
|
|
100
|
+
return { x: rect.x - margin, y: rect.y - margin, width: rect.width + margin * 2, height: rect.height + margin * 2 };
|
|
101
|
+
}
|
|
102
|
+
/** Whether a point lies inside a rect, edges included. */
|
|
103
|
+
function contains(rect, point) {
|
|
104
|
+
return point.x >= rect.x && point.x <= rect.x + rect.width && point.y >= rect.y && point.y <= rect.y + rect.height;
|
|
105
|
+
}
|
package/dist/locate.js
CHANGED
|
@@ -10,6 +10,13 @@ import { compileSelector } from './selector.js';
|
|
|
10
10
|
/** Resolves one expression to the nodes it currently matches; `platform` is the one agent-device selectors match for. */
|
|
11
11
|
export function resolveExpression(expression, index, platform) {
|
|
12
12
|
switch (expression.kind) {
|
|
13
|
+
case 'union': {
|
|
14
|
+
const matches = new Set([
|
|
15
|
+
...resolveExpression(expression.left, index, platform),
|
|
16
|
+
...resolveExpression(expression.right, index, platform),
|
|
17
|
+
]);
|
|
18
|
+
return index.filter((entry) => matches.has(entry));
|
|
19
|
+
}
|
|
13
20
|
case 'query': {
|
|
14
21
|
const candidates = expression.scope === undefined ? index : descendantsOf(resolveExpression(expression.scope, index, platform), index);
|
|
15
22
|
const full = candidates.filter((entry) => matchesQuery(entry, expression.query, false));
|
package/dist/nodes.d.ts
CHANGED
|
@@ -32,6 +32,8 @@ export interface ProjectedSnapshot {
|
|
|
32
32
|
readonly index: readonly ProjectedNode[];
|
|
33
33
|
readonly viewport: ViewportSize | undefined;
|
|
34
34
|
}
|
|
35
|
+
/** Kinds that span the screen and hold everything else. */
|
|
36
|
+
export declare const SCREEN_KINDS: ReadonlySet<string>;
|
|
35
37
|
/** One element-type spelling for `NavigationBar`, `XCUIElementTypeNavigationBar`, `navigation-bar`, and `android.widget.NavigationBar` alike. */
|
|
36
38
|
export declare function normalizeKind(type: string): string;
|
|
37
39
|
/**
|
package/dist/nodes.js
CHANGED
|
@@ -66,7 +66,8 @@ const TEXT_INPUT_ROLES = new Set(['textbox', 'searchbox', 'combobox']);
|
|
|
66
66
|
const CHECKABLE_ROLES = new Set(['switch', 'checkbox']);
|
|
67
67
|
const CHECKED_VALUES = new Set(['1', 'on', 'true', 'checked', 'selected']);
|
|
68
68
|
const UNCHECKED_VALUES = new Set(['0', 'off', 'false', 'unchecked']);
|
|
69
|
-
|
|
69
|
+
/** Kinds that span the screen and hold everything else. */
|
|
70
|
+
export const SCREEN_KINDS = new Set(['application', 'app', 'window']);
|
|
70
71
|
/**
|
|
71
72
|
* The role descriptions React Native writes into an iOS view's accessibility
|
|
72
73
|
* value for the roles UIKit has no trait for, onto the contract role each
|
package/dist/surface.d.ts
CHANGED
|
@@ -113,8 +113,10 @@ export declare class AgentDeviceSurface {
|
|
|
113
113
|
/**
|
|
114
114
|
* The screen's logical size as last learned from a snapshot with geometry
|
|
115
115
|
* or from the device itself, so a snapshot without geometry (an empty
|
|
116
|
-
* screen before any app is open, a sparse tree)
|
|
117
|
-
* every observation
|
|
116
|
+
* screen before any app is open, a sparse tree) or with an app drawn
|
|
117
|
+
* smaller than the screen still reports the viewport every observation
|
|
118
|
+
* must carry. A rotation replaces it; any other size needs the device's
|
|
119
|
+
* screen to agree, see `viewportFor`.
|
|
118
120
|
*/
|
|
119
121
|
private knownViewport;
|
|
120
122
|
constructor(options: MobileOptions, createClient: ClientFactory);
|
|
@@ -277,13 +279,27 @@ export declare class AgentDeviceSurface {
|
|
|
277
279
|
* reason instead, so a wait that ends on an unreadable screen says so.
|
|
278
280
|
*/
|
|
279
281
|
private locateSnapshot;
|
|
282
|
+
/**
|
|
283
|
+
* Projects a snapshot and learns its viewport when it is plainly the
|
|
284
|
+
* screen's: the first one the session reads, the one it knows, or that one
|
|
285
|
+
* rotated. A tree of any other size may be an app drawn smaller than the
|
|
286
|
+
* screen (behind Notification Center, iOS shrinks it to a third), so an
|
|
287
|
+
* observation's `viewportFor` checks it against the device before it
|
|
288
|
+
* replaces the known one.
|
|
289
|
+
*/
|
|
280
290
|
private project;
|
|
281
291
|
/**
|
|
282
|
-
* The viewport of a snapshot: its own geometry
|
|
283
|
-
* session learned, else the device's logical screen
|
|
284
|
-
* screenshot. A
|
|
285
|
-
*
|
|
286
|
-
*
|
|
292
|
+
* The viewport of a snapshot: its own geometry when that is the screen,
|
|
293
|
+
* else the last one this session learned, else the device's logical screen
|
|
294
|
+
* size read off one screenshot. A tree whose size is neither the known one
|
|
295
|
+
* nor that one rotated replaces it only when the screenshot's size, in
|
|
296
|
+
* either orientation, agrees: a screenshot reports the portrait size even
|
|
297
|
+
* in landscape, so it confirms a size but cannot pick the orientation. Only
|
|
298
|
+
* a simulator's screenshot reports a logical size; on a real device the
|
|
299
|
+
* tree's size is taken as it comes. A device whose size cannot be learned
|
|
300
|
+
* at all fails the observation: the harness
|
|
301
|
+
* measures every rect and point against the viewport, so an invented one
|
|
302
|
+
* would misplace every tap.
|
|
287
303
|
*/
|
|
288
304
|
private viewportFor;
|
|
289
305
|
/** The device's logical screen size as its screenshot reports it; undefined when it reports none. */
|
|
@@ -377,21 +393,25 @@ export declare class AgentDeviceSurface {
|
|
|
377
393
|
dismissKeyboard(signal: AbortSignal, input?: InputBudget): Promise<void>;
|
|
378
394
|
/**
|
|
379
395
|
* A keyboard with no dismiss key goes the way a user sends it away: a short
|
|
380
|
-
* drag
|
|
381
|
-
*
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
*
|
|
396
|
+
* drag on the screen, which a scroll view turns into a dismissal, each
|
|
397
|
+
* followed by a fresh look at whether the keyboard is still there.
|
|
398
|
+
* Horizontal first: on a vertical scroll view it is no scroll, so React
|
|
399
|
+
* Native's scroll view takes it as the touch outside the keyboard that
|
|
400
|
+
* dismisses by default, where a vertical drag scrolls the content and
|
|
385
401
|
* dismisses only under `keyboardDismissMode="on-drag"`. Vertical second,
|
|
386
402
|
* for the scroll views that dismiss on a drag alone. Maestro's
|
|
387
403
|
* `hideKeyboard` drags in the other order, and on a React Native screen
|
|
388
404
|
* that costs a drag plus the settle after it. The simulator's "Speed up
|
|
389
405
|
* your typing" tip, which sits over a keyboard the first time it appears,
|
|
390
|
-
* goes through its Continue button first. A drag
|
|
391
|
-
*
|
|
392
|
-
*
|
|
393
|
-
*
|
|
394
|
-
*
|
|
406
|
+
* goes through its Continue button first. A drag that starts on a control
|
|
407
|
+
* presses it (a horizontal one is no scroll, so the control keeps the
|
|
408
|
+
* touch), so each drag starts where the latest capture shows none, inside
|
|
409
|
+
* a scroll view when the screen has one, see {@link keyboardDragStart}, and
|
|
410
|
+
* a screen with no such point is refused rather than touched. The sequence stops the moment the keyboard is gone,
|
|
411
|
+
* and a keyboard that outlives both drags fails naming what does work.
|
|
412
|
+
* Each look is the observation's own capture, sparse-tree retries
|
|
413
|
+
* included: a one-node tree has no keys and would pass for a dismissed
|
|
414
|
+
* keyboard.
|
|
395
415
|
*/
|
|
396
416
|
private dismissKeyboardByGesture;
|
|
397
417
|
/**
|
package/dist/surface.js
CHANGED
|
@@ -19,6 +19,7 @@ import { isAppNotRunning, isSnapshotPresentationFailure, isUnreadableCapture, ru
|
|
|
19
19
|
import { pointerInteraction, DEFAULT_LONG_PRESS_MS } from './actions.js';
|
|
20
20
|
import { resolveExpression } from './locate.js';
|
|
21
21
|
import { isWithin, normalizeKind, projectSnapshot, ROOT_ID, screenRoot, screenTitle, } from './nodes.js';
|
|
22
|
+
import { keyboardDragStart } from './keyboard.js';
|
|
22
23
|
import { DEVICE_PERMISSIONS } from './options.js';
|
|
23
24
|
import { maskPng } from './png.js';
|
|
24
25
|
import { deviceLabel, pinnedApp } from './bindings.js';
|
|
@@ -37,6 +38,19 @@ const KEYBOARD_TYPES = new Set(['keyboard', 'key']);
|
|
|
37
38
|
* Maestro's `hideKeyboard` uses.
|
|
38
39
|
*/
|
|
39
40
|
const KEYBOARD_DRAG_RATIO = 0.03;
|
|
41
|
+
/** What a test does instead when the engine cannot send the keyboard away. */
|
|
42
|
+
const KEYBOARD_ALTERNATIVES = 'Tap the app\'s own Done or close control, or press Enter on the field when submitting is what you want.';
|
|
43
|
+
/**
|
|
44
|
+
* The top edge of the soft keyboard: the band the capture's producer
|
|
45
|
+
* measured, else the highest keyboard or key element, else the bottom of the
|
|
46
|
+
* screen.
|
|
47
|
+
*/
|
|
48
|
+
function keyboardTop(raw, viewport) {
|
|
49
|
+
if (raw.keyboard?.kind === 'visible' || raw.keyboard?.kind === 'derived')
|
|
50
|
+
return raw.keyboard.frame.y;
|
|
51
|
+
const keys = (raw.nodes ?? []).filter((node) => node.rect !== undefined && KEYBOARD_TYPES.has(normalizeKind(node.type ?? '')));
|
|
52
|
+
return Math.min(viewport.height, ...keys.map((node) => node.rect.y));
|
|
53
|
+
}
|
|
40
54
|
/**
|
|
41
55
|
* Whether the capture's producer measured the soft keyboard on screen:
|
|
42
56
|
* undefined when it did not look or could not.
|
|
@@ -79,6 +93,11 @@ function isTruncated(raw) {
|
|
|
79
93
|
* the action are acted on at once.
|
|
80
94
|
*/
|
|
81
95
|
const DEFAULT_TRANSITION_MS = 500;
|
|
96
|
+
/** Whether two viewports are one screen, in the same orientation or rotated; rects from the tree carry fractions. */
|
|
97
|
+
function sameScreen(a, b) {
|
|
98
|
+
const near = (x, y) => Math.abs(x - y) < 1;
|
|
99
|
+
return (near(a.width, b.width) && near(a.height, b.height)) || (near(a.width, b.height) && near(a.height, b.width));
|
|
100
|
+
}
|
|
82
101
|
/** The centre of a rect in logical pixels. */
|
|
83
102
|
function centreOf(rect) {
|
|
84
103
|
return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 };
|
|
@@ -234,8 +253,10 @@ export class AgentDeviceSurface {
|
|
|
234
253
|
/**
|
|
235
254
|
* The screen's logical size as last learned from a snapshot with geometry
|
|
236
255
|
* or from the device itself, so a snapshot without geometry (an empty
|
|
237
|
-
* screen before any app is open, a sparse tree)
|
|
238
|
-
* every observation
|
|
256
|
+
* screen before any app is open, a sparse tree) or with an app drawn
|
|
257
|
+
* smaller than the screen still reports the viewport every observation
|
|
258
|
+
* must carry. A rotation replaces it; any other size needs the device's
|
|
259
|
+
* screen to agree, see `viewportFor`.
|
|
239
260
|
*/
|
|
240
261
|
knownViewport;
|
|
241
262
|
constructor(options, createClient) {
|
|
@@ -740,6 +761,14 @@ export class AgentDeviceSurface {
|
|
|
740
761
|
return undefined;
|
|
741
762
|
}
|
|
742
763
|
}
|
|
764
|
+
/**
|
|
765
|
+
* Projects a snapshot and learns its viewport when it is plainly the
|
|
766
|
+
* screen's: the first one the session reads, the one it knows, or that one
|
|
767
|
+
* rotated. A tree of any other size may be an app drawn smaller than the
|
|
768
|
+
* screen (behind Notification Center, iOS shrinks it to a third), so an
|
|
769
|
+
* observation's `viewportFor` checks it against the device before it
|
|
770
|
+
* replaces the known one.
|
|
771
|
+
*/
|
|
743
772
|
project(raw) {
|
|
744
773
|
const projected = projectSnapshot(raw.nodes ?? [], {
|
|
745
774
|
mintId: () => {
|
|
@@ -749,21 +778,38 @@ export class AgentDeviceSurface {
|
|
|
749
778
|
viewport: raw.viewport,
|
|
750
779
|
});
|
|
751
780
|
this.latestIndex = projected.index;
|
|
752
|
-
|
|
753
|
-
|
|
781
|
+
const read = projected.viewport;
|
|
782
|
+
if (read !== undefined && (this.knownViewport === undefined || sameScreen(read, this.knownViewport)))
|
|
783
|
+
this.knownViewport = read;
|
|
754
784
|
return projected;
|
|
755
785
|
}
|
|
756
786
|
/**
|
|
757
|
-
* The viewport of a snapshot: its own geometry
|
|
758
|
-
* session learned, else the device's logical screen
|
|
759
|
-
* screenshot. A
|
|
760
|
-
*
|
|
761
|
-
*
|
|
787
|
+
* The viewport of a snapshot: its own geometry when that is the screen,
|
|
788
|
+
* else the last one this session learned, else the device's logical screen
|
|
789
|
+
* size read off one screenshot. A tree whose size is neither the known one
|
|
790
|
+
* nor that one rotated replaces it only when the screenshot's size, in
|
|
791
|
+
* either orientation, agrees: a screenshot reports the portrait size even
|
|
792
|
+
* in landscape, so it confirms a size but cannot pick the orientation. Only
|
|
793
|
+
* a simulator's screenshot reports a logical size; on a real device the
|
|
794
|
+
* tree's size is taken as it comes. A device whose size cannot be learned
|
|
795
|
+
* at all fails the observation: the harness
|
|
796
|
+
* measures every rect and point against the viewport, so an invented one
|
|
797
|
+
* would misplace every tap.
|
|
762
798
|
*/
|
|
763
799
|
async viewportFor(projected, signal) {
|
|
764
|
-
const
|
|
765
|
-
|
|
766
|
-
|
|
800
|
+
const read = projected.viewport;
|
|
801
|
+
const known = this.knownViewport;
|
|
802
|
+
if (known !== undefined) {
|
|
803
|
+
if (read === undefined)
|
|
804
|
+
return known;
|
|
805
|
+
if (!sameScreen(read, known)) {
|
|
806
|
+
const probed = await this.probeViewport(signal);
|
|
807
|
+
if (probed !== undefined && !sameScreen(read, probed))
|
|
808
|
+
return known;
|
|
809
|
+
}
|
|
810
|
+
this.knownViewport = read;
|
|
811
|
+
return read;
|
|
812
|
+
}
|
|
767
813
|
const probed = await this.probeViewport(signal);
|
|
768
814
|
if (probed === undefined) {
|
|
769
815
|
throw new EngineError('ENGINE_FAILURE', 'the device reported no screen geometry: the snapshot has no bounds and the screenshot no logical size, so the viewport is unknown', { retryable: false });
|
|
@@ -817,7 +863,7 @@ export class AgentDeviceSurface {
|
|
|
817
863
|
*/
|
|
818
864
|
traceScreen(raw, projected) {
|
|
819
865
|
const sink = this.screenSink;
|
|
820
|
-
const viewport =
|
|
866
|
+
const viewport = this.knownViewport;
|
|
821
867
|
if (sink === undefined || viewport === undefined)
|
|
822
868
|
return;
|
|
823
869
|
const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected));
|
|
@@ -1144,40 +1190,46 @@ export class AgentDeviceSurface {
|
|
|
1144
1190
|
}
|
|
1145
1191
|
/**
|
|
1146
1192
|
* A keyboard with no dismiss key goes the way a user sends it away: a short
|
|
1147
|
-
* drag
|
|
1148
|
-
*
|
|
1149
|
-
*
|
|
1150
|
-
*
|
|
1151
|
-
*
|
|
1193
|
+
* drag on the screen, which a scroll view turns into a dismissal, each
|
|
1194
|
+
* followed by a fresh look at whether the keyboard is still there.
|
|
1195
|
+
* Horizontal first: on a vertical scroll view it is no scroll, so React
|
|
1196
|
+
* Native's scroll view takes it as the touch outside the keyboard that
|
|
1197
|
+
* dismisses by default, where a vertical drag scrolls the content and
|
|
1152
1198
|
* dismisses only under `keyboardDismissMode="on-drag"`. Vertical second,
|
|
1153
1199
|
* for the scroll views that dismiss on a drag alone. Maestro's
|
|
1154
1200
|
* `hideKeyboard` drags in the other order, and on a React Native screen
|
|
1155
1201
|
* that costs a drag plus the settle after it. The simulator's "Speed up
|
|
1156
1202
|
* your typing" tip, which sits over a keyboard the first time it appears,
|
|
1157
|
-
* goes through its Continue button first. A drag
|
|
1158
|
-
*
|
|
1159
|
-
*
|
|
1160
|
-
*
|
|
1161
|
-
*
|
|
1203
|
+
* goes through its Continue button first. A drag that starts on a control
|
|
1204
|
+
* presses it (a horizontal one is no scroll, so the control keeps the
|
|
1205
|
+
* touch), so each drag starts where the latest capture shows none, inside
|
|
1206
|
+
* a scroll view when the screen has one, see {@link keyboardDragStart}, and
|
|
1207
|
+
* a screen with no such point is refused rather than touched. The sequence stops the moment the keyboard is gone,
|
|
1208
|
+
* and a keyboard that outlives both drags fails naming what does work.
|
|
1209
|
+
* Each look is the observation's own capture, sparse-tree retries
|
|
1210
|
+
* included: a one-node tree has no keys and would pass for a dismissed
|
|
1211
|
+
* keyboard.
|
|
1162
1212
|
*/
|
|
1163
1213
|
async dismissKeyboardByGesture(signal, input) {
|
|
1164
1214
|
let raw = await this.dismissKeyboardTip(await this.snapshot(signal, false), signal, input);
|
|
1165
1215
|
if (!keyboardShowing(raw))
|
|
1166
1216
|
return;
|
|
1167
|
-
const
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1217
|
+
for (const axis of ['horizontal', 'vertical']) {
|
|
1218
|
+
const projected = projectSnapshot(raw.nodes ?? [], { mintId: () => 'keyboard', viewport: raw.viewport });
|
|
1219
|
+
const viewport = await this.viewportFor(projected, signal);
|
|
1220
|
+
const from = keyboardDragStart(projected, viewport, keyboardTop(raw, viewport));
|
|
1221
|
+
if (from === undefined) {
|
|
1222
|
+
throw new EngineError('UNSUPPORTED_CAPABILITY', `keyboard dismissal failed: the keyboard has no dismiss key, and every point above it (in a scroll view, when the screen has one) is on a control, so a drag would press one. ${KEYBOARD_ALTERNATIVES}`, { retryable: false });
|
|
1223
|
+
}
|
|
1224
|
+
const to = axis === 'horizontal'
|
|
1225
|
+
? { x: from.x - viewport.width * KEYBOARD_DRAG_RATIO, y: from.y }
|
|
1226
|
+
: { x: from.x, y: from.y - viewport.height * KEYBOARD_DRAG_RATIO };
|
|
1227
|
+
await this.command('keyboard.dismiss', (client) => sendInput(input, () => client.interactions.swipe({ from, to })), signal);
|
|
1176
1228
|
raw = await this.snapshot(signal, false);
|
|
1177
1229
|
if (!keyboardShowing(raw))
|
|
1178
1230
|
return;
|
|
1179
1231
|
}
|
|
1180
|
-
throw new EngineError('UNSUPPORTED_CAPABILITY',
|
|
1232
|
+
throw new EngineError('UNSUPPORTED_CAPABILITY', `keyboard dismissal failed: the keyboard has no dismiss key and stayed up through a horizontal and a vertical drag. ${KEYBOARD_ALTERNATIVES}`, { retryable: false });
|
|
1181
1233
|
}
|
|
1182
1234
|
/**
|
|
1183
1235
|
* The simulator shows a "Speed up your typing" tip over the keyboard the
|
|
@@ -1294,7 +1346,7 @@ export class AgentDeviceSurface {
|
|
|
1294
1346
|
};
|
|
1295
1347
|
const projected = this.project(await this.snapshotOrEmpty(operation, false));
|
|
1296
1348
|
const data = await this.rawScreenshot(signal);
|
|
1297
|
-
const masked = redactSecure(data, projected,
|
|
1349
|
+
const masked = redactSecure(data, projected, this.knownViewport);
|
|
1298
1350
|
if (masked === undefined) {
|
|
1299
1351
|
throw new EngineError('ENGINE_FAILURE', 'a secure field on screen could not be masked; screenshot withheld', {
|
|
1300
1352
|
retryable: false,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@e2e-dev/mobile",
|
|
3
|
-
"version": "0.12.4
|
|
3
|
+
"version": "0.12.4",
|
|
4
4
|
"description": "Mobile engine (iOS and Android) for e2e agentic end-to-end testing, built on agent-device",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"e2e",
|
|
@@ -51,13 +51,13 @@
|
|
|
51
51
|
"NOTICE"
|
|
52
52
|
],
|
|
53
53
|
"dependencies": {
|
|
54
|
-
"agent-device": "0.21.
|
|
54
|
+
"agent-device": "0.21.29",
|
|
55
55
|
"pngjs": "7.0.0",
|
|
56
56
|
"zod": "4.6.1"
|
|
57
57
|
},
|
|
58
58
|
"peerDependencies": {
|
|
59
59
|
"ai": "^7.0.0",
|
|
60
|
-
"e2e": "0.
|
|
60
|
+
"e2e": ">=0.15.0 <1"
|
|
61
61
|
},
|
|
62
62
|
"peerDependenciesMeta": {
|
|
63
63
|
"ai": {
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"@types/node": "26.6.2",
|
|
69
69
|
"@types/pngjs": "6.0.5",
|
|
70
70
|
"ai": "7.0.107",
|
|
71
|
-
"e2e": "0.21.3
|
|
71
|
+
"e2e": "0.21.3",
|
|
72
72
|
"typescript": "7.0.2",
|
|
73
73
|
"vitest": "5.0.1"
|
|
74
74
|
},
|