device-devtools-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +327 -0
- package/LICENSE +21 -0
- package/README.md +491 -0
- package/VERSION +1 -0
- package/bin/device-devtools-mcp.js +106 -0
- package/bin/devicetools +315 -0
- package/config.example.json +63 -0
- package/integrations/agent-pointer.sh +48 -0
- package/integrations/claude/SKILL.md +8 -0
- package/integrations/cursor/devicetools.mdc +8 -0
- package/integrations/gemini/GEMINI.md +5 -0
- package/integrations/mcp/README.md +184 -0
- package/integrations/mcp/mcp.json +10 -0
- package/integrations/mcp/reference.sh +119 -0
- package/integrations/mcp/selftest.sh +158 -0
- package/integrations/mcp/server.sh +343 -0
- package/package.json +50 -0
- package/scripts/android/app.sh +241 -0
- package/scripts/android/back.sh +73 -0
- package/scripts/android/controls.sh +56 -0
- package/scripts/android/devices.sh +77 -0
- package/scripts/android/doctor.sh +253 -0
- package/scripts/android/lib.sh +699 -0
- package/scripts/android/logs.sh +289 -0
- package/scripts/android/permission.sh +119 -0
- package/scripts/android/settings.sh +63 -0
- package/scripts/android/setup.sh +97 -0
- package/scripts/android/tree.awk +166 -0
- package/scripts/android/tree.sh +127 -0
- package/scripts/android/type.sh +191 -0
- package/scripts/common/find.sh +95 -0
- package/scripts/common/key.sh +65 -0
- package/scripts/common/lib.sh +14 -0
- package/scripts/common/measure.sh +170 -0
- package/scripts/common/open.sh +131 -0
- package/scripts/common/screenshot.sh +102 -0
- package/scripts/common/scroll.sh +177 -0
- package/scripts/common/snapshot.sh +69 -0
- package/scripts/common/swipe.sh +171 -0
- package/scripts/common/tap.sh +226 -0
- package/scripts/common/wait.sh +212 -0
- package/scripts/common/waypoint.sh +141 -0
- package/scripts/dispatch.sh +21 -0
- package/scripts/flow.sh +266 -0
- package/scripts/init.sh +101 -0
- package/scripts/ios/app.sh +404 -0
- package/scripts/ios/back.sh +95 -0
- package/scripts/ios/controls.sh +68 -0
- package/scripts/ios/devices.sh +80 -0
- package/scripts/ios/doctor.sh +386 -0
- package/scripts/ios/lib.sh +864 -0
- package/scripts/ios/logs.sh +272 -0
- package/scripts/ios/permission.sh +108 -0
- package/scripts/ios/settings.sh +76 -0
- package/scripts/ios/setup.sh +175 -0
- package/scripts/ios/tree.sh +128 -0
- package/scripts/ios/type.sh +178 -0
- package/scripts/lib.sh +1032 -0
- package/scripts/links.tsv +44 -0
- package/scripts/relink.sh +121 -0
- package/scripts/run.sh +415 -0
- package/scripts/selftest.sh +1709 -0
- package/scripts/snapshot.awk +362 -0
- package/scripts/verify-npm-package.js +133 -0
- package/tests/fixtures/ios-contacts-list.expected +52 -0
- package/tests/fixtures/ios-contacts-list.rows +140 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# swipe.sh — drag from one point to another.
|
|
3
|
+
#
|
|
4
|
+
# swipe.sh <up|down|left|right> [--duration MS]
|
|
5
|
+
# swipe.sh <from> <to> [--duration MS]
|
|
6
|
+
# swipe.sh --direction <up|down|left|right> [--duration MS]
|
|
7
|
+
#
|
|
8
|
+
# Both endpoints are selectors. An element selector resolves to that element's
|
|
9
|
+
# centre, exactly as tap.sh does, and is subject to the same rule: it must match
|
|
10
|
+
# EXACTLY ONE visible element or nothing happens.
|
|
11
|
+
#
|
|
12
|
+
# swipe.sh label:Inbox xy:200,100
|
|
13
|
+
# swipe.sh xy:214,700 xy:214,200 --duration 300
|
|
14
|
+
#
|
|
15
|
+
# --direction describes where the FINGER travels. That is the opposite reading
|
|
16
|
+
# from scroll.sh, where the direction is where the content you want lives:
|
|
17
|
+
#
|
|
18
|
+
# swipe --direction up == scroll down
|
|
19
|
+
#
|
|
20
|
+
# Both exist because one is a gesture and the other is an intent, and collapsing
|
|
21
|
+
# them into a single word is how everyone spends their first hour scrolling the
|
|
22
|
+
# wrong way.
|
|
23
|
+
#
|
|
24
|
+
# --duration is the press-and-hold time before the drag begins, in milliseconds.
|
|
25
|
+
# It is not the travel time. Short values read as a flick and scroll with
|
|
26
|
+
# momentum; longer ones read as a deliberate drag and do not. The default of
|
|
27
|
+
# 200ms is a flick, which is what "swipe" usually means.
|
|
28
|
+
#
|
|
29
|
+
# --snapshot prints the screen this left behind and renumbers the uids, so the
|
|
30
|
+
# read that would have been the next call is folded into this one. In a measured
|
|
31
|
+
# session — log in, open Settings — twelve calls did the work and six of them
|
|
32
|
+
# were snapshots taken only to find out what the previous action produced.
|
|
33
|
+
#
|
|
34
|
+
# exit 0 swiped
|
|
35
|
+
# exit 1 usage error
|
|
36
|
+
# exit 3 the device driver is unreachable — run doctor.sh
|
|
37
|
+
# exit 4 an endpoint selector did not resolve to exactly one element
|
|
38
|
+
|
|
39
|
+
# SHARED VERB — one copy, reached from scripts/ios/ and scripts/android/ by
|
|
40
|
+
# symlink. The symlink's own directory decides which lib.sh the line below
|
|
41
|
+
# resolves to, which is the whole mechanism.
|
|
42
|
+
#
|
|
43
|
+
# It qualifies as shared because it touches the screen only through the driver
|
|
44
|
+
# primitives (drv_tap, drv_drag, drv_fingerprint*) and the lookup helpers every
|
|
45
|
+
# adapter implements. Nothing here may learn that WebDriverAgent or UiAutomator2
|
|
46
|
+
# exists; the moment a verb needs to, it stops being shared and moves back into
|
|
47
|
+
# the adapter that needs it.
|
|
48
|
+
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
|
|
49
|
+
|
|
50
|
+
FROM=""
|
|
51
|
+
TO=""
|
|
52
|
+
SWIPE_DIR=""
|
|
53
|
+
DURATION_MS=200
|
|
54
|
+
|
|
55
|
+
while [ $# -gt 0 ]; do
|
|
56
|
+
case "$1" in
|
|
57
|
+
--duration)
|
|
58
|
+
[ $# -ge 2 ] || die "--duration needs a number of milliseconds" 1
|
|
59
|
+
case "$2" in (*[!0-9]*|"") die "--duration needs a non-negative integer, got: $2" 1 ;; esac
|
|
60
|
+
DURATION_MS="$2"; shift 2 ;;
|
|
61
|
+
--snapshot) act_snapshot_on; shift ;;
|
|
62
|
+
-h|--help) awk 'NR > 1 { if (!/^#/) exit; sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}"; exit 0 ;;
|
|
63
|
+
--direction)
|
|
64
|
+
[ $# -ge 2 ] || die "--direction needs up, down, left or right" 1
|
|
65
|
+
case "$2" in up|down|left|right) ;; *) die "--direction must be up, down, left or right, got: $2" 1 ;; esac
|
|
66
|
+
SWIPE_DIR="$2"; shift 2 ;;
|
|
67
|
+
-*) die "unknown argument: $1 (usage: swipe.sh <up|down|left|right> | swipe.sh <from> <to> [--duration MS])" 1 ;;
|
|
68
|
+
*)
|
|
69
|
+
if [ -z "$FROM" ]; then FROM="$1"
|
|
70
|
+
elif [ -z "$TO" ]; then TO="$1"
|
|
71
|
+
else die "swipe.sh takes exactly two selectors, got a third: $1" 1
|
|
72
|
+
fi
|
|
73
|
+
shift ;;
|
|
74
|
+
esac
|
|
75
|
+
done
|
|
76
|
+
|
|
77
|
+
# A LONE DIRECTION WORD IS A DIRECTION, NOT AN ENDPOINT.
|
|
78
|
+
#
|
|
79
|
+
# `scroll.sh right` has always meant the direction; `swipe.sh right` used to
|
|
80
|
+
# mean "start at the element matching 'right'" and then failed for want of a
|
|
81
|
+
# second endpoint. Two shapes for the same idea, and the MCP layer sends the
|
|
82
|
+
# positional form, so every swipe an agent asked for died on the difference.
|
|
83
|
+
#
|
|
84
|
+
# Decided after parsing rather than during it, so `swipe.sh up down` is still
|
|
85
|
+
# two endpoints rather than a direction plus a stray.
|
|
86
|
+
if [ -z "$SWIPE_DIR" ] && [ -n "$FROM" ] && [ -z "$TO" ]; then
|
|
87
|
+
case "$FROM" in
|
|
88
|
+
up|down|left|right) SWIPE_DIR="$FROM"; FROM="" ;;
|
|
89
|
+
esac
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
if [ -n "$SWIPE_DIR" ]; then
|
|
93
|
+
[ -z "$FROM" ] && [ -z "$TO" ] \
|
|
94
|
+
|| die "--direction and explicit endpoints cannot be combined — use one or the other" 1
|
|
95
|
+
else
|
|
96
|
+
[ -n "$FROM" ] && [ -n "$TO" ] \
|
|
97
|
+
|| die "missing endpoint (usage: swipe.sh <from> <to> [--duration MS] | swipe.sh --direction <dir>)" 1
|
|
98
|
+
fi
|
|
99
|
+
|
|
100
|
+
load_config
|
|
101
|
+
|
|
102
|
+
SRC=""
|
|
103
|
+
|
|
104
|
+
# resolve_point <selector> <role> — sets PT_X, PT_Y and PT_DESC.
|
|
105
|
+
resolve_point() {
|
|
106
|
+
local sel="$1" role="$2" matches n all cands
|
|
107
|
+
parse_selector "$sel"
|
|
108
|
+
|
|
109
|
+
if [ "$SEL_KIND" = xy ]; then
|
|
110
|
+
parse_xy "$SEL_VALUE"
|
|
111
|
+
PT_X="$XY_X"; PT_Y="$XY_Y"; PT_DESC="point"
|
|
112
|
+
return 0
|
|
113
|
+
fi
|
|
114
|
+
|
|
115
|
+
# One tree read serves both endpoints: two reads cost ~3.8s and, worse, could
|
|
116
|
+
# resolve against two different screens.
|
|
117
|
+
[ -n "$SRC" ] || SRC="$(fetch_source)"
|
|
118
|
+
|
|
119
|
+
matches="$(match_elements "$SRC" "$SEL_KIND" "$SEL_VALUE" || true)"
|
|
120
|
+
if [ -z "$matches" ]; then
|
|
121
|
+
cands="$(candidate_elements "$SRC" "$SEL_VALUE" 5 | join_lines '; ' || true)"
|
|
122
|
+
[ -z "$cands" ] \
|
|
123
|
+
&& die "$role endpoint: nothing matches $SEL_DESC — run $(as_cmd tree)" 4
|
|
124
|
+
die "$role endpoint: nothing matches $SEL_DESC — did you mean: $cands" 4
|
|
125
|
+
fi
|
|
126
|
+
|
|
127
|
+
n="$(printf '%s\n' "$matches" | wc -l | tr -d ' ')"
|
|
128
|
+
if [ "$n" -gt 1 ]; then
|
|
129
|
+
all="$(printf '%s\n' "$matches" \
|
|
130
|
+
| awk -F'|' 'NR <= 5 { printf "[%d] %s at %s,%s,%s,%s\n", NR - 1, $5, $1, $2, $3, $4 }' | join_lines '; ')"
|
|
131
|
+
die "$role endpoint: $SEL_DESC matches $n elements, refusing to guess: $all" 4
|
|
132
|
+
fi
|
|
133
|
+
|
|
134
|
+
PT_X="$(printf '%s' "$matches" | awk -F'|' '{ print $1 + $3 / 2 }')"
|
|
135
|
+
PT_Y="$(printf '%s' "$matches" | awk -F'|' '{ print $2 + $4 / 2 }')"
|
|
136
|
+
PT_DESC="$(printf '%s' "$matches" | awk -F'|' '{ print $5 }')"
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if [ -n "$SWIPE_DIR" ]; then
|
|
140
|
+
read -r WIN_W WIN_H <<< "$(window_size)"
|
|
141
|
+
# Kept off the very edges: starting at x=0 triggers the system back gesture,
|
|
142
|
+
# and the extremes of y are the status bar and the home indicator.
|
|
143
|
+
CX=$(( WIN_W / 2 )); CY=$(( WIN_H / 2 ))
|
|
144
|
+
NEAR_Y=$(( WIN_H * 70 / 100 )); FAR_Y=$(( WIN_H * 30 / 100 ))
|
|
145
|
+
NEAR_X=$(( WIN_W * 80 / 100 )); FAR_X=$(( WIN_W * 20 / 100 ))
|
|
146
|
+
case "$SWIPE_DIR" in
|
|
147
|
+
up) FX=$CX; FY=$NEAR_Y; TX=$CX; TY=$FAR_Y ;;
|
|
148
|
+
down) FX=$CX; FY=$FAR_Y; TX=$CX; TY=$NEAR_Y ;;
|
|
149
|
+
left) FX=$NEAR_X; FY=$CY; TX=$FAR_X; TY=$CY ;;
|
|
150
|
+
right) FX=$FAR_X; FY=$CY; TX=$NEAR_X; TY=$CY ;;
|
|
151
|
+
esac
|
|
152
|
+
FROM="--direction $SWIPE_DIR"; TO=""
|
|
153
|
+
FD="point"; TD="point"
|
|
154
|
+
else
|
|
155
|
+
resolve_point "$FROM" from
|
|
156
|
+
FX="$PT_X"; FY="$PT_Y"; FD="$PT_DESC"
|
|
157
|
+
resolve_point "$TO" to
|
|
158
|
+
TX="$PT_X"; TY="$PT_Y"; TD="$PT_DESC"
|
|
159
|
+
fi
|
|
160
|
+
|
|
161
|
+
act_before
|
|
162
|
+
err="$(drv_drag "$FX" "$FY" "$TX" "$TY" "$DURATION_MS" || true)"
|
|
163
|
+
[ -z "$err" ] || die "swipe from $FX,$FY to $TX,$TY failed — $err" 3
|
|
164
|
+
|
|
165
|
+
if [ -n "$SWIPE_DIR" ]; then
|
|
166
|
+
act_report SWIPE "$SWIPE_DIR — $FX,$FY → $TX,$TY over ${DURATION_MS}ms"
|
|
167
|
+
journal_append swipe "$SWIPE_DIR" "" "" "" "" "$FX" "$FY" 0 0 "$DT_ACT_HASH"
|
|
168
|
+
else
|
|
169
|
+
printf 'OK swiped %s → %s — %s %s,%s → %s %s,%s over %sms\n' \
|
|
170
|
+
"$FROM" "$TO" "$FD" "$FX" "$FY" "$TD" "$TX" "$TY" "$DURATION_MS"
|
|
171
|
+
fi
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# tap.sh — tap exactly one thing, or refuse.
|
|
3
|
+
#
|
|
4
|
+
# tap.sh <selector> [--index N] [--count N]
|
|
5
|
+
#
|
|
6
|
+
# selector is one of:
|
|
7
|
+
# id:<identifier> accessibilityIdentifier, exact match
|
|
8
|
+
# label:<text> accessibility label, exact match
|
|
9
|
+
# text:<text> the element's value, exact match. This is how fields that
|
|
10
|
+
# carry neither an identifier nor a label are addressed at
|
|
11
|
+
# all — a password field with only a placeholder, say.
|
|
12
|
+
# kind:<Type> the element's class, the word snapshot prints first on the
|
|
13
|
+
# line: kind:SecureTextField. For a control with no
|
|
14
|
+
# identifier, no label and no value — usually with --index.
|
|
15
|
+
#
|
|
16
|
+
# EVERY SELECTOR MATCHES WHOLE. `text:OTP` does not find a label that contains
|
|
17
|
+
# OTP; `snapshot --grep OTP` does. Reading may guess, acting may not — a
|
|
18
|
+
# substring match is how a tap lands on the caption instead of the button.
|
|
19
|
+
# xy:<x>,<y> raw point, in the same coordinate space tree.sh prints
|
|
20
|
+
#
|
|
21
|
+
# --index N act on the Nth match, zero-based, in tree order
|
|
22
|
+
# --count N tap the same point N times, N up to 20
|
|
23
|
+
#
|
|
24
|
+
# WHY --count EXISTS, AND WHY IT RESOLVES ONCE.
|
|
25
|
+
#
|
|
26
|
+
# A six-digit code is six taps on one key. Six separate calls is six round trips
|
|
27
|
+
# and six identical tool calls, which some agent harnesses treat as a loop to
|
|
28
|
+
# break out of — one refused to make the sixth call at all. So the repetition
|
|
29
|
+
# happens here.
|
|
30
|
+
#
|
|
31
|
+
# The element is resolved once and the same point is tapped N times. Re-reading
|
|
32
|
+
# the tree between taps would be slower and would also be wrong: a keypad key
|
|
33
|
+
# does not move, and a uid that went stale mid-sequence should stop the sequence
|
|
34
|
+
# rather than silently retarget it.
|
|
35
|
+
#
|
|
36
|
+
# For id:, label: and text: the selector must resolve to EXACTLY ONE visible
|
|
37
|
+
# element, unless --index says which one. Zero matches, or several without an
|
|
38
|
+
# index, and this exits non-zero naming what it saw, without tapping anything.
|
|
39
|
+
#
|
|
40
|
+
# That strictness is the entire point. A wrong tap is worse than a stop: the
|
|
41
|
+
# stop is visible and recoverable, while the wrong tap leaves the caller
|
|
42
|
+
# reasoning confidently about a screen it is no longer on, and every step after
|
|
43
|
+
# it compounds the error. Guessing "probably the first one" is how that happens.
|
|
44
|
+
#
|
|
45
|
+
# --index is the way through, and it is deliberately explicit. Other tools
|
|
46
|
+
# resolve ambiguity implicitly and are usually right; when tree order changes,
|
|
47
|
+
# the implicit choice silently starts acting on something else while the test
|
|
48
|
+
# keeps passing. Written into the test, the intent survives.
|
|
49
|
+
#
|
|
50
|
+
# On success one line on stdout:
|
|
51
|
+
# OK tapped <selector> — <resolved by ...> <Type> at x,y,w,h → tapped cx,cy
|
|
52
|
+
#
|
|
53
|
+
# --snapshot prints the screen this left behind and renumbers the uids, so the
|
|
54
|
+
# read that would have been the next call is folded into this one. In a measured
|
|
55
|
+
# session — log in, open Settings — twelve calls did the work and six of them
|
|
56
|
+
# were snapshots taken only to find out what the previous action produced.
|
|
57
|
+
#
|
|
58
|
+
# exit 0 tapped
|
|
59
|
+
# exit 1 usage error
|
|
60
|
+
# exit 3 the device driver is unreachable — run doctor.sh
|
|
61
|
+
# exit 4 the selector did not resolve to exactly one element
|
|
62
|
+
|
|
63
|
+
# SHARED VERB — one copy, reached from scripts/ios/ and scripts/android/ by
|
|
64
|
+
# symlink. The symlink's own directory decides which lib.sh the line below
|
|
65
|
+
# resolves to, which is the whole mechanism.
|
|
66
|
+
#
|
|
67
|
+
# It qualifies as shared because it touches the screen only through the driver
|
|
68
|
+
# primitives (drv_tap, drv_drag, drv_fingerprint*) and the lookup helpers every
|
|
69
|
+
# adapter implements. Nothing here may learn that WebDriverAgent or UiAutomator2
|
|
70
|
+
# exists; the moment a verb needs to, it stops being shared and moves back into
|
|
71
|
+
# the adapter that needs it.
|
|
72
|
+
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
|
|
73
|
+
|
|
74
|
+
SELECTOR=""
|
|
75
|
+
INDEX=""
|
|
76
|
+
COUNT=1
|
|
77
|
+
|
|
78
|
+
while [ $# -gt 0 ]; do
|
|
79
|
+
case "$1" in
|
|
80
|
+
--count)
|
|
81
|
+
[ $# -ge 2 ] || die "--count needs a number" 1
|
|
82
|
+
case "$2" in (*[!0-9]*|""|0) die "--count needs a positive integer, got: $2" 1 ;; esac
|
|
83
|
+
# Bounded, because the documented limit has to be enforced somewhere and
|
|
84
|
+
# an unbounded count is a way to hammer a real device by typo.
|
|
85
|
+
[ "$2" -le 20 ] || die "--count is capped at 20, got: $2" 1
|
|
86
|
+
COUNT="$2"; shift 2 ;;
|
|
87
|
+
--index)
|
|
88
|
+
[ $# -ge 2 ] || die "--index needs a number" 1
|
|
89
|
+
case "$2" in (*[!0-9]*|"") die "--index needs a non-negative integer, got: $2" 1 ;; esac
|
|
90
|
+
INDEX="$2"; shift 2 ;;
|
|
91
|
+
--snapshot) act_snapshot_on; shift ;;
|
|
92
|
+
-h|--help) awk 'NR > 1 { if (!/^#/) exit; sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}"; exit 0 ;;
|
|
93
|
+
-*) die "unknown argument: $1 (usage: tap.sh <selector> [--index N] [--count N])" 1 ;;
|
|
94
|
+
*)
|
|
95
|
+
[ -z "$SELECTOR" ] || die "only one selector is supported, got '$SELECTOR' and '$1'" 1
|
|
96
|
+
SELECTOR="$1"; shift ;;
|
|
97
|
+
esac
|
|
98
|
+
done
|
|
99
|
+
|
|
100
|
+
[ -n "$SELECTOR" ] || die "missing selector (usage: tap.sh <selector> [--index N] [--count N])" 1
|
|
101
|
+
|
|
102
|
+
load_config
|
|
103
|
+
parse_selector "$SELECTOR"
|
|
104
|
+
|
|
105
|
+
if [ "$SEL_KIND" = xy ]; then
|
|
106
|
+
parse_xy "$SEL_VALUE"
|
|
107
|
+
CX="$XY_X"
|
|
108
|
+
CY="$XY_Y"
|
|
109
|
+
# A uid arrives here as a point, because that is what uid_resolve produces
|
|
110
|
+
# after proving the element is still where it was numbered. Reporting it as
|
|
111
|
+
# "resolved by xy" would describe the mechanism instead of what was asked for,
|
|
112
|
+
# and the two are not the same claim: a raw point was never checked against
|
|
113
|
+
# anything, and a uid was.
|
|
114
|
+
case "$SEL_DESC" in
|
|
115
|
+
uid\ *) RESOLVED="resolved by $SEL_DESC"; DETAIL="verified still at $CX,$CY" ;;
|
|
116
|
+
*) RESOLVED="resolved by xy"; DETAIL="point" ;;
|
|
117
|
+
esac
|
|
118
|
+
else
|
|
119
|
+
src="$(fetch_source)"
|
|
120
|
+
# What the screen says right now, and only that. There is no fingerprint to
|
|
121
|
+
# fall back on any more: a screen that has moved on is read again, which costs
|
|
122
|
+
# one cheap call, instead of being matched against a recording from some
|
|
123
|
+
# earlier day.
|
|
124
|
+
matches="$(match_or_holding "$src" "$SEL_KIND" "$SEL_VALUE" || true)"
|
|
125
|
+
resolve_state
|
|
126
|
+
|
|
127
|
+
if [ -z "$matches" ]; then
|
|
128
|
+
# AN EXACT HIT THAT IS MERELY COVERED BEATS A LIST OF NEAR MISSES.
|
|
129
|
+
#
|
|
130
|
+
# This branch used to end at "nothing on screen contains it" while
|
|
131
|
+
# `snapshot --grep` was listing the very thing, because snapshot searches
|
|
132
|
+
# the whole tree and this searches what a user could touch. Both are right;
|
|
133
|
+
# only one of them was saying so.
|
|
134
|
+
note="$(invisible_note "$src" "$SEL_KIND" "$SEL_VALUE" || true)"
|
|
135
|
+
[ -n "$note" ] && die "$SEL_DESC matches nothing that can be tapped — $note" 4
|
|
136
|
+
|
|
137
|
+
# A selector matches a value WHOLE; --grep searches inside one. Said here
|
|
138
|
+
# because the list below otherwise reads as a contradiction: this line says
|
|
139
|
+
# nothing matched and then prints things that plainly contain the words.
|
|
140
|
+
cands="$(candidate_elements "$src" "$SEL_VALUE" 5 | join_lines '; ' || true)"
|
|
141
|
+
if [ -n "$cands" ]; then
|
|
142
|
+
die "nothing matches $SEL_DESC — a selector matches whole, and these merely contain it: $cands" 4
|
|
143
|
+
fi
|
|
144
|
+
if [ "$SEL_KIND" = text ]; then
|
|
145
|
+
note="$(holding_absence "$src" "$SEL_VALUE")"
|
|
146
|
+
[ -n "$note" ] && die "no control matched '$SEL_VALUE' — $note" 4
|
|
147
|
+
fi
|
|
148
|
+
if [ "$SEL_KIND" = kind ]; then
|
|
149
|
+
die "no visible $SEL_VALUE on this screen. What is here: $(kinds_on_screen "$src")" 4
|
|
150
|
+
fi
|
|
151
|
+
die "nothing matches $SEL_DESC and nothing on screen contains it — run $(as_cmd snapshot)" 4
|
|
152
|
+
fi
|
|
153
|
+
|
|
154
|
+
n="$(printf '%s\n' "$matches" | wc -l | tr -d ' ')"
|
|
155
|
+
if [ -n "$INDEX" ]; then
|
|
156
|
+
[ "$INDEX" -lt "$n" ] \
|
|
157
|
+
|| die "--index $INDEX is out of range: $SEL_DESC matches $n element(s), so the last index is $((n - 1))" 4
|
|
158
|
+
matches="$(printf '%s\n' "$matches" | sed -n "$((INDEX + 1))p")"
|
|
159
|
+
elif [ "$n" -gt 1 ]; then
|
|
160
|
+
# The numbers are the values --index takes, printed because a list without
|
|
161
|
+
# them is an invitation to guess, and guessing is the thing this refusal
|
|
162
|
+
# exists to prevent.
|
|
163
|
+
all="$(printf '%s\n' "$matches" \
|
|
164
|
+
| awk -F'|' 'NR <= 5 { printf "[%d] %s at %s,%s,%s,%s\n", NR - 1, $5, $1, $2, $3, $4 }' | join_lines '; ')"
|
|
165
|
+
die "$SEL_DESC matches $n elements, refusing to guess: $all — use --index N, or id:/xy:" 4
|
|
166
|
+
fi
|
|
167
|
+
|
|
168
|
+
IFS='|' read -r X Y W H TYPE EID ELB EVAL <<< "$matches"
|
|
169
|
+
CX=$(( X + W / 2 ))
|
|
170
|
+
CY=$(( Y + H / 2 ))
|
|
171
|
+
# Not `"...$([ -n "$INDEX" ] && printf ...)"`. A command substitution whose
|
|
172
|
+
# last command is false makes the whole assignment fail, and under `set -e`
|
|
173
|
+
# that ends the script with status 1 and no message at all.
|
|
174
|
+
if [ -n "$INDEX" ]; then
|
|
175
|
+
RESOLVED="resolved by $SEL_KIND index $INDEX of $n"
|
|
176
|
+
elif [ "$RESOLVE_HOW" = holding ]; then
|
|
177
|
+
RESOLVED="resolved by the control holding that text"
|
|
178
|
+
else
|
|
179
|
+
RESOLVED="resolved by $SEL_KIND"
|
|
180
|
+
fi
|
|
181
|
+
DETAIL="$TYPE at $X,$Y,$W,$H"
|
|
182
|
+
fi
|
|
183
|
+
|
|
184
|
+
# A TAP THE KEYBOARD SWALLOWS IS NOT A TAP, AND MUST NOT BE REPORTED AS ONE.
|
|
185
|
+
#
|
|
186
|
+
# Found on a real login screen: the submit button sits at the bottom, the
|
|
187
|
+
# keyboard is up because the step before typed a password, and the driver still
|
|
188
|
+
# reports the button visible and at its usual place. The tap goes to the
|
|
189
|
+
# keyboard. `tap` printed OK, the app did nothing, and the suite stayed green
|
|
190
|
+
# over an action that never happened — the worst failure available, because
|
|
191
|
+
# nothing about it looks like a failure.
|
|
192
|
+
#
|
|
193
|
+
# Refusing costs one screen's worth of nothing on iOS, where the keyboard is
|
|
194
|
+
# already in the tree, and one adb call on Android. Both are cheaper than a
|
|
195
|
+
# green run that means nothing.
|
|
196
|
+
#
|
|
197
|
+
# Tapping the keyboard itself stays allowed: a key, or the keyboard as a whole,
|
|
198
|
+
# is the thing being aimed at rather than the thing in the way.
|
|
199
|
+
if [ "$SEL_KIND" != xy ]; then
|
|
200
|
+
# Settling for the label when the control itself is merely unreachable is the
|
|
201
|
+
# one failure that leaves no trace: one match is not ambiguous, so nothing is
|
|
202
|
+
# refused, and the run goes green over an action nobody performed.
|
|
203
|
+
twin="$(unhittable_twin "$src" "$SEL_KIND" "$SEL_VALUE" "$X" "$Y" "$W" "$H" 2>/dev/null || true)"
|
|
204
|
+
[ -z "$twin" ] || die "$SEL_KIND '$SEL_VALUE' also names $twin, which is on screen but not hittable — the only match left is $DETAIL, and tapping that acts on the label instead of the control; dismiss whatever covers it, or scroll it into view" 4
|
|
205
|
+
|
|
206
|
+
kb="$(keyboard_blocks "$src" "$X" "$Y" "$W" "$H" 2>/dev/null || true)"
|
|
207
|
+
[ -z "$kb" ] || die "the keyboard covers $DETAIL — a tap at $CX,$CY would land on the keyboard at ${kb// /,} and the app would see nothing; dismiss the keyboard first, or scroll the control clear of it" 4
|
|
208
|
+
fi
|
|
209
|
+
|
|
210
|
+
act_before
|
|
211
|
+
i=1
|
|
212
|
+
while [ "$i" -le "$COUNT" ]; do
|
|
213
|
+
err="$(drv_tap "$CX" "$CY" || true)"
|
|
214
|
+
[ -z "$err" ] || die "tap $i of $COUNT at $CX,$CY failed — $err" 3
|
|
215
|
+
i=$((i + 1))
|
|
216
|
+
done
|
|
217
|
+
|
|
218
|
+
if [ "$COUNT" -eq 1 ]; then
|
|
219
|
+
act_report TAP "$SELECTOR — $RESOLVED, $DETAIL → tapped $CX,$CY"
|
|
220
|
+
else
|
|
221
|
+
act_report TAP "$SELECTOR — $RESOLVED, $DETAIL → tapped $CX,$CY ${COUNT} times"
|
|
222
|
+
fi
|
|
223
|
+
# X..EVAL are set only on the resolving path; a raw point has no element behind
|
|
224
|
+
# it, and the journal records that honestly rather than inventing one.
|
|
225
|
+
journal_append tap "$SEL_KIND:$SEL_VALUE" "${TYPE-}" "${EID-}" "${ELB-}" "${EVAL-}" \
|
|
226
|
+
"${X-0}" "${Y-0}" "${W-0}" "${H-0}" "$DT_ACT_HASH"
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# wait.sh — block until the screen satisfies a condition.
|
|
3
|
+
#
|
|
4
|
+
# wait.sh <selector> [--gone] [--timeout N]
|
|
5
|
+
#
|
|
6
|
+
# selector is id:<identifier>, label:<text>, text:<value>, kind:<Type>, or a
|
|
7
|
+
# plain string. EVERY ONE OF THEM MATCHES WHOLE — `text:OTP` does not find a
|
|
8
|
+
# label that merely contains OTP. `snapshot --grep` is the one that searches
|
|
9
|
+
# inside, because reading may guess and acting may not.
|
|
10
|
+
#
|
|
11
|
+
# screen:<name> wait until the SCREEN line reads exactly that name, or
|
|
12
|
+
# screen:#<hash> exactly that hash. This is how a flow says where it expects
|
|
13
|
+
# to be starting from, which is the assertion that stops it
|
|
14
|
+
# tapping into a screen it was not written for.
|
|
15
|
+
#
|
|
16
|
+
# (default) wait until the selector matches at least one element
|
|
17
|
+
# --gone wait until it matches none
|
|
18
|
+
# --timeout N seconds; defaults to timeouts.wait_seconds from the config
|
|
19
|
+
#
|
|
20
|
+
# This exists so that nothing in this repo ever synchronises with `sleep`. A
|
|
21
|
+
# fixed sleep is a guess about a device you cannot see: too short and the agent
|
|
22
|
+
# acts on the previous screen, too long and every step pays for the worst case.
|
|
23
|
+
# Polling a real condition is both faster and correct.
|
|
24
|
+
#
|
|
25
|
+
# One tree fetch costs about 1.9s against the device this was built on, which
|
|
26
|
+
# dominates timeouts.poll_ms entirely — the real interval is roughly
|
|
27
|
+
# 1.9s + poll_ms. Lowering poll_ms below a second buys nothing. Expect elapsed
|
|
28
|
+
# times reported here to be quantised to about two seconds.
|
|
29
|
+
#
|
|
30
|
+
# On success one line on stdout:
|
|
31
|
+
# OK <selector> — <present|gone> after <N>s (<detail>)
|
|
32
|
+
#
|
|
33
|
+
# --snapshot prints the screen this left behind and renumbers the uids, so the
|
|
34
|
+
# read that would have been the next call is folded into this one. In a measured
|
|
35
|
+
# session — log in, open Settings — twelve calls did the work and six of them
|
|
36
|
+
# were snapshots taken only to find out what the previous action produced.
|
|
37
|
+
#
|
|
38
|
+
# exit 0 the condition held
|
|
39
|
+
# exit 1 usage error
|
|
40
|
+
# exit 3 the device driver is unreachable — run doctor.sh
|
|
41
|
+
# exit 4 the condition did not hold before the timeout
|
|
42
|
+
|
|
43
|
+
# SHARED VERB — one copy, reached from scripts/ios/ and scripts/android/ by
|
|
44
|
+
# symlink. The symlink's own directory decides which lib.sh the line below
|
|
45
|
+
# resolves to, which is the whole mechanism.
|
|
46
|
+
#
|
|
47
|
+
# It qualifies as shared because it touches the screen only through the driver
|
|
48
|
+
# primitives (drv_tap, drv_drag, drv_fingerprint*) and the lookup helpers every
|
|
49
|
+
# adapter implements. Nothing here may learn that WebDriverAgent or UiAutomator2
|
|
50
|
+
# exists; the moment a verb needs to, it stops being shared and moves back into
|
|
51
|
+
# the adapter that needs it.
|
|
52
|
+
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
|
|
53
|
+
|
|
54
|
+
SELECTOR=""
|
|
55
|
+
GONE=0
|
|
56
|
+
TIMEOUT=""
|
|
57
|
+
|
|
58
|
+
while [ $# -gt 0 ]; do
|
|
59
|
+
case "$1" in
|
|
60
|
+
--gone) GONE=1; shift ;;
|
|
61
|
+
--timeout)
|
|
62
|
+
[ $# -ge 2 ] || die "--timeout needs a number of seconds" 1
|
|
63
|
+
case "$2" in (*[!0-9]*|"") die "--timeout needs a positive integer, got: $2" 1 ;; esac
|
|
64
|
+
[ "$2" -gt 0 ] || die "--timeout must be greater than zero" 1
|
|
65
|
+
TIMEOUT="$2"; shift 2 ;;
|
|
66
|
+
--snapshot) act_snapshot_on; shift ;;
|
|
67
|
+
-h|--help) awk 'NR > 1 { if (!/^#/) exit; sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}"; exit 0 ;;
|
|
68
|
+
-*) die "unknown argument: $1 (usage: wait.sh <selector> [--gone] [--timeout N])" 1 ;;
|
|
69
|
+
*)
|
|
70
|
+
[ -z "$SELECTOR" ] || die "only one selector is supported, got '$SELECTOR' and '$1'" 1
|
|
71
|
+
SELECTOR="$1"; shift ;;
|
|
72
|
+
esac
|
|
73
|
+
done
|
|
74
|
+
|
|
75
|
+
[ -n "$SELECTOR" ] || die "missing selector (usage: wait.sh <selector> [--gone] [--timeout N])" 1
|
|
76
|
+
|
|
77
|
+
load_config
|
|
78
|
+
|
|
79
|
+
# screen:<name> — WAIT FOR A PLACE, NOT FOR A CONTROL.
|
|
80
|
+
#
|
|
81
|
+
# Handled before parse_selector, and deliberately not added to the selector
|
|
82
|
+
# grammar: a screen is not an element, so `tap screen:Home` has no meaning and
|
|
83
|
+
# every verb that took a selector would need a guard against it. One verb can
|
|
84
|
+
# ask this question, so one verb knows the spelling.
|
|
85
|
+
#
|
|
86
|
+
# The name is the one snapshot prints on its SCREEN line, matched exactly —
|
|
87
|
+
# a substring would make `screen:Home` pass on `HomeOnboarding`, and an assertion
|
|
88
|
+
# that passes on the wrong screen is worse than none. `screen:#6360` matches the
|
|
89
|
+
# hash instead, which is what to use when the app publishes no useful title.
|
|
90
|
+
#
|
|
91
|
+
# This is what makes a flow able to check where it is starting from. Logging in
|
|
92
|
+
# has three different paths depending on whether the device is activated, and a
|
|
93
|
+
# flow that taps into the wrong one produces a session's worth of confident
|
|
94
|
+
# nonsense.
|
|
95
|
+
case "$SELECTOR" in
|
|
96
|
+
screen:*)
|
|
97
|
+
WANT="${SELECTOR#screen:}"
|
|
98
|
+
[ -n "$WANT" ] || die "screen: needs a name or a #hash — got '$SELECTOR'" 1
|
|
99
|
+
[ "$GONE" -eq 0 ] || die "--gone makes no sense for a screen: you are always on one" 1
|
|
100
|
+
[ -n "$TIMEOUT" ] || TIMEOUT="$(cfg '.timeouts.wait_seconds' '15')"
|
|
101
|
+
POLL_MS="$(cfg '.timeouts.poll_ms' '400')"
|
|
102
|
+
poll_s="$(awk -v ms="$POLL_MS" 'BEGIN { printf "%.3f", ms / 1000 }')"
|
|
103
|
+
SECONDS=0
|
|
104
|
+
while :; do
|
|
105
|
+
split_state "$(screen_state || true)"
|
|
106
|
+
case "$WANT" in
|
|
107
|
+
\#*) [ "#$ST_HASH" = "$WANT" ] && { printf 'OK %s — arrived after %ss ("%s")\n' "$SELECTOR" "$SECONDS" "$ST_NAME"; act_snapshot_now; exit 0; } ;;
|
|
108
|
+
*) [ "$ST_NAME" = "$WANT" ] && { printf 'OK %s — arrived after %ss (#%s)\n' "$SELECTOR" "$SECONDS" "$ST_HASH"; act_snapshot_now; exit 0; } ;;
|
|
109
|
+
esac
|
|
110
|
+
[ "$SECONDS" -lt "$TIMEOUT" ] || break
|
|
111
|
+
sleep "$poll_s"
|
|
112
|
+
done
|
|
113
|
+
if [ -z "$ST_HASH" ]; then
|
|
114
|
+
die "timeout after ${TIMEOUT}s: the screen could not be read at all — run $(as_cmd doctor)" 3
|
|
115
|
+
fi
|
|
116
|
+
die "timeout after ${TIMEOUT}s: this is \"$ST_NAME\" #$ST_HASH, not $SELECTOR" 4
|
|
117
|
+
;;
|
|
118
|
+
esac
|
|
119
|
+
|
|
120
|
+
parse_selector "$SELECTOR"
|
|
121
|
+
|
|
122
|
+
# Waiting on a coordinate cannot mean anything: a point is always "there".
|
|
123
|
+
# Failing here is better than looping for the full timeout and reporting a
|
|
124
|
+
# condition that was never checkable.
|
|
125
|
+
[ "$SEL_KIND" != xy ] \
|
|
126
|
+
|| die "xy selectors cannot be waited on — use id: or label:, or wait for an element near that point" 1
|
|
127
|
+
|
|
128
|
+
[ -n "$TIMEOUT" ] || TIMEOUT="$(cfg '.timeouts.wait_seconds' '15')"
|
|
129
|
+
POLL_MS="$(cfg '.timeouts.poll_ms' '400')"
|
|
130
|
+
poll_s="$(awk -v ms="$POLL_MS" 'BEGIN { printf "%.3f", ms / 1000 }')"
|
|
131
|
+
|
|
132
|
+
SECONDS=0
|
|
133
|
+
last_count=0
|
|
134
|
+
while :; do
|
|
135
|
+
src="$(fetch_source)"
|
|
136
|
+
# The containing-control rule belongs in the loop, not after the timeout: a
|
|
137
|
+
# caption the driver calls invisible is on screen from the first poll, and
|
|
138
|
+
# making the step sit out the whole timeout before finding it would turn a
|
|
139
|
+
# correct answer into a slow one. The fingerprint stays after the timeout,
|
|
140
|
+
# where it cannot bind to something that merely resembles the target while the
|
|
141
|
+
# screen is still building.
|
|
142
|
+
matches="$(match_or_holding "$src" "$SEL_KIND" "$SEL_VALUE" || true)"
|
|
143
|
+
|
|
144
|
+
# THERE USED TO BE AN EARLY EXIT HERE, AND IT NEVER RAN.
|
|
145
|
+
#
|
|
146
|
+
# It was guarded by `[ -n "$(baseline_file)" ]` — a function belonging to the
|
|
147
|
+
# test layer that was deleted, so every poll ran `baseline_file` as a command,
|
|
148
|
+
# got "command not found" on stderr, and read the condition as false. Eight
|
|
149
|
+
# lines of that per wait, on a phone, for months, while the block they were
|
|
150
|
+
# guarding did nothing at all.
|
|
151
|
+
#
|
|
152
|
+
# Not resurrected. What it did was give up early once the screen had stopped
|
|
153
|
+
# changing, on the theory that whatever was going to appear had appeared —
|
|
154
|
+
# which is false for anything arriving over the network, and this app waits on
|
|
155
|
+
# a network. The timeout is the honest deadline, and the caller sets it.
|
|
156
|
+
if [ -z "$matches" ]; then last_count=0; else last_count="$(printf '%s\n' "$matches" | wc -l | tr -d ' ')"; fi
|
|
157
|
+
|
|
158
|
+
if [ "$GONE" -eq 1 ]; then
|
|
159
|
+
if [ "$last_count" -eq 0 ]; then
|
|
160
|
+
printf 'OK %s — gone after %ss\n' "$SELECTOR" "$SECONDS"
|
|
161
|
+
act_snapshot_now
|
|
162
|
+
exit 0
|
|
163
|
+
fi
|
|
164
|
+
elif [ "$last_count" -gt 0 ]; then
|
|
165
|
+
detail="$(printf '%s\n' "$matches" | awk -F'|' 'NR == 1 { printf "%s at %s,%s,%s,%s", $5, $1, $2, $3, $4 }')"
|
|
166
|
+
if [ "$last_count" -gt 1 ]; then detail="$last_count matches, first: $detail"; fi
|
|
167
|
+
resolve_state
|
|
168
|
+
if [ "$RESOLVE_HOW" = holding ]; then
|
|
169
|
+
detail="$detail, the control holding that text"
|
|
170
|
+
fi
|
|
171
|
+
printf 'OK %s — present after %ss (%s)\n' "$SELECTOR" "$SECONDS" "$detail"
|
|
172
|
+
act_snapshot_now
|
|
173
|
+
exit 0
|
|
174
|
+
fi
|
|
175
|
+
|
|
176
|
+
[ "$SECONDS" -lt "$TIMEOUT" ] || break
|
|
177
|
+
sleep "$poll_s"
|
|
178
|
+
done
|
|
179
|
+
|
|
180
|
+
# Say what was actually on screen. An agent that only learns "it timed out"
|
|
181
|
+
# has to spend a screenshot to find out why; one that learns the nearest
|
|
182
|
+
# candidates can usually recover on the spot.
|
|
183
|
+
if [ "$GONE" -eq 1 ]; then
|
|
184
|
+
die "timeout after ${TIMEOUT}s: $SELECTOR still present ($last_count match(es))" 4
|
|
185
|
+
fi
|
|
186
|
+
|
|
187
|
+
note="$(invisible_note "$src" "$SEL_KIND" "$SEL_VALUE" || true)"
|
|
188
|
+
[ -n "$note" ] && die "timeout after ${TIMEOUT}s: $SELECTOR never became actionable — $note" 4
|
|
189
|
+
|
|
190
|
+
if [ "$SEL_KIND" = text ]; then
|
|
191
|
+
note="$(holding_absence "$src" "$SEL_VALUE")"
|
|
192
|
+
[ -n "$note" ] && die "timeout after ${TIMEOUT}s: no control matched '$SEL_VALUE' — $note" 4
|
|
193
|
+
fi
|
|
194
|
+
# "NEAR MATCHES" AFTER "NEVER APPEARED" READS LIKE A CONTRADICTION, AND IT WAS
|
|
195
|
+
# ONE — REPORTED AS SUCH.
|
|
196
|
+
#
|
|
197
|
+
# `wait text:OTP` timed out and then listed a StaticText whose value contains
|
|
198
|
+
# OTP. Both lines are true and the pair is unreadable, because the rule joining
|
|
199
|
+
# them was never stated: A SELECTOR MATCHES A VALUE WHOLE. --grep searches
|
|
200
|
+
# inside one, which is why reading finds things acting cannot.
|
|
201
|
+
#
|
|
202
|
+
# That difference is deliberate and stays. Substring matching is what makes a
|
|
203
|
+
# tap land on the label instead of the button — the failure this tool is
|
|
204
|
+
# arranged around. So the rule is said out loud instead.
|
|
205
|
+
cands="$(candidate_elements "$src" "$SEL_VALUE" 3 | join_lines '; ' || true)"
|
|
206
|
+
if [ -n "$cands" ]; then
|
|
207
|
+
die "timeout after ${TIMEOUT}s: nothing has the whole value '$SEL_VALUE' — a selector matches a value whole, while --grep searches inside one. '$SEL_VALUE' is inside: $cands. Name one of those in full, or use id:/kind:" 4
|
|
208
|
+
fi
|
|
209
|
+
if [ "$SEL_KIND" = kind ]; then
|
|
210
|
+
die "no visible $SEL_VALUE on this screen. What is here: $(kinds_on_screen "$src")" 4
|
|
211
|
+
fi
|
|
212
|
+
die "timeout after ${TIMEOUT}s: $SELECTOR never appeared — no element contains '$SEL_VALUE'" 4
|