mcp-mobile-interaction 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -7
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/tools/double-tap.js +14 -6
- package/dist/tools/double-tap.js.map +1 -1
- package/dist/tools/get-screen-state.js +2 -2
- package/dist/tools/get-screen-state.js.map +1 -1
- package/dist/tools/get-ui-tree.js +39 -4
- package/dist/tools/get-ui-tree.js.map +1 -1
- package/dist/tools/long-press.js +14 -6
- package/dist/tools/long-press.js.map +1 -1
- package/dist/tools/screenshot.js +2 -2
- package/dist/tools/screenshot.js.map +1 -1
- package/dist/tools/swipe.js +16 -13
- package/dist/tools/swipe.js.map +1 -1
- package/dist/tools/tap-element.js +80 -30
- package/dist/tools/tap-element.js.map +1 -1
- package/dist/tools/tap.js +14 -6
- package/dist/tools/tap.js.map +1 -1
- package/dist/tools/wait-for-element-gone.d.ts +2 -0
- package/dist/tools/wait-for-element-gone.js +96 -0
- package/dist/tools/wait-for-element-gone.js.map +1 -0
- package/dist/tools/wait-for-element.js +15 -15
- package/dist/tools/wait-for-element.js.map +1 -1
- package/dist/utils/element-matcher.d.ts +18 -0
- package/dist/utils/element-matcher.js +47 -0
- package/dist/utils/element-matcher.js.map +1 -0
- package/dist/utils/format-response.d.ts +3 -0
- package/dist/utils/format-response.js +1 -1
- package/dist/utils/format-response.js.map +1 -1
- package/dist/utils/image.d.ts +6 -2
- package/dist/utils/image.js +3 -0
- package/dist/utils/image.js.map +1 -1
- package/package.json +6 -2
package/README.md
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# mcp-mobile-interaction
|
|
2
2
|
|
|
3
|
+
[](https://github.com/pablonortiz/mcp-mobile-interaction/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.npmjs.com/package/mcp-mobile-interaction)
|
|
5
|
+
[](https://www.npmjs.com/package/mcp-mobile-interaction)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
|
|
3
8
|
An MCP (Model Context Protocol) server that lets Claude interact with Android and iOS devices/emulators. Take screenshots, tap, swipe, type, inspect UI elements, and more — no Appium required.
|
|
4
9
|
|
|
5
10
|
## Prerequisites
|
|
@@ -65,7 +70,7 @@ All tools accept a `platform` parameter (`"android"` or `"ios"`) and an optional
|
|
|
65
70
|
|------|-------------|
|
|
66
71
|
| `list_devices` | List connected devices and emulators/simulators |
|
|
67
72
|
| `screenshot` | Capture a screenshot (returns base64 JPEG) |
|
|
68
|
-
| `get_ui_tree` | Get a flat list of UI elements with
|
|
73
|
+
| `get_ui_tree` | Get a flat list of UI elements with optional filters (`only_clickable`, `only_with_text`, `type_filter`, `resource_id_contains`) |
|
|
69
74
|
| `get_screen_info` | Get screen dimensions, density, and orientation |
|
|
70
75
|
| `get_screen_state` | Get UI tree + screenshot in a single call (saves a round-trip) |
|
|
71
76
|
|
|
@@ -73,23 +78,48 @@ All tools accept a `platform` parameter (`"android"` or `"ios"`) and an optional
|
|
|
73
78
|
|
|
74
79
|
| Tool | Description |
|
|
75
80
|
|------|-------------|
|
|
76
|
-
| `tap` | Tap at (x, y) coordinates |
|
|
77
|
-
| `double_tap` | Double-tap at (x, y) coordinates |
|
|
81
|
+
| `tap` | Tap at (x, y) coordinates (native resolution) |
|
|
82
|
+
| `double_tap` | Double-tap at (x, y) coordinates (native resolution) |
|
|
78
83
|
| `long_press` | Long-press at (x, y) with configurable duration |
|
|
79
84
|
| `swipe` | Swipe between coordinates or by direction (up/down/left/right) |
|
|
80
85
|
| `type_text` | Type text into the focused input field |
|
|
81
86
|
| `press_key` | Press a key (home, back, enter, delete, volume_up, volume_down, power, tab, recent_apps) |
|
|
82
87
|
| `launch_app` | Launch an app by package name / bundle ID |
|
|
83
88
|
| `open_url` | Open a URL or deep link |
|
|
84
|
-
| `tap_element` | Find
|
|
89
|
+
| `tap_element` | Find element by text, resource_id, or type and tap it. Supports `scroll_to_find` and `wait_for` |
|
|
85
90
|
|
|
86
91
|
### Waiting Tools
|
|
87
92
|
|
|
88
93
|
| Tool | Description |
|
|
89
94
|
|------|-------------|
|
|
90
|
-
| `wait_for_element` | Poll until an element matching text/type criteria appears on screen |
|
|
95
|
+
| `wait_for_element` | Poll until an element matching text/type/resource_id criteria appears on screen |
|
|
96
|
+
| `wait_for_element_gone` | Poll until a matching element disappears (loading spinners, skeletons, dialogs) |
|
|
91
97
|
| `wait_for_stable` | Poll until the screen stops changing (two consecutive UI snapshots match) |
|
|
92
98
|
|
|
99
|
+
## Coordinate System
|
|
100
|
+
|
|
101
|
+
Screenshots are scaled down by default (`scale=0.5`) to save bandwidth, while `get_ui_tree` and all coordinate-based tools (`tap`, `double_tap`, `long_press`, `swipe`) work in **native device resolution**.
|
|
102
|
+
|
|
103
|
+
Every screenshot response includes the native dimensions and scale factor to make this explicit:
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
Screenshot captured (540x1140, scale=0.5 of native 1080x2280).
|
|
107
|
+
Coordinate tools expect native resolution — multiply screenshot pixel
|
|
108
|
+
positions by 2 to convert, or pass screenshot_scale=0.5.
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
**Two ways to handle this:**
|
|
112
|
+
|
|
113
|
+
1. **Manual conversion** — multiply the position you see in the screenshot by `1/scale` (e.g. `×2` for `scale=0.5`)
|
|
114
|
+
2. **Automatic conversion** — pass `screenshot_scale` to coordinate tools and they convert for you:
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
tap(x=270, y=570, screenshot_scale=0.5)
|
|
118
|
+
→ taps at native (540, 1140)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
The `screenshot_scale` parameter is available on `tap`, `double_tap`, `long_press`, and `swipe`.
|
|
122
|
+
|
|
93
123
|
## Observe Mode
|
|
94
124
|
|
|
95
125
|
All 8 action tools (`tap`, `double_tap`, `long_press`, `swipe`, `type_text`, `press_key`, `launch_app`, `open_url`) support optional **observe** parameters that capture the screen state after the action completes — returning the result in a single round-trip instead of two:
|
|
@@ -130,7 +160,23 @@ Claude will call `screenshot` with `platform: "android"` and display the image.
|
|
|
130
160
|
"Open Settings on my iOS simulator, then scroll down and tap General"
|
|
131
161
|
```
|
|
132
162
|
|
|
133
|
-
Claude will use `launch_app
|
|
163
|
+
Claude will use `launch_app` and `tap_element` with `scroll_to_find: true` to navigate.
|
|
164
|
+
|
|
165
|
+
### Tap elements without visible text
|
|
166
|
+
|
|
167
|
+
```
|
|
168
|
+
"Tap the start session button"
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Claude will use `tap_element` with `resource_id: "start-session-button"` to find icon buttons by their resource ID.
|
|
172
|
+
|
|
173
|
+
### Wait for loading to finish
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
"Tap 'Picking Flow', wait for the loading to finish, then tap the first session"
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
Claude will use `tap_element`, then `wait_for_element_gone` with the loading indicator's resource_id, then proceed.
|
|
134
180
|
|
|
135
181
|
### Run a test flow efficiently
|
|
136
182
|
|
|
@@ -146,7 +192,7 @@ Claude will use `tap_element` with `observe_stabilize: true` and `wait_for_eleme
|
|
|
146
192
|
"What buttons are visible on the screen?"
|
|
147
193
|
```
|
|
148
194
|
|
|
149
|
-
Claude will use `get_ui_tree` to list
|
|
195
|
+
Claude will use `get_ui_tree` with `only_clickable: true` to list only interactive elements.
|
|
150
196
|
|
|
151
197
|
## How It Works
|
|
152
198
|
|
package/dist/index.js
CHANGED
|
@@ -15,13 +15,13 @@ import { registerLaunchAppTool } from "./tools/launch-app.js";
|
|
|
15
15
|
import { registerOpenUrlTool } from "./tools/open-url.js";
|
|
16
16
|
import { registerWaitForElementTool } from "./tools/wait-for-element.js";
|
|
17
17
|
import { registerWaitForStableTool } from "./tools/wait-for-stable.js";
|
|
18
|
+
import { registerWaitForElementGoneTool } from "./tools/wait-for-element-gone.js";
|
|
18
19
|
import { registerTapElementTool } from "./tools/tap-element.js";
|
|
19
20
|
import { registerGetScreenStateTool } from "./tools/get-screen-state.js";
|
|
20
21
|
const server = new McpServer({
|
|
21
22
|
name: "mcp-mobile-interaction",
|
|
22
|
-
version: "1.
|
|
23
|
+
version: "1.2.0",
|
|
23
24
|
});
|
|
24
|
-
// Register all 16 tools
|
|
25
25
|
registerListDevicesTool(server);
|
|
26
26
|
registerScreenshotTool(server);
|
|
27
27
|
registerTapTool(server);
|
|
@@ -36,6 +36,7 @@ registerLaunchAppTool(server);
|
|
|
36
36
|
registerOpenUrlTool(server);
|
|
37
37
|
registerWaitForElementTool(server);
|
|
38
38
|
registerWaitForStableTool(server);
|
|
39
|
+
registerWaitForElementGoneTool(server);
|
|
39
40
|
registerTapElementTool(server);
|
|
40
41
|
registerGetScreenStateTool(server);
|
|
41
42
|
async function main() {
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAEzE,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;IAC3B,IAAI,EAAE,wBAAwB;IAC9B,OAAO,EAAE,OAAO;CACjB,CAAC,CAAC;AAEH,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAClF,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAEzE,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;IAC3B,IAAI,EAAE,wBAAwB;IAC9B,OAAO,EAAE,OAAO;CACjB,CAAC,CAAC;AAEH,uBAAuB,CAAC,MAAM,CAAC,CAAC;AAChC,sBAAsB,CAAC,MAAM,CAAC,CAAC;AAC/B,eAAe,CAAC,MAAM,CAAC,CAAC;AACxB,qBAAqB,CAAC,MAAM,CAAC,CAAC;AAC9B,qBAAqB,CAAC,MAAM,CAAC,CAAC;AAC9B,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAC1B,oBAAoB,CAAC,MAAM,CAAC,CAAC;AAC7B,oBAAoB,CAAC,MAAM,CAAC,CAAC;AAC7B,qBAAqB,CAAC,MAAM,CAAC,CAAC;AAC9B,yBAAyB,CAAC,MAAM,CAAC,CAAC;AAClC,qBAAqB,CAAC,MAAM,CAAC,CAAC;AAC9B,mBAAmB,CAAC,MAAM,CAAC,CAAC;AAC5B,0BAA0B,CAAC,MAAM,CAAC,CAAC;AACnC,yBAAyB,CAAC,MAAM,CAAC,CAAC;AAClC,8BAA8B,CAAC,MAAM,CAAC,CAAC;AACvC,sBAAsB,CAAC,MAAM,CAAC,CAAC;AAC/B,0BAA0B,CAAC,MAAM,CAAC,CAAC;AAEnC,KAAK,UAAU,IAAI;IACjB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,OAAO,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;AAClE,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/tools/double-tap.js
CHANGED
|
@@ -10,8 +10,14 @@ export function registerDoubleTapTool(server) {
|
|
|
10
10
|
.string()
|
|
11
11
|
.optional()
|
|
12
12
|
.describe("Device ID. Omit to use the first connected device."),
|
|
13
|
-
x: z.number().
|
|
14
|
-
y: z.number().
|
|
13
|
+
x: z.number().describe("X coordinate to double-tap (in native device resolution by default)"),
|
|
14
|
+
y: z.number().describe("Y coordinate to double-tap (in native device resolution by default)"),
|
|
15
|
+
screenshot_scale: z
|
|
16
|
+
.number()
|
|
17
|
+
.min(0.1)
|
|
18
|
+
.max(1.0)
|
|
19
|
+
.optional()
|
|
20
|
+
.describe("If coordinates come from a scaled screenshot, provide the scale factor (e.g. 0.5). Coordinates will be auto-converted to native resolution."),
|
|
15
21
|
observe: z
|
|
16
22
|
.enum(["none", "ui_tree", "screenshot", "both"])
|
|
17
23
|
.optional()
|
|
@@ -25,12 +31,14 @@ export function registerDoubleTapTool(server) {
|
|
|
25
31
|
.boolean()
|
|
26
32
|
.optional()
|
|
27
33
|
.describe("If true, wait for UI to stabilize instead of fixed delay. Default: false"),
|
|
28
|
-
}, async ({ platform, device_id, x, y, observe, observe_delay_ms, observe_stabilize }) => {
|
|
34
|
+
}, async ({ platform, device_id, x, y, screenshot_scale, observe, observe_delay_ms, observe_stabilize }) => {
|
|
35
|
+
const nativeX = screenshot_scale ? Math.round(x / screenshot_scale) : Math.round(x);
|
|
36
|
+
const nativeY = screenshot_scale ? Math.round(y / screenshot_scale) : Math.round(y);
|
|
29
37
|
if (platform === "android") {
|
|
30
|
-
await android.doubleTap(
|
|
38
|
+
await android.doubleTap(nativeX, nativeY, device_id);
|
|
31
39
|
}
|
|
32
40
|
else {
|
|
33
|
-
await ios.doubleTap(
|
|
41
|
+
await ios.doubleTap(nativeX, nativeY, device_id);
|
|
34
42
|
}
|
|
35
43
|
const observation = await performObservation({
|
|
36
44
|
mode: observe ?? "none",
|
|
@@ -40,7 +48,7 @@ export function registerDoubleTapTool(server) {
|
|
|
40
48
|
stabilize: observe_stabilize,
|
|
41
49
|
});
|
|
42
50
|
return {
|
|
43
|
-
content: buildResponseContent(`Double-tapped at (${
|
|
51
|
+
content: buildResponseContent(`Double-tapped at (${nativeX}, ${nativeY}) on ${platform} device${screenshot_scale ? ` (converted from screenshot coords ${x},${y} with scale=${screenshot_scale})` : ""}`, observation),
|
|
44
52
|
};
|
|
45
53
|
});
|
|
46
54
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"double-tap.js","sourceRoot":"","sources":["../../src/tools/double-tap.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,UAAU,qBAAqB,CAAC,MAAiB;IACrD,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,0DAA0D,EAC1D;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"double-tap.js","sourceRoot":"","sources":["../../src/tools/double-tap.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,UAAU,qBAAqB,CAAC,MAAiB;IACrD,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,0DAA0D,EAC1D;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qEAAqE,CAAC;QAC7F,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qEAAqE,CAAC;QAC7F,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,CAAC,GAAG,CAAC;aACR,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CACP,6IAA6I,CAC9I;QACH,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aAC/C,QAAQ,EAAE;aACV,QAAQ,CACP,kDAAkD,CACnD;QACH,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,2CAA2C,CAAC;QACxD,iBAAiB,EAAE,CAAC;aACjB,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CACP,0EAA0E,CAC3E;KACJ,EACD,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,gBAAgB,EAAE,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,EAAE,EAAE;QACtG,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpF,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAEpF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;YAC3C,IAAI,EAAE,OAAO,IAAI,MAAM;YACvB,QAAQ;YACR,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,gBAAgB,IAAI,GAAG;YAChC,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,oBAAoB,CAC3B,qBAAqB,OAAO,KAAK,OAAO,QAAQ,QAAQ,UAAU,gBAAgB,CAAC,CAAC,CAAC,sCAAsC,CAAC,IAAI,CAAC,eAAe,gBAAgB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAC1K,WAAW,CACZ;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -44,7 +44,7 @@ export function registerGetScreenStateTool(server) {
|
|
|
44
44
|
});
|
|
45
45
|
}
|
|
46
46
|
if (screenshotBuffer) {
|
|
47
|
-
const { base64, width, height } = await compressScreenshot(screenshotBuffer);
|
|
47
|
+
const { base64, width, height, nativeWidth, nativeHeight, scale } = await compressScreenshot(screenshotBuffer);
|
|
48
48
|
content.push({
|
|
49
49
|
type: "image",
|
|
50
50
|
data: base64,
|
|
@@ -52,7 +52,7 @@ export function registerGetScreenStateTool(server) {
|
|
|
52
52
|
});
|
|
53
53
|
content.push({
|
|
54
54
|
type: "text",
|
|
55
|
-
text: `Screenshot captured (${width}x${height})
|
|
55
|
+
text: `Screenshot captured (${width}x${height}, scale=${scale} of native ${nativeWidth}x${nativeHeight}). Coordinate tools expect native resolution — multiply screenshot pixel positions by ${Math.round(1 / scale)} to convert, or pass screenshot_scale=${scale}.`,
|
|
56
56
|
});
|
|
57
57
|
}
|
|
58
58
|
return { content };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get-screen-state.js","sourceRoot":"","sources":["../../src/tools/get-screen-state.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,MAAM,UAAU,0BAA0B,CAAC,MAAiB;IAC1D,MAAM,CAAC,IAAI,CACT,kBAAkB,EAClB,gIAAgI,EAChI;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aACvC,QAAQ,EAAE;aACV,QAAQ,CAAC,gCAAgC,CAAC;QAC7C,SAAS,EAAE,CAAC;aACT,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CAAC,kFAAkF,CAAC;KAChG,EACD,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE;QACpD,MAAM,IAAI,GAAG,OAAO,IAAI,MAAM,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,CAAC;QACvD,MAAM,cAAc,GAAG,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,MAAM,CAAC;QAEhE,sBAAsB;QACtB,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACjD,QAAQ;gBACN,CAAC,CAAC,QAAQ,KAAK,SAAS;oBACtB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;oBAC9B,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC;gBAC5B,CAAC,CAAC,SAAS;YACb,cAAc;gBACZ,CAAC,CAAC,QAAQ,KAAK,SAAS;oBACtB,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC/B,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;gBAC7B,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,OAAO,GAGT,EAAE,CAAC;QAEP,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC;YAC9D,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,YAAY,QAAQ,CAAC,MAAM,gBAAgB,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;aACrF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,kBAAkB,
|
|
1
|
+
{"version":3,"file":"get-screen-state.js","sourceRoot":"","sources":["../../src/tools/get-screen-state.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,MAAM,UAAU,0BAA0B,CAAC,MAAiB;IAC1D,MAAM,CAAC,IAAI,CACT,kBAAkB,EAClB,gIAAgI,EAChI;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aACvC,QAAQ,EAAE;aACV,QAAQ,CAAC,gCAAgC,CAAC;QAC7C,SAAS,EAAE,CAAC;aACT,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CAAC,kFAAkF,CAAC;KAChG,EACD,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE;QACpD,MAAM,IAAI,GAAG,OAAO,IAAI,MAAM,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,CAAC;QACvD,MAAM,cAAc,GAAG,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,MAAM,CAAC;QAEhE,sBAAsB;QACtB,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACjD,QAAQ;gBACN,CAAC,CAAC,QAAQ,KAAK,SAAS;oBACtB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;oBAC9B,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC;gBAC5B,CAAC,CAAC,SAAS;YACb,cAAc;gBACZ,CAAC,CAAC,QAAQ,KAAK,SAAS;oBACtB,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;oBAC/B,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC;gBAC7B,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,OAAO,GAGT,EAAE,CAAC;QAEP,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC;YAC9D,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,YAAY,QAAQ,CAAC,MAAM,gBAAgB,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;aACrF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,MAAM,kBAAkB,CAC1F,gBAAgB,CACjB,CAAC;YACF,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAgB;gBACtB,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,YAAqB;aAChC,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,wBAAwB,KAAK,IAAI,MAAM,WAAW,KAAK,cAAc,WAAW,IAAI,YAAY,yFAAyF,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,yCAAyC,KAAK,GAAG;aACtQ,CAAC,CAAC;QACL,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -2,21 +2,56 @@ import { z } from "zod";
|
|
|
2
2
|
import * as android from "../platforms/android.js";
|
|
3
3
|
import * as ios from "../platforms/ios.js";
|
|
4
4
|
export function registerGetUiTreeTool(server) {
|
|
5
|
-
server.tool("get_ui_tree", "Get a simplified flat list of UI elements on the current screen. Each element includes type, text, bounds, center coordinates (for tapping), and whether it is clickable.", {
|
|
5
|
+
server.tool("get_ui_tree", "Get a simplified flat list of UI elements on the current screen. Each element includes type, text, bounds, center coordinates (for tapping), and whether it is clickable. Supports optional filters to reduce noise.", {
|
|
6
6
|
platform: z.enum(["android", "ios"]).describe("Target platform"),
|
|
7
7
|
device_id: z
|
|
8
8
|
.string()
|
|
9
9
|
.optional()
|
|
10
10
|
.describe("Device ID. Omit to use the first connected device."),
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
only_clickable: z
|
|
12
|
+
.boolean()
|
|
13
|
+
.optional()
|
|
14
|
+
.describe("If true, only return clickable elements. Default: false"),
|
|
15
|
+
only_with_text: z
|
|
16
|
+
.boolean()
|
|
17
|
+
.optional()
|
|
18
|
+
.describe("If true, only return elements with non-empty text. Default: false"),
|
|
19
|
+
type_filter: z
|
|
20
|
+
.array(z.string())
|
|
21
|
+
.optional()
|
|
22
|
+
.describe('Only return elements whose type contains one of these strings (case-insensitive). E.g. ["Button", "EditText", "TextView"]'),
|
|
23
|
+
resource_id_contains: z
|
|
24
|
+
.string()
|
|
25
|
+
.optional()
|
|
26
|
+
.describe("Only return elements whose resource_id contains this substring (case-insensitive)"),
|
|
27
|
+
}, async ({ platform, device_id, only_clickable, only_with_text, type_filter, resource_id_contains }) => {
|
|
28
|
+
let elements = platform === "android"
|
|
13
29
|
? await android.getUiTree(device_id)
|
|
14
30
|
: await ios.getUiTree(device_id);
|
|
31
|
+
const totalCount = elements.length;
|
|
32
|
+
if (only_clickable) {
|
|
33
|
+
elements = elements.filter((el) => el.clickable);
|
|
34
|
+
}
|
|
35
|
+
if (only_with_text) {
|
|
36
|
+
elements = elements.filter((el) => el.text.trim() !== "");
|
|
37
|
+
}
|
|
38
|
+
if (type_filter && type_filter.length > 0) {
|
|
39
|
+
const lowerFilters = type_filter.map((t) => t.toLowerCase());
|
|
40
|
+
elements = elements.filter((el) => lowerFilters.some((f) => el.type.toLowerCase().includes(f)));
|
|
41
|
+
}
|
|
42
|
+
if (resource_id_contains) {
|
|
43
|
+
const lower = resource_id_contains.toLowerCase();
|
|
44
|
+
elements = elements.filter((el) => (el.resource_id ?? "").toLowerCase().includes(lower));
|
|
45
|
+
}
|
|
46
|
+
const hasFilters = only_clickable || only_with_text || type_filter || resource_id_contains;
|
|
47
|
+
const header = hasFilters
|
|
48
|
+
? `${elements.length} elements (filtered from ${totalCount} total):`
|
|
49
|
+
: `${elements.length} elements:`;
|
|
15
50
|
return {
|
|
16
51
|
content: [
|
|
17
52
|
{
|
|
18
53
|
type: "text",
|
|
19
|
-
text: JSON.stringify(elements, null, 2)
|
|
54
|
+
text: `${header}\n${JSON.stringify(elements, null, 2)}`,
|
|
20
55
|
},
|
|
21
56
|
],
|
|
22
57
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get-ui-tree.js","sourceRoot":"","sources":["../../src/tools/get-ui-tree.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAE3C,MAAM,UAAU,qBAAqB,CAAC,MAAiB;IACrD,MAAM,CAAC,IAAI,CACT,aAAa,EACb,
|
|
1
|
+
{"version":3,"file":"get-ui-tree.js","sourceRoot":"","sources":["../../src/tools/get-ui-tree.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAE3C,MAAM,UAAU,qBAAqB,CAAC,MAAiB;IACrD,MAAM,CAAC,IAAI,CACT,aAAa,EACb,sNAAsN,EACtN;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,cAAc,EAAE,CAAC;aACd,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CAAC,yDAAyD,CAAC;QACtE,cAAc,EAAE,CAAC;aACd,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CAAC,mEAAmE,CAAC;QAChF,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CAAC,2HAA2H,CAAC;QACxI,oBAAoB,EAAE,CAAC;aACpB,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,mFAAmF,CAAC;KACjG,EACD,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,oBAAoB,EAAE,EAAE,EAAE;QACnG,IAAI,QAAQ,GACV,QAAQ,KAAK,SAAS;YACpB,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;YACpC,CAAC,CAAC,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAErC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC;QAEnC,IAAI,cAAc,EAAE,CAAC;YACnB,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,cAAc,EAAE,CAAC;YACnB,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YAC7D,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAChC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAC5D,CAAC;QACJ,CAAC;QACD,IAAI,oBAAoB,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,oBAAoB,CAAC,WAAW,EAAE,CAAC;YACjD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAChC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CACrD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,cAAc,IAAI,cAAc,IAAI,WAAW,IAAI,oBAAoB,CAAC;QAC3F,MAAM,MAAM,GAAG,UAAU;YACvB,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,4BAA4B,UAAU,UAAU;YACpE,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,YAAY,CAAC;QAEnC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,GAAG,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;iBACxD;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
package/dist/tools/long-press.js
CHANGED
|
@@ -10,8 +10,14 @@ export function registerLongPressTool(server) {
|
|
|
10
10
|
.string()
|
|
11
11
|
.optional()
|
|
12
12
|
.describe("Device ID. Omit to use the first connected device."),
|
|
13
|
-
x: z.number().
|
|
14
|
-
y: z.number().
|
|
13
|
+
x: z.number().describe("X coordinate to long-press (in native device resolution by default)"),
|
|
14
|
+
y: z.number().describe("Y coordinate to long-press (in native device resolution by default)"),
|
|
15
|
+
screenshot_scale: z
|
|
16
|
+
.number()
|
|
17
|
+
.min(0.1)
|
|
18
|
+
.max(1.0)
|
|
19
|
+
.optional()
|
|
20
|
+
.describe("If coordinates come from a scaled screenshot, provide the scale factor (e.g. 0.5). Coordinates will be auto-converted to native resolution."),
|
|
15
21
|
duration_ms: z
|
|
16
22
|
.number()
|
|
17
23
|
.int()
|
|
@@ -30,13 +36,15 @@ export function registerLongPressTool(server) {
|
|
|
30
36
|
.boolean()
|
|
31
37
|
.optional()
|
|
32
38
|
.describe("If true, wait for UI to stabilize instead of fixed delay. Default: false"),
|
|
33
|
-
}, async ({ platform, device_id, x, y, duration_ms, observe, observe_delay_ms, observe_stabilize }) => {
|
|
39
|
+
}, async ({ platform, device_id, x, y, screenshot_scale, duration_ms, observe, observe_delay_ms, observe_stabilize }) => {
|
|
34
40
|
const duration = duration_ms ?? 1000;
|
|
41
|
+
const nativeX = screenshot_scale ? Math.round(x / screenshot_scale) : Math.round(x);
|
|
42
|
+
const nativeY = screenshot_scale ? Math.round(y / screenshot_scale) : Math.round(y);
|
|
35
43
|
if (platform === "android") {
|
|
36
|
-
await android.longPress(
|
|
44
|
+
await android.longPress(nativeX, nativeY, duration, device_id);
|
|
37
45
|
}
|
|
38
46
|
else {
|
|
39
|
-
await ios.longPress(
|
|
47
|
+
await ios.longPress(nativeX, nativeY, duration, device_id);
|
|
40
48
|
}
|
|
41
49
|
const observation = await performObservation({
|
|
42
50
|
mode: observe ?? "none",
|
|
@@ -46,7 +54,7 @@ export function registerLongPressTool(server) {
|
|
|
46
54
|
stabilize: observe_stabilize,
|
|
47
55
|
});
|
|
48
56
|
return {
|
|
49
|
-
content: buildResponseContent(`Long-pressed at (${
|
|
57
|
+
content: buildResponseContent(`Long-pressed at (${nativeX}, ${nativeY}) for ${duration}ms on ${platform} device${screenshot_scale ? ` (converted from screenshot coords ${x},${y} with scale=${screenshot_scale})` : ""}`, observation),
|
|
50
58
|
};
|
|
51
59
|
});
|
|
52
60
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"long-press.js","sourceRoot":"","sources":["../../src/tools/long-press.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,UAAU,qBAAqB,CAAC,MAAiB;IACrD,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,0DAA0D,EAC1D;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"long-press.js","sourceRoot":"","sources":["../../src/tools/long-press.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,UAAU,qBAAqB,CAAC,MAAiB;IACrD,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,0DAA0D,EAC1D;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qEAAqE,CAAC;QAC7F,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qEAAqE,CAAC;QAC7F,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,CAAC,GAAG,CAAC;aACR,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CACP,6IAA6I,CAC9I;QACH,WAAW,EAAE,CAAC;aACX,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,2DAA2D,CAAC;QACxE,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aAC/C,QAAQ,EAAE;aACV,QAAQ,CACP,kDAAkD,CACnD;QACH,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,2CAA2C,CAAC;QACxD,iBAAiB,EAAE,CAAC;aACjB,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CACP,0EAA0E,CAC3E;KACJ,EACD,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,EAAE,EAAE;QACnH,MAAM,QAAQ,GAAG,WAAW,IAAI,IAAI,CAAC;QACrC,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpF,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAEpF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;YAC3C,IAAI,EAAE,OAAO,IAAI,MAAM;YACvB,QAAQ;YACR,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,gBAAgB,IAAI,GAAG;YAChC,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,oBAAoB,CAC3B,oBAAoB,OAAO,KAAK,OAAO,SAAS,QAAQ,SAAS,QAAQ,UAAU,gBAAgB,CAAC,CAAC,CAAC,sCAAsC,CAAC,IAAI,CAAC,eAAe,gBAAgB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAC3L,WAAW,CACZ;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
package/dist/tools/screenshot.js
CHANGED
|
@@ -25,7 +25,7 @@ export function registerScreenshotTool(server) {
|
|
|
25
25
|
const rawBuffer = platform === "android"
|
|
26
26
|
? await android.screenshot(device_id)
|
|
27
27
|
: await ios.screenshot(device_id);
|
|
28
|
-
const { base64, width, height } = await compressScreenshot(rawBuffer, {
|
|
28
|
+
const { base64, width, height, nativeWidth, nativeHeight } = await compressScreenshot(rawBuffer, {
|
|
29
29
|
quality: quality ?? 50,
|
|
30
30
|
scale: scale ?? 0.5,
|
|
31
31
|
});
|
|
@@ -38,7 +38,7 @@ export function registerScreenshotTool(server) {
|
|
|
38
38
|
},
|
|
39
39
|
{
|
|
40
40
|
type: "text",
|
|
41
|
-
text: `Screenshot captured (${width}x${height},
|
|
41
|
+
text: `Screenshot captured (${width}x${height}, scale=${scale ?? 0.5} of native ${nativeWidth}x${nativeHeight}). Coordinate tools (tap, swipe, etc.) expect native resolution — multiply screenshot pixel positions by ${Math.round(1 / (scale ?? 0.5))} to convert, or pass screenshot_scale=${scale ?? 0.5}.`,
|
|
42
42
|
},
|
|
43
43
|
],
|
|
44
44
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,MAAM,UAAU,sBAAsB,CAAC,MAAiB;IACtD,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,0GAA0G,EAC1G;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,OAAO,EAAE,CAAC;aACP,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,mCAAmC,CAAC;QAChD,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,GAAG,CAAC,GAAG,CAAC;aACR,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,sCAAsC,CAAC;KACpD,EACD,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;QAChD,MAAM,SAAS,GACb,QAAQ,KAAK,SAAS;YACpB,CAAC,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YACrC,CAAC,CAAC,MAAM,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAEtC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,kBAAkB,CAAC,SAAS,EAAE;
|
|
1
|
+
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,MAAM,UAAU,sBAAsB,CAAC,MAAiB;IACtD,MAAM,CAAC,IAAI,CACT,YAAY,EACZ,0GAA0G,EAC1G;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,OAAO,EAAE,CAAC;aACP,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,mCAAmC,CAAC;QAChD,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,GAAG,CAAC,GAAG,CAAC;aACR,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,sCAAsC,CAAC;KACpD,EACD,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;QAChD,MAAM,SAAS,GACb,QAAQ,KAAK,SAAS;YACpB,CAAC,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC;YACrC,CAAC,CAAC,MAAM,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAEtC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,GAAG,MAAM,kBAAkB,CAAC,SAAS,EAAE;YAC/F,OAAO,EAAE,OAAO,IAAI,EAAE;YACtB,KAAK,EAAE,KAAK,IAAI,GAAG;SACpB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,OAAgB;oBACtB,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,YAAqB;iBAChC;gBACD;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,wBAAwB,KAAK,IAAI,MAAM,WAAW,KAAK,IAAI,GAAG,cAAc,WAAW,IAAI,YAAY,4GAA4G,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,yCAAyC,KAAK,IAAI,GAAG,GAAG;iBAChT;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
package/dist/tools/swipe.js
CHANGED
|
@@ -12,24 +12,26 @@ export function registerSwipeTool(server) {
|
|
|
12
12
|
.describe("Device ID. Omit to use the first connected device."),
|
|
13
13
|
start_x: z
|
|
14
14
|
.number()
|
|
15
|
-
.int()
|
|
16
15
|
.optional()
|
|
17
|
-
.describe("Start X coordinate. Required if direction is not set."),
|
|
16
|
+
.describe("Start X coordinate in native device resolution. Required if direction is not set."),
|
|
18
17
|
start_y: z
|
|
19
18
|
.number()
|
|
20
|
-
.int()
|
|
21
19
|
.optional()
|
|
22
|
-
.describe("Start Y coordinate. Required if direction is not set."),
|
|
20
|
+
.describe("Start Y coordinate in native device resolution. Required if direction is not set."),
|
|
23
21
|
end_x: z
|
|
24
22
|
.number()
|
|
25
|
-
.int()
|
|
26
23
|
.optional()
|
|
27
|
-
.describe("End X coordinate. Required if direction is not set."),
|
|
24
|
+
.describe("End X coordinate in native device resolution. Required if direction is not set."),
|
|
28
25
|
end_y: z
|
|
29
26
|
.number()
|
|
30
|
-
.int()
|
|
31
27
|
.optional()
|
|
32
|
-
.describe("End Y coordinate. Required if direction is not set."),
|
|
28
|
+
.describe("End Y coordinate in native device resolution. Required if direction is not set."),
|
|
29
|
+
screenshot_scale: z
|
|
30
|
+
.number()
|
|
31
|
+
.min(0.1)
|
|
32
|
+
.max(1.0)
|
|
33
|
+
.optional()
|
|
34
|
+
.describe("If coordinates come from a scaled screenshot, provide the scale factor (e.g. 0.5). Coordinates will be auto-converted to native resolution."),
|
|
33
35
|
direction: z
|
|
34
36
|
.enum(["up", "down", "left", "right"])
|
|
35
37
|
.optional()
|
|
@@ -52,8 +54,9 @@ export function registerSwipeTool(server) {
|
|
|
52
54
|
.boolean()
|
|
53
55
|
.optional()
|
|
54
56
|
.describe("If true, wait for UI to stabilize instead of fixed delay. Default: false"),
|
|
55
|
-
}, async ({ platform, device_id, start_x, start_y, end_x, end_y, direction, duration_ms, observe, observe_delay_ms, observe_stabilize, }) => {
|
|
57
|
+
}, async ({ platform, device_id, start_x, start_y, end_x, end_y, screenshot_scale, direction, duration_ms, observe, observe_delay_ms, observe_stabilize, }) => {
|
|
56
58
|
const duration = duration_ms ?? 300;
|
|
59
|
+
const scaleFn = (v) => screenshot_scale ? Math.round(v / screenshot_scale) : Math.round(v);
|
|
57
60
|
let sx, sy, ex, ey;
|
|
58
61
|
if (direction) {
|
|
59
62
|
// Auto-compute coordinates based on screen info
|
|
@@ -106,10 +109,10 @@ export function registerSwipeTool(server) {
|
|
|
106
109
|
isError: true,
|
|
107
110
|
};
|
|
108
111
|
}
|
|
109
|
-
sx = start_x;
|
|
110
|
-
sy = start_y;
|
|
111
|
-
ex = end_x;
|
|
112
|
-
ey = end_y;
|
|
112
|
+
sx = scaleFn(start_x);
|
|
113
|
+
sy = scaleFn(start_y);
|
|
114
|
+
ex = scaleFn(end_x);
|
|
115
|
+
ey = scaleFn(end_y);
|
|
113
116
|
}
|
|
114
117
|
if (platform === "android") {
|
|
115
118
|
await android.swipe(sx, sy, ex, ey, duration, device_id);
|
package/dist/tools/swipe.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"swipe.js","sourceRoot":"","sources":["../../src/tools/swipe.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,UAAU,iBAAiB,CAAC,MAAiB;IACjD,MAAM,CAAC,IAAI,CACT,OAAO,EACP,kIAAkI,EAClI;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,OAAO,EAAE,CAAC;aACP,MAAM,EAAE;aACR,
|
|
1
|
+
{"version":3,"file":"swipe.js","sourceRoot":"","sources":["../../src/tools/swipe.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,UAAU,iBAAiB,CAAC,MAAiB;IACjD,MAAM,CAAC,IAAI,CACT,OAAO,EACP,kIAAkI,EAClI;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,OAAO,EAAE,CAAC;aACP,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,mFAAmF,CAAC;QAChG,OAAO,EAAE,CAAC;aACP,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,mFAAmF,CAAC;QAChG,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,iFAAiF,CAAC;QAC9F,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,iFAAiF,CAAC;QAC9F,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,CAAC,GAAG,CAAC;aACR,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CACP,6IAA6I,CAC9I;QACH,SAAS,EAAE,CAAC;aACT,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;aACrC,QAAQ,EAAE;aACV,QAAQ,CACP,gGAAgG,CACjG;QACH,WAAW,EAAE,CAAC;aACX,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,qDAAqD,CAAC;QAClE,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aAC/C,QAAQ,EAAE;aACV,QAAQ,CACP,kDAAkD,CACnD;QACH,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,2CAA2C,CAAC;QACxD,iBAAiB,EAAE,CAAC;aACjB,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CACP,0EAA0E,CAC3E;KACJ,EACD,KAAK,EAAE,EACL,QAAQ,EACR,SAAS,EACT,OAAO,EACP,OAAO,EACP,KAAK,EACL,KAAK,EACL,gBAAgB,EAChB,SAAS,EACT,WAAW,EACX,OAAO,EACP,gBAAgB,EAChB,iBAAiB,GAClB,EAAE,EAAE;QACH,MAAM,QAAQ,GAAG,WAAW,IAAI,GAAG,CAAC;QACpC,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnG,IAAI,EAAU,EAAE,EAAU,EAAE,EAAU,EAAE,EAAU,CAAC;QAEnD,IAAI,SAAS,EAAE,CAAC;YACd,gDAAgD;YAChD,MAAM,UAAU,GACd,QAAQ,KAAK,SAAS;gBACpB,CAAC,CAAC,MAAM,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC;gBACxC,CAAC,CAAC,MAAM,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;YAEzC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC5C,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;YAElD,QAAQ,SAAS,EAAE,CAAC;gBAClB,KAAK,IAAI;oBACP,EAAE,GAAG,EAAE,CAAC;oBAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;oBAAC,EAAE,GAAG,EAAE,CAAC;oBAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;oBACnD,MAAM;gBACR,KAAK,MAAM;oBACT,EAAE,GAAG,EAAE,CAAC;oBAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;oBAAC,EAAE,GAAG,EAAE,CAAC;oBAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;oBACnD,MAAM;gBACR,KAAK,MAAM;oBACT,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;oBAAC,EAAE,GAAG,EAAE,CAAC;oBAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;oBAAC,EAAE,GAAG,EAAE,CAAC;oBACnD,MAAM;gBACR,KAAK,OAAO;oBACV,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;oBAAC,EAAE,GAAG,EAAE,CAAC;oBAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;oBAAC,EAAE,GAAG,EAAE,CAAC;oBACnD,MAAM;YACV,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IACE,OAAO,KAAK,SAAS;gBACrB,OAAO,KAAK,SAAS;gBACrB,KAAK,KAAK,SAAS;gBACnB,KAAK,KAAK,SAAS,EACnB,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,2FAA2F;yBAClG;qBACF;oBACD,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;YACD,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YACtB,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YACtB,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3D,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;YAC3C,IAAI,EAAE,OAAO,IAAI,MAAM;YACvB,QAAQ;YACR,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,gBAAgB,IAAI,GAAG;YAChC,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,oBAAoB,CAC3B,gBAAgB,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,QAAQ,SAAS,QAAQ,SAAS,EACvF,WAAW,CACZ;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -3,8 +3,33 @@ import * as android from "../platforms/android.js";
|
|
|
3
3
|
import * as ios from "../platforms/ios.js";
|
|
4
4
|
import { performObservation } from "../utils/observe.js";
|
|
5
5
|
import { buildResponseContent } from "../utils/format-response.js";
|
|
6
|
+
import { matchElement, describeCriteria } from "../utils/element-matcher.js";
|
|
7
|
+
function getUiTree(platform, deviceId) {
|
|
8
|
+
return platform === "android"
|
|
9
|
+
? android.getUiTree(deviceId)
|
|
10
|
+
: ios.getUiTree(deviceId);
|
|
11
|
+
}
|
|
12
|
+
function tap(platform, x, y, deviceId) {
|
|
13
|
+
return platform === "android"
|
|
14
|
+
? android.tap(x, y, deviceId)
|
|
15
|
+
: ios.tap(x, y, deviceId);
|
|
16
|
+
}
|
|
17
|
+
async function swipeDown(platform, deviceId) {
|
|
18
|
+
const screenInfo = platform === "android"
|
|
19
|
+
? await android.getScreenInfo(deviceId)
|
|
20
|
+
: await ios.getScreenInfo(deviceId);
|
|
21
|
+
const cx = Math.round(screenInfo.width / 2);
|
|
22
|
+
const cy = Math.round(screenInfo.height / 2);
|
|
23
|
+
const dist = Math.round(screenInfo.height * 0.3);
|
|
24
|
+
if (platform === "android") {
|
|
25
|
+
await android.swipe(cx, cy + dist, cx, cy - dist, 300, deviceId);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
await ios.swipe(cx, cy + dist, cx, cy - dist, 300, deviceId);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
6
31
|
export function registerTapElementTool(server) {
|
|
7
|
-
server.tool("tap_element", "Find a UI element by text and tap its center. Combines get_ui_tree + tap in one call. Optionally waits for the element to
|
|
32
|
+
server.tool("tap_element", "Find a UI element by text, resource_id, or type and tap its center. Combines get_ui_tree + tap in one call. Optionally waits for the element, or scrolls to find it.", {
|
|
8
33
|
platform: z.enum(["android", "ios"]).describe("Target platform"),
|
|
9
34
|
device_id: z
|
|
10
35
|
.string()
|
|
@@ -18,6 +43,10 @@ export function registerTapElementTool(server) {
|
|
|
18
43
|
.string()
|
|
19
44
|
.optional()
|
|
20
45
|
.describe("Tap element whose text matches exactly"),
|
|
46
|
+
resource_id: z
|
|
47
|
+
.string()
|
|
48
|
+
.optional()
|
|
49
|
+
.describe("Tap element whose resource_id contains this substring (case-insensitive). Useful for icon buttons without visible text."),
|
|
21
50
|
index: z
|
|
22
51
|
.number()
|
|
23
52
|
.int()
|
|
@@ -27,6 +56,15 @@ export function registerTapElementTool(server) {
|
|
|
27
56
|
.boolean()
|
|
28
57
|
.optional()
|
|
29
58
|
.describe("If true, poll until the element appears before tapping. Default: false"),
|
|
59
|
+
scroll_to_find: z
|
|
60
|
+
.boolean()
|
|
61
|
+
.optional()
|
|
62
|
+
.describe("If true, scroll down iteratively to find the element before tapping. Default: false"),
|
|
63
|
+
max_scrolls: z
|
|
64
|
+
.number()
|
|
65
|
+
.int()
|
|
66
|
+
.optional()
|
|
67
|
+
.describe("Maximum number of scrolls when scroll_to_find is true. Default: 5"),
|
|
30
68
|
timeout_ms: z
|
|
31
69
|
.number()
|
|
32
70
|
.int()
|
|
@@ -45,13 +83,14 @@ export function registerTapElementTool(server) {
|
|
|
45
83
|
.boolean()
|
|
46
84
|
.optional()
|
|
47
85
|
.describe("If true, wait for UI to stabilize after tap. Default: false"),
|
|
48
|
-
}, async ({ platform, device_id, text_contains, text_exact, index: matchIndex, wait_for, timeout_ms, observe, observe_delay_ms, observe_stabilize, }) => {
|
|
49
|
-
|
|
86
|
+
}, async ({ platform, device_id, text_contains, text_exact, resource_id, index: matchIndex, wait_for, scroll_to_find, max_scrolls, timeout_ms, observe, observe_delay_ms, observe_stabilize, }) => {
|
|
87
|
+
const criteria = { text_exact, text_contains, resource_id };
|
|
88
|
+
if (!text_contains && !text_exact && !resource_id) {
|
|
50
89
|
return {
|
|
51
90
|
content: [
|
|
52
91
|
{
|
|
53
92
|
type: "text",
|
|
54
|
-
text: "Error: Provide
|
|
93
|
+
text: "Error: Provide at least one of text_contains, text_exact, or resource_id to identify the element.",
|
|
55
94
|
},
|
|
56
95
|
],
|
|
57
96
|
isError: true,
|
|
@@ -59,23 +98,40 @@ export function registerTapElementTool(server) {
|
|
|
59
98
|
}
|
|
60
99
|
const targetIndex = matchIndex ?? 0;
|
|
61
100
|
const timeout = timeout_ms ?? 10_000;
|
|
62
|
-
function matchElement(el) {
|
|
63
|
-
if (text_exact !== undefined && el.text !== text_exact)
|
|
64
|
-
return false;
|
|
65
|
-
if (text_contains !== undefined &&
|
|
66
|
-
!el.text.toLowerCase().includes(text_contains.toLowerCase()))
|
|
67
|
-
return false;
|
|
68
|
-
return true;
|
|
69
|
-
}
|
|
70
101
|
let target;
|
|
71
|
-
if (
|
|
102
|
+
if (scroll_to_find) {
|
|
103
|
+
// Scroll down iteratively to find element
|
|
104
|
+
const scrollLimit = max_scrolls ?? 5;
|
|
105
|
+
for (let i = 0; i <= scrollLimit; i++) {
|
|
106
|
+
const tree = await getUiTree(platform, device_id);
|
|
107
|
+
const matches = tree.filter((el) => matchElement(el, criteria));
|
|
108
|
+
if (matches.length > targetIndex) {
|
|
109
|
+
target = matches[targetIndex];
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
if (i < scrollLimit) {
|
|
113
|
+
await swipeDown(platform, device_id);
|
|
114
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (!target) {
|
|
118
|
+
return {
|
|
119
|
+
content: [
|
|
120
|
+
{
|
|
121
|
+
type: "text",
|
|
122
|
+
text: `Element not found after ${scrollLimit} scrolls (${describeCriteria(criteria)}).`,
|
|
123
|
+
},
|
|
124
|
+
],
|
|
125
|
+
isError: true,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
else if (wait_for) {
|
|
72
130
|
// Poll until element appears
|
|
73
131
|
const start = Date.now();
|
|
74
132
|
while (Date.now() - start < timeout) {
|
|
75
|
-
const tree = platform
|
|
76
|
-
|
|
77
|
-
: await ios.getUiTree(device_id);
|
|
78
|
-
const matches = tree.filter(matchElement);
|
|
133
|
+
const tree = await getUiTree(platform, device_id);
|
|
134
|
+
const matches = tree.filter((el) => matchElement(el, criteria));
|
|
79
135
|
if (matches.length > targetIndex) {
|
|
80
136
|
target = matches[targetIndex];
|
|
81
137
|
break;
|
|
@@ -87,7 +143,7 @@ export function registerTapElementTool(server) {
|
|
|
87
143
|
content: [
|
|
88
144
|
{
|
|
89
145
|
type: "text",
|
|
90
|
-
text: `Timeout after ${timeout}ms: element not found (
|
|
146
|
+
text: `Timeout after ${timeout}ms: element not found (${describeCriteria(criteria)})`,
|
|
91
147
|
},
|
|
92
148
|
],
|
|
93
149
|
isError: true,
|
|
@@ -96,16 +152,14 @@ export function registerTapElementTool(server) {
|
|
|
96
152
|
}
|
|
97
153
|
else {
|
|
98
154
|
// Single attempt
|
|
99
|
-
const tree = platform
|
|
100
|
-
|
|
101
|
-
: await ios.getUiTree(device_id);
|
|
102
|
-
const matches = tree.filter(matchElement);
|
|
155
|
+
const tree = await getUiTree(platform, device_id);
|
|
156
|
+
const matches = tree.filter((el) => matchElement(el, criteria));
|
|
103
157
|
if (matches.length === 0) {
|
|
104
158
|
return {
|
|
105
159
|
content: [
|
|
106
160
|
{
|
|
107
161
|
type: "text",
|
|
108
|
-
text: `Element not found (
|
|
162
|
+
text: `Element not found (${describeCriteria(criteria)}). ${tree.length} elements on screen.`,
|
|
109
163
|
},
|
|
110
164
|
],
|
|
111
165
|
isError: true,
|
|
@@ -125,12 +179,7 @@ export function registerTapElementTool(server) {
|
|
|
125
179
|
target = matches[targetIndex];
|
|
126
180
|
}
|
|
127
181
|
// Tap the element's center
|
|
128
|
-
|
|
129
|
-
await android.tap(target.center_x, target.center_y, device_id);
|
|
130
|
-
}
|
|
131
|
-
else {
|
|
132
|
-
await ios.tap(target.center_x, target.center_y, device_id);
|
|
133
|
-
}
|
|
182
|
+
await tap(platform, target.center_x, target.center_y, device_id);
|
|
134
183
|
const observation = await performObservation({
|
|
135
184
|
mode: observe ?? "none",
|
|
136
185
|
platform,
|
|
@@ -138,8 +187,9 @@ export function registerTapElementTool(server) {
|
|
|
138
187
|
delayMs: observe_delay_ms ?? 500,
|
|
139
188
|
stabilize: observe_stabilize,
|
|
140
189
|
});
|
|
190
|
+
const label = target.text || target.resource_id || target.type;
|
|
141
191
|
return {
|
|
142
|
-
content: buildResponseContent(`Tapped element "${
|
|
192
|
+
content: buildResponseContent(`Tapped element "${label}" (${target.type}) at (${target.center_x}, ${target.center_y}) on ${platform} device`, observation),
|
|
143
193
|
};
|
|
144
194
|
});
|
|
145
195
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tap-element.js","sourceRoot":"","sources":["../../src/tools/tap-element.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAE3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"tap-element.js","sourceRoot":"","sources":["../../src/tools/tap-element.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAE3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAsB,MAAM,6BAA6B,CAAC;AAEjG,SAAS,SAAS,CAAC,QAAgB,EAAE,QAAiB;IACpD,OAAO,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC;QAC7B,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,GAAG,CAAC,QAAgB,EAAE,CAAS,EAAE,CAAS,EAAE,QAAiB;IACpE,OAAO,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC;QAC7B,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,QAAiB;IAC1D,MAAM,UAAU,GACd,QAAQ,KAAK,SAAS;QACpB,CAAC,CAAC,MAAM,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC;QACvC,CAAC,CAAC,MAAM,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAExC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IAEjD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;IACnE,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAiB;IACtD,MAAM,CAAC,IAAI,CACT,aAAa,EACb,sKAAsK,EACtK;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,aAAa,EAAE,CAAC;aACb,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,mEAAmE,CAAC;QAChF,UAAU,EAAE,CAAC;aACV,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,wCAAwC,CAAC;QACrD,WAAW,EAAE,CAAC;aACX,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,yHAAyH,CAAC;QACtI,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,4DAA4D,CAAC;QACzE,QAAQ,EAAE,CAAC;aACR,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CAAC,wEAAwE,CAAC;QACrF,cAAc,EAAE,CAAC;aACd,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CAAC,qFAAqF,CAAC;QAClG,WAAW,EAAE,CAAC;aACX,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,mEAAmE,CAAC;QAChF,UAAU,EAAE,CAAC;aACV,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,qDAAqD,CAAC;QAClE,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aAC/C,QAAQ,EAAE;aACV,QAAQ,CAAC,mDAAmD,CAAC;QAChE,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,2CAA2C,CAAC;QACxD,iBAAiB,EAAE,CAAC;aACjB,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CAAC,6DAA6D,CAAC;KAC3E,EACD,KAAK,EAAE,EACL,QAAQ,EACR,SAAS,EACT,aAAa,EACb,UAAU,EACV,WAAW,EACX,KAAK,EAAE,UAAU,EACjB,QAAQ,EACR,cAAc,EACd,WAAW,EACX,UAAU,EACV,OAAO,EACP,gBAAgB,EAChB,iBAAiB,GAClB,EAAE,EAAE;QACH,MAAM,QAAQ,GAAkB,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC;QAE3E,IAAI,CAAC,aAAa,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW,EAAE,CAAC;YAClD,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,mGAAmG;qBAC1G;iBACF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,UAAU,IAAI,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,UAAU,IAAI,MAAM,CAAC;QAErC,IAAI,MAA6B,CAAC;QAElC,IAAI,cAAc,EAAE,CAAC;YACnB,0CAA0C;YAC1C,MAAM,WAAW,GAAG,WAAW,IAAI,CAAC,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAClD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChE,IAAI,OAAO,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;oBACjC,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;oBAC9B,MAAM;gBACR,CAAC;gBACD,IAAI,CAAC,GAAG,WAAW,EAAE,CAAC;oBACpB,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;oBACrC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;YACH,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,2BAA2B,WAAW,aAAa,gBAAgB,CAAC,QAAQ,CAAC,IAAI;yBACxF;qBACF;oBACD,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,EAAE,CAAC;YACpB,6BAA6B;YAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE,CAAC;gBACpC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAClD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChE,IAAI,OAAO,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;oBACjC,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;oBAC9B,MAAM;gBACR,CAAC;gBACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3D,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,iBAAiB,OAAO,0BAA0B,gBAAgB,CAAC,QAAQ,CAAC,GAAG;yBACtF;qBACF;oBACD,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,iBAAiB;YACjB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,sBAAsB,gBAAgB,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,MAAM,sBAAsB;yBAC9F;qBACF;oBACD,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;gBAClC,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,QAAQ,OAAO,CAAC,MAAM,8BAA8B,WAAW,aAAa;yBACnF;qBACF;oBACD,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;QAChC,CAAC;QAED,2BAA2B;QAC3B,MAAM,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAEjE,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;YAC3C,IAAI,EAAE,OAAO,IAAI,MAAM;YACvB,QAAQ;YACR,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,gBAAgB,IAAI,GAAG;YAChC,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC;QAC/D,OAAO;YACL,OAAO,EAAE,oBAAoB,CAC3B,mBAAmB,KAAK,MAAM,MAAM,CAAC,IAAI,SAAS,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,QAAQ,QAAQ,SAAS,EAC9G,WAAW,CACZ;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
package/dist/tools/tap.js
CHANGED
|
@@ -10,8 +10,14 @@ export function registerTapTool(server) {
|
|
|
10
10
|
.string()
|
|
11
11
|
.optional()
|
|
12
12
|
.describe("Device ID. Omit to use the first connected device."),
|
|
13
|
-
x: z.number().
|
|
14
|
-
y: z.number().
|
|
13
|
+
x: z.number().describe("X coordinate to tap (in native device resolution by default)"),
|
|
14
|
+
y: z.number().describe("Y coordinate to tap (in native device resolution by default)"),
|
|
15
|
+
screenshot_scale: z
|
|
16
|
+
.number()
|
|
17
|
+
.min(0.1)
|
|
18
|
+
.max(1.0)
|
|
19
|
+
.optional()
|
|
20
|
+
.describe("If coordinates come from a scaled screenshot, provide the scale factor (e.g. 0.5). Coordinates will be auto-converted to native resolution."),
|
|
15
21
|
observe: z
|
|
16
22
|
.enum(["none", "ui_tree", "screenshot", "both"])
|
|
17
23
|
.optional()
|
|
@@ -25,12 +31,14 @@ export function registerTapTool(server) {
|
|
|
25
31
|
.boolean()
|
|
26
32
|
.optional()
|
|
27
33
|
.describe("If true, wait for UI to stabilize instead of fixed delay. Default: false"),
|
|
28
|
-
}, async ({ platform, device_id, x, y, observe, observe_delay_ms, observe_stabilize }) => {
|
|
34
|
+
}, async ({ platform, device_id, x, y, screenshot_scale, observe, observe_delay_ms, observe_stabilize }) => {
|
|
35
|
+
const nativeX = screenshot_scale ? Math.round(x / screenshot_scale) : Math.round(x);
|
|
36
|
+
const nativeY = screenshot_scale ? Math.round(y / screenshot_scale) : Math.round(y);
|
|
29
37
|
if (platform === "android") {
|
|
30
|
-
await android.tap(
|
|
38
|
+
await android.tap(nativeX, nativeY, device_id);
|
|
31
39
|
}
|
|
32
40
|
else {
|
|
33
|
-
await ios.tap(
|
|
41
|
+
await ios.tap(nativeX, nativeY, device_id);
|
|
34
42
|
}
|
|
35
43
|
const observation = await performObservation({
|
|
36
44
|
mode: observe ?? "none",
|
|
@@ -40,7 +48,7 @@ export function registerTapTool(server) {
|
|
|
40
48
|
stabilize: observe_stabilize,
|
|
41
49
|
});
|
|
42
50
|
return {
|
|
43
|
-
content: buildResponseContent(`Tapped at (${
|
|
51
|
+
content: buildResponseContent(`Tapped at (${nativeX}, ${nativeY}) on ${platform} device${screenshot_scale ? ` (converted from screenshot coords ${x},${y} with scale=${screenshot_scale})` : ""}`, observation),
|
|
44
52
|
};
|
|
45
53
|
});
|
|
46
54
|
}
|
package/dist/tools/tap.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tap.js","sourceRoot":"","sources":["../../src/tools/tap.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,MAAM,CAAC,IAAI,CACT,KAAK,EACL,mDAAmD,EACnD;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"tap.js","sourceRoot":"","sources":["../../src/tools/tap.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,UAAU,eAAe,CAAC,MAAiB;IAC/C,MAAM,CAAC,IAAI,CACT,KAAK,EACL,mDAAmD,EACnD;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8DAA8D,CAAC;QACtF,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8DAA8D,CAAC;QACtF,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,CAAC,GAAG,CAAC;aACR,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CACP,6IAA6I,CAC9I;QACH,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aAC/C,QAAQ,EAAE;aACV,QAAQ,CACP,kDAAkD,CACnD;QACH,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,2CAA2C,CAAC;QACxD,iBAAiB,EAAE,CAAC;aACjB,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CACP,0EAA0E,CAC3E;KACJ,EACD,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,gBAAgB,EAAE,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,EAAE,EAAE;QACtG,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpF,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAEpF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;YAC3C,IAAI,EAAE,OAAO,IAAI,MAAM;YACvB,QAAQ;YACR,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,gBAAgB,IAAI,GAAG;YAChC,SAAS,EAAE,iBAAiB;SAC7B,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,oBAAoB,CAC3B,cAAc,OAAO,KAAK,OAAO,QAAQ,QAAQ,UAAU,gBAAgB,CAAC,CAAC,CAAC,sCAAsC,CAAC,IAAI,CAAC,eAAe,gBAAgB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EACnK,WAAW,CACZ;SACF,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import * as android from "../platforms/android.js";
|
|
3
|
+
import * as ios from "../platforms/ios.js";
|
|
4
|
+
import { performObservation } from "../utils/observe.js";
|
|
5
|
+
import { buildResponseContent } from "../utils/format-response.js";
|
|
6
|
+
import { matchElement, hasCriteria, describeCriteria } from "../utils/element-matcher.js";
|
|
7
|
+
export function registerWaitForElementGoneTool(server) {
|
|
8
|
+
server.tool("wait_for_element_gone", "Poll the UI tree until an element matching the criteria disappears from screen. Useful for waiting until loading indicators, skeletons, or dialogs go away.", {
|
|
9
|
+
platform: z.enum(["android", "ios"]).describe("Target platform"),
|
|
10
|
+
device_id: z
|
|
11
|
+
.string()
|
|
12
|
+
.optional()
|
|
13
|
+
.describe("Device ID. Omit to use the first connected device."),
|
|
14
|
+
text_contains: z
|
|
15
|
+
.string()
|
|
16
|
+
.optional()
|
|
17
|
+
.describe("Wait for element with this text substring to disappear (case-insensitive)"),
|
|
18
|
+
text_exact: z
|
|
19
|
+
.string()
|
|
20
|
+
.optional()
|
|
21
|
+
.describe("Wait for element with this exact text to disappear"),
|
|
22
|
+
resource_id: z
|
|
23
|
+
.string()
|
|
24
|
+
.optional()
|
|
25
|
+
.describe("Wait for element with this resource_id substring to disappear (case-insensitive)"),
|
|
26
|
+
type_contains: z
|
|
27
|
+
.string()
|
|
28
|
+
.optional()
|
|
29
|
+
.describe("Filter by element type containing this substring (case-insensitive)"),
|
|
30
|
+
timeout_ms: z
|
|
31
|
+
.number()
|
|
32
|
+
.int()
|
|
33
|
+
.optional()
|
|
34
|
+
.describe("Maximum time to wait in ms. Default: 10000"),
|
|
35
|
+
poll_interval_ms: z
|
|
36
|
+
.number()
|
|
37
|
+
.int()
|
|
38
|
+
.optional()
|
|
39
|
+
.describe("Polling interval in ms. Default: 500"),
|
|
40
|
+
observe: z
|
|
41
|
+
.enum(["none", "ui_tree", "screenshot", "both"])
|
|
42
|
+
.optional()
|
|
43
|
+
.describe("Capture screen state after element disappears. Default: none"),
|
|
44
|
+
}, async ({ platform, device_id, text_contains, text_exact, resource_id, type_contains, timeout_ms, poll_interval_ms, observe, }) => {
|
|
45
|
+
const criteria = {
|
|
46
|
+
text_exact,
|
|
47
|
+
text_contains,
|
|
48
|
+
resource_id,
|
|
49
|
+
type_contains,
|
|
50
|
+
};
|
|
51
|
+
if (!hasCriteria(criteria)) {
|
|
52
|
+
return {
|
|
53
|
+
content: [
|
|
54
|
+
{
|
|
55
|
+
type: "text",
|
|
56
|
+
text: "Error: Provide at least one criterion (text_contains, text_exact, resource_id, or type_contains).",
|
|
57
|
+
},
|
|
58
|
+
],
|
|
59
|
+
isError: true,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
const timeout = timeout_ms ?? 10_000;
|
|
63
|
+
const pollInterval = poll_interval_ms ?? 500;
|
|
64
|
+
const start = Date.now();
|
|
65
|
+
while (Date.now() - start < timeout) {
|
|
66
|
+
const tree = platform === "android"
|
|
67
|
+
? await android.getUiTree(device_id)
|
|
68
|
+
: await ios.getUiTree(device_id);
|
|
69
|
+
const matches = tree.filter((el) => matchElement(el, criteria));
|
|
70
|
+
if (matches.length === 0) {
|
|
71
|
+
const elapsed = Date.now() - start;
|
|
72
|
+
const observation = await performObservation({
|
|
73
|
+
mode: observe ?? "none",
|
|
74
|
+
platform,
|
|
75
|
+
deviceId: device_id,
|
|
76
|
+
delayMs: 0,
|
|
77
|
+
stabilize: false,
|
|
78
|
+
});
|
|
79
|
+
return {
|
|
80
|
+
content: buildResponseContent(`Element gone after ${elapsed}ms (${describeCriteria(criteria)})`, observation),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
content: [
|
|
87
|
+
{
|
|
88
|
+
type: "text",
|
|
89
|
+
text: `Timeout after ${timeout}ms: element still present (${describeCriteria(criteria)}).`,
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
isError: true,
|
|
93
|
+
};
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=wait-for-element-gone.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wait-for-element-gone.js","sourceRoot":"","sources":["../../src/tools/wait-for-element-gone.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,gBAAgB,EAAsB,MAAM,6BAA6B,CAAC;AAE9G,MAAM,UAAU,8BAA8B,CAAC,MAAiB;IAC9D,MAAM,CAAC,IAAI,CACT,uBAAuB,EACvB,6JAA6J,EAC7J;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,aAAa,EAAE,CAAC;aACb,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,2EAA2E,CAAC;QACxF,UAAU,EAAE,CAAC;aACV,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,WAAW,EAAE,CAAC;aACX,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,kFAAkF,CAAC;QAC/F,aAAa,EAAE,CAAC;aACb,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,qEAAqE,CAAC;QAClF,UAAU,EAAE,CAAC;aACV,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,4CAA4C,CAAC;QACzD,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,sCAAsC,CAAC;QACnD,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aAC/C,QAAQ,EAAE;aACV,QAAQ,CAAC,8DAA8D,CAAC;KAC5E,EACD,KAAK,EAAE,EACL,QAAQ,EACR,SAAS,EACT,aAAa,EACb,UAAU,EACV,WAAW,EACX,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,OAAO,GACR,EAAE,EAAE;QACH,MAAM,QAAQ,GAAkB;YAC9B,UAAU;YACV,aAAa;YACb,WAAW;YACX,aAAa;SACd,CAAC;QAEF,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,mGAAmG;qBAC1G;iBACF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,UAAU,IAAI,MAAM,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,IAAI,GAAG,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE,CAAC;YACpC,MAAM,IAAI,GACR,QAAQ,KAAK,SAAS;gBACpB,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;gBACpC,CAAC,CAAC,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAErC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;YAEhE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;gBAEnC,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;oBAC3C,IAAI,EAAE,OAAO,IAAI,MAAM;oBACvB,QAAQ;oBACR,QAAQ,EAAE,SAAS;oBACnB,OAAO,EAAE,CAAC;oBACV,SAAS,EAAE,KAAK;iBACjB,CAAC,CAAC;gBAEH,OAAO;oBACL,OAAO,EAAE,oBAAoB,CAC3B,sBAAsB,OAAO,OAAO,gBAAgB,CAAC,QAAQ,CAAC,GAAG,EACjE,WAAW,CACZ;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACpE,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,iBAAiB,OAAO,8BAA8B,gBAAgB,CAAC,QAAQ,CAAC,IAAI;iBAC3F;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -4,6 +4,7 @@ import * as ios from "../platforms/ios.js";
|
|
|
4
4
|
import { performObservation } from "../utils/observe.js";
|
|
5
5
|
import { filterUiElements } from "../utils/ui-filter.js";
|
|
6
6
|
import { buildResponseContent } from "../utils/format-response.js";
|
|
7
|
+
import { matchElement, describeCriteria } from "../utils/element-matcher.js";
|
|
7
8
|
export function registerWaitForElementTool(server) {
|
|
8
9
|
server.tool("wait_for_element", "Poll the UI tree until an element matching the criteria appears on screen. Returns matched elements and optionally the full UI tree or screenshot.", {
|
|
9
10
|
platform: z.enum(["android", "ios"]).describe("Target platform"),
|
|
@@ -19,6 +20,10 @@ export function registerWaitForElementTool(server) {
|
|
|
19
20
|
.string()
|
|
20
21
|
.optional()
|
|
21
22
|
.describe("Wait for element whose text matches exactly"),
|
|
23
|
+
resource_id: z
|
|
24
|
+
.string()
|
|
25
|
+
.optional()
|
|
26
|
+
.describe("Wait for element whose resource_id contains this substring (case-insensitive)"),
|
|
22
27
|
type_contains: z
|
|
23
28
|
.string()
|
|
24
29
|
.optional()
|
|
@@ -41,29 +46,24 @@ export function registerWaitForElementTool(server) {
|
|
|
41
46
|
.enum(["none", "ui_tree", "screenshot", "both"])
|
|
42
47
|
.optional()
|
|
43
48
|
.describe("Additional observation after element found. Default: none"),
|
|
44
|
-
}, async ({ platform, device_id, text_contains, text_exact, type_contains, clickable, timeout_ms, poll_interval_ms, observe, }) => {
|
|
49
|
+
}, async ({ platform, device_id, text_contains, text_exact, resource_id, type_contains, clickable, timeout_ms, poll_interval_ms, observe, }) => {
|
|
45
50
|
const timeout = timeout_ms ?? 10_000;
|
|
46
51
|
const pollInterval = poll_interval_ms ?? 500;
|
|
47
52
|
const start = Date.now();
|
|
53
|
+
const criteria = {
|
|
54
|
+
text_exact,
|
|
55
|
+
text_contains,
|
|
56
|
+
resource_id,
|
|
57
|
+
type_contains,
|
|
58
|
+
clickable,
|
|
59
|
+
};
|
|
48
60
|
let lastTree = [];
|
|
49
61
|
while (Date.now() - start < timeout) {
|
|
50
62
|
const tree = platform === "android"
|
|
51
63
|
? await android.getUiTree(device_id)
|
|
52
64
|
: await ios.getUiTree(device_id);
|
|
53
65
|
lastTree = tree;
|
|
54
|
-
const matches = tree.filter((el) =>
|
|
55
|
-
if (text_exact !== undefined && el.text !== text_exact)
|
|
56
|
-
return false;
|
|
57
|
-
if (text_contains !== undefined &&
|
|
58
|
-
!el.text.toLowerCase().includes(text_contains.toLowerCase()))
|
|
59
|
-
return false;
|
|
60
|
-
if (type_contains !== undefined &&
|
|
61
|
-
!el.type.toLowerCase().includes(type_contains.toLowerCase()))
|
|
62
|
-
return false;
|
|
63
|
-
if (clickable !== undefined && el.clickable !== clickable)
|
|
64
|
-
return false;
|
|
65
|
-
return true;
|
|
66
|
-
});
|
|
66
|
+
const matches = tree.filter((el) => matchElement(el, criteria));
|
|
67
67
|
if (matches.length > 0) {
|
|
68
68
|
const observation = await performObservation({
|
|
69
69
|
mode: observe ?? "none",
|
|
@@ -85,7 +85,7 @@ export function registerWaitForElementTool(server) {
|
|
|
85
85
|
content: [
|
|
86
86
|
{
|
|
87
87
|
type: "text",
|
|
88
|
-
text: `Timeout after ${timeout}ms: no element found matching criteria. Last UI tree (${filtered.length} relevant elements):\n${JSON.stringify(filtered, null, 2)}`,
|
|
88
|
+
text: `Timeout after ${timeout}ms: no element found matching criteria (${describeCriteria(criteria)}). Last UI tree (${filtered.length} relevant elements):\n${JSON.stringify(filtered, null, 2)}`,
|
|
89
89
|
},
|
|
90
90
|
],
|
|
91
91
|
isError: true,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wait-for-element.js","sourceRoot":"","sources":["../../src/tools/wait-for-element.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAE3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"wait-for-element.js","sourceRoot":"","sources":["../../src/tools/wait-for-element.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,OAAO,MAAM,yBAAyB,CAAC;AACnD,OAAO,KAAK,GAAG,MAAM,qBAAqB,CAAC;AAE3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAsB,MAAM,6BAA6B,CAAC;AAEjG,MAAM,UAAU,0BAA0B,CAAC,MAAiB;IAC1D,MAAM,CAAC,IAAI,CACT,kBAAkB,EAClB,oJAAoJ,EACpJ;QACE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAChE,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,oDAAoD,CAAC;QACjE,aAAa,EAAE,CAAC;aACb,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,wEAAwE,CAAC;QACrF,UAAU,EAAE,CAAC;aACV,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,6CAA6C,CAAC;QAC1D,WAAW,EAAE,CAAC;aACX,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,+EAA+E,CAAC;QAC5F,aAAa,EAAE,CAAC;aACb,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,qEAAqE,CAAC;QAClF,SAAS,EAAE,CAAC;aACT,OAAO,EAAE;aACT,QAAQ,EAAE;aACV,QAAQ,CAAC,uDAAuD,CAAC;QACpE,UAAU,EAAE,CAAC;aACV,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,4CAA4C,CAAC;QACzD,gBAAgB,EAAE,CAAC;aAChB,MAAM,EAAE;aACR,GAAG,EAAE;aACL,QAAQ,EAAE;aACV,QAAQ,CAAC,sCAAsC,CAAC;QACnD,OAAO,EAAE,CAAC;aACP,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;aAC/C,QAAQ,EAAE;aACV,QAAQ,CAAC,2DAA2D,CAAC;KACzE,EACD,KAAK,EAAE,EACL,QAAQ,EACR,SAAS,EACT,aAAa,EACb,UAAU,EACV,WAAW,EACX,aAAa,EACb,SAAS,EACT,UAAU,EACV,gBAAgB,EAChB,OAAO,GACR,EAAE,EAAE;QACH,MAAM,OAAO,GAAG,UAAU,IAAI,MAAM,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,IAAI,GAAG,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAkB;YAC9B,UAAU;YACV,aAAa;YACb,WAAW;YACX,aAAa;YACb,SAAS;SACV,CAAC;QAEF,IAAI,QAAQ,GAAgB,EAAE,CAAC;QAE/B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE,CAAC;YACpC,MAAM,IAAI,GACR,QAAQ,KAAK,SAAS;gBACpB,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;gBACpC,CAAC,CAAC,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAErC,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC;YAEhE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;oBAC3C,IAAI,EAAE,OAAO,IAAI,MAAM;oBACvB,QAAQ;oBACR,QAAQ,EAAE,SAAS;oBACnB,OAAO,EAAE,CAAC;oBACV,SAAS,EAAE,KAAK;iBACjB,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,SAAS,OAAO,CAAC,MAAM,8BAA8B,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBAEpI,OAAO;oBACL,OAAO,EAAE,oBAAoB,CAAC,SAAS,EAAE,WAAW,CAAC;iBACtD,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACpE,CAAC;QAED,yDAAyD;QACzD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC5C,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,iBAAiB,OAAO,2CAA2C,gBAAgB,CAAC,QAAQ,CAAC,oBAAoB,QAAQ,CAAC,MAAM,yBAAyB,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;iBACnM;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { UiElement } from "../types.js";
|
|
2
|
+
export interface MatchCriteria {
|
|
3
|
+
text_exact?: string;
|
|
4
|
+
text_contains?: string;
|
|
5
|
+
resource_id?: string;
|
|
6
|
+
type_contains?: string;
|
|
7
|
+
clickable?: boolean;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Returns true if the element matches all provided criteria.
|
|
11
|
+
* Undefined criteria are ignored (match all).
|
|
12
|
+
*/
|
|
13
|
+
export declare function matchElement(el: UiElement, criteria: MatchCriteria): boolean;
|
|
14
|
+
/**
|
|
15
|
+
* Returns true if at least one criterion was provided.
|
|
16
|
+
*/
|
|
17
|
+
export declare function hasCriteria(criteria: MatchCriteria): boolean;
|
|
18
|
+
export declare function describeCriteria(criteria: MatchCriteria): string;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Returns true if the element matches all provided criteria.
|
|
3
|
+
* Undefined criteria are ignored (match all).
|
|
4
|
+
*/
|
|
5
|
+
export function matchElement(el, criteria) {
|
|
6
|
+
if (criteria.text_exact !== undefined && el.text !== criteria.text_exact)
|
|
7
|
+
return false;
|
|
8
|
+
if (criteria.text_contains !== undefined &&
|
|
9
|
+
!el.text.toLowerCase().includes(criteria.text_contains.toLowerCase()))
|
|
10
|
+
return false;
|
|
11
|
+
if (criteria.resource_id !== undefined &&
|
|
12
|
+
!(el.resource_id ?? "")
|
|
13
|
+
.toLowerCase()
|
|
14
|
+
.includes(criteria.resource_id.toLowerCase()))
|
|
15
|
+
return false;
|
|
16
|
+
if (criteria.type_contains !== undefined &&
|
|
17
|
+
!el.type.toLowerCase().includes(criteria.type_contains.toLowerCase()))
|
|
18
|
+
return false;
|
|
19
|
+
if (criteria.clickable !== undefined && el.clickable !== criteria.clickable)
|
|
20
|
+
return false;
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Returns true if at least one criterion was provided.
|
|
25
|
+
*/
|
|
26
|
+
export function hasCriteria(criteria) {
|
|
27
|
+
return (criteria.text_exact !== undefined ||
|
|
28
|
+
criteria.text_contains !== undefined ||
|
|
29
|
+
criteria.resource_id !== undefined ||
|
|
30
|
+
criteria.type_contains !== undefined ||
|
|
31
|
+
criteria.clickable !== undefined);
|
|
32
|
+
}
|
|
33
|
+
export function describeCriteria(criteria) {
|
|
34
|
+
const parts = [];
|
|
35
|
+
if (criteria.text_exact !== undefined)
|
|
36
|
+
parts.push(`text_exact: "${criteria.text_exact}"`);
|
|
37
|
+
if (criteria.text_contains !== undefined)
|
|
38
|
+
parts.push(`text_contains: "${criteria.text_contains}"`);
|
|
39
|
+
if (criteria.resource_id !== undefined)
|
|
40
|
+
parts.push(`resource_id: "${criteria.resource_id}"`);
|
|
41
|
+
if (criteria.type_contains !== undefined)
|
|
42
|
+
parts.push(`type_contains: "${criteria.type_contains}"`);
|
|
43
|
+
if (criteria.clickable !== undefined)
|
|
44
|
+
parts.push(`clickable: ${criteria.clickable}`);
|
|
45
|
+
return parts.join(", ");
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=element-matcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"element-matcher.js","sourceRoot":"","sources":["../../src/utils/element-matcher.ts"],"names":[],"mappings":"AAUA;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,EAAa,EAAE,QAAuB;IACjE,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,UAAU;QACtE,OAAO,KAAK,CAAC;IACf,IACE,QAAQ,CAAC,aAAa,KAAK,SAAS;QACpC,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAErE,OAAO,KAAK,CAAC;IACf,IACE,QAAQ,CAAC,WAAW,KAAK,SAAS;QAClC,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;aACpB,WAAW,EAAE;aACb,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QAE/C,OAAO,KAAK,CAAC;IACf,IACE,QAAQ,CAAC,aAAa,KAAK,SAAS;QACpC,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAErE,OAAO,KAAK,CAAC;IACf,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,CAAC,SAAS,KAAK,QAAQ,CAAC,SAAS;QACzE,OAAO,KAAK,CAAC;IACf,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,QAAuB;IACjD,OAAO,CACL,QAAQ,CAAC,UAAU,KAAK,SAAS;QACjC,QAAQ,CAAC,aAAa,KAAK,SAAS;QACpC,QAAQ,CAAC,WAAW,KAAK,SAAS;QAClC,QAAQ,CAAC,aAAa,KAAK,SAAS;QACpC,QAAQ,CAAC,SAAS,KAAK,SAAS,CACjC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAuB;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS;QACnC,KAAK,CAAC,IAAI,CAAC,gBAAgB,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC;IACrD,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;QACtC,KAAK,CAAC,IAAI,CAAC,mBAAmB,QAAQ,CAAC,aAAa,GAAG,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS;QACpC,KAAK,CAAC,IAAI,CAAC,iBAAiB,QAAQ,CAAC,WAAW,GAAG,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;QACtC,KAAK,CAAC,IAAI,CAAC,mBAAmB,QAAQ,CAAC,aAAa,GAAG,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS;QAClC,KAAK,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;IACjD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -22,7 +22,7 @@ export function buildResponseContent(confirmationText, observation) {
|
|
|
22
22
|
});
|
|
23
23
|
content.push({
|
|
24
24
|
type: "text",
|
|
25
|
-
text: `Screenshot captured (${observation.screenshot.width}x${observation.screenshot.height})
|
|
25
|
+
text: `Screenshot captured (${observation.screenshot.width}x${observation.screenshot.height}, scale=${observation.screenshot.scale} of native ${observation.screenshot.nativeWidth}x${observation.screenshot.nativeHeight}). Coordinate tools expect native resolution — multiply screenshot pixel positions by ${Math.round(1 / observation.screenshot.scale)} to convert, or pass screenshot_scale=${observation.screenshot.scale}.`,
|
|
26
26
|
});
|
|
27
27
|
}
|
|
28
28
|
return content;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format-response.js","sourceRoot":"","sources":["../../src/utils/format-response.ts"],"names":[],"mappings":"AAWA;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,gBAAwB,EACxB,WAA+B;IAE/B,MAAM,OAAO,GAAkB;QAC7B,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,gBAAgB,EAAE;KAClD,CAAC;IAEF,IAAI,CAAC,WAAW;QAAE,OAAO,OAAO,CAAC;IAEjC,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,MAAe;YACrB,IAAI,EAAE,gBAAgB,WAAW,CAAC,MAAM,CAAC,MAAM,mBAAmB,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SAChH,CAAC,CAAC;IACL,CAAC;IAED,IAAI,WAAW,CAAC,UAAU,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,OAAgB;YACtB,IAAI,EAAE,WAAW,CAAC,UAAU,CAAC,MAAM;YACnC,QAAQ,EAAE,YAAqB;SAChC,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,MAAe;YACrB,IAAI,EAAE,wBAAwB,WAAW,CAAC,UAAU,CAAC,KAAK,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,GAAG;
|
|
1
|
+
{"version":3,"file":"format-response.js","sourceRoot":"","sources":["../../src/utils/format-response.ts"],"names":[],"mappings":"AAWA;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,gBAAwB,EACxB,WAA+B;IAE/B,MAAM,OAAO,GAAkB;QAC7B,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,gBAAgB,EAAE;KAClD,CAAC;IAEF,IAAI,CAAC,WAAW;QAAE,OAAO,OAAO,CAAC;IAEjC,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,MAAe;YACrB,IAAI,EAAE,gBAAgB,WAAW,CAAC,MAAM,CAAC,MAAM,mBAAmB,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SAChH,CAAC,CAAC;IACL,CAAC;IAED,IAAI,WAAW,CAAC,UAAU,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,OAAgB;YACtB,IAAI,EAAE,WAAW,CAAC,UAAU,CAAC,MAAM;YACnC,QAAQ,EAAE,YAAqB;SAChC,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,MAAe;YACrB,IAAI,EAAE,wBAAwB,WAAW,CAAC,UAAU,CAAC,KAAK,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,WAAW,WAAW,CAAC,UAAU,CAAC,KAAK,cAAc,WAAW,CAAC,UAAU,CAAC,WAAW,IAAI,WAAW,CAAC,UAAU,CAAC,YAAY,yFAAyF,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,yCAAyC,WAAW,CAAC,UAAU,CAAC,KAAK,GAAG;SACva,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/utils/image.d.ts
CHANGED
|
@@ -2,8 +2,12 @@ export interface CompressOptions {
|
|
|
2
2
|
quality?: number;
|
|
3
3
|
scale?: number;
|
|
4
4
|
}
|
|
5
|
-
export
|
|
5
|
+
export interface CompressResult {
|
|
6
6
|
base64: string;
|
|
7
7
|
width: number;
|
|
8
8
|
height: number;
|
|
9
|
-
|
|
9
|
+
nativeWidth: number;
|
|
10
|
+
nativeHeight: number;
|
|
11
|
+
scale: number;
|
|
12
|
+
}
|
|
13
|
+
export declare function compressScreenshot(input: Buffer, options?: CompressOptions): Promise<CompressResult>;
|
package/dist/utils/image.js
CHANGED
package/dist/utils/image.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image.js","sourceRoot":"","sources":["../../src/utils/image.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"image.js","sourceRoot":"","sources":["../../src/utils/image.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAgB1B,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAa,EACb,OAAyB;IAEzB,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,GAAG,CAAC;IAEpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAExC,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC;IACzC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAG,MAAM,KAAK;SACvB,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC3B,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;SACjB,QAAQ,EAAE,CAAC;IAEd,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACjC,KAAK,EAAE,QAAQ;QACf,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,SAAS;QACtB,YAAY,EAAE,UAAU;QACxB,KAAK;KACN,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mcp-mobile-interaction",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "MCP server for interacting with Android and iOS devices/emulators — screenshots, taps, swipes, typing, UI inspection, and more.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
],
|
|
13
13
|
"scripts": {
|
|
14
14
|
"build": "tsc && chmod +x dist/index.js",
|
|
15
|
-
"prepublishOnly": "npm run build"
|
|
15
|
+
"prepublishOnly": "npm run build",
|
|
16
|
+
"test": "node --experimental-vm-modules node_modules/.bin/jest"
|
|
16
17
|
},
|
|
17
18
|
"keywords": [
|
|
18
19
|
"mcp",
|
|
@@ -39,7 +40,10 @@
|
|
|
39
40
|
"zod": "^3.24.2"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
43
|
+
"@types/jest": "^30.0.0",
|
|
42
44
|
"@types/node": "^22.13.4",
|
|
45
|
+
"jest": "^30.2.0",
|
|
46
|
+
"ts-jest": "^29.4.6",
|
|
43
47
|
"typescript": "^5.7.3"
|
|
44
48
|
}
|
|
45
49
|
}
|