appium-mcp 1.71.6 → 1.72.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +60 -15
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +21 -0
- package/dist/server.js.map +1 -1
- package/dist/tools/ai/ai.d.ts +3 -0
- package/dist/tools/ai/ai.d.ts.map +1 -0
- package/dist/tools/ai/ai.js +30 -0
- package/dist/tools/ai/ai.js.map +1 -0
- package/dist/tools/ai/config.d.ts +15 -0
- package/dist/tools/ai/config.d.ts.map +1 -0
- package/dist/tools/ai/config.js +31 -0
- package/dist/tools/ai/config.js.map +1 -0
- package/dist/tools/ai/handlers/find-element.d.ts +5 -0
- package/dist/tools/ai/handlers/find-element.d.ts.map +1 -0
- package/dist/tools/ai/handlers/find-element.js +46 -0
- package/dist/tools/ai/handlers/find-element.js.map +1 -0
- package/dist/tools/ai/schema.d.ts +12 -0
- package/dist/tools/ai/schema.d.ts.map +1 -0
- package/dist/tools/ai/schema.js +20 -0
- package/dist/tools/ai/schema.js.map +1 -0
- package/dist/tools/gestures/handlers/tap.d.ts.map +1 -1
- package/dist/tools/gestures/handlers/tap.js +7 -0
- package/dist/tools/gestures/handlers/tap.js.map +1 -1
- package/dist/tools/gestures/schema.d.ts +1 -1
- package/dist/tools/gestures/schema.d.ts.map +1 -1
- package/dist/tools/gestures/schema.js +14 -7
- package/dist/tools/gestures/schema.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +11 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/interactions/find.d.ts +1 -3
- package/dist/tools/interactions/find.d.ts.map +1 -1
- package/dist/tools/interactions/find.js +27 -85
- package/dist/tools/interactions/find.js.map +1 -1
- package/dist/tools/test-generation/generate-tests.js +1 -1
- package/dist/tools/test-generation/generate-tests.js.map +1 -1
- package/package.json +1 -1
- package/server.json +2 -2
- package/src/resources/submodules.zip +0 -0
- package/src/server.ts +31 -0
- package/src/tools/README.md +2 -1
- package/src/tools/ai/ai.ts +36 -0
- package/src/tools/ai/config.ts +35 -0
- package/src/tools/ai/handlers/find-element.ts +74 -0
- package/src/tools/ai/schema.ts +30 -0
- package/src/tools/gestures/handlers/tap.ts +9 -0
- package/src/tools/gestures/schema.ts +17 -7
- package/src/tools/index.ts +14 -0
- package/src/tools/interactions/find.ts +33 -119
- package/src/tools/test-generation/generate-tests.ts +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
## [1.72.1](https://github.com/appium/appium-mcp/compare/v1.72.0...v1.72.1) (2026-05-01)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
* **find-elements:** gate ai instructions behind a config env ([#309](https://github.com/appium/appium-mcp/issues/309)) ([07f1b22](https://github.com/appium/appium-mcp/commit/07f1b22d070c8c541df887ef6502c36e70f307c3))
|
|
6
|
+
|
|
7
|
+
## [1.72.0](https://github.com/appium/appium-mcp/compare/v1.71.6...v1.72.0) (2026-05-01)
|
|
8
|
+
|
|
9
|
+
### Features
|
|
10
|
+
|
|
11
|
+
* add APPIUM_MCP_ON_CLIENT_DISCONNECT toggle ([#306](https://github.com/appium/appium-mcp/issues/306)) ([b77cae4](https://github.com/appium/appium-mcp/commit/b77cae4fe6753599f4c04cfedbb6e1b362c23ed2))
|
|
12
|
+
|
|
1
13
|
## [1.71.6](https://github.com/appium/appium-mcp/compare/v1.71.5...v1.71.6) (2026-05-01)
|
|
2
14
|
|
|
3
15
|
### Bug Fixes
|
package/README.md
CHANGED
|
@@ -143,10 +143,12 @@ This will automatically configure the MCP server for use with Claude Code. Make
|
|
|
143
143
|
| `CAPABILITIES_CONFIG` | Optional | Absolute path to a `capabilities.json` file with per-platform capability presets |
|
|
144
144
|
| `SCREENSHOTS_DIR` | Optional | Directory where screenshots and screen recordings are saved. Defaults to the current working directory |
|
|
145
145
|
| `NO_UI` | Optional | Set to `true` or `1` to disable HTML UI components — faster responses, fewer tokens. See [NO_UI Mode](#no_ui-mode) |
|
|
146
|
+
| `APPIUM_MCP_ON_CLIENT_DISCONNECT` | Optional | Session cleanup when the MCP client disconnects: `delete_all` (default) deletes **MCP-owned** Appium sessions (`safeDeleteAllSessions`); `skip` keeps those sessions across disconnects (e.g. HTTP/stream clients that reconnect). Attached/remote sessions are not removed by this path. See [MCP disconnect behavior](#mcp-disconnect-behavior). |
|
|
146
147
|
| `APPIUM_MCP_WDA_APP_PATH` | Optional | Absolute path to a pre-extracted `WebDriverAgentRunner-Runner.app` bundle. When set, `prepare_ios_simulator` skips all GitHub downloads and uses this bundle directly — useful in environments where external downloads are blocked |
|
|
147
148
|
| `REMOTE_SERVER_URL_ALLOW_REGEX` | Optional | Regex pattern that remote Appium server URLs must match. Defaults to `^https?://` |
|
|
148
|
-
| `
|
|
149
|
-
| `
|
|
149
|
+
| `AI_VISION_ENABLED` | Optional | Set to `true` to register the `appium_ai` tool (vision-based element finding). When unset or `false`, the AI tool is **not registered** and the LLM has no way to invoke vision-based finding. Requires `AI_VISION_API_BASE_URL` and `AI_VISION_API_KEY` to also be set, otherwise the server fails to start. |
|
|
150
|
+
| `AI_VISION_API_BASE_URL` | Required when `AI_VISION_ENABLED=true` | Base URL of the OpenAI-compatible vision model API |
|
|
151
|
+
| `AI_VISION_API_KEY` | Required when `AI_VISION_ENABLED=true` | API key for the vision model provider |
|
|
150
152
|
| `AI_VISION_MODEL` | Optional | Vision model name (default: `Qwen3-VL-235B-A22B-Instruct`) |
|
|
151
153
|
| `AI_VISION_COORD_TYPE` | Optional | Coordinate type: `normalized` (default) or `absolute` |
|
|
152
154
|
| `AI_VISION_IMAGE_MAX_WIDTH` | Optional | Max image width in pixels before compression (default: `1080`) |
|
|
@@ -227,7 +229,9 @@ To start recording, call `appium_screen_recording` with `action="start"`. You ma
|
|
|
227
229
|
|
|
228
230
|
### AI Vision Element Finding
|
|
229
231
|
|
|
230
|
-
Configure AI-powered element finding using vision models.
|
|
232
|
+
Configure AI-powered element finding using vision models. When enabled, a separate tool — **`appium_ai`** — is registered alongside `appium_find_element`. It exposes `action=find_element`, which locates UI elements from natural-language descriptions and returns a coordinate UUID (`ai-element:x,y:bbox`) that can be passed to `appium_gesture` (`tap` / `double_tap` / `long_press`).
|
|
233
|
+
|
|
234
|
+
**This feature is opt-in.** When `AI_VISION_ENABLED` is unset or `false`, the `appium_ai` tool is **not registered** and the LLM has no way to invoke vision-based finding — keeping `appium_find_element` purely traditional. This deliberate gating prevents the model from defaulting to a slow, paid vision call when a stable locator (accessibility id, resource-id, etc.) would do the job.
|
|
231
235
|
|
|
232
236
|
**Required Environment Variables:**
|
|
233
237
|
|
|
@@ -236,6 +240,7 @@ Configure AI-powered element finding using vision models. This feature allows yo
|
|
|
236
240
|
"appium-mcp": {
|
|
237
241
|
"env": {
|
|
238
242
|
"ANDROID_HOME": "/path/to/android/sdk",
|
|
243
|
+
"AI_VISION_ENABLED": "true",
|
|
239
244
|
"AI_VISION_API_BASE_URL": "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
|
240
245
|
"AI_VISION_API_KEY": "your_api_key_here"
|
|
241
246
|
}
|
|
@@ -243,6 +248,8 @@ Configure AI-powered element finding using vision models. This feature allows yo
|
|
|
243
248
|
}
|
|
244
249
|
```
|
|
245
250
|
|
|
251
|
+
If `AI_VISION_ENABLED=true` is set without both API vars, the server fails to start with a clear error message — misconfiguration is surfaced immediately rather than mid-test.
|
|
252
|
+
|
|
246
253
|
**Optional Environment Variables:**
|
|
247
254
|
|
|
248
255
|
See the [Environment Variables](#environment-variables) table above for the full list of `AI_VISION_*` options and their defaults.
|
|
@@ -321,6 +328,14 @@ The following tools return lightweight text-only responses when NO_UI is enabled
|
|
|
321
328
|
- ✅ Scripted automation where human interaction is not needed
|
|
322
329
|
- ❌ Interactive debugging and exploration (keep UI enabled for better experience)
|
|
323
330
|
|
|
331
|
+
#### MCP disconnect behavior
|
|
332
|
+
|
|
333
|
+
By default (`APPIUM_MCP_ON_CLIENT_DISCONNECT` unset or `delete_all`), when the **MCP client disconnects**, this server **deletes every MCP-owned Appium session** (the same sessions `safeDeleteAllSessions` targets) so embedded drivers are not left running after a short-lived assistant run. **Attached** sessions (`ownership=attached`) are unchanged by this teardown.
|
|
334
|
+
|
|
335
|
+
HTTP and streamable MCP clients may **disconnect briefly** (reconnect, reload, proxy). If that tears down drivers you still need, set `APPIUM_MCP_ON_CLIENT_DISCONNECT` to `skip` in your MCP server `env` (same pattern as `NO_UI` above). With `skip`, sessions **survive** disconnect until you call `appium_session_management` with `action=delete`, or you stop the Appium server / process.
|
|
336
|
+
|
|
337
|
+
**Tradeoff:** `skip` can leave **orphaned sessions** on your Appium server if nothing cleans up — use it when disconnect is not the same as “automation finished.”
|
|
338
|
+
|
|
324
339
|
## 🎯 Available Tools
|
|
325
340
|
|
|
326
341
|
MCP Appium provides a comprehensive set of tools organized into the following categories:
|
|
@@ -356,7 +371,8 @@ The default regex pattern allows any URL that starts with `http://` or `https://
|
|
|
356
371
|
|
|
357
372
|
| Tool | Description |
|
|
358
373
|
| --------------------- | -------------------------------------------------------------------------------------------- |
|
|
359
|
-
| `appium_find_element` | Find a specific element using traditional locator strategies
|
|
374
|
+
| `appium_find_element` | Find a specific element using traditional locator strategies. **Strategy priority**: `accessibility id` > `id` > platform-native (`-ios predicate string` / `-ios class chain` on iOS, `-android uiautomator` on Android) > `xpath` (last resort — slow & brittle). To scroll until an element appears, use **`appium_gesture`** with **`action=scroll_to_element`** (same `strategy` / `selector` as find). |
|
|
375
|
+
| `appium_ai` | **Opt-in (gated by `AI_VISION_ENABLED=true`).** Vision-based element finding — fallback for when traditional locators don't work. `action=find_element` takes a natural-language `instruction` (e.g., "yellow search button at bottom") and returns a coordinate UUID consumable by **`appium_gesture`** (`tap` / `double_tap` / `long_press`). See [AI Vision Element Finding](#ai-vision-element-finding) for setup. |
|
|
360
376
|
| `appium_gesture` | Perform a touch gesture. `action` = `tap`, `double_tap`, `long_press`, `scroll`, `swipe`, `pinch_zoom`, or **`scroll_to_element`**. **`scroll_to_element`** scrolls vertically (`direction` = `up` \| `down`) until the locator matches, **page source stops changing** after a scroll (end of list), or **`maxScrollAttempts`** (default 10, max 80). Optional **`scrollDistance`** (0.05–1) or **`scrollDistancePreset`** = `small` \| `medium` \| `large`. Supports element UUIDs and raw coordinates for other actions. For swipe, use `speed` = `slow` \| `normal` \| `fast` (fast for pull-to-refresh). |
|
|
361
377
|
| `appium_drag_and_drop` | Perform a drag and drop gesture from a source location to a target location (supports element-to-element, element-to-coordinates, coordinates-to-element, and coordinates-to-coordinates) |
|
|
362
378
|
| `appium_perform_actions` | Execute raw W3C Actions API sequences for custom multi-touch gestures (rotate, three-finger swipe, edge swipes, precise timing). Prefer `appium_gesture` for standard gestures. |
|
|
@@ -421,7 +437,22 @@ This example demonstrates a complete e-commerce checkout flow that can be automa
|
|
|
421
437
|
|
|
422
438
|
### AI-Powered Element Finding Examples
|
|
423
439
|
|
|
424
|
-
**Traditional Mode
|
|
440
|
+
**Traditional Mode — prefer stable identifiers:**
|
|
441
|
+
|
|
442
|
+
Try strategies in priority order: `accessibility id` first, then `id`, then platform-native predicates (`-ios predicate string` / `-ios class chain` on iOS, `-android uiautomator` on Android). Reach for `xpath` only when nothing more stable exists.
|
|
443
|
+
|
|
444
|
+
```json
|
|
445
|
+
{
|
|
446
|
+
"tool": "appium_find_element",
|
|
447
|
+
"arguments": {
|
|
448
|
+
"strategy": "accessibility id",
|
|
449
|
+
"selector": "search-button"
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
`xpath` fallback (when no accessibility id, resource-id, or platform-native predicate works):
|
|
455
|
+
|
|
425
456
|
```json
|
|
426
457
|
{
|
|
427
458
|
"tool": "appium_find_element",
|
|
@@ -449,29 +480,43 @@ This example demonstrates a complete e-commerce checkout flow that can be automa
|
|
|
449
480
|
|
|
450
481
|
Use **`scrollDistance`** (0.05–1) instead of **`scrollDistancePreset`** when you want an exact fraction. Then call **`appium_find_element`** with the same `strategy` / `selector` to obtain the element id.
|
|
451
482
|
|
|
452
|
-
**AI Mode (Natural Language)
|
|
483
|
+
**AI Mode (Natural Language) — requires `AI_VISION_ENABLED=true`:**
|
|
484
|
+
|
|
485
|
+
When the AI tool is enabled, use **`appium_ai`** (not `appium_find_element`) for vision-based finding:
|
|
486
|
+
|
|
453
487
|
```json
|
|
454
488
|
{
|
|
455
|
-
"tool": "
|
|
489
|
+
"tool": "appium_ai",
|
|
490
|
+
"arguments": {
|
|
491
|
+
"action": "find_element",
|
|
492
|
+
"instruction": "yellow search button at the bottom of the screen"
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
```
|
|
496
|
+
|
|
497
|
+
The returned UUID (`ai-element:x,y:bbox`) flows directly into `appium_gesture`:
|
|
498
|
+
|
|
499
|
+
```json
|
|
500
|
+
{
|
|
501
|
+
"tool": "appium_gesture",
|
|
456
502
|
"arguments": {
|
|
457
|
-
"
|
|
458
|
-
"
|
|
503
|
+
"action": "tap",
|
|
504
|
+
"elementUUID": "ai-element:540,2280:480,2240,600,2320"
|
|
459
505
|
}
|
|
460
506
|
}
|
|
461
507
|
```
|
|
462
508
|
|
|
463
|
-
**More
|
|
509
|
+
**More instruction examples:**
|
|
464
510
|
- `"username input field at top"`
|
|
465
511
|
- `"settings icon in top-right corner"`
|
|
466
512
|
- `"red delete button next to the item"`
|
|
467
513
|
- `"blue submit button at bottom"`
|
|
468
514
|
- `"profile picture in navigation bar"`
|
|
469
515
|
|
|
470
|
-
**
|
|
471
|
-
- **
|
|
472
|
-
- **
|
|
473
|
-
-
|
|
474
|
-
- **Works Across Languages**: Describe in any language you're comfortable with
|
|
516
|
+
**When to reach for `appium_ai` vs `appium_find_element`:**
|
|
517
|
+
- **Prefer `appium_find_element`** whenever a stable accessibility id, resource-id, or unique text exists — faster, free, deterministic.
|
|
518
|
+
- **Use `appium_ai`** only when the element has no stable identifier, the page source is unavailable, or you must locate by visual cues (color, position, icon).
|
|
519
|
+
- See [AI Vision Element Finding](#ai-vision-element-finding) for setup and configuration.
|
|
475
520
|
|
|
476
521
|
### Working in Your Native Language
|
|
477
522
|
|
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AA0BlC,QAAA,MAAM,MAAM,+CAKV,CAAC;AA2CH,eAAe,MAAM,CAAC"}
|
package/dist/server.js
CHANGED
|
@@ -3,6 +3,21 @@ import registerTools from './tools/index.js';
|
|
|
3
3
|
import registerResources from './resources/index.js';
|
|
4
4
|
import { listSessions, safeDeleteAllSessions } from './session-store.js';
|
|
5
5
|
import log from './logger.js';
|
|
6
|
+
/**
|
|
7
|
+
* MCP disconnect policy for Appium sessions tracked by this server.
|
|
8
|
+
* - delete_all (default): end every owned session when the MCP client disconnects (avoids leaked drivers).
|
|
9
|
+
* - skip: keep sessions across disconnects — needed for flaky HTTP/stream clients that reconnect briefly.
|
|
10
|
+
*/
|
|
11
|
+
function disconnectSessionPolicyFromEnv() {
|
|
12
|
+
const raw = process.env.APPIUM_MCP_ON_CLIENT_DISCONNECT?.trim().toLowerCase();
|
|
13
|
+
if (raw === 'skip') {
|
|
14
|
+
return 'skip';
|
|
15
|
+
}
|
|
16
|
+
if (raw !== 'delete_all') {
|
|
17
|
+
log.warn(`APPIUM_MCP_ON_CLIENT_DISCONNECT="${raw}" is not recognized (expected delete_all or skip); defaulting to delete_all`);
|
|
18
|
+
}
|
|
19
|
+
return 'delete_all';
|
|
20
|
+
}
|
|
6
21
|
const server = new FastMCP({
|
|
7
22
|
name: 'MCP Appium',
|
|
8
23
|
version: '1.0.0',
|
|
@@ -16,7 +31,13 @@ server.on('connect', (event) => {
|
|
|
16
31
|
});
|
|
17
32
|
server.on('disconnect', async (event) => {
|
|
18
33
|
log.info('Client disconnected:', event.session);
|
|
34
|
+
const policy = disconnectSessionPolicyFromEnv();
|
|
19
35
|
const ownedSessions = listSessions().filter((session) => session.ownership === 'owned');
|
|
36
|
+
if (ownedSessions.length > 0 && policy === 'skip') {
|
|
37
|
+
log.info(`${ownedSessions.length} owned session(s) retained after MCP disconnect ` +
|
|
38
|
+
'(APPIUM_MCP_ON_CLIENT_DISCONNECT=skip). Delete explicitly via appium_session_management (action=delete) when finished.');
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
20
41
|
if (ownedSessions.length > 0) {
|
|
21
42
|
try {
|
|
22
43
|
log.info(`${ownedSessions.length} owned session(s) detected on disconnect, cleaning up...`);
|
package/dist/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,aAAa,MAAM,kBAAkB,CAAC;AAC7C,OAAO,iBAAiB,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,GAAG,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,aAAa,MAAM,kBAAkB,CAAC;AAC7C,OAAO,iBAAiB,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,GAAG,MAAM,aAAa,CAAC;AAI9B;;;;GAIG;AACH,SAAS,8BAA8B;IACrC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9E,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;QACzB,GAAG,CAAC,IAAI,CACN,oCAAoC,GAAG,6EAA6E,CACrH,CAAC;IACJ,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,MAAM,GAAG,IAAI,OAAO,CAAC;IACzB,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,OAAO;IAChB,YAAY,EACV,mPAAmP;CACtP,CAAC,CAAC;AAEH,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAC1B,aAAa,CAAC,MAAM,CAAC,CAAC;AAEtB,oDAAoD;AACpD,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;IAC7B,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;AAC/C,CAAC,CAAC,CAAC;AAEH,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;IACtC,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,8BAA8B,EAAE,CAAC;IAEhD,MAAM,aAAa,GAAG,YAAY,EAAE,CAAC,MAAM,CACzC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,KAAK,OAAO,CAC3C,CAAC;IAEF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QAClD,GAAG,CAAC,IAAI,CACN,GAAG,aAAa,CAAC,MAAM,kDAAkD;YACvE,wHAAwH,CAC3H,CAAC;QACF,OAAO;IACT,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,GAAG,CAAC,IAAI,CACN,GAAG,aAAa,CAAC,MAAM,0DAA0D,CAClF,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,qBAAqB,EAAE,CAAC;YACnD,GAAG,CAAC,IAAI,CACN,GAAG,YAAY,oDAAoD,CACpE,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,CAAC,KAAK,CAAC,0CAA0C,EAAE,KAAK,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,eAAe,MAAM,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai.d.ts","sourceRoot":"","sources":["../../../src/tools/ai/ai.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAiB,OAAO,EAAE,MAAM,SAAS,CAAC;AAKtD,MAAM,CAAC,OAAO,UAAU,EAAE,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CA8BhD"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { resolveDriver } from '../tool-response.js';
|
|
2
|
+
import { AI_ACTIONS, aiSchema } from './schema.js';
|
|
3
|
+
import { handleFindElement } from './handlers/find-element.js';
|
|
4
|
+
export default function ai(server) {
|
|
5
|
+
server.addTool({
|
|
6
|
+
name: 'appium_ai',
|
|
7
|
+
description: `Vision-based AI capabilities (FALLBACK - use only when traditional tools cannot locate the element). ` +
|
|
8
|
+
`Use 'action' to choose: ${AI_ACTIONS.join(', ')}. ` +
|
|
9
|
+
`find_element: locate an element from a natural-language description; returns a coordinate UUID (ai-element:x,y:bbox) usable with appium_gesture. ` +
|
|
10
|
+
`Prefer appium_find_element with xpath/id/accessibility id/text first. ` +
|
|
11
|
+
`Reach for this tool only when the element has no stable identifier, the page source is unavailable, or you need to locate by purely visual cues (color, position, icon).`,
|
|
12
|
+
parameters: aiSchema,
|
|
13
|
+
annotations: {
|
|
14
|
+
readOnlyHint: true,
|
|
15
|
+
openWorldHint: true,
|
|
16
|
+
},
|
|
17
|
+
execute: async (args, _context) => {
|
|
18
|
+
const resolved = resolveDriver(args.sessionId);
|
|
19
|
+
if (!resolved.ok) {
|
|
20
|
+
return resolved.result;
|
|
21
|
+
}
|
|
22
|
+
const { driver } = resolved;
|
|
23
|
+
switch (args.action) {
|
|
24
|
+
case 'find_element':
|
|
25
|
+
return handleFindElement(driver, args);
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=ai.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ai.js","sourceRoot":"","sources":["../../../src/tools/ai/ai.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAe,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAE/D,MAAM,CAAC,OAAO,UAAU,EAAE,CAAC,MAAe;IACxC,MAAM,CAAC,OAAO,CAAC;QACb,IAAI,EAAE,WAAW;QACjB,WAAW,EACT,uGAAuG;YACvG,2BAA2B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACpD,mJAAmJ;YACnJ,wEAAwE;YACxE,0KAA0K;QAC5K,UAAU,EAAE,QAAQ;QACpB,WAAW,EAAE;YACX,YAAY,EAAE,IAAI;YAClB,aAAa,EAAE,IAAI;SACpB;QACD,OAAO,EAAE,KAAK,EACZ,IAAY,EACZ,QAA6C,EACrB,EAAE;YAC1B,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,OAAO,QAAQ,CAAC,MAAM,CAAC;YACzB,CAAC;YACD,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;YAE5B,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,KAAK,cAAc;oBACjB,OAAO,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI tool enablement and configuration.
|
|
3
|
+
*
|
|
4
|
+
* The AI tool is gated behind AI_VISION_ENABLED to prevent the LLM from
|
|
5
|
+
* defaulting to vision-based finding when a traditional locator would work,
|
|
6
|
+
* and to keep cost/latency surprises out of the default experience.
|
|
7
|
+
*
|
|
8
|
+
* Contract:
|
|
9
|
+
* - AI_VISION_ENABLED !== 'true' → tool is not registered.
|
|
10
|
+
* - AI_VISION_ENABLED === 'true' → AI_VISION_API_BASE_URL and
|
|
11
|
+
* AI_VISION_API_KEY MUST be set, or server startup fails fast.
|
|
12
|
+
*/
|
|
13
|
+
export declare function isAIEnabled(): boolean;
|
|
14
|
+
export declare function assertAIConfig(): void;
|
|
15
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/tools/ai/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,wBAAgB,WAAW,IAAI,OAAO,CAErC;AAED,wBAAgB,cAAc,IAAI,IAAI,CAWrC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI tool enablement and configuration.
|
|
3
|
+
*
|
|
4
|
+
* The AI tool is gated behind AI_VISION_ENABLED to prevent the LLM from
|
|
5
|
+
* defaulting to vision-based finding when a traditional locator would work,
|
|
6
|
+
* and to keep cost/latency surprises out of the default experience.
|
|
7
|
+
*
|
|
8
|
+
* Contract:
|
|
9
|
+
* - AI_VISION_ENABLED !== 'true' → tool is not registered.
|
|
10
|
+
* - AI_VISION_ENABLED === 'true' → AI_VISION_API_BASE_URL and
|
|
11
|
+
* AI_VISION_API_KEY MUST be set, or server startup fails fast.
|
|
12
|
+
*/
|
|
13
|
+
const ENABLED_FLAG = 'AI_VISION_ENABLED';
|
|
14
|
+
const REQUIRED_WHEN_ENABLED = [
|
|
15
|
+
'AI_VISION_API_BASE_URL',
|
|
16
|
+
'AI_VISION_API_KEY',
|
|
17
|
+
];
|
|
18
|
+
export function isAIEnabled() {
|
|
19
|
+
return process.env[ENABLED_FLAG]?.toLowerCase() === 'true';
|
|
20
|
+
}
|
|
21
|
+
export function assertAIConfig() {
|
|
22
|
+
if (!isAIEnabled()) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
const missing = REQUIRED_WHEN_ENABLED.filter((name) => !process.env[name]);
|
|
26
|
+
if (missing.length > 0) {
|
|
27
|
+
throw new Error(`${ENABLED_FLAG}=true but required env vars are missing: ${missing.join(', ')}. ` +
|
|
28
|
+
`Set them or unset ${ENABLED_FLAG} to disable the AI tool.`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../../src/tools/ai/config.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,YAAY,GAAG,mBAAmB,CAAC;AACzC,MAAM,qBAAqB,GAAG;IAC5B,wBAAwB;IACxB,mBAAmB;CACX,CAAC;AAEX,MAAM,UAAU,WAAW;IACzB,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,WAAW,EAAE,KAAK,MAAM,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IACD,MAAM,OAAO,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,GAAG,YAAY,4CAA4C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YAC/E,qBAAqB,YAAY,0BAA0B,CAC9D,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ContentResult } from 'fastmcp';
|
|
2
|
+
import type { DriverInstance } from '../../../session-store.js';
|
|
3
|
+
import type { AIArgs } from '../schema.js';
|
|
4
|
+
export declare function handleFindElement(driver: DriverInstance, args: AIArgs): Promise<ContentResult>;
|
|
5
|
+
//# sourceMappingURL=find-element.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"find-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAShE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAY3C,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,aAAa,CAAC,CA+CxB"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { imageUtil } from '@appium/support';
|
|
2
|
+
import { getScreenshot } from '../../../command.js';
|
|
3
|
+
import { AIVisionFinder } from '../../../ai-finder/vision-finder.js';
|
|
4
|
+
import log from '../../../logger.js';
|
|
5
|
+
import { errorResult, textResultWithPrimaryElementId, toolErrorMessage, } from '../../tool-response.js';
|
|
6
|
+
// Module-level singleton: ensures the LRU cache persists across tool calls.
|
|
7
|
+
// Creating a new AIVisionFinder() on every call would reset the cache each time.
|
|
8
|
+
let _finderInstance = null;
|
|
9
|
+
function getAIVisionFinder() {
|
|
10
|
+
if (!_finderInstance) {
|
|
11
|
+
_finderInstance = new AIVisionFinder();
|
|
12
|
+
}
|
|
13
|
+
return _finderInstance;
|
|
14
|
+
}
|
|
15
|
+
export async function handleFindElement(driver, args) {
|
|
16
|
+
if (!args.instruction) {
|
|
17
|
+
return errorResult('instruction is required for action=find_element. ' +
|
|
18
|
+
'Example: { action: "find_element", instruction: "yellow search button at bottom" }');
|
|
19
|
+
}
|
|
20
|
+
try {
|
|
21
|
+
log.info(`Finding element using AI with instruction: "${args.instruction}"`);
|
|
22
|
+
const screenshotBase64 = await getScreenshot(driver);
|
|
23
|
+
const imageBuffer = Buffer.from(screenshotBase64, 'base64');
|
|
24
|
+
const sharp = imageUtil.requireSharp();
|
|
25
|
+
const metadata = await sharp(imageBuffer).metadata();
|
|
26
|
+
if (!metadata.width || !metadata.height) {
|
|
27
|
+
throw new Error('Failed to get image dimensions from screenshot');
|
|
28
|
+
}
|
|
29
|
+
const { width, height } = metadata;
|
|
30
|
+
const finder = getAIVisionFinder();
|
|
31
|
+
const result = await finder.findElement(screenshotBase64, args.instruction, width, height);
|
|
32
|
+
// Format: "ai-element:{x},{y}:{bbox}" — consumed by appium_gesture handlers.
|
|
33
|
+
const elementUUID = `ai-element:${result.center.x},${result.center.y}:${result.bbox.join(',')}`;
|
|
34
|
+
let detail = `Successfully found "${result.target}" at coordinates (${result.center.x}, ${result.center.y}) using AI vision.`;
|
|
35
|
+
if (result.annotatedImagePath) {
|
|
36
|
+
detail += ` Vision image: ${result.annotatedImagePath}`;
|
|
37
|
+
}
|
|
38
|
+
return textResultWithPrimaryElementId(elementUUID, detail);
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
const errorMessage = toolErrorMessage(err);
|
|
42
|
+
log.error('AI find_element failed:', errorMessage);
|
|
43
|
+
return errorResult(`AI find_element failed. Error: ${errorMessage}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=find-element.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"find-element.js","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AACrE,OAAO,GAAG,MAAM,oBAAoB,CAAC;AACrC,OAAO,EACL,WAAW,EACX,8BAA8B,EAC9B,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAGhC,4EAA4E;AAC5E,iFAAiF;AACjF,IAAI,eAAe,GAA0B,IAAI,CAAC;AAClD,SAAS,iBAAiB;IACxB,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,eAAe,GAAG,IAAI,cAAc,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAsB,EACtB,IAAY;IAEZ,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACtB,OAAO,WAAW,CAChB,mDAAmD;YACjD,oFAAoF,CACvF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,GAAG,CAAC,IAAI,CACN,+CAA+C,IAAI,CAAC,WAAW,GAAG,CACnE,CAAC;QAEF,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QAErD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,YAAY,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC;QAErD,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAEnC,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CACrC,gBAAgB,EAChB,IAAI,CAAC,WAAW,EAChB,KAAK,EACL,MAAM,CACP,CAAC;QAEF,6EAA6E;QAC7E,MAAM,WAAW,GAAG,cAAc,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAEhG,IAAI,MAAM,GAAG,uBAAuB,MAAM,CAAC,MAAM,qBAAqB,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC;QAC9H,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;YAC9B,MAAM,IAAI,kBAAkB,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC1D,CAAC;QAED,OAAO,8BAA8B,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC3C,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,YAAY,CAAC,CAAC;QACnD,OAAO,WAAW,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const AI_ACTIONS: readonly ["find_element"];
|
|
3
|
+
export type AIAction = (typeof AI_ACTIONS)[number];
|
|
4
|
+
export declare const aiSchema: z.ZodObject<{
|
|
5
|
+
action: z.ZodEnum<{
|
|
6
|
+
find_element: "find_element";
|
|
7
|
+
}>;
|
|
8
|
+
instruction: z.ZodOptional<z.ZodString>;
|
|
9
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
10
|
+
}, z.core.$strip>;
|
|
11
|
+
export type AIArgs = z.infer<typeof aiSchema>;
|
|
12
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/tools/ai/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,UAAU,2BAA4B,CAAC;AACpD,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,eAAO,MAAM,QAAQ;;;;;;iBAsBnB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export const AI_ACTIONS = ['find_element'];
|
|
3
|
+
export const aiSchema = z.object({
|
|
4
|
+
action: z
|
|
5
|
+
.enum(AI_ACTIONS)
|
|
6
|
+
.describe(`AI capability to invoke. ` +
|
|
7
|
+
`find_element: locate an element from a natural-language description using a vision model. ` +
|
|
8
|
+
`Returns a coordinate UUID (format: ai-element:x,y:bbox) usable with appium_gesture (tap/double_tap/long_press).`),
|
|
9
|
+
instruction: z
|
|
10
|
+
.string()
|
|
11
|
+
.optional()
|
|
12
|
+
.describe(`Natural-language description of the target element. ` +
|
|
13
|
+
`Required for: find_element. ` +
|
|
14
|
+
`Examples: "yellow search button at bottom", "username input field at top", "settings icon in top-right corner".`),
|
|
15
|
+
sessionId: z
|
|
16
|
+
.string()
|
|
17
|
+
.optional()
|
|
18
|
+
.describe('Session ID to target. If omitted, uses the active session.'),
|
|
19
|
+
});
|
|
20
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/tools/ai/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,cAAc,CAAU,CAAC;AAGpD,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,UAAU,CAAC;SAChB,QAAQ,CACP,2BAA2B;QACzB,4FAA4F;QAC5F,iHAAiH,CACpH;IAEH,WAAW,EAAE,CAAC;SACX,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,sDAAsD;QACpD,8BAA8B;QAC9B,iHAAiH,CACpH;IAEH,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,4DAA4D,CAAC;CAC1E,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tap.d.ts","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/tap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"tap.d.ts","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/tap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAehE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAShD,wBAAsB,SAAS,CAC7B,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CAoCxB;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CAmDxB;AAED,wBAAsB,eAAe,CACnC,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CA6DxB"}
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
import { getPlatformName, PLATFORM } from '../../../session-store.js';
|
|
2
2
|
import { elementClick, execute, getElementRect, performActions, } from '../../../command.js';
|
|
3
3
|
import { errorResult, textResult, textResultWithPrimaryElementId, toolErrorMessage, } from '../../tool-response.js';
|
|
4
|
+
import { isAIEnabled } from '../../ai/config.js';
|
|
4
5
|
const AI_ELEMENT_PREFIX = 'ai-element:';
|
|
6
|
+
const AI_DISABLED_REJECTION = `Received an ai-element: UUID, but the appium_ai tool is not registered ` +
|
|
7
|
+
`(AI_VISION_ENABLED is not set to true). Use appium_find_element to get a real ` +
|
|
8
|
+
`element UUID, or enable AI_VISION_ENABLED=true with the required AI_VISION_* keys.`;
|
|
5
9
|
export async function handleTap(driver, args) {
|
|
6
10
|
try {
|
|
7
11
|
if (args.elementUUID) {
|
|
8
12
|
if (args.elementUUID.startsWith(AI_ELEMENT_PREFIX)) {
|
|
13
|
+
if (!isAIEnabled()) {
|
|
14
|
+
return errorResult(AI_DISABLED_REJECTION);
|
|
15
|
+
}
|
|
9
16
|
const parsed = parseAiElementCoords(args.elementUUID);
|
|
10
17
|
if ('error' in parsed) {
|
|
11
18
|
return errorResult(parsed.error);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tap.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/tap.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EACL,YAAY,EACZ,OAAO,EACP,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,WAAW,EACX,UAAU,EACV,8BAA8B,EAC9B,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"tap.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/tap.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,EACL,YAAY,EACZ,OAAO,EACP,cAAc,EACd,cAAc,GACf,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,WAAW,EACX,UAAU,EACV,8BAA8B,EAC9B,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,iBAAiB,GAAG,aAAa,CAAC;AAExC,MAAM,qBAAqB,GACzB,yEAAyE;IACzE,gFAAgF;IAChF,oFAAoF,CAAC;AAEvF,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,MAAsB,EACtB,IAAiB;IAEjB,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACnD,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBACnB,OAAO,WAAW,CAAC,qBAAqB,CAAC,CAAC;gBAC5C,CAAC;gBACD,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACtD,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;oBACtB,OAAO,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnC,CAAC;gBACD,MAAM,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3D,OAAO,8BAA8B,CACnC,IAAI,CAAC,WAAW,EAChB,kDAAkD,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,IAAI,CAC5E,CAAC;YACJ,CAAC;YACD,MAAM,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7C,OAAO,8BAA8B,CACnC,IAAI,CAAC,WAAW,EAChB,+BAA+B,IAAI,CAAC,WAAW,GAAG,CACnD,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACjD,OAAO,WAAW,CAChB,+DAA+D,CAChE,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,OAAO,UAAU,CACf,uCAAuC,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAC7D,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,WAAW,CAAC,0BAA0B,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAsB,EACtB,IAAiB;IAEjB,IAAI,CAAC;QACH,IAAI,CAAS,CAAC;QACd,IAAI,CAAS,CAAC;QAEd,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,QAAQ,KAAK,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAC9B,MAAM,OAAO,CAAC,MAAM,EAAE,mBAAmB,EAAE;oBACzC,SAAS,EAAE,IAAI,CAAC,WAAW;iBAC5B,CAAC,CAAC;gBACH,OAAO,8BAA8B,CACnC,IAAI,CAAC,WAAW,EAChB,sCAAsC,IAAI,CAAC,WAAW,GAAG,CAC1D,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACxE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;YACb,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;QACf,CAAC;aAAM,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACxD,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACX,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACb,CAAC;aAAM,CAAC;YACN,OAAO,WAAW,CAChB,sEAAsE,CACvE,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,CAAC,MAAM,EAAE;YAC3B;gBACE,IAAI,EAAE,SAAS;gBACf,EAAE,EAAE,SAAS;gBACb,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;gBACpC,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;oBAC1C,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;oBAClC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;oBAChC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE;oBAChC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;oBAClC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;oBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;iBACjC;aACF;SACF,CAAC,CAAC;QACH,OAAO,UAAU,CAAC,kCAAkC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,WAAW,CAChB,iCAAiC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CACzD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAsB,EACtB,IAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QACvC,IAAI,QAAQ,GAAG,GAAG,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;YACvC,OAAO,WAAW,CAChB,uDAAuD,CACxD,CAAC;QACJ,CAAC;QAED,IAAI,CAAS,CAAC;QACd,IAAI,CAAS,CAAC;QAEd,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,QAAQ,KAAK,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,MAAM,OAAO,CAAC,MAAM,EAAE,sBAAsB,EAAE;wBAC5C,SAAS,EAAE,IAAI,CAAC,WAAW;wBAC3B,QAAQ,EAAE,QAAQ,GAAG,IAAI;qBAC1B,CAAC,CAAC;oBACH,OAAO,8BAA8B,CACnC,IAAI,CAAC,WAAW,EAChB,qCAAqC,IAAI,CAAC,WAAW,QAAQ,QAAQ,KAAK,CAC3E,CAAC;gBACJ,CAAC;gBAAC,MAAM,CAAC;oBACP,uCAAuC;gBACzC,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACxE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;YACb,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;QACf,CAAC;aAAM,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACxD,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACX,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACb,CAAC;aAAM,CAAC;YACN,OAAO,WAAW,CAChB,sEAAsE,CACvE,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,CAAC,MAAM,EAAE;YAC3B;gBACE,IAAI,EAAE,SAAS;gBACf,EAAE,EAAE,SAAS;gBACb,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;gBACpC,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;oBAC1C,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;oBAClC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE;oBAC3B,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;iBACjC;aACF;SACF,CAAC,CAAC;QACH,OAAO,UAAU,CACf,iCAAiC,CAAC,KAAK,CAAC,SAAS,QAAQ,KAAK,CAC/D,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,WAAW,CAChB,iCAAiC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CACzD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAAY;IAEZ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,KAAK,EAAE,iCAAiC,EAAE,CAAC;IACtD,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,KAAK,EAAE,wCAAwC,EAAE,CAAC;IAC7D,CAAC;IACD,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,KAAK,EAAE,8CAA8C,EAAE,CAAC;IACnE,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AAClB,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,OAAO;QACL;YACE,IAAI,EAAE,SAAS;YACf,EAAE,EAAE,SAAS;YACb,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE;YACpC,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;gBAC1C,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,EAAE;gBAClC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE;gBAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;aACjC;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAAsB,EACtB,WAAmB;IAEnB,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACvD,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QACtC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;KACxC,CAAC;AACJ,CAAC"}
|
|
@@ -5,7 +5,7 @@ export declare const SWIPE_SPEEDS: readonly ["slow", "normal", "fast"];
|
|
|
5
5
|
export type SwipeSpeed = (typeof SWIPE_SPEEDS)[number];
|
|
6
6
|
export declare const SCROLL_DISTANCE_PRESETS: readonly ["small", "medium", "large"];
|
|
7
7
|
export type ScrollDistancePreset = (typeof SCROLL_DISTANCE_PRESETS)[number];
|
|
8
|
-
export declare const LOCATOR_STRATEGIES: readonly ["
|
|
8
|
+
export declare const LOCATOR_STRATEGIES: readonly ["accessibility id", "id", "-ios predicate string", "-ios class chain", "-android uiautomator", "xpath", "name", "class name", "css selector"];
|
|
9
9
|
export declare const gestureSchema: z.ZodObject<{
|
|
10
10
|
action: z.ZodEnum<{
|
|
11
11
|
tap: "tap";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/tools/gestures/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/tools/gestures/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,eAAO,MAAM,eAAe,oGAQlB,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7D,eAAO,MAAM,YAAY,qCAAsC,CAAC;AAChE,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,uBAAuB,uCAAwC,CAAC;AAC7E,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE5E,eAAO,MAAM,kBAAkB,yJAUrB,CAAC;AAEX,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAyJxB,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC"}
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { elementUUIDScheme } from '../../schema.js';
|
|
3
|
+
import { isAIEnabled } from '../ai/config.js';
|
|
4
|
+
const AI_UUID_HINT = isAIEnabled()
|
|
5
|
+
? `Supports AI coordinate UUIDs (format: ai-element:x,y:bbox) returned by appium_ai. `
|
|
6
|
+
: '';
|
|
3
7
|
export const GESTURE_ACTIONS = [
|
|
4
8
|
'tap',
|
|
5
9
|
'double_tap',
|
|
@@ -12,15 +16,15 @@ export const GESTURE_ACTIONS = [
|
|
|
12
16
|
export const SWIPE_SPEEDS = ['slow', 'normal', 'fast'];
|
|
13
17
|
export const SCROLL_DISTANCE_PRESETS = ['small', 'medium', 'large'];
|
|
14
18
|
export const LOCATOR_STRATEGIES = [
|
|
15
|
-
'
|
|
19
|
+
'accessibility id',
|
|
16
20
|
'id',
|
|
21
|
+
'-ios predicate string',
|
|
22
|
+
'-ios class chain',
|
|
23
|
+
'-android uiautomator',
|
|
24
|
+
'xpath',
|
|
17
25
|
'name',
|
|
18
26
|
'class name',
|
|
19
|
-
'accessibility id',
|
|
20
27
|
'css selector',
|
|
21
|
-
'-android uiautomator',
|
|
22
|
-
'-ios predicate string',
|
|
23
|
-
'-ios class chain',
|
|
24
28
|
];
|
|
25
29
|
export const gestureSchema = z.object({
|
|
26
30
|
action: z
|
|
@@ -37,7 +41,8 @@ export const gestureSchema = z.object({
|
|
|
37
41
|
`Optional scrollDistance (0.05–1) or scrollDistancePreset (small|medium|large).`),
|
|
38
42
|
elementUUID: elementUUIDScheme
|
|
39
43
|
.optional()
|
|
40
|
-
.describe(`UUID of the element to act on.
|
|
44
|
+
.describe(`UUID of the element to act on. ` +
|
|
45
|
+
AI_UUID_HINT +
|
|
41
46
|
`Used by: tap, double_tap, long_press, pinch_zoom. ` +
|
|
42
47
|
`For scroll/swipe, when provided with direction, the gesture is calculated relative to this element instead of the whole screen.`),
|
|
43
48
|
x: z
|
|
@@ -102,7 +107,9 @@ export const gestureSchema = z.object({
|
|
|
102
107
|
strategy: z
|
|
103
108
|
.enum(LOCATOR_STRATEGIES)
|
|
104
109
|
.optional()
|
|
105
|
-
.describe(`Locator strategy. Required for: scroll_to_element
|
|
110
|
+
.describe(`Locator strategy. Required for: scroll_to_element. ` +
|
|
111
|
+
`Priority: accessibility id > id > platform-native (-ios predicate string / -ios class chain on iOS, -android uiautomator on Android) > xpath (LAST RESORT — slow on iOS XCUITest, brittle) > name > class name > css selector (webview only). ` +
|
|
112
|
+
`Same ranking as appium_find_element.`),
|
|
106
113
|
selector: z
|
|
107
114
|
.string()
|
|
108
115
|
.optional()
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/tools/gestures/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/tools/gestures/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,MAAM,YAAY,GAAG,WAAW,EAAE;IAChC,CAAC,CAAC,oFAAoF;IACtF,CAAC,CAAC,EAAE,CAAC;AAEP,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK;IACL,YAAY;IACZ,YAAY;IACZ,QAAQ;IACR,OAAO;IACP,YAAY;IACZ,mBAAmB;CACX,CAAC;AAIX,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAU,CAAC;AAGhE,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAU,CAAC;AAG7E,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,kBAAkB;IAClB,IAAI;IACJ,uBAAuB;IACvB,kBAAkB;IAClB,sBAAsB;IACtB,OAAO;IACP,MAAM;IACN,YAAY;IACZ,cAAc;CACN,CAAC;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,eAAe,CAAC;SACrB,QAAQ,CACP,sBAAsB;QACpB,uCAAuC;QACvC,uFAAuF;QACvF,sEAAsE;QACtE,0DAA0D;QAC1D,2GAA2G;QAC3G,4FAA4F;QAC5F,6GAA6G;QAC7G,yIAAyI;QACzI,gFAAgF,CACnF;IAEH,WAAW,EAAE,iBAAiB;SAC3B,QAAQ,EAAE;SACV,QAAQ,CACP,iCAAiC;QAC/B,YAAY;QACZ,oDAAoD;QACpD,iIAAiI,CACpI;IAEH,CAAC,EAAE,CAAC;SACD,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,gBAAgB;QACd,4EAA4E;QAC5E,oFAAoF;QACpF,mFAAmF,CACtF;IACH,CAAC,EAAE,CAAC;SACD,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,gBAAgB;QACd,+CAA+C;QAC/C,2DAA2D;QAC3D,mFAAmF,CACtF;IACH,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,uEAAuE,CACxE;IACH,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,uEAAuE,CACxE;IAEH,SAAS,EAAE,CAAC;SACT,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;SACrC,QAAQ,EAAE;SACV,QAAQ,CACP,gGAAgG;QAC9F,+FAA+F,CAClG;IAEH,KAAK,EAAE,CAAC;SACL,IAAI,CAAC,YAAY,CAAC;SAClB,QAAQ,EAAE;SACV,QAAQ,CACP,8KAA8K,CAC/K;IAEH,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,KAAK,CAAC;SACV,QAAQ,EAAE;SACV,QAAQ,CACP,2FAA2F;QACzF,oFAAoF,CACvF;IAEH,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,wJAAwJ,CACzJ;IACH,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,8EAA8E,CAC/E;IAEH,QAAQ,EAAE,CAAC;SACR,IAAI,CAAC,kBAAkB,CAAC;SACxB,QAAQ,EAAE;SACV,QAAQ,CACP,qDAAqD;QACnD,gPAAgP;QAChP,sCAAsC,CACzC;IACH,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,0DAA0D,CAAC;IAEvE,iBAAiB,EAAE,CAAC;SACjB,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CACP,oGAAoG,CACrG;IAEH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,EAAE;SACV,QAAQ,CACP,oGAAoG;QAClG,qGAAqG,CACxG;IAEH,oBAAoB,EAAE,CAAC;SACpB,IAAI,CAAC,uBAAuB,CAAC;SAC7B,QAAQ,EAAE;SACV,QAAQ,CACP,0HAA0H;QACxH,qCAAqC,CACxC;IAEH,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,4DAA4D,CAAC;CAC1E,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,KAAK,EAAiB,OAAO,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,KAAK,EAAiB,OAAO,EAAE,MAAM,SAAS,CAAC;AAwCtD,MAAM,CAAC,OAAO,UAAU,aAAa,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAuJ3D"}
|
package/dist/tools/index.js
CHANGED
|
@@ -32,6 +32,8 @@ import screenRecording from './interactions/screen-recording.js';
|
|
|
32
32
|
import app from './app-management/app.js';
|
|
33
33
|
import mobilePermissions from './app-management/permissions.js';
|
|
34
34
|
import context from './context/context.js';
|
|
35
|
+
import ai from './ai/ai.js';
|
|
36
|
+
import { isAIEnabled, assertAIConfig } from './ai/config.js';
|
|
35
37
|
export default function registerTools(server) {
|
|
36
38
|
// Wrap addTool to inject logging around tool execution
|
|
37
39
|
const originalAddTool = server.addTool.bind(server);
|
|
@@ -153,6 +155,15 @@ export default function registerTools(server) {
|
|
|
153
155
|
// Documentation
|
|
154
156
|
answerAppium(server);
|
|
155
157
|
appiumSkills(server);
|
|
158
|
+
// AI (vision-based fallback) — gated; only registered when explicitly enabled.
|
|
159
|
+
assertAIConfig();
|
|
160
|
+
if (isAIEnabled()) {
|
|
161
|
+
ai(server);
|
|
162
|
+
log.info('appium_ai tool registered (AI_VISION_ENABLED=true)');
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
log.info('appium_ai tool NOT registered (set AI_VISION_ENABLED=true to enable)');
|
|
166
|
+
}
|
|
156
167
|
log.info('All tools registered');
|
|
157
168
|
}
|
|
158
169
|
function sessionIdFromToolArgs(args) {
|