appium-mcp 1.95.7 → 1.95.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/tools/ai/handlers/find-element.d.ts.map +1 -1
- package/dist/tools/ai/handlers/find-element.js +23 -3
- package/dist/tools/ai/handlers/find-element.js.map +1 -1
- package/dist/tools/gestures/handlers/ai-element.d.ts +2 -1
- package/dist/tools/gestures/handlers/ai-element.d.ts.map +1 -1
- package/dist/tools/gestures/handlers/ai-element.js +1 -1
- package/dist/tools/gestures/handlers/ai-element.js.map +1 -1
- package/package.json +1 -1
- package/server.json +2 -2
- package/src/tools/ai/handlers/find-element.ts +24 -3
- package/src/tools/gestures/handlers/ai-element.ts +3 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## [1.95.8](https://github.com/appium/appium-mcp/compare/v1.95.7...v1.95.8) (2026-10-10)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
* map AI vision coordinates to iOS touch space ([#564](https://github.com/appium/appium-mcp/issues/564)) ([2ed139e](https://github.com/appium/appium-mcp/commit/2ed139ee6c11a74f2a67fcdb03ec9951752fa92f))
|
|
6
|
+
|
|
1
7
|
## [1.95.7](https://github.com/appium/appium-mcp/compare/v1.95.6...v1.95.7) (2026-10-10)
|
|
2
8
|
|
|
3
9
|
### Bug Fixes
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"find-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"find-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,SAAS,CAAC;AAM3C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,cAAc,CAAC;AAMzC,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAyDpG"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { imageUtil } from '@appium/support';
|
|
2
2
|
import { AIVisionFinder } from '../../../ai-finder/vision-finder.js';
|
|
3
|
-
import { getScreenshot } from '../../../command.js';
|
|
3
|
+
import { getScreenshot, getWindowSize } from '../../../command.js';
|
|
4
4
|
import log from '../../../logger.js';
|
|
5
|
+
import { getPlatformName, PLATFORM } from '../../../session-store.js';
|
|
5
6
|
import { errorResult, textResultWithPrimaryElementId, toolErrorMessage } from '../../tool-response.js';
|
|
6
7
|
// Module-level singleton: ensures the LRU cache persists across tool calls.
|
|
7
8
|
// Creating a new AIVisionFinder() on every call would reset the cache each time.
|
|
@@ -22,9 +23,28 @@ export async function handleFindElement(driver, args) {
|
|
|
22
23
|
const { width, height } = metadata;
|
|
23
24
|
const finder = getAIVisionFinder();
|
|
24
25
|
const result = await finder.findElement(screenshotBase64, instruction, width, height);
|
|
26
|
+
// Vision results and annotations use screenshot pixels, but iOS touch actions
|
|
27
|
+
// use logical screen coordinates. Do not mutate the finder's cached result.
|
|
28
|
+
let center = result.center;
|
|
29
|
+
let bbox = result.bbox;
|
|
30
|
+
if (getPlatformName(driver) === PLATFORM.ios) {
|
|
31
|
+
const screen = await getWindowSize(driver);
|
|
32
|
+
if (!(screen.width > 0 && screen.height > 0)) {
|
|
33
|
+
throw new Error('Failed to get screen dimensions for AI coordinate mapping');
|
|
34
|
+
}
|
|
35
|
+
const scaleX = screen.width / width;
|
|
36
|
+
const scaleY = screen.height / height;
|
|
37
|
+
center = { x: Math.floor(center.x * scaleX), y: Math.floor(center.y * scaleY) };
|
|
38
|
+
bbox = [
|
|
39
|
+
Math.floor(bbox[0] * scaleX),
|
|
40
|
+
Math.floor(bbox[1] * scaleY),
|
|
41
|
+
Math.ceil(bbox[2] * scaleX),
|
|
42
|
+
Math.ceil(bbox[3] * scaleY),
|
|
43
|
+
];
|
|
44
|
+
}
|
|
25
45
|
// Format: "ai-element:{x},{y}:{bbox}" — consumed by appium_gesture handlers.
|
|
26
|
-
const elementUUID = `ai-element:${
|
|
27
|
-
let detail = `Successfully found "${result.target}" at coordinates (${
|
|
46
|
+
const elementUUID = `ai-element:${center.x},${center.y}:${bbox.join(',')}`;
|
|
47
|
+
let detail = `Successfully found "${result.target}" at coordinates (${center.x}, ${center.y}) using AI vision.`;
|
|
28
48
|
if (result.annotatedImagePath) {
|
|
29
49
|
detail += ` Vision image: ${result.annotatedImagePath}`;
|
|
30
50
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"find-element.js","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAG1C,OAAO,EAAC,cAAc,EAAC,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"find-element.js","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAG1C,OAAO,EAAC,cAAc,EAAC,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAAC,aAAa,EAAE,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACjE,OAAO,GAAG,MAAM,oBAAoB,CAAC;AACrC,OAAO,EAAC,eAAe,EAAE,QAAQ,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,8BAA8B,EAAE,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAGrG,4EAA4E;AAC5E,iFAAiF;AACjF,IAAI,eAAe,GAA0B,IAAI,CAAC;AAElD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAsB,EAAE,IAAY;IAC1E,uEAAuE;IACvE,8EAA8E;IAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,WAAqB,CAAC;IAE/C,IAAI,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,+CAA+C,WAAW,GAAG,CAAC,CAAC;QAExE,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QAErD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,YAAY,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC;QAErD,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,EAAC,KAAK,EAAE,MAAM,EAAC,GAAG,QAAQ,CAAC;QAEjC,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,gBAAgB,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAEtF,8EAA8E;QAC9E,4EAA4E;QAC5E,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC3B,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACvB,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,GAAG,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAC/E,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;YACtC,MAAM,GAAG,EAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,EAAC,CAAC;YAC9E,IAAI,GAAG;gBACL,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;aAC5B,CAAC;QACJ,CAAC;QAED,6EAA6E;QAC7E,MAAM,WAAW,GAAG,cAAc,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAE3E,IAAI,MAAM,GAAG,uBAAuB,MAAM,CAAC,MAAM,qBAAqB,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,oBAAoB,CAAC;QAChH,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;YAC9B,MAAM,IAAI,kBAAkB,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC1D,CAAC;QAED,OAAO,8BAA8B,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC3C,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,YAAY,CAAC,CAAC;QACnD,OAAO,WAAW,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,eAAe,GAAG,IAAI,cAAc,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC"}
|
|
@@ -9,7 +9,8 @@ import type { Rect } from '@appium/types';
|
|
|
9
9
|
*
|
|
10
10
|
* where (cx, cy) is the visual centre of the target and (x0,y0,x1,y1) is
|
|
11
11
|
* the bounding box returned by the vision model. Both are pixel
|
|
12
|
-
* coordinates in the
|
|
12
|
+
* coordinates in the driver's touch coordinate space (logical screen
|
|
13
|
+
* coordinates on iOS), NOT real WebDriver
|
|
13
14
|
* element ids — so they MUST NOT be passed to driver.getElementRect or
|
|
14
15
|
* to platform-native commands like `mobile: doubleTap`, `mobile: pinch`,
|
|
15
16
|
* etc., which require a real element id.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/ai-element.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,eAAe,CAAC;AACxC
|
|
1
|
+
{"version":3,"file":"ai-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/ai-element.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,eAAe,CAAC;AACxC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,SAAS,CAAC;AAG3C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,2BAA2B,CAAC;AAG9D,eAAO,MAAM,iBAAiB,gBAAgB,CAAC;AAE/C,eAAO,MAAM,qBAAqB,QAGoD,CAAC;AAEvF,eAAO,MAAM,sBAAsB,QAGgD,CAAC;AAEpF,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAC,CAAC;IAC/B,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAaF,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,IAAI,MAAM,CAExE;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,CA6B9E;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,CAAC,CAepH;AAED,wBAAgB,gBAAgB,IAAI,aAAa,CAEhD;AAED,wBAAgB,gCAAgC,IAAI,aAAa,CAEhE;AAED,wBAAgB,mCAAmC,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,GAAG,SAAS,CAK9G"}
|
|
@@ -13,7 +13,7 @@ export const AI_WEBDRIVER_REJECTION = `That is an ai-element token from appium_a
|
|
|
13
13
|
* makes start/end identical after rounding. Keep a minimum inset area so those
|
|
14
14
|
* gestures still move visibly while staying centred on the parsed point.
|
|
15
15
|
*
|
|
16
|
-
* Values are
|
|
16
|
+
* Values are touch coordinates; callers can clamp against the window if needed.
|
|
17
17
|
*/
|
|
18
18
|
const AI_FALLBACK_RECT_WIDTH = 100;
|
|
19
19
|
const AI_FALLBACK_RECT_HEIGHT = 100;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai-element.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/ai-element.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ai-element.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/ai-element.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAC,WAAW,EAAC,MAAM,wBAAwB,CAAC;AAEnD,MAAM,CAAC,MAAM,iBAAiB,GAAG,aAAa,CAAC;AAE/C,MAAM,CAAC,MAAM,qBAAqB,GAChC,yEAAyE;IACzE,gFAAgF;IAChF,oFAAoF,CAAC;AAEvF,MAAM,CAAC,MAAM,sBAAsB,GACjC,0EAA0E;IAC1E,sEAAsE;IACtE,iFAAiF,CAAC;AAOpF;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAEpC,MAAM,UAAU,eAAe,CAAC,IAAwB;IACtD,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;AACxE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,EAAC,KAAK,EAAE,sDAAsD,EAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QACjD,OAAO;YACL,KAAK,EAAE,iEAAiE,KAAK,CAAC,CAAC,CAAC,KAAK;SACtF,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,GAAG,gBAAgB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACpC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACzG,IAAI,GAAG;gBACL,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;gBACV,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;gBACV,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACxB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;aAC1B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAC,MAAM,EAAE,EAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAC,EAAE,IAAI,EAAC,CAAC;AACxC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAsB,EAAE,WAAmB;IACjF,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IACD,IAAI,CAAC;QACH,OAAO,MAAM,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,WAAW,CAAC,qBAAqB,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,gCAAgC;IAC9C,OAAO,WAAW,CAAC,sBAAsB,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,WAA+B;IACjF,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;QACjC,OAAO,gCAAgC,EAAE,CAAC;IAC5C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAU,EAAE,EAAU;IAC9C,MAAM,CAAC,GAAG,sBAAsB,CAAC;IACjC,MAAM,CAAC,GAAG,uBAAuB,CAAC;IAClC,OAAO;QACL,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,CAAC;KACV,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "appium-mcp",
|
|
3
|
-
"version": "1.95.
|
|
3
|
+
"version": "1.95.8",
|
|
4
4
|
"description": "Intelligent MCP server providing AI assistants with powerful tools and resources for Appium mobile automation",
|
|
5
5
|
"bugs": {
|
|
6
6
|
"url": "https://github.com/appium/appium-mcp/issues"
|
package/server.json
CHANGED
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
"name": "io.github.appium/appium-mcp",
|
|
4
4
|
"title": "MCP Appium - Mobile Development and Automation Server",
|
|
5
5
|
"description": "MCP server for Appium mobile automation on iOS and Android devices with test creation tools.",
|
|
6
|
-
"version": "1.95.
|
|
6
|
+
"version": "1.95.8",
|
|
7
7
|
"packages": [
|
|
8
8
|
{
|
|
9
9
|
"registryType": "npm",
|
|
10
10
|
"identifier": "appium-mcp",
|
|
11
|
-
"version": "1.95.
|
|
11
|
+
"version": "1.95.8",
|
|
12
12
|
"transport": {
|
|
13
13
|
"type": "stdio"
|
|
14
14
|
}
|
|
@@ -2,8 +2,9 @@ import {imageUtil} from '@appium/support';
|
|
|
2
2
|
import type {ContentResult} from 'fastmcp';
|
|
3
3
|
|
|
4
4
|
import {AIVisionFinder} from '../../../ai-finder/vision-finder.js';
|
|
5
|
-
import {getScreenshot} from '../../../command.js';
|
|
5
|
+
import {getScreenshot, getWindowSize} from '../../../command.js';
|
|
6
6
|
import log from '../../../logger.js';
|
|
7
|
+
import {getPlatformName, PLATFORM} from '../../../session-store.js';
|
|
7
8
|
import type {DriverInstance} from '../../../session-store.js';
|
|
8
9
|
import {errorResult, textResultWithPrimaryElementId, toolErrorMessage} from '../../tool-response.js';
|
|
9
10
|
import type {AIArgs} from '../schema.js';
|
|
@@ -35,10 +36,30 @@ export async function handleFindElement(driver: DriverInstance, args: AIArgs): P
|
|
|
35
36
|
const finder = getAIVisionFinder();
|
|
36
37
|
const result = await finder.findElement(screenshotBase64, instruction, width, height);
|
|
37
38
|
|
|
39
|
+
// Vision results and annotations use screenshot pixels, but iOS touch actions
|
|
40
|
+
// use logical screen coordinates. Do not mutate the finder's cached result.
|
|
41
|
+
let center = result.center;
|
|
42
|
+
let bbox = result.bbox;
|
|
43
|
+
if (getPlatformName(driver) === PLATFORM.ios) {
|
|
44
|
+
const screen = await getWindowSize(driver);
|
|
45
|
+
if (!(screen.width > 0 && screen.height > 0)) {
|
|
46
|
+
throw new Error('Failed to get screen dimensions for AI coordinate mapping');
|
|
47
|
+
}
|
|
48
|
+
const scaleX = screen.width / width;
|
|
49
|
+
const scaleY = screen.height / height;
|
|
50
|
+
center = {x: Math.floor(center.x * scaleX), y: Math.floor(center.y * scaleY)};
|
|
51
|
+
bbox = [
|
|
52
|
+
Math.floor(bbox[0] * scaleX),
|
|
53
|
+
Math.floor(bbox[1] * scaleY),
|
|
54
|
+
Math.ceil(bbox[2] * scaleX),
|
|
55
|
+
Math.ceil(bbox[3] * scaleY),
|
|
56
|
+
];
|
|
57
|
+
}
|
|
58
|
+
|
|
38
59
|
// Format: "ai-element:{x},{y}:{bbox}" — consumed by appium_gesture handlers.
|
|
39
|
-
const elementUUID = `ai-element:${
|
|
60
|
+
const elementUUID = `ai-element:${center.x},${center.y}:${bbox.join(',')}`;
|
|
40
61
|
|
|
41
|
-
let detail = `Successfully found "${result.target}" at coordinates (${
|
|
62
|
+
let detail = `Successfully found "${result.target}" at coordinates (${center.x}, ${center.y}) using AI vision.`;
|
|
42
63
|
if (result.annotatedImagePath) {
|
|
43
64
|
detail += ` Vision image: ${result.annotatedImagePath}`;
|
|
44
65
|
}
|
|
@@ -9,7 +9,8 @@ import type {Rect} from '@appium/types';
|
|
|
9
9
|
*
|
|
10
10
|
* where (cx, cy) is the visual centre of the target and (x0,y0,x1,y1) is
|
|
11
11
|
* the bounding box returned by the vision model. Both are pixel
|
|
12
|
-
* coordinates in the
|
|
12
|
+
* coordinates in the driver's touch coordinate space (logical screen
|
|
13
|
+
* coordinates on iOS), NOT real WebDriver
|
|
13
14
|
* element ids — so they MUST NOT be passed to driver.getElementRect or
|
|
14
15
|
* to platform-native commands like `mobile: doubleTap`, `mobile: pinch`,
|
|
15
16
|
* etc., which require a real element id.
|
|
@@ -48,7 +49,7 @@ export type ParsedAiElement = {
|
|
|
48
49
|
* makes start/end identical after rounding. Keep a minimum inset area so those
|
|
49
50
|
* gestures still move visibly while staying centred on the parsed point.
|
|
50
51
|
*
|
|
51
|
-
* Values are
|
|
52
|
+
* Values are touch coordinates; callers can clamp against the window if needed.
|
|
52
53
|
*/
|
|
53
54
|
const AI_FALLBACK_RECT_WIDTH = 100;
|
|
54
55
|
const AI_FALLBACK_RECT_HEIGHT = 100;
|