appium-mcp 1.95.7 → 1.95.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## [1.95.8](https://github.com/appium/appium-mcp/compare/v1.95.7...v1.95.8) (2026-10-10)
2
+
3
+ ### Bug Fixes
4
+
5
+ * map AI vision coordinates to iOS touch space ([#564](https://github.com/appium/appium-mcp/issues/564)) ([2ed139e](https://github.com/appium/appium-mcp/commit/2ed139ee6c11a74f2a67fcdb03ec9951752fa92f))
6
+
1
7
  ## [1.95.7](https://github.com/appium/appium-mcp/compare/v1.95.6...v1.95.7) (2026-10-10)
2
8
 
3
9
  ### Bug Fixes
@@ -1 +1 @@
1
- {"version":3,"file":"find-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,SAAS,CAAC;AAK3C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,cAAc,CAAC;AAMzC,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAqCpG"}
1
+ {"version":3,"file":"find-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,SAAS,CAAC;AAM3C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,cAAc,CAAC;AAMzC,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAyDpG"}
@@ -1,7 +1,8 @@
1
1
  import { imageUtil } from '@appium/support';
2
2
  import { AIVisionFinder } from '../../../ai-finder/vision-finder.js';
3
- import { getScreenshot } from '../../../command.js';
3
+ import { getScreenshot, getWindowSize } from '../../../command.js';
4
4
  import log from '../../../logger.js';
5
+ import { getPlatformName, PLATFORM } from '../../../session-store.js';
5
6
  import { errorResult, textResultWithPrimaryElementId, toolErrorMessage } from '../../tool-response.js';
6
7
  // Module-level singleton: ensures the LRU cache persists across tool calls.
7
8
  // Creating a new AIVisionFinder() on every call would reset the cache each time.
@@ -22,9 +23,28 @@ export async function handleFindElement(driver, args) {
22
23
  const { width, height } = metadata;
23
24
  const finder = getAIVisionFinder();
24
25
  const result = await finder.findElement(screenshotBase64, instruction, width, height);
26
+ // Vision results and annotations use screenshot pixels, but iOS touch actions
27
+ // use logical screen coordinates. Do not mutate the finder's cached result.
28
+ let center = result.center;
29
+ let bbox = result.bbox;
30
+ if (getPlatformName(driver) === PLATFORM.ios) {
31
+ const screen = await getWindowSize(driver);
32
+ if (!(screen.width > 0 && screen.height > 0)) {
33
+ throw new Error('Failed to get screen dimensions for AI coordinate mapping');
34
+ }
35
+ const scaleX = screen.width / width;
36
+ const scaleY = screen.height / height;
37
+ center = { x: Math.floor(center.x * scaleX), y: Math.floor(center.y * scaleY) };
38
+ bbox = [
39
+ Math.floor(bbox[0] * scaleX),
40
+ Math.floor(bbox[1] * scaleY),
41
+ Math.ceil(bbox[2] * scaleX),
42
+ Math.ceil(bbox[3] * scaleY),
43
+ ];
44
+ }
25
45
  // Format: "ai-element:{x},{y}:{bbox}" — consumed by appium_gesture handlers.
26
- const elementUUID = `ai-element:${result.center.x},${result.center.y}:${result.bbox.join(',')}`;
27
- let detail = `Successfully found "${result.target}" at coordinates (${result.center.x}, ${result.center.y}) using AI vision.`;
46
+ const elementUUID = `ai-element:${center.x},${center.y}:${bbox.join(',')}`;
47
+ let detail = `Successfully found "${result.target}" at coordinates (${center.x}, ${center.y}) using AI vision.`;
28
48
  if (result.annotatedImagePath) {
29
49
  detail += ` Vision image: ${result.annotatedImagePath}`;
30
50
  }
@@ -1 +1 @@
1
- {"version":3,"file":"find-element.js","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAG1C,OAAO,EAAC,cAAc,EAAC,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAClD,OAAO,GAAG,MAAM,oBAAoB,CAAC;AAErC,OAAO,EAAC,WAAW,EAAE,8BAA8B,EAAE,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAGrG,4EAA4E;AAC5E,iFAAiF;AACjF,IAAI,eAAe,GAA0B,IAAI,CAAC;AAElD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAsB,EAAE,IAAY;IAC1E,uEAAuE;IACvE,8EAA8E;IAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,WAAqB,CAAC;IAE/C,IAAI,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,+CAA+C,WAAW,GAAG,CAAC,CAAC;QAExE,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QAErD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,YAAY,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC;QAErD,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,EAAC,KAAK,EAAE,MAAM,EAAC,GAAG,QAAQ,CAAC;QAEjC,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,gBAAgB,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAEtF,6EAA6E;QAC7E,MAAM,WAAW,GAAG,cAAc,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAEhG,IAAI,MAAM,GAAG,uBAAuB,MAAM,CAAC,MAAM,qBAAqB,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC;QAC9H,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;YAC9B,MAAM,IAAI,kBAAkB,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC1D,CAAC;QAED,OAAO,8BAA8B,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC3C,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,YAAY,CAAC,CAAC;QACnD,OAAO,WAAW,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,eAAe,GAAG,IAAI,cAAc,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC"}
1
+ {"version":3,"file":"find-element.js","sourceRoot":"","sources":["../../../../src/tools/ai/handlers/find-element.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAG1C,OAAO,EAAC,cAAc,EAAC,MAAM,qCAAqC,CAAC;AACnE,OAAO,EAAC,aAAa,EAAE,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACjE,OAAO,GAAG,MAAM,oBAAoB,CAAC;AACrC,OAAO,EAAC,eAAe,EAAE,QAAQ,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,8BAA8B,EAAE,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAGrG,4EAA4E;AAC5E,iFAAiF;AACjF,IAAI,eAAe,GAA0B,IAAI,CAAC;AAElD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAsB,EAAE,IAAY;IAC1E,uEAAuE;IACvE,8EAA8E;IAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,WAAqB,CAAC;IAE/C,IAAI,CAAC;QACH,GAAG,CAAC,IAAI,CAAC,+CAA+C,WAAW,GAAG,CAAC,CAAC;QAExE,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QAErD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,YAAY,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC;QAErD,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,EAAC,KAAK,EAAE,MAAM,EAAC,GAAG,QAAQ,CAAC;QAEjC,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,gBAAgB,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAEtF,8EAA8E;QAC9E,4EAA4E;QAC5E,IAAI,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC3B,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACvB,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,GAAG,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;YAC3C,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAC/E,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;YACtC,MAAM,GAAG,EAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,EAAC,CAAC;YAC9E,IAAI,GAAG;gBACL,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;aAC5B,CAAC;QACJ,CAAC;QAED,6EAA6E;QAC7E,MAAM,WAAW,GAAG,cAAc,MAAM,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAE3E,IAAI,MAAM,GAAG,uBAAuB,MAAM,CAAC,MAAM,qBAAqB,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,oBAAoB,CAAC;QAChH,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;YAC9B,MAAM,IAAI,kBAAkB,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC1D,CAAC;QAED,OAAO,8BAA8B,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC3C,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,YAAY,CAAC,CAAC;QACnD,OAAO,WAAW,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,eAAe,GAAG,IAAI,cAAc,EAAE,CAAC;IACzC,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC"}
@@ -9,7 +9,8 @@ import type { Rect } from '@appium/types';
9
9
  *
10
10
  * where (cx, cy) is the visual centre of the target and (x0,y0,x1,y1) is
11
11
  * the bounding box returned by the vision model. Both are pixel
12
- * coordinates in the screenshot's coordinate space, NOT real WebDriver
12
+ * coordinates in the driver's touch coordinate space (logical screen
13
+ * coordinates on iOS), NOT real WebDriver
13
14
  * element ids — so they MUST NOT be passed to driver.getElementRect or
14
15
  * to platform-native commands like `mobile: doubleTap`, `mobile: pinch`,
15
16
  * etc., which require a real element id.
@@ -1 +1 @@
1
- {"version":3,"file":"ai-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/ai-element.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,eAAe,CAAC;AACxC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,SAAS,CAAC;AAG3C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,2BAA2B,CAAC;AAG9D,eAAO,MAAM,iBAAiB,gBAAgB,CAAC;AAE/C,eAAO,MAAM,qBAAqB,QAGoD,CAAC;AAEvF,eAAO,MAAM,sBAAsB,QAGgD,CAAC;AAEpF,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAC,CAAC;IAC/B,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAaF,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,IAAI,MAAM,CAExE;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,CA6B9E;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,CAAC,CAepH;AAED,wBAAgB,gBAAgB,IAAI,aAAa,CAEhD;AAED,wBAAgB,gCAAgC,IAAI,aAAa,CAEhE;AAED,wBAAgB,mCAAmC,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,GAAG,SAAS,CAK9G"}
1
+ {"version":3,"file":"ai-element.d.ts","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/ai-element.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,eAAe,CAAC;AACxC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,SAAS,CAAC;AAG3C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,2BAA2B,CAAC;AAG9D,eAAO,MAAM,iBAAiB,gBAAgB,CAAC;AAE/C,eAAO,MAAM,qBAAqB,QAGoD,CAAC;AAEvF,eAAO,MAAM,sBAAsB,QAGgD,CAAC;AAEpF,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAC,CAAC;IAC/B,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAaF,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,IAAI,MAAM,CAExE;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,CA6B9E;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,CAAC,CAepH;AAED,wBAAgB,gBAAgB,IAAI,aAAa,CAEhD;AAED,wBAAgB,gCAAgC,IAAI,aAAa,CAEhE;AAED,wBAAgB,mCAAmC,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,GAAG,SAAS,CAK9G"}
@@ -13,7 +13,7 @@ export const AI_WEBDRIVER_REJECTION = `That is an ai-element token from appium_a
13
13
  * makes start/end identical after rounding. Keep a minimum inset area so those
14
14
  * gestures still move visibly while staying centred on the parsed point.
15
15
  *
16
- * Values are screenshot pixels; callers can clamp against the window if needed.
16
+ * Values are touch coordinates; callers can clamp against the window if needed.
17
17
  */
18
18
  const AI_FALLBACK_RECT_WIDTH = 100;
19
19
  const AI_FALLBACK_RECT_HEIGHT = 100;
@@ -1 +1 @@
1
- {"version":3,"file":"ai-element.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/ai-element.ts"],"names":[],"mappings":"AAuBA,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAC,WAAW,EAAC,MAAM,wBAAwB,CAAC;AAEnD,MAAM,CAAC,MAAM,iBAAiB,GAAG,aAAa,CAAC;AAE/C,MAAM,CAAC,MAAM,qBAAqB,GAChC,yEAAyE;IACzE,gFAAgF;IAChF,oFAAoF,CAAC;AAEvF,MAAM,CAAC,MAAM,sBAAsB,GACjC,0EAA0E;IAC1E,sEAAsE;IACtE,iFAAiF,CAAC;AAOpF;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAEpC,MAAM,UAAU,eAAe,CAAC,IAAwB;IACtD,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;AACxE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,EAAC,KAAK,EAAE,sDAAsD,EAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QACjD,OAAO;YACL,KAAK,EAAE,iEAAiE,KAAK,CAAC,CAAC,CAAC,KAAK;SACtF,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,GAAG,gBAAgB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACpC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACzG,IAAI,GAAG;gBACL,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;gBACV,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;gBACV,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACxB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;aAC1B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAC,MAAM,EAAE,EAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAC,EAAE,IAAI,EAAC,CAAC;AACxC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAsB,EAAE,WAAmB;IACjF,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IACD,IAAI,CAAC;QACH,OAAO,MAAM,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,WAAW,CAAC,qBAAqB,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,gCAAgC;IAC9C,OAAO,WAAW,CAAC,sBAAsB,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,WAA+B;IACjF,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;QACjC,OAAO,gCAAgC,EAAE,CAAC;IAC5C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAU,EAAE,EAAU;IAC9C,MAAM,CAAC,GAAG,sBAAsB,CAAC;IACjC,MAAM,CAAC,GAAG,uBAAuB,CAAC;IAClC,OAAO;QACL,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,CAAC;KACV,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"ai-element.js","sourceRoot":"","sources":["../../../../src/tools/gestures/handlers/ai-element.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAC,WAAW,EAAC,MAAM,wBAAwB,CAAC;AAEnD,MAAM,CAAC,MAAM,iBAAiB,GAAG,aAAa,CAAC;AAE/C,MAAM,CAAC,MAAM,qBAAqB,GAChC,yEAAyE;IACzE,gFAAgF;IAChF,oFAAoF,CAAC;AAEvF,MAAM,CAAC,MAAM,sBAAsB,GACjC,0EAA0E;IAC1E,sEAAsE;IACtE,iFAAiF,CAAC;AAOpF;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAEpC,MAAM,UAAU,eAAe,CAAC,IAAwB;IACtD,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;AACxE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,EAAC,KAAK,EAAE,sDAAsD,EAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QACjD,OAAO;YACL,KAAK,EAAE,iEAAiE,KAAK,CAAC,CAAC,CAAC,KAAK;SACtF,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,GAAG,gBAAgB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACpC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACzG,IAAI,GAAG;gBACL,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;gBACV,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;gBACV,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACxB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;aAC1B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAC,MAAM,EAAE,EAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAC,EAAE,IAAI,EAAC,CAAC;AACxC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAsB,EAAE,WAAmB;IACjF,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IACD,IAAI,CAAC;QACH,OAAO,MAAM,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO;YACL,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,WAAW,CAAC,qBAAqB,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,gCAAgC;IAC9C,OAAO,WAAW,CAAC,sBAAsB,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,WAA+B;IACjF,IAAI,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;QACjC,OAAO,gCAAgC,EAAE,CAAC;IAC5C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAU,EAAE,EAAU;IAC9C,MAAM,CAAC,GAAG,sBAAsB,CAAC;IACjC,MAAM,CAAC,GAAG,uBAAuB,CAAC;IAClC,OAAO;QACL,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;QACzB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,CAAC;KACV,CAAC;AACJ,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "appium-mcp",
3
- "version": "1.95.7",
3
+ "version": "1.95.8",
4
4
  "description": "Intelligent MCP server providing AI assistants with powerful tools and resources for Appium mobile automation",
5
5
  "bugs": {
6
6
  "url": "https://github.com/appium/appium-mcp/issues"
package/server.json CHANGED
@@ -3,12 +3,12 @@
3
3
  "name": "io.github.appium/appium-mcp",
4
4
  "title": "MCP Appium - Mobile Development and Automation Server",
5
5
  "description": "MCP server for Appium mobile automation on iOS and Android devices with test creation tools.",
6
- "version": "1.95.7",
6
+ "version": "1.95.8",
7
7
  "packages": [
8
8
  {
9
9
  "registryType": "npm",
10
10
  "identifier": "appium-mcp",
11
- "version": "1.95.7",
11
+ "version": "1.95.8",
12
12
  "transport": {
13
13
  "type": "stdio"
14
14
  }
@@ -2,8 +2,9 @@ import {imageUtil} from '@appium/support';
2
2
  import type {ContentResult} from 'fastmcp';
3
3
 
4
4
  import {AIVisionFinder} from '../../../ai-finder/vision-finder.js';
5
- import {getScreenshot} from '../../../command.js';
5
+ import {getScreenshot, getWindowSize} from '../../../command.js';
6
6
  import log from '../../../logger.js';
7
+ import {getPlatformName, PLATFORM} from '../../../session-store.js';
7
8
  import type {DriverInstance} from '../../../session-store.js';
8
9
  import {errorResult, textResultWithPrimaryElementId, toolErrorMessage} from '../../tool-response.js';
9
10
  import type {AIArgs} from '../schema.js';
@@ -35,10 +36,30 @@ export async function handleFindElement(driver: DriverInstance, args: AIArgs): P
35
36
  const finder = getAIVisionFinder();
36
37
  const result = await finder.findElement(screenshotBase64, instruction, width, height);
37
38
 
39
+ // Vision results and annotations use screenshot pixels, but iOS touch actions
40
+ // use logical screen coordinates. Do not mutate the finder's cached result.
41
+ let center = result.center;
42
+ let bbox = result.bbox;
43
+ if (getPlatformName(driver) === PLATFORM.ios) {
44
+ const screen = await getWindowSize(driver);
45
+ if (!(screen.width > 0 && screen.height > 0)) {
46
+ throw new Error('Failed to get screen dimensions for AI coordinate mapping');
47
+ }
48
+ const scaleX = screen.width / width;
49
+ const scaleY = screen.height / height;
50
+ center = {x: Math.floor(center.x * scaleX), y: Math.floor(center.y * scaleY)};
51
+ bbox = [
52
+ Math.floor(bbox[0] * scaleX),
53
+ Math.floor(bbox[1] * scaleY),
54
+ Math.ceil(bbox[2] * scaleX),
55
+ Math.ceil(bbox[3] * scaleY),
56
+ ];
57
+ }
58
+
38
59
  // Format: "ai-element:{x},{y}:{bbox}" — consumed by appium_gesture handlers.
39
- const elementUUID = `ai-element:${result.center.x},${result.center.y}:${result.bbox.join(',')}`;
60
+ const elementUUID = `ai-element:${center.x},${center.y}:${bbox.join(',')}`;
40
61
 
41
- let detail = `Successfully found "${result.target}" at coordinates (${result.center.x}, ${result.center.y}) using AI vision.`;
62
+ let detail = `Successfully found "${result.target}" at coordinates (${center.x}, ${center.y}) using AI vision.`;
42
63
  if (result.annotatedImagePath) {
43
64
  detail += ` Vision image: ${result.annotatedImagePath}`;
44
65
  }
@@ -9,7 +9,8 @@ import type {Rect} from '@appium/types';
9
9
  *
10
10
  * where (cx, cy) is the visual centre of the target and (x0,y0,x1,y1) is
11
11
  * the bounding box returned by the vision model. Both are pixel
12
- * coordinates in the screenshot's coordinate space, NOT real WebDriver
12
+ * coordinates in the driver's touch coordinate space (logical screen
13
+ * coordinates on iOS), NOT real WebDriver
13
14
  * element ids — so they MUST NOT be passed to driver.getElementRect or
14
15
  * to platform-native commands like `mobile: doubleTap`, `mobile: pinch`,
15
16
  * etc., which require a real element id.
@@ -48,7 +49,7 @@ export type ParsedAiElement = {
48
49
  * makes start/end identical after rounding. Keep a minimum inset area so those
49
50
  * gestures still move visibly while staying centred on the parsed point.
50
51
  *
51
- * Values are screenshot pixels; callers can clamp against the window if needed.
52
+ * Values are touch coordinates; callers can clamp against the window if needed.
52
53
  */
53
54
  const AI_FALLBACK_RECT_WIDTH = 100;
54
55
  const AI_FALLBACK_RECT_HEIGHT = 100;