appium-mcp 1.74.2 → 1.74.3

package/src/tools/gestures/handlers/ai-element.ts

@@ -0,0 +1,140 @@
+/**
+ * Helpers for the special "ai-element:" UUIDs produced by the appium_ai
+ * find_element handler.
+ *
+ * The format is:
+ *
+ *   ai-element:<cx>,<cy>:<x0>,<y0>,<x1>,<y1>
+ *
+ * where (cx, cy) is the visual centre of the target and (x0,y0,x1,y1) is
+ * the bounding box returned by the vision model. Both are pixel
+ * coordinates in the screenshot's coordinate space, NOT real WebDriver
+ * element ids — so they MUST NOT be passed to driver.getElementRect or
+ * to platform-native commands like `mobile: doubleTap`, `mobile: pinch`,
+ * etc., which require a real element id.
+ *
+ * Centralising the prefix, the parser, and the rect-resolution helper
+ * here keeps every gesture handler honest: if it ever needs a rect for
+ * a UUID, it goes through `resolveTargetRect` and gets the right thing
+ * for both AI and traditional UUIDs.
+ */
+import type { ContentResult } from 'fastmcp';
+import type { Rect } from '@appium/types';
+import { getElementRect } from '../../../command.js';
+import type { DriverInstance } from '../../../session-store.js';
+import { errorResult } from '../../tool-response.js';
+
+export const AI_ELEMENT_PREFIX = 'ai-element:';
+
+export const AI_DISABLED_REJECTION =
+  `Received an ai-element: UUID, but the appium_ai tool is not registered ` +
+  `(AI_VISION_ENABLED is not set to true). Use appium_find_element to get a real ` +
+  `element UUID, or enable AI_VISION_ENABLED=true with the required AI_VISION_* keys.`;
+
+export type ParsedAiElement = {
+  center: { x: number; y: number };
+  rect: Rect;
+};
+
+/**
+ * When centre-only AI UUIDs have no bbox, direction-based swipe/scroll uses
+ * `rect.width` / `rect.height` with 0.2 / 0.8 edges; an effective 1x1 rect
+ * makes start/end identical after rounding. Keep a minimum inset area so those
+ * gestures still move visibly while staying centred on the parsed point.
+ *
+ * Values are screenshot pixels; callers can clamp against the window if needed.
+ */
+const AI_FALLBACK_RECT_WIDTH = 100;
+const AI_FALLBACK_RECT_HEIGHT = 100;
+
+export function isAiElementUUID(uuid: string | undefined): uuid is string {
+  return typeof uuid === 'string' && uuid.startsWith(AI_ELEMENT_PREFIX);
+}
+
+/**
+ * Parse an `ai-element:` UUID into a centre point and a synthetic rect.
+ *
+ * If the bbox segment is present and well-formed, the rect describes the
+ * full bounding box. Otherwise we fall back to a small rect centred on
+ * `(cx, cy)`, so directional swipe/scroll still has usable start/end deltas.
+ */
+export function parseAiElement(
+  uuid: string
+): ParsedAiElement | { error: string } {
+  const parts = uuid.split(':');
+  if (parts.length < 2) {
+    return { error: 'Invalid ai-element UUID: missing coordinate segment.' };
+  }
+
+  const [cxStr, cyStr] = (parts[1] ?? '').split(',');
+  const cx = Number.parseInt(cxStr ?? '', 10);
+  const cy = Number.parseInt(cyStr ?? '', 10);
+  if (!Number.isFinite(cx) || !Number.isFinite(cy)) {
+    return {
+      error: `Invalid ai-element UUID: centre coordinates are not numbers ('${parts[1]}').`,
+    };
+  }
+
+  let rect = rectAroundCenter(cx, cy);
+  if (parts[2]) {
+    const bbox = parts[2].split(',').map((v) => Number.parseInt(v, 10));
+    if (
+      bbox.length === 4 &&
+      bbox.every((n) => Number.isFinite(n)) &&
+      bbox[2] > bbox[0] &&
+      bbox[3] > bbox[1]
+    ) {
+      rect = {
+        x: bbox[0],
+        y: bbox[1],
+        width: bbox[2] - bbox[0],
+        height: bbox[3] - bbox[1],
+      };
+    }
+  }
+
+  return { center: { x: cx, y: cy }, rect };
+}
+
+/**
+ * Single dispatcher for "give me a rect for this UUID":
+ *   - ai-element UUID → rect synthesised from the bbox / centre fallback, no driver call
+ *   - traditional UUID → `getElementRect` on the driver
+ *
+ * Returns `{ error }` for malformed AI UUIDs or for driver lookup failures (`getElementRect`
+ * rejects). Call sites avoid try/catch and handle both cases the same way.
+ */
+export async function resolveTargetRect(
+  driver: DriverInstance,
+  elementUUID: string
+): Promise<Rect | { error: string }> {
+  if (isAiElementUUID(elementUUID)) {
+    const parsed = parseAiElement(elementUUID);
+    if ('error' in parsed) {
+      return parsed;
+    }
+    return parsed.rect;
+  }
+  try {
+    return await getElementRect(driver, elementUUID);
+  } catch (err: unknown) {
+    return {
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+export function aiDisabledResult(): ContentResult {
+  return errorResult(AI_DISABLED_REJECTION);
+}
+
+function rectAroundCenter(cx: number, cy: number): Rect {
+  const w = AI_FALLBACK_RECT_WIDTH;
+  const h = AI_FALLBACK_RECT_HEIGHT;
+  return {
+    x: cx - Math.floor(w / 2),
+    y: cy - Math.floor(h / 2),
+    width: w,
+    height: h,
+  };
+}

package/src/tools/gestures/handlers/pinch.ts

@@ -1,17 +1,18 @@
 import type { ContentResult } from 'fastmcp';
 import type { DriverInstance } from '../../../session-store.js';
 import { getPlatformName, PLATFORM } from '../../../session-store.js';
-import {
-  execute,
-  getElementRect,
-  getWindowRect,
-  performActions,
-} from '../../../command.js';
+import { execute, getWindowRect, performActions } from '../../../command.js';
 import {
   errorResult,
   textResult,
   toolErrorMessage,
 } from '../../tool-response.js';
+import { isAIEnabled } from '../../ai/config.js';
+import {
+  aiDisabledResult,
+  isAiElementUUID,
+  resolveTargetRect,
+} from './ai-element.js';
 import type { GestureArgs } from '../schema.js';
 
 const DEFAULT_VELOCITY = 2.2;
@@ -96,6 +97,14 @@ export async function handlePinchZoom(
   }
   const useCustomCoords =
     !args.elementUUID && args.x !== undefined && args.y !== undefined;
+  // ai-element UUIDs are coordinate UUIDs, not real element ids: drive the
+  // gesture from the bbox and skip the iOS/Android native paths that need
+  // a real `elementId`.
+  const isAiUUID = isAiElementUUID(args.elementUUID);
+  if (isAiUUID && !isAIEnabled()) {
+    return aiDisabledResult();
+  }
+  const useCoordPath = useCustomCoords || isAiUUID;
 
   try {
     const platform = getPlatformName(driver);
@@ -107,7 +116,10 @@ export async function handlePinchZoom(
 
     if (args.elementUUID) {
       windowRect = await getWindowRect(driver);
-      const rect = await getElementRect(driver, args.elementUUID);
+      const rect = await resolveTargetRect(driver, args.elementUUID);
+      if ('error' in rect) {
+        return errorResult(rect.error);
+      }
       ({ cx, cy, spread } = resolveElementPinchTarget(rect, windowRect));
     } else if (useCustomCoords) {
       windowRect = await getWindowRect(driver);
@@ -151,10 +163,11 @@ export async function handlePinchZoom(
           ],
         },
       ]);
-    } else if (useCustomCoords && platform === PLATFORM.android) {
-      // Zoom in at a custom center on Android: use the native pinchOpenGesture
-      // with a region centered at (cx, cy). spread is pre-clamped so the region
-      // fits within the window.
+    } else if (useCoordPath && platform === PLATFORM.android) {
+      // Zoom in at a coordinate-driven center on Android: use the native
+      // pinchOpenGesture with a region centered at (cx, cy). spread is
+      // pre-clamped so the region fits within the window. Also covers
+      // ai-element UUIDs, whose synthetic rect is used to pick the region.
       const percent = Math.min(0.99, 1 - 1 / scale);
       await execute(driver, 'mobile: pinchOpenGesture', {
         left: cx - spread,
@@ -163,8 +176,10 @@ export async function handlePinchZoom(
         height: 2 * spread,
         percent,
       });
-    } else if (useCustomCoords) {
-      // Zoom in at a custom center on iOS using W3C Actions
+    } else if (useCoordPath) {
+      // Zoom in at a coordinate-driven center on iOS using W3C Actions.
+      // Same path is used for ai-element UUIDs because `mobile: pinch`
+      // requires a real elementId we don't have.
       const startSpread = Math.max(1, Math.floor(spread / scale));
       const endSpread = spread;
       const duration = Math.floor((1 / Math.abs(velocity)) * 1000);
@@ -224,11 +239,13 @@ export async function handlePinchZoom(
     }
 
     const direction = scale < 1 ? 'out' : 'in';
-    const target = args.elementUUID
-      ? `element ${args.elementUUID}`
-      : useCustomCoords
-        ? `coordinates (${cx}, ${cy})`
-        : 'screen';
+    const target = isAiUUID
+      ? `AI element coordinates (${cx}, ${cy})`
+      : args.elementUUID
+        ? `element ${args.elementUUID}`
+        : useCustomCoords
+          ? `coordinates (${cx}, ${cy})`
+          : 'screen';
     return textResult(
       `Successfully pinched ${direction} (scale=${scale}) on ${target}.`
     );

package/src/tools/gestures/handlers/swipe-scroll.ts

@@ -1,17 +1,18 @@
 import type { ContentResult } from 'fastmcp';
 import type { DriverInstance } from '../../../session-store.js';
 import { getPlatformName, PLATFORM } from '../../../session-store.js';
-import {
-  execute,
-  getElementRect,
-  getWindowRect,
-  performActions,
-} from '../../../command.js';
+import { execute, getWindowRect, performActions } from '../../../command.js';
 import {
   errorResult,
   textResult,
   toolErrorMessage,
 } from '../../tool-response.js';
+import { isAIEnabled } from '../../ai/config.js';
+import {
+  aiDisabledResult,
+  isAiElementUUID,
+  resolveTargetRect,
+} from './ai-element.js';
 import type { GestureArgs } from '../schema.js';
 
 type Direction = 'up' | 'down' | 'left' | 'right';
@@ -123,6 +124,9 @@ export async function handleScroll(
   args: GestureArgs
 ): Promise<ContentResult> {
   try {
+    if (isAiElementUUID(args.elementUUID) && !isAIEnabled()) {
+      return aiDisabledResult();
+    }
     const platform = getPlatformName(driver);
     // Android scroll follows the scrollbar convention (down = reveal content below),
     // so flip the user's direction before computing the W3C drag coords.
@@ -164,6 +168,9 @@ export async function handleSwipe(
   args: GestureArgs
 ): Promise<ContentResult> {
   try {
+    if (isAiElementUUID(args.elementUUID) && !isAIEnabled()) {
+      return aiDisabledResult();
+    }
     const coordsResult = await resolveCoords(driver, args);
     if ('error' in coordsResult) {
       return errorResult(`swipe: ${coordsResult.error}`);
@@ -252,7 +259,13 @@ async function resolveCoords(
 ): Promise<Coords | { error: string }> {
   if (args.direction) {
     if (args.elementUUID) {
-      const rect = await getElementRect(driver, args.elementUUID);
+      // ai-element UUIDs are coordinate UUIDs, not real element ids; their
+      // rect is synthesised from the bbox in the UUID itself rather than
+      // fetched from the driver.
+      const rect = await resolveTargetRect(driver, args.elementUUID);
+      if ('error' in rect) {
+        return rect;
+      }
       return coordsForDirection(args.direction, {
         x: rect.x,
         y: rect.y,

package/src/tools/gestures/handlers/tap.ts

@@ -14,33 +14,32 @@ import {
   toolErrorMessage,
 } from '../../tool-response.js';
 import { isAIEnabled } from '../../ai/config.js';
+import {
+  aiDisabledResult,
+  isAiElementUUID,
+  parseAiElement,
+} from './ai-element.js';
 import type { GestureArgs } from '../schema.js';
 
-const AI_ELEMENT_PREFIX = 'ai-element:';
-
-const AI_DISABLED_REJECTION =
-  `Received an ai-element: UUID, but the appium_ai tool is not registered ` +
-  `(AI_VISION_ENABLED is not set to true). Use appium_find_element to get a real ` +
-  `element UUID, or enable AI_VISION_ENABLED=true with the required AI_VISION_* keys.`;
-
 export async function handleTap(
   driver: DriverInstance,
   args: GestureArgs
 ): Promise<ContentResult> {
   try {
     if (args.elementUUID) {
-      if (args.elementUUID.startsWith(AI_ELEMENT_PREFIX)) {
+      if (isAiElementUUID(args.elementUUID)) {
         if (!isAIEnabled()) {
-          return errorResult(AI_DISABLED_REJECTION);
+          return aiDisabledResult();
         }
-        const parsed = parseAiElementCoords(args.elementUUID);
+        const parsed = parseAiElement(args.elementUUID);
         if ('error' in parsed) {
           return errorResult(parsed.error);
         }
-        await performActions(driver, w3cTapAt(parsed.x, parsed.y));
+        const { x, y } = parsed.center;
+        await performActions(driver, w3cTapAt(x, y));
         return textResultWithPrimaryElementId(
           args.elementUUID,
-          `Successfully tapped at AI element coordinates (${parsed.x}, ${parsed.y}).`
+          `Successfully tapped at AI element coordinates (${x}, ${y}).`
         );
       }
       await elementClick(driver, args.elementUUID);
@@ -77,9 +76,22 @@ export async function handleDoubleTap(
     // other JS-bridged) gesture handlers. Resolve elements to coordinates
     // and use the same W3C sequence everywhere, verified against vodqa.
     if (args.elementUUID) {
-      const coords = await resolveCoordsFromElement(driver, args.elementUUID);
-      x = coords.x;
-      y = coords.y;
+      if (isAiElementUUID(args.elementUUID)) {
+        if (!isAIEnabled()) {
+          return aiDisabledResult();
+        }
+        const parsed = parseAiElement(args.elementUUID);
+        if ('error' in parsed) {
+          return errorResult(parsed.error);
+        }
+        // ai-element UUIDs are coordinates, not real element ids — use W3C
+        // double-tap at the parsed centre (same as main for real elements).
+        ({ x, y } = parsed.center);
+      } else {
+        const coords = await resolveCoordsFromElement(driver, args.elementUUID);
+        x = coords.x;
+        y = coords.y;
+      }
     } else if (args.x !== undefined && args.y !== undefined) {
       x = args.x;
       y = args.y;
@@ -128,24 +140,37 @@ export async function handleLongPress(
     let y: number;
 
     if (args.elementUUID) {
-      const platform = getPlatformName(driver);
-      if (platform === PLATFORM.ios) {
-        try {
-          await execute(driver, 'mobile: touchAndHold', {
-            elementId: args.elementUUID,
-            duration: duration / 1000,
-          });
-          return textResultWithPrimaryElementId(
-            args.elementUUID,
-            `Successfully long pressed element ${args.elementUUID} for ${duration}ms.`
-          );
-        } catch {
-          // fall through to W3C Actions fallback
+      if (isAiElementUUID(args.elementUUID)) {
+        if (!isAIEnabled()) {
+          return aiDisabledResult();
         }
+        const parsed = parseAiElement(args.elementUUID);
+        if ('error' in parsed) {
+          return errorResult(parsed.error);
+        }
+        // ai-element UUIDs are coordinates, not real element ids — skip the
+        // iOS `mobile: touchAndHold` fast-path and long-press at the parsed centre.
+        ({ x, y } = parsed.center);
+      } else {
+        const platform = getPlatformName(driver);
+        if (platform === PLATFORM.ios) {
+          try {
+            await execute(driver, 'mobile: touchAndHold', {
+              elementId: args.elementUUID,
+              duration: duration / 1000,
+            });
+            return textResultWithPrimaryElementId(
+              args.elementUUID,
+              `Successfully long pressed element ${args.elementUUID} for ${duration}ms.`
+            );
+          } catch {
+            // fall through to W3C Actions fallback
+          }
+        }
+        const coords = await resolveCoordsFromElement(driver, args.elementUUID);
+        x = coords.x;
+        y = coords.y;
       }
-      const coords = await resolveCoordsFromElement(driver, args.elementUUID);
-      x = coords.x;
-      y = coords.y;
     } else if (args.x !== undefined && args.y !== undefined) {
       x = args.x;
       y = args.y;
@@ -178,25 +203,6 @@ export async function handleLongPress(
   }
 }
 
-function parseAiElementCoords(
-  uuid: string
-): { x: number; y: number } | { error: string } {
-  const parts = uuid.split(':');
-  if (parts.length < 2) {
-    return { error: 'Invalid AI element UUID format.' };
-  }
-  const coords = parts[1].split(',');
-  if (coords.length < 2) {
-    return { error: 'Invalid AI element coordinates format.' };
-  }
-  const x = parseInt(coords[0], 10);
-  const y = parseInt(coords[1], 10);
-  if (isNaN(x) || isNaN(y)) {
-    return { error: 'Invalid AI element coordinates: not numbers.' };
-  }
-  return { x, y };
-}
-
 function w3cTapAt(x: number, y: number) {
   return [
     {