@midscene/ios 1.8.0 → 1.8.1

package/dist/es/mcp-server.mjs

@@ -4,8 +4,8 @@ import { MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE } from "@midscene/shared/env";
 import { getDebug } from "@midscene/shared/logger";
 import { mergeAndNormalizeAppNameMapping, normalizeForComparison } from "@midscene/shared/utils";
 import node_assert from "node:assert";
-import { getMidsceneLocationSchema, z } from "@midscene/core";
-import { defineAction, defineActionClearInput, defineActionCursorMove, defineActionDoubleClick, defineActionDragAndDrop, defineActionKeyboardPress, defineActionLongPress, defineActionPinch, defineActionScroll, defineActionSwipe, defineActionTap, normalizeMobileSwipeParam, normalizePinchParam } from "@midscene/core/device";
+import { z } from "@midscene/core";
+import { createDefaultMobileActions, defineAction } from "@midscene/core/device";
 import { sleep } from "@midscene/core/utils";
 import { DEFAULT_WDA_PORT } from "@midscene/shared/constants";
 import { createImgBase64ByFormat } from "@midscene/shared/img";
@@ -649,16 +649,6 @@ function _define_property(obj, key, value) {
     return obj;
 }
 const debugDevice = getDebug('ios:device');
-const iosInputParamSchema = z.object({
-    value: z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-    autoDismissKeyboard: z.boolean().optional().describe('Whether to dismiss the keyboard after input. Defaults to true if not specified. Set to false to keep the keyboard visible after input.'),
-    mode: z.preprocess((val)=>'append' === val ? 'typeOnly' : val, z["enum"]([
-        'replace',
-        'clear',
-        'typeOnly'
-    ]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - clear the field without inputting new text.')),
-    locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
-});
 const WDA_HTTP_METHODS = [
     'GET',
     'POST',
@@ -667,96 +657,39 @@ const WDA_HTTP_METHODS = [
 ];
 const DEFAULT_WDA_MJPEG_PORT = 9100;
 class IOSDevice {
+    async tapPoint(point) {
+        debugDevice(`tap at coordinates (${point.x}, ${point.y})`);
+        await this.wdaBackend.tap(Math.round(point.x), Math.round(point.y));
+    }
+    async doubleTapPoint(point) {
+        await this.wdaBackend.doubleTap(Math.round(point.x), Math.round(point.y));
+    }
+    async longPressPoint(point, duration = 1000) {
+        await this.wdaBackend.longPress(Math.round(point.x), Math.round(point.y), duration);
+    }
+    async swipePoint(start, end, duration = 500) {
+        await this.wdaBackend.swipe(Math.round(start.x), Math.round(start.y), Math.round(end.x), Math.round(end.y), duration);
+    }
+    async clearInputAt(point) {
+        if (point) {
+            await this.tapPoint(point);
+            await sleep(100);
+        }
+        debugDevice('Attempting to clear input with WebDriver Clear API');
+        const cleared = await this.wdaBackend.clearActiveElement();
+        cleared ? debugDevice('Successfully cleared input with WebDriver Clear API') : debugDevice('WebDriver Clear API returned false (no active element or clear failed)');
+    }
     actionSpace() {
+        const mobileActionContext = {
+            input: this.inputPrimitives,
+            size: ()=>this.size(),
+            sleep: async (timeMs)=>{
+                await sleep(timeMs);
+            },
+            getDefaultAutoDismissKeyboard: ()=>this.options?.autoDismissKeyboard
+        };
         const defaultActions = [
-            defineActionTap(async (param)=>{
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot tap');
-                await this.mouseClick(element.center[0], element.center[1]);
-            }),
-            defineActionDoubleClick(async (param)=>{
-                const element = param.locate;
-                node_assert(element, 'Element not found, cannot double click');
-                await this.doubleTap(element.center[0], element.center[1]);
-            }),
-            defineAction({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: iosInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    const element = param.locate;
-                    if ('typeOnly' !== param.mode) await this.clearInput(element);
-                    if ('clear' === param.mode) return;
-                    if (!param || !param.value) return;
-                    const autoDismissKeyboard = param.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
-                    await this.typeText(param.value, {
-                        autoDismissKeyboard
-                    });
-                }
-            }),
-            defineActionScroll(async (param)=>{
-                const element = param.locate;
-                const startingPoint = element ? {
-                    left: element.center[0],
-                    top: element.center[1]
-                } : void 0;
-                const scrollToEventName = param?.scrollType;
-                if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
-                else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
-                else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
-                else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
-                else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
-                else {
-                    if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance || void 0, startingPoint);
-                    else if ('left' === param.direction) await this.scrollLeft(param.distance || void 0, startingPoint);
-                    else if ('right' === param.direction) await this.scrollRight(param.distance || void 0, startingPoint);
-                    else throw new Error(`Unknown scroll direction: ${param.direction}`);
-                    else await this.scrollDown(param?.distance || void 0, startingPoint);
-                    await sleep(500);
-                }
-            }),
-            defineActionDragAndDrop(async (param)=>{
-                const from = param.from;
-                const to = param.to;
-                node_assert(from, 'missing "from" param for drag and drop');
-                node_assert(to, 'missing "to" param for drag and drop');
-                await this.swipe(from.center[0], from.center[1], to.center[0], to.center[1], 1000);
-            }),
-            defineActionSwipe(async (param)=>{
-                const { startPoint, endPoint, duration, repeatCount } = normalizeMobileSwipeParam(param, await this.size());
-                for(let i = 0; i < repeatCount; i++)await this.swipe(startPoint.x, startPoint.y, endPoint.x, endPoint.y, duration);
-            }),
-            defineActionKeyboardPress(async (param)=>{
-                await this.pressKey(param.keyName);
-            }),
-            defineActionCursorMove(async (param)=>{
-                const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
-                const times = param.times ?? 1;
-                for(let i = 0; i < times; i++){
-                    await this.pressKey(arrowKey);
-                    await sleep(100);
-                }
-            }),
-            defineActionLongPress(async (param)=>{
-                const element = param.locate;
-                node_assert(element, 'LongPress requires an element to be located');
-                const [x, y] = element.center;
-                await this.longPress(x, y, param?.duration);
-            }),
-            defineActionPinch(async (param)=>{
-                const { centerX, centerY, startDistance, endDistance, duration } = normalizePinchParam(param, await this.size());
-                await this.wdaBackend.pinch(centerX, centerY, startDistance, endDistance, duration);
-            }),
-            defineActionClearInput(async (param)=>{
-                await this.clearInput(param.locate);
-            })
+            ...createDefaultMobileActions(mobileActionContext)
         ];
         const platformSpecificActions = Object.values(createPlatformActions(this));
         const customActions = this.customActions || [];
@@ -766,6 +699,27 @@ class IOSDevice {
             ...customActions
         ];
     }
+    async performActionScroll(param) {
+        const element = param.locate;
+        const startingPoint = element ? {
+            left: element.center[0],
+            top: element.center[1]
+        } : void 0;
+        const scrollToEventName = param?.scrollType;
+        if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
+        else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
+        else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
+        else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
+        else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
+        else {
+            if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance || void 0, startingPoint);
+            else if ('left' === param.direction) await this.scrollLeft(param.distance || void 0, startingPoint);
+            else if ('right' === param.direction) await this.scrollRight(param.distance || void 0, startingPoint);
+            else throw new Error(`Unknown scroll direction: ${param.direction}`);
+            else await this.scrollDown(param?.distance || void 0, startingPoint);
+            await sleep(500);
+        }
+    }
     describe() {
         return this.description || `Device ID: ${this.deviceId}`;
     }
@@ -877,35 +831,31 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
         }
     }
     async clearInput(element) {
-        if (element) {
-            await this.tap(element.center[0], element.center[1]);
-            await sleep(100);
-        }
-        debugDevice('Attempting to clear input with WebDriver Clear API');
-        const cleared = await this.wdaBackend.clearActiveElement();
-        cleared ? debugDevice('Successfully cleared input with WebDriver Clear API') : debugDevice('WebDriver Clear API returned false (no active element or clear failed)');
+        await this.clearInputAt(element ? {
+            x: element.center[0],
+            y: element.center[1]
+        } : void 0);
     }
     async url() {
         return '';
     }
     async tap(x, y) {
-        await this.wdaBackend.tap(Math.round(x), Math.round(y));
-    }
-    async mouseClick(x, y) {
-        debugDevice(`mouseClick at coordinates (${x}, ${y})`);
-        await this.tap(x, y);
-    }
-    async doubleTap(x, y) {
-        await this.wdaBackend.doubleTap(Math.round(x), Math.round(y));
-    }
-    async tripleTap(x, y) {
-        await this.wdaBackend.tripleTap(Math.round(x), Math.round(y));
-    }
-    async longPress(x, y, duration = 1000) {
-        await this.wdaBackend.longPress(Math.round(x), Math.round(y), duration);
+        await this.tapPoint({
+            x,
+            y
+        });
     }
     async swipe(fromX, fromY, toX, toY, duration = 500) {
-        await this.wdaBackend.swipe(Math.round(fromX), Math.round(fromY), Math.round(toX), Math.round(toY), duration);
+        await this.swipeCoordinates(fromX, fromY, toX, toY, duration);
+    }
+    async swipeCoordinates(fromX, fromY, toX, toY, duration = 500) {
+        await this.swipePoint({
+            x: fromX,
+            y: fromY
+        }, {
+            x: toX,
+            y: toY
+        }, duration);
     }
     async typeText(text, options) {
         if (!text) return;
@@ -934,7 +884,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             y: Math.round(height / 2)
         };
         const scrollDistance = Math.round(distance || height / 3);
-        await this.swipe(start.x, start.y, start.x, start.y + scrollDistance);
+        await this.swipeCoordinates(start.x, start.y, start.x, start.y + scrollDistance);
     }
     async scrollDown(distance, startPoint) {
         const { width, height } = await this.size();
@@ -946,7 +896,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             y: Math.round(height / 2)
         };
         const scrollDistance = Math.round(distance || height / 3);
-        await this.swipe(start.x, start.y, start.x, start.y - scrollDistance);
+        await this.swipeCoordinates(start.x, start.y, start.x, start.y - scrollDistance);
     }
     async scrollLeft(distance, startPoint) {
         const { width, height } = await this.size();
@@ -958,7 +908,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             y: Math.round(height / 2)
         };
         const scrollDistance = Math.round(distance || 0.7 * width);
-        await this.swipe(start.x, start.y, start.x + scrollDistance, start.y);
+        await this.swipeCoordinates(start.x, start.y, start.x + scrollDistance, start.y);
     }
     async scrollRight(distance, startPoint) {
         const { width, height } = await this.size();
@@ -970,7 +920,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             y: Math.round(height / 2)
         };
         const scrollDistance = Math.round(distance || 0.7 * width);
-        await this.swipe(start.x, start.y, start.x - scrollDistance, start.y);
+        await this.swipeCoordinates(start.x, start.y, start.x - scrollDistance, start.y);
     }
     async scrollUntilTop(startPoint) {
         debugDevice('Using screenshot-based scroll detection for better reliability');
@@ -1066,16 +1016,16 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             debugDevice(`Performing scroll: ${direction}, distance: ${scrollDistance}`);
             switch(direction){
                 case 'up':
-                    await this.swipe(start.x, start.y, start.x, start.y + scrollDistance, 300);
+                    await this.swipeCoordinates(start.x, start.y, start.x, start.y + scrollDistance, 300);
                     break;
                 case 'down':
-                    await this.swipe(start.x, start.y, start.x, start.y - scrollDistance, 300);
+                    await this.swipeCoordinates(start.x, start.y, start.x, start.y - scrollDistance, 300);
                     break;
                 case 'left':
-                    await this.swipe(start.x, start.y, start.x + scrollDistance, start.y, 300);
+                    await this.swipeCoordinates(start.x, start.y, start.x + scrollDistance, start.y, 300);
                     break;
                 case 'right':
-                    await this.swipe(start.x, start.y, start.x - scrollDistance, start.y, 300);
+                    await this.swipeCoordinates(start.x, start.y, start.x - scrollDistance, start.y, 300);
                     break;
             }
             debugDevice('Waiting for scroll and inertia to complete...');
@@ -1132,7 +1082,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             const centerX = Math.round(windowSize.width / 2);
             const startY = Math.round(0.9 * windowSize.height);
             const endY = Math.round(0.5 * windowSize.height);
-            await this.swipe(centerX, startY, centerX, endY, 300);
+            await this.swipeCoordinates(centerX, startY, centerX, endY, 300);
             debugDevice('Dismissed keyboard with swipe up gesture from bottom of screen');
             await sleep(500);
             return true;
@@ -1209,6 +1159,45 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
         _define_property(this, "interfaceType", 'ios');
         _define_property(this, "uri", void 0);
         _define_property(this, "options", void 0);
+        _define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: (point)=>this.tapPoint(point),
+                doubleClick: (point)=>this.doubleTapPoint(point),
+                longPress: (point, opts)=>this.longPressPoint(point, opts?.duration),
+                dragAndDrop: (from, to)=>this.swipePoint(from, to, 1000)
+            },
+            keyboard: {
+                keyboardPress: (keyName)=>this.pressKey(keyName),
+                typeText: async (value, opts)=>{
+                    const target = opts?.target;
+                    if (target && opts?.replace !== false) await this.clearInput(target);
+                    else if (target) await this.tapPoint({
+                        x: target.center[0],
+                        y: target.center[1]
+                    });
+                    if (opts?.focusOnly) return;
+                    await this.typeText(value, opts);
+                },
+                clearInput: (target)=>this.clearInput(target),
+                cursorMove: async (direction, times = 1)=>{
+                    const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
+                    for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
+                }
+            },
+            touch: {
+                swipe: async (start, end, opts)=>{
+                    const duration = opts?.duration ?? 300;
+                    const repeat = opts?.repeat ?? 1;
+                    for(let i = 0; i < repeat; i++)await this.swipePoint(start, end, duration);
+                },
+                pinch: async (center, opts)=>{
+                    await this.wdaBackend.pinch(Math.round(center.x), Math.round(center.y), opts.startDistance, opts.endDistance, opts.duration);
+                }
+            },
+            scroll: {
+                scroll: (param)=>this.performActionScroll(param)
+            }
+        });
         this.deviceId = 'pending-connection';
         this.options = options;
         this.customActions = options?.customActions;
@@ -1451,7 +1440,7 @@ class IOSMCPServer extends BaseMCPServer {
     constructor(toolsManager){
         super({
             name: '@midscene/ios-mcp',
-            version: "1.8.0",
+            version: "1.8.1",
             description: 'Control the iOS device using natural language commands'
         }, toolsManager);
     }