@midscene/ios 1.8.0 → 1.8.1

package/dist/lib/mcp-server.js

@@ -688,16 +688,6 @@ function _define_property(obj, key, value) {
     return obj;
 }
 const debugDevice = (0, logger_namespaceObject.getDebug)('ios:device');
-const iosInputParamSchema = core_namespaceObject.z.object({
-    value: core_namespaceObject.z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-    autoDismissKeyboard: core_namespaceObject.z.boolean().optional().describe('Whether to dismiss the keyboard after input. Defaults to true if not specified. Set to false to keep the keyboard visible after input.'),
-    mode: core_namespaceObject.z.preprocess((val)=>'append' === val ? 'typeOnly' : val, core_namespaceObject.z["enum"]([
-        'replace',
-        'clear',
-        'typeOnly'
-    ]).default('replace').optional().describe('Input mode: "replace" (default) - clear the field and input the value; "typeOnly" - type the value directly without clearing the field first; "clear" - clear the field without inputting new text.')),
-    locate: (0, core_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be filled').optional()
-});
 const WDA_HTTP_METHODS = [
     'GET',
     'POST',
@@ -706,96 +696,39 @@ const WDA_HTTP_METHODS = [
 ];
 const DEFAULT_WDA_MJPEG_PORT = 9100;
 class IOSDevice {
+    async tapPoint(point) {
+        debugDevice(`tap at coordinates (${point.x}, ${point.y})`);
+        await this.wdaBackend.tap(Math.round(point.x), Math.round(point.y));
+    }
+    async doubleTapPoint(point) {
+        await this.wdaBackend.doubleTap(Math.round(point.x), Math.round(point.y));
+    }
+    async longPressPoint(point, duration = 1000) {
+        await this.wdaBackend.longPress(Math.round(point.x), Math.round(point.y), duration);
+    }
+    async swipePoint(start, end, duration = 500) {
+        await this.wdaBackend.swipe(Math.round(start.x), Math.round(start.y), Math.round(end.x), Math.round(end.y), duration);
+    }
+    async clearInputAt(point) {
+        if (point) {
+            await this.tapPoint(point);
+            await (0, core_utils_namespaceObject.sleep)(100);
+        }
+        debugDevice('Attempting to clear input with WebDriver Clear API');
+        const cleared = await this.wdaBackend.clearActiveElement();
+        cleared ? debugDevice('Successfully cleared input with WebDriver Clear API') : debugDevice('WebDriver Clear API returned false (no active element or clear failed)');
+    }
     actionSpace() {
+        const mobileActionContext = {
+            input: this.inputPrimitives,
+            size: ()=>this.size(),
+            sleep: async (timeMs)=>{
+                await (0, core_utils_namespaceObject.sleep)(timeMs);
+            },
+            getDefaultAutoDismissKeyboard: ()=>this.options?.autoDismissKeyboard
+        };
         const defaultActions = [
-            (0, device_namespaceObject.defineActionTap)(async (param)=>{
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot tap');
-                await this.mouseClick(element.center[0], element.center[1]);
-            }),
-            (0, device_namespaceObject.defineActionDoubleClick)(async (param)=>{
-                const element = param.locate;
-                external_node_assert_default()(element, 'Element not found, cannot double click');
-                await this.doubleTap(element.center[0], element.center[1]);
-            }),
-            (0, device_namespaceObject.defineAction)({
-                name: 'Input',
-                description: 'Input text into the input field',
-                interfaceAlias: 'aiInput',
-                paramSchema: iosInputParamSchema,
-                sample: {
-                    value: 'test@example.com',
-                    locate: {
-                        prompt: 'the email input field'
-                    }
-                },
-                call: async (param)=>{
-                    const element = param.locate;
-                    if ('typeOnly' !== param.mode) await this.clearInput(element);
-                    if ('clear' === param.mode) return;
-                    if (!param || !param.value) return;
-                    const autoDismissKeyboard = param.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
-                    await this.typeText(param.value, {
-                        autoDismissKeyboard
-                    });
-                }
-            }),
-            (0, device_namespaceObject.defineActionScroll)(async (param)=>{
-                const element = param.locate;
-                const startingPoint = element ? {
-                    left: element.center[0],
-                    top: element.center[1]
-                } : void 0;
-                const scrollToEventName = param?.scrollType;
-                if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
-                else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
-                else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
-                else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
-                else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
-                else {
-                    if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance || void 0, startingPoint);
-                    else if ('left' === param.direction) await this.scrollLeft(param.distance || void 0, startingPoint);
-                    else if ('right' === param.direction) await this.scrollRight(param.distance || void 0, startingPoint);
-                    else throw new Error(`Unknown scroll direction: ${param.direction}`);
-                    else await this.scrollDown(param?.distance || void 0, startingPoint);
-                    await (0, core_utils_namespaceObject.sleep)(500);
-                }
-            }),
-            (0, device_namespaceObject.defineActionDragAndDrop)(async (param)=>{
-                const from = param.from;
-                const to = param.to;
-                external_node_assert_default()(from, 'missing "from" param for drag and drop');
-                external_node_assert_default()(to, 'missing "to" param for drag and drop');
-                await this.swipe(from.center[0], from.center[1], to.center[0], to.center[1], 1000);
-            }),
-            (0, device_namespaceObject.defineActionSwipe)(async (param)=>{
-                const { startPoint, endPoint, duration, repeatCount } = (0, device_namespaceObject.normalizeMobileSwipeParam)(param, await this.size());
-                for(let i = 0; i < repeatCount; i++)await this.swipe(startPoint.x, startPoint.y, endPoint.x, endPoint.y, duration);
-            }),
-            (0, device_namespaceObject.defineActionKeyboardPress)(async (param)=>{
-                await this.pressKey(param.keyName);
-            }),
-            (0, device_namespaceObject.defineActionCursorMove)(async (param)=>{
-                const arrowKey = 'left' === param.direction ? 'ArrowLeft' : 'ArrowRight';
-                const times = param.times ?? 1;
-                for(let i = 0; i < times; i++){
-                    await this.pressKey(arrowKey);
-                    await (0, core_utils_namespaceObject.sleep)(100);
-                }
-            }),
-            (0, device_namespaceObject.defineActionLongPress)(async (param)=>{
-                const element = param.locate;
-                external_node_assert_default()(element, 'LongPress requires an element to be located');
-                const [x, y] = element.center;
-                await this.longPress(x, y, param?.duration);
-            }),
-            (0, device_namespaceObject.defineActionPinch)(async (param)=>{
-                const { centerX, centerY, startDistance, endDistance, duration } = (0, device_namespaceObject.normalizePinchParam)(param, await this.size());
-                await this.wdaBackend.pinch(centerX, centerY, startDistance, endDistance, duration);
-            }),
-            (0, device_namespaceObject.defineActionClearInput)(async (param)=>{
-                await this.clearInput(param.locate);
-            })
+            ...(0, device_namespaceObject.createDefaultMobileActions)(mobileActionContext)
         ];
         const platformSpecificActions = Object.values(createPlatformActions(this));
         const customActions = this.customActions || [];
@@ -805,6 +738,27 @@ class IOSDevice {
             ...customActions
         ];
     }
+    async performActionScroll(param) {
+        const element = param.locate;
+        const startingPoint = element ? {
+            left: element.center[0],
+            top: element.center[1]
+        } : void 0;
+        const scrollToEventName = param?.scrollType;
+        if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
+        else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
+        else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
+        else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
+        else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
+        else {
+            if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance || void 0, startingPoint);
+            else if ('left' === param.direction) await this.scrollLeft(param.distance || void 0, startingPoint);
+            else if ('right' === param.direction) await this.scrollRight(param.distance || void 0, startingPoint);
+            else throw new Error(`Unknown scroll direction: ${param.direction}`);
+            else await this.scrollDown(param?.distance || void 0, startingPoint);
+            await (0, core_utils_namespaceObject.sleep)(500);
+        }
+    }
     describe() {
         return this.description || `Device ID: ${this.deviceId}`;
     }
@@ -916,35 +870,31 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
         }
     }
     async clearInput(element) {
-        if (element) {
-            await this.tap(element.center[0], element.center[1]);
-            await (0, core_utils_namespaceObject.sleep)(100);
-        }
-        debugDevice('Attempting to clear input with WebDriver Clear API');
-        const cleared = await this.wdaBackend.clearActiveElement();
-        cleared ? debugDevice('Successfully cleared input with WebDriver Clear API') : debugDevice('WebDriver Clear API returned false (no active element or clear failed)');
+        await this.clearInputAt(element ? {
+            x: element.center[0],
+            y: element.center[1]
+        } : void 0);
     }
     async url() {
         return '';
     }
     async tap(x, y) {
-        await this.wdaBackend.tap(Math.round(x), Math.round(y));
-    }
-    async mouseClick(x, y) {
-        debugDevice(`mouseClick at coordinates (${x}, ${y})`);
-        await this.tap(x, y);
-    }
-    async doubleTap(x, y) {
-        await this.wdaBackend.doubleTap(Math.round(x), Math.round(y));
-    }
-    async tripleTap(x, y) {
-        await this.wdaBackend.tripleTap(Math.round(x), Math.round(y));
-    }
-    async longPress(x, y, duration = 1000) {
-        await this.wdaBackend.longPress(Math.round(x), Math.round(y), duration);
+        await this.tapPoint({
+            x,
+            y
+        });
     }
     async swipe(fromX, fromY, toX, toY, duration = 500) {
-        await this.wdaBackend.swipe(Math.round(fromX), Math.round(fromY), Math.round(toX), Math.round(toY), duration);
+        await this.swipeCoordinates(fromX, fromY, toX, toY, duration);
+    }
+    async swipeCoordinates(fromX, fromY, toX, toY, duration = 500) {
+        await this.swipePoint({
+            x: fromX,
+            y: fromY
+        }, {
+            x: toX,
+            y: toY
+        }, duration);
     }
     async typeText(text, options) {
         if (!text) return;
@@ -973,7 +923,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             y: Math.round(height / 2)
         };
         const scrollDistance = Math.round(distance || height / 3);
-        await this.swipe(start.x, start.y, start.x, start.y + scrollDistance);
+        await this.swipeCoordinates(start.x, start.y, start.x, start.y + scrollDistance);
     }
     async scrollDown(distance, startPoint) {
         const { width, height } = await this.size();
@@ -985,7 +935,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             y: Math.round(height / 2)
         };
         const scrollDistance = Math.round(distance || height / 3);
-        await this.swipe(start.x, start.y, start.x, start.y - scrollDistance);
+        await this.swipeCoordinates(start.x, start.y, start.x, start.y - scrollDistance);
     }
     async scrollLeft(distance, startPoint) {
         const { width, height } = await this.size();
@@ -997,7 +947,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             y: Math.round(height / 2)
         };
         const scrollDistance = Math.round(distance || 0.7 * width);
-        await this.swipe(start.x, start.y, start.x + scrollDistance, start.y);
+        await this.swipeCoordinates(start.x, start.y, start.x + scrollDistance, start.y);
     }
     async scrollRight(distance, startPoint) {
         const { width, height } = await this.size();
@@ -1009,7 +959,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             y: Math.round(height / 2)
         };
         const scrollDistance = Math.round(distance || 0.7 * width);
-        await this.swipe(start.x, start.y, start.x - scrollDistance, start.y);
+        await this.swipeCoordinates(start.x, start.y, start.x - scrollDistance, start.y);
     }
     async scrollUntilTop(startPoint) {
         debugDevice('Using screenshot-based scroll detection for better reliability');
@@ -1105,16 +1055,16 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             debugDevice(`Performing scroll: ${direction}, distance: ${scrollDistance}`);
             switch(direction){
                 case 'up':
-                    await this.swipe(start.x, start.y, start.x, start.y + scrollDistance, 300);
+                    await this.swipeCoordinates(start.x, start.y, start.x, start.y + scrollDistance, 300);
                     break;
                 case 'down':
-                    await this.swipe(start.x, start.y, start.x, start.y - scrollDistance, 300);
+                    await this.swipeCoordinates(start.x, start.y, start.x, start.y - scrollDistance, 300);
                     break;
                 case 'left':
-                    await this.swipe(start.x, start.y, start.x + scrollDistance, start.y, 300);
+                    await this.swipeCoordinates(start.x, start.y, start.x + scrollDistance, start.y, 300);
                     break;
                 case 'right':
-                    await this.swipe(start.x, start.y, start.x - scrollDistance, start.y, 300);
+                    await this.swipeCoordinates(start.x, start.y, start.x - scrollDistance, start.y, 300);
                     break;
             }
             debugDevice('Waiting for scroll and inertia to complete...');
@@ -1171,7 +1121,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             const centerX = Math.round(windowSize.width / 2);
             const startY = Math.round(0.9 * windowSize.height);
             const endY = Math.round(0.5 * windowSize.height);
-            await this.swipe(centerX, startY, centerX, endY, 300);
+            await this.swipeCoordinates(centerX, startY, centerX, endY, 300);
             debugDevice('Dismissed keyboard with swipe up gesture from bottom of screen');
             await (0, core_utils_namespaceObject.sleep)(500);
             return true;
@@ -1248,6 +1198,45 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
         _define_property(this, "interfaceType", 'ios');
         _define_property(this, "uri", void 0);
         _define_property(this, "options", void 0);
+        _define_property(this, "inputPrimitives", {
+            pointer: {
+                tap: (point)=>this.tapPoint(point),
+                doubleClick: (point)=>this.doubleTapPoint(point),
+                longPress: (point, opts)=>this.longPressPoint(point, opts?.duration),
+                dragAndDrop: (from, to)=>this.swipePoint(from, to, 1000)
+            },
+            keyboard: {
+                keyboardPress: (keyName)=>this.pressKey(keyName),
+                typeText: async (value, opts)=>{
+                    const target = opts?.target;
+                    if (target && opts?.replace !== false) await this.clearInput(target);
+                    else if (target) await this.tapPoint({
+                        x: target.center[0],
+                        y: target.center[1]
+                    });
+                    if (opts?.focusOnly) return;
+                    await this.typeText(value, opts);
+                },
+                clearInput: (target)=>this.clearInput(target),
+                cursorMove: async (direction, times = 1)=>{
+                    const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
+                    for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
+                }
+            },
+            touch: {
+                swipe: async (start, end, opts)=>{
+                    const duration = opts?.duration ?? 300;
+                    const repeat = opts?.repeat ?? 1;
+                    for(let i = 0; i < repeat; i++)await this.swipePoint(start, end, duration);
+                },
+                pinch: async (center, opts)=>{
+                    await this.wdaBackend.pinch(Math.round(center.x), Math.round(center.y), opts.startDistance, opts.endDistance, opts.duration);
+                }
+            },
+            scroll: {
+                scroll: (param)=>this.performActionScroll(param)
+            }
+        });
         this.deviceId = 'pending-connection';
         this.options = options;
         this.customActions = options?.customActions;
@@ -1491,7 +1480,7 @@ class IOSMCPServer extends mcp_namespaceObject.BaseMCPServer {
     constructor(toolsManager){
         super({
             name: '@midscene/ios-mcp',
-            version: "1.8.0",
+            version: "1.8.1",
             description: 'Control the iOS device using natural language commands'
         }, toolsManager);
     }

package/dist/types/index.d.ts

@@ -8,8 +8,8 @@ import { DeviceAction } from '@midscene/core';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import { InitArgSpec } from '@midscene/shared/mcp/base-tools';
 import { InterfaceType } from '@midscene/core';
-import { IOSDeviceInputOpt } from '@midscene/core/device';
 import { IOSDeviceOpt } from '@midscene/core/device';
+import { MobileInputPrimitives } from '@midscene/core/device';
 import { overrideAIConfig } from '@midscene/shared/env';
 import { PlaygroundPlatformDescriptor } from '@midscene/playground';
 import { Point } from '@midscene/core';
@@ -88,7 +88,14 @@ export declare class IOSDevice implements AbstractInterface {
     interfaceType: InterfaceType;
     uri: string | undefined;
     options?: IOSDeviceOpt;
+    readonly inputPrimitives: MobileInputPrimitives;
+    private tapPoint;
+    private doubleTapPoint;
+    private longPressPoint;
+    private swipePoint;
+    private clearInputAt;
     actionSpace(): DeviceAction<any>[];
+    private performActionScroll;
     constructor(options?: IOSDeviceOpt);
     describe(): string;
     getConnectedDeviceInfo(): Promise<{
@@ -128,13 +135,10 @@ export declare class IOSDevice implements AbstractInterface {
     clearInput(element?: ElementInfo): Promise<void>;
     url(): Promise<string>;
     tap(x: number, y: number): Promise<void>;
-    mouseClick(x: number, y: number): Promise<void>;
-    doubleTap(x: number, y: number): Promise<void>;
-    tripleTap(x: number, y: number): Promise<void>;
-    longPress(x: number, y: number, duration?: number): Promise<void>;
     swipe(fromX: number, fromY: number, toX: number, toY: number, duration?: number): Promise<void>;
-    typeText(text: string, options?: IOSDeviceInputOpt): Promise<void>;
-    pressKey(key: string): Promise<void>;
+    private swipeCoordinates;
+    private typeText;
+    private pressKey;
     scrollUp(distance?: number, startPoint?: Point): Promise<void>;
     scrollDown(distance?: number, startPoint?: Point): Promise<void>;
     scrollLeft(distance?: number, startPoint?: Point): Promise<void>;

package/dist/types/mcp-server.d.ts

@@ -9,10 +9,10 @@ import { DeviceAction } from '@midscene/core';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import { InitArgSpec } from '@midscene/shared/mcp/base-tools';
 import { InterfaceType } from '@midscene/core';
-import { IOSDeviceInputOpt } from '@midscene/core/device';
 import { IOSDeviceOpt } from '@midscene/core/device';
 import { LaunchMCPServerOptions } from '@midscene/shared/mcp';
 import { LaunchMCPServerResult } from '@midscene/shared/mcp';
+import { MobileInputPrimitives } from '@midscene/core/device';
 import { Point } from '@midscene/core';
 import { Size } from '@midscene/core';
 import { Tool } from '@midscene/shared/mcp';
@@ -82,7 +82,14 @@ declare class IOSDevice implements AbstractInterface {
     interfaceType: InterfaceType;
     uri: string | undefined;
     options?: IOSDeviceOpt;
+    readonly inputPrimitives: MobileInputPrimitives;
+    private tapPoint;
+    private doubleTapPoint;
+    private longPressPoint;
+    private swipePoint;
+    private clearInputAt;
     actionSpace(): DeviceAction<any>[];
+    private performActionScroll;
     constructor(options?: IOSDeviceOpt);
     describe(): string;
     getConnectedDeviceInfo(): Promise<{
@@ -122,13 +129,10 @@ declare class IOSDevice implements AbstractInterface {
     clearInput(element?: ElementInfo): Promise<void>;
     url(): Promise<string>;
     tap(x: number, y: number): Promise<void>;
-    mouseClick(x: number, y: number): Promise<void>;
-    doubleTap(x: number, y: number): Promise<void>;
-    tripleTap(x: number, y: number): Promise<void>;
-    longPress(x: number, y: number, duration?: number): Promise<void>;
     swipe(fromX: number, fromY: number, toX: number, toY: number, duration?: number): Promise<void>;
-    typeText(text: string, options?: IOSDeviceInputOpt): Promise<void>;
-    pressKey(key: string): Promise<void>;
+    private swipeCoordinates;
+    private typeText;
+    private pressKey;
     scrollUp(distance?: number, startPoint?: Point): Promise<void>;
     scrollDown(distance?: number, startPoint?: Point): Promise<void>;
     scrollLeft(distance?: number, startPoint?: Point): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/ios",
-  "version": "1.8.0",
+  "version": "1.8.1",
   "description": "iOS automation library for Midscene",
   "keywords": [
     "iOS UI automation",
@@ -43,10 +43,10 @@
   "dependencies": {
     "@inquirer/prompts": "^7.8.6",
     "open": "10.1.0",
-    "@midscene/core": "1.8.0",
-    "@midscene/webdriver": "1.8.0",
-    "@midscene/shared": "1.8.0",
-    "@midscene/playground": "1.8.0"
+    "@midscene/core": "1.8.1",
+    "@midscene/shared": "1.8.1",
+    "@midscene/playground": "1.8.1",
+    "@midscene/webdriver": "1.8.1"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",

package/static/index.html

@@ -1 +1 @@
-<!doctype html><html><head><link rel="icon" href="/favicon.ico"><title>Midscene Playground</title><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><script defer src="/static/js/lib-react.7b1abe58.js"></script><script defer src="/static/js/889.c8e2e995.js"></script><script defer src="/static/js/index.7d3d953d.js"></script><link href="/static/css/index.63b028da.css" rel="stylesheet"></head><body><div id="root"></div></body></html>
+<!doctype html><html><head><link rel="icon" href="/favicon.ico"><title>Midscene Playground</title><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><script defer src="/static/js/lib-react.7b1abe58.js"></script><script defer src="/static/js/596.5426be9e.js"></script><script defer src="/static/js/index.acaa5ec1.js"></script><link href="/static/css/index.26c9c911.css" rel="stylesheet"></head><body><div id="root"></div></body></html>