@midscene/ios 0.30.10 → 1.0.0

package/dist/es/index.mjs

@@ -154,9 +154,8 @@ class IOSWebDriverClient extends WebDriverClient {
         this.ensureSession();
         debugIOS('Getting active element');
         try {
-            var _response_value, _response_value1;
             const response = await this.makeRequest('GET', `/session/${this.sessionId}/element/active`);
-            const elementId = (null == (_response_value = response.value) ? void 0 : _response_value.ELEMENT) || (null == (_response_value1 = response.value) ? void 0 : _response_value1['element-6066-11e4-a52e-4f735466cecf']) || response.ELEMENT || response['element-6066-11e4-a52e-4f735466cecf'];
+            const elementId = response.value?.ELEMENT || response.value?.['element-6066-11e4-a52e-4f735466cecf'] || response.ELEMENT || response['element-6066-11e4-a52e-4f735466cecf'];
             if (elementId) {
                 debugIOS(`Got active element ID: ${elementId}`);
                 return elementId;
@@ -233,8 +232,17 @@ class IOSWebDriverClient extends WebDriverClient {
             });
             debugIOS(`Tapped at coordinates (${x}, ${y})`);
         } catch (error) {
-            debugIOS(`Failed to tap at (${x}, ${y}): ${error}`);
-            throw new Error(`Failed to tap at coordinates: ${error}`);
+            debugIOS(`New tap endpoint failed, trying legacy endpoint: ${error}`);
+            try {
+                await this.makeRequest('POST', `/session/${this.sessionId}/wda/tap/0`, {
+                    x,
+                    y
+                });
+                debugIOS(`Tapped at coordinates (${x}, ${y}) using legacy endpoint`);
+            } catch (fallbackError) {
+                debugIOS(`Failed to tap at (${x}, ${y}): ${fallbackError}`);
+                throw new Error(`Failed to tap at coordinates: ${fallbackError}`);
+            }
         }
     }
     async swipe(fromX, fromY, toX, toY, duration = 500) {
@@ -307,13 +315,34 @@ class IOSWebDriverClient extends WebDriverClient {
         debugIOS(`Triple tapped at coordinates (${x}, ${y})`);
     }
     async getScreenScale() {
-        var _screenResponse_value;
-        const screenResponse = await this.makeRequest('GET', '/wda/screen');
-        if (null == screenResponse ? void 0 : null == (_screenResponse_value = screenResponse.value) ? void 0 : _screenResponse_value.scale) {
-            debugIOS(`Got screen scale from WDA screen endpoint: ${screenResponse.value.scale}`);
-            return screenResponse.value.scale;
+        this.ensureSession();
+        try {
+            const screenResponse = await this.makeRequest('GET', `/session/${this.sessionId}/wda/screen`);
+            if (screenResponse?.value?.scale) {
+                debugIOS(`Got screen scale from WDA screen endpoint: ${screenResponse.value.scale}`);
+                return screenResponse.value.scale;
+            }
+        } catch (error) {
+            debugIOS(`Failed to get screen scale from /wda/screen: ${error}`);
         }
-        debugIOS('No screen scale found in WDA screen response');
+        try {
+            debugIOS('Calculating screen scale from screenshot and window size');
+            const [screenshotBase64, windowSize] = await Promise.all([
+                this.takeScreenshot(),
+                this.getWindowSize()
+            ]);
+            const { jimpFromBase64 } = await import("@midscene/shared/img");
+            const screenshotImg = await jimpFromBase64(screenshotBase64);
+            const screenshotWidth = screenshotImg.bitmap.width;
+            const screenshotHeight = screenshotImg.bitmap.height;
+            const scale = Math.max(screenshotWidth, screenshotHeight) / Math.max(windowSize.width, windowSize.height);
+            const roundedScale = Math.round(scale);
+            debugIOS(`Calculated screen scale: ${roundedScale} (screenshot: ${screenshotWidth}x${screenshotHeight}, window: ${windowSize.width}x${windowSize.height})`);
+            return roundedScale;
+        } catch (error) {
+            debugIOS(`Failed to calculate screen scale: ${error}`);
+        }
+        debugIOS('No screen scale found');
         return null;
     }
     async createSession(capabilities) {
@@ -340,6 +369,10 @@ class IOSWebDriverClient extends WebDriverClient {
             debugIOS(`Failed to apply iOS session configuration: ${error}`);
         }
     }
+    async executeRequest(method, endpoint, data) {
+        this.ensureSession();
+        return this.makeRequest(method, endpoint, data);
+    }
 }
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
@@ -352,6 +385,12 @@ function _define_property(obj, key, value) {
     return obj;
 }
 const debugDevice = getDebug('ios:device');
+const WDA_HTTP_METHODS = [
+    'GET',
+    'POST',
+    'DELETE',
+    'PUT'
+];
 class IOSDevice {
     actionSpace() {
         const defaultActions = [
@@ -371,7 +410,7 @@ class IOSDevice {
                 interfaceAlias: 'aiInput',
                 paramSchema: z.object({
                     value: z.string().describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-                    autoDismissKeyboard: z.boolean().optional().describe('If true, the keyboard will be dismissed after the input is completed. Do not set it unless the user asks you to do so.'),
+                    autoDismissKeyboard: z.boolean().optional().describe('Whether to dismiss the keyboard after input. Defaults to true if not specified. Set to false to keep the keyboard visible after input.'),
                     mode: z["enum"]([
                         'replace',
                         'clear',
@@ -380,14 +419,13 @@ class IOSDevice {
                     locate: getMidsceneLocationSchema().describe('The input field to be filled').optional()
                 }),
                 call: async (param)=>{
-                    var _this_options;
                     const element = param.locate;
                     if (element) {
                         if ('append' !== param.mode) await this.clearInput(element);
                     }
                     if ('clear' === param.mode) return;
                     if (!param || !param.value) return;
-                    const autoDismissKeyboard = param.autoDismissKeyboard ?? (null == (_this_options = this.options) ? void 0 : _this_options.autoDismissKeyboard);
+                    const autoDismissKeyboard = param.autoDismissKeyboard ?? this.options?.autoDismissKeyboard;
                     await this.typeText(param.value, {
                         autoDismissKeyboard
                     });
@@ -399,18 +437,18 @@ class IOSDevice {
                     left: element.center[0],
                     top: element.center[1]
                 } : void 0;
-                const scrollToEventName = null == param ? void 0 : param.scrollType;
-                if ('untilTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
-                else if ('untilBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
-                else if ('untilRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
-                else if ('untilLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
-                else if ('once' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
+                const scrollToEventName = param?.scrollType;
+                if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
+                else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
+                else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
+                else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
+                else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
                 else {
-                    if ((null == param ? void 0 : param.direction) !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance || void 0, startingPoint);
+                    if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance || void 0, startingPoint);
                     else if ('left' === param.direction) await this.scrollLeft(param.distance || void 0, startingPoint);
                     else if ('right' === param.direction) await this.scrollRight(param.distance || void 0, startingPoint);
                     else throw new Error(`Unknown scroll direction: ${param.direction}`);
-                    else await this.scrollDown((null == param ? void 0 : param.distance) || void 0, startingPoint);
+                    else await this.scrollDown(param?.distance || void 0, startingPoint);
                     await sleep(500);
                 }
             }),
@@ -424,22 +462,6 @@ class IOSDevice {
             defineActionKeyboardPress(async (param)=>{
                 await this.pressKey(param.keyName);
             }),
-            defineAction({
-                name: 'IOSHomeButton',
-                description: 'Trigger the system "home" operation on iOS devices',
-                paramSchema: z.object({}),
-                call: async ()=>{
-                    await this.home();
-                }
-            }),
-            defineAction({
-                name: 'IOSAppSwitcher',
-                description: 'Trigger the system "app switcher" operation on iOS devices',
-                paramSchema: z.object({}),
-                call: async ()=>{
-                    await this.appSwitcher();
-                }
-            }),
             defineAction({
                 name: 'IOSLongPress',
                 description: 'Trigger a long press on the screen at specified coordinates on iOS devices',
@@ -451,7 +473,7 @@ class IOSDevice {
                     const element = param.locate;
                     node_assert(element, 'IOSLongPress requires an element to be located');
                     const [x, y] = element.center;
-                    await this.longPress(x, y, null == param ? void 0 : param.duration);
+                    await this.longPress(x, y, param?.duration);
                 }
             }),
             defineActionClearInput(async (param)=>{
@@ -460,9 +482,11 @@ class IOSDevice {
                 await this.clearInput(element);
             })
         ];
+        const platformSpecificActions = Object.values(createPlatformActions(this));
         const customActions = this.customActions || [];
         return [
             ...defaultActions,
+            ...platformSpecificActions,
             ...customActions
         ];
     }
@@ -479,7 +503,7 @@ class IOSDevice {
             await this.wdaManager.start();
             await this.wdaBackend.createSession();
             const deviceInfo = await this.wdaBackend.getDeviceInfo();
-            if (null == deviceInfo ? void 0 : deviceInfo.udid) {
+            if (deviceInfo?.udid) {
                 this.deviceId = deviceInfo.udid;
                 debugDevice(`Updated device ID to real UDID: ${this.deviceId}`);
             }
@@ -587,9 +611,8 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
         await this.wdaBackend.swipe(Math.round(fromX), Math.round(fromY), Math.round(toX), Math.round(toY), duration);
     }
     async typeText(text, options) {
-        var _this_options;
         if (!text) return;
-        const shouldAutoDismissKeyboard = (null == options ? void 0 : options.autoDismissKeyboard) ?? (null == (_this_options = this.options) ? void 0 : _this_options.autoDismissKeyboard) ?? true;
+        const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
         debugDevice(`Typing text: "${text}"`);
         try {
             await sleep(200);
@@ -791,20 +814,30 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
     }
     async hideKeyboard(keyNames) {
         try {
-            if (keyNames && keyNames.length > 0) {
-                debugDevice(`Using keyNames to dismiss keyboard: ${keyNames.join(', ')}`);
-                await this.wdaBackend.dismissKeyboard(keyNames);
-                debugDevice('Dismissed keyboard using provided keyNames');
-                await sleep(300);
+            const dismissKeys = keyNames && keyNames.length > 0 ? keyNames : [
+                'return',
+                'done',
+                'go',
+                'search',
+                'next',
+                'send'
+            ];
+            debugDevice(`Attempting to dismiss keyboard using WDA API with keys: ${dismissKeys.join(', ')}`);
+            try {
+                await this.wdaBackend.dismissKeyboard(dismissKeys);
+                debugDevice('Successfully dismissed keyboard using WDA API');
+                await sleep(500);
                 return true;
+            } catch (wdaError) {
+                debugDevice(`WDA dismissKeyboard failed, falling back to swipe gesture: ${wdaError}`);
             }
             const windowSize = await this.wdaBackend.getWindowSize();
             const centerX = Math.round(windowSize.width / 2);
-            const startY = Math.round(0.33 * windowSize.height);
-            const endY = Math.round(0.33 * windowSize.height + 10);
-            await this.swipe(centerX, startY, centerX, endY, 50);
-            debugDevice('Dismissed keyboard with swipe down gesture at screen one-third position');
-            await sleep(300);
+            const startY = Math.round(0.9 * windowSize.height);
+            const endY = Math.round(0.5 * windowSize.height);
+            await this.swipe(centerX, startY, centerX, endY, 300);
+            debugDevice('Dismissed keyboard with swipe up gesture from bottom of screen');
+            await sleep(500);
             return true;
         } catch (error) {
             debugDevice(`Failed to hide keyboard: ${error}`);
@@ -833,6 +866,7 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
     async openUrlViaSafari(url) {
         try {
             debugDevice(`Opening URL via Safari: ${url}`);
+            await this.wdaBackend.terminateApp('com.apple.mobilesafari');
             await this.wdaBackend.launchApp('com.apple.mobilesafari');
             await sleep(2000);
             await this.typeText(url);
@@ -850,6 +884,9 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
             throw new Error(`Failed to open URL via Safari: ${error}`);
         }
     }
+    async runWdaRequest(method, endpoint, data) {
+        return await this.wdaBackend.executeRequest(method, endpoint, data);
+    }
     async destroy() {
         if (this.destroyed) return;
         try {
@@ -875,9 +912,9 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
         _define_property(this, "options", void 0);
         this.deviceId = 'pending-connection';
         this.options = options;
-        this.customActions = null == options ? void 0 : options.customActions;
-        const wdaPort = (null == options ? void 0 : options.wdaPort) || DEFAULT_WDA_PORT;
-        const wdaHost = (null == options ? void 0 : options.wdaHost) || 'localhost';
+        this.customActions = options?.customActions;
+        const wdaPort = options?.wdaPort || DEFAULT_WDA_PORT;
+        const wdaHost = options?.wdaHost || 'localhost';
         this.wdaBackend = new IOSWebDriverClient({
             port: wdaPort,
             host: wdaHost
@@ -885,6 +922,44 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
         this.wdaManager = WDAManager.getInstance(wdaPort, wdaHost);
     }
 }
+const runWdaRequestParamSchema = z.object({
+    method: z["enum"](WDA_HTTP_METHODS).describe('HTTP method (GET, POST, DELETE, PUT)'),
+    endpoint: z.string().describe('WebDriver API endpoint'),
+    data: z.object({}).passthrough().optional().describe('Optional request body data as JSON object')
+});
+const launchParamSchema = z.string().describe('App bundle ID or URL to launch');
+const createPlatformActions = (device)=>({
+        RunWdaRequest: defineAction({
+            name: 'RunWdaRequest',
+            description: 'Execute WebDriverAgent API request directly on iOS device',
+            interfaceAlias: 'runWdaRequest',
+            paramSchema: runWdaRequestParamSchema,
+            call: async (param)=>await device.runWdaRequest(param.method, param.endpoint, param.data)
+        }),
+        Launch: defineAction({
+            name: 'Launch',
+            description: 'Launch an iOS app or URL',
+            interfaceAlias: 'launch',
+            paramSchema: launchParamSchema,
+            call: async (param)=>{
+                await device.launch(param);
+            }
+        }),
+        IOSHomeButton: defineAction({
+            name: 'IOSHomeButton',
+            description: 'Trigger the system "home" operation on iOS devices',
+            call: async ()=>{
+                await device.home();
+            }
+        }),
+        IOSAppSwitcher: defineAction({
+            name: 'IOSAppSwitcher',
+            description: 'Trigger the system "app switcher" operation on iOS devices',
+            call: async ()=>{
+                await device.appSwitcher();
+            }
+        })
+    });
 const execAsync = promisify(exec);
 const debugUtils = getDebug('ios:utils');
 function checkMacOSPlatform() {
@@ -931,24 +1006,35 @@ async function checkIOSEnvironment() {
         };
     }
 }
+function agent_define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
 const debugAgent = getDebug('ios:agent');
 class IOSAgent extends Agent {
-    async launch(uri) {
-        const device = this.page;
-        await device.launch(uri);
+    createActionWrapper(name) {
+        const action = this.wrapActionInActionSpace(name);
+        return (...args)=>action(args[0]);
+    }
+    constructor(device, opts){
+        super(device, opts), agent_define_property(this, "launch", void 0), agent_define_property(this, "runWdaRequest", void 0), agent_define_property(this, "home", void 0), agent_define_property(this, "appSwitcher", void 0);
+        this.launch = this.createActionWrapper('Launch');
+        this.runWdaRequest = this.createActionWrapper('RunWdaRequest');
+        this.home = this.createActionWrapper('IOSHomeButton');
+        this.appSwitcher = this.createActionWrapper('IOSAppSwitcher');
     }
 }
 async function agentFromWebDriverAgent(opts) {
     debugAgent('Creating iOS agent with WebDriverAgent auto-detection');
     const envCheck = await checkIOSEnvironment();
     if (!envCheck.available) throw new Error(`iOS environment not available: ${envCheck.error}`);
-    const device = new IOSDevice({
-        autoDismissKeyboard: null == opts ? void 0 : opts.autoDismissKeyboard,
-        customActions: null == opts ? void 0 : opts.customActions,
-        wdaPort: null == opts ? void 0 : opts.wdaPort,
-        wdaHost: null == opts ? void 0 : opts.wdaHost,
-        useWDA: null == opts ? void 0 : opts.useWDA
-    });
+    const device = new IOSDevice(opts || {});
     await device.connect();
     return new IOSAgent(device, opts);
 }