@midscene/web 0.12.9-beta-20250324113701.0 → 0.13.0

package/README.md

@@ -4,13 +4,25 @@ Automate browser actions, extract data, and perform assertions using AI. It offe
 
 See https://midscenejs.com/ for details.
 
-## Android prerequisites
+## iOS/Android prerequisites
 
-Android is driven by adb, so you need install adb first:
-- [CLI](https://developer.android.com/tools/adb)
+iOS/Android is driven by appium, so you need the following Appium tool chain first:
+- [CLI](https://appium.io/docs/en/latest/quickstart/install/)
+- Driver
+  - [iOS](https://github.com/appium/appium-xcuitest-driver)
+  - [Android](https://github.com/appium/appium-uiautomator2-driver)
+- [Plugin](https://github.com/appium/appium/tree/master/packages/universal-xml-plugin)
+
+then execute the command to start appium server:
+
+```bash
+appium --use-plugins=universal-xml
+```
+
+now you can use run tests for iOS/Android devices:
 
 ```bash
-npm run test:ai -- adb
+npm run test:ai -- appium
 ```
 
 ## License

package/dist/es/{android.js → appium.js}

@@ -1169,7 +1169,7 @@ var PageTaskExecutor = class {
     let planningTask = this.planningTaskFromPrompt(userPrompt, cacheGroup);
     let result;
     let replanCount = 0;
-    const logLog = [];
+    const logList = [];
     while (planningTask) {
       if (replanCount > replanningCountLimit) {
         const errorMsg = "Replanning too many times, please split the task into multiple steps";
@@ -1204,7 +1204,7 @@ var PageTaskExecutor = class {
         };
       }
       if (planResult?.log) {
-        logLog.push(planResult.log);
+        logList.push(planResult.log);
       }
       if (!planResult.more_actions_needed_by_instruction) {
         planningTask = null;
@@ -1213,7 +1213,7 @@ var PageTaskExecutor = class {
       planningTask = this.planningTaskFromPrompt(
         userPrompt,
         cacheGroup,
-        logLog.join("\n")
+        logList.length > 0 ? `- ${logList.join("\n- ")}` : void 0
       );
       replanCount++;
     }
@@ -1832,154 +1832,48 @@ ${errors}`);
   }
 };
 
-// src/android/page.ts
+// src/appium/page.ts
 import fs from "fs";
-import path from "path";
 import { getTmpFile } from "@midscene/core/utils";
-import { resizeImg } from "@midscene/shared/img";
-import { getDebug as getDebug3 } from "@midscene/shared/utils";
-import { ADB } from "appium-adb";
-var debugPage = getDebug3("android:page");
+import {
+  clientExtractTextWithPosition
+} from "@midscene/shared/extractor";
+import { base64Encoded, resizeImg } from "@midscene/shared/img";
+import { DOMParser } from "@xmldom/xmldom";
+function buttonToNumber(button) {
+  return button === "left" ? 0 : button === "middle" ? 1 : 2;
+}
 var Page = class {
-  constructor({ deviceId }) {
-    this.screenSize = null;
-    this.yadbPushed = false;
-    this.deviceRatio = 1;
-    this.pageType = "android";
-    this.deviceId = deviceId;
-    this.adbPromise = this.initAdb();
-  }
-  async initAdb() {
-    try {
-      debugPage(`Initializing ADB with device ID: ${this.deviceId}`);
-      const adb = await ADB.createADB({
-        udid: this.deviceId,
-        adbExecTimeout: 6e4
-      });
-      debugPage("ADB initialized successfully");
-      return adb;
-    } catch (error) {
-      console.error(`Failed to initialize ADB: ${error}`);
-      throw error;
-    }
-  }
-  async getAdb() {
-    return this.adbPromise;
-  }
-  async execYadb(keyboardContent) {
-    await this.pushYadb();
-    const adb = await this.getAdb();
-    await adb.shell(
-      `app_process -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "${keyboardContent}"`
-    );
+  constructor(browser) {
+    this.pageType = "appium";
+    this.browser = browser;
   }
   async getElementsInfo() {
-    return [];
+    const pageSource = await this.browser.getPageSource();
+    const { width, height } = await this.browser.getWindowSize();
+    const parser = new DOMParser();
+    const doc = parser.parseFromString(pageSource, "text/xml");
+    const infos = clientExtractTextWithPosition(doc).filter(
+      (element) => element.rect.height !== height && element.rect.width !== width && element.rect.left !== 0 && element.rect.top !== 0 && element.attributes.visible === "true"
+    );
+    return infos;
   }
   async getElementsNodeTree() {
-    return {
-      node: null,
-      children: []
-    };
+    throw new Error("Not implemented");
   }
   async size() {
-    if (this.screenSize) {
-      return this.screenSize;
-    }
-    try {
-      const adb = await this.getAdb();
-      const screenSize = await adb.getScreenSize();
-      let width;
-      let height;
-      if (typeof screenSize === "string") {
-        const match = screenSize.match(/(\d+)x(\d+)/);
-        if (!match || match.length < 3) {
-          throw new Error(`Unable to parse screen size: ${screenSize}`);
-        }
-        width = Number.parseInt(match[1], 10);
-        height = Number.parseInt(match[2], 10);
-      } else if (typeof screenSize === "object" && screenSize !== null) {
-        const sizeObj = screenSize;
-        if ("width" in sizeObj && "height" in sizeObj) {
-          width = Number(sizeObj.width);
-          height = Number(sizeObj.height);
-        } else {
-          throw new Error(
-            `Invalid screen size object: ${JSON.stringify(screenSize)}`
-          );
-        }
-      } else {
-        throw new Error(`Invalid screen size format: ${screenSize}`);
-      }
-      try {
-        const densityNum = await adb.getScreenDensity();
-        this.deviceRatio = Number(densityNum) / 160;
-        debugPage(`Device display ratio: ${this.deviceRatio}`);
-      } catch (error) {
-        console.warn("Error getting device density:", error);
-      }
-      const { x: logicalWidth, y: logicalHeight } = this.reverseAdjustCoordinates(width, height);
-      this.screenSize = {
-        width: logicalWidth,
-        height: logicalHeight
-      };
-      return this.screenSize;
-    } catch (error) {
-      console.error("Error getting screen size:", error);
-      return { width: 720, height: 1280 };
-    }
-  }
-  /**
-   * Convert logical coordinates to physical coordinates, handling device ratio
-   * @param x Logical X coordinate
-   * @param y Logical Y coordinate
-   * @returns Physical coordinate point
-   */
-  adjustCoordinates(x, y) {
-    const ratio = this.deviceRatio;
-    return {
-      x: Math.round(x * ratio),
-      y: Math.round(y * ratio)
-    };
-  }
-  /**
-   * Convert physical coordinates to logical coordinates, handling device ratio
-   * @param x Physical X coordinate
-   * @param y Physical Y coordinate
-   * @returns Logical coordinate point
-   */
-  reverseAdjustCoordinates(x, y) {
-    const ratio = this.deviceRatio;
-    return {
-      x: Math.round(x / ratio),
-      y: Math.round(y / ratio)
-    };
+    return this.browser.getWindowSize();
   }
   async screenshotBase64() {
-    try {
-      debugPage("screenshotBase64 begin");
-      const { width, height } = await this.size();
-      const adb = await this.getAdb();
-      let screenshotBuffer;
-      try {
-        screenshotBuffer = await adb.takeScreenshot(null);
-      } catch (error) {
-        const screenshotPath = getTmpFile("png");
-        await adb.shell("screencap -p /sdcard/screenshot.png");
-        await adb.pull("/sdcard/screenshot.png", screenshotPath);
-        screenshotBuffer = await fs.promises.readFile(screenshotPath);
-      }
-      const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, {
-        width,
-        height
-      });
-      const result = `data:image/jpeg;base64,${resizedScreenshotBuffer.toString("base64")}`;
-      debugPage("screenshotBase64 end");
-      return result;
-    } catch (error) {
-      console.error("Error taking screenshot:", error);
-      throw error;
-    }
+    const { width, height } = await this.size();
+    const path = getTmpFile("png");
+    const screenshotBuffer = await this.browser.saveScreenshot(path);
+    const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, {
+      width,
+      height
+    });
+    fs.writeFileSync(path, resizedScreenshotBuffer);
+    return base64Encoded(path);
   }
   get mouse() {
     return {
@@ -1989,6 +1883,7 @@ var Page = class {
       drag: (from, to) => this.mouseDrag(from, to)
     };
   }
+  // Object that includes keyboard and mouse operations
   get keyboard() {
     return {
       type: (text) => this.keyboardType(text),
@@ -1999,218 +1894,221 @@ var Page = class {
     if (!element) {
       return;
     }
-    try {
-      await this.pushYadb();
-      const adb = await this.getAdb();
-      await adb.shell(
-        'app_process -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "~CLEAR~"'
-      );
-    } catch (error) {
-      console.error("Error clearing input:", error);
-    }
+    const ele = await this.browser.$(element.locator);
+    const blank = " ";
+    await this.keyboardType(blank);
+    await ele.clearValue();
   }
-  async url() {
-    try {
-      const adb = await this.getAdb();
-      const { appPackage, appActivity } = await adb.getFocusedPackageAndActivity();
-      return `${appPackage}/${appActivity}`;
-    } catch (error) {
-      console.error("Error getting URL:", error);
-      return "";
+  url() {
+    const platformName = this.browser.capabilities.platformName?.toLowerCase();
+    if (platformName === "ios") {
+      const bundleId = this.browser.capabilities.bundleId;
+      return bundleId;
+    }
+    if (platformName === "android") {
+      const appActivity = this.browser.capabilities.appActivity;
+      return appActivity;
     }
+    return "";
   }
+  // Scroll to top element
   async scrollUntilTop(startingPoint) {
     if (startingPoint) {
       await this.mouse.move(startingPoint.left, startingPoint.top);
     }
-    await this.mouseWheel(0, 9999999, 100);
+    await this.mouseWheel(0, -9999999, 100);
   }
+  // Scroll to bottom element
   async scrollUntilBottom(startingPoint) {
     if (startingPoint) {
       await this.mouse.move(startingPoint.left, startingPoint.top);
     }
-    await this.mouseWheel(0, -9999999, 100);
+    await this.mouseWheel(0, 9999999, 100);
   }
   async scrollUntilLeft(startingPoint) {
     if (startingPoint) {
       await this.mouse.move(startingPoint.left, startingPoint.top);
     }
-    await this.mouseWheel(9999999, 0, 100);
+    await this.mouseWheel(-9999999, 0, 100);
   }
   async scrollUntilRight(startingPoint) {
     if (startingPoint) {
       await this.mouse.move(startingPoint.left, startingPoint.top);
     }
-    await this.mouseWheel(-9999999, 0, 100);
+    await this.mouseWheel(9999999, 0, 100);
   }
+  // Scroll up one screen
   async scrollUp(distance, startingPoint) {
-    const { height } = await this.size();
+    const { height } = await this.browser.getWindowSize();
     const scrollDistance = distance || height * 0.7;
     if (startingPoint) {
       await this.mouse.move(startingPoint.left, startingPoint.top);
     }
-    await this.mouseWheel(0, scrollDistance, 1e3);
+    await this.mouseWheel(0, -scrollDistance, 1e3);
   }
+  // Scroll down one screen
   async scrollDown(distance, startingPoint) {
-    const { height } = await this.size();
+    const { height } = await this.browser.getWindowSize();
     const scrollDistance = distance || height * 0.7;
     if (startingPoint) {
       await this.mouse.move(startingPoint.left, startingPoint.top);
     }
-    await this.mouseWheel(0, -scrollDistance, 1e3);
+    await this.mouseWheel(0, scrollDistance, 1e3);
   }
   async scrollLeft(distance, startingPoint) {
-    const { width } = await this.size();
+    const { width } = await this.browser.getWindowSize();
     const scrollDistance = distance || width * 0.7;
     if (startingPoint) {
       await this.mouse.move(startingPoint.left, startingPoint.top);
     }
-    await this.mouseWheel(scrollDistance, 0, 1e3);
+    await this.mouseWheel(-scrollDistance, 0, 1e3);
   }
   async scrollRight(distance, startingPoint) {
-    const { width } = await this.size();
+    const { width } = await this.browser.getWindowSize();
     const scrollDistance = distance || width * 0.7;
     if (startingPoint) {
       await this.mouse.move(startingPoint.left, startingPoint.top);
     }
-    await this.mouseWheel(-scrollDistance, 0, 1e3);
-  }
-  async pushYadb() {
-    if (!this.yadbPushed) {
-      const adb = await this.getAdb();
-      const yadbBin = path.join(__dirname, "../../bin/yadb");
-      await adb.push(yadbBin, "/data/local/tmp");
-      this.yadbPushed = true;
-    }
+    await this.mouseWheel(scrollDistance, 0, 1e3);
   }
   async keyboardType(text) {
-    if (!text)
+    const actions = [];
+    for (const char of text) {
+      actions.push({ type: "keyDown", value: char });
+      actions.push({ type: "keyUp", value: char });
+    }
+    if (!actions.length) {
       return;
-    try {
-      const adb = await this.getAdb();
-      const isChinese = /[\p{Script=Han}\p{sc=Hani}]/u.test(text);
-      if (!isChinese) {
-        await adb.inputText(text);
-        return;
-      }
-      await this.execYadb(text);
-    } catch (error) {
-      console.error("Error typing text:", error);
-      throw error;
     }
+    await this.browser.performActions([
+      {
+        type: "key",
+        id: "keyboard",
+        actions
+      }
+    ]);
   }
   async keyboardPress(key) {
-    const keyCodeMap = {
-      Enter: 66,
-      Backspace: 67,
-      Tab: 61,
-      ArrowUp: 19,
-      ArrowDown: 20,
-      ArrowLeft: 21,
-      ArrowRight: 22,
-      Escape: 111,
-      Home: 3,
-      End: 123
-    };
-    const adb = await this.getAdb();
-    const keyCode = keyCodeMap[key];
-    if (keyCode !== void 0) {
-      await adb.keyevent(keyCode);
-    } else {
-      if (key.length === 1) {
-        const asciiCode = key.toUpperCase().charCodeAt(0);
-        if (asciiCode >= 65 && asciiCode <= 90) {
-          await adb.keyevent(asciiCode - 36);
-        }
+    await this.browser.performActions([
+      {
+        type: "key",
+        id: "keyboard",
+        actions: [
+          { type: "keyDown", value: key },
+          { type: "keyUp", value: key }
+        ]
       }
-    }
+    ]);
   }
   async keyboardPressAction(action) {
     if (Array.isArray(action)) {
       for (const act of action) {
-        await this.keyboardPress(act.key);
+        await this.browser.performActions([
+          {
+            type: "key",
+            id: "keyboard",
+            actions: [
+              { type: "keyDown", value: act.key },
+              { type: "keyUp", value: act.key }
+            ]
+          }
+        ]);
       }
     } else {
-      await this.keyboardPress(action.key);
+      await this.browser.performActions([
+        {
+          type: "key",
+          id: "keyboard",
+          actions: [
+            { type: "keyDown", value: action.key },
+            { type: "keyUp", value: action.key }
+          ]
+        }
+      ]);
     }
   }
   async mouseClick(x, y, button = "left") {
-    try {
-      if (button !== "left") {
-        console.warn(
-          `ADB only supports left mouse button clicks. Ignored request for ${button} button.`
-        );
+    await this.mouseMove(x, y);
+    await this.browser.performActions([
+      {
+        type: "pointer",
+        id: "mouse",
+        parameters: { pointerType: "mouse" },
+        actions: [
+          { type: "pointerMove", duration: 0, x, y },
+          { type: "pointerDown", button: buttonToNumber(button) },
+          { type: "pause", duration: 100 },
+          { type: "pointerUp", button: buttonToNumber(button) }
+        ]
       }
-      await this.mouseMove(x, y);
-      const adb = await this.getAdb();
-      const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
-      await adb.shell(`input tap ${adjustedX} ${adjustedY}`);
-    } catch (error) {
-      console.error("Error clicking:", error);
-    }
+    ]);
   }
   async mouseMove(x, y) {
-    return Promise.resolve();
+    await this.browser.performActions([
+      {
+        type: "pointer",
+        id: "mouse",
+        parameters: { pointerType: "mouse" },
+        actions: [{ type: "pointerMove", duration: 0, x, y }]
+      }
+    ]);
   }
   async mouseDrag(from, to) {
-    try {
-      const adb = await this.getAdb();
-      const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
-      const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
-      await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} 300`);
-    } catch (error) {
-      console.error("Error dragging:", error);
-    }
+    await this.browser.performActions([
+      {
+        type: "pointer",
+        id: "mouse",
+        parameters: { pointerType: "mouse" },
+        actions: [
+          { type: "pointerMove", duration: 0, x: from.x, y: from.y },
+          { type: "pointerDown", button: 0 },
+          { type: "pointerMove", duration: 500, x: to.x, y: to.y },
+          { type: "pointerUp", button: 0 }
+        ]
+      }
+    ]);
   }
   async mouseWheel(deltaX, deltaY, duration = 1e3) {
-    try {
-      const { width, height } = await this.size();
-      const n = 4;
-      const startX = deltaX < 0 ? (n - 1) * (width / n) : width / n;
-      const startY = deltaY < 0 ? (n - 1) * (height / n) : height / n;
-      const maxNegativeDeltaX = startX;
-      const maxPositiveDeltaX = (n - 1) * (width / n);
-      const maxNegativeDeltaY = startY;
-      const maxPositiveDeltaY = (n - 1) * (height / n);
-      deltaX = Math.max(
-        -maxNegativeDeltaX,
-        Math.min(deltaX, maxPositiveDeltaX)
-      );
-      deltaY = Math.max(
-        -maxNegativeDeltaY,
-        Math.min(deltaY, maxPositiveDeltaY)
-      );
-      const endX = startX + deltaX;
-      const endY = startY + deltaY;
-      const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(
-        startX,
-        startY
-      );
-      const { x: adjustedEndX, y: adjustedEndY } = this.adjustCoordinates(
-        endX,
-        endY
-      );
-      const adb = await this.getAdb();
-      await adb.shell(
-        `input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${duration}`
-      );
-    } catch (error) {
-      console.error("Error scrolling:", error);
-    }
+    const n = 4;
+    const windowSize = await this.browser.getWindowSize();
+    const startX = deltaX < 0 ? (n - 1) * (windowSize.width / n) : windowSize.width / n;
+    const startY = deltaY < 0 ? (n - 1) * (windowSize.height / n) : windowSize.height / n;
+    const maxNegativeDeltaX = startX;
+    const maxPositiveDeltaX = (n - 1) * (windowSize.width / n);
+    const maxNegativeDeltaY = startY;
+    const maxPositiveDeltaY = (n - 1) * (windowSize.height / n);
+    deltaX = Math.max(-maxNegativeDeltaX, Math.min(deltaX, maxPositiveDeltaX));
+    deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
+    await this.browser.performActions([
+      {
+        type: "pointer",
+        id: "finger1",
+        parameters: { pointerType: "touch" },
+        actions: [
+          { type: "pointerMove", duration: 0, x: startX, y: startY },
+          { type: "pointerDown", button: 0 },
+          { type: "pause", duration },
+          {
+            type: "pointerMove",
+            duration,
+            origin: "pointer",
+            // Use 'pointer' as the starting point
+            x: deltaX,
+            // X offset relative to the starting point
+            y: deltaY
+            // Y offset relative to the starting point
+          },
+          { type: "pointerUp", button: 0 }
+        ]
+      }
+    ]);
   }
   async destroy() {
-    try {
-      const adb = await this.getAdb();
-      await adb.shell("rm -f /sdcard/screenshot.png");
-      await adb.shell("rm -f /sdcard/window_dump.xml");
-    } catch (error) {
-      console.error("Error during cleanup:", error);
-    }
   }
 };
 export {
-  PageAgent as AndroidAgent,
-  Page as AndroidPage
+  PageAgent as AppiumAgent,
+  Page as AppiumPage
 };
 
-//# sourceMappingURL=android.js.map
+//# sourceMappingURL=appium.js.map