android-midscene-automation 0.2.3 → 1.1.0

package/artemis-main/artemis/drivers/__init__.py

@@ -0,0 +1,48 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal Device Driver framework for ARTEMIS."""
+
+from artemis.drivers.base import (
+    BaseDeviceDriver,
+    KeyCode,
+    ScreenData,
+    SwipeDirection,
+)
+from artemis.drivers.factory import create_driver, get_driver
+from artemis.drivers.mock.mock_driver import MockDeviceDriver
+from artemis.drivers.types import (
+    Bounds,
+    CoordinatesSelectorRequest,
+    DeviceInfo,
+    PercentagesSelectorRequest,
+    SwipeRequest,
+    TapOutput,
+)
+
+__all__ = [
+    "BaseDeviceDriver",
+    "KeyCode",
+    "ScreenData",
+    "SwipeDirection",
+    "Bounds",
+    "CoordinatesSelectorRequest",
+    "PercentagesSelectorRequest",
+    "SwipeRequest",
+    "TapOutput",
+    "DeviceInfo",
+    "create_driver",
+    "get_driver",
+    "MockDeviceDriver",
+]

package/artemis-main/artemis/drivers/android/adb_driver.py

@@ -0,0 +1,507 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Concrete Android Device Driver implementation using ADB and UIAutomator2."""
+
+import asyncio
+import base64
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Literal
+
+from adbutils import AdbClient, AdbDevice
+from artemis.config.paths import get_temp_dir
+from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection
+from artemis.toolchain import find_ffmpeg, find_scrcpy
+from artemis.utils.video import build_scrcpy_record_command
+from artemis.utils.ui_filter import filter_ui_hierarchy
+from third_party.mobile_use.clients.ui_automator_client import (
+    UIAutomatorClient,
+    parse_hierarchy_xml_to_elements,
+)
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+# KeyCode mapping from string name to Android keycode integer
+ANDROID_KEYCODE_MAP: dict[str, int] = {
+    "home": 3,
+    "back": 4,
+    "enter": 66,
+    "delete": 67,
+    "power": 26,
+    "app_switch": 187,
+    "volume_up": 24,
+    "volume_down": 25,
+}
+
+
+def _escape_for_adb_text(s: str) -> str:
+    """Escapes special characters for adb shell input text."""
+    return (
+        s.replace("\\", "\\\\")
+        .replace('"', '\\"')
+        .replace("'", "\\'")
+        .replace("`", "\\`")
+        .replace("$", "\\$")
+        .replace("&", "\\&")
+        .replace("|", "\\|")
+        .replace(";", "\\;")
+        .replace("<", "\\<")
+        .replace(">", "\\>")
+        .replace("(", "\\(")
+        .replace(")", "\\)")
+        .replace("*", "\\*")
+        .replace("?", "\\?")
+        .replace("~", "\\~")
+        .replace(" ", "%s")
+    )
+
+
+class AndroidAdbDriver(BaseDeviceDriver):
+    """Android device driver using ADB and UIAutomator2."""
+
+    def __init__(
+        self,
+        device_id: str,
+        adb_client: AdbClient,
+        ui_adb_client: UIAutomatorClient | None = None,
+        width: int = 1080,
+        height: int = 2400,
+    ):
+        self._device_id = device_id
+        self._adb_client = adb_client
+        self._ui_adb_client = ui_adb_client
+        self._width = width
+        self._height = height
+        self._device: AdbDevice | None = None
+        self._recording_process: asyncio.subprocess.Process | None = None
+        self._recording_output_path: Path | None = None
+
+    @property
+    def device_id(self) -> str:
+        return self._device_id
+
+    @property
+    def screen_size(self) -> tuple[int, int]:
+        return (self._width, self._height)
+
+    @property
+    def device(self) -> AdbDevice:
+        if self._device is None:
+            self._device = self._adb_client.device(serial=self._device_id)
+        return self._device
+
+    async def connect(self) -> None:
+        """Verify device availability."""
+        try:
+            state = self.device.get_state()
+            logger.info(f"Android device '{self._device_id}' connected (state: {state})")
+        except Exception as e:
+            logger.warning(f"Connecting to Android device '{self._device_id}': {e}")
+
+    async def disconnect(self) -> None:
+        """Clean up ongoing recordings and tunnels."""
+        if self._recording_process:
+            await self.stop_video_recording()
+        if self._ui_adb_client and hasattr(self._ui_adb_client, "disconnect"):
+            await asyncio.to_thread(self._ui_adb_client.disconnect)
+
+    async def get_screen_data(self, skip_settling: bool = False) -> ScreenData:
+        """Captures screenshot and XML hierarchy concurrently."""
+        if not skip_settling:
+            await asyncio.sleep(0.3)
+
+        # 1. Capture raw screenshot
+        screenshot_bytes: bytes = b""
+        screenshot_base64: str = ""
+        ui_hierarchy_xml: str | None = None
+        ui_elements: list[dict[str, Any]] = []
+
+        # Check if ui_adb_client has direct screen data support
+        if self._ui_adb_client and hasattr(self._ui_adb_client, "get_screen_data"):
+            try:
+                raw_ui_data = self._ui_adb_client.get_screen_data()
+                if (
+                    hasattr(raw_ui_data, "base64")
+                    and isinstance(raw_ui_data.base64, str)
+                    and raw_ui_data.base64
+                ):
+                    screenshot_base64 = raw_ui_data.base64
+                    screenshot_bytes = base64.b64decode(screenshot_base64)
+                if hasattr(raw_ui_data, "width") and isinstance(raw_ui_data.width, int):
+                    self._width = raw_ui_data.width
+                if hasattr(raw_ui_data, "height") and isinstance(raw_ui_data.height, int):
+                    self._height = raw_ui_data.height
+                if hasattr(raw_ui_data, "hierarchy_xml") and isinstance(
+                    raw_ui_data.hierarchy_xml, str
+                ):
+                    ui_hierarchy_xml = raw_ui_data.hierarchy_xml
+                if hasattr(raw_ui_data, "elements"):
+                    raw_elements = raw_ui_data.elements
+                    if isinstance(raw_elements, dict):
+                        ui_elements = [raw_elements]
+                    elif isinstance(raw_elements, list):
+                        ui_elements = raw_elements
+            except Exception as e:
+                logger.debug(f"Direct ui_adb_client.get_screen_data error: {e}")
+
+        if not screenshot_base64 or not isinstance(screenshot_base64, str):
+            try:
+                pil_img = await asyncio.to_thread(self.device.screenshot)
+                if hasattr(pil_img, "save"):
+                    buf = BytesIO()
+                    pil_img.save(buf, format="PNG")
+                    screenshot_bytes = buf.getvalue()
+                    self._width, self._height = pil_img.size
+                    screenshot_base64 = base64.b64encode(screenshot_bytes).decode("utf-8")
+            except Exception as e:
+                logger.warning(f"Device screenshot capture failed on {self._device_id}: {e}")
+
+        if not screenshot_base64 or not isinstance(screenshot_base64, str):
+            # Fallback for headless testing environments
+            screenshot_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+            screenshot_bytes = base64.b64decode(screenshot_base64)
+
+        # 2. Capture UI Hierarchy via UIAutomator
+        if self._ui_adb_client and not ui_elements:
+            try:
+                if hasattr(self._ui_adb_client, "get_hierarchy"):
+                    res = self._ui_adb_client.get_hierarchy()
+                    hierarchy = await res if asyncio.iscoroutine(res) else res
+                    if isinstance(hierarchy, str):
+                        ui_hierarchy_xml = hierarchy
+                        ui_elements = parse_hierarchy_xml_to_elements(hierarchy)
+                    else:
+                        ui_elements = hierarchy
+                elif hasattr(self._ui_adb_client, "get_ui_elements"):
+                    res = self._ui_adb_client.get_ui_elements()
+                    ui_elements = await res if asyncio.iscoroutine(res) else res
+            except Exception as e:
+                logger.debug(f"UI hierarchy extraction failed: {e}")
+
+        if isinstance(ui_elements, dict):
+            clean_ui_elements = [ui_elements]
+        elif isinstance(ui_elements, list):
+            clean_ui_elements = ui_elements
+        else:
+            clean_ui_elements = []
+
+        w = self._width if isinstance(self._width, int) else 1080
+        h = self._height if isinstance(self._height, int) else 2400
+        clean_ui_elements = filter_ui_hierarchy(
+            clean_ui_elements,
+            screen_width=w,
+            screen_height=h,
+        )
+        clean_b64 = (
+            screenshot_base64
+            if isinstance(screenshot_base64, str) and screenshot_base64
+            else "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+        )
+        clean_bytes = (
+            screenshot_bytes
+            if isinstance(screenshot_bytes, bytes) and screenshot_bytes
+            else base64.b64decode(clean_b64)
+        )
+
+        return ScreenData(
+            screenshot_bytes=clean_bytes,
+            screenshot_base64=clean_b64,
+            ui_hierarchy_xml=ui_hierarchy_xml,
+            ui_elements=clean_ui_elements,
+            width=w,
+            height=h,
+            platform="android",
+        )
+
+    async def tap(
+        self,
+        x: int,
+        y: int,
+        duration_ms: int = 100,
+        times: int = 1,
+        delay_ms: int = 100,
+    ) -> bool:
+        try:
+            if duration_ms >= 500:
+                cmd = f"input swipe {x} {y} {x} {y} {duration_ms}"
+            else:
+                if times <= 1:
+                    cmd = f"input tap {x} {y}"
+                else:
+                    taps = [f"input tap {x} {y}"] * times
+                    cmd = f" && sleep {delay_ms / 1000.0:.3f} && ".join(taps)
+
+            logger.info(f"[ADB] {cmd}")
+            await asyncio.to_thread(self.device.shell, cmd)
+            return True
+        except Exception as e:
+            logger.error(f"Tap failed at ({x}, {y}): {e}")
+            return False
+
+    async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool:
+        return await self.tap(x=x, y=y, duration_ms=duration_ms)
+
+    async def swipe(
+        self,
+        start_x: int,
+        start_y: int,
+        end_x: int,
+        end_y: int,
+        duration_ms: int = 800,
+    ) -> bool:
+        try:
+            cmd = f"input swipe {start_x} {start_y} {end_x} {end_y} {duration_ms}"
+            logger.info(f"[ADB] {cmd}")
+            await asyncio.to_thread(self.device.shell, cmd)
+            return True
+        except Exception as e:
+            logger.error(f"Swipe failed from ({start_x},{start_y}) to ({end_x},{end_y}): {e}")
+            return False
+
+    async def swipe_direction(
+        self,
+        direction: SwipeDirection | Literal["up", "down", "left", "right"],
+        duration_ms: int = 800,
+    ) -> bool:
+        dir_str = str(direction).lower()
+        mid_x = int(
+            self._width * 0.6
+        )  # Lock to 60% width to avoid edge gestures and alphabet fast-scroll sidebar
+        mid_y = self._height // 2
+
+        if dir_str == "up":
+            # Drag bottom to top -> scrolls down (0.7 -> 0.3 leaves ~50-60% overlap, 800ms prevents fling)
+            return await self.swipe(
+                mid_x, int(self._height * 0.7), mid_x, int(self._height * 0.3), duration_ms
+            )
+        elif dir_str == "down":
+            # Drag top to bottom -> scrolls up
+            return await self.swipe(
+                mid_x, int(self._height * 0.3), mid_x, int(self._height * 0.7), duration_ms
+            )
+        elif dir_str == "left":
+            # Drag right to left -> scrolls right
+            return await self.swipe(
+                int(self._width * 0.75), mid_y, int(self._width * 0.25), mid_y, duration_ms
+            )
+        elif dir_str == "right":
+            # Drag left to right -> scrolls left
+            return await self.swipe(
+                int(self._width * 0.25), mid_y, int(self._width * 0.75), mid_y, duration_ms
+            )
+        return False
+
+    async def input_text(self, text: str, clear_existing: bool = True) -> bool:
+        try:
+            if clear_existing:
+                # Safe & robust clearing: Move to End -> Shift+Home selection -> Delete -> Fallback backspaces
+                clear_cmd = (
+                    "input keyevent 123 && "
+                    "input keyevent --meta 1 122 && "
+                    "input keyevent 67 && "
+                    "input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
+                )
+                await asyncio.to_thread(self.device.shell, clear_cmd)
+            else:
+                # Append mode: move cursor to the very end of existing text
+                await asyncio.to_thread(
+                    self.device.shell,
+                    "input keyevent 123",  # KEYCODE_MOVE_END
+                )
+
+            # Normalize literal escaped newlines from LLM / tool call serialization
+            norm_text = text.replace(r"\r\n", "\n").replace(r"\n", "\n").replace(r"\r", "\n")
+
+            # 1. Tier 1: Try clipboard injection + KEYCODE_PASTE (Zero IME interference, preserves multiline, works for all charsets)
+            if self._ui_adb_client:
+                try:
+                    set_clip_ok = False
+                    if hasattr(self._ui_adb_client, "set_clipboard"):
+                        set_clip_ok = self._ui_adb_client.set_clipboard(norm_text)
+                    elif hasattr(self._ui_adb_client, "_device") and self._ui_adb_client._device:
+                        self._ui_adb_client._device.set_clipboard(norm_text)
+                        set_clip_ok = True
+
+                    if set_clip_ok:
+                        await asyncio.to_thread(self.device.shell, "input keyevent 279")
+                        return True
+                except Exception as e:
+                    logger.debug(f"Clipboard paste fallback to ADB input: {e}")
+
+            # 2. Tier 2: Check if ADBKeyboard is currently active
+            try:
+                default_ime = await asyncio.to_thread(
+                    self.device.shell, "settings get secure default_input_method"
+                )
+                if "adbkeyboard" in str(default_ime).lower():
+                    b64_text = base64.b64encode(norm_text.encode("utf-8")).decode("utf-8")
+                    broadcast_cmd = f"am broadcast -a ADB_INPUT_B64 --es msg '{b64_text}'"
+                    await asyncio.to_thread(self.device.shell, broadcast_cmd)
+                    return True
+            except Exception as e:
+                # ADBKeyboard probe/broadcast failed; fall through to native input.
+                logger.debug(f"ADBKeyboard IME path failed, falling back to ADB input: {e}")
+
+            # 3. Tier 3: Universal Native ADB input text fallback
+            lines = norm_text.split("\n")
+            for i, line in enumerate(lines):
+                if i > 0:
+                    # Send Enter key between lines
+                    await asyncio.to_thread(self.device.shell, "input keyevent 66")
+                if line:
+                    escaped = _escape_for_adb_text(line)
+                    await asyncio.to_thread(self.device.shell, f"input text {escaped}")
+            return True
+        except Exception as e:
+            logger.error(f"Input text failed for '{text}': {e}")
+            return False
+
+    async def press_key(self, key: KeyCode | str | int) -> bool:
+        try:
+            keycode_val = key
+            if isinstance(key, (KeyCode, str)):
+                key_name = str(key).lower().replace("keycode.", "")
+                keycode_val = ANDROID_KEYCODE_MAP.get(key_name, key)
+
+            await asyncio.to_thread(self.device.shell, f"input keyevent {keycode_val}")
+            return True
+        except Exception as e:
+            logger.error(f"Press key failed for '{key}': {e}")
+            return False
+
+    async def launch_app(self, package_name: str) -> bool:
+        try:
+            cmd = f"monkey -p {package_name} -c android.intent.category.LAUNCHER 1"
+            await asyncio.to_thread(self.device.shell, cmd)
+            return True
+        except Exception as e:
+            logger.error(f"Launch app failed for '{package_name}': {e}")
+            return False
+
+    async def stop_app(self, package_name: str) -> bool:
+        try:
+            await asyncio.to_thread(self.device.shell, f"am force-stop {package_name}")
+            return True
+        except Exception as e:
+            logger.error(f"Stop app failed for '{package_name}': {e}")
+            return False
+
+    async def get_current_package(self) -> str | None:
+        try:
+            # 1. Try modern current_app from adbutils
+            if hasattr(self.device, "current_app"):
+                try:
+                    app_info = await asyncio.to_thread(self.device.current_app)
+                    if app_info and getattr(app_info, "package", None):
+                        return app_info.package
+                except Exception as e:
+                    # current_app is unsupported on some devices; use dumpsys below.
+                    logger.debug(f"current_app query failed, falling back to dumpsys: {e}")
+
+            # 2. Structured dumpsys extraction
+            out = await asyncio.to_thread(
+                self.device.shell, "dumpsys window displays | grep -E 'mCurrentFocus|mFocusedApp'"
+            )
+            for line in str(out).splitlines():
+                if "/" in line:
+                    for token in line.split():
+                        if "/" in token and "." in token:
+                            clean = token.split("/")[0].strip("{} ,")
+                            if clean and not clean.startswith("Window") and "." in clean:
+                                return clean
+        except Exception as e:
+            logger.debug(f"Error querying current package: {e}")
+        return None
+
+    async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str:
+        try:
+            return await asyncio.wait_for(
+                asyncio.to_thread(self.device.shell, command),
+                timeout=timeout_seconds,
+            )
+        except Exception as e:
+            return f"Error: {e}"
+
+    async def start_video_recording(self, output_dir: Path | None = None) -> None:
+        """Starts screen recording via scrcpy in background."""
+        out_dir = output_dir or get_temp_dir("recordings")
+        out_dir.mkdir(parents=True, exist_ok=True)
+        self._recording_mkv_path = out_dir / "recording.mkv"
+        self._recording_output_path = out_dir / "recording.mp4"
+        logger.info(f"Starting scrcpy video recording to {self._recording_mkv_path}...")
+        try:
+            scrcpy_bin = find_scrcpy()
+            cmd = build_scrcpy_record_command(
+                scrcpy_bin,
+                self.device_id,
+                self._recording_mkv_path,
+                lock_capture_orientation=False,
+            )
+            self._scrcpy_process = await asyncio.create_subprocess_exec(
+                *cmd,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            await asyncio.sleep(0.5)
+        except Exception as e:
+            logger.warning(f"Failed to start scrcpy subprocess: {e}")
+
+    async def stop_video_recording(self) -> str | None:
+        """Stops background video capture and returns local recording file path."""
+        if hasattr(self, "_scrcpy_process") and self._scrcpy_process:
+            try:
+                self._scrcpy_process.terminate()
+                await asyncio.wait_for(self._scrcpy_process.wait(), timeout=5.0)
+            except (TimeoutError, ProcessLookupError, OSError) as e:
+                # Already exited, or did not stop within the timeout; a lingering
+                # scrcpy may keep the MKV file locked on Windows.
+                logger.debug(f"scrcpy process did not terminate cleanly: {e}")
+            self._scrcpy_process = None
+
+        mkv = getattr(self, "_recording_mkv_path", None)
+        mp4 = getattr(self, "_recording_output_path", None)
+
+        if mkv and mkv.exists() and mp4:
+            try:
+                proc = await asyncio.create_subprocess_exec(
+                    find_ffmpeg(),
+                    "-y",
+                    "-i",
+                    str(mkv),
+                    "-c",
+                    "copy",
+                    "-movflags",
+                    "+faststart",
+                    str(mp4),
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                )
+                await proc.wait()
+                if mp4.exists() and mp4.stat().st_size > 0:
+                    try:
+                        mkv.unlink()
+                    except OSError:
+                        # Best-effort cleanup of the intermediate MKV file.
+                        pass
+                    return str(mp4)
+            except Exception as e:
+                logger.warning(f"Failed to convert MKV to MP4 in AdbDriver: {e}")
+
+        if mp4 and mp4.exists():
+            return str(mp4)
+        if mkv and mkv.exists():
+            return str(mkv)
+        return None

package/artemis-main/artemis/drivers/android/hierarchy.py

@@ -0,0 +1,64 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Android XML Layout Hierarchy parser and semantic node extractor."""
+
+import re
+from typing import Any
+import xml.etree.ElementTree as ET
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def parse_ui_hierarchy(xml_string: str) -> list[dict[str, Any]]:
+    """Parses raw Android XML hierarchy dump into structured interactive UI nodes."""
+    if not xml_string:
+        return []
+
+    elements: list[dict[str, Any]] = []
+    try:
+        root = ET.fromstring(xml_string)
+        for node in root.iter():
+            attribs = node.attrib
+            bounds_str = attribs.get("bounds", "")
+            match = re.match(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]", bounds_str)
+            if not match:
+                continue
+
+            x1, y1, x2, y2 = map(int, match.groups())
+            # Filter zero-area nodes
+            if x1 >= x2 or y1 >= y2:
+                continue
+
+            text = attribs.get("text") or attribs.get("content-desc") or ""
+            class_name = attribs.get("class", "")
+            resource_id = attribs.get("resource-id", "")
+            clickable = attribs.get("clickable") == "true"
+
+            if text or clickable or resource_id:
+                elements.append(
+                    {
+                        "text": text,
+                        "resource_id": resource_id,
+                        "class": class_name,
+                        "clickable": clickable,
+                        "bounds": [x1, y1, x2, y2],
+                        "center": [(x1 + x2) // 2, (y1 + y2) // 2],
+                    }
+                )
+    except Exception as e:
+        logger.warning(f"Failed to parse XML hierarchy: {e}")
+
+    return elements