android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/third_party/mobile_use/clients/ui_automator_client.py

@@ -0,0 +1,482 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""UIAutomator2 client for Android device screen data retrieval.
+
+Provides an alternative to the Maestro-based screen API with direct device
+access.
+Handles Maestro blocker detection and removal before connecting.
+"""
+
+import base64
+from io import BytesIO
+import re as _re
+import subprocess
+import time
+from typing import TYPE_CHECKING
+import xml.etree.ElementTree as ET
+
+from PIL import Image
+from pydantic import BaseModel
+import uiautomator2 as u2
+
+from artemis.runtime.adb_endpoint import adb_command
+from artemis.runtime.awake_service import ensure_device_awake
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from uiautomator2 import Device
+
+logger = get_logger(__name__)
+
+
+MAESTRO_PACKAGE = "dev.mobile.maestro"
+
+# Boolean attributes of a UIAutomator hierarchy node.
+UIAUTOMATOR_BOOL_ATTRIBUTES = (
+    "checkable",
+    "checked",
+    "clickable",
+    "enabled",
+    "focusable",
+    "focused",
+    "scrollable",
+    "long-clickable",
+    "password",
+    "selected",
+)
+
+
+class UIAutomatorScreenData(BaseModel):
+    """Screen data response from UIAutomator2."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+
+    base64: str
+    hierarchy_xml: str
+    elements: list[dict]
+    width: int
+    height: int
+
+
+def parse_hierarchy_xml_to_elements(hierarchy_xml: str) -> list[dict]:
+    """Parse uiautomator2 XML hierarchy into a flat list of element dictionaries.
+
+    The output format matches the existing screen API format with attributes
+    like:
+    - resource-id
+    - text
+    - content-desc
+    - bounds (as string "[x1,y1][x2,y2]")
+    - class
+    - package
+    - checkable, checked, clickable, enabled, focusable, focused
+    - scrollable, long-clickable, password, selected
+
+    Args:
+        hierarchy_xml: XML string from uiautomator2.dump_hierarchy()
+
+    Returns:
+        Flat list of element dictionaries
+    """
+    elements: list[dict] = []
+
+    try:
+        root = ET.fromstring(hierarchy_xml)
+    except ET.ParseError as e:
+        logger.error(f"Failed to parse hierarchy XML: {e}")
+        return elements
+
+    def _extract_element(node: ET.Element) -> None:
+        """Recursively extract elements from XML nodes."""
+        element: dict = {}
+
+        for attr_name, attr_value in node.attrib.items():
+            if attr_name == "resource-id":
+                element["resource-id"] = attr_value
+            elif attr_name == "text":
+                element["text"] = attr_value
+            elif attr_name == "content-desc":
+                element["content-desc"] = attr_value
+                element["accessibilityText"] = attr_value
+            elif attr_name == "bounds":
+                element["bounds"] = attr_value
+
+                match = _re.match(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]", attr_value)
+                if match:
+                    x1, y1, x2, y2 = map(int, match.groups())
+                    element["parsed_bounds"] = {
+                        "left": x1,
+                        "top": y1,
+                        "right": x2,
+                        "bottom": y2,
+                    }
+            elif attr_name == "class":
+                element["class"] = attr_value
+            elif attr_name == "package":
+                element["package"] = attr_value
+            elif attr_name in UIAUTOMATOR_BOOL_ATTRIBUTES:
+                element[attr_name] = attr_value
+            else:
+                element[attr_name] = attr_value
+
+        if element:
+            elements.append(element)
+
+        for child in node:
+            _extract_element(child)
+
+    _extract_element(root)
+
+    return elements
+
+
+def pil_to_base64(img: Image.Image, format: str = "JPEG", quality: int = 80) -> str:
+    """Convert PIL Image to base64 string with high-quality compression."""
+    buffer = BytesIO()
+    if format.upper() in ("JPEG", "JPG"):
+        format = "JPEG"
+        if img.mode in ("RGBA", "P"):
+            img = img.convert("RGB")
+        img.save(buffer, format=format, quality=quality, optimize=True)
+    else:
+        img.save(buffer, format=format)
+    return base64.b64encode(buffer.getvalue()).decode("utf-8")
+
+
+def _is_package_installed(device_id: str, pkg: str) -> bool:
+    """Check if a package is installed on the device."""
+    try:
+        result = subprocess.run(
+            adb_command(["-s", device_id, "shell", "pm", "list", "packages"]),
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if result.returncode != 0:
+            logger.warning(f"Failed to list packages: {result.stderr}")
+            return False
+
+        lines = result.stdout.splitlines()
+        target = f"package:{pkg}"
+        return target in lines
+    except subprocess.TimeoutExpired:
+        logger.warning("Timeout checking installed packages")
+        return False
+    except (OSError, subprocess.SubprocessError) as e:
+        logger.warning(f"Error checking installed packages: {e}")
+        return False
+
+
+def _uninstall_package(device_id: str, pkg: str) -> bool:
+    """Uninstall a package from the device for the current user."""
+    try:
+        result = subprocess.run(
+            adb_command(
+                [
+                    "-s",
+                    device_id,
+                    "shell",
+                    "pm",
+                    "uninstall",
+                    "--user",
+                    "0",
+                    pkg,
+                ]
+            ),
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        if result.returncode == 0:
+            logger.info(f"Successfully uninstalled {pkg}")
+            return True
+        else:
+            logger.warning(f"Failed to uninstall {pkg}: {result.stderr}")
+            return False
+    except subprocess.TimeoutExpired:
+        logger.warning(f"Timeout uninstalling {pkg}")
+        return False
+    except Exception as e:
+        logger.warning(f"Error uninstalling {pkg}: {e}")
+        return False
+
+
+def _ensure_maestro_not_installed(device_id: str) -> None:
+    """Check if Maestro is installed and uninstall it.
+
+    Maestro conflicts with uiautomator2 - if Maestro's package is installed,
+    u2.connect() will fail. This function ensures Maestro is removed before
+    attempting to connect.
+    """
+    if _is_package_installed(device_id, MAESTRO_PACKAGE):
+        logger.warning(
+            f"Maestro ({MAESTRO_PACKAGE}) detected - uninstalling to enable UIAutomator2..."
+        )
+        _uninstall_package(device_id, MAESTRO_PACKAGE)
+
+
+class UIAutomatorClient:
+    """UIAutomator2 client for Android screen data retrieval.
+
+    This client uses uiautomator2 library for direct device communication,
+    providing faster hierarchy and screenshot retrieval compared to Maestro.
+
+    Important: Maestro must not be installed on the device as it conflicts
+    with uiautomator2. This client automatically handles Maestro removal.
+    """
+
+    def __init__(self, device_id: str):
+        """Initialize the UIAutomator client.
+
+        Args:
+            device_id: The Android device serial number (e.g., "emulator-5554")
+        """
+        self._device_id = device_id
+        self._device: Device | None = None
+        self._awake_strategy: str | None = None
+
+    def _ensure_connected(self) -> "Device":
+        """Ensure connection to the device, handling Maestro blocker.
+
+        Returns:
+            Connected uiautomator2 Device instance
+        """
+        if self._device is not None:
+            try:
+                # Quick check if connection is still alive
+                self._device.info
+                return self._device
+            except Exception:
+                logger.warning("UIAutomator2 connection lost, reconnecting...")
+                self._device = None
+
+        # Ensure Maestro is not blocking us
+        _ensure_maestro_not_installed(self._device_id)
+
+        # Enroll the device in the containing Artemis service lifetime before
+        # UIAutomator2 reads the screen. Client disconnects do not release it.
+        if self._awake_strategy is None:
+            self._awake_strategy = ensure_device_awake(self._device_id)
+
+        # Retry connections while ADB recovers from a reboot or USB disconnect.
+        logger.info(f"Connecting UIAutomator2 to device: {self._device_id}")
+        last_error: Exception | None = None
+        for attempt in range(3):
+            if attempt:
+                time.sleep(1.0 * attempt)
+            try:
+                self._device = u2.connect(self._device_id)
+                break
+            except Exception as exc:
+                last_error = exc
+                logger.warning(
+                    f"UIAutomator2 connect attempt {attempt + 1}/3 to"
+                    f" {self._device_id} failed: {exc}"
+                )
+        else:
+            self._awake_strategy = None
+            raise last_error
+        logger.info("UIAutomator2 connected successfully")
+        return self._device
+
+    def connect(self) -> None:
+        """Connect and activate this session's awake strategy without an action."""
+        self._ensure_connected()
+
+    def press_key(self, key: str):
+        """Press a key on the device.
+
+        Args:
+            key: Key to press (e.g., "home", "back", "enter"...)
+        """
+        device = self._ensure_connected()
+        return device.press(key=key)
+
+    def send_text(self, text: str) -> None:
+        """Send text input to the device using FastInputIME.
+
+        This method supports special characters (e.g., 'ö') that ADB shell
+        input text cannot handle. The original IME is restored after input.
+
+        Args:
+            text: The text to input
+        """
+
+        device = self._ensure_connected()
+        device.set_fastinput_ime(True)
+        # Allow IME to initialize
+        time.sleep(0.3)
+        try:
+            device.send_keys(text)
+            # Give FastInputIME time to process the broadcast and commit text
+            # before switching it off and killing it.
+            time.sleep(0.5)
+        finally:
+            device.set_fastinput_ime(False)
+
+    def clear_text(self) -> bool:
+        """Clear text of the currently focused element using UIAutomator2.
+
+        Returns True on success, False on failure.
+        """
+        try:
+            device = self._ensure_connected()
+            device(focused=True).clear_text()
+            return True
+        except Exception as e:
+            logger.error(f"UIAutomator2 clear_text failed: {e}")
+            return False
+
+    def set_clipboard(self, text: str) -> bool:
+        """Set text to system clipboard via UIAutomator2.
+
+        Args:
+            text: The string to set in the device clipboard.
+
+        Returns:
+            True if successful, False otherwise.
+        """
+        try:
+            device = self._ensure_connected()
+            device.set_clipboard(text)
+            return True
+        except Exception as e:
+            logger.debug(f"UIAutomator2 set_clipboard failed: {e}")
+            return False
+
+    def get_hierarchy(self) -> str:
+        """Get the UI hierarchy XML from the device.
+
+        Returns:
+            Compressed UI hierarchy XML string
+        """
+        device = self._ensure_connected()
+        return device.dump_hierarchy(compressed=True)
+
+    def get_screenshot(self) -> Image.Image | None:
+        """Capture a screenshot from the device.
+
+        Uses adb screencap for more reliable capture of all layers (including
+        popups), falling back to UIAutomator2 if it fails.
+
+        Returns:
+            PIL Image or None if capture failed
+        """
+        try:
+            result = subprocess.run(
+                adb_command(["-s", self._device_id, "exec-out", "screencap", "-p"]),
+                stdin=subprocess.DEVNULL,
+                capture_output=True,
+                timeout=10,
+                check=True,
+            )
+            return Image.open(BytesIO(result.stdout))
+        except (OSError, subprocess.SubprocessError) as e:
+            logger.warning(
+                f"Failed to capture screenshot via adb: {e}, falling back to uiautomator2"
+            )
+            device = self._ensure_connected()
+            return device.screenshot()
+
+    def get_screenshot_base64(self) -> str | None:
+        """Capture a screenshot and return as base64 string.
+
+        Returns:
+            Base64 encoded PNG screenshot or None if capture failed
+        """
+        screenshot = self.get_screenshot()
+        if screenshot is None:
+            return None
+        return pil_to_base64(screenshot, format="JPEG")
+
+    def get_screen_data(self) -> UIAutomatorScreenData:
+        """Get complete screen data including screenshot and hierarchy.
+
+        Returns:
+            UIAutomatorScreenData with screenshot, hierarchy, elements, and
+            dimensions
+        """
+        device = self._ensure_connected()
+
+        # Get screenshot
+        screenshot = self.get_screenshot()
+        if screenshot is None:
+            raise RuntimeError("Failed to capture screenshot")
+
+        # Get hierarchy
+        hierarchy_xml = device.dump_hierarchy(compressed=True)
+
+        # Parse XML to flat elements list
+        elements = parse_hierarchy_xml_to_elements(hierarchy_xml)
+
+        return UIAutomatorScreenData(
+            base64=pil_to_base64(screenshot, format="JPEG"),
+            hierarchy_xml=hierarchy_xml,
+            elements=elements,
+            width=screenshot.width,
+            height=screenshot.height,
+        )
+
+    def disconnect(self, stop_server: bool = False) -> None:
+        """Disconnect this client without ending the host's awake lifetime.
+
+        ``stop_server`` also kills uiautomator2's device server. Its UiAutomation
+        connection suppresses every accessibility service (the Artemis helper
+        included) for as long as it lives, so a caller that prefers the helper
+        must release it; the pure UIAutomator2 backend keeps it for fast reconnects.
+        """
+        device = self._device
+        self._awake_strategy = None
+        self._device = None
+        if stop_server and device is not None:
+            self.stop_server(device)
+        logger.info("UIAutomator2 client disconnected")
+
+    def stop_server(self, device: "Device | None" = None) -> bool:
+        """Kill uiautomator2's on-device server (see :meth:`disconnect`)."""
+        device = device or self._device
+        if device is None:
+            return False
+        try:
+            device.stop_uiautomator()
+            logger.info(f"UIAutomator2 device server stopped on {self._device_id}")
+            return True
+        except (
+            OSError,
+            RuntimeError,
+            subprocess.SubprocessError,
+            u2.exceptions.DeviceError,
+        ) as exc:
+            logger.debug(f"Stopping the UIAutomator2 server on {self._device_id} failed: {exc}")
+            return False
+
+
+def get_client(device_id: str) -> UIAutomatorClient:
+    """Factory function to create a UIAutomatorClient.
+
+    Args:
+        device_id: The Android device serial number
+
+    Returns:
+        UIAutomatorClient instance
+    """
+    return UIAutomatorClient(device_id=device_id)

package/artemis-main/third_party/mobile_use/config/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

package/artemis-main/third_party/mobile_use/config/llm.py

@@ -0,0 +1,183 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""LLM provider models, the base LLM configuration model and its loaders."""
+
+from collections.abc import Callable
+import os
+from pathlib import Path
+from typing import Any, Literal
+
+import google.auth
+from google.auth.exceptions import DefaultCredentialsError
+from pydantic import BaseModel, ValidationError
+
+from artemis.config.constants import AgentNode, LLMProvider, LLMUtilsNode
+from artemis.config.settings import settings
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.utils.file import load_jsonc
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+LLMUtilsNodeWithFallback = LLMUtilsNode
+AgentNodeWithFallback = AgentNode
+
+
+def validate_vertex_ai_credentials() -> None:
+    """Validate Google Application Default Credentials for VertexAI provider."""
+    try:
+        _, project = google.auth.default()
+        if not project:
+            raise Exception("VertexAI requires a Google Cloud project to be set.")
+    except DefaultCredentialsError as e:
+        raise Exception(
+            f"VertexAI requires valid Google Application Default Credentials (ADC): {e}"
+        )
+
+
+class LLM(BaseModel):
+    """Base model representing an LLM model provider and runtime parameters."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    provider: LLMProvider
+    model: str
+    temperature: float | None = None
+    thinking_budget: int | None = None
+    thinking_level: Literal["minimal", "low", "medium", "high"] | None = None
+    reasoning_effort: Literal["none", "low", "medium", "high"] | None = None
+    include_thoughts: bool | None = None
+    enable_grounding: bool | None = None
+
+    def validate_provider(self, name: str) -> None:
+        """Ensure the required API key or credentials exist in settings for this provider."""
+        if self.provider == "openai":
+            if not settings.OPENAI_API_KEY:
+                raise Exception(f"{name} requires OPENAI_API_KEY in .env")
+        elif self.provider == "google":
+            if not settings.GOOGLE_API_KEY:
+                raise Exception(f"{name} requires GOOGLE_API_KEY in .env")
+        elif self.provider == "vertexai":
+            validate_vertex_ai_credentials()
+        elif self.provider == "anthropic":
+            if not (settings.ANTHROPIC_API_KEY or os.environ.get("ANTHROPIC_API_KEY")):
+                raise Exception(f"{name} requires ANTHROPIC_API_KEY in .env")
+        elif self.provider == "openrouter":
+            if not settings.OPEN_ROUTER_API_KEY:
+                raise Exception(f"{name} requires OPEN_ROUTER_API_KEY in .env")
+        elif self.provider == "xai":
+            if not settings.XAI_API_KEY:
+                raise Exception(f"{name} requires XAI_API_KEY in .env")
+
+    def __str__(self) -> str:
+        return f"{self.provider}/{self.model}"
+
+
+class LLMWithFallback(LLM):
+    """LLM configuration with automatic secondary fallback and timeout specs."""
+
+    fallback: LLM
+    fix_model: str | None = None
+    timeout: float | None = None
+
+    def __str__(self) -> str:
+        return f"{self.provider}/{self.model} (fallback: {self.fallback})"
+
+
+class LLMConfigUtils(BaseModel):
+    """Configuration container for auxiliary utility agents/nodes."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    outputter: LLMWithFallback
+    hopper: LLMWithFallback
+    video_analyzer: LLMWithFallback | None = None
+    object_detector: LLMWithFallback | None = None
+
+
+class LLMConfigBase(BaseModel):
+    """Base LLM configuration: planner, utility nodes and their accessors."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    planner: LLMWithFallback
+    utils: LLMConfigUtils
+
+    def __str__(self) -> str:
+        return f"""
+📃 Planner: {self.planner}
+🧩 Utils:
+    🔽 Hopper: {self.utils.hopper}
+    📝 Outputter: {self.utils.outputter}
+    🎬 Video Analyzer: {self.utils.video_analyzer or "Not configured"}
+    👁️ Object Detector: {self.utils.object_detector or "Not configured"}
+"""
+
+    def validate_providers(self) -> None:
+        """Validate credentials of the planner and the utility nodes."""
+        self.planner.validate_provider("Planner")
+        self.utils.outputter.validate_provider("Outputter")
+        self.utils.hopper.validate_provider("Hopper")
+        if self.utils.video_analyzer:
+            self.utils.video_analyzer.validate_provider("VideoAnalyzer")
+        if self.utils.object_detector:
+            self.utils.object_detector.validate_provider("ObjectDetector")
+
+    def get_utils(self, item: LLMUtilsNode) -> LLMWithFallback:
+        """Retrieve model configuration for a specific utility node."""
+        value = getattr(self.utils, item)
+        if value is None:
+            raise ValueError(
+                f"Utils '{item}' is not configured. Please add it to your LLM "
+                "config or enable it via AgentConfigBuilder."
+            )
+        return value
+
+
+def initialize_llm_config[C: LLMConfigBase](parse: Callable[[], C]) -> C:
+    """Parse the LLM config with ``parse`` and validate provider credentials."""
+    llm_config = parse()
+    llm_config.validate_providers()
+    logger.success("LLM config initialized")
+    return llm_config
+
+
+def load_llm_config_override[C: LLMConfigBase](
+    path: Path,
+    get_default: Callable[[], C],
+    merge: Callable[[C, dict[str, Any]], C],
+) -> C:
+    """Merge the JSON/JSONC overrides at ``path`` onto the default config.
+
+    Falls back to the default config when the file is missing or the merged
+    config is invalid.
+    """
+    default_config = get_default()
+
+    override_config_dict: dict[str, Any] = {}
+    if path.exists():
+        logger.info(f"Loading custom LLM config from {path.resolve()}...")
+        with open(path, encoding="utf-8") as f:
+            override_config_dict = load_jsonc(f)
+    else:
+        logger.warning(f"Custom LLM config not found at {path} - using default config")
+
+    try:
+        return merge(default_config, override_config_dict)
+    except ValidationError as e:
+        logger.error(f"Invalid LLM config: {e}")
+        logger.info("Falling back to default config")
+        return default_config