android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/drivers/android/input_ime.py

@@ -0,0 +1,120 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Android IME and keyboard text typing helper."""
+
+import asyncio
+import base64
+from typing import Any
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def _escape_for_adb_text(s: str) -> str:
+    """Escapes special characters for adb shell input text."""
+    return (
+        s.replace("\\", "\\\\")
+        .replace('"', '\\"')
+        .replace("'", "\\'")
+        .replace("`", "\\`")
+        .replace("$", "\\$")
+        .replace("&", "\\&")
+        .replace("|", "\\|")
+        .replace(";", "\\;")
+        .replace("<", "\\<")
+        .replace(">", "\\>")
+        .replace("(", "\\(")
+        .replace(")", "\\)")
+        .replace("*", "\\*")
+        .replace("?", "\\?")
+        .replace("~", "\\~")
+        .replace(" ", "%s")
+    )
+
+
+class AndroidInputIME:
+    """Handles text input, unicode typing, and key combination injection."""
+
+    def __init__(self, adb_device: Any, ui_client: Any = None):
+        self.device = adb_device
+        self.ui_client = ui_client
+
+    async def type_text(self, text: str, clear_existing: bool = True) -> bool:
+        """Types text using adb input text, UIAutomator, or broadcast IME."""
+        try:
+            if clear_existing:
+                # Safe & robust clearing: Move to End -> Shift+Home selection -> Delete -> Fallback backspaces
+                clear_cmd = (
+                    "input keyevent 123 && "
+                    "input keyevent --meta 1 122 && "
+                    "input keyevent 67 && "
+                    "input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
+                )
+                await asyncio.to_thread(self.device.shell, clear_cmd)
+            else:
+                # Append mode: move cursor to the end of existing text
+                await asyncio.to_thread(
+                    self.device.shell,
+                    "input keyevent 123",  # KEYCODE_MOVE_END
+                )
+
+            # 1. Fast path for ASCII text: Use ADB input text directly.
+            if text.isascii():
+                lines = text.split("\n")
+                for i, line in enumerate(lines):
+                    if i > 0:
+                        # Send Enter key between lines
+                        await asyncio.to_thread(self.device.shell, "input keyevent 66")
+                    if line:
+                        escaped = _escape_for_adb_text(line)
+                        await asyncio.to_thread(self.device.shell, f"input text {escaped}")
+                return True
+
+            # 2. For non-ASCII / Unicode text: Try UIAutomator client
+            if self.ui_client and hasattr(self.ui_client, "send_text"):
+                try:
+                    res = self.ui_client.send_text(text)
+                    if asyncio.iscoroutine(res):
+                        await res
+                    return True
+                except Exception as e:
+                    logger.warning(f"UIAutomator send_text failed: {e}")
+
+            # 3. Check if ADBKeyboard is currently active
+            try:
+                default_ime = await asyncio.to_thread(
+                    self.device.shell, "settings get secure default_input_method"
+                )
+                if "adbkeyboard" in str(default_ime).lower():
+                    b64_text = base64.b64encode(text.encode("utf-8")).decode("utf-8")
+                    broadcast_cmd = f"am broadcast -a ADB_INPUT_B64 --es msg '{b64_text}'"
+                    await asyncio.to_thread(self.device.shell, broadcast_cmd)
+                    return True
+            except Exception as e:
+                # ADBKeyboard probe/broadcast failed; fall through to ADB input.
+                logger.debug(f"ADBKeyboard IME path failed, falling back to ADB input: {e}")
+
+            # 4. Fallback: ADB input text
+            lines = text.split("\n")
+            for i, line in enumerate(lines):
+                if i > 0:
+                    await asyncio.to_thread(self.device.shell, "input keyevent 66")
+                if line:
+                    escaped = _escape_for_adb_text(line)
+                    await asyncio.to_thread(self.device.shell, f"input text {escaped}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to type text '{text}': {e}")
+            return False

package/artemis-main/artemis/drivers/android/recorder.py

@@ -0,0 +1,57 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Android screen recording and video transcode utilities."""
+
+import asyncio
+from pathlib import Path
+from typing import Any
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ScreenRecorder:
+    """Manages background screen recording sessions using adb screenrecord."""
+
+    def __init__(self, adb_device: Any, output_path: Path):
+        self.device = adb_device
+        self.output_path = output_path
+        self.is_recording = False
+        self._remote_path = "/sdcard/artemis_record.mp4"
+
+    async def start(self) -> None:
+        """Starts remote screen recording."""
+        self.is_recording = True
+        logger.info("Starting screen recording on device...")
+        # Start background screenrecord
+        asyncio.create_task(
+            asyncio.to_thread(self.device.shell, f"screenrecord {self._remote_path}")
+        )
+
+    async def stop(self) -> Path | None:
+        """Stops screen recording and pulls file to host."""
+        if not self.is_recording:
+            return None
+        self.is_recording = False
+        logger.info("Stopping screen recording...")
+        try:
+            await asyncio.to_thread(self.device.shell, "pkill -2 screenrecord")
+            await asyncio.sleep(1.0)
+            self.output_path.parent.mkdir(parents=True, exist_ok=True)
+            await asyncio.to_thread(self.device.sync.pull, self._remote_path, str(self.output_path))
+            return self.output_path
+        except Exception as e:
+            logger.warning(f"Failed to pull screen recording: {e}")
+            return None

package/artemis-main/artemis/drivers/base.py

@@ -0,0 +1,290 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal Base Device Driver Protocol.
+
+Defines the contract for hardware and simulator device interaction,
+abstracting low-level protocols (ADB, UIAutomator2, WebDriver, Cloud RPC)
+behind a clean, unified async interface.
+"""
+
+from abc import ABC, abstractmethod
+import asyncio
+from enum import Enum
+from pathlib import Path
+import re
+import time
+from typing import Any, Literal
+from pydantic import BaseModel, ConfigDict, Field
+
+from third_party.mobile_use.controllers.types import ElementQuery
+
+
+class SwipeDirection(str, Enum):
+    UP = "up"
+    DOWN = "down"
+    LEFT = "left"
+    RIGHT = "right"
+
+
+class KeyCode(str, Enum):
+    HOME = "home"
+    BACK = "back"
+    ENTER = "enter"
+    DELETE = "delete"
+    POWER = "power"
+    APP_SWITCH = "app_switch"
+    VOLUME_UP = "volume_up"
+    VOLUME_DOWN = "volume_down"
+
+
+class ScreenData(BaseModel):
+    """Encapsulates raw and parsed UI screen observation data."""
+
+    screenshot_bytes: bytes = Field(..., description="Raw PNG/JPEG screenshot bytes")
+    screenshot_base64: str = Field(..., description="Base64 encoded screenshot string")
+    ui_hierarchy_xml: str | None = Field(default=None, description="Raw UI layout XML hierarchy")
+    ui_elements: list[dict[str, Any]] = Field(
+        default_factory=list, description="Parsed element node list"
+    )
+    width: int = Field(default=1080, description="Device screen width in pixels")
+    height: int = Field(default=2400, description="Device screen height in pixels")
+    timestamp: float = Field(default_factory=time.time, description="Capture timestamp")
+    platform: str = Field(default="android", description="Platform identifier")
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class BaseDeviceDriver(ABC):
+    """Abstract Base Class for mobile device and emulator drivers."""
+
+    @property
+    @abstractmethod
+    def device_id(self) -> str:
+        """Returns the unique hardware or simulator serial/identifier."""
+        ...
+
+    @property
+    @abstractmethod
+    def screen_size(self) -> tuple[int, int]:
+        """Returns (width, height) in physical device pixels."""
+        ...
+
+    @abstractmethod
+    async def connect(self) -> None:
+        """Establishes connection to the target device."""
+        ...
+
+    @abstractmethod
+    async def disconnect(self) -> None:
+        """Closes connection and releases allocated resources."""
+        ...
+
+    @abstractmethod
+    async def get_screen_data(self, skip_settling: bool = False) -> ScreenData:
+        """Captures real-time screenshot and UI element hierarchy.
+
+        Args:
+            skip_settling: If True, captures immediately without waiting for screen stabilization.
+        """
+        ...
+
+    @abstractmethod
+    async def tap(
+        self,
+        x: int,
+        y: int,
+        duration_ms: int = 100,
+        times: int = 1,
+        delay_ms: int = 100,
+    ) -> bool:
+        """Performs single or multiple tap interactions at absolute pixel coordinates.
+
+        Args:
+            x: Absolute horizontal coordinate in pixels.
+            y: Absolute vertical coordinate in pixels.
+            duration_ms: Touch hold duration in milliseconds.
+            times: Number of consecutive taps (e.g. 2 for double tap, 7 for dev options).
+            delay_ms: Delay in milliseconds between consecutive taps.
+        """
+        ...
+
+    @abstractmethod
+    async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool:
+        """Performs a long press at absolute pixel coordinates."""
+        ...
+
+    @abstractmethod
+    async def swipe(
+        self,
+        start_x: int,
+        start_y: int,
+        end_x: int,
+        end_y: int,
+        duration_ms: int = 800,
+    ) -> bool:
+        """Performs a drag/swipe gesture from start to end pixel coordinates."""
+        ...
+
+    @abstractmethod
+    async def swipe_direction(
+        self,
+        direction: SwipeDirection | Literal["up", "down", "left", "right"],
+        duration_ms: int = 800,
+    ) -> bool:
+        """Performs standard cardinal scrolling gesture across device viewport."""
+        ...
+
+    @abstractmethod
+    async def input_text(self, text: str, clear_existing: bool = True) -> bool:
+        """Types text into the currently focused input field."""
+        ...
+
+    @abstractmethod
+    async def press_key(self, key: KeyCode | str | int) -> bool:
+        """Simulates physical or soft key press (e.g. HOME, BACK, ENTER)."""
+        ...
+
+    @abstractmethod
+    async def launch_app(self, package_name: str) -> bool:
+        """Launches target application package into the foreground."""
+        ...
+
+    @abstractmethod
+    async def stop_app(self, package_name: str) -> bool:
+        """Force-stops target application package."""
+        ...
+
+    @abstractmethod
+    async def get_current_package(self) -> str | None:
+        """Retrieves currently focused foreground application package name."""
+        ...
+
+    @abstractmethod
+    async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str:
+        """Executes a system shell command directly on the device."""
+        ...
+
+    @abstractmethod
+    async def start_video_recording(self, output_dir: Path | None = None) -> None:
+        """Starts dynamic screen video capture in the background."""
+        ...
+
+    @abstractmethod
+    async def stop_video_recording(self) -> str | None:
+        """Stops background video capture and returns local recording file path."""
+        ...
+
+    async def wait_for_delay(self, seconds: float = 1.0) -> bool:
+        """Waits for specified duration in seconds."""
+        await asyncio.sleep(seconds)
+        return True
+
+    async def tap_normalized(
+        self,
+        norm_x: int,
+        norm_y: int,
+        long_press: bool = False,
+        duration_ms: int = 1000,
+        times: int = 1,
+        delay_ms: int = 100,
+    ) -> bool:
+        """Taps at normalized (0-1000 scale) screen coordinates."""
+        width, height = self.screen_size
+        abs_x = int(max(0, min(width - 1, norm_x * width / 1000.0)))
+        abs_y = int(max(0, min(height - 1, norm_y * height / 1000.0)))
+        if long_press:
+            return await self.long_press(abs_x, abs_y, duration_ms=duration_ms)
+        return await self.tap(abs_x, abs_y, duration_ms=100, times=times, delay_ms=delay_ms)
+
+    async def swipe_normalized(
+        self,
+        start_norm: list[int],
+        end_norm: list[int],
+        duration_ms: int = 800,
+    ) -> bool:
+        """Swipes between normalized (0-1000 scale) screen coordinates."""
+        width, height = self.screen_size
+        sx = int(max(0, min(width - 1, start_norm[0] * width / 1000.0)))
+        sy = int(max(0, min(height - 1, start_norm[1] * height / 1000.0)))
+        ex = int(max(0, min(width - 1, end_norm[0] * width / 1000.0)))
+        ey = int(max(0, min(height - 1, end_norm[1] * height / 1000.0)))
+        return await self.swipe(sx, sy, ex, ey, duration_ms=duration_ms)
+
+    async def find_element(
+        self, query: ElementQuery, screen_data: ScreenData | None = None
+    ) -> tuple[dict[str, Any] | None, list[int] | None, str | None]:
+        """Finds matching UI element in screen hierarchy by resource_id or text."""
+        resource_id, text, index = query.resource_id, query.text, query.index
+        data = screen_data or await self.get_screen_data()
+        elements = data.ui_elements or []
+
+        matched = []
+        for elem in elements:
+            r_id = elem.get("resource_id") or elem.get("resource-id") or ""
+            t_val = elem.get("text") or elem.get("content-desc") or ""
+
+            if resource_id and resource_id in r_id:
+                matched.append(elem)
+            elif text and text.lower() in t_val.lower():
+                matched.append(elem)
+
+        if not matched:
+            return None, None, f"Element not found ({query.describe()})"
+
+        if index >= len(matched):
+            return None, None, f"Element index {index} out of range (matched {len(matched)})"
+
+        elem = matched[index]
+        bounds = elem.get("bounds")
+        parsed_bounds = elem.get("parsed_bounds")
+        numeric_bounds: list[int] | None = None
+        if isinstance(parsed_bounds, dict):
+            keys = ("left", "top", "right", "bottom")
+            if all(isinstance(parsed_bounds.get(key), (int, float)) for key in keys):
+                numeric_bounds = [int(parsed_bounds[key]) for key in keys]
+        if numeric_bounds is None and isinstance(bounds, (list, tuple)) and len(bounds) == 4:
+            if all(isinstance(value, (int, float)) for value in bounds):
+                numeric_bounds = [int(value) for value in bounds]
+        if numeric_bounds is None and isinstance(bounds, str):
+            match = re.fullmatch(
+                r"\[\s*(-?\d+)\s*,\s*(-?\d+)\s*\]\[\s*(-?\d+)\s*,\s*(-?\d+)\s*\]",
+                bounds,
+            )
+            if match:
+                numeric_bounds = [int(value) for value in match.groups()]
+        # Bounds belong to the freshly captured hierarchy. Some UIAutomator
+        # adapters also attach a precomputed center which can lag one layout
+        # transition behind (for example when an expanding drawer shifts a
+        # keypad). Always derive the tap point from live bounds when possible;
+        # retain center only for adapters that do not expose bounds.
+        if numeric_bounds is not None:
+            center = [
+                (numeric_bounds[0] + numeric_bounds[2]) // 2,
+                (numeric_bounds[1] + numeric_bounds[3]) // 2,
+            ]
+        else:
+            center = elem.get("center")
+        return elem, center, None
+
+    async def tap_element(
+        self, query: ElementQuery, long_press: bool = False, duration_ms: int = 1000
+    ) -> bool:
+        """Finds and taps a UI element by resource_id or text."""
+        _, center, error = await self.find_element(query)
+        if error or not center:
+            return False
+        if long_press:
+            return await self.long_press(center[0], center[1], duration_ms=duration_ms)
+        return await self.tap(center[0], center[1])

package/artemis-main/artemis/drivers/cloud/cloud_driver.py

@@ -0,0 +1,117 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Remote Cloud Mobile Device Driver."""
+
+from pathlib import Path
+from typing import Literal
+from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class CloudDeviceDriver(BaseDeviceDriver):
+    """Driver communicating with remote Android devices via Cloud Gateway RPC."""
+
+    def __init__(
+        self,
+        device_id: str = "cloud-device",
+        gateway_url: str = "http://127.0.0.1:8000",
+        session_id: str = "default-cloud-session",
+        width: int = 1080,
+        height: int = 2400,
+    ):
+        self._device_id = device_id
+        self.gateway_url = gateway_url
+        self.session_id = session_id
+        self._width = width
+        self._height = height
+
+    @property
+    def device_id(self) -> str:
+        return self._device_id
+
+    @property
+    def screen_size(self) -> tuple[int, int]:
+        return (self._width, self._height)
+
+    async def connect(self) -> None:
+        logger.info(f"Connecting to Cloud Device {self._device_id} at {self.gateway_url}")
+
+    async def disconnect(self) -> None:
+        logger.info(f"Disconnecting Cloud Device {self._device_id}")
+
+    async def get_screen_data(self, skip_settling: bool = False) -> ScreenData:
+        return ScreenData(
+            screenshot_bytes=b"",
+            screenshot_base64="",
+            width=self._width,
+            height=self._height,
+            platform="cloud",
+        )
+
+    async def tap(
+        self,
+        x: int,
+        y: int,
+        duration_ms: int = 100,
+        times: int = 1,
+        delay_ms: int = 100,
+    ) -> bool:
+        return True
+
+    async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool:
+        return True
+
+    async def swipe(
+        self,
+        start_x: int,
+        start_y: int,
+        end_x: int,
+        end_y: int,
+        duration_ms: int = 400,
+    ) -> bool:
+        return True
+
+    async def swipe_direction(
+        self,
+        direction: SwipeDirection | Literal["up", "down", "left", "right"],
+        duration_ms: int = 400,
+    ) -> bool:
+        return True
+
+    async def input_text(self, text: str, clear_existing: bool = True) -> bool:
+        return True
+
+    async def press_key(self, key: KeyCode | str | int) -> bool:
+        return True
+
+    async def launch_app(self, package_name: str) -> bool:
+        return True
+
+    async def stop_app(self, package_name: str) -> bool:
+        return True
+
+    async def get_current_package(self) -> str | None:
+        return None
+
+    async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str:
+        return f"cloud_output: {command}"
+
+    async def start_video_recording(self, output_dir: Path | None = None) -> None:
+        pass
+
+    async def stop_video_recording(self) -> str | None:
+        return None

package/artemis-main/artemis/drivers/factory.py

@@ -0,0 +1,80 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Device Driver Factory & Registry."""
+
+import os
+from typing import TYPE_CHECKING
+
+from adbutils import AdbClient
+
+from artemis.config import settings
+from artemis.drivers.android.adb_driver import AndroidAdbDriver
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.drivers.mock.mock_driver import MockDeviceDriver
+from third_party.mobile_use.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from artemis.context import ArtemisContext
+
+logger = get_logger(__name__)
+
+
+def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver:
+    """Instantiates the appropriate BaseDeviceDriver based on the runtime context."""
+    # 1. Cloud mode check. Cloud devices are reached through the gateway's
+    # RemoteUIAutomatorClient; ARTEMIS_HIERARCHY_BACKEND does not apply there
+    # because the Accessibility Helper needs a local adb forward.
+    if os.environ.get("ARTEMIS_CLOUD_MODE") == "1":
+        if ctx.adb_client is None:
+            from cloud_service.virtualization import RemoteAdbClient
+
+            ctx.adb_client = RemoteAdbClient()
+        if ctx.ui_adb_client is None:
+            from cloud_service.virtualization import RemoteUIAutomatorClient
+
+            ctx.ui_adb_client = RemoteUIAutomatorClient(adb_client=ctx.adb_client)
+
+    # 2. Mock mode check
+    if (
+        getattr(ctx.device, "mobile_platform", None) == "mock"
+        or os.environ.get("ARTEMIS_MOCK_DRIVER") == "1"
+    ):
+        return MockDeviceDriver(
+            device_id=ctx.device.device_id if ctx.device else "mock-device",
+            width=ctx.device.device_width if ctx.device else 1080,
+            height=ctx.device.device_height if ctx.device else 2400,
+        )
+
+    # 3. Default Android ADB driver
+    if ctx.adb_client is None:
+        ctx.adb_client = AdbClient(
+            host=settings.ADB_HOST or "localhost", port=settings.ADB_PORT or 5037
+        )
+
+    return AndroidAdbDriver(
+        device_id=ctx.device.device_id,
+        adb_client=ctx.adb_client,
+        ui_adb_client=getattr(ctx, "ui_adb_client", None),
+        width=ctx.device.device_width,
+        height=ctx.device.device_height,
+    )
+
+
+def get_driver(ctx: "ArtemisContext") -> BaseDeviceDriver:
+    """Cached accessor for device driver in the current context."""
+    if not hasattr(ctx, "_active_driver") or getattr(ctx, "_active_driver") is None:
+        driver = create_driver(ctx)
+        setattr(ctx, "_active_driver", driver)
+    return getattr(ctx, "_active_driver")