android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/artemis/mcp/actuators/mock.py

@@ -0,0 +1,70 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Mock actuator for tests: real actuator code paths over an in-memory device.
+
+Subclasses :class:`AdbActuator` deliberately, so tests exercise the same coordinate
+conversion and controller dispatch as production -- only the driver is in-memory.
+``capabilities``/``extensions`` are overridable to model partial and extended
+backends.
+"""
+
+from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
+from artemis.drivers.mock.mock_driver import MockDeviceDriver
+from artemis.mcp.action_manifest import DEVICE_ACTIONS, ExtensionTool
+from artemis.mcp.actuators.adb import AdbActuator
+from artemis.platform import platform
+
+__all__ = ["MockActuator"]
+
+
+class MockActuator(AdbActuator):
+    """An :class:`AdbActuator` bound to a :class:`MockDeviceDriver`."""
+
+    def __init__(
+        self,
+        width: int = 1080,
+        height: int = 2400,
+        capabilities: frozenset[str] | None = None,
+        extensions: list[ExtensionTool] | None = None,
+    ):
+        driver = MockDeviceDriver(width=width, height=height)
+        ctx = ArtemisContext(
+            trace_id="mock-actuator",
+            device=DeviceContext(
+                host_platform=platform.os_type.name,
+                mobile_platform=DevicePlatform.ANDROID,
+                device_id=driver.device_id,
+                device_width=width,
+                device_height=height,
+            ),
+        )
+        # get_driver(ctx) returns a cached `_active_driver` when present
+        # (artemis/drivers/factory.py), the established test seam.
+        ctx._active_driver = driver
+        super().__init__(ctx)
+        self.driver = driver
+        self._capabilities = capabilities if capabilities is not None else DEVICE_ACTIONS
+        self._extensions = list(extensions or [])
+
+    def capabilities(self) -> frozenset[str]:
+        return self._capabilities
+
+    def extensions(self) -> list[ExtensionTool]:
+        return self._extensions
+
+    @property
+    def action_history(self) -> list[dict]:
+        """The mock driver's recorded actions, for assertions."""
+        return self.driver.action_history

package/artemis-main/artemis/mcp/adb_server.py

@@ -0,0 +1,490 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Callable
+import json
+import logging
+import os
+from pathlib import Path
+import sys
+import textwrap
+from typing import Any
+
+# Ensure repository root is in sys.path when executed directly or via MCP runner
+_REPO_ROOT = str(Path(__file__).resolve().parent.parent.parent)
+if _REPO_ROOT not in sys.path:
+    sys.path.insert(0, _REPO_ROOT)
+
+# pylint: disable=wrong-import-position
+from adbutils import AdbClient
+from mcp.server.fastmcp import Context, FastMCP
+
+logger = logging.getLogger(__name__)
+
+try:
+    from mcp.server.fastmcp.server import Settings as FastMCPSettings
+
+    FastMCPSettings.model_rebuild()
+except Exception as exc:  # pylint: disable=broad-exception-caught
+    # Best-effort compatibility shim for FastMCP/pydantic version drift.
+    logger.debug("FastMCP Settings model_rebuild skipped: %s", exc, exc_info=True)
+
+from artemis.clients.screen_client_factory import create_screen_client
+from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
+from artemis.controllers.unified_controller import UnifiedMobileController
+from artemis.platform import platform
+from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries
+
+
+def configure_stdio_mode() -> None:
+    """Applies process-wide settings required when serving MCP over stdio.
+
+    These are destructive to the host process (console logging is removed and the
+    DataEngine IPC port is cleared), so they must only run when this module owns the
+    process -- i.e. from the ``__main__`` block. Importing this module for its tools
+    or for :func:`_get_controller` must leave the caller's logging and environment
+    untouched.
+    """
+    # Redirect stdio to prevent logs from breaking MCP JSON-RPC protocol and causing deadlocks.
+    try:
+        from artemis.config import settings
+
+        log_path = Path(settings.TRACES_PATH) / "mcp_server.log"
+    except Exception:
+        log_path = Path("traces") / "mcp_server.log"
+
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+
+    for name in ["artemis", __name__]:
+        log_instance = logging.getLogger(name)
+        log_instance.setLevel(logging.DEBUG)
+        log_instance.propagate = False
+        # Remove standard streams handlers
+        log_instance.handlers = [
+            h for h in log_instance.handlers if not isinstance(h, logging.StreamHandler)
+        ]
+        fh = logging.FileHandler(log_path, encoding="utf-8")
+        fh.setFormatter(logging.Formatter("%(asctime)s | %(name)s | %(levelname)s | %(message)s"))
+        log_instance.addHandler(fh)
+
+    logger.info("MCP Server Python process fully started. Configuring FastMCP service...")
+
+    # Avoid concurrent SQLite WAL locks by blocking child processes from connecting
+    # to the primary DataEngine.
+    os.environ["ARTEMIS_IPC_PORT"] = ""
+
+
+# Create minimal MCP server
+mcp = FastMCP("Android_ADB_Controller")
+
+
+def _tool_description(fn: Callable[..., Any]) -> str:
+    """Returns ``fn``'s docstring normalized identically on every Python version.
+
+    FastMCP publishes ``__doc__`` verbatim as the tool description. Python 3.13+
+    strips the common indentation of docstring continuation lines at compile time,
+    while 3.12 keeps it, so the schema external MCP clients see would otherwise
+    depend on the interpreter. This reproduces the 3.13 form (first line as-is,
+    remaining lines dedented, whitespace-only lines emptied); it is a no-op there.
+    """
+    first, sep, rest = (fn.__doc__ or "").partition("\n")
+    return first + sep + textwrap.dedent(rest)
+
+
+def _tool() -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    """``mcp.tool()`` with a Python-version-independent description."""
+
+    def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+        return mcp.tool(description=_tool_description(fn))(fn)
+
+    return decorator
+
+
+_GLOBAL_CONTROLLER = None
+_CONTROLLERS: dict[str, Any] = {}
+
+
+def _get_controller(device_serial: str | None = None):
+    """Lazy-load device controller on-demand, caching per device serial."""
+    global _GLOBAL_CONTROLLER, _CONTROLLERS
+    target_serial = (
+        device_serial or os.environ.get("ARTEMIS_DEVICE_ID") or os.environ.get("ADB_DEVICE_SERIAL")
+    )
+    if target_serial and target_serial in _CONTROLLERS:
+        return _CONTROLLERS[target_serial]
+    if not target_serial and _GLOBAL_CONTROLLER is not None:
+        return _GLOBAL_CONTROLLER
+
+    logger.info("Initializing lazy device controller...")
+    if os.environ.get("ARTEMIS_CLOUD_MODE") == "1":
+        logger.info(
+            "GCP CLOUD MODE active: initializing CloudMobileDeviceController via UnifiedMobileController..."
+        )
+        resolved_serial = target_serial or "cloud_device"
+        ctx = ArtemisContext(
+            device=DeviceContext(
+                platform=DevicePlatform.ANDROID,
+                device_id=resolved_serial,
+                width=1080,
+                height=2400,
+            )
+        )
+        controller = UnifiedMobileController(ctx=ctx)
+        if resolved_serial:
+            _CONTROLLERS[resolved_serial] = controller
+        if _GLOBAL_CONTROLLER is None:
+            _GLOBAL_CONTROLLER = controller
+        return controller
+
+    host = os.environ.get("ADB_HOST", "localhost")
+    port_str = os.environ.get("ADB_PORT", "5037")
+    port = int(port_str) if port_str.isdigit() else 5037
+    if "ADB_SERVER_SOCKET" not in os.environ and (host != "localhost" or port != 5037):
+        os.environ["ADB_SERVER_SOCKET"] = f"tcp:{host}:{port}"
+    adb = AdbClient(host=host, port=port)
+
+    devices = adb.device_list()
+    if not devices:
+        raise Exception(f"No Android devices found at {host}:{port}")
+
+    if target_serial:
+        matched = [d for d in devices if d.serial == target_serial]
+        if not matched:
+            raise Exception(
+                f"Target Android device '{target_serial}' not found at"
+                f" {host}:{port} among available devices."
+            )
+        device = matched[0]
+    else:
+        device = devices[0]
+    device_id = device.serial
+
+    # Observer path: the helper is used only when it is already installed and
+    # running; nothing is installed from here (that happens inside a task's
+    # device-lock boundary or via `artemis helper install`).
+    ui_client = create_screen_client(device_id)
+    try:
+        ui_data = ui_client.get_screen_data()
+        width, height = ui_data.width, ui_data.height
+    except Exception as e:
+        logger.warning(f"Failed initial screen data check, using defaults: {e}")
+        width, height = 1080, 2400
+
+    ctx = ArtemisContext(
+        trace_id="mcp-session",
+        device=DeviceContext(
+            host_platform=platform.os_type.name,
+            mobile_platform=DevicePlatform.ANDROID,
+            device_id=device_id,
+            device_width=width,
+            device_height=height,
+        ),
+        adb_client=adb,
+        ui_adb_client=ui_client,
+    )
+
+    controller = UnifiedMobileController(ctx)
+    if device_id:
+        _CONTROLLERS[device_id] = controller
+    if _GLOBAL_CONTROLLER is None:
+        _GLOBAL_CONTROLLER = controller
+    logger.info(f"Lazy device controller fully initialized for device: {device_id}")
+    return controller
+
+
+# Shared with the in-process actuator layer; this stdio server keeps its legacy
+# pixel-space contract for external MCP clients, but the element/focus logic has a
+# single implementation in artemis.mcp.actuators.adb.
+from artemis.mcp.actuators.adb import (  # noqa: E402  pylint: disable=wrong-import-position
+    ensure_focus_at_coords as _ensure_focus_at_coords,
+)
+
+
+@_tool()
+async def tap(
+    ctx: Context,
+    coordinates: list[int],
+    times: int = 1,
+    delay_ms: int = 100,
+) -> str:
+    """Taps on the screen at coordinates.
+
+    'coordinates' is a list [x, y].
+    'times' is the number of consecutive clicks (default 1).
+    'delay_ms' is the delay in milliseconds between consecutive clicks (default
+    100).
+    """
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    if len(coordinates) != 2:
+        return "Error: coordinates must be [x, y]"
+
+    result = await controller.tap_at(
+        x=coordinates[0], y=coordinates[1], times=times, delay_ms=delay_ms
+    )
+
+    if result.error:
+        return f"Error: {result.error}"
+    return "Success"
+
+
+@_tool()
+async def long_press_on(
+    ctx: Context,
+    coordinates: list[int],
+    duration: int = 1000,
+) -> str:
+    """Long presses on the screen at coordinates.
+
+    'coordinates' is a list [x, y].
+    'duration' is in milliseconds (default 1000).
+    """
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    if len(coordinates) != 2:
+        return "Error: coordinates must be [x, y]"
+
+    result = await controller.tap_at(
+        x=coordinates[0],
+        y=coordinates[1],
+        long_press=True,
+        long_press_duration=duration,
+    )
+
+    if result.error:
+        return f"Error: {result.error}"
+    return "Success"
+
+
+@_tool()
+async def swipe(
+    ctx: Context,
+    coordinates: list[int],
+    duration: int = 400,
+) -> str:
+    """Swipes from start coordinates to end coordinates.
+
+    'coordinates' is a list [start_x, start_y, end_x, end_y].
+    'duration' is in milliseconds (default 400).
+
+    Set duration >= 1000 to drag-and-drop. Drag slightly past the target
+    position to trigger reordering.
+    """
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    if len(coordinates) != 4:
+        return "Error: coordinates must be [start_x, start_y, end_x, end_y]"
+
+    error = await controller.swipe_coords(
+        start_x=coordinates[0],
+        start_y=coordinates[1],
+        end_x=coordinates[2],
+        end_y=coordinates[3],
+        duration=duration,
+    )
+    if error:
+        return f"Error: {error}"
+    return "Success"
+
+
+@_tool()
+async def back(ctx: Context) -> str:
+    """Simulates pressing the system back button."""
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    success = await controller.go_back()
+    return "Success" if success else "Failed"
+
+
+@_tool()
+async def launch_app(ctx: Context, package_name: str) -> str:
+    """Launches an application by its Android package name with retries and smart polling."""
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    success, error_msg = await launch_app_with_retries(controller.ctx, package_name)
+    return "Success" if success else f"Failed: {error_msg}"
+
+
+@_tool()
+async def stop_app(ctx: Context, package_name: str) -> str:
+    """Force stops an application by its Android package name."""
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    success = await controller.terminate_app(package_name)
+    return "Success" if success else "Failed"
+
+
+@_tool()
+async def open_link(ctx: Context, url: str) -> str:
+    """Opens a URL or deep link on the device."""
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    success = await controller.open_url(url)
+    return "Success" if success else "Failed"
+
+
+@_tool()
+async def focus_and_input_text(
+    ctx: Context,
+    coordinates: list[int],
+    text: str,
+    clear_before_input: bool = False,
+) -> str:
+    """Focuses on a UI element at coordinates and inputs text.
+
+    'coordinates' is a list [x, y] to tap first to gain focus.
+    'clear_before_input' if True, clears all existing text before typing. If False, appends text at the end of existing content.
+    'text' supports multi-line content with '\\n'.
+    """
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    if len(coordinates) != 2:
+        return "Error: coordinates must be [x, y]"
+
+    err = await _ensure_focus_at_coords(controller, coordinates[0], coordinates[1])
+    if err:
+        return f"Error focusing element: {err}"
+
+    if clear_before_input:
+        success = await controller.erase_text()
+        if not success:
+            return "Failed to clear existing text"
+    else:
+        # Move cursor to the end for reliable append
+        await controller.press_key("123")
+
+    success = await controller.type_text(text, clear_existing=False)
+    return "Success" if success else "Failed"
+
+
+@_tool()
+async def focus_and_clear_text(
+    ctx: Context,
+    coordinates: list[int],
+) -> str:
+    """Focuses on a UI element at coordinates and clears its text content.
+
+    'coordinates' is a list [x, y] to tap first to gain focus.
+    """
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    if len(coordinates) != 2:
+        return "Error: coordinates must be [x, y]"
+
+    err = await _ensure_focus_at_coords(controller, coordinates[0], coordinates[1])
+    if err:
+        return f"Error focusing element: {err}"
+
+    success = await controller.erase_text()
+    return "Success" if success else "Failed"
+
+
+@_tool()
+async def erase_one_char(ctx: Context) -> str:
+    """Erases a single character (simulates Backspace)."""
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    success = await controller.erase_text(nb_chars=1)
+    return "Success" if success else "Failed"
+
+
+@_tool()
+async def press_key(ctx: Context, keycode: str) -> str:
+    """Presses a specific Android key event (e.g., KEYCODE_ENTER, KEYCODE_HOME)."""
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    try:
+        success = await controller.press_key(keycode)
+        return "Success" if success else "Failed"
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@_tool()
+async def take_screenshot(ctx: Context) -> str:
+    """Takes a screenshot of the device screen.
+
+    Returns the screenshot as a base64 encoded JPEG string.
+    """
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        return f"Error: Controller lazy initialization failed: {e}"
+
+    try:
+        screenshot_b64 = await controller.take_screenshot()
+        return screenshot_b64
+    except Exception as e:
+        return f"Error: {e}"
+
+
+@_tool()
+async def get_ui_hierarchy(ctx: Context) -> str:
+    """Retrieves the current UI elements hierarchy from the device."""
+    try:
+        controller = _get_controller()
+    except Exception as e:
+        logger.exception("Failed to initialize controller in get_ui_hierarchy")
+        raise e
+
+    try:
+        elements = await controller.get_ui_elements()
+        return json.dumps(elements)
+    except Exception as e:
+        logger.exception("Failed to get UI elements in get_ui_hierarchy")
+        raise e
+
+
+if __name__ == "__main__":
+    from artemis.runtime import shutdown_awake_service, start_awake_service
+
+    configure_stdio_mode()
+    start_awake_service()
+    try:
+        mcp.run(transport="stdio")
+    finally:
+        shutdown_awake_service()

package/artemis-main/artemis/mcp/observation.py

@@ -0,0 +1,133 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Screen observation: capture, fuse, index, and persist -- without LangGraph State.
+
+This is the act-then-observe half that used to be welded into every executor action
+(``capture_screenshot_and_parse_ui``). Split out so that actuation and observation are
+separate calls: the Flash adapter observes after each action, while the
+Validator -- which runs its own UI-change polling -- simply never calls it.
+"""
+
+import asyncio
+import base64
+import time
+
+from artemis.config import get_temp_dir
+from artemis.context import ArtemisContext
+from artemis.controllers.unified_controller import UnifiedMobileController
+from artemis.mcp.action_types import ActionCode, ObserveResult
+from artemis.utils.ocr_xml_fusion import fuse_ocr_with_xml
+from artemis.utils.visualization import format_minimal_list_with_elements
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+__all__ = ["observe"]
+
+
+async def observe(
+    ctx: ArtemisContext,
+    controller: UnifiedMobileController,
+    settle_ms: int = 400,
+    ocr_results: list | None = None,
+) -> tuple[ObserveResult, bytes | None]:
+    """Captures and indexes the current screen.
+
+    Returns the structured observation plus the raw screenshot bytes (an in-process
+    convenience; the serialized ``ObserveResult`` carries only the persisted path).
+
+    Mirrors the historical ``capture_screenshot_and_parse_ui`` behavior exactly:
+    a settling delay, screen capture, XML fusion and element indexing, persistence via
+    the data engine with a temp-file fallback -- and the same failure semantics: a
+    failed capture yields ``ok=False``; a failed hierarchy parse still returns the
+    screenshot with ``hierarchy_ok=False``.
+    """
+    try:
+        if settle_ms > 0:
+            logger.info(f"Delaying {settle_ms / 1000:.1f}s before capturing screen state...")
+            await asyncio.sleep(settle_ms / 1000.0)
+        else:
+            logger.info("Retrieving screen data directly (skip settling)...")
+        device_data = await controller.get_screen_data()
+        screenshot_bytes = base64.b64decode(device_data.base64)
+    except Exception as e:
+        logger.error(f"Failed to capture screen state: {e}")
+        return (
+            ObserveResult(
+                ok=False,
+                code=ActionCode.DEVICE_ERROR,
+                message=f"Failed to capture screen state: {e}",
+                hierarchy_ok=False,
+            ),
+            None,
+        )
+
+    xml_hierarchy = None
+    ocr_results = ocr_results or []
+    elements: list[dict] = []
+    hierarchy_ok = True
+    width = height = None
+    try:
+        xml_hierarchy = device_data.elements
+        width = device_data.width or 1080
+        height = device_data.height or 2400
+        if ctx.device:
+            ctx.device.device_width = width
+            ctx.device.device_height = height
+
+        fused_xml = fuse_ocr_with_xml(xml_hierarchy, ocr_results)
+        minimal_list_str, elements, _ = format_minimal_list_with_elements(fused_xml, width, height)
+    except Exception as e:
+        logger.error(f"Failed to fetch or fuse UI hierarchy: {e}")
+        minimal_list_str = "Not available due to hierarchy error."
+        hierarchy_ok = False
+
+    screenshot_path = None
+    if ctx.data_engine:
+        try:
+            image_name = ctx.data_engine.get_or_create_image(
+                screenshot_bytes,
+                ui_tree=xml_hierarchy,
+                ocr_result=ocr_results,
+            )
+            screenshot_path = str(ctx.data_engine.get_image_path(image_name))
+        except Exception as e:
+            logger.error(f"Failed to save image in DataEngine: {e}")
+
+    if not screenshot_path:
+        try:
+            temp_dir = get_temp_dir("screenshots")
+            temp_file = temp_dir / f"screenshot_{int(time.time())}.jpg"
+            temp_file.write_bytes(screenshot_bytes)
+            screenshot_path = str(temp_file)
+        except Exception as e:
+            logger.error(f"Failed to save fallback screenshot: {e}")
+
+    from pathlib import Path
+
+    return (
+        ObserveResult(
+            ok=True,
+            message="Screen observed.",
+            screenshot_path=screenshot_path,
+            image_name=Path(screenshot_path).stem if screenshot_path else None,
+            width=width,
+            height=height,
+            elements_text=minimal_list_str,
+            elements=elements,
+            hierarchy_ok=hierarchy_ok,
+        ),
+        screenshot_bytes,
+    )