android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/third_party/mobile_use/controllers/unified_controller.py

@@ -0,0 +1,183 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Device-control facade base: the upstream ``UnifiedMobileController`` API.
+
+Artemis extends it as ``artemis.controllers.unified_controller.UnifiedMobileController``
+with scrcpy video recording and timeline segment extraction.
+"""
+
+from typing import Any
+
+from artemis.context import ArtemisContext
+from artemis.controllers.device_controller import ScreenDataResponse
+from artemis.drivers.base import BaseDeviceDriver
+from third_party.mobile_use.controllers.types import ElementQuery, TapOutput
+
+
+class UnifiedMobileControllerBase:
+    def __init__(self, ctx: ArtemisContext, driver: BaseDeviceDriver):
+        self.ctx = ctx
+        self._driver: BaseDeviceDriver = driver
+
+    @property
+    def driver(self) -> BaseDeviceDriver:
+        return self._driver
+
+    @property
+    def controller(self) -> Any:
+        return self._driver
+
+    async def tap_at(
+        self,
+        x: int,
+        y: int,
+        long_press: bool = False,
+        long_press_duration: int = 1000,
+        times: int = 1,
+        delay_ms: int = 100,
+    ) -> TapOutput:
+        try:
+            if long_press:
+                success = await self._driver.long_press(x, y, duration_ms=long_press_duration)
+            else:
+                success = await self._driver.tap(
+                    x, y, duration_ms=100, times=times, delay_ms=delay_ms
+                )
+            return TapOutput(error=None if success else f"Tap failed at ({x}, {y})")
+        except Exception as e:
+            return TapOutput(error=str(e))
+
+    async def tap_percentage(
+        self,
+        x_percent: int,
+        y_percent: int,
+        long_press: bool = False,
+        long_press_duration: int = 1000,
+    ) -> TapOutput:
+        """Tap at percentage-based coordinates (0 to 100)."""
+        norm_x = int(x_percent * 10)
+        norm_y = int(y_percent * 10)
+        success = await self._driver.tap_normalized(
+            norm_x, norm_y, long_press=long_press, duration_ms=long_press_duration
+        )
+        return TapOutput(
+            error=None if success else f"Tap percentage failed at ({x_percent}%, {y_percent}%)"
+        )
+
+    async def tap_element(
+        self,
+        resource_id: str | None = None,
+        text: str | None = None,
+        index: int = 0,
+        long_press: bool = False,
+        long_press_duration: int = 1000,
+    ) -> TapOutput:
+        """Tap on a UI element by finding it in the hierarchy."""
+        query = ElementQuery(resource_id=resource_id, text=text, index=index)
+        success = await self._driver.tap_element(
+            query, long_press=long_press, duration_ms=long_press_duration
+        )
+        return TapOutput(error=None if success else f"Failed to tap element ({query.describe()})")
+
+    async def swipe_coords(
+        self,
+        start_x: int,
+        start_y: int,
+        end_x: int,
+        end_y: int,
+        duration: int = 400,
+    ) -> str | None:
+        """Swipe between two coordinate points."""
+        success = await self._driver.swipe(start_x, start_y, end_x, end_y, duration_ms=duration)
+        return None if success else f"Swipe failed from ({start_x},{start_y}) to ({end_x},{end_y})"
+
+    async def type_text(self, text: str, clear_existing: bool = True) -> bool:
+        return await self._driver.input_text(text, clear_existing=clear_existing)
+
+    async def take_screenshot(self) -> str:
+        screen_data = await self._driver.get_screen_data()
+        return screen_data.screenshot_base64
+
+    async def launch_app(self, package_or_bundle_id: str) -> bool:
+        return await self._driver.launch_app(package_or_bundle_id)
+
+    async def terminate_app(self, package_or_bundle_id: str | None) -> bool:
+        if not package_or_bundle_id:
+            return False
+        return await self._driver.stop_app(package_or_bundle_id)
+
+    async def open_url(self, url: str) -> bool:
+        await self._driver.execute_shell(f"am start -a android.intent.action.VIEW -d '{url}'")
+        return True
+
+    async def go_back(self) -> bool:
+        return await self._driver.press_key("back")
+
+    async def go_home(self) -> bool:
+        return await self._driver.press_key("home")
+
+    async def press_enter(self) -> bool:
+        return await self._driver.press_key("enter")
+
+    async def press_key(self, keycode: str) -> bool:
+        return await self._driver.press_key(keycode)
+
+    async def erase_text(self, nb_chars: int | None = None) -> bool:
+        if nb_chars is not None and nb_chars > 0:
+            for _ in range(nb_chars):
+                await self._driver.press_key("delete")
+            return True
+        # Best-effort full clear: End -> Ctrl+A -> Delete
+        try:
+            clear_cmd = (
+                "input keyevent 123 && "
+                "input keycombination 113 29 && input keyevent 67 && "
+                "input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
+            )
+            await self._driver.execute_shell(clear_cmd)
+        except Exception:
+            for _ in range(30):
+                await self._driver.press_key("delete")
+        return True
+
+    async def get_ui_elements(self) -> list[dict]:
+        screen_data = await self._driver.get_screen_data()
+        return screen_data.ui_elements or []
+
+    async def get_screen_data(self) -> "ScreenDataResponse":
+        """Get screen data including screenshot, UI hierarchy, dimensions, and platform."""
+        data = await self._driver.get_screen_data()
+        return ScreenDataResponse(
+            base64=data.screenshot_base64,
+            elements=data.ui_elements or [],
+            width=data.width,
+            height=data.height,
+            platform=data.platform,
+        )
+
+    async def find_element(
+        self,
+        resource_id: str | None = None,
+        text: str | None = None,
+        index: int = 0,
+    ) -> tuple[dict | None, str | None]:
+        elem, _, error = await self._driver.find_element(
+            ElementQuery(resource_id=resource_id, text=text, index=index)
+        )
+        return elem, error

package/artemis-main/third_party/mobile_use/graph.py

@@ -0,0 +1,54 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""State reducer and convergence node used when building the agent graph."""
+
+from collections.abc import Callable, Hashable
+from typing import Any
+
+from langgraph.graph import StateGraph
+
+CONVERGENCE_NODE = "convergence"
+
+
+def take_last(a, b):
+    """Reducer function keeping the latest value."""
+    return b
+
+
+def convergence_node(state: Any):
+    """Convergence point for parallel execution paths."""
+    return {}
+
+
+def add_convergence_node(graph_builder: StateGraph) -> None:
+    """Add the deferred node where parallel branches join."""
+    graph_builder.add_node(node=CONVERGENCE_NODE, action=convergence_node, defer=True)
+
+
+def add_convergence_edges(
+    graph_builder: StateGraph,
+    gate: Callable[..., Hashable],
+    path_map: dict[Hashable, str],
+) -> None:
+    """Add the conditional edges leaving the convergence node, chosen by ``gate``."""
+    graph_builder.add_conditional_edges(
+        source=CONVERGENCE_NODE,
+        path=gate,
+        path_map=path_map,
+    )

package/artemis-main/third_party/mobile_use/llm_service.py

@@ -0,0 +1,193 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Helpers for LLM calls, from upstream ``services/llm.py``.
+
+``LLMWaitNotice`` waits for a call, tells the user when it is slow and enforces
+a hard timeout. ``FallbackRunner`` retries a failed call on a fallback model.
+``GetLLM`` is the type of ``get_llm``.
+
+Artemis passes in its own behaviour (pause detection, error classification,
+telemetry); this module does not import artemis at runtime.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable, Coroutine
+from contextlib import AbstractContextManager, nullcontext
+import logging
+from typing import TYPE_CHECKING, Any, Literal, Protocol, overload
+
+if TYPE_CHECKING:
+    from langchain_core.language_models.chat_models import BaseChatModel
+
+    from artemis.config import AgentNodeWithFallback, LLMUtilsNode, LLMUtilsNodeWithFallback
+    from artemis.context import ArtemisContext
+
+_logger = logging.getLogger(__name__)
+
+WAITING_MESSAGE = "Waiting for LLM call response..."
+
+
+class LLMWaitNotice:
+    """Wait for LLM calls, report slow ones and enforce a hard timeout.
+
+    Args:
+        notify: Called with :data:`WAITING_MESSAGE` once a call outlives its notice delay.
+        on_timeout: Called with the timeout message right before ``TimeoutError`` is raised.
+        hold: While it returns ``True`` the hard-timeout clock is held at zero
+            (e.g. the task is paused by the user).
+        default_hard_timeout: Hard timeout in seconds when the call does not pass
+            one; ``None`` waits indefinitely.
+        poll_interval: Seconds between ``hold``/timeout checks after the notice.
+    """
+
+    def __init__(
+        self,
+        *,
+        notify: Callable[[str], object],
+        on_timeout: Callable[[str], object],
+        hold: Callable[[], bool] = lambda: False,
+        default_hard_timeout: float | None = None,
+        poll_interval: float = 1.0,
+    ) -> None:
+        self._notify = notify
+        self._on_timeout = on_timeout
+        self._hold = hold
+        self._default_hard_timeout = default_hard_timeout
+        self._poll_interval = poll_interval
+
+    async def __call__[T](
+        self,
+        llm_call: Coroutine[Any, Any, T],
+        timeout_seconds: float = 10,
+        hard_timeout: float | None = None,
+    ) -> T:
+        """Run ``llm_call``; show a notice after ``timeout_seconds``.
+
+        The call is cancelled if the caller is cancelled or the hard timeout
+        (counted from the start of the call) expires.
+        """
+        if hard_timeout is None:
+            hard_timeout = self._default_hard_timeout
+        llm_task = asyncio.create_task(llm_call)
+        try:
+            # asyncio.wait never raises on timeout and needs no timer task.
+            done, _ = await asyncio.wait({llm_task}, timeout=timeout_seconds)
+            if llm_task in done:
+                return llm_task.result()
+
+            self._notify(WAITING_MESSAGE)
+            loop = asyncio.get_running_loop()
+            budget = None if hard_timeout is None else max(0.0, hard_timeout - timeout_seconds)
+            start_time = loop.time()
+            while True:
+                done, _ = await asyncio.wait({llm_task}, timeout=self._poll_interval)
+                if llm_task in done:
+                    return llm_task.result()
+                if self._hold():
+                    start_time = loop.time()
+                    continue
+                if budget is not None and loop.time() - start_time > budget:
+                    message = f"LLM call timed out after {hard_timeout} seconds."
+                    self._on_timeout(message)
+                    raise TimeoutError(message)
+        except BaseException:
+            if not llm_task.done():
+                llm_task.cancel()
+            await asyncio.gather(llm_task, return_exceptions=True)
+            raise
+
+
+class FallbackRunner:
+    """Runs a main LLM call and switches to a fallback call when it fails.
+
+    The base class falls back on every ``Exception`` and on a ``None`` result.
+    Subclasses narrow that down and report switches by overriding the hooks
+    :meth:`main_scope`, :meth:`fallback_reason` and :meth:`on_fallback`.
+    """
+
+    def main_scope(self) -> AbstractContextManager[object]:
+        """Context active while the main call runs; exited before the fallback starts."""
+        return nullcontext()
+
+    def fallback_reason(self, error: Exception) -> tuple[str, str | None] | None:
+        """``(reason, category)`` if ``error`` should trigger the fallback, ``None`` to re-raise."""
+        return "error", None
+
+    def on_fallback(self, reason: str, category: str | None, error: str | None) -> None:
+        """Report a switch to the fallback call."""
+        detail = f": {error}" if error else ""
+        _logger.warning(f"❗ Main LLM inference failed ({reason}{detail}). Falling back...")
+
+    async def run[T](
+        self,
+        main_call: Callable[[], Awaitable[T]],
+        fallback_call: Callable[[], Awaitable[T]],
+        none_should_fallback: bool = True,
+    ) -> T:
+        try:
+            with self.main_scope():
+                result = await main_call()
+        except Exception as e:
+            decision = self.fallback_reason(e)
+            if decision is None:
+                raise
+            self.on_fallback(decision[0], decision[1], str(e))
+            return await fallback_call()
+
+        if result is None and none_should_fallback:
+            self.on_fallback("empty_result", None, None)
+            return await fallback_call()
+        return result
+
+
+class GetLLM(Protocol):
+    """Call signature of ``get_llm``: resolve the chat model configured for a node."""
+
+    @overload
+    def __call__(
+        self,
+        ctx: ArtemisContext,
+        name: AgentNodeWithFallback,
+        *,
+        use_fallback: bool = False,
+        temperature: float | None = None,
+    ) -> BaseChatModel: ...
+
+    @overload
+    def __call__(
+        self,
+        ctx: ArtemisContext,
+        name: LLMUtilsNode,
+        *,
+        is_utils: Literal[True],
+        temperature: float | None = None,
+    ) -> BaseChatModel: ...
+
+    @overload
+    def __call__(
+        self,
+        ctx: ArtemisContext,
+        name: LLMUtilsNodeWithFallback,
+        *,
+        is_utils: Literal[True],
+        use_fallback: bool = False,
+        temperature: float | None = None,
+    ) -> BaseChatModel: ...

package/artemis-main/third_party/mobile_use/main.py

@@ -0,0 +1,130 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Single-task automation driver (upstream ``run_automation``).
+
+Used by the ``artemis run`` CLI command in ``artemis.interfaces.cli.commands.run``.
+"""
+
+import os
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from artemis import Agent, Builders
+from artemis.config import initialize_llm_config, settings
+from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
+from third_party.mobile_use.sdk.types.task import AgentProfile
+from third_party.mobile_use.utils.video import check_ffmpeg_available
+
+# CLI parameters of the ``run`` command.
+GoalArgument = Annotated[str, typer.Argument(help="The main goal for the agent to achieve.")]
+TestNameOption = Annotated[
+    str | None,
+    typer.Option(
+        "--test-name",
+        "-n",
+        help="Name of the test run for trace recording.",
+    ),
+]
+TracesPathOption = Annotated[
+    str | None,
+    typer.Option(
+        "--traces-path",
+        "-t",
+        help="Directory where execution traces are persisted.",
+    ),
+]
+OutputDescriptionOption = Annotated[
+    str | None,
+    typer.Option(
+        "--output-description",
+        "-o",
+        help="Natural language or schema description of the expected output.",
+    ),
+]
+VideoRecordingToolsOption = Annotated[
+    bool | None,
+    typer.Option(
+        "--with-video-recording-tools/--without-video-recording-tools",
+        help=(
+            "Enable or disable dynamic video recording and screen analysis "
+            "tools (auto-detected if omitted)."
+        ),
+    ),
+]
+
+
+def ensure_video_recording_available(with_video_recording_tools: bool | None) -> None:
+    """Fail early when video recording tools are requested but ffmpeg is missing."""
+    if with_video_recording_tools:
+        check_ffmpeg_available()
+
+
+def new_default_config_builder() -> AgentConfigBuilder:
+    """Return a config builder set up with the default LLM profile and ADB server."""
+    llm_config = initialize_llm_config()
+    agent_profile = AgentProfile(name="default", llm_config=llm_config)
+    config = Builders.AgentConfig.with_default_profile(profile=agent_profile)
+    if settings.ADB_HOST:
+        config.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT)
+    return config
+
+
+async def run_automation(
+    config: AgentConfigBuilder,
+    goal: str,
+    session_id: str | None = None,
+    locked_app_package: str | None = None,
+    test_name: str | None = None,
+    traces_output_path_str: str | None = None,
+    output_description: str | None = None,
+    profile: str | None = None,
+    app_path: str | None = None,
+) -> None:
+    """Initialize an agent from ``config``, run ``goal`` as a single task, then clean up."""
+    agent: Agent | None = None
+    try:
+        agent = Agent(config=config.build(), session_id=session_id)
+        await agent.init(
+            retry_count=int(os.getenv("ARTEMIS_HEALTH_RETRIES", 5)),
+            retry_wait_seconds=int(os.getenv("ARTEMIS_HEALTH_DELAY", 2)),
+        )
+
+        task = agent.new_task(goal)
+        if locked_app_package:
+            task.with_locked_app_package(locked_app_package)
+        if test_name:
+            trace_path = traces_output_path_str or str(settings.TRACES_PATH)
+            task.with_name(test_name).with_trace_recording(path=trace_path)
+        if output_description:
+            task.with_output_description(output_description)
+        if profile:
+            task.using_profile(profile)
+        if app_path:
+            task.with_app_path(Path(app_path))
+
+        llm_result_path = os.getenv("RESULTS_OUTPUT_PATH", None)
+        if llm_result_path:
+            task.with_llm_output_saving(path=llm_result_path)
+
+        await agent.run_task(request=task.build())
+    finally:
+        if agent is not None:
+            await agent.clean()

package/artemis-main/third_party/mobile_use/sdk/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.