android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/third_party/mobile_use/sdk/types/exceptions.py

@@ -0,0 +1,109 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Exceptions for the ARTEMIS SDK.
+
+This module defines the exception hierarchy used throughout the Artemis SDK.
+"""
+
+from typing import Literal
+
+
+class ArtemisError(Exception):
+    """Base exception class for all ARTEMIS SDK exceptions."""
+
+    def __init__(self, message="An error occurred in the Artemis SDK"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class DeviceError(ArtemisError):
+    """Exception raised for errors related to mobile devices."""
+
+    def __init__(self, message="A device-related error occurred"):
+        super().__init__(message)
+
+
+class DeviceNotFoundError(DeviceError):
+    """Exception raised when no mobile device is found."""
+
+    def __init__(self, message="No mobile device found"):
+        super().__init__(message)
+
+
+class ServerError(ArtemisError):
+    """Exception raised for errors related to ARTEMIS servers."""
+
+    def __init__(self, message="A server-related error occurred"):
+        super().__init__(message)
+
+
+class ServerStartupError(ServerError):
+    """Exception raised when ARTEMIS servers fail to start."""
+
+    def __init__(self, server_name=None, message=None):
+        if server_name and not message:
+            message = f"Failed to start {server_name}"
+        elif not message:
+            message = "Failed to start Artemis servers"
+        super().__init__(message)
+        self.server_name = server_name
+
+
+class AgentError(ArtemisError):
+    """Exception raised for errors related to the ARTEMIS agent."""
+
+    def __init__(self, message="An agent-related error occurred"):
+        super().__init__(message)
+
+
+class AgentNotInitializedError(AgentError):
+    """Exception raised when attempting operations on an uninitialized agent."""
+
+    def __init__(self, message="Agent is not initialized. Call init() first"):
+        super().__init__(message)
+
+
+class AgentTaskRequestError(AgentError):
+    """Exception raised when a requested task is invalid."""
+
+    def __init__(self, message="An agent task-related error occurred"):
+        super().__init__(message)
+
+
+class AgentProfileNotFoundError(AgentTaskRequestError):
+    """Exception raised when an agent profile is not found."""
+
+    def __init__(self, profile_name: str):
+        super().__init__(f"Agent profile {profile_name} not found")
+
+
+EXECUTABLES = Literal["adb"]
+
+
+class ExecutableNotFoundError(ArtemisError):
+    """Exception raised when a required executable is not found."""
+
+    def __init__(self, executable_name: EXECUTABLES):
+        install_instructions: dict[EXECUTABLES, str] = {
+            "adb": "https://developer.android.com/tools/adb",
+        }
+        message = f"Required executable '{executable_name}' not found in PATH."
+        if executable_name in install_instructions:
+            message += f"\nTo install it, please visit: {install_instructions[executable_name]}"
+        super().__init__(message)

package/artemis-main/third_party/mobile_use/sdk/types/task.py

@@ -0,0 +1,234 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Task-related type definitions for the ARTEMIS SDK."""
+
+from collections.abc import Callable, Coroutine
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Generic, Literal, TypeVar, overload
+
+from artemis.config import LLMConfig, get_default_llm_config
+from artemis.constants import RECURSION_LIMIT
+from artemis.context import DeviceContext
+from artemis.sdk.utils import load_llm_config_override
+from pydantic import BaseModel, Field
+from artemis.utils.cython_compat import CyFunctionDetector
+
+TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"]
+
+
+class AgentProfile(BaseModel):
+    """Represents a ARTEMIS agent profile.
+
+    Attributes:
+        name: Name of the agent - used to reference the agent when running
+          tasks.
+        llm_config: LLM configuration for the agent.
+    """
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+
+    name: str
+    llm_config: LLMConfig = Field(default_factory=get_default_llm_config)
+
+    @overload
+    def __init__(self, *, name: str, llm_config: LLMConfig): ...
+
+    @overload
+    def __init__(self, *, name: str, from_file: str): ...
+
+    def __init__(
+        self,
+        *,
+        name: str,
+        llm_config: LLMConfig | None = None,
+        from_file: str | None = None,
+        **kwargs,
+    ):
+        kwargs["name"] = name
+        if from_file:
+            kwargs["llm_config"] = load_llm_config_override(Path(from_file))
+        elif llm_config:
+            kwargs["llm_config"] = llm_config
+        else:
+            raise ValueError("Either llm_config or from_file must be provided")
+        super().__init__(**kwargs)
+
+    def __str__(self):
+        return f"Profile {self.name}:\n{self.llm_config}"
+
+
+T = TypeVar("T", bound=BaseModel)
+TOutput = TypeVar("TOutput", bound=BaseModel | None)
+
+
+class TaskRequestBase(BaseModel):
+    """Defines common parameters of a mobile automation task request."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    max_steps: int = RECURSION_LIMIT
+    record_trace: bool = True
+    trace_path: Path = Path("traces")
+    llm_output_path: Path | None = None
+
+
+class TaskRequestCommon(TaskRequestBase):
+    """Defines common parameters for any task request."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    max_steps: int = RECURSION_LIMIT
+    locked_app_package: str | None = None
+    app_path: Path | None = None
+    """Path to an app to install before running the task.
+    
+    For Android: Path to an APK file.
+    
+    The app will be installed automatically before the task starts.
+    """
+
+
+class TaskRequest(TaskRequestCommon, Generic[TOutput]):
+    """Defines the format of a mobile automation task request.
+
+    Attributes:
+        goal: Natural language description of the goal to achieve
+        profile: Optional agent profile to use for executing the task
+        task_name: Optional name for the task
+        output_description: Optional natural language description of expected
+          output format
+        output_format: Optional pydantic model for the output format of the task
+        max_steps: Maximum number of steps the agent can take (default: 20)
+        record_trace: Whether to record a trace (screenshots, actions) of the
+          execution (default: False)
+        trace_path: Directory path to save trace data if recording is enabled
+        llm_output_path: Path to save LLM output data
+    """
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    goal: str
+    profile: str | None = None
+    task_name: str | None = None
+    output_description: str | None = None
+    output_format: type[TOutput] | None = None
+    enable_remote_tracing: bool = False
+
+
+class TaskResult(BaseModel):
+    """Result of a mobile automation task.
+
+    Attributes:
+        content: Raw result content (could be text or structured data)
+        error: Error message if the task failed
+        execution_time_seconds: How long the task took to execute
+        steps_taken: Number of steps the agent took to complete the task
+    """
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    content: Any = None
+    error: str | None = None
+    execution_time_seconds: float
+    steps_taken: int
+
+    def get_as_model(self, model_class: type[T]) -> T:
+        """Parse the content into a Pydantic model instance.
+
+        Args:
+            model_class: The Pydantic model class to parse the data into
+
+        Returns:
+            An instance of the specified model class
+
+        Raises:
+            ValueError: If content is None or not compatible with the model
+        """
+        if self.content is None:
+            raise ValueError("No content available to parse into a model")
+        if isinstance(self.content, model_class):
+            return self.content
+        return model_class.model_validate(self.content)
+
+
+class Task(BaseModel):
+    """A mobile automation task to be executed.
+
+    Attributes:
+        id: Unique identifier for the task
+        device: Information about the target device
+        status: Current status of the task execution
+        request: User task request
+        created_at: ISO timestamp when the task was created
+        ended_at: ISO timestamp when the task ended
+    """
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    id: str
+    device: DeviceContext
+    status: TaskRunStatus
+    status_message: str | None = None
+    on_status_changed: Callable[[TaskRunStatus, str | None, Any | None], Coroutine] | None = None
+    request: TaskRequest
+    created_at: datetime
+    ended_at: datetime | None = None
+    result: TaskResult | None = None
+
+    async def finalize(
+        self,
+        content: Any | None = None,
+        state: dict | None = None,
+        error: str | None = None,
+        cancelled: bool = False,
+    ):
+        new_status: TaskRunStatus = "completed" if error is None else "failed"
+        if new_status == "failed" and cancelled:
+            new_status = "cancelled"
+        message = "Task completed successfully"
+        if new_status == "failed":
+            message = "Task failed" + (f": {error}" if error else "")
+        elif new_status == "cancelled":
+            message = "Task cancelled" + (f": {error}" if error else "")
+        await self.set_status(status=new_status, message=message, output=content or error)
+        self.ended_at = datetime.now()
+
+        duration = self.ended_at - self.created_at
+        steps_taken = -1
+        if state is not None:
+            metadata = state.get("metadata", None)
+            if metadata:
+                steps_taken = metadata.get("step_count", -1)
+
+        self.result = TaskResult(
+            content=content,
+            error=error,
+            execution_time_seconds=duration.total_seconds(),
+            steps_taken=steps_taken,
+        )
+
+    def get_name(self) -> str:
+        return self.request.task_name or self.id
+
+    async def set_status(
+        self,
+        status: TaskRunStatus,
+        message: str | None = None,
+        output: Any | None = None,
+    ):
+        self.status = status
+        self.status_message = message
+        if self.on_status_changed:
+            await self.on_status_changed(status, message, output)

package/artemis-main/third_party/mobile_use/testing.py

@@ -0,0 +1,59 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Pytest fixtures for agent node tests (upstream ``agents/outputter/test_outputter.py``).
+
+Test-only: imports pytest, so it must not be imported by runtime code.
+"""
+
+from unittest.mock import Mock
+
+import pytest
+
+
+class DummyState:
+    def __init__(self, messages, initial_goal, operator_raw_data=None):
+        self.messages = messages
+        self.initial_goal = initial_goal
+        self.operator_raw_data = operator_raw_data
+
+
+@pytest.fixture
+def mock_context():
+    """Create a properly mocked context with all required fields."""
+    from artemis.config import LLM
+    from artemis.context import ArtemisContext
+
+    ctx = Mock(spec=ArtemisContext)
+    ctx.llm_config = {
+        "planner": LLM(provider="openai", model="gpt-5-nano"),
+        "operator": LLM(provider="openai", model="gpt-5-nano"),
+        "validator": LLM(provider="openai", model="gpt-5-nano"),
+    }
+    ctx.device = Mock()
+    ctx.data_engine = None
+    return ctx
+
+
+@pytest.fixture
+def mock_state():
+    """Create a mock state with test data."""
+    return DummyState(
+        messages=[],
+        initial_goal="Find a green product on my website",
+    )

package/artemis-main/third_party/mobile_use/tools/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

package/artemis-main/third_party/mobile_use/tools/index.py

@@ -0,0 +1,44 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Instantiate LangChain tools from tool wrappers."""
+
+from collections.abc import Callable
+
+from langchain_core.tools import BaseTool
+
+from artemis.context import ArtemisContext
+from third_party.mobile_use.tools.tool_wrapper import CompositeToolWrapper, ToolWrapper
+
+
+def get_tools_from_wrappers(
+    ctx: "ArtemisContext",
+    wrappers: list[ToolWrapper],
+    wrap_tool: Callable[[BaseTool], BaseTool] | None = None,
+) -> list[BaseTool]:
+    """Instantiate and wrap LangChain tools from a list of ToolWrappers."""
+    tools: list[BaseTool] = []
+    for wrapper in wrappers:
+        if wrapper.is_available_fn is not None and not wrapper.is_available_fn(ctx):
+            continue
+        if isinstance(wrapper, CompositeToolWrapper):
+            new_tools = wrapper.composite_tools_fn_getter(ctx)
+        else:
+            new_tools = [wrapper.tool_fn_getter(ctx)]
+        tools.extend(wrap_tool(t) if wrap_tool else t for t in new_tools)
+    return tools

package/artemis-main/third_party/mobile_use/tools/mobile/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

package/artemis-main/third_party/mobile_use/tools/mobile/launch_app.py

@@ -0,0 +1,89 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Resolve a natural-language app name to an installed package via the Hopper agent."""
+
+from artemis.context import ArtemisContext
+from third_party.mobile_use.agents.hopper.hopper import HopperOutput, hopper
+from third_party.mobile_use.controllers.platform_specific_commands_controller import (
+    list_packages_async,
+)
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+LAUNCH_APP_DOCSTRING = (
+    "[ACTION] Finds and launches an application on the device using its natural language name."
+)
+
+
+async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool = True) -> str | None:
+    """Finds the package name for a given application name.
+
+    Returns None if package not found or on error.
+    """
+    package_cache = getattr(ctx, "package_cache", None)
+    if package_cache is None or not isinstance(package_cache, dict):
+        try:
+            ctx.package_cache = {}
+            package_cache = ctx.package_cache
+        except Exception:  # pylint: disable=broad-exception-caught
+            package_cache = {}
+
+    if isinstance(package_cache, dict) and app_name in package_cache:
+        logger.info(f"Cache hit for app '{app_name}': {package_cache[app_name]}")
+        return package_cache[app_name]
+
+    try:
+        all_packages = await list_packages_async(ctx=ctx)
+        package_set = {p.strip() for p in all_packages.split("\n") if p.strip()}
+
+        # Fast path: If app_name is already directly an installed package name
+        if app_name in package_set:
+            if isinstance(package_cache, dict):
+                package_cache[app_name] = app_name
+            return app_name
+
+        hopper_output: HopperOutput = await hopper(
+            ctx=ctx,
+            request=(f"I'm looking for the package name of the following app: '{app_name}'"),
+            data=all_packages,
+            use_fallback=use_fallback,
+        )
+        if not hopper_output.found or not hopper_output.output:
+            if isinstance(package_cache, dict):
+                package_cache[app_name] = None
+            return None
+
+        package_name = hopper_output.output.strip()
+        if package_name not in package_set:
+            logger.warning(
+                f"Hopper returned package '{package_name}' for '{app_name}', "
+                "but it is NOT physically installed on the device!"
+            )
+            if isinstance(package_cache, dict):
+                package_cache[app_name] = None
+            return None
+
+        if isinstance(package_cache, dict):
+            package_cache[app_name] = package_name
+        return package_name
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logger.error(f"Failed to find package for '{app_name}': {e}")
+        return None

package/artemis-main/third_party/mobile_use/tools/tool_wrapper.py

@@ -0,0 +1,52 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Tool wrappers: a LangChain tool factory and the messages shown for its result."""
+
+from collections.abc import Callable
+
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel
+
+from artemis.context import ArtemisContext
+from artemis.utils.cython_compat import CyFunctionDetector
+
+
+class ToolWrapper(BaseModel):
+    """Wrapper holding a tool factory and lifecycle callbacks."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    tool_fn_getter: Callable[[ArtemisContext], BaseTool]
+    on_success_fn: Callable[..., str]
+    on_failure_fn: Callable[..., str]
+    is_available_fn: Callable[[ArtemisContext], bool] | None = None
+
+
+class CompositeToolWrapper(ToolWrapper):
+    """Wrapper holding a composite tool factory and lifecycle callbacks."""
+
+    composite_tools_fn_getter: Callable[[ArtemisContext], list[BaseTool]]
+
+
+def make_save_note_wrapper(tool_fn_getter: Callable[[ArtemisContext], BaseTool]) -> ToolWrapper:
+    """Wrapper for the scratchpad ``save_note`` tool with its outcome messages."""
+    return ToolWrapper(
+        tool_fn_getter=tool_fn_getter,
+        on_success_fn=lambda key: f"Saved note '{key}'.",
+        on_failure_fn=lambda key: f"Failed to save note '{key}'.",
+    )

package/artemis-main/third_party/mobile_use/utils/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.