android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/third_party/mobile_use/sdk/builders/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

package/artemis-main/third_party/mobile_use/sdk/builders/agent_config_builder.py

@@ -0,0 +1,225 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Builder for AgentConfig objects using a fluent interface."""
+
+import copy
+from typing import Any, ClassVar, Self
+
+from langchain_core.callbacks.base import Callbacks
+from artemis.config import get_default_llm_config
+from artemis.sdk.constants import DEFAULT_PROFILE_NAME
+from third_party.mobile_use.context import DevicePlatform
+from third_party.mobile_use.sdk.types.agent import AgentConfigBase, ServerConfig
+from third_party.mobile_use.sdk.types.task import AgentProfile, TaskRequestCommon
+
+
+class AgentConfigBuilderBase:
+    """Builder class providing a fluent interface for creating AgentConfig objects.
+
+    This builder allows for step-by-step construction of an AgentConfig with
+    clear methods that make the configuration process intuitive and type-safe.
+
+    Examples:
+        >>> builder = AgentConfigBuilder()
+        >>> config = (builder
+        ...     .add_profile(AgentProfile(name="HighReasoning",
+        llm_config=LLMConfig(...)))
+        ...     .add_profile(AgentProfile(name="LowReasoning",
+        llm_config=LLMConfig(...)))
+        ...     .for_device(DevicePlatform.ANDROID, "device123")
+        ...     .with_default_task_config(TaskRequestCommon(max_steps=30))
+        ...     .with_default_profile("HighReasoning")
+        ...     .build()
+        ... )
+
+    Subclasses extend the built configuration by setting ``config_class`` and
+    overriding :meth:`_extra_config_fields`.
+    """
+
+    config_class: ClassVar[type[AgentConfigBase]] = AgentConfigBase
+
+    def __init__(self, servers: ServerConfig):
+        """Initialize an empty AgentConfigBuilder."""
+        self._agent_profiles: dict[str, AgentProfile] = {}
+        self._task_request_defaults: TaskRequestCommon | None = None
+        self._default_profile: str | AgentProfile | None = None
+        self._device_id: str | None = None
+        self._device_platform: DevicePlatform | None = None
+        self._servers: ServerConfig = servers
+        self._graph_config_callbacks: Callbacks = None
+        self._cloud_mobile_id_or_ref: str | None = None
+
+    def add_profile(self, profile: AgentProfile, validate: bool = True) -> Self:
+        """Add an agent profile to the ARTEMIS agent.
+
+        Args:
+            profile: The agent profile to add
+        """
+        self._agent_profiles[profile.name] = profile
+        if validate:
+            profile.llm_config.validate_providers()
+        return self
+
+    def add_profiles(
+        self,
+        profiles: list[AgentProfile],
+        validate: bool = True,
+    ) -> Self:
+        """Add multiple agent profiles to the ARTEMIS agent.
+
+        Args:
+            profiles: List of agent profiles to add
+        """
+        for profile in profiles:
+            self.add_profile(profile=profile, validate=validate)
+        return self
+
+    def with_default_profile(self, profile: str | AgentProfile) -> Self:
+        """Set the default agent profile used for tasks.
+
+        Args:
+            profile: The name or instance of the default agent profile
+        """
+        self._default_profile = profile
+        return self
+
+    def for_device(
+        self,
+        platform_or_device_id: DevicePlatform | str,
+        device_id: str | None = None,
+    ) -> Self:
+        """Configure the ARTEMIS agent for a specific device.
+
+        Supports both:
+            builder.for_device(DevicePlatform.ANDROID, "emulator-5554")
+        and:
+            builder.for_device("emulator-5554")  (defaults to DevicePlatform.ANDROID)
+
+        Args:
+            platform_or_device_id: DevicePlatform or unique identifier for the device
+            device_id: The unique identifier for the device (if platform was passed first)
+        """
+        if self._cloud_mobile_id_or_ref is not None:
+            raise ValueError(
+                "Device ID cannot be set when a cloud mobile is already"
+                " configured.\n> for_device() and for_cloud_mobile() are"
+                " mutually exclusive"
+            )
+        if isinstance(platform_or_device_id, DevicePlatform):
+            self._device_platform = platform_or_device_id
+            self._device_id = device_id
+        else:
+            self._device_platform = DevicePlatform.ANDROID
+            self._device_id = str(platform_or_device_id)
+        return self
+
+    def with_default_task_config(self, config: TaskRequestCommon) -> Self:
+        """Set the default task configuration.
+
+        Args:
+            config: The task configuration to use as default
+        """
+        self._task_request_defaults = copy.deepcopy(config)
+        return self
+
+    def with_adb_server(self, host: str, port: int | None = None) -> Self:
+        """Set the ADB server host and port.
+
+        Args:
+            host: The ADB server host
+            port: The ADB server port
+        """
+        self._servers.adb_host = host
+        if port is not None:
+            self._servers.adb_port = port
+        return self
+
+    def with_servers(self, servers: ServerConfig) -> Self:
+        """Set the server settings.
+
+        Args:
+            servers: The server settings to use
+        """
+        self._servers = copy.deepcopy(servers)
+        return self
+
+    def with_graph_config_callbacks(self, callbacks: Callbacks) -> Self:
+        """Set the graph config callbacks.
+
+        Args:
+            callbacks: The graph config callbacks to use
+        """
+        self._graph_config_callbacks = callbacks
+        return self
+
+    def _extra_config_fields(self) -> dict[str, Any]:
+        """Fields a subclass adds to (or overrides in) the built configuration."""
+        return {}
+
+    def build(self, validate_profiles: bool = True) -> AgentConfigBase:
+        """Build the AgentConfig object.
+
+        Args:
+            default_profile: Name of the default agent profile to use
+
+        Returns:
+            A configured AgentConfig object
+
+        Raises:
+            ValueError: If default_profile is specified but not found in
+            configured profiles
+        """
+        nb_profiles = len(self._agent_profiles)
+
+        if isinstance(self._default_profile, str):
+            profile_name = self._default_profile
+            default_profile = self._agent_profiles.get(profile_name, None)
+            if default_profile is None:
+                raise ValueError(f"Profile '{profile_name}' not found in configured agents")
+        elif isinstance(self._default_profile, AgentProfile):
+            default_profile = self._default_profile
+            if default_profile.name not in self._agent_profiles:
+                self.add_profile(default_profile, validate=validate_profiles)
+        elif nb_profiles <= 0:
+            llm_config = get_default_llm_config()
+            default_profile = AgentProfile(
+                name=DEFAULT_PROFILE_NAME,
+                llm_config=llm_config,
+            )
+            self.add_profile(default_profile, validate=validate_profiles)
+        elif nb_profiles == 1:
+            # Select the only one available
+            default_profile = next(iter(self._agent_profiles.values()))
+        else:
+            available_profiles = ", ".join(self._agent_profiles.keys())
+            raise ValueError(
+                f"You must call with_default_profile() to select one among: {available_profiles}"
+            )
+
+        fields: dict[str, Any] = {
+            "agent_profiles": self._agent_profiles,
+            "task_request_defaults": self._task_request_defaults or TaskRequestCommon(),
+            "default_profile": default_profile,
+            "device_id": self._device_id,
+            "device_platform": self._device_platform,
+            "servers": self._servers,
+            "graph_config_callbacks": self._graph_config_callbacks,
+        }
+        fields.update(self._extra_config_fields())
+        return self.config_class(**fields)

package/artemis-main/third_party/mobile_use/sdk/builders/index.py

@@ -0,0 +1,33 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
+from third_party.mobile_use.sdk.builders.task_request_builder import TaskRequestCommonBuilder
+
+
+class BuildersWrapper:
+    @property
+    def AgentConfig(self) -> AgentConfigBuilder:
+        return AgentConfigBuilder()
+
+    @property
+    def TaskDefaults(self) -> TaskRequestCommonBuilder:
+        return TaskRequestCommonBuilder()
+
+
+Builders = BuildersWrapper()

package/artemis-main/third_party/mobile_use/sdk/builders/task_request_builder.py

@@ -0,0 +1,243 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Builder for TaskRequest objects using a fluent interface."""
+
+from pathlib import Path
+from typing import Generic, TypeVar, cast
+
+try:
+    from typing import Self
+except ImportError:
+    from typing import Self
+
+from artemis.constants import RECURSION_LIMIT
+from artemis.sdk.types.agent import AgentProfile
+from pydantic import BaseModel
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.sdk.types.task import TaskRequest, TaskRequestCommon
+
+TIn = TypeVar("TIn", bound=BaseModel | None)
+TOut = TypeVar("TOut", bound=BaseModel)
+
+
+class TaskRequestCommonBuilder(BaseModel):
+    """Builder class providing a fluent interface for creating TaskRequestCommon objects."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+
+    def __init__(self):
+        self._max_steps = RECURSION_LIMIT
+        self._record_trace = True
+        self._trace_path = Path("traces")
+        self._llm_output_path: Path | None = None
+        self._locked_app_package: str | None = None
+        self._app_path: Path | None = None
+
+    def with_max_steps(self, max_steps: int) -> Self:
+        """Set the maximum number of steps the task can take.
+
+        Args:
+            max_steps: Maximum number of steps
+        """
+        self._max_steps = max_steps
+        return self
+
+    def with_trace_recording(self, enabled: bool = True, path: str | None = None) -> Self:
+        """Configure trace recording for the task.
+
+        Traces record screenshots and actions during execution.
+
+        Args:
+            enabled: Whether to enable trace recording
+            path: Directory path where traces should be saved
+        """
+        self._record_trace = enabled
+        if enabled and path:
+            self._trace_path = Path(path)
+        return self
+
+    def with_llm_output_saving(self, path: str) -> Self:
+        """Configure LLM output saving for the task.
+
+        Args:
+            path: Path where to save the LLM output message
+        """
+        self._llm_output_path = Path(path)
+        return self
+
+    def with_locked_app_package(self, package_name: str) -> Self:
+        """Set the app package to lock execution to.
+
+        This ensures the specified app is launched and in the foreground before
+        the agentic loop starts.
+
+        Args:
+            package_name: Package name (Android, e.g., 'com.whatsapp')
+        """
+        self._locked_app_package = package_name
+        return self
+
+    def with_app_path(self, app_path: str | Path) -> Self:
+        """Set the path to an app to install before running the task.
+
+        For Android: Path to an APK file.
+
+        The app will be installed automatically before the task starts.
+
+        Args:
+            app_path: Path to the app file to install
+        """
+        self._app_path = Path(app_path) if isinstance(app_path, str) else app_path
+        return self
+
+    def build(self) -> TaskRequestCommon:
+        """Build the TaskRequestCommon object.
+
+        Returns:
+            A configured TaskRequestCommon object
+
+        Raises:
+            ValueError: If required fields are missing
+        """
+        return TaskRequestCommon(
+            max_steps=self._max_steps,
+            record_trace=self._record_trace,
+            trace_path=self._trace_path,
+            llm_output_path=self._llm_output_path,
+            locked_app_package=self._locked_app_package,
+            app_path=self._app_path,
+        )
+
+
+class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]):
+    """Builder class providing a fluent interface for creating TaskRequest objects.
+
+    This builder allows for step-by-step construction of a TaskRequest with
+    clear methods that make the configuration process intuitive and type-safe.
+
+    Examples:
+        >>> builder = TaskRequestBuilder[None](goal="Open Gmail and check unread
+        emails")
+        >>> task_request = (
+        ...     builder
+        ...     .with_max_steps(30)
+        ...     .using_profile("LowReasoning")
+        ...     .with_output_description("A list of email subjects and senders")
+        ...     .build()
+        ... )
+    """
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+
+    def __init__(self, goal: str):
+        """Initialize an empty TaskRequestBuilder."""
+        super().__init__()
+        self._goal = goal
+        self._profile: str | AgentProfile | None = None
+        self._name: str | None = None
+        self._output_description = None
+        self._output_format: type[TIn] | None = None
+
+    @classmethod
+    def from_common(cls, goal: str, common: TaskRequestCommon):
+        res = cls(goal=goal)
+        res._max_steps = common.max_steps
+        res._record_trace = common.record_trace
+        res._trace_path = common.trace_path
+        res._llm_output_path = common.llm_output_path
+        res._locked_app_package = common.locked_app_package
+        res._app_path = common.app_path
+        return res
+
+    def using_profile(self, profile: str | AgentProfile) -> "TaskRequestBuilder[TIn]":
+        """Set the agent profile for executing the task.
+
+        Args:
+            profile: The agent profile to use
+        """
+        self._profile = profile
+        return self
+
+    def with_name(self, name: str) -> "TaskRequestBuilder[TIn]":
+        """Set the name of the task - useful when recording traces.
+
+        Otherwise, a random name will be generated.
+
+        Args:
+            name: Name of the task
+        """
+        self._name = name
+        return self
+
+    def without_llm_output_saving(self) -> Self:
+        """Disable LLM output saving for the task."""
+        self._llm_output_path = None
+        return self
+
+    def with_output_description(self, description: str) -> "TaskRequestBuilder[TIn]":
+        """Set the description of the expected output format.
+
+        This is especially useful for data extraction tasks.
+
+        Args:
+            description: Description of the expected output format
+        """
+        self._output_description = description
+        return self
+
+    def with_output_format(self, output_format: type[TOut]) -> "TaskRequestBuilder[TOut]":
+        """Set the pydantic model for the expected output format.
+
+        Args:
+            output_format: Pydantic model instance defining the output format
+        """
+        self._output_format = output_format  # type: ignore
+        return cast(TaskRequestBuilder[TOut], self)
+
+    def build(self) -> TaskRequest[TIn]:
+        """Build the TaskRequest object.
+
+        Returns:
+            A configured TaskRequest object
+
+        Raises:
+            ValueError: If required fields are missing
+        """
+        if not self._goal:
+            raise ValueError("Task goal is required")
+
+        if self._output_format and self._output_description:
+            raise ValueError("Output format and description are mutually exclusive")
+
+        task_request = TaskRequest(
+            goal=self._goal,
+            profile=self._profile.name
+            if isinstance(self._profile, AgentProfile)
+            else self._profile,
+            task_name=self._name,
+            output_description=self._output_description,
+            output_format=self._output_format,
+            max_steps=self._max_steps,
+            record_trace=self._record_trace,
+            trace_path=self._trace_path,
+            llm_output_path=self._llm_output_path,
+            locked_app_package=self._locked_app_package,
+            app_path=self._app_path,
+        )
+        return task_request

package/artemis-main/third_party/mobile_use/sdk/types/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

package/artemis-main/third_party/mobile_use/sdk/types/agent.py

@@ -0,0 +1,98 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Agent configuration base models.
+
+Artemis extends :class:`AgentConfigBase` as ``artemis.sdk.types.agent.AgentConfig``.
+"""
+
+from typing import Literal
+from urllib.parse import urlparse
+
+from langchain_core.callbacks.base import Callbacks
+from pydantic import BaseModel
+
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.context import DevicePlatform
+from third_party.mobile_use.sdk.types.task import AgentProfile, TaskRequestCommon
+
+
+class ApiBaseUrl(BaseModel):
+    """Defines an API base URL."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    scheme: Literal["http", "https"]
+    host: str
+    port: int | None = None
+
+    def __eq__(self, other):
+        if not isinstance(other, ApiBaseUrl):
+            return False
+        return self.to_url() == other.to_url()
+
+    def to_url(self):
+        return (
+            f"{self.scheme}://{self.host}:{self.port}"
+            if self.port is not None
+            else f"{self.scheme}://{self.host}"
+        )
+
+    @classmethod
+    def from_url(cls, url: str) -> "ApiBaseUrl":
+        parsed_url = urlparse(url)
+        if parsed_url.scheme not in ["http", "https"]:
+            raise ValueError(f"Invalid scheme: {parsed_url.scheme}")
+        if parsed_url.hostname is None:
+            raise ValueError("Invalid hostname")
+        return cls(
+            scheme=parsed_url.scheme,  # type: ignore
+            host=parsed_url.hostname,
+            port=parsed_url.port,
+        )
+
+
+class ServerConfig(BaseModel):
+    """Configuration for the required servers."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+
+    adb_host: str
+    adb_port: int
+
+
+class AgentConfigBase(BaseModel):
+    """ARTEMIS agent configuration.
+
+    Attributes:
+        agent_profiles: Map an agent profile name to its configuration.
+        task_config_defaults: Default task request configuration.
+        default_profile: default profile to use for tasks
+        device_id: Specific device to target (if None, first available is used).
+        device_platform: Platform of the device to target.
+        servers: Custom server configurations.
+    """
+
+    agent_profiles: dict[str, AgentProfile]
+    task_request_defaults: TaskRequestCommon
+    default_profile: AgentProfile
+    device_id: str | None = None
+    device_platform: DevicePlatform | None = None
+    servers: ServerConfig
+    graph_config_callbacks: Callbacks = None
+
+    model_config = {"arbitrary_types_allowed": True}