android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/third_party/mobile_use/config/output.py

@@ -0,0 +1,184 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Structured output schemas, event recording, and output file management."""
+
+import json
+import os
+from pathlib import Path
+from typing import Annotated, Any
+import warnings
+
+from pydantic import BaseModel, Field, model_validator
+
+from artemis.config.constants import (
+    ENV_EVENTS_OUTPUT_PATH,
+    ENV_RESULTS_OUTPUT_PATH,
+)
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class OutputConfig(BaseModel):
+    """Configuration for structured output formatting and schemas."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    structured_output: Annotated[
+        type[BaseModel] | dict | None,
+        Field(
+            default=None,
+            description=(
+                "Optional structured schema (as a BaseModel or dict) to shape the output. "
+                "If provided, it takes precedence over 'output_description'."
+            ),
+        ),
+    ]
+    output_description: Annotated[
+        str | None,
+        Field(
+            default=None,
+            description=(
+                "Optional natural language description of the expected output format. "
+                "Used only if 'structured_output' is not provided. "
+                "Example: 'Output a JSON with 3 keys: color, price, websiteUrl'."
+            ),
+        ),
+    ]
+    enable_outputter: Annotated[
+        bool | None,
+        Field(
+            default=None,
+            description="Explicitly enable or disable Outputter post-execution synthesis.",
+        ),
+    ] = None
+    force_synthesis: Annotated[
+        bool,
+        Field(
+            default=False,
+            description="Whether to run Outputter synthesis even without explicit output_description or schema.",
+        ),
+    ] = False
+
+    def __str__(self) -> str:
+        s_builder = ""
+        if self.structured_output:
+            s_builder += f"Structured Output: {self.structured_output}\n"
+        if self.output_description:
+            s_builder += f"Output Description: {self.output_description}\n"
+        if self.output_description and self.structured_output:
+            s_builder += (
+                "Both 'structured_output' and 'output_description' are provided. "
+                "'structured_output' will take precedence.\n"
+            )
+        return s_builder
+
+    @model_validator(mode="after")
+    def warn_if_both_outputs_provided(self) -> "OutputConfig":
+        if self.structured_output and self.output_description:
+            warnings.warn(
+                "Both 'structured_output' and 'output_description' are provided. "
+                "'structured_output' will take precedence.",
+                stacklevel=2,
+            )
+        return self
+
+    def needs_structured_format(self, default_enabled: bool = True) -> bool:
+        """Returns True if Outputter should run to synthesize/structure the output."""
+        if self.enable_outputter is False:
+            return False
+        if self.enable_outputter is True or self.force_synthesis:
+            return True
+        if not default_enabled:
+            return False
+        return bool(self.structured_output or self.output_description)
+
+    def structured_output_schema(self) -> dict | type[BaseModel] | None:
+        """The structured output as a schema accepted by ``with_structured_output``."""
+        so = self.structured_output
+        if isinstance(so, dict):
+            return so
+        elif isinstance(so, BaseModel):
+            return type(so)
+        elif isinstance(so, type) and issubclass(so, BaseModel):
+            return so
+        return None
+
+    def bind_structured_output(self, llm: Any, llm_fallback: Any) -> tuple[Any, Any] | None:
+        """Bind the structured output schema to the main and fallback models.
+
+        Returns None when no structured output is configured.
+        """
+        schema = self.structured_output_schema()
+        if schema is None:
+            return None
+        return llm.with_structured_output(schema), llm_fallback.with_structured_output(schema)
+
+
+def prepare_output_files() -> tuple[str | None, str | None]:
+    """Validate and prepare events and results output files from environment variables."""
+    events_output_path = os.getenv(ENV_EVENTS_OUTPUT_PATH)
+    results_output_path = os.getenv(ENV_RESULTS_OUTPUT_PATH)
+
+    def validate_and_prepare_file(file_path: str) -> str | None:
+        if not file_path:
+            return None
+
+        path_obj = Path(file_path)
+
+        if path_obj.exists() and path_obj.is_dir():
+            logger.error(f"Error: Path '{file_path}' points to an existing directory, not a file.")
+            return None
+
+        if not path_obj.suffix or file_path.endswith(("/", "\\")):
+            logger.error(f"Error: Path '{file_path}' appears to be a directory path, not a file.")
+            return None
+
+        try:
+            path_obj.parent.mkdir(parents=True, exist_ok=True)
+            path_obj.touch(exist_ok=True)
+            return file_path
+        except OSError as e:
+            logger.error(f"Error creating file '{file_path}': {e}")
+            return None
+
+    validated_events_path = (
+        validate_and_prepare_file(events_output_path) if events_output_path else None
+    )
+    validated_results_path = (
+        validate_and_prepare_file(results_output_path) if results_output_path else None
+    )
+
+    return validated_events_path, validated_results_path
+
+
+def record_events(output_path: Path | None, events: list[str] | BaseModel | Any) -> None:
+    """Serialize and write execution events to disk."""
+    if not output_path:
+        return
+
+    if isinstance(events, str):
+        events_content = events
+    elif isinstance(events, BaseModel):
+        events_content = events.model_dump_json(indent=2)
+    else:
+        events_content = json.dumps(events, indent=2)
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(events_content)

package/artemis-main/third_party/mobile_use/context.py

@@ -0,0 +1,119 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Context variables for global state management.
+
+Uses ContextVar to avoid prop drilling and maintain clean function signatures.
+"""
+
+from pathlib import Path
+from typing import Any, Literal
+
+try:
+    from enum import StrEnum
+except ImportError:
+    from enum import Enum
+
+    class StrEnum(str, Enum):
+        pass
+
+
+from pydantic import BaseModel, ConfigDict
+
+
+class AppLaunchResult(BaseModel):
+    """Result of initial app launch attempt."""
+
+    model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
+
+    locked_app_package: str
+    locked_app_initial_launch_success: bool | None
+    locked_app_initial_launch_error: str | None
+
+
+class DevicePlatform(StrEnum):
+    """Mobile device platform enumeration."""
+
+    ANDROID = "android"
+
+
+class DeviceContext(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        extra="allow",
+    )
+
+    host_platform: Literal["WINDOWS", "LINUX", "DARWIN", "MACOS"] | str = "DARWIN"
+    mobile_platform: DevicePlatform = DevicePlatform.ANDROID
+    device_id: str = "default-device"
+
+    device_width: int = 1080
+    device_height: int = 2400
+
+    def to_str(self):
+        return (
+            f"Host platform: {self.host_platform}\n"
+            f"Mobile platform: {self.mobile_platform.value}\n"
+            f"Device ID: {self.device_id}\n"
+            f"Device width: {self.device_width}\n"
+            f"Device height: {self.device_height}\n"
+        )
+
+
+class ExecutionSetupBase(BaseModel):
+    """Execution setup for a task."""
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        extra="allow",
+    )
+
+    traces_path: Path | None = None
+    trace_name: str | None = None
+    enable_remote_tracing: bool = False
+    app_lock_status: AppLaunchResult | None = None
+
+    def get_locked_app_package(self) -> str | None:
+        """Get the locked app package name if app locking is enabled.
+
+        Returns:
+            The locked app package name, or None if app locking is not enabled.
+        """
+        if self.app_lock_status:
+            return self.app_lock_status.locked_app_package
+        return None
+
+
+class DeviceClientAccessors:
+    """Checked accessors for the device clients held by a context model.
+
+    The inheriting model declares the ``adb_client`` and ``ui_adb_client`` fields.
+    """
+
+    adb_client: Any
+    ui_adb_client: Any
+
+    def get_adb_client(self) -> Any:
+        if self.adb_client is None:
+            raise ValueError("No ADB client in context.")
+        return self.adb_client
+
+    def get_ui_adb_client(self) -> Any:
+        if self.ui_adb_client is None:
+            raise ValueError("No UIAutomator client in context.")
+        return self.ui_adb_client

package/artemis-main/third_party/mobile_use/controllers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

package/artemis-main/third_party/mobile_use/controllers/platform_specific_commands_controller.py

@@ -0,0 +1,151 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+import os
+from shutil import which
+import time
+from typing import Any
+
+from artemis.context import ArtemisContext, DevicePlatform
+from third_party.mobile_use.utils.logger import ArtemisLogger, get_logger
+from third_party.mobile_use.utils.shell_utils import run_shell_command_on_host
+
+logger = get_logger(__name__)
+
+
+def get_adb_device(ctx: ArtemisContext) -> Any:
+    """Retrieve the ADB device instance (AdbDevice locally or RemoteAdbDevice in Cloud mode)."""
+    if ctx.adb_client is None:
+        if os.environ.get("ARTEMIS_CLOUD_MODE") == "1":
+            from cloud_service.virtualization import RemoteAdbClient
+
+            ctx.adb_client = RemoteAdbClient()
+            return ctx.adb_client.device(serial=ctx.device.device_id)
+        return None
+
+    if ctx.device.mobile_platform != DevicePlatform.ANDROID:
+        return None
+
+    adb = ctx.get_adb_client()
+    return adb.device(serial=ctx.device.device_id)
+
+
+def get_first_device(
+    logger: ArtemisLogger | None = None,
+) -> tuple[str | None, DevicePlatform | None, None]:
+    """Gets the first available device, prioritizing idle and unassigned devices."""
+    try:
+        from artemis.runtime import device_pool
+
+        chosen = device_pool.select_device()
+        if chosen:
+            return chosen, DevicePlatform.ANDROID, None
+    except Exception as exc:
+        if logger:
+            logger.debug(f"Device pool selection fallback: {exc}")
+
+    if which("adb"):
+        try:
+            android_output = run_shell_command_on_host("adb devices")
+            lines = android_output.strip().split("\n")
+            for line in lines:
+                if "device" in line and not line.startswith("List of devices"):
+                    return line.split()[0], DevicePlatform.ANDROID, None
+        except RuntimeError as e:
+            if logger:
+                logger.error(f"ADB command failed: {e}")
+
+    return None, None, None
+
+
+def get_device_date(ctx: ArtemisContext) -> str:
+    device = get_adb_device(ctx)
+    if not device:
+        return time.strftime("%Y-%m-%d %H:%M:%S")
+    try:
+        return str(device.shell("date")).strip()
+    except Exception as e:
+        logger.debug(f"Failed to get device date: {e}")
+        return time.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def list_packages(ctx: ArtemisContext) -> str:
+    """List installed packages dynamically from the physical/remote device."""
+    device = get_adb_device(ctx)
+    if not device:
+        return ""
+
+    try:
+        cmd = ["pm", "list", "packages", "-f"]
+        raw_output = str(device.shell(" ".join(cmd)))
+    except Exception as e:
+        logger.error(f"Failed to query package list from device: {e}")
+        return ""
+
+    lines = raw_output.strip().split("\n")
+    packages = []
+    for line in lines:
+        if "=" in line:
+            package_name = line.split("=")[-1].strip()
+            if package_name:
+                packages.append(package_name)
+        elif line.startswith("package:"):
+            package_name = line.split("package:")[-1].strip()
+            if package_name:
+                packages.append(package_name)
+
+    return "\n".join(sorted(packages))
+
+
+async def list_packages_async(ctx: ArtemisContext) -> str:
+    return list_packages(ctx)
+
+
+def get_current_foreground_package(ctx: ArtemisContext) -> str | None:
+    """Get the package name of the currently focused/foreground app dynamically."""
+    device = get_adb_device(ctx)
+    if not device:
+        return None
+
+    try:
+        app_info = device.current_app()
+        if app_info and app_info.package:
+            return app_info.package
+    except Exception as e:
+        logger.debug(f"device.current_app failed: {e}. Falling back to dumpsys.")
+
+    try:
+        output = str(device.shell("dumpsys window | grep mCurrentFocus"))
+        if "mCurrentFocus=" in output:
+            segment = output.split("mCurrentFocus=")[-1]
+            if "/" in segment:
+                tokens = segment.split()
+                for token in tokens:
+                    if "." in token and not token.startswith("Window"):
+                        package = token.split("/")[0]
+                        package = package.rstrip("}")
+                        if package and "." in package:
+                            return package
+    except Exception as e:
+        logger.debug(f"Fallback dumpsys parsing failed: {e}")
+
+    return None
+
+
+async def get_current_foreground_package_async(ctx: ArtemisContext) -> str | None:
+    return get_current_foreground_package(ctx)

package/artemis-main/third_party/mobile_use/controllers/types.py

@@ -0,0 +1,92 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Geometry, element selector and tap result models shared by device drivers."""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TapOutput(BaseModel):
+    """Output from tap operations."""
+
+    error: str | None = Field(default=None, description="Error message if tap failed")
+    success: bool = Field(default=True, description="Whether the tap was successfully dispatched")
+
+
+class ElementQuery(BaseModel):
+    """Selects a UI element by resource id or text; ``index`` picks among the matches."""
+
+    model_config = ConfigDict(frozen=True)
+    resource_id: str | None = None
+    text: str | None = None
+    index: int = 0
+
+    def describe(self) -> str:
+        return f"resource_id={self.resource_id}, text={self.text}"
+
+
+class CoordinatesSelectorRequest(BaseModel):
+    """Absolute pixel coordinate pair."""
+
+    model_config = ConfigDict(extra="forbid")
+    x: int
+    y: int
+
+    def to_str(self) -> str:
+        return f"{self.x}, {self.y}"
+
+
+class PercentagesSelectorRequest(BaseModel):
+    """Normalized percentage coordinate pair (0-100%)."""
+
+    model_config = ConfigDict(extra="forbid")
+    x_percent: int = Field(ge=0, le=100, description="X percentage (0-100)")
+    y_percent: int = Field(ge=0, le=100, description="Y percentage (0-100)")
+
+    def to_str(self) -> str:
+        return f"{self.x_percent}%, {self.y_percent}%"
+
+    def to_coords(self, width: int, height: int) -> CoordinatesSelectorRequest:
+        """Convert percentages to absolute pixel coordinates."""
+        x = min(max(int(width * self.x_percent / 100), 0), max(0, width - 1))
+        y = min(max(int(height * self.y_percent / 100), 0), max(0, height - 1))
+        return CoordinatesSelectorRequest(x=x, y=y)
+
+
+class Bounds(BaseModel):
+    """Bounding box defined by top-left (x1, y1) and bottom-right (x2, y2)."""
+
+    x1: int
+    y1: int
+    x2: int
+    y2: int
+
+    @property
+    def width(self) -> int:
+        return max(0, self.x2 - self.x1)
+
+    @property
+    def height(self) -> int:
+        return max(0, self.y2 - self.y1)
+
+    def get_center(self) -> CoordinatesSelectorRequest:
+        """Get the center point of the bounds."""
+        return CoordinatesSelectorRequest(
+            x=(self.x1 + self.x2) // 2,
+            y=(self.y1 + self.y2) // 2,
+        )