android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/third_party/mobile_use/utils/app_launch_utils.py

@@ -0,0 +1,217 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+"""Utilities for handling app locking and initial app launch logic."""
+
+import asyncio
+
+from artemis.context import AppLaunchResult, ArtemisContext
+from artemis.controllers.unified_controller import UnifiedMobileController
+from artemis.data_engine.trace import TraceSpan
+from artemis.utils.app_launch_utils import current_foreground, observe_foreground
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+async def _poll_for_app_ready(
+    ctx: ArtemisContext,
+    app_package: str,
+    max_poll_seconds: int = 15,
+    poll_interval: float = 1.0,
+) -> tuple[bool, str | None]:
+    """Poll for app to be ready after launch.
+
+    Treats mCurrentFocus=null as a loading state and keeps polling.
+    Only fails if we get a different (non-null) package or timeout.
+
+    Args:
+        ctx: Mobile use context
+        app_package: Expected package name
+        max_poll_seconds: Maximum time to poll (default: 15s)
+        poll_interval: Time between polls (default: 1s)
+
+    Returns:
+        Tuple of (success: bool, error_message: str | None)
+    """
+    polls = int(max_poll_seconds / poll_interval)
+
+    for i in range(polls):
+        ready, current_package, task = await observe_foreground(ctx, app_package)
+
+        if ready:
+            if current_package == app_package:
+                logger.success(f"App {app_package} is ready (took ~{i * poll_interval:.1f}s)")
+            else:
+                logger.success(
+                    f"App {app_package} is ready (focused window belongs to"
+                    f" '{current_package}', but the foreground {task.describe()} is owned"
+                    f" by {app_package}, took ~{i * poll_interval:.1f}s)"
+                )
+            return True, None
+
+        if current_package is None:
+            logger.debug(f"Poll {i + 1}/{polls}: App loading (mCurrentFocus=null)...")
+        else:
+            logger.debug(
+                f"Poll {i + 1}/{polls}: Wrong app in foreground (expected"
+                f" '{app_package}', got '{current_package}', foreground"
+                f" {task.describe() if task else 'task unknown'}). Still waiting..."
+            )
+
+        if i < polls - 1:
+            await asyncio.sleep(poll_interval)
+
+    current_package, task = await current_foreground(ctx)
+    error_msg = (
+        f"Timeout waiting for {app_package} to load after {max_poll_seconds}s. "
+        f"Current foreground: {current_package}; foreground"
+        f" {task.describe() if task else 'task unknown'}"
+    )
+    logger.error(error_msg)
+    return False, error_msg
+
+
+async def launch_app_with_retries(
+    ctx: ArtemisContext,
+    app_package: str,
+    max_retries: int = 3,
+    max_poll_seconds: int = 15,
+) -> tuple[bool, str | None]:
+    """Launch an app with retry logic and smart polling.
+
+    Args:
+        ctx: Mobile use context
+        app_package: Package name (Android) to launch
+        max_retries: Maximum number of launch attempts (default: 3)
+        max_poll_seconds: Maximum time to wait for app to load per attempt
+          (default: 15s)
+
+    Returns:
+        Tuple of (success: bool, error_message: str | None)
+    """
+
+    for attempt in range(1, max_retries + 1):
+        logger.info(f"Launch attempt {attempt}/{max_retries} for app {app_package}")
+
+        with TraceSpan(
+            name=f"Launch Attempt {attempt}",
+            trace_type="span",
+            ctx=ctx,
+        ) as span:
+            span.payload = {"attempt": attempt, "app_package": app_package}
+
+            controller = UnifiedMobileController(ctx)
+            if attempt > 1:
+                logger.warning(
+                    f"Attempt {attempt - 1} failed. Force stopping"
+                    f" '{app_package}' to clear frozen state before retrying..."
+                )
+                await controller.terminate_app(app_package)
+                await asyncio.sleep(1.0)
+
+            launch_success = await controller.launch_app(app_package)
+            if not launch_success:
+                error_msg = f"Failed to execute launch command for {app_package}"
+                logger.error(error_msg)
+                span.status = "failed"
+                span.error = error_msg
+                if attempt == max_retries:
+                    return False, error_msg
+                await asyncio.sleep(2)
+                continue
+
+            await asyncio.sleep(1)
+
+            success, error_msg = await _poll_for_app_ready(ctx, app_package, max_poll_seconds)
+
+            if success:
+                span.status = "success"
+                span.result = "App is ready"
+                return True, None
+
+            span.status = "failed"
+            span.error = error_msg
+
+            if attempt < max_retries:
+                logger.warning(f"Attempt {attempt} failed: {error_msg}. Retrying...")
+                await asyncio.sleep(1)
+
+    error_msg = f"Failed to launch {app_package} after {max_retries} attempts"
+    logger.error(error_msg)
+    return False, error_msg
+
+
+async def _handle_initial_app_launch(
+    ctx: ArtemisContext,
+    locked_app_package: str,
+) -> AppLaunchResult:
+    """Handle initial app launch verification and launching if needed.
+
+    If locked_app_package is set:
+    1. Check if the app is already in the foreground
+    2. If not, attempt to launch it (with retries)
+    3. Return status with success/error information
+
+    Args:
+        ctx: Mobile use context
+        locked_app_package: Package name (Android) to lock to
+
+    Returns:
+        AppLaunchResult with launch status and error information
+    """
+    if not locked_app_package:
+        error_msg = f"Invalid locked_app_package: '{locked_app_package}'"
+        logger.error(error_msg)
+        return AppLaunchResult(
+            locked_app_package=locked_app_package,
+            locked_app_initial_launch_success=False,
+            locked_app_initial_launch_error=error_msg,
+        )
+
+    logger.info(f"Starting initial app launch for package: {locked_app_package}")
+
+    try:
+        already_foreground, current_package, _ = await observe_foreground(ctx, locked_app_package)
+        logger.info(f"Current foreground app: {current_package}")
+
+        if already_foreground:
+            logger.info(f"App {locked_app_package} is already in foreground")
+            return AppLaunchResult(
+                locked_app_package=locked_app_package,
+                locked_app_initial_launch_success=True,
+                locked_app_initial_launch_error=None,
+            )
+
+        logger.info(f"App {locked_app_package} not in foreground, attempting to launch")
+        success, error_msg = await launch_app_with_retries(ctx, locked_app_package)
+
+        return AppLaunchResult(
+            locked_app_package=locked_app_package,
+            locked_app_initial_launch_success=success,
+            locked_app_initial_launch_error=error_msg,
+        )
+
+    except Exception as e:
+        error_msg = f"Exception during initial app launch: {str(e)}"
+        logger.error(error_msg)
+        return AppLaunchResult(
+            locked_app_package=locked_app_package,
+            locked_app_initial_launch_success=False,
+            locked_app_initial_launch_error=error_msg,
+        )

package/artemis-main/third_party/mobile_use/utils/cli_helpers.py

@@ -0,0 +1,60 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+from shutil import which
+import sys
+
+from adbutils import AdbClient
+from rich.console import Console
+
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def display_device_status(console: Console, adb_client: AdbClient | None = None):
+    """Checks for connected devices and displays the status."""
+    console.print("\n[bold]📱 Device Status[/bold]")
+    devices = None
+    if adb_client is not None:
+        devices = adb_client.device_list()
+    if devices:
+        console.print("✅ [bold green]Android device(s) connected:[/bold green]")
+        for device in devices:
+            console.print(f"  - {device.serial}")
+    else:
+        console.print("❌ [bold red]No Android device found.[/bold red]")
+        command = "emulator -avd <avd_name>"
+        if sys.platform not in ["win32", "darwin"]:
+            command = f"./{command}"
+            console.print(
+                f"You can start an emulator using a command like: [bold]'{command}'[/bold]"
+            )
+
+
+def display_local_device_status(console: Console, host: str | None, port: int | None) -> None:
+    """Display connected Android devices, using a local ADB client when ``adb`` is available."""
+    adb_client = None
+    try:
+        if which("adb"):
+            adb_client = AdbClient(host=host or "localhost", port=port or 5037)
+    except Exception as exc:
+        # Optional cosmetic device-status display; run continues without it.
+        logger.debug(f"Could not create ADB client for device status display: {exc}")
+
+    display_device_status(console, adb_client=adb_client)

package/artemis-main/third_party/mobile_use/utils/decorators.py

@@ -0,0 +1,158 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from functools import wraps
+import inspect
+from typing import Any, TypeVar, cast, overload
+
+R = TypeVar("R")
+
+
+def wrap_with_callbacks_sync(
+    fn: Callable[..., R],
+    *,
+    before: Callable[..., None] | None = None,
+    on_success: Callable[[R], None] | None = None,
+    on_failure: Callable[[Exception], None] | None = None,
+    suppress_exceptions: bool = False,
+) -> Callable[..., R]:
+    @wraps(fn)
+    def wrapper(*args: Any, **kwargs: Any) -> R:
+        if before:
+            before()
+        try:
+            result = fn(*args, **kwargs)
+            if on_success:
+                on_success(result)
+            return result
+        except Exception as e:
+            if on_failure:
+                on_failure(e)
+            if suppress_exceptions:
+                return None  # type: ignore
+            raise
+
+    return wrapper
+
+
+def wrap_with_callbacks_async(
+    fn: Callable[..., Awaitable[R]],
+    *,
+    before: Callable[..., None] | None = None,
+    on_success: Callable[[R], None] | None = None,
+    on_failure: Callable[[Exception], None] | None = None,
+    suppress_exceptions: bool = False,
+) -> Callable[..., Awaitable[R]]:
+    @wraps(fn)
+    async def wrapper(*args: Any, **kwargs: Any) -> R:
+        if before:
+            before()
+        try:
+            result = await fn(*args, **kwargs)
+            if on_success:
+                on_success(result)
+            return result
+        except Exception as e:
+            if on_failure:
+                on_failure(e)
+            if suppress_exceptions:
+                return None  # type: ignore
+            raise
+
+    return wrapper
+
+
+@overload
+def wrap_with_callbacks(
+    fn: Callable[..., Awaitable[R]],
+    *,
+    before: Callable[[], None] | None = ...,
+    on_success: Callable[[R], None] | None = ...,
+    on_failure: Callable[[Exception], None] | None = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[..., Awaitable[R]]: ...
+
+
+@overload
+def wrap_with_callbacks(
+    *,
+    before: Callable[..., None] | None = ...,
+    on_success: Callable[[Any], None] | None = ...,
+    on_failure: Callable[[Exception], None] | None = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[[Callable[..., R]], Callable[..., R]]: ...
+
+
+@overload
+def wrap_with_callbacks(
+    fn: Callable[..., R],
+    *,
+    before: Callable[[], None] | None = ...,
+    on_success: Callable[[R], None] | None = ...,
+    on_failure: Callable[[Exception], None] | None = ...,
+    suppress_exceptions: bool = ...,
+) -> Callable[..., R]: ...
+
+
+def wrap_with_callbacks(
+    fn: Callable[..., Any] | None = None,
+    *,
+    before: Callable[[], None] | None = None,
+    on_success: Callable[[Any], None] | None = None,
+    on_failure: Callable[[Exception], None] | None = None,
+    suppress_exceptions: bool = False,
+) -> Any:
+    def decorator(func: Callable[..., Any]) -> Any:
+        if inspect.iscoroutinefunction(func):
+            return wrap_with_callbacks_async(
+                cast(Callable[..., Awaitable[Any]], func),
+                before=before,
+                on_success=on_success,
+                on_failure=on_failure,
+                suppress_exceptions=suppress_exceptions,
+            )
+        else:
+            return wrap_with_callbacks_sync(
+                cast(Callable[..., Any], func),
+                before=before,
+                on_success=on_success,
+                on_failure=on_failure,
+                suppress_exceptions=suppress_exceptions,
+            )
+
+    if fn is None:
+        return decorator
+    else:
+        return decorator(fn)
+
+
+def agent_lifecycle_logging(
+    agent_name: str, logger: Any
+) -> Callable[[Callable[..., R]], Callable[..., R]]:
+    """Log the start, success and failure of an agent node call.
+
+    ``agent_name`` is the display name, e.g. ``"Planner"``.
+    """
+    label = f"{agent_name} Agent"
+    return wrap_with_callbacks(
+        before=lambda: logger.info(f"Starting {label}..."),
+        on_success=lambda _: logger.success(label),
+        on_failure=lambda _: logger.error(label),
+    )

package/artemis-main/third_party/mobile_use/utils/file.py

@@ -0,0 +1,31 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+import json
+import re
+from typing import IO
+
+
+def strip_json_comments(text: str) -> str:
+    text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE)
+    text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
+    return text
+
+
+def load_jsonc(file: IO) -> dict:
+    return json.loads(strip_json_comments(file.read()))

package/artemis-main/third_party/mobile_use/utils/logger.py

@@ -0,0 +1,216 @@
+# Copyright 2025-2026 Minitap, Inc.
+# Modifications Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Originally from mobile-use (https://github.com/minitap-ai/mobile-use).
+# See third_party/mobile_use/METADATA for the upstream source and local modifications.
+
+from enum import Enum
+import logging
+import os
+from pathlib import Path
+import queue
+import sys
+import threading
+
+from colorama import Fore, Style, init
+
+from artemis.data_engine.context_vars import CURRENT_TRACE_ID
+
+init(autoreset=True)
+
+
+class LogLevel(Enum):
+    DEBUG = ("DEBUG", Fore.MAGENTA, "🔍")
+    INFO = ("INFO", Fore.WHITE, "ℹ")
+    SUCCESS = ("SUCCESS", Fore.GREEN, "✓")
+    WARNING = ("WARNING", Fore.YELLOW, "⚠")
+    ERROR = ("ERROR", Fore.RED, "❌")
+    CRITICAL = ("CRITICAL", Fore.RED + Style.BRIGHT, "💥")
+
+
+class DataEngineHandler(logging.Handler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._log_queue = queue.Queue(maxsize=5000)
+        self._worker_thread = threading.Thread(target=self._drain_queue, daemon=True)
+        self._worker_thread.start()
+
+    def _drain_queue(self):
+        while True:
+            item = self._log_queue.get()
+            if item is None:
+                break
+            engine_mod = sys.modules.get("artemis.data_engine.engine")
+            if engine_mod and getattr(engine_mod, "_CURRENT_DATA_ENGINE", None):
+                _CURRENT_DATA_ENGINE = engine_mod._CURRENT_DATA_ENGINE
+                if _CURRENT_DATA_ENGINE and _CURRENT_DATA_ENGINE.current_session_id:
+                    record_name, message, level_name, formatted, parent_id = item
+                    _CURRENT_DATA_ENGINE.record_trace(
+                        type="log",
+                        name=record_name,
+                        payload={
+                            "message": message,
+                            "level": level_name,
+                            "formatted": formatted,
+                        },
+                        status="success",
+                        parent_trace_id=parent_id,
+                    )
+            self._log_queue.task_done()
+
+    def emit(self, record):
+        engine_mod = sys.modules.get("artemis.data_engine.engine")
+        if not engine_mod or not getattr(engine_mod, "_CURRENT_DATA_ENGINE", None):
+            return
+        _CURRENT_DATA_ENGINE = engine_mod._CURRENT_DATA_ENGINE
+
+        if _CURRENT_DATA_ENGINE and _CURRENT_DATA_ENGINE.current_session_id:
+            log_entry = self.format(record)
+            log_level = getattr(record, "log_level", None)
+            if log_level and hasattr(log_level, "value"):
+                level_name, _color, _symbol = log_level.value
+            else:
+                level_name = record.levelname
+
+            parent_id = CURRENT_TRACE_ID.get()
+
+            try:
+                self._log_queue.put_nowait(
+                    (
+                        record.name,
+                        record.getMessage(),
+                        level_name,
+                        log_entry,
+                        parent_id,
+                    )
+                )
+            except queue.Full as e:
+                raise RuntimeError(
+                    "DataEngine log queue is full - cannot accept more log"
+                    " entries without dropping records"
+                ) from e
+
+
+class ArtemisLogger:
+    def __init__(
+        self,
+        name: str,
+        console_level: str = "INFO",
+    ):
+        """Initialize the ARTEMIS logger.
+
+        Args:
+            name: Logger name (usually __name__)
+            console_level: Minimum level for console output
+        """
+        self.name = name
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(logging.DEBUG)
+
+        self.logger.handlers.clear()
+        self.logger.propagate = False
+
+        self._setup_console_handler(console_level)
+
+        # Add DataEngineHandler
+        data_engine_handler = DataEngineHandler()
+        data_engine_handler.setLevel(logging.DEBUG)
+        formatter = logging.Formatter(fmt="%(asctime)s | %(name)s | %(levelname)s | %(message)s")
+        data_engine_handler.setFormatter(formatter)
+        self.logger.addHandler(data_engine_handler)
+
+        if os.environ.get("ARTEMIS_MCP_SERVER") == "true":
+            traces_dir = os.environ.get("ARTEMIS_TRACES_PATH", "traces")
+            log_path = Path(traces_dir) / "mcp_server.log"
+            log_path.parent.mkdir(parents=True, exist_ok=True)
+            file_handler = logging.FileHandler(log_path, encoding="utf-8")
+            file_handler.setLevel(logging.DEBUG)
+            file_handler.setFormatter(formatter)
+            self.logger.addHandler(file_handler)
+
+    def _setup_console_handler(self, level: str):
+        console_handler = logging.StreamHandler(sys.stderr)
+        console_handler.setLevel(getattr(logging, level.upper()))
+
+        console_formatter = ColoredFormatter()
+        console_handler.setFormatter(console_formatter)
+
+        self.logger.addHandler(console_handler)
+
+    def debug(self, message: str, **kwargs):
+        self.logger.debug(message, extra={"log_level": LogLevel.DEBUG}, **kwargs)
+
+    def info(self, message: str, **kwargs):
+        self.logger.info(message, extra={"log_level": LogLevel.INFO}, **kwargs)
+
+    def success(self, message: str, **kwargs):
+        self.logger.info(message, extra={"log_level": LogLevel.SUCCESS}, **kwargs)
+
+    def warning(self, message: str, **kwargs):
+        self.logger.warning(message, extra={"log_level": LogLevel.WARNING}, **kwargs)
+
+    def error(self, message: str, **kwargs):
+        self.logger.error(message, extra={"log_level": LogLevel.ERROR}, **kwargs)
+
+    def critical(self, message: str, **kwargs):
+        self.logger.critical(message, extra={"log_level": LogLevel.CRITICAL}, **kwargs)
+
+    def header(self, message: str, **_kwargs):
+        separator = "=" * 60
+        colored_separator = f"{Fore.CYAN}{separator}{Style.RESET_ALL}"
+        colored_message = f"{Fore.CYAN}{message}{Style.RESET_ALL}"
+
+        print(colored_separator, file=sys.stderr)
+        print(colored_message, file=sys.stderr)
+        print(colored_separator, file=sys.stderr)
+        self.logger.info(f"\n{separator}\n{message}\n{separator}")
+
+
+class ColoredFormatter(logging.Formatter):
+    def format(self, record):
+        log_level = getattr(record, "log_level", LogLevel.INFO)
+        _level_name, color, symbol = log_level.value
+
+        colored_message = f"{color}{symbol} {record.getMessage()}{Style.RESET_ALL}"
+
+        return colored_message
+
+
+_loggers = {}
+
+
+def get_logger(
+    name: str,
+    console_level: str = "INFO",
+) -> ArtemisLogger:
+    """Get or create a logger instance.
+
+    Args:
+        name: Logger name (usually __name__)
+        log_file: Path to log file (defaults to logs/{name}.log)
+        console_level: Minimum level for console output
+        file_level: Minimum level for file output
+        enable_file_logging: Whether to enable file logging
+
+    Returns:
+        ArtemisLogger instance
+    """
+    if name not in _loggers:
+        _loggers[name] = ArtemisLogger(
+            name=name,
+            console_level=console_level,
+        )
+
+    return _loggers[name]