android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/mcp_server/notifiers/file.py

@@ -0,0 +1,69 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""File-based notification recorder."""
+
+import json
+import logging
+import os
+import time
+from typing import Any
+
+from artemis.runtime import trace_store
+from mcp_server.notifiers.base import BaseNotifier
+
+logger = logging.getLogger("mcp_server.notifiers.file")
+
+
+class FileNotifier(BaseNotifier):
+    """Notifier that records all notification events directly to the trace directory."""
+
+    @property
+    def name(self) -> str:
+        return "file"
+
+    def is_available(self) -> bool:
+        return True
+
+    def notify(
+        self,
+        conversation_id: str,
+        message: str,
+        title: str | None = None,
+        event_type: str = "completed",
+        payload: dict[str, Any] | None = None,
+    ) -> bool:
+        trace_id = (payload or {}).get("trace_id") or (payload or {}).get("session_id")
+        if not trace_id:
+            return False
+
+        try:
+            trace_dir = trace_store.get_trace_dir(trace_id)
+            os.makedirs(trace_dir, exist_ok=True)
+            log_file = os.path.join(trace_dir, "notifications.jsonl")
+
+            entry = {
+                "timestamp": time.time(),
+                "event_type": event_type,
+                "title": title,
+                "conversation_id": conversation_id,
+                "message": message,
+                "payload": payload or {},
+            }
+            with open(log_file, "a", encoding="utf-8") as f:
+                f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+            return True
+        except Exception as e:
+            logger.debug(f"File notification record failed: {e}")
+            return False

package/artemis-main/mcp_server/notifiers/script.py

@@ -0,0 +1,90 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Custom Script / Command Hook Notifier for universal IDE and editor integration."""
+
+import logging
+import os
+import subprocess
+from typing import Any
+
+from mcp_server.notifiers.base import BaseNotifier
+
+logger = logging.getLogger("mcp_server.notifiers.script")
+
+
+class ScriptNotifier(BaseNotifier):
+    """Notifier that executes a user-defined command or script when an event occurs.
+
+    This adapter enables universal integration with any custom IDE, editor (Neovim/Emacs),
+    or automation platform by allowing users to define ARTEMIS_NOTIFY_CMD or MCP_NOTIFY_COMMAND.
+    Placeholders like {title}, {message}, {conversation_id}, {event_type}, and {trace_id}
+    are automatically replaced before execution.
+    """
+
+    ENV_VARS = [
+        "ARTEMIS_NOTIFY_CMD",
+        "MCP_NOTIFY_COMMAND",
+    ]
+
+    @property
+    def name(self) -> str:
+        return "script"
+
+    def _get_command_template(self) -> str | None:
+        for var in self.ENV_VARS:
+            val = os.getenv(var)
+            if val and val.strip():
+                return val.strip()
+        return None
+
+    def is_available(self) -> bool:
+        return self._get_command_template() is not None
+
+    def notify(
+        self,
+        conversation_id: str,
+        message: str,
+        title: str | None = None,
+        event_type: str = "completed",
+        payload: dict[str, Any] | None = None,
+    ) -> bool:
+        cmd_template = self._get_command_template()
+        if not cmd_template:
+            return False
+
+        trace_id = (payload or {}).get("trace_id", "")
+        formatted_title = title or f"Artemis Task {event_type.capitalize()}"
+
+        # Replace template placeholders safely
+        try:
+            cmd = (
+                cmd_template.replace("{title}", str(formatted_title))
+                .replace("{message}", str(message))
+                .replace("{conversation_id}", str(conversation_id))
+                .replace("{event_type}", str(event_type))
+                .replace("{trace_id}", str(trace_id))
+            )
+            subprocess.run(
+                cmd,
+                shell=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=10,
+            )
+            logger.info(f"Custom script notification command executed: {cmd[:60]}...")
+            return True
+        except Exception as e:
+            logger.warning(f"Failed to execute custom script notification: {e}")
+            return False

package/artemis-main/mcp_server/notifiers/webhook.py

@@ -0,0 +1,87 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Webhook Notifier for OpenClaw, CI pipelines, and external gateways."""
+
+import json
+import logging
+import os
+import time
+from typing import Any
+import urllib.request
+
+from mcp_server.notifiers.base import BaseNotifier
+
+logger = logging.getLogger("mcp_server.notifiers.webhook")
+
+
+class WebhookNotifier(BaseNotifier):
+    """Notifier that posts execution events to HTTP/HTTPS webhooks (e.g. OpenClaw / CI / Slack)."""
+
+    ENV_VARS = [
+        "OPENCLAW_WEBHOOK_URL",
+        "MCP_NOTIFICATION_WEBHOOK",
+        "ARTEMIS_WEBHOOK_URL",
+    ]
+
+    @property
+    def name(self) -> str:
+        return "webhook"
+
+    def _get_webhook_url(self) -> str | None:
+        for var in self.ENV_VARS:
+            val = os.getenv(var)
+            if val and val.startswith(("http://", "https://")):
+                return val
+        return None
+
+    def is_available(self) -> bool:
+        return self._get_webhook_url() is not None
+
+    def notify(
+        self,
+        conversation_id: str,
+        message: str,
+        title: str | None = None,
+        event_type: str = "completed",
+        payload: dict[str, Any] | None = None,
+    ) -> bool:
+        url = self._get_webhook_url()
+        if not url:
+            return False
+
+        data = {
+            "event": event_type,
+            "title": title or f"Artemis Task {event_type.capitalize()}",
+            "conversation_id": conversation_id,
+            "message": message,
+            "timestamp": time.time(),
+            "payload": payload or {},
+        }
+
+        try:
+            req_data = json.dumps(data).encode("utf-8")
+            req = urllib.request.Request(
+                url,
+                data=req_data,
+                headers={"Content-Type": "application/json", "User-Agent": "Artemis-MCP/3.0"},
+                method="POST",
+            )
+            with urllib.request.urlopen(req, timeout=5) as response:
+                if response.status in (200, 201, 202, 204):
+                    logger.info(f"Webhook notification delivered to {url}.")
+                    return True
+        except Exception as e:
+            logger.warning(f"Failed to deliver webhook notification to {url}: {e}")
+        return False

package/artemis-main/mcp_server/rules.md

@@ -0,0 +1,68 @@
+# Mobile Testing Mindset (ARTEMIS Integration)
+
+Write executable tests based on device behavior verified with **ARTEMIS**. Before authoring a test, explore the target application and confirm its screens, transitions, and interactions. Use ARTEMIS device actions or ADB commands to investigate software and hardware behavior instead of assuming how an interaction works.
+
+### 1. The Runnable Code Principle & ARTEMIS Exploration
+- **When tasked with authoring tests**, deliver runnable test code with verified interactions and explicit wait conditions.
+- Before writing any test code, you must use the ARTEMIS MCP tools to interactively run and explore the target application. This allows you to discover the exact sequence of UI states, transitions, and required interactions.
+- Analyze the user's target testing framework to exploit its native capabilities and maximize test stability.
+- **Timing & Latency Management (Exploration vs. Execution)**:
+  - **Precise Timing in Final Code**: While ARTEMIS's AI exploration inherently involves model latency and is not strictly time-precise, you must bridge this gap in your final deliverables. Use ARTEMIS to discover and verify the interaction path, then implement exact, deterministic timing and wait conditions (`sleep`, explicit/implicit waits) in your authored test scripts, as local test execution runs without LLM overhead.
+  - **Compensating for Model Latency During Exploration**: When delegating exploratory tasks to ARTEMIS that involve waiting periods (e.g., waiting 30 seconds for a page load or timer), adjust the requested wait duration in your task description based on the selected model's natural step interval:
+    - **Flash Model**: The average processing interval between steps is roughly **5 seconds**. For a required 30-second delay, instruct the agent to wait for approximately **25 seconds**.
+    - **Pro Model**: The processing interval between turns is typically **~30 seconds** (due to multi-agent planning and verification). The natural pipeline delay often covers the required waiting time without adding long explicit delay commands.
+  - **Pragmatic Timing Judgment**: When a user request mentions performing an action for a specific duration (e.g., "stay on this screen for 2 minutes"), evaluate whether exact timing is functionally critical. Often, these durations are rough guidelines rather than strict test constraints—exercise flexibility and pragmatic engineering judgment to achieve the verification goal efficiently.
+
+
+### 2. ARTEMIS Closed-Loop Architecture Mastery
+Deeply understand and select between ARTEMIS's dual execution models (**ARTEMIS Flash** and **ARTEMIS Pro**) based on the task scenario, and master their corresponding workflows:
+
+- **ARTEMIS Flash (Fast / Reactive Model)**:
+  - **Applicable Scenarios**: Designed for simple, highly deterministic, lightweight UI operations or direct automation workflows. There is no step cap by default (history is compressed rather than truncated), so task length alone is not a reason to choose Pro; choose Pro when the task needs a persistent plan, verified checkpoints, notes or a written report, ADB / log diagnostics, or multi-branch exploration.
+  - **Execution Mechanism (Reactive Loop)**: Does not enter the LangGraph multi-node orchestration: there is no Planner, no pre-execution safety net, and no Checker. `FlashRunner` receives the user's goal, the latest UI element list, and the live screenshot, performs "Observe-Think-Act" with a single LLM, and calls action tools until it reports the task status. It chains taps into one `click_sequence` to catch transient UI (auto-fading control bars, toasts) before they expire, and it can call `ask_explorer` for element grounding, `search_history` / `replay_steps` / `get_step_screenshot` to look up compressed earlier steps, and `video_analyzer` over the session recording. History lives in the same session transcript ledger as Pro (session-relative `T+mm:ss` clock, screenshots folded into visual summaries, older steps chunked into eras).
+
+- **ARTEMIS Pro (Deep / Multi-Agent Graph-Driven Closed-Loop Model)**:
+  - **Applicable Scenarios**: Designed for long-range, highly complex, dynamic multi-branch tasks; continuous monitoring / polling loops; and tasks requiring deep system diagnostics (ADB, logs, video), verified checkpoints, or a detailed written report.
+  - **Execution Mechanism (Plan-Execute-Verify-Summarize Closed-Loop)**:
+    - **Planner**: Deconstructs complex, high-level testing goals into a living Markdown task plan with milestones and `verify` / `assert` check items. Later milestone edits get an advisory review (a hint plus reason back to the Operator; the plan is never rolled back).
+    - **Operator**: Consumes the plan, analyzes the current screen state (screenshots plus the UI element list, with the Explorer for grounding at the `explorer_mode` perception depth: `flash` 1-shot, `pro` 3-turn, `ultra` deep zoom), keeps notes, and executes precise device interactions; it can also recall compressed history, analyze the session recording, and run ADB commands as a supplement to UI actions.
+    - **Safety Net & Execution Incidents**: A single turn-ending action is vetted by a pre-execution safety net (XML-first, pixel fallback). A multi-action turn is a **fast-action burst** that fires back to back without the safety net, which is how the Operator defeats turn latency on transient UI. When an action is intercepted or fails, the system opens an **execution incident** that persists in the Operator's context (with the consecutive-failure count shown as a fact) until a later action executes successfully; recovery is the Operator's own decision and there is no separate repair agent.
+    - **Checker (Verification)**: A read-only verifier with the same observation tools as the Operator audits plan-declared checkpoints and performs an exit final review against the original goal; `verification_level` selects the depth (`off`, `final` = exit review only and the default, `checkpoints` = every checkpoint plus exit review, `strict` = checkpoints with a larger repair budget where a failed assert halts the run).
+    - **Outputter (Optional)**: Synthesizes the entire execution trace into a human-readable report detailing every action step and visual result.
+
+### 3. Device & Environment Constraints & Multi-Device Management
+- **Device Selection & Multi-Device Execution**: ARTEMIS supports multi-device execution and per-device concurrency. You can control device targeting via two modes:
+  - **Direct Device Specification**: Explicitly provide the target phone's serial number via `device_serial` to `mobile_run_task` or `mobile_get_device_state`. Tasks targeting distinct devices run concurrently without blocking each other.
+  - **Automatic Device Selection**: When `device_serial` is omitted or set to `None`, ARTEMIS automatically selects an available connected device or allocates an idle device from the device pool.
+- **Prioritize User Choice for Device Selection**:
+  - When multiple connected devices or emulators are detected, or whenever device selection is ambiguous, **YOU MUST PRIORITIZE ASKING THE USER** to select or confirm their preferred device serial before launching a task.
+- **Device Diagnosis with `adb devices`**:
+  - Use `adb devices` (or `adb devices -l`) via bash command execution to diagnose attached hardware, inspect connection status (`device`, `unauthorized`, `offline`), and retrieve device serials and models whenever preparing tasks or troubleshooting device issues.
+- **ADB & File Transfer**: While ARTEMIS excels at device automation, it operates within the device boundary. To extract diagnostic files, logs, or test artifacts from the mobile device to the host PC for further analysis, you must manually execute appropriate `adb` commands (e.g., `adb -s <serial> pull ...`).
+- **Hardware Prerequisites**: Running ARTEMIS requires at least one physically connected, fully authorized Android device (e.g., a Pixel phone) or an active emulator.
+- **Per-Device Mutual Exclusion**: ARTEMIS manages per-device execution mutexes (`DeviceExecutionLock`). A device can only execute a single task at a time (FIFO queue), while different devices can execute tasks in parallel.
+
+### 4. Robust Test Code Design (The "Dynamic-First, Coordinate-Fallback" Philosophy)
+*If your task involves authoring test code, you must adhere to the following design principles for maximum reliability:*
+- **Adapt to Framework Capabilities**: You must first assess what locating mechanisms the user's test framework supports (e.g., resource IDs, XPath, text matching, OCR, image template matching, or absolute/relative coordinates).
+- **The Core Principle: Dynamic-First, Coordinate-Fallback**:
+  - Wherever supported by the framework, **always prioritize dynamic element locating** (using IDs, text, OCR, etc.) to ensure the test code can withstand UI layout drifts and resolution changes.
+  - Use **absolute coordinates as a reliable fallback** to guarantee execution success when dynamic locators fail or are unavailable.
+- **Implementing Resilient Locating Patterns**:
+  - **Dual-Capable Frameworks**: If the framework supports both dynamic and coordinate-based locating, implement the **Try-Catch Fallback Pattern**:
+    1. **Try**: Attempt to interact with the element using dynamic locators (IDs, OCR, text) for maximum resilience against UI changes.
+    2. **Catch**: If the dynamic attempt fails, fall back to the precise absolute coordinates verified during your ARTEMIS exploration.
+  - **Coordinate-Only Frameworks**: If the framework only supports coordinates, ensure the coordinates are well-documented, and where possible, parameterized or made relative to screen boundaries to mitigate resolution differences.
+  - **Dynamic-Only Frameworks**: If the framework does not support coordinate-based clicks, focus entirely on generating highly robust dynamic locators, leveraging ARTEMIS's element descriptions and XML tree analysis.
+
+### 5. Environment Self-Diagnosis (`mobile_diagnose`)
+- **Diagnose before guessing**: Whenever an ARTEMIS tool returns an error, a task fails to start or is rejected, no device is found, or the user reports that ARTEMIS "doesn't work" in the IDE, call `mobile_diagnose` first. Do not troubleshoot by re-running `mobile_run_task` on trial and error, and do not ask the user to run `adb` commands by hand before you have the report.
+- **If `mobile_diagnose` is not in your tool list at all**, the MCP server itself failed to start: check the IDE's MCP server log and run `uv run artemis doctor` in the project directory (the CLI shares the same checks).
+- **Follow `next_steps` in order**: The report lists fixes in dependency order (Python runtime → config → MCP host → LLM credentials → device → optional video toolchain). `Run:` lines are single, local, non-destructive shell commands (one command per line, no `&&`) you may execute yourself (ask before installing software with winget / brew / apt). `Guidance:` lines describe something only the user can do (unlock the phone, tap "Allow" on the USB-debugging prompt, plug in a cable, free a port, tell you which IDE they use) — relay them verbatim. `Docs:` lines are references.
+- **Let ARTEMIS self-heal first**: Call `mobile_diagnose(attempt_fix=true)` before asking for manual ADB intervention; it regenerates corrupted ADB RSA keys, restarts the ADB server when that is safe (no device ready, no task holding a device), and removes stale device locks / queue tickets left behind by crashed runners.
+- **Deeper checks on demand**: `verify_credentials=true` validates the configured API keys live (~12s; use when a key exists but tasks fail with auth / quota errors; result in `credentials`). `probe_device=true` runs a true end-to-end check — screenshot plus UI hierarchy through UIAutomator (~20s; use when the device shows connected but observation or tasks fail; result in `device_probe`).
+- **Emulators**: `launch_avd="<name>"` starts an installed Android Virtual Device in the background via ARTEMIS's emulator manager and returns immediately; boot takes 1-3 minutes, so re-run `mobile_diagnose` after ~60s. Never run `emulator -avd` in the shell yourself — it is a foreground process and hangs the shell.
+- **Busy devices**: `tasks: {active, queued}` lists tasks currently holding or waiting for devices. A busy device means wait, or stop the task with `mobile_manage_task(action="stop", trace_id=...)`.
+- **Credentials never go through the chat**: If the LLM key is missing, ask the user to add it to the `.env` file at the path the report shows (`host.env_file`) or to the MCP server's `env` block, then to restart the MCP server. `artemis init` is interactive and cannot be driven from a tool call.
+- **Restart after configuration changes**: Settings are read when the server starts. After editing `.env`, `config/artemis.jsonc`, or the MCP client configuration, reload the MCP server in the IDE before calling `mobile_diagnose` again; otherwise it reports the old state.
+- **Failed tasks**: `logs.last_failed_task` gives the trace id, its `stderr_log`, and `recent_errors` (the error lines from that stderr, so you need not open the file); read them before re-running the task. Repeat `mobile_diagnose` until `verdict` is `ready` or `degraded` before delegating tasks again (`degraded` means tasks can run but something optional is off: video tools, the OCR key, a secondary credential, or a host warning such as an interpreter mismatch or a squatted daemon port; relay its `[OPTIONAL]` steps to the user, do not loop on them).

package/artemis-main/mcp_server/server.py

@@ -0,0 +1,106 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal Model Context Protocol (MCP) Server for ARTEMIS."""
+
+import datetime
+import os
+import sys
+
+# Ensure repository root is in sys.path
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+# Debug logging and stderr tee to capture crashes without breaking MCP stdio JSON-RPC
+_LOG_MAX_BYTES = 5 * 1024 * 1024
+
+
+def _rotate_log(path: str, max_bytes: int = _LOG_MAX_BYTES) -> None:
+    """Rotates the log to <path>.old once it exceeds max_bytes, so long-lived
+    server processes cannot grow the debug logs without bound."""
+    try:
+        if os.path.exists(path) and os.path.getsize(path) > max_bytes:
+            old_path = path + ".old"
+            if os.path.exists(old_path):
+                os.remove(old_path)
+            os.replace(path, old_path)
+    except OSError:
+        # Best-effort rotation; a locked or vanished log file is tolerable.
+        pass
+
+
+try:
+    _mcp_log_dir = os.path.join(PROJECT_ROOT, "scratch")
+    os.makedirs(_mcp_log_dir, exist_ok=True)
+
+    _launch_log = os.path.join(_mcp_log_dir, "mcp_launch_debug.log")
+    _rotate_log(_launch_log)
+    with open(_launch_log, "a", encoding="utf-8") as _f:
+        _f.write(
+            f"[{datetime.datetime.now()}] Artemis MCP Server launched! sys.argv: {sys.argv}, CWD: {os.getcwd()}\n"
+        )
+
+    class StderrTee:
+        def __init__(self, original, log_path, max_bytes=_LOG_MAX_BYTES):
+            self.original = original
+            self.log_path = log_path
+            self.max_bytes = max_bytes
+            _rotate_log(log_path, max_bytes)
+            self.log_file = open(log_path, "a", encoding="utf-8")
+
+        def write(self, data):
+            self.original.write(data)
+            self.log_file.write(data)
+            self.log_file.flush()
+            if self.log_file.tell() > self.max_bytes:
+                self.log_file.close()
+                _rotate_log(self.log_path, self.max_bytes)
+                self.log_file = open(self.log_path, "a", encoding="utf-8")
+
+        def flush(self):
+            self.original.flush()
+            self.log_file.flush()
+
+    sys.stderr = StderrTee(sys.stderr, os.path.join(_mcp_log_dir, "mcp_stderr.log"))
+except OSError:
+    # Debug logging and the stderr tee are optional diagnostics; they must
+    # never prevent the MCP stdio server from starting.
+    pass
+
+# 1. Import shared FastMCP instance
+from mcp_server.base import mcp
+
+# 2. Import tools to register them
+import mcp_server.tools  # noqa: F401
+from artemis.runtime import shutdown_awake_service, start_awake_service
+
+
+import threading
+
+
+def main(transport: str = "stdio", host: str = "127.0.0.1", port: int = 8001):
+    """Main entrypoint to run the Artemis Mobile Agent MCP server."""
+    threading.Thread(target=start_awake_service, daemon=True, name="artemis-awake-init").start()
+    try:
+        if transport.lower() == "sse":
+            mcp.run(transport="sse", host=host, port=port)
+        else:
+            mcp.run(transport="stdio")
+    finally:
+        shutdown_awake_service()
+
+
+if __name__ == "__main__":
+    main()

package/artemis-main/mcp_server/tools/__init__.py

@@ -0,0 +1,29 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""MCP Tools package for ARTEMIS."""
+
+from mcp_server.tools.device_state import mobile_get_device_state
+from mcp_server.tools.diagnose import mobile_diagnose
+from mcp_server.tools.inspect_trace import mobile_inspect_trace
+from mcp_server.tools.task_manager import mobile_manage_task
+from mcp_server.tools.task_runner import mobile_run_task
+
+__all__ = [
+    "mobile_run_task",
+    "mobile_manage_task",
+    "mobile_get_device_state",
+    "mobile_inspect_trace",
+    "mobile_diagnose",
+]

package/artemis-main/mcp_server/tools/device_state.py

@@ -0,0 +1,107 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""MCP Tool: mobile_get_device_state."""
+
+import base64
+import os
+
+from mcp_server.base import mcp
+from artemis.mcp.adb_server import _get_controller
+from mcp_server.utils import env_utils
+from artemis.utils.ocr_xml_fusion import (
+    fuse_ocr_with_xml,
+    _detect_status_bar_height,
+    _crop_image_remove_status_bar,
+    _map_coordinates_back,
+)
+from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
+from artemis.utils.visualization import format_minimal_list_with_elements
+
+
+@mcp.tool()
+async def mobile_get_device_state(view_type: str, device_serial: str | None = None) -> str:
+    """Real-time mobile device state observer (for debugging and validation).
+
+    Retrieves a real-time screenshot or a simplified UI element tree from the
+    target device — useful for inspecting device status or tracking a
+    subagent's progress.
+
+    Args:
+        view_type: Observation type:
+          - "screenshot": captures the screen, saves it to the workspace, and
+            returns the image's local file URI.
+          - "hierarchy": returns the simplified text-labeled element list —
+            exactly what the automation subagent sees when making decisions.
+        device_serial: Optional device serial (e.g. "emulator-5554") to inspect
+          a specific device; omitted → the default connected device. With
+          several devices attached, confirm the target with the user
+          (`adb devices -l` lists serials).
+    """
+    try:
+        controller = _get_controller(device_serial=device_serial)
+        device_width = controller.ctx.device.device_width
+        device_height = controller.ctx.device.device_height
+    except Exception as e:
+        return f"Error: Failed to initialize/lock Android device controller: {e}"
+
+    try:
+        device_data = await controller.get_screen_data()
+        latest_screenshot_b64 = device_data.base64
+        xml_hierarchy = device_data.elements
+
+        project_root = env_utils.get_project_root()
+
+        if view_type == "screenshot":
+            screenshot_bytes = base64.b64decode(latest_screenshot_b64)
+            device_id = controller.ctx.device.device_id
+            safe_device_id = "".join(
+                [c if c.isalnum() or c in ("-", "_") else "_" for c in device_id]
+            )
+            screenshot_filename = f"live_screenshot_{safe_device_id}.jpg"
+            screenshot_path = os.path.join(project_root, screenshot_filename)
+
+            with open(screenshot_path, "wb") as f:
+                f.write(screenshot_bytes)
+
+            return f"file://{screenshot_path}"
+
+        elif view_type == "hierarchy":
+            ocr_results = []
+            if is_ocr_configured():
+                try:
+                    screen_height = device_data.height
+                    status_bar_height = _detect_status_bar_height(xml_hierarchy, screen_height)
+                    if status_bar_height > 0:
+                        cropped_b64, _, _ = _crop_image_remove_status_bar(
+                            latest_screenshot_b64, status_bar_height
+                        )
+                        raw_ocr_results = await perform_ocr(cropped_b64)
+                        ocr_results = _map_coordinates_back(raw_ocr_results, status_bar_height)
+                    else:
+                        ocr_results = await perform_ocr(latest_screenshot_b64)
+                except Exception:
+                    ocr_results = []
+
+            fused_xml = fuse_ocr_with_xml(xml_hierarchy, ocr_results)
+            hierarchy_text, elements, labels = format_minimal_list_with_elements(
+                fused_xml, width=device_width, height=device_height
+            )
+            return hierarchy_text
+
+        else:
+            return f"Error: Invalid view_type '{view_type}'. Supported types are 'screenshot' and 'hierarchy'."
+
+    except Exception as e:
+        return f"Error: An unexpected error occurred while communicating with the device: {e}"