android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/mcp/action_manifest.py

@@ -0,0 +1,320 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contract layer between actuator backends and the agent-facing tool surface.
+
+This module is the single place that declares *what a backend must provide*. It sits
+between the actuator implementations (``artemis/mcp/actuators/``) and the agents that
+consume tools, so that a backend -- ADB today, a robot arm tomorrow -- can omit tools
+it cannot perform, and add tools nobody anticipated, without any agent knowing.
+
+Two rules govern the split between required and optional:
+
+1. A tool is REQUIRED only when a prompt depends on it *structurally*, i.e. the prompt
+   cannot be assembled around its absence. Today that is ``click_sequence`` alone: the
+   Flash runner prompt's "Time-Sensitive Tasks" teaching is built around "atomic
+   chained execution to defeat turn latency", which *is* ``click_sequence``. (The Pro
+   Operator needs no such tool: a multi-action turn is executed by the Validator as an
+   unvetted fast-action burst.)
+2. Everything else is OPTIONAL, because every other prompt reference lives in a tool
+   enumeration slot or a self-contained instruction block that the assembly layer
+   (``artemis/agents/prompt_assembly.py``) renders conditionally. An optional tool that
+   a backend does not implement disappears from the declarations *and* from the prompt,
+   so it costs the model nothing.
+
+``tests/unit/mcp/test_action_manifest.py`` re-derives rule 2 by scanning the prompt
+sources, so adding a tool name to a prompt without classifying it here fails CI.
+"""
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+__all__ = [
+    "ActuatorContractError",
+    "AGENT_ROLES",
+    "ALL_KNOWN_TOOLS",
+    "BACKEND_INDEPENDENT_TOOLS",
+    "DEVICE_ACTIONS",
+    "ExtensionTool",
+    "INTERNAL_ACTIONS",
+    "MIN_EXTENSION_DESCRIPTION_LENGTH",
+    "OPTIONAL_ACTIONS",
+    "REQUIRED_ACTIONS",
+    "available_device_actions",
+    "filter_declarations",
+    "validate_actuator",
+]
+
+
+# --- Tool classification -------------------------------------------------------------
+
+#: Device actions a backend MUST implement. See rule 1 in the module docstring.
+#:
+#: ``click_sequence`` implies the ability to tap, so a backend providing it can
+#: trivially provide ``click`` as well; ``click`` is nonetheless optional because no
+#: prompt depends on it structurally once the enumeration slots are assembled.
+REQUIRED_ACTIONS: frozenset[str] = frozenset(
+    {
+        "click_sequence",
+    }
+)
+
+#: Device actions a backend MAY implement. Absent ones vanish from both the tool
+#: declarations and the rendered prompts.
+OPTIONAL_ACTIONS: frozenset[str] = frozenset(
+    {
+        "click",
+        "long_press",
+        "input_text",
+        "swipe",
+        "press_key",
+        "manage_app",
+        "wait_for_delay",
+        # The four below appear in no prompt at all. `wait_for_text` corroborates the
+        # classification: the former legacy executor dispatched it, but it has never had
+        # a ToolDeclaration, i.e. it has always been unreachable by any LLM.
+        "wait_for_text",
+        "open_link",
+        "erase_one_char",
+        "focus_and_clear_text",
+    }
+)
+
+#: Used by the client-side adapter only; never declared to any LLM, so the
+#: "no cognitive load" rule does not apply. The adapter cannot function without them,
+#: so they are validated as strictly as REQUIRED_ACTIONS.
+INTERNAL_ACTIONS: frozenset[str] = frozenset(
+    {
+        "observe_screen",
+        "take_screenshot",
+        "get_ui_hierarchy",
+    }
+)
+
+#: Tools that are not device control at all -- notes, sub-agents, diagnostics, and the
+#: loop-termination sentinels. They are unaffected by which actuator is installed.
+#:
+#: ``run_adb_command`` and ``manage_app`` ride the *system channel* rather than physical
+#: actuation: a robot arm driving a handset still has ADB attached, so ``am start``
+#: suffices and no icon-hunting is required. (``manage_app`` is nonetheless classified
+#: as an optional device action above, since it is declared through the same path.)
+BACKEND_INDEPENDENT_TOOLS: frozenset[str] = frozenset(
+    {
+        "read_note",
+        "list_notes",
+        "save_note",
+        "update_note",
+        "append_note",
+        "ask_explorer",
+        "ask_diagnoser",
+        "ask_committee",
+        "video_analyzer",
+        "run_adb_command",
+        "manage_task",
+        "analyze_task_output",
+        "search_history",
+        "replay_steps",
+        "get_step_screenshot",
+        "report_task_status",
+    }
+)
+
+#: Everything an actuator may legitimately claim in ``capabilities()``.
+DEVICE_ACTIONS: frozenset[str] = REQUIRED_ACTIONS | OPTIONAL_ACTIONS | INTERNAL_ACTIONS
+
+#: Every name the manifest knows about. Extension tools must not collide with these.
+ALL_KNOWN_TOOLS: frozenset[str] = DEVICE_ACTIONS | BACKEND_INDEPENDENT_TOOLS
+
+#: Agent roles an extension tool may target.
+AGENT_ROLES: frozenset[str] = frozenset(
+    {
+        "operator",
+        "flash",
+        "validator",
+    }
+)
+
+#: Extension descriptions carry the entire usage contract (see ``ExtensionTool``), so a
+#: one-word description is a bug rather than a terse style choice.
+MIN_EXTENSION_DESCRIPTION_LENGTH = 40
+
+
+class ActuatorContractError(RuntimeError):
+    """Raised when an actuator does not satisfy the manifest contract.
+
+    Always raised at session start rather than mid-run, so a misconfigured backend
+    fails before the agent has taken any action on the device.
+    """
+
+
+@dataclass(frozen=True)
+class ExtensionTool:
+    """A tool contributed by a backend that the manifest does not know about.
+
+    Extension tools are invisible to the prompt layer by construction: prompts are
+    static text that cannot anticipate future backends, so an extension gets **no**
+    prompt-level teaching. Its ``description`` must therefore be self-contained --
+    stating when to reach for it and what it does -- or the model will never use it
+    correctly. :func:`validate_actuator` enforces a length floor as a crude guard.
+    """
+
+    name: str
+    description: str
+    input_schema: dict[str, Any]
+    handler: Callable[..., Awaitable[Any]]
+    targets: frozenset[str] = field(default=AGENT_ROLES)
+
+
+# --- Contract validation -------------------------------------------------------------
+
+
+def validate_actuator(actuator: Any) -> None:
+    """Checks an actuator against the manifest, raising on any violation.
+
+    Called once from ``get_action_session`` before the first device interaction.
+    """
+    caps = frozenset(actuator.capabilities())
+
+    missing_required = REQUIRED_ACTIONS - caps
+    if missing_required:
+        raise ActuatorContractError(
+            f"{type(actuator).__name__} does not implement required action(s): "
+            f"{sorted(missing_required)}. These are structurally depended on by prompts "
+            f"and cannot be assembled away."
+        )
+
+    missing_internal = INTERNAL_ACTIONS - caps
+    if missing_internal:
+        raise ActuatorContractError(
+            f"{type(actuator).__name__} does not implement internal action(s): "
+            f"{sorted(missing_internal)}. The client-side adapter cannot observe the "
+            f"device without them."
+        )
+
+    unknown = caps - DEVICE_ACTIONS
+    if unknown:
+        raise ActuatorContractError(
+            f"{type(actuator).__name__} claims capabilities unknown to the manifest: "
+            f"{sorted(unknown)}. Backend-specific tools belong in extensions(), not "
+            f"capabilities()."
+        )
+
+    _validate_extensions(actuator)
+
+
+def _validate_extensions(actuator: Any) -> None:
+    """Validates extension tools for name collisions and self-sufficient descriptions."""
+    extensions = list(actuator.extensions())
+    seen: set[str] = set()
+
+    for ext in extensions:
+        if ext.name in ALL_KNOWN_TOOLS:
+            raise ActuatorContractError(
+                f"Extension tool '{ext.name}' collides with a manifest tool of the same "
+                f"name. Extension names must be unique across the whole tool surface."
+            )
+        if ext.name in seen:
+            raise ActuatorContractError(
+                f"{type(actuator).__name__} registers extension '{ext.name}' more than once."
+            )
+        seen.add(ext.name)
+
+        unknown_targets = frozenset(ext.targets) - AGENT_ROLES
+        if unknown_targets:
+            raise ActuatorContractError(
+                f"Extension tool '{ext.name}' targets unknown agent role(s): "
+                f"{sorted(unknown_targets)}. Valid roles: {sorted(AGENT_ROLES)}."
+            )
+
+        if len(ext.description.strip()) < MIN_EXTENSION_DESCRIPTION_LENGTH:
+            raise ActuatorContractError(
+                f"Extension tool '{ext.name}' has a description of "
+                f"{len(ext.description.strip())} characters. Extensions receive no "
+                f"prompt-level guidance, so the description must state when to use the "
+                f"tool and what it does (minimum "
+                f"{MIN_EXTENSION_DESCRIPTION_LENGTH} characters)."
+            )
+
+
+# --- Surface derivation --------------------------------------------------------------
+
+
+def available_device_actions(actuator: Any) -> frozenset[str]:
+    """Returns the LLM-facing device actions this actuator supports.
+
+    Internal actions are excluded: they are adapter plumbing and never reach a model.
+    """
+    caps = frozenset(actuator.capabilities())
+    return (REQUIRED_ACTIONS | OPTIONAL_ACTIONS) & caps
+
+
+def filter_declarations(
+    declarations: list[Any],
+    actuator: Any,
+    agent: str,
+) -> list[Any]:
+    """Filters a declaration list down to what ``actuator`` actually provides.
+
+    Declarations for known tools keep coming from the hand-tuned ``ToolDeclaration``
+    constants -- this layer only removes entries, it never rewrites descriptions.
+    Extension declarations are appended from the actuator itself, since the manifest
+    cannot know them in advance.
+
+    Args:
+        declarations: Declarations the agent would use if every tool existed.
+        actuator: The installed backend.
+        agent: One of :data:`AGENT_ROLES`; selects which extensions apply.
+
+    Returns:
+        Declarations in their original order, with unavailable device actions dropped
+        and this agent's extension tools appended.
+    """
+    if agent not in AGENT_ROLES:
+        raise ValueError(f"Unknown agent role '{agent}'. Valid roles: {sorted(AGENT_ROLES)}.")
+
+    available = available_device_actions(actuator)
+
+    kept = [
+        decl
+        for decl in declarations
+        # Only device actions are gated. Backend-independent tools and anything the
+        # manifest does not classify (e.g. an agent's own bespoke tool) pass through.
+        if _declaration_name(decl) not in DEVICE_ACTIONS or _declaration_name(decl) in available
+    ]
+
+    for ext in actuator.extensions():
+        if agent in ext.targets:
+            kept.append(_extension_declaration(ext))
+
+    return kept
+
+
+def _declaration_name(declaration: Any) -> str:
+    """Reads a tool name off either a ToolDeclaration mapping or a LangChain tool."""
+    name = getattr(declaration, "name", None)
+    if name is None and isinstance(declaration, dict):
+        name = declaration.get("name")
+    return name or ""
+
+
+def _extension_declaration(ext: ExtensionTool) -> Any:
+    """Renders an ExtensionTool into the same ToolDeclaration shape as known tools."""
+    from artemis.core.tool_declaration import ToolDeclaration
+
+    return ToolDeclaration(
+        name=ext.name,
+        description=ext.description,
+        parameters=ext.input_schema,
+    )

package/artemis-main/artemis/mcp/action_names.py

@@ -0,0 +1,172 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The one place where the historical verb split is resolved.
+
+Canonical tool names are the agent-facing ones (``click``, ``long_press``,
+``input_text``, ``manage_app``, ...) because those are what the prompts teach and the
+prompts are the contract. The Operator/Validator-internal vocabulary
+(``tap``, ``long_press_on``, ``focus_and_input_text``, ``launch_app``/``stop_app``,
+``back``) is translated here and nowhere else.
+
+``to_canonical_call`` converts a Validator action item (the dicts Operator writes into
+``state.structured_decisions``) into a canonical ``(name, kwargs)`` pair for the
+actuator/session layer, using the ``normalized_coordinates`` the Operator already
+computed so the canonical surface never needs a pixel escape hatch.
+"""
+
+from typing import Any
+
+__all__ = ["OPERATOR_ACTION_TO_CANONICAL", "to_canonical_call"]
+
+#: Operator-internal action verb -> canonical tool name.
+OPERATOR_ACTION_TO_CANONICAL: dict[str, str] = {
+    "tap": "click",
+    "long_press_on": "long_press",
+    "swipe": "swipe",
+    "focus_and_input_text": "input_text",
+    "focus_and_clear_text": "focus_and_clear_text",
+    "erase_one_char": "erase_one_char",
+    "press_key": "press_key",
+    "back": "press_key",
+    "launch_app": "manage_app",
+    "stop_app": "manage_app",
+    "open_link": "open_link",
+    "wait_for_delay": "wait_for_delay",
+}
+
+
+def _norm_pair(
+    action_item: dict[str, Any], dims: tuple[int, int] | None = None
+) -> tuple[int, int] | None:
+    """Reads the Operator-computed normalized [nx, ny] off an action item.
+
+    Falls back to converting the pixel ``coordinates`` when the item predates the
+    ``normalized_coordinates`` field and screen ``dims`` are provided.
+    """
+    coords = action_item.get("normalized_coordinates")
+    if isinstance(coords, (list, tuple)) and len(coords) >= 2:
+        return int(coords[0]), int(coords[1])
+
+    pixel = action_item.get("coordinates")
+    if dims and isinstance(pixel, (list, tuple)) and len(pixel) >= 2:
+        width, height = dims
+        nx = int(max(0, min(1000, round(int(pixel[0]) * 1000 / max(1, width)))))
+        ny = int(max(0, min(1000, round(int(pixel[1]) * 1000 / max(1, height)))))
+        return nx, ny
+    return None
+
+
+def to_canonical_call(
+    action_item: dict[str, Any], dims: tuple[int, int] | None = None
+) -> tuple[str, dict[str, Any]]:
+    """Translates a Validator action item into a canonical (tool, kwargs) call.
+
+    Args:
+        action_item: One entry of the Operator's structured decisions.
+        dims: Optional (width, height) for pixel->normalized fallback on legacy items
+            lacking ``normalized_coordinates``.
+
+    Raises:
+        ValueError: for an unknown action verb or an item missing the coordinates the
+            canonical call requires.
+    """
+    verb = str(action_item.get("action", ""))
+    canonical = OPERATOR_ACTION_TO_CANONICAL.get(verb)
+    if canonical is None:
+        raise ValueError(f"Unsupported action: {verb}")
+
+    if verb == "tap":
+        pair = _norm_pair(action_item, dims)
+        if pair is None:
+            raise ValueError("Invalid coords")
+        return "click", {
+            "target": [pair[0], pair[1]],
+            "times": action_item.get("times") or action_item.get("click_times") or 1,
+            "delay_ms": action_item.get("delay_ms") or action_item.get("delay") or 100,
+        }
+
+    if verb == "long_press_on":
+        pair = _norm_pair(action_item, dims)
+        if pair is None:
+            raise ValueError("Invalid coords")
+        return "long_press", {
+            "target": [pair[0], pair[1]],
+            "duration_ms": action_item.get("duration", 1000),
+        }
+
+    if verb == "swipe":
+        coords = action_item.get("normalized_coordinates")
+        if not (isinstance(coords, (list, tuple)) and len(coords) == 4):
+            pixel = action_item.get("coordinates")
+            if dims and isinstance(pixel, (list, tuple)) and len(pixel) == 4:
+                width, height = dims
+                coords = [
+                    int(max(0, min(1000, round(int(pixel[0]) * 1000 / max(1, width))))),
+                    int(max(0, min(1000, round(int(pixel[1]) * 1000 / max(1, height))))),
+                    int(max(0, min(1000, round(int(pixel[2]) * 1000 / max(1, width))))),
+                    int(max(0, min(1000, round(int(pixel[3]) * 1000 / max(1, height))))),
+                ]
+            else:
+                raise ValueError("Invalid coords")
+        return "swipe", {
+            "start": [int(coords[0]), int(coords[1])],
+            "end": [int(coords[2]), int(coords[3])],
+            "duration_ms": action_item.get("duration", 400),
+        }
+
+    if verb == "focus_and_input_text":
+        pair = _norm_pair(action_item, dims)
+        if pair is None:
+            raise ValueError("Invalid coords")
+        return "input_text", {
+            "text": action_item.get("text", ""),
+            "target": [pair[0], pair[1]],
+            "clear_exist": bool(action_item.get("clear_before_input", False)),
+        }
+
+    if verb == "focus_and_clear_text":
+        pair = _norm_pair(action_item, dims)
+        if pair is None:
+            raise ValueError("Invalid coords")
+        return "focus_and_clear_text", {"target": [pair[0], pair[1]]}
+
+    if verb == "erase_one_char":
+        return "erase_one_char", {}
+
+    if verb == "press_key":
+        keycode = str(action_item.get("keycode", ""))
+        # Operator emits Android-style KEYCODE_* names; the canonical tool takes the
+        # bare key word for the common keys. Anything else keeps its original spelling
+        # so the driver can forward it verbatim (arbitrary KEYCODE_* / numeric codes).
+        bare = keycode.removeprefix("KEYCODE_").lower() if keycode else ""
+        known = {"home", "back", "enter", "delete", "tab", "search", "menu", "app_switch"}
+        return "press_key", {"key": bare if bare in known else keycode}
+
+    if verb == "back":
+        return "press_key", {"key": "back"}
+
+    if verb in ("launch_app", "stop_app"):
+        return "manage_app", {
+            "action": "launch" if verb == "launch_app" else "stop",
+            "app_name": action_item.get("app_name", ""),
+        }
+
+    if verb == "open_link":
+        return "open_link", {"url": action_item.get("url", "")}
+
+    if verb == "wait_for_delay":
+        return "wait_for_delay", {"time_in_ms": action_item.get("time_in_ms", 0)}
+
+    raise ValueError(f"Unsupported action: {verb}")

package/artemis-main/artemis/mcp/action_server.py

@@ -0,0 +1,202 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The unified device action MCP server.
+
+``build_action_server`` turns an actuator into a ``FastMCP`` server that registers
+**only** the tools the actuator implements, plus its extension tools -- so
+``list_tools()`` is clean by construction and an absent optional tool is invisible to
+every consumer.
+
+Return protocol: every device action returns a ``CallToolResult`` whose
+``structuredContent`` is an ``ActionResult`` -- ``ok``/``code`` carry the real status
+(killing the historical ``"Error" in text`` sniffing) while ``content`` keeps the
+historical human-readable message. ``isError`` is reserved for protocol-level
+failures; a device refusing an action is ``isError=False, ok=False`` so it reaches the
+model as an observation, not an exception.
+
+Exceptions raised by the actuator are wrapped here with the historical
+``"Error during X: {e}"`` wording the agents' transcripts already contain.
+"""
+
+import asyncio
+from typing import Annotated, Any
+
+from mcp.server.fastmcp import FastMCP
+from mcp.types import CallToolResult, ImageContent, TextContent
+
+from artemis.mcp.action_manifest import validate_actuator
+from artemis.mcp.action_specs import (
+    exception_prefix,
+    make_wire_handler,
+    wire_dialects,
+)
+from artemis.mcp.action_types import ActionCode, ActionResult, ObserveResult
+from artemis.mcp.actuators.base import Actuator
+from artemis.mcp.observation import observe as observe_impl
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+__all__ = ["build_action_server"]
+
+
+def _wrap(res: ActionResult) -> CallToolResult:
+    return CallToolResult(
+        content=[TextContent(type="text", text=res.message)],
+        structuredContent=res.model_dump(mode="json"),
+        isError=False,
+    )
+
+
+def _wrap_exception(action: str, exc: Exception) -> CallToolResult:
+    return _wrap(
+        ActionResult.failure(action, f"{exception_prefix(action)}: {exc}", detail=repr(exc))
+    )
+
+
+def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> FastMCP:
+    """Builds a FastMCP server exposing exactly what ``actuator`` implements.
+
+    Raises ``ActuatorContractError`` when the actuator violates the manifest, so a
+    misconfigured backend fails before serving anything.
+    """
+    validate_actuator(actuator)
+    caps = actuator.capabilities()
+    mcp = FastMCP(name)
+
+    # --- Device actions (registered only when implemented) ---------------------------
+    # Declarations and argument conversion both come from the canonical manifest
+    # (artemis/mcp/action_specs.py); this server contributes only the transport
+    # wrapping, so the served schema cannot drift from the manifest.
+
+    for spec in wire_dialects():
+        if spec.name in caps:
+            mcp.add_tool(
+                make_wire_handler(spec, actuator, _wrap, _wrap_exception),
+                name=spec.name,
+                description=spec.wire.description,
+            )
+
+    # --- Internal observation tools (never declared to an LLM) -----------------------
+
+    if "observe_screen" in caps:
+
+        @mcp.tool()
+        async def observe_screen(
+            settle_ms: int = 400, include_image: bool = False
+        ) -> Annotated[CallToolResult, ObserveResult]:
+            """Capture and index the current screen (adapter-internal)."""
+            try:
+                obs, img_bytes = await observe_impl(
+                    getattr(actuator, "ctx", None),
+                    actuator.controller,
+                    settle_ms=settle_ms,
+                )
+                content: list[Any] = [TextContent(type="text", text=obs.message)]
+                if include_image and img_bytes:
+                    import base64 as _b64
+
+                    content.append(
+                        ImageContent(
+                            type="image",
+                            data=_b64.b64encode(img_bytes).decode("utf-8"),
+                            mimeType="image/jpeg",
+                        )
+                    )
+                return CallToolResult(
+                    content=content,
+                    structuredContent=obs.model_dump(mode="json"),
+                    isError=False,
+                )
+            except Exception as e:
+                obs = ObserveResult(
+                    ok=False,
+                    code=ActionCode.DEVICE_ERROR,
+                    message=f"Error during observe_screen: {e}",
+                    hierarchy_ok=False,
+                )
+                return CallToolResult(
+                    content=[TextContent(type="text", text=obs.message)],
+                    structuredContent=obs.model_dump(mode="json"),
+                    isError=False,
+                )
+
+    # Timeouts on the two observation tools run INSIDE the tool, around the actuator
+    # call only. They must never be expressed as a transport-level read timeout
+    # (``read_timeout_seconds``) or a caller-side ``asyncio.wait_for``: both produce a
+    # late response to an abandoned request, which crashes the mcp-1.29 client
+    # receive loop (``_handle_response`` sends into the request's already-closed
+    # response stream) and bricks the in-memory session for every later caller.
+    # A server-side timeout yields a normal error payload and no late response.
+
+    if "take_screenshot" in caps:
+
+        @mcp.tool()
+        async def take_screenshot(timeout_ms: int | None = None) -> CallToolResult:
+            """Return the current screen as base64 (adapter-internal, no persistence)."""
+            try:
+                coro = actuator.take_screenshot()
+                if timeout_ms:
+                    coro = asyncio.wait_for(coro, timeout=timeout_ms / 1000.0)
+                b64 = await coro
+                payload: dict[str, Any] = {"ok": True, "image_b64": b64}
+            except TimeoutError:
+                payload = {"ok": False, "error": f"screenshot timed out after {timeout_ms}ms"}
+            except Exception as e:
+                payload = {"ok": False, "error": str(e)}
+            return CallToolResult(
+                content=[
+                    TextContent(type="text", text="screenshot" if payload["ok"] else str(payload))
+                ],
+                structuredContent=payload,
+                isError=False,
+            )
+
+    if "get_ui_hierarchy" in caps:
+
+        @mcp.tool()
+        async def get_ui_hierarchy(timeout_ms: int | None = None) -> CallToolResult:
+            """Return the raw UI element hierarchy (adapter-internal)."""
+            try:
+                coro = actuator.get_ui_elements()
+                if timeout_ms:
+                    coro = asyncio.wait_for(coro, timeout=timeout_ms / 1000.0)
+                elements = await coro
+                payload: dict[str, Any] = {"ok": True, "elements": elements}
+            except TimeoutError:
+                payload = {
+                    "ok": False,
+                    "error": f"ui hierarchy fetch timed out after {timeout_ms}ms",
+                }
+            except Exception as e:
+                payload = {"ok": False, "error": str(e)}
+            return CallToolResult(
+                content=[
+                    TextContent(type="text", text="ui_hierarchy" if payload["ok"] else str(payload))
+                ],
+                structuredContent=payload,
+                isError=False,
+            )
+
+    # --- Backend extension tools -----------------------------------------------------
+
+    for ext in actuator.extensions():
+        mcp.add_tool(ext.handler, name=ext.name, description=ext.description)
+
+    logger.info(
+        f"Action server '{name}' built with {len(caps)} device actions and"
+        f" {len(actuator.extensions())} extension tools."
+    )
+    return mcp