android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/agents/validator/tool_declarations.py

@@ -0,0 +1,332 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal Tool Declarations and Execution Results for Artemis.
+
+Execution itself lives in :class:`artemis.mcp.action_executor.McpActionExecutor`
+(Flash) and the Validator's action session (Pro); this module keeps the shared
+result envelope, the observation helpers both use, and the tool declarations.
+"""
+
+import base64
+import json
+from typing import Any, Literal
+
+from google.genai import types as genai_types
+from langchain_core.messages import ToolMessage
+from pydantic import BaseModel, Field
+
+from artemis.agents.explorer.constants import (
+    ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION,
+    ASK_EXPLORER_DESCRIPTION,
+    ASK_EXPLORER_QUERY_DESCRIPTION,
+    ASK_EXPLORER_TOOL_NAME,
+)
+from artemis.context import ArtemisContext
+from artemis.controllers.unified_controller import UnifiedMobileController
+from artemis.core.tool_declaration import ToolDeclaration
+from artemis.mcp.action_specs import tool_declaration
+from artemis.mcp.observation import observe
+from artemis.graph.state import State
+from artemis.utils.notes import (
+    LIST_NOTES_DOCSTRING,
+    READ_NOTE_ARG_KEY_DESC,
+    READ_NOTE_DOCSTRING,
+)
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ToolExecutionResult(BaseModel):
+    """Standardized result produced by executing a mobile automation or utility tool."""
+
+    tool_call_id: str
+    tool_name: str
+    status: Literal["success", "error", "cancelled"]
+    text_summary: str
+    screenshot_bytes: bytes | None = None
+    screenshot_path: str | None = None
+    ui_elements_text: str | None = None
+    raw_result: Any = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    def to_langchain_tool_message(self) -> ToolMessage:
+        """Converts execution result into a standardized multimodal LangChain ToolMessage."""
+        content_blocks: list[dict[str, Any]] = [
+            {"type": "text", "text": self.text_summary or f"Action '{self.tool_name}' dispatched."}
+        ]
+        if self.ui_elements_text:
+            content_blocks.append(
+                {
+                    "type": "text",
+                    "text": f"--- UI Element List ---\n{self.ui_elements_text}",
+                }
+            )
+        if self.screenshot_bytes:
+            b64_img = base64.b64encode(self.screenshot_bytes).decode("utf-8")
+            content_blocks.append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
+                }
+            )
+        return ToolMessage(
+            tool_call_id=self.tool_call_id,
+            name=self.tool_name,
+            content=content_blocks,
+            status=self.status,
+        )
+
+
+def prune_intermediate_screenshots(messages: list[Any]) -> None:
+    """Prunes heavy binary screenshot blocks from intermediate observation messages.
+
+    Keeps the latest screenshot across the conversation history to preserve context budget.
+    Supports both LangChain BaseMessage and google.genai.types.Content formats.
+    """
+    if not messages:
+        return
+
+    # Check for google.genai.types.Content objects
+    if hasattr(messages[0], "parts"):
+        user_indices_with_images = []
+        for idx in range(1, len(messages)):
+            content = messages[idx]
+            if getattr(content, "role", None) == "user" and getattr(content, "parts", None):
+                for part in content.parts:
+                    if getattr(part, "inline_data", None) is not None:
+                        user_indices_with_images.append(idx)
+                        break
+
+        if len(user_indices_with_images) > 1:
+            indices_to_prune = user_indices_with_images[:-1]
+            for idx in indices_to_prune:
+                content = messages[idx]
+                new_parts = []
+                for part in content.parts:
+                    if getattr(part, "inline_data", None) is not None:
+                        try:
+                            new_parts.append(
+                                genai_types.Part.from_text(
+                                    text=(
+                                        "[Screenshot of intermediate step omitted for performance]"
+                                    )
+                                )
+                            )
+                        except Exception as exc:
+                            logger.debug(
+                                "Could not build placeholder text part for pruned screenshot;"
+                                " dropping it: %s",
+                                exc,
+                                exc_info=True,
+                            )
+                    else:
+                        new_parts.append(part)
+                content.parts = new_parts
+        return
+
+    # LangChain BaseMessage format
+    last_img_msg_idx = -1
+    for idx in range(len(messages) - 1, -1, -1):
+        msg = messages[idx]
+        msg_content = getattr(msg, "content", None)
+        if isinstance(msg_content, list):
+            has_image = any(
+                isinstance(block, dict) and block.get("type") in ("image_url", "image")
+                for block in msg_content
+            )
+            if has_image:
+                last_img_msg_idx = idx
+                break
+
+    for idx in range(len(messages)):
+        if idx == last_img_msg_idx:
+            continue
+        msg = messages[idx]
+        msg_content = getattr(msg, "content", None)
+        if isinstance(msg_content, list):
+            msg.content = [
+                b
+                for b in msg_content
+                if not (isinstance(b, dict) and b.get("type") in ("image_url", "image"))
+            ]
+
+
+async def capture_screenshot_and_parse_ui(
+    ctx: ArtemisContext,
+    state: State,
+    controller: UnifiedMobileController,
+    skip_settling: bool = False,
+) -> tuple[str | None, bytes | None, str | None]:
+    """Captures screenshot and parses fused XML tree after optional screen settling delay.
+
+    Thin adapter over :func:`artemis.mcp.observation.observe` that adds the LangGraph
+    ``State`` write-back (indexed points/elements) the agents rely on.
+    """
+    obs, screenshot_bytes = await observe(ctx, controller, settle_ms=0 if skip_settling else 400)
+    if not obs.ok:
+        return None, None, None
+
+    if obs.hierarchy_ok:
+        state.indexed_points = [el["center"] for el in obs.elements]
+        state.indexed_elements = obs.elements
+
+    return obs.screenshot_path, screenshot_bytes, obs.elements_text
+
+
+def normalize_coordinate_target(target: Any) -> int | list[int] | Any:
+    """Normalizes coordinate targets or index references to standard types.
+
+    Supports:
+    - Integer index: e.g. 1 or "1" -> 1
+    - Coordinate list: e.g. [500, 280] or ["500", "280"] -> [500, 280]
+    - Serialized coordinate string: e.g. "[500, 280]" or "500, 280" -> [500, 280]
+    """
+    if isinstance(target, int):
+        return target
+
+    if isinstance(target, (list, tuple)):
+        if len(target) == 2:
+            try:
+                return [int(target[0]), int(target[1])]
+            except (ValueError, TypeError):
+                pass
+        elif len(target) == 1:
+            return normalize_coordinate_target(target[0])
+
+    if isinstance(target, str):
+        cleaned = target.strip()
+        if cleaned.isdigit() or (cleaned.startswith("-") and cleaned[1:].isdigit()):
+            return int(cleaned)
+        try:
+            parsed = json.loads(cleaned)
+            return normalize_coordinate_target(parsed)
+        except (json.JSONDecodeError, TypeError):
+            pass
+        if "," in cleaned:
+            parts = [p.strip().strip("[]() ") for p in cleaned.split(",")]
+            if len(parts) == 2:
+                try:
+                    return [int(parts[0]), int(parts[1])]
+                except (ValueError, TypeError):
+                    pass
+
+    return target
+
+
+# Device-action declarations are generated from the canonical manifest
+# (artemis/mcp/action_specs.py); the historical constant names remain the public API.
+CLICK_TOOL = tool_declaration("click")
+
+CLICK_SEQUENCE_TOOL = tool_declaration("click_sequence")
+
+LONG_PRESS_TOOL = tool_declaration("long_press")
+
+INPUT_TEXT_TOOL = tool_declaration("input_text")
+
+SWIPE_TOOL = tool_declaration("swipe")
+
+PRESS_KEY_TOOL = tool_declaration("press_key")
+
+READ_NOTE_TOOL = ToolDeclaration(
+    name="read_note",
+    description=READ_NOTE_DOCSTRING,
+    parameters={
+        "type": "object",
+        "properties": {
+            "key": {
+                "type": "string",
+                "description": READ_NOTE_ARG_KEY_DESC,
+            },
+            "start_line": {
+                "type": "integer",
+                "description": "Start line to read (1-indexed, inclusive).",
+            },
+            "end_line": {
+                "type": "integer",
+                "description": "End line to read (1-indexed, inclusive).",
+            },
+        },
+        "required": ["key"],
+    },
+)
+
+LIST_NOTES_TOOL = ToolDeclaration(
+    name="list_notes",
+    description=LIST_NOTES_DOCSTRING,
+    parameters={"type": "object", "properties": {}},
+)
+
+MANAGE_APP_TOOL = tool_declaration("manage_app")
+
+WAIT_FOR_DELAY_TOOL = tool_declaration("wait_for_delay")
+
+REPORT_TASK_STATUS_TOOL = ToolDeclaration(
+    name="report_task_status",
+    description=(
+        "[REPORT] Use this tool to return your final answer when you are done"
+        " with the task. This is the ONLY way to return your final conclusion."
+    ),
+    parameters={
+        "type": "object",
+        "properties": {
+            "status": {
+                "type": "string",
+                "description": (
+                    "Whether the task was completed successfully ('completed') or"
+                    " failed/unreachable ('failed')."
+                ),
+            },
+            "explanation": {
+                "type": "string",
+                "description": "Provide the final explanation or summary of the task execution.",
+            },
+        },
+        "required": ["status", "explanation"],
+    },
+)
+
+# Same contract as the Operator's LangChain tool (artemis/tools/explorer_tool.py):
+# the Explorer tier is a user setting, so no tier argument is exposed here.
+ASK_EXPLORER_TOOL = ToolDeclaration(
+    name=ASK_EXPLORER_TOOL_NAME,
+    description=ASK_EXPLORER_DESCRIPTION,
+    parameters={
+        "type": "object",
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": ASK_EXPLORER_QUERY_DESCRIPTION,
+            },
+            "context_feedback": {
+                "type": "string",
+                "description": ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION,
+            },
+        },
+        "required": ["query"],
+    },
+)
+
+VALIDATOR_TOOLS_DECLARATION: list[ToolDeclaration] = [
+    CLICK_TOOL,
+    LONG_PRESS_TOOL,
+    INPUT_TEXT_TOOL,
+    SWIPE_TOOL,
+    PRESS_KEY_TOOL,
+    READ_NOTE_TOOL,
+    LIST_NOTES_TOOL,
+    MANAGE_APP_TOOL,
+    WAIT_FOR_DELAY_TOOL,
+]

package/artemis-main/artemis/agents/validator/validator.py

@@ -0,0 +1,188 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""ValidatorNode facade: action execution + pre-execution safety net.
+
+The implementation is split by responsibility:
+
+- :mod:`artemis.agents.validator.execution_loop` -- per-turn loop
+  orchestration, screenshot/trace bookkeeping, failure-repair handling.
+- :mod:`artemis.agents.validator.action_execution` -- decision parsing,
+  turn-initial screenshot loading, single-action dispatch.
+- :mod:`artemis.agents.validator.precondition_xml` -- XML-hierarchy safety
+  net (element matching, coordinate self-healing, failure taxonomy).
+- :mod:`artemis.agents.validator.precondition_pixel` -- pixel/VLM safety net.
+
+This module remains the stable import path: ``ValidatorNode``,
+``ValidationErrorCategory``, and the patchable seams (``get_action_session``,
+``get_llm``, ``VALIDATOR_POLL_TIMEOUT``, ``VALIDATOR_POLL_INTERVAL``) are all
+resolved here so existing imports and ``mock.patch`` targets keep working.
+"""
+
+import asyncio
+import traceback
+from uuid import UUID
+
+from artemis.agents.validator import (
+    action_execution,
+    execution_loop,
+    precondition_pixel,
+    precondition_xml,
+)
+from artemis.agents.validator.categories import ValidationErrorCategory
+from artemis.constants import (  # noqa: F401  (patchable seams; the UI-change
+    # polling path that consumed the poll constants was removed as dead code,
+    # but the names stay importable/patchable at this module path.)
+    VALIDATOR_POLL_INTERVAL,
+    VALIDATOR_POLL_TIMEOUT,
+    VALIDATOR_UI_HIERARCHY_TIMEOUT,
+)
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace
+from artemis.graph.state import State
+from artemis.graph.visibility import strict_state
+from artemis.mcp.action_session import ActionSession, get_action_session
+from artemis.services.llm import acomplete_structured, get_llm  # noqa: F401
+from third_party.mobile_use.utils.decorators import agent_lifecycle_logging
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+__all__ = ["ValidatorNode", "ValidationErrorCategory"]
+
+
+class ValidatorNode:
+    """Node responsible for executing actions on the device and verifying the results.
+
+    Despite its name, this node handles:
+    1. Parsing structured decisions from the Operator.
+    2. Executing these actions via an MCP session (calling tools like tap,
+    swipe, etc.).
+    3. Validating action preconditions against the live screen (safety net).
+    4. Triggering failure analysis and local repair if actions fail.
+
+    Post-hoc goal verification lives elsewhere: the Checker performs the
+    checkpoint/final audits at subgoal completion and task exit.
+    """
+
+    def __init__(self, ctx: ArtemisContext):
+        self.ctx = ctx
+
+    async def _get_mcp_session(self) -> ActionSession:
+        """Returns the in-process unified action session (created lazily on ctx).
+
+        The previous stdio-subprocess spawn -- and its local-controller fallback,
+        which on Windows was the *only* path that ever ran -- is gone: the server now
+        lives in-process, so there is no spawn to fail and exactly one execution path.
+        """
+        return await get_action_session(self.ctx)
+
+    def _parse_decisions(self, structured_decisions: str) -> tuple[list[dict] | None, str | None]:
+        return action_execution.parse_decisions(structured_decisions)
+
+    async def _get_initial_screenshot(self, session, state: State) -> tuple[str, str | None]:
+        return await action_execution.read_initial_screenshot(state)
+
+    async def _execute_validation_loop(self, state: State):
+        return await execution_loop.run_validation_loop(self, state)
+
+    @agent_lifecycle_logging("Validator", logger)
+    @trace(type="agent", name="validator")
+    async def __call__(self, state: State):
+        state = strict_state(state, "validator")
+
+        step_id = None
+        if state.current_step_id:
+            step_id = UUID(state.current_step_id)
+
+        original_step_id = getattr(self.ctx.data_engine, "current_step_id", None)
+        if self.ctx.data_engine and step_id:
+            self.ctx.data_engine.current_step_id = step_id
+
+        try:
+            return await self._execute_validation_loop(state)
+        except asyncio.CancelledError:
+            # In-process session teardown happens in ArtemisContext.__aexit__; there
+            # is no subprocess left to kill.
+            logger.warning("Validator task cancelled (Stop requested).")
+            raise
+        except Exception as e:
+            err_stack = traceback.format_exc()
+            logger.critical(f"CRITICAL ERROR in Validator execution loop: {e}\n{err_stack}")
+            if self.ctx.data_engine and step_id:
+                try:
+                    self.ctx.data_engine.update_step_execution_result(
+                        step_id,
+                        {
+                            "status": "error",
+                            "error_msg": str(e),
+                            "traceback": err_stack,
+                        },
+                    )
+                except Exception as record_err:
+                    logger.debug(
+                        "Could not persist validator error status for step %s: %s",
+                        step_id,
+                        record_err,
+                        exc_info=True,
+                    )
+            raise e
+        finally:
+            if self.ctx.data_engine:
+                self.ctx.data_engine.current_step_id = original_step_id
+
+    async def _exec_action(self, session: ActionSession, action_item: dict) -> tuple[bool, str]:
+        """Executes one Operator action item through the unified action session."""
+        return await action_execution.exec_action(self.ctx, session, action_item)
+
+    @trace(type="tool", name="safety_net_validation")
+    async def _validate_action_precondition(
+        self, session, action_item: dict, state: State | None = None
+    ) -> tuple[bool, ValidationErrorCategory, str]:
+        """Safety Net: XML-hierarchy validation with a short retry loop."""
+        return await precondition_xml.validate_action_precondition(
+            self, session, action_item, state
+        )
+
+    async def _validate_action_precondition_single(
+        self, session, action_item: dict, state: State | None = None
+    ) -> tuple[bool, ValidationErrorCategory, str]:
+        """Safety Net: single-shot XML-hierarchy validation."""
+        return await precondition_xml.validate_action_precondition_single(
+            self.ctx, session, action_item, state
+        )
+
+    @trace(type="tool", name="safety_net_pixel_validation")
+    async def _validate_action_precondition_pixel(
+        self,
+        session,
+        action_item: dict,
+        pre_screenshot_b64: str,
+        original_coords: list[int] | None = None,
+        state: State | None = None,
+    ) -> tuple[bool, ValidationErrorCategory, str]:
+        """Pixel-level Safety Net: VLM comparison of the target crop.
+
+        ``get_llm`` is passed through from this module's namespace so tests
+        patching ``artemis.agents.validator.validator.get_llm`` take effect.
+        """
+        return await precondition_pixel.validate_action_precondition_pixel(
+            self.ctx,
+            session,
+            action_item,
+            pre_screenshot_b64,
+            original_coords,
+            state,
+            get_llm_fn=get_llm,
+        )

package/artemis-main/artemis/agents/video_analyzer/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Video analyzer utility for analyzing video content with Gemini models."""