android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/artemis/tools/system/adb_shell.py

@@ -0,0 +1,31 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from artemis.tools.command_tool import (
+    RunAdbCommandArgs,
+    RunAdbCommandTool,
+    RunAdbCommand,
+    run_adb_command,
+)
+
+# Backward-compatibility alias
+AdbShellArgs = RunAdbCommandArgs
+
+__all__ = [
+    "RunAdbCommandArgs",
+    "RunAdbCommandTool",
+    "RunAdbCommand",
+    "run_adb_command",
+    "AdbShellArgs",
+]

package/artemis-main/artemis/tools/tool_wrapper.py

@@ -0,0 +1,270 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Callable
+import functools
+import inspect
+import re
+import time
+from typing import Annotated, Any, Literal, get_args, get_origin
+import uuid
+
+from langchain_core.messages import BaseMessage, HumanMessage, ToolMessage
+from langchain_core.tools import BaseTool
+from langchain_core.tools.base import InjectedToolCallId
+from langgraph.prebuilt import InjectedState
+
+from artemis.core.tool_failure import is_tool_failure
+from artemis.data_engine import engine as engine_mod
+from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize
+from artemis.graph.state import State
+from third_party.mobile_use.tools.tool_wrapper import CompositeToolWrapper, ToolWrapper
+
+__all__ = [
+    "CompositeToolWrapper",
+    "ToolWrapper",
+    "get_tool_result_content",
+    "invoke_tool_with_injection",
+    "resolve_image_carrier",
+    "split_multimodal_result",
+    "tool_result_messages",
+]
+
+
+# pylint: disable=too-many-branches,too-many-locals,too-many-statements
+async def invoke_tool_with_injection(
+    tool: BaseTool,
+    args: dict,
+    tool_call_id: str,
+    state: State | None = None,
+    record_trace: bool = True,
+) -> Any:
+    """Invokes a LangChain tool, manually injecting arguments marked with
+
+    InjectedState or InjectedToolCallId if calling the underlying function
+    directly.
+    Automatically records type='tool' trace in DataEngine.
+    """
+    final_args = dict(args)
+
+    def _real_callable(candidate: Any) -> Callable | None:
+        """Return a routine or partial whose signature supports direct injection."""
+        target = candidate.func if isinstance(candidate, functools.partial) else candidate
+        if inspect.isroutine(target):
+            return candidate
+        return None
+
+    # For StructuredTool, the original function might be in tool.func or tool.coroutine
+    func = None
+    if getattr(tool, "coroutine", None):
+        func = _real_callable(tool.coroutine)
+    if func is None and getattr(tool, "func", None):
+        func = _real_callable(tool.func)
+
+    if func:
+        try:
+            sig = inspect.signature(func)
+            for param_name, param in sig.parameters.items():
+                annotation = param.annotation
+                # Handle Annotated types
+                if get_origin(annotation) is Annotated:
+                    args_list = get_args(annotation)
+                    if InjectedState in args_list:
+                        final_args[param_name] = state
+                    elif InjectedToolCallId in args_list:
+                        final_args[param_name] = tool_call_id
+        except (ValueError, TypeError):
+            # If inspection fails, fall back to just using provided args
+            pass
+
+        trace_id = uuid.uuid4()
+        parent_id = CURRENT_TRACE_ID.get()
+        token = CURRENT_TRACE_ID.set(trace_id)
+        start_time = time.time()
+
+        live_engine = getattr(engine_mod, "_CURRENT_DATA_ENGINE", None)
+
+        if record_trace and live_engine:
+            step_id = getattr(live_engine, "current_step_id", None)
+            live_engine.record_trace(
+                type="tool",
+                name=tool.name,
+                payload={"args": {k: smart_serialize(v) for k, v in args.items()}},
+                status="running",
+                parent_trace_id=parent_id,
+                step_id=step_id,
+                trace_id=trace_id,
+            )
+
+        execution_status = "running"
+        execution_result = None
+        execution_error = None
+        try:
+            # Call the function
+            if inspect.iscoroutinefunction(func):
+                result = await func(**final_args)
+            else:
+                result = func(**final_args)
+            execution_result = result
+            if is_tool_failure(result):
+                # A structural failure is traced as one; the text is the error.
+                execution_status = "failed"
+                execution_error = result
+            else:
+                execution_status = "success"
+            return result
+        except Exception as e:
+            execution_status = "failed"
+            execution_error = e
+            raise e
+        finally:
+            duration = time.time() - start_time
+            if record_trace and live_engine:
+                step_id = getattr(live_engine, "current_step_id", None)
+                payload = {"args": {k: smart_serialize(v) for k, v in args.items()}}
+
+                if execution_status == "success":
+                    payload["result"] = smart_serialize(get_tool_result_content(execution_result))
+                elif execution_status == "failed":
+                    payload["error"] = str(execution_error)
+                else:
+                    execution_status = "terminated"
+                    payload["error"] = "Execution terminated unexpectedly."
+
+                live_engine.record_trace(
+                    type="tool",
+                    name=tool.name,
+                    payload=payload,
+                    status=execution_status,
+                    duration=duration,
+                    parent_trace_id=parent_id,
+                    step_id=step_id,
+                    trace_id=trace_id,
+                )
+            CURRENT_TRACE_ID.reset(token)
+    else:
+        # Fallback to ainvoke if no direct func available; tools that declare
+        # a "state" argument still receive the graph state.
+        tool_args = getattr(tool, "args", None)
+        if state is not None and isinstance(tool_args, dict) and "state" in tool_args:
+            final_args["state"] = state
+        return await tool.ainvoke(final_args)
+
+
+def get_tool_result_content(result: Any) -> Any:
+    """Extracts the content from a tool result, handling ToolMessage returns."""
+    if isinstance(result, ToolMessage):
+        return result.content
+    return result
+
+
+# --- Multimodal tool results ------------------------------------------------------------
+
+ImageCarrier = Literal["tool", "human"]
+
+#: Providers whose tool-result messages may carry image parts directly.
+_TOOL_IMAGE_PROVIDERS = frozenset({"google", "anthropic"})
+
+_STEP_RE = re.compile(r"\bstep\s+(\d+)", re.IGNORECASE)
+
+
+def split_multimodal_result(result: Any) -> tuple[str, list[dict[str, Any]]]:
+    """Splits a tool result into its text and its image content blocks.
+
+    Plain strings have no images; content-block lists contribute their text
+    blocks (joined by newlines) and their ``image_url`` / ``image`` blocks.
+    ``ToolMessage`` results are unwrapped first.
+    """
+    content = get_tool_result_content(result)
+    if content is None:
+        return "", []
+    if isinstance(content, list):
+        texts: list[str] = []
+        images: list[dict[str, Any]] = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") in ("image_url", "image"):
+                images.append(block)
+            elif isinstance(block, dict) and block.get("type") == "text":
+                texts.append(str(block.get("text") or ""))
+            else:
+                texts.append(str(block))
+        return "\n".join(t for t in texts if t), images
+    return content if isinstance(content, str) else str(content), []
+
+
+def resolve_image_carrier(llm_or_provider: Any) -> ImageCarrier:
+    """Which message carries images returned by a tool for this LLM.
+
+    Gemini (direct API) and Anthropic accept image parts inside the tool
+    result message itself (``"tool"``). Vertex AI and the OpenAI-compatible
+    providers (openai / openrouter / xai / ...) reject them there, so the
+    image travels in a ``HumanMessage`` that immediately follows the textual
+    tool result (``"human"``). Accepts a provider name, a provider enum, or an
+    LLM whose ``endpoint.provider`` names the provider that actually issues
+    the request (a fallback model is judged by its own provider).
+    """
+    provider: Any = llm_or_provider
+    if provider is not None and not isinstance(provider, str):
+        # An LLM (RobustChatModelWrapper) names its provider on ``endpoint``;
+        # anything without an endpoint is taken as the provider itself.
+        endpoint = getattr(provider, "endpoint", None)
+        if endpoint is not None:
+            provider = getattr(endpoint, "provider", None)
+    value = str(getattr(provider, "value", provider) or "").lower()
+    return "tool" if value in _TOOL_IMAGE_PROVIDERS else "human"
+
+
+def tool_result_messages(
+    tool_call_id: str,
+    result: Any,
+    *,
+    name: str | None = None,
+    status: str = "success",
+    image_carrier: ImageCarrier | None = None,
+    llm: Any = None,
+) -> list[BaseMessage]:
+    """The conversation messages that deliver one tool result to the model.
+
+    Text-only results become a single ``ToolMessage``. Results carrying images
+    (e.g. ``get_step_screenshot``) are delivered according to the image
+    carrier — given explicitly or resolved from ``llm`` — so every agent loop
+    handles multimodal tool results the same way and no loop ever pastes
+    base64 into a text field. The tool call id, name and status are preserved.
+    """
+    text, images = split_multimodal_result(result)
+    extra = {"name": name} if name else {}
+    if not images:
+        content = get_tool_result_content(result)
+        if not isinstance(content, (str, list)):
+            content = text
+        return [ToolMessage(tool_call_id=tool_call_id, content=content, status=status, **extra)]
+
+    carrier = image_carrier or resolve_image_carrier(llm)
+    if carrier == "tool":
+        blocks: list[dict[str, Any]] = [{"type": "text", "text": text}] if text else []
+        return [
+            ToolMessage(
+                tool_call_id=tool_call_id, content=[*blocks, *images], status=status, **extra
+            )
+        ]
+
+    match = _STEP_RE.search(text)
+    caption = f"[Screenshot returned by {name or 'the tool'}"
+    if match:
+        caption += f" for step {match.group(1)}"
+    caption += "]"
+    return [
+        ToolMessage(tool_call_id=tool_call_id, content=text, status=status, **extra),
+        HumanMessage(content=[{"type": "text", "text": caption}, *images]),
+    ]

package/artemis-main/artemis/tools/video_tool.py

@@ -0,0 +1,261 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Video Analyzer utility for analyzing video content using Gemini models.
+
+This utility sends video files to video-capable Gemini models for analysis
+and returns text descriptions based on the provided prompt.
+It also exposes a tool for agents to perform structured video analysis.
+"""
+
+from typing import Any, Literal
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field
+
+from artemis.core.tool_failure import ToolFailure
+from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
+from artemis.context import ArtemisContext
+from artemis.core.tool_declaration import ToolDeclaration
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.graph.state import State
+from artemis.tools.base import ArtemisTool, ToolCategory
+from artemis.tools.tool_wrapper import ToolWrapper
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class VideoAnalyzerArgs(BaseModel):
+    """Arguments schema for video analyzer tool."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    time_description: str = Field(
+        ...,
+        description=(
+            "The specific time range to analyze, relative to the start of the"
+            " recording (e.g., 'from 5s to 12s', 'from 20s to the end')."
+        ),
+    )
+    purpose: str = Field(
+        ...,
+        description=(
+            "The specific query, intent, or question you want the video"
+            " analyzer to answer about the visual or audio content in this"
+            " segment."
+        ),
+    )
+
+
+OPERATOR_VIDEO_ANALYZER_DOCSTRING = (
+    "[VIDEO] Use this tool to analyze any specific time range of the video"
+    " recording. This tool delegates to a video analysis subagent"
+    " with full access to the continuous screen recording of your entire"
+    " execution session.\n\n"
+    "- This tool overcomes your limitation of only receiving discrete"
+    " screenshots. Common use cases include (but are not limited to):\n"
+    "  * Watching media files: Play the video on the device screen, use"
+    " `wait_for_delay` until the video finishes playing, and then call this"
+    " tool to analyze the recording.\n"
+    "  * Bridging the gap: Inspecting what happened between your last action"
+    " and the current screenshot.\n"
+    "  * Memory retrieval: Recalling fast-moving or transient information from"
+    " a previous step.\n"
+    "  * Catching rapid changes: Investigating on-screen information that"
+    " changes too quickly for static screenshots to capture.\n"
+    "  - Please specify a narrow time range when possible to ensure faster"
+    " processing.\n"
+    "  - If only audio analysis is needed, state it clearly in your purpose.\n"
+)
+
+#: Flash-profile declaration of the same tool (FlashRunner binds JSON-schema
+#: declarations rather than LangChain tools). Description and parameters are
+#: derived from the operator contract above so the two never drift.
+VIDEO_ANALYZER_TOOL = ToolDeclaration(
+    name="video_analyzer",
+    description=OPERATOR_VIDEO_ANALYZER_DOCSTRING,
+    parameters={
+        "type": "object",
+        "properties": {
+            field_name: {
+                "type": "string",
+                "description": field_info.description or "",
+            }
+            for field_name, field_info in VideoAnalyzerArgs.model_fields.items()
+        },
+        "required": list(VideoAnalyzerArgs.model_fields),
+    },
+)
+
+DIAGNOSER_VIDEO_ANALYZER_DOCSTRING = (
+    "[DIAGNOSTIC] Analyzes a video segment for failure diagnosis and"
+    " forensics.\n\n"
+    "Use this to verify if an action succeeded, check for error popups, or"
+    " investigate visual anomalies when a step fails.\n"
+    "Provide a narrow time range (e.g., 'from 10s to 15s') to ensure speed.\n"
+)
+
+
+class VideoAnalyzerTool(ArtemisTool):
+    """Universal tool for analyzing video content using Gemini models."""
+
+    def __init__(
+        self,
+        role: Literal["operator", "diagnoser"] = "operator",
+        description: str | None = None,
+        category: ToolCategory = "perception",
+    ):
+        chosen_description = description or (
+            OPERATOR_VIDEO_ANALYZER_DOCSTRING
+            if role == "operator"
+            else DIAGNOSER_VIDEO_ANALYZER_DOCSTRING
+        )
+        super().__init__(
+            name="video_analyzer",
+            description=chosen_description,
+            args_schema=VideoAnalyzerArgs,
+            category=category,
+        )
+        self.role = role
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,  # pylint: disable=unused-argument
+        ctx: ArtemisContext | None = None,
+        time_description: str | None = None,
+        purpose: str | None = None,
+        tool_call_id: str | None = None,
+        state: State | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        time_desc = (
+            time_description
+            if time_description is not None
+            else (kwargs.get("time_description") or kwargs.get("TimeDescription") or "")
+        )
+        purp = (
+            purpose
+            if purpose is not None
+            else (kwargs.get("purpose") or kwargs.get("Purpose") or "")
+        )
+        tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
+        st = state if state is not None else kwargs.get("state")
+
+        try:
+            logger.info(f"video_analyzer called for range: {time_desc}")
+            if ctx is None:
+                raise ValueError("ArtemisContext is required for VideoAnalyzer.")
+            agent = VideoAnalyzer(ctx)
+            agent_outcome, status = await agent.run(time_desc, purp)
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.error(f"Error running video analyzer: {e}")
+            agent_outcome = f"Error running video analyzer: {e}"
+            status = "error"
+
+        if st is not None:
+            return ToolMessage(
+                tool_call_id=tcid or "",
+                content=agent_outcome,
+                status=status,
+            )
+
+        if status == "failed":
+            return f"Video analysis failed: {agent_outcome}"
+        return agent_outcome
+
+
+# Universal tool instance & aliases
+video_analyzer = VideoAnalyzerTool()
+
+
+class VideoAnalyzerPureTool(ArtemisTool):
+    """Universal pure tool for video analysis returning text results directly."""
+
+    def __init__(self):
+        super().__init__(
+            name="video_analyzer_pure",
+            description=OPERATOR_VIDEO_ANALYZER_DOCSTRING,
+            args_schema=VideoAnalyzerArgs,
+            category="perception",
+        )
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,  # pylint: disable=unused-argument
+        ctx: ArtemisContext | None = None,
+        time_description: str | None = None,
+        purpose: str | None = None,
+        **kwargs: Any,
+    ) -> str:
+        time_desc = (
+            time_description
+            if time_description is not None
+            else (kwargs.get("time_description") or kwargs.get("TimeDescription") or "")
+        )
+        purp = (
+            purpose
+            if purpose is not None
+            else (kwargs.get("purpose") or kwargs.get("Purpose") or "")
+        )
+
+        try:
+            logger.info(f"video_analyzer (pure) called for range: {time_desc}")
+            if ctx is None:
+                raise ValueError("ArtemisContext is required for VideoAnalyzer.")
+            agent = VideoAnalyzer(ctx)
+            agent_outcome, status = await agent.run(time_desc, purp)
+            if status == "failed":
+                return ToolFailure(f"Video analysis failed: {agent_outcome}")
+            return agent_outcome
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.error(f"Error in pure video_analyzer tool: {e}")
+            return ToolFailure(f"Error running video analyzer: {e}")
+
+
+# Universal pure tool instance & aliases
+video_analyzer_pure = VideoAnalyzerPureTool()
+VideoAnalyzerPure = VideoAnalyzerPureTool
+
+
+def get_video_analyzer_tool(
+    ctx: ArtemisContext, role: Literal["operator", "diagnoser"] = "operator"
+) -> BaseTool:
+    """Creates a tool that allows agents to perform structured video analysis.
+
+    The tool acts as a Main Agent following the Dynamic Video Analyzer plan.
+    """
+    tool_inst = VideoAnalyzerTool(role=role)
+    return trace_langchain_tool(tool_inst.to_langchain_tool(ctx), ctx)
+
+
+def get_video_analyzer_tool_pure(ctx: ArtemisContext) -> BaseTool:
+    """Creates a pure LangChain tool for video analysis that returns the text result directly.
+
+    Suitable for agents running outside of LangGraph (like the outputter).
+    """
+    return trace_langchain_tool(
+        video_analyzer_pure.to_langchain_tool(ctx, name="video_analyzer"), ctx
+    )
+
+
+video_analyzer_wrapper = ToolWrapper(
+    tool_fn_getter=get_video_analyzer_tool,
+    on_success_fn=lambda output: f"Video Analyzer replied:\n{output}",
+    on_failure_fn=lambda error: f"Video Analyzer failed: {error}",
+)

package/artemis-main/artemis/tools/wait_tool.py

@@ -0,0 +1,97 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+from typing import Any
+
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field
+
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.tools.base import ArtemisTool, ToolCategory
+from artemis.tools.tool_wrapper import ToolWrapper
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class WaitArgs(BaseModel):
+    """Arguments schema for wait tool."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    seconds: int = Field(
+        ...,
+        description=("The duration to wait/sleep in seconds. Minimum is 1, maximum is 60."),
+    )
+
+
+WAIT_DOCSTRING = (
+    "[SHELL] Pauses execution for a specified duration in seconds.\n\n"
+    "Use this to wait for active background operations to progress."
+)
+
+
+class WaitTool(ArtemisTool):
+    """Universal tool for pausing execution for a specified duration in seconds."""
+
+    def __init__(self, category: ToolCategory = "system"):
+        super().__init__(
+            name="wait",
+            description=WAIT_DOCSTRING,
+            args_schema=WaitArgs,
+            category=category,
+        )
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,  # pylint: disable=unused-argument
+        ctx: ArtemisContext | None = None,  # pylint: disable=unused-argument
+        seconds: int | None = None,
+        **kwargs: Any,
+    ) -> str:
+        sec = (
+            seconds
+            if seconds is not None
+            else (kwargs.get("seconds") or kwargs.get("Seconds") or 1)
+        )
+        if sec < 1:
+            sec = 1
+        elif sec > 60:
+            sec = 60
+
+        logger.info(f"Diagnoser waiting for {sec} seconds...")
+        await asyncio.sleep(sec)
+        return f"Waited {sec} seconds."
+
+
+# Universal tool instance & aliases
+wait = WaitTool()
+Wait = WaitTool
+
+
+def get_wait_tool(ctx: ArtemisContext) -> BaseTool:
+    """Exports wait as a LangChain BaseTool."""
+    return trace_langchain_tool(wait.to_langchain_tool(ctx), ctx)
+
+
+wait_wrapper = ToolWrapper(
+    tool_fn_getter=get_wait_tool,
+    on_success_fn=lambda seconds: f"Waited {seconds}s",
+    on_failure_fn=lambda err: f"Wait failed: {err}",
+)
+export_wait_tool = get_wait_tool