android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/artemis/tools/mobile/read_hierarchy.py

@@ -0,0 +1,108 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+from typing import Any
+
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel
+
+from artemis.core.tool_failure import ToolFailure
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.graph.state import State
+from artemis.tools.base import ArtemisTool, ToolCategory
+from artemis.tools.tool_wrapper import ToolWrapper
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class GetUiHierarchyArgs(BaseModel):
+    """Arguments schema for getting UI hierarchy."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+
+
+GET_UI_HIERARCHY_DOCSTRING = (
+    "[DIAGNOSTIC] Retrieves the current screen UI hierarchy XML using a persistent"
+    " UI Automator client.\n\nUse this instead of running 'uiautomator dump'"
+    " command via adb shell, as it is much faster and more stable."
+)
+
+
+class GetUiHierarchyTool(ArtemisTool):
+    """Universal tool for retrieving screen UI hierarchy XML."""
+
+    def __init__(self, category: ToolCategory = "diagnostic"):
+        super().__init__(
+            name="get_ui_hierarchy",
+            description=GET_UI_HIERARCHY_DOCSTRING,
+            args_schema=GetUiHierarchyArgs,
+            category=category,
+        )
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,
+        ctx: ArtemisContext | None = None,
+        tool_call_id: str | None = None,  # pylint: disable=unused-argument
+        state: State | None = None,  # pylint: disable=unused-argument
+        **kwargs: Any,
+    ) -> str:
+        try:
+            logger.info("get_ui_hierarchy tool called.")
+            if ctx is not None and getattr(ctx, "ui_adb_client", None) is not None:
+                xml_hierarchy = ctx.ui_adb_client.get_hierarchy()
+                if inspect.iscoroutine(xml_hierarchy):
+                    xml_hierarchy = await xml_hierarchy
+                return str(xml_hierarchy)
+            if driver is not None and hasattr(driver, "get_screen_data"):
+                screen_data = await driver.get_screen_data()
+                xml_hierarchy = getattr(screen_data, "ui_hierarchy_xml", None)
+                if xml_hierarchy:
+                    return str(xml_hierarchy)
+                return ToolFailure("Error retrieving UI hierarchy: No UI hierarchy in screen data.")
+            if driver is not None and hasattr(driver, "get_ui_hierarchy"):
+                xml_hierarchy = await driver.get_ui_hierarchy()
+                if inspect.iscoroutine(xml_hierarchy):
+                    xml_hierarchy = await xml_hierarchy
+                return str(xml_hierarchy)
+            return ToolFailure(
+                "Error retrieving UI hierarchy: No UI Automator client or driver provided."
+            )
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.error(f"Failed to get UI hierarchy: {e}")
+            return ToolFailure(f"Error retrieving UI hierarchy: {e}")
+
+
+# Universal tool instance & aliases
+get_ui_hierarchy = GetUiHierarchyTool()
+GetUiHierarchy = GetUiHierarchyTool
+GetUIHierarchy = GetUiHierarchyTool
+
+
+def get_ui_hierarchy_tool(ctx: ArtemisContext) -> BaseTool:
+    """Exports get_ui_hierarchy as a LangChain BaseTool."""
+    return trace_langchain_tool(get_ui_hierarchy.to_langchain_tool(ctx), ctx)
+
+
+ui_hierarchy_wrapper = ToolWrapper(
+    tool_fn_getter=get_ui_hierarchy_tool,
+    on_success_fn=lambda output: output,
+    on_failure_fn=lambda error: f"Failed to retrieve UI hierarchy: {error}",
+)

package/artemis-main/artemis/tools/mobile/read_logs.py

@@ -0,0 +1,116 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any
+
+from langchain_core.tools.base import BaseTool
+from pydantic import BaseModel, Field
+
+from artemis.core.tool_failure import ToolFailure
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.graph.state import State
+from artemis.tools.base import ArtemisTool, ToolCategory
+from artemis.tools.mobile.log_utils import fetch_and_filter_logs
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ReadLogsArgs(BaseModel):
+    """Arguments schema for read_logs tool."""
+
+    lines: int | None = Field(
+        default=200,
+        description=(
+            "Number of recent log lines to read. Default is 200. Use larger"
+            " numbers only if necessary."
+        ),
+    )
+    since_time: str | None = Field(
+        default=None,
+        description=(
+            "Target timestamp in 'MM-DD HH:MM:SS.ms' format OR relative seconds"
+            " (e.g., '15.5') to read logs since. If provided, we will try to"
+            " fetch logs starting from this time."
+        ),
+    )
+    until_time: str | None = Field(
+        default=None,
+        description=(
+            "Target timestamp in 'MM-DD HH:MM:SS.ms' format OR relative seconds"
+            " (e.g., '20.0') to read logs until. If provided, we will filter"
+            " logs up to this time."
+        ),
+    )
+
+
+READ_LOGS_DOCSTRING = (
+    "[DIAGNOSTIC] Fetches raw Android logs.\n\n"
+    "- Use when: You need to scan logs for a time window or see the sequence"
+    " of events.\n"
+    "- Input: lines (int, optional): Number of lines to read (default 200)."
+    " since_time (str, optional): Timestamp or relative seconds to read since."
+    " until_time (str, optional): Timestamp or relative seconds to read until.\n"
+    "- Output: Raw log text."
+)
+
+
+class ReadLogsTool(ArtemisTool):
+    """Universal tool for fetching raw Android logs."""
+
+    def __init__(self, category: ToolCategory = "diagnostic"):
+        super().__init__(
+            name="read_logs",
+            description=READ_LOGS_DOCSTRING,
+            args_schema=ReadLogsArgs,
+            category=category,
+        )
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,  # pylint: disable=unused-argument
+        ctx: ArtemisContext | None = None,
+        lines: int | None = 200,
+        since_time: str | None = None,
+        until_time: str | None = None,
+        tool_call_id: str | None = None,  # pylint: disable=unused-argument
+        state: State | None = None,  # pylint: disable=unused-argument
+        **kwargs: Any,
+    ) -> str:
+        try:
+            num_lines = 200 if lines is None else lines
+            logs = fetch_and_filter_logs(
+                ctx=ctx,
+                lines=num_lines,
+                since_time=since_time,
+                until_time=until_time,
+            )
+            return logs
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.error(f"Failed to read logs: {e}")
+            return ToolFailure(f"Failed to read logs: {e}")
+
+
+# Universal tool instance & aliases
+read_logs = ReadLogsTool()
+ReadLogs = ReadLogsTool
+ReadLogsToolAlias = ReadLogsTool
+
+
+def get_read_logs_tool(ctx: ArtemisContext) -> BaseTool:
+    """Exports read_logs as a LangChain BaseTool."""
+    return trace_langchain_tool(read_logs.to_langchain_tool(ctx), ctx)

package/artemis-main/artemis/tools/mobile/search_logs.py

@@ -0,0 +1,260 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import Any
+
+from langchain_core.tools.base import BaseTool
+from pydantic import BaseModel, Field
+
+from artemis.core.tool_failure import ToolFailure
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.graph.state import State
+from artemis.tools.base import ArtemisTool, ToolCategory
+from artemis.tools.mobile.log_utils import fetch_and_filter_logs
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+#: How many line ranges a merged block's header lists before it is cut.
+_MAX_HEADER_RANGES = 8
+
+
+def _compress_line_numbers(indices: list[int]) -> str:
+    """Renders sorted line numbers as ranges: ``6``, ``69-79``, ``699-798``.
+
+    Consecutive runs collapse into ``start-end``; at most
+    ``_MAX_HEADER_RANGES`` ranges are listed, the rest replaced by ``...``.
+    """
+    runs: list[str] = []
+    start = prev = indices[0]
+    for idx in indices[1:]:
+        if idx == prev + 1:
+            prev = idx
+            continue
+        runs.append(str(start) if start == prev else f"{start}-{prev}")
+        start = prev = idx
+    runs.append(str(start) if start == prev else f"{start}-{prev}")
+    if len(runs) > _MAX_HEADER_RANGES:
+        return ", ".join(runs[:_MAX_HEADER_RANGES]) + ", ..."
+    return ", ".join(runs)
+
+
+# pylint: disable=too-many-branches,too-many-locals,too-many-statements
+def search_and_merge_logs(
+    logs: str,
+    keyword: str,
+    context_lines: int = 50,
+    is_regex: bool = False,
+) -> str:
+    """Searches for keyword/pattern in logs and merges overlapping context windows."""
+    log_lines = logs.splitlines()
+
+    if is_regex:
+        try:
+            pattern = re.compile(keyword, re.IGNORECASE)
+        except re.error as e:
+            return ToolFailure(f"Error: Invalid regex: {e}")
+    else:
+        keyword_lower = keyword.lower()
+
+    # 1. Collect all raw match ranges
+    ranges = []
+    for i, line in enumerate(log_lines):
+        matched = False
+        if is_regex:
+            if pattern.search(line):
+                matched = True
+        else:
+            if keyword_lower in line.lower():
+                matched = True
+
+        if matched:
+            start = max(0, i - context_lines)
+            end = min(len(log_lines), i + context_lines + 1)
+            ranges.append((start, end, i))
+
+    if not ranges:
+        return f"No matches found for '{keyword}'."
+
+    # 2. Merge overlapping/adjacent ranges
+    ranges.sort(key=lambda x: x[0])
+
+    merged_ranges = []
+    curr_start, curr_end, curr_matches = (
+        ranges[0][0],
+        ranges[0][1],
+        [ranges[0][2]],
+    )
+
+    for start, end, match_idx in ranges[1:]:
+        if start <= curr_end:  # Overlaps or adjacent
+            curr_end = max(curr_end, end)
+            curr_matches.append(match_idx)
+        else:
+            merged_ranges.append((curr_start, curr_end, curr_matches))
+            curr_start = start
+            curr_end = end
+            curr_matches = [match_idx]
+    merged_ranges.append((curr_start, curr_end, curr_matches))
+
+    # 3. Format output
+    matches = []
+    total_lines = 0
+    max_return_lines = 300
+
+    truncated = False
+    for start, end, match_indices in merged_ranges:
+        if total_lines >= max_return_lines:
+            truncated = True
+            break
+
+        match_str = _compress_line_numbers(match_indices)
+        matches.append(f"--- Match at line(s) {match_str} ---")
+        for idx in range(start, end):
+            line_content = log_lines[idx]
+            if idx in match_indices:
+                matches.append(f"{idx:4d}: [MATCH] {line_content}")
+            else:
+                matches.append(f"{idx:4d}:         {line_content}")
+
+            total_lines += 1
+            if total_lines >= max_return_lines:
+                truncated = True
+                break
+
+        matches.append("-" * 20)
+
+    if truncated:
+        matches.append(
+            "\n... [SYSTEM WARNING: Output truncated to protect Context Window."
+            " Too many matches exceeded the 300 lines limit. You MUST use a"
+            " more specific keyword, narrow the time range, or use"
+            " context_lines=0.] ..."
+        )
+
+    return "\n".join(matches)
+
+
+class SearchLogsArgs(BaseModel):
+    """Arguments schema for search_logs tool."""
+
+    keyword: str = Field(
+        ...,
+        description=("The keyword or regular expression to search for. Case-insensitive."),
+    )
+    is_regex: bool = Field(
+        default=False,
+        description=("Whether to treat keyword as a regular expression. Default is False."),
+    )
+    lines: int | None = Field(
+        default=10000,
+        description=("Number of recent log lines to search in. Default is 10000."),
+    )
+    since_time: str | None = Field(
+        default=None,
+        description=(
+            "Target timestamp in 'MM-DD HH:MM:SS.ms' format OR relative seconds"
+            " (e.g., '15.5') to search logs since. If provided, we will search"
+            " logs starting from this time."
+        ),
+    )
+    until_time: str | None = Field(
+        default=None,
+        description=(
+            "Target timestamp in 'MM-DD HH:MM:SS.ms' format OR relative seconds"
+            " (e.g., '20.0') to search logs until. If provided, we will filter"
+            " logs up to this time."
+        ),
+    )
+    context_lines: int | None = Field(
+        default=0,
+        description=(
+            "Number of lines to include before and after each match for"
+            " context. Default is 0. Only increase if you need more context"
+            " around a very specific match."
+        ),
+    )
+
+
+SEARCH_LOGS_DOCSTRING = (
+    "[DIAGNOSTIC] Searches logs for a keyword or regex pattern.\n\n"
+    "- Use when: Looking for specific errors, tags, or keywords without"
+    " reading all logs.\n"
+    "- Input: keyword (str): Search pattern. is_regex (bool, optional):"
+    " Treat keyword as regex. context_lines (int, optional): Lines around"
+    " match. since_time (str, optional): Timestamp or relative seconds to"
+    " search since. until_time (str, optional): Timestamp or relative seconds"
+    " to search until.\n"
+    "- Output: Matching lines with context."
+)
+
+
+class SearchLogsTool(ArtemisTool):
+    """Universal tool for searching logs for keywords or regex patterns."""
+
+    def __init__(self, category: ToolCategory = "diagnostic"):
+        super().__init__(
+            name="search_logs",
+            description=SEARCH_LOGS_DOCSTRING,
+            args_schema=SearchLogsArgs,
+            category=category,
+        )
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,  # pylint: disable=unused-argument
+        ctx: ArtemisContext | None = None,
+        keyword: str = "",
+        is_regex: bool = False,
+        lines: int | None = 10000,
+        since_time: str | None = None,
+        until_time: str | None = None,
+        context_lines: int | None = 0,
+        tool_call_id: str | None = None,  # pylint: disable=unused-argument
+        state: State | None = None,  # pylint: disable=unused-argument
+        **kwargs: Any,
+    ) -> str:
+        try:
+            num_lines = 10000 if lines is None else lines
+            ctx_lines = 0 if context_lines is None else context_lines
+            logs = fetch_and_filter_logs(
+                ctx=ctx,
+                lines=num_lines,
+                since_time=since_time,
+                until_time=until_time,
+            )
+            return search_and_merge_logs(
+                logs=logs,
+                keyword=keyword,
+                context_lines=ctx_lines,
+                is_regex=is_regex,
+            )
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.error(f"Failed to search logs: {e}")
+            return ToolFailure(f"Error searching logs: {e}")
+
+
+# Universal tool instance & aliases
+search_logs = SearchLogsTool()
+SearchLogs = SearchLogsTool
+SearchLogsToolAlias = SearchLogsTool
+
+
+def get_search_logs_tool(ctx: ArtemisContext) -> BaseTool:
+    """Exports search_logs as a LangChain BaseTool."""
+    return trace_langchain_tool(search_logs.to_langchain_tool(ctx), ctx)