android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/artemis/agents/log_analyzer/log_analyzer.py

@@ -0,0 +1,403 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Annotated, Any
+from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
+from langchain_core.tools import BaseTool, StructuredTool
+from pydantic import BaseModel, Field
+from artemis.utils.cython_compat import CyFunctionDetector
+
+
+class SpawnLogReaderArgs(BaseModel):
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    specific_query: str = Field(
+        ...,
+        description=(
+            "Specific instructions for the sub-agent, including what to look for and analyze."
+        ),
+    )
+
+
+from langgraph.prebuilt import InjectedState
+
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import (
+    CURRENT_TRACE_ID,
+    TraceSpan,
+    trace,
+)
+from artemis.graph.state import State
+from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
+from artemis.tools.index import get_tool_by_name
+from artemis.tools.mobile.log_utils import fetch_and_filter_logs
+from artemis.tools.mobile.read_logs import get_read_logs_tool
+from artemis.tools.mobile.search_logs import (
+    get_search_logs_tool,
+    search_and_merge_logs,
+)
+from artemis.tools.tool_wrapper import (
+    get_tool_result_content,
+    invoke_tool_with_injection,
+)
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class LogAnalyzerNode:
+    def __init__(self, ctx: ArtemisContext):
+        self.ctx = ctx
+        self._log_cache = {}
+        self._raw_log_cache = {}
+
+    def _wrap_with_caching(self, tool: BaseTool) -> BaseTool:
+        """Wraps a tool with caching logic based on tool name and arguments."""
+        original_coroutine = tool.coroutine
+
+        async def cached_coroutine(**kwargs):
+            if tool.name == "read_logs":
+                # Intercept read_logs to cache raw logs
+                lines = kwargs.get("lines", 200)
+                # Since we might have since_time/until_time, let's keep it simple or key by all.
+                # The instruction says: "cache the returned string based on the lines parameter."
+                # We can also include since_time and until_time in the cache key to be safe, but let's follow the lines-based requirement.
+                cache_key = (
+                    lines,
+                    kwargs.get("since_time"),
+                    kwargs.get("until_time"),
+                )
+                if cache_key in self._raw_log_cache:
+                    logger.info(f"Raw log cache hit for read_logs with key {cache_key}")
+                    return self._raw_log_cache[cache_key]
+
+                result = await original_coroutine(**kwargs)
+                if isinstance(result, str) and not is_tool_failure(result):
+                    self._raw_log_cache[cache_key] = result
+                return result
+
+            elif tool.name == "search_logs":
+                lines = kwargs.get("lines", 10000)
+                since_time = kwargs.get("since_time")
+                until_time = kwargs.get("until_time")
+                raw_cache_key = (lines, since_time, until_time)
+
+                # Check if we have the raw logs cached
+                if raw_cache_key in self._raw_log_cache:
+                    logger.info(
+                        "Raw log cache hit for search_logs with key"
+                        f" {raw_cache_key}. Running local search."
+                    )
+                    raw_logs = self._raw_log_cache[raw_cache_key]
+                    # Run search_and_merge_logs locally
+
+                    try:
+                        return search_and_merge_logs(
+                            logs=raw_logs,
+                            keyword=kwargs.get("keyword"),
+                            context_lines=kwargs.get("context_lines", 0),
+                            is_regex=kwargs.get("is_regex", False),
+                        )
+                    except Exception as e:
+                        logger.error(f"Failed to search logs locally: {e}")
+                        return ToolFailure(f"Error searching logs: {e}")
+
+                # If not cached, fetch them once via read_logs (or the original tool, which will fetch them)
+                # Note: search_logs itself fetches and filters. We can let the original_coroutine run,
+                # but to cache the raw logs, we need the raw logs.
+                # Wait, search_logs returns the SEARCHED results, not the raw logs.
+                # So if we just run original_coroutine, we don't get the raw logs to cache them!
+                # To cache them, we should fetch them using get_read_logs_tool or fetch_and_filter_logs,
+                # cache them, and then run search_and_merge_logs.
+                logger.info(
+                    "Raw log cache miss for search_logs with key"
+                    f" {raw_cache_key}. Fetching raw logs first."
+                )
+
+                try:
+                    raw_logs = fetch_and_filter_logs(
+                        ctx=self.ctx,
+                        lines=lines,
+                        since_time=since_time,
+                        until_time=until_time,
+                    )
+                    self._raw_log_cache[raw_cache_key] = raw_logs
+
+                    return search_and_merge_logs(
+                        logs=raw_logs,
+                        keyword=kwargs.get("keyword"),
+                        context_lines=kwargs.get("context_lines", 0),
+                        is_regex=kwargs.get("is_regex", False),
+                    )
+                except Exception as e:
+                    logger.error(f"Failed to fetch and search logs: {e}")
+                    return ToolFailure(f"Error searching logs: {e}")
+
+            # Fallback to standard tool caching for other tools
+            serialized_args = json.dumps(kwargs, sort_keys=True)
+            cache_key = f"{tool.name}:{serialized_args}"
+            if cache_key in self._log_cache:
+                logger.info(f"Cache hit for tool {tool.name} with args {kwargs}")
+                return self._log_cache[cache_key]
+
+            logger.info(
+                f"Cache miss for tool {tool.name} with args {kwargs}. Invoking original tool."
+            )
+            result = await original_coroutine(**kwargs)
+            self._log_cache[cache_key] = result
+            return result
+
+        return StructuredTool.from_function(
+            coroutine=cached_coroutine,
+            name=tool.name,
+            description=tool.description,
+            args_schema=tool.args_schema,
+        )
+
+    async def _run_agent_loop(
+        self,
+        base_llm: Any,
+        tools_for_binding: list[Any],
+        tools: list[BaseTool],
+        current_messages: list[Any],
+        state: State,
+        max_iterations: int = 10,
+        agent_name: str = "Agent",
+    ) -> str:
+        outcome = "No result"
+
+        for i in range(max_iterations):
+            if i == max_iterations - 1:
+                current_messages.append(
+                    HumanMessage(
+                        content=(
+                            "[WARNING] This is your final iteration; all tools"
+                            " are stripped, and you must provide your final"
+                            " answer/summary directly."
+                        )
+                    )
+                )
+                llm = base_llm.bind_tools(tools=[])
+            else:
+                if i > 0:
+                    current_messages.append(
+                        HumanMessage(
+                            content=(
+                                "[WARNING] You have not completed the analysis"
+                                f" yet (iteration {i + 1} of {max_iterations})."
+                            )
+                        )
+                    )
+                llm = base_llm.bind_tools(tools=tools_for_binding)
+
+            response = await invoke_llm_with_timeout_message(acomplete(llm, current_messages))
+
+            if response.content:
+                outcome = response.content
+
+            if not response.tool_calls:
+                break
+
+            if i == max_iterations - 1:
+                tool_names = ", ".join(tc["name"] for tc in response.tool_calls)
+                warning = (
+                    f"\n\n[Warning: {agent_name} reached the maximum iteration"
+                    f" limit of {max_iterations} and could not execute tool(s):"
+                    f" {tool_names}]"
+                )
+                if outcome == "No result":
+                    outcome = (
+                        f"{agent_name} reached the maximum iteration limit of"
+                        f" {max_iterations} while attempting to call tools:"
+                        f" {tool_names}"
+                    )
+                else:
+                    outcome += warning
+                break
+
+            current_messages.append(response)
+
+            # Handle tool calls in parallel
+            async def run_tool(tc) -> ToolMessage:
+                tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
+                logger.info(f"{agent_name} requested tool: {tool_name}")
+
+                result = ""
+                status = "success"
+                try:
+                    if ":" in tool_name:
+                        tool_to_run = get_tool_by_name(tool_name, tools)
+                    else:
+                        tool_to_run = next((t for t in tools if t.name == tool_name), None)
+                    if tool_to_run:
+                        args = dict(tc["args"])
+                        with TraceSpan(name=tool_name, ctx=self.ctx) as span:
+                            span.payload = {"args": args}
+                            try:
+                                result_obj = await invoke_tool_with_injection(
+                                    tool=tool_to_run,
+                                    args=args,
+                                    tool_call_id=tc["id"],
+                                    state=state,
+                                )
+                                result = get_tool_result_content(result_obj)
+                                if isinstance(result, list):
+                                    result = "\n".join(map(str, result))
+                                elif not isinstance(result, str):
+                                    result = str(result)
+                                span.result = result
+                                if is_tool_failure(result_obj):
+                                    status = "error"
+                                    span.status = "failed"
+                                    span.error = result
+                            except Exception as e:
+                                status = "error"
+                                span.status = "failed"
+                                span.error = str(e)
+                                raise e
+                    else:
+                        result = f"Error: Tool {tool_name} not supported"
+                        status = "error"
+                except Exception as e:
+                    logger.error(f"Error running tool {tool_name}: {e}")
+                    result = f"Error running tool {tool_name}: {e}"
+                    status = "error"
+
+                return ToolMessage(
+                    tool_call_id=tc["id"],
+                    content=result,
+                    status=status,
+                )
+
+            active_tool_calls = [
+                tc
+                for tc in response.tool_calls
+                if (tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"])
+                != "google_search"
+            ]
+            tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in active_tool_calls))
+
+            for tm in tool_outputs:
+                current_messages.append(tm)
+
+        return outcome
+
+    @trace(type="agent", name="log_analyzer")
+    async def run(self, prompt: str, state: State) -> str:
+        """Runs the main Log Analyzer and returns its summary."""
+        prompt_path = Path(__file__).parent.joinpath("log_analyzer.md")
+        if not prompt_path.exists():
+            raise FileNotFoundError(f"System prompt file not found at {prompt_path}")
+
+        system_prompt = prompt_path.read_text(encoding="utf-8")
+
+        llm = get_llm(ctx=self.ctx, name="log_analyzer")
+
+        all_tools = [
+            self._wrap_with_caching(get_read_logs_tool(self.ctx)),
+            self._wrap_with_caching(get_search_logs_tool(self.ctx)),
+            self._get_spawn_log_reader_tool(),
+        ]
+
+        current_messages = [
+            SystemMessage(content=system_prompt),
+            HumanMessage(content=prompt),
+        ]
+
+        return await self._run_agent_loop(
+            base_llm=llm,
+            tools_for_binding=all_tools,
+            tools=all_tools,
+            current_messages=current_messages,
+            state=state,
+            max_iterations=10,
+            agent_name="Log Analyzer",
+        )
+
+    @trace(type="agent", name="log_reader_sub_agent")
+    async def _run_log_reader(self, specific_query: str, state: State) -> str:
+        """Runs the Log Reader Sub-Agent and returns its summary."""
+        sub_prompt_path = Path(__file__).parent.joinpath("log_reader_sub_agent.md")
+        if not sub_prompt_path.exists():
+            sub_prompt_path.write_text(
+                "# Log Reader Sub-Agent\nYou are a specialized agent for"
+                " reading and analyzing Android logs.\nYour job is to fulfill"
+                " the specific prompt given by the Main Agent.\nYou have access"
+                " to tools: `read_logs`.\nAnalyze the logs carefully and"
+                " provide a clear, structured summary of your findings.\n",
+                encoding="utf-8",
+            )
+
+        sub_system_prompt = sub_prompt_path.read_text(encoding="utf-8")
+
+        llm = get_llm(ctx=self.ctx, name="log_reader_sub_agent")
+
+        sub_tools = [self._wrap_with_caching(get_read_logs_tool(self.ctx))]
+
+        # Extract the list of keys (metadata) currently stored in _log_cache.
+        cache_keys = list(self._log_cache.keys())
+
+        current_messages = [
+            SystemMessage(content=sub_system_prompt),
+        ]
+
+        # Append a system message listing the available cache keys so the sub-agent knows what data can be retrieved instantly.
+        if cache_keys:
+            current_messages.append(
+                SystemMessage(
+                    content=(
+                        "The following log reads are already cached and can be"
+                        " retrieved instantly without cost:\n"
+                    )
+                    + "\n".join(f"- {k}" for k in cache_keys)
+                )
+            )
+
+        current_messages.append(HumanMessage(content=specific_query))
+
+        return await self._run_agent_loop(
+            base_llm=llm,
+            tools_for_binding=sub_tools,
+            tools=sub_tools,
+            current_messages=current_messages,
+            state=state,
+            max_iterations=10,
+            agent_name="Log Reader Sub-Agent",
+        )
+
+    def _get_spawn_log_reader_tool(self) -> BaseTool:
+
+        async def spawn_log_reader(
+            specific_query: str,
+            state: Annotated[State, InjectedState] = None,
+        ) -> str:
+            """[SPAWN SUB-AGENT] Delegates complex log analysis to a sub-agent.
+
+            - Use when: Task requires deep analysis, cross-referencing, or
+            reading huge log volumes. - Note: The sub-agent has no search tools
+            and can only read raw logs. - Input: specific_query (str):
+            Instructions for the sub-agent. - Output: A structured summary
+            report from the sub-agent.
+            """
+            logger.info(f"Spawning Log Reader Sub-Agent with query: '{specific_query}'")
+            return await self._run_log_reader(specific_query, state)
+
+        return StructuredTool.from_function(
+            coroutine=spawn_log_reader,
+            name="spawn_log_reader",
+            args_schema=SpawnLogReaderArgs,
+        )

package/artemis-main/artemis/agents/log_analyzer/log_reader_sub_agent.md

@@ -0,0 +1,8 @@
+# Log Deep Analysis Instructions
+
+Fulfill the specific log analysis prompt by reading and synthesizing Android logs.
+
+## Guidelines
+1. **Direct Answer**: Address the exact questions or patterns requested in the prompt.
+2. **Synthesis**: Do not output raw, unsorted logs. Provide a structured summary of findings.
+3. **Precision**: Include relevant timestamps, line numbers, and error codes to support your conclusions.

package/artemis-main/artemis/agents/log_analyzer/output_analyzer.md

@@ -0,0 +1,11 @@
+# Task Output Analyzer
+
+You are a specialized agent for analyzing the stdout/stderr output of a command executed on an Android device.
+The user (another agent) will ask you a question about the command's execution output.
+Analyze the output carefully to answer their question.
+
+You have access to the following tools to read and search the full output:
+- `read_task_output`: Reads a range of lines from the output.
+- `search_task_output`: Searches the output for a keyword or regex.
+
+Be concise, accurate, and direct in your answer.

package/artemis-main/artemis/agents/log_analyzer/output_analyzer.py

@@ -0,0 +1,163 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+from typing import Annotated
+from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
+from langchain_core.tools import tool
+from artemis.core.tool_failure import ToolFailure, is_tool_failure
+from artemis.context import ArtemisContext
+from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
+from artemis.tools.mobile.search_logs import search_and_merge_logs
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class TaskOutputAnalyzerNode:
+    """Agent that analyzes large command/task execution outputs using LLM and tools."""
+
+    def __init__(self, ctx: ArtemisContext):
+        self.ctx = ctx
+
+    async def run(
+        self,
+        command: str,
+        output_text: str,
+        query: str,
+    ) -> str:
+        prompt_path = Path(__file__).parent.joinpath("output_analyzer.md")
+        if not prompt_path.exists():
+            raise FileNotFoundError(f"System prompt file not found at {prompt_path}")
+
+        system_prompt = prompt_path.read_text(encoding="utf-8")
+
+        # Get the LLM for output_analyzer (falls back to log_analyzer)
+        llm = get_llm(ctx=self.ctx, name="output_analyzer")
+
+        # Local tools created with closure on output_text
+        @tool
+        def read_task_output(
+            start_line: Annotated[int, "The 1-indexed line number to start reading from."],
+            end_line: Annotated[int, "The 1-indexed line number to stop reading at (inclusive)."],
+        ) -> str:
+            """Reads a specific range of lines from the task output.
+
+            Use this tool when you know the line number range to read a specific
+            portion of the output.
+            """
+            lines = output_text.splitlines()
+            if not lines:
+                return "Task output is empty."
+            start = max(1, start_line)
+            end = min(len(lines), end_line)
+            if start > end:
+                return (
+                    f"Error: start_line {start_line} is greater than end_line"
+                    f" {end_line} or task output length {len(lines)}."
+                )
+            segment = lines[start - 1 : end]
+            return "\n".join(segment)
+
+        @tool
+        def search_task_output(
+            keyword: Annotated[
+                str,
+                "The keyword or regular expression to search for in the output.",
+            ],
+            is_regex: Annotated[
+                bool,
+                "Whether to treat the keyword as a regular expression. Default is False.",
+            ] = False,
+            context_lines: Annotated[
+                int,
+                "Number of context lines to include before and after the"
+                " matched line. Default is 5.",
+            ] = 5,
+        ) -> str:
+            """Searches the task output for a keyword or regular expression, returning matching lines and context.
+
+            Use this tool to quickly locate error messages, warnings, or
+            specific key-value pairs in large outputs.
+            """
+            return search_and_merge_logs(
+                logs=output_text,
+                keyword=keyword,
+                context_lines=context_lines,
+                is_regex=is_regex,
+            )
+
+        tools = [read_task_output, search_task_output]
+        llm = llm.bind_tools(tools=tools)
+
+        # Optimize prompt context with preview if output is large
+        lines = output_text.splitlines()
+        if len(lines) <= 400:
+            log_preview = f"Full Output:\n{output_text}"
+        else:
+            first_part = "\n".join(lines[:200])
+            last_part = "\n".join(lines[-200:])
+            log_preview = (
+                f"Note: Output is very large ({len(lines)} lines). Here is a"
+                " preview of the start and end of the output:\n\n--- START OF"
+                f" OUTPUT ---\n{first_part}\n--- TRUNCATED {len(lines) - 400}"
+                f" LINES ---\n--- END OF OUTPUT ---\n{last_part}\n\nYou can use"
+                " `read_task_output` or `search_task_output` to fetch any"
+                " lines in the middle if needed."
+            )
+
+        user_content = f"Command Executed: {command}\n\n{log_preview}\n\nQuestion: {query}"
+
+        current_messages = [
+            SystemMessage(content=system_prompt),
+            HumanMessage(content=user_content),
+        ]
+
+        max_iterations = 5
+        outcome = "Failed to analyze output."
+
+        for _ in range(max_iterations):
+            response = await invoke_llm_with_timeout_message(llm.ainvoke(current_messages))
+
+            if response.content:
+                outcome = response.content
+
+            if not response.tool_calls:
+                break
+
+            current_messages.append(response)
+
+            for tc in response.tool_calls:
+                tool_name = tc["name"]
+                args = tc["args"]
+                logger.info(f"Output Analyzer requested tool: {tool_name}")
+
+                tool_to_run = next((t for t in tools if t.name == tool_name), None)
+                if tool_to_run:
+                    try:
+                        result = tool_to_run.invoke(args)
+                    except Exception as e:
+                        result = ToolFailure(f"Failed to run tool {tool_name}: {e}")
+                else:
+                    result = ToolFailure(f"Error: Tool {tool_name} not found.")
+
+                current_messages.append(
+                    ToolMessage(
+                        tool_call_id=tc["id"],
+                        content=str(result),
+                        status="error" if is_tool_failure(result) else "success",
+                    )
+                )
+
+        return outcome

package/artemis-main/artemis/agents/object_detector/object_detector.json

@@ -0,0 +1,9 @@
+{
+  "templates": [
+    "Point to the following objects in the provided image: {labels_str}.",
+    "Detect the location of the following objects in the image: {labels_str}.",
+    "Locate the following UI components on the screen: {labels_str}."
+  ],
+  "global_timeout": 30.0,
+  "instructions": "Return a JSON list containing the detected objects. Each object must have a 'point' array [y, x] (normalized to 0-1000) and a 'label' string.\n\nExample output:\n[\n  {\"point\": [500, 250], \"label\": \"search icon\"}\n]\n\nIf no objects are found, return an empty list []."
+}