android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/agents/history_analyzer/history_analyzer.py

@@ -0,0 +1,182 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""History Analyzer: answers natural-language questions about the session history.
+
+Mounts the shared history tools (``search_history`` / ``replay_steps`` /
+``get_step_screenshot``) plus the read-only note tools; tool results enter the
+conversation through ``tool_result_messages`` so a fetched screenshot travels
+in the carrier the model's provider accepts.
+"""
+
+from pathlib import Path
+
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from langchain_core.tools import BaseTool
+
+from artemis.core.tool_failure import is_tool_failure
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import TraceSpan, trace
+from artemis.memory.context_policy import build_history_for
+from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
+from artemis.tools.history import get_history_tools
+from artemis.tools.scratchpad import (
+    get_list_notes_tool_pure,
+    get_read_note_tool_pure,
+)
+from artemis.tools.tool_wrapper import (
+    invoke_tool_with_injection,
+    split_multimodal_result,
+    tool_result_messages,
+)
+from artemis.utils.notes import get_note_file_path
+from artemis.utils.task_tree import get_active_subgoal_hashes
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+_FALLBACK_SYSTEM_PROMPT = (
+    "You are a History Analyzer. Your role is to analyze the execution history"
+    " of a session and answer user queries in natural language.\nIf you need"
+    " the full record of specific steps, use the `replay_steps` tool; use"
+    " `search_history` to locate steps by keyword and `get_step_screenshot` to"
+    " look at a step's screen."
+)
+
+
+class HistoryAnalyzer:
+    def __init__(self, ctx: ArtemisContext):
+        self.ctx = ctx
+
+    def _build_tools(self) -> list[BaseTool]:
+        return [
+            *get_history_tools(self.ctx),
+            get_list_notes_tool_pure(self.ctx),
+            get_read_note_tool_pure(self.ctx),
+        ]
+
+    @trace(type="agent", name="history_analyzer")
+    async def run(self, query: str) -> str:
+        if not self.ctx.data_engine:
+            return "Error: DataEngine is not available to retrieve history."
+
+        # 1. Fetch all steps from Data Engine
+        history_steps = self.ctx.data_engine.get_agent_friendly_steps()
+        if not history_steps:
+            return "No history recorded for this session yet."
+
+        # 2. Build the plan and history or step list
+        plan_and_history = ""
+        try:
+            current_plan = ""
+            current_path = get_note_file_path(self.ctx.data_engine.base_dir, "task_plan")
+            if current_path.exists():
+                current_plan = current_path.read_text(encoding="utf-8")
+
+            if current_plan:
+                active_subgoal_hash, _ = get_active_subgoal_hashes(current_plan)
+                plan_and_history = build_history_for(
+                    "history_analyzer",
+                    current_plan,
+                    history_steps,
+                    active_subgoal_hash,
+                    engine=self.ctx.data_engine,
+                )
+        except Exception as e:
+            logger.error(f"Failed to build plan and history in HistoryAnalyzer: {e}")
+
+        if not plan_and_history:
+            # Fallback to a simple list
+            plan_and_history = "\n".join(
+                [
+                    f"- Step {s.get('step_number')} ({s.get('relative_time')}):"
+                    f" {s.get('summary') or 'No summary'}"
+                    for s in history_steps
+                ]
+            )
+
+        # 3. Load system prompt
+        prompt_path = Path(__file__).parent.joinpath("history_analyzer.md")
+        if prompt_path.exists():
+            system_prompt_template = prompt_path.read_text(encoding="utf-8")
+        else:
+            system_prompt_template = _FALLBACK_SYSTEM_PROMPT
+
+        system_message_content = (
+            system_prompt_template + f"\n\n### Task Plan and Execution History:\n{plan_and_history}"
+        )
+
+        # 4. Prepare LLM and bind tools
+        llm = get_llm(ctx=self.ctx, name="history_analyzer")
+        tools = self._build_tools()
+        llm = llm.bind_tools(tools=tools)
+
+        # 5. ReAct Loop
+        messages: list[BaseMessage] = [
+            SystemMessage(content=system_message_content),
+            HumanMessage(content=query),
+        ]
+
+        max_turns = 5
+        for turn in range(max_turns):
+            logger.info(f"HistoryAnalyzer ReAct turn {turn + 1}")
+
+            response = await invoke_llm_with_timeout_message(acomplete(llm, messages))
+            messages.append(response)
+
+            if not response.tool_calls:
+                return (
+                    response.content.strip()
+                    if isinstance(response.content, str)
+                    else str(response.content)
+                )
+
+            # Process tool calls
+            for tc in response.tool_calls:
+                tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
+                args = dict(tc.get("args") or {})
+                tool = next((t for t in tools if t.name == tool_name), None)
+
+                with TraceSpan(name=tool_name, ctx=self.ctx) as span:
+                    span.payload = {"args": args}
+                    if tool is None:
+                        result = f"Error: Tool {tool_name} is not supported."
+                        status = "error"
+                        span.status = "failed"
+                        span.error = result
+                    else:
+                        logger.info(f"HistoryAnalyzer executing tool {tool_name} with args: {args}")
+                        try:
+                            result = await invoke_tool_with_injection(
+                                tool=tool, args=args, tool_call_id=tc["id"]
+                            )
+                            text, _ = split_multimodal_result(result)
+                            status = "error" if is_tool_failure(result) else "success"
+                            span.result = text
+                            if status == "error":
+                                span.status = "failed"
+                                span.error = text
+                        except Exception as e:
+                            span.status = "failed"
+                            span.error = str(e)
+                            result = f"Error running {tool_name}: {e}"
+                            status = "error"
+
+                # A step screenshot travels in the carrier the model's provider
+                # accepts; text results stay a single ToolMessage.
+                messages.extend(
+                    tool_result_messages(tc["id"], result, name=tool_name, status=status, llm=llm)
+                )
+
+        return "Error: HistoryAnalyzer failed to resolve the query within maximum turns."

package/artemis-main/artemis/agents/image_processor/image_processor.md

@@ -0,0 +1,74 @@
+# IDENTITY
+You are the expert UI Image Processor, a coding agent that can write Python scripts to analyze and modify images (like color filtering, cropping, highlighting, coloring, or performing custom image processing).
+
+# CONTEXT
+Target Image ID: `img_0` (This is the original input screenshot)
+Instruction: {instruction}
+Intermediate Artifact Save Path: `{intermediate_artifact_save_path}`
+If you want to save intermediate steps to inspect them or load them later, you must call `canvas.save(final=False)`. Any intermediate images saved this way will be automatically loaded and fed back to you as visual context at the beginning of your next turn, and can be retrieved in subsequent runs by instantiating a new canvas: `ImageCanvas(image_id, '{intermediate_artifact_save_path}')`.
+
+
+# GOAL
+Your goal is to perform image analysis, filter, crop, draw annotations/markers, or modify the image to visually highlight or isolate specific features as requested. Write a Python script using OpenCV (`cv2`), `numpy`, or other standard libraries to analyze and modify the target image `img_0` according to the `Instruction`. The final output should be the modified image. Call `submit_result` tool after you have successfully completed the task.
+
+# CRITICAL BOUNDARIES FOR GEOMETRIC TRANSFORMATIONS
+1. **DO NOT** use numpy array slicing (e.g. `img[y:y+h, x:x+w]`) to crop the image.
+2. **DO NOT** use `cv2.resize()` to scale the image.
+Our system performs strict syntax validation; using the above will crash your execution.
+
+If you MUST crop or scale the image to isolate small details or improve reading/detection downstream, you MUST use the pre-imported `ImageCanvas` tool exactly as follows:
+
+### Snippet 1: Saving & Resuming Intermediate Steps (Iterative Workflow)
+Use this to crop/zoom into specific areas and save them so you can visually inspect them in your next turn, or load them back to create new branches.
+```python
+from artemis.utils.cv_canvas import ImageCanvas
+
+# 1. Start from the original image:
+canvas = ImageCanvas("img_0", "{intermediate_artifact_save_path}")
+
+# 2. Crop and scale:
+canvas.crop(x=100, y=200, w=300, h=300)
+canvas.resize_by_factor(2.0)
+
+# 3. Save as an intermediate step for verification:
+# This will auto-generate 'img_1' and save it in the intermediate directory.
+# In the next turn, you will receive 'img_1' as visual confirmation.
+canvas.save(final=False)
+
+# --- In a subsequent turn, if you want to resume from 'img_1' ---
+# canvas = ImageCanvas('img_1', '{intermediate_artifact_save_path}')
+```
+
+### Snippet 2: Finalizing and Saving the Output
+Use this to draw your final annotations, highlights, or markers, and save the final result to be returned.
+```python
+from artemis.utils.cv_canvas import ImageCanvas
+
+# 1. Load the original or a cropped intermediate step to finalize:
+canvas = ImageCanvas("img_1", "{intermediate_artifact_save_path}")
+
+# 2. Draw annotations, highlight elements, or apply filters:
+# # Convert normalized coordinates (0-1000) to pixel coordinates:
+# x, y = canvas.normalized_to_pixel_coords(nx, ny)
+# Draw specific verification dots and register their labels with unique label_idx (e.g. V1):
+# canvas.draw_dot(100, 200, 1)
+
+# 3. Save the final result (this is required to complete the task):
+# This will auto-generate the next sequential image ID (e.g., 'img_2') and mark it as a final output.
+canvas.save(final=True)
+```
+
+### Snippet 3: Simple Modification Without Cropping or Scaling
+If you do NOT need to crop or scale, but just want to draw annotations or apply filters, you can still use `ImageCanvas` for consistency:
+```python
+from artemis.utils.cv_canvas import ImageCanvas
+
+canvas = ImageCanvas("img_0", "{intermediate_artifact_save_path}")
+# ... modify canvas.img ...
+canvas.save(final=True)
+```
+
+# EXECUTION LOOP
+1. Write and execute your script using the `execute_python` tool.
+2. Read the stdout/stderr. If there are errors, fix your code and execute again.
+3. Once you have successfully called `canvas.save(final=True)` on your final output image(s), call the `submit_result` tool with a summary to finish your job.

package/artemis-main/artemis/agents/image_processor/image_processor.py

@@ -0,0 +1,322 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal Multi-Model Image Processor for Artemis."""
+
+import base64
+import json
+from pathlib import Path
+import shutil
+import time
+import uuid
+
+from langchain_core.messages import (
+    BaseMessage,
+    HumanMessage,
+    SystemMessage,
+    ToolMessage,
+)
+
+from artemis.config import settings
+from artemis.context import ArtemisContext
+from artemis.core.tool_declaration import ToolDeclaration
+from artemis.data_engine.trace import TraceSpan, trace
+from artemis.services.llm import get_llm
+from artemis.utils.python_executor import PythonExecutor
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+EXECUTE_PYTHON_TOOL = ToolDeclaration(
+    name="execute_python",
+    description=(
+        "Executes the provided Python code within an isolated"
+        " Jupyter environment and returns the resulting stdout,"
+        " stderr, and traceback outputs."
+    ),
+    parameters={
+        "type": "object",
+        "properties": {
+            "code": {
+                "type": "string",
+                "description": "The Python code to execute",
+            }
+        },
+        "required": ["code"],
+    },
+)
+
+SUBMIT_RESULT_TOOL = ToolDeclaration(
+    name="submit_result",
+    description=("Submits a summary of the modifications to successfully complete the task."),
+    parameters={
+        "type": "object",
+        "properties": {
+            "summary": {
+                "type": "string",
+                "description": "Summary of what you found or changed.",
+            }
+        },
+        "required": ["summary"],
+    },
+)
+
+
+class ImageProcessor:
+    def __init__(self, ctx: ArtemisContext):
+        self.ctx = ctx
+
+    @trace(type="agent", name="image_processor")
+    async def run(self, instruction: str, target_image_path: str) -> dict:
+        try:
+            llm = get_llm(self.ctx, name="image_processor")
+        except Exception:
+            llm = get_llm(self.ctx, name="operator")
+
+        base_dir = settings.TRACES_PATH
+        image_processor_dir = base_dir / "images" / "image_processor"
+        image_processor_dir.mkdir(parents=True, exist_ok=True)
+        intermediate_artifact_save_path = image_processor_dir / f"intermediates_{int(time.time())}"
+        intermediate_artifact_save_path.mkdir(parents=True, exist_ok=True)
+        intermediate_artifact_save_path_str = str(intermediate_artifact_save_path.resolve())
+
+        max_iterations = 5
+
+        prompt_path = Path(__file__).parent / "image_processor.md"
+        prompt_template = prompt_path.read_text(encoding="utf-8")
+        prompt_template = (
+            prompt_template.replace("{target_image_path}", target_image_path)
+            .replace("{instruction}", instruction)
+            .replace(
+                "{intermediate_artifact_save_path}",
+                intermediate_artifact_save_path_str,
+            )
+            .replace("{max_iterations}", str(max_iterations))
+        )
+
+        tools_declaration = [EXECUTE_PYTHON_TOOL, SUBMIT_RESULT_TOOL]
+
+        with open(target_image_path, "rb") as f:
+            img_bytes = f.read()
+        img_b64 = base64.b64encode(img_bytes).decode("utf-8")
+
+        messages: list[BaseMessage] = [
+            SystemMessage(content=prompt_template),
+            HumanMessage(
+                content=[
+                    {"type": "text", "text": "Here is the target image. Begin writing your code."},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"},
+                    },
+                ]
+            ),
+        ]
+
+        img_0_path = intermediate_artifact_save_path / "img_0.jpg"
+        try:
+            shutil.copy2(target_image_path, img_0_path)
+        except Exception as e:
+            logger.error(f"Failed to copy original screenshot to intermediate pool: {e}")
+            img_0_path = Path(target_image_path)
+
+        self.intermediate_image_pool = {
+            "img_0": {
+                "image_id": "img_0",
+                "path": str(img_0_path.resolve()),
+                "turn_id": 0,
+                "transform": {
+                    "offset_x": 0.0,
+                    "offset_y": 0.0,
+                    "scale_x": 1.0,
+                    "scale_y": 1.0,
+                },
+                "annotations": {},
+            }
+        }
+
+        pool_json_path = intermediate_artifact_save_path / "intermediate_transforms.json"
+        sync_pool = {
+            "img_0": {
+                "image_id": "img_0",
+                "path": self.intermediate_image_pool["img_0"]["path"],
+                "transform": self.intermediate_image_pool["img_0"]["transform"],
+                "annotations": {},
+            }
+        }
+        with open(pool_json_path, "w") as f:
+            json.dump(sync_pool, f)
+
+        executor = PythonExecutor(session_dir=image_processor_dir)
+        iterations = 0
+        final_result = None
+
+        try:
+            while iterations < max_iterations:
+                iterations += 1
+                logger.info(f"ImageProcessor Iteration {iterations}")
+
+                # Inject newly created intermediate images from last turn
+                if self.intermediate_image_pool:
+                    image_blocks = []
+                    last_turn_id = iterations - 1
+                    for img_id, entry in self.intermediate_image_pool.items():
+                        if entry["turn_id"] == last_turn_id:
+                            try:
+                                path = Path(entry["path"])
+                                if path.exists():
+                                    b64_data = base64.b64encode(path.read_bytes()).decode("utf-8")
+                                    is_output = entry.get("is_output", False)
+                                    label = (
+                                        f"img_id: {img_id} (Output)"
+                                        if is_output
+                                        else f"img_id: {img_id}"
+                                    )
+                                    image_blocks.append(
+                                        {
+                                            "type": "image_url",
+                                            "image_url": {
+                                                "url": f"data:image/jpeg;base64,{b64_data}"
+                                            },
+                                        }
+                                    )
+                                    image_blocks.append({"type": "text", "text": f"\n{label}\n"})
+                            except Exception as e:
+                                logger.error(
+                                    f"Failed to read intermediate image {entry['path']}: {e}"
+                                )
+
+                    if image_blocks:
+                        if (
+                            messages
+                            and isinstance(messages[-1], HumanMessage)
+                            and isinstance(messages[-1].content, list)
+                        ):
+                            messages[-1].content.extend(image_blocks)
+                        else:
+                            messages.append(HumanMessage(content=image_blocks))
+
+                if iterations == max_iterations - 1:
+                    warning_msg = (
+                        "\n[WARNING] This is your next-to-last iteration. You"
+                        " must complete your task and call `submit_result` now"
+                        " or in the next turn, as no further code execution"
+                        " will be allowed after that."
+                    )
+                    messages.append(HumanMessage(content=warning_msg))
+                elif iterations == max_iterations:
+                    warning_msg = (
+                        "\n[WARNING] This is your final iteration. You MUST"
+                        " call `submit_result` now to complete the task. No"
+                        " other tools are available."
+                    )
+                    messages.append(HumanMessage(content=warning_msg))
+
+                current_tools = tools_declaration
+                if iterations == max_iterations:
+                    current_tools = [t for t in tools_declaration if t.name == "submit_result"]
+
+                bound_llm = llm.bind_tools(current_tools)
+
+                with TraceSpan(name="image_processor_call", ctx=self.ctx):
+                    response = await bound_llm.ainvoke(messages)
+
+                tool_calls = response.tool_calls or []
+                if not tool_calls:
+                    break
+
+                messages.append(response)
+
+                submit_called = False
+                for tc in tool_calls:
+                    name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
+                    args = tc.get("args") or {}
+                    tc_id = tc.get("id") or str(uuid.uuid4())
+
+                    if name == "submit_result":
+                        final_result = args
+                        submit_called = True
+                        messages.append(
+                            ToolMessage(
+                                tool_call_id=tc_id,
+                                name=name,
+                                content=json.dumps({"status": "submitted"}),
+                                status="success",
+                            )
+                        )
+                        break
+
+                    elif name == "execute_python":
+                        code = args.get("code", "")
+                        logger.info("Executing Python code via PythonExecutor")
+                        exec_output = executor.execute(code)
+
+                        messages.append(
+                            ToolMessage(
+                                tool_call_id=tc_id,
+                                name=name,
+                                content=exec_output,
+                                status="success",
+                            )
+                        )
+
+                        if pool_json_path.exists():
+                            try:
+                                with open(pool_json_path) as f:
+                                    loaded_pool = json.load(f)
+                                for img_id, entry in loaded_pool.items():
+                                    if img_id not in self.intermediate_image_pool:
+                                        self.intermediate_image_pool[img_id] = {
+                                            "image_id": img_id,
+                                            "path": entry["path"],
+                                            "turn_id": iterations,
+                                            "transform": entry["transform"],
+                                            "is_output": entry.get("is_output", False),
+                                            "annotations": entry.get("annotations", {}),
+                                        }
+                            except Exception as e:
+                                logger.error(f"Failed to read pool_json_path: {e}")
+
+                if submit_called:
+                    break
+
+        finally:
+            executor.close()
+
+        outputs = [
+            entry
+            for entry in self.intermediate_image_pool.values()
+            if entry.get("is_output") is True
+        ]
+
+        if not outputs:
+            sorted_keys = sorted(
+                self.intermediate_image_pool.keys(),
+                key=lambda k: (
+                    int(k.split("_")[1])
+                    if k.startswith("img_") and k.split("_")[1].isdigit()
+                    else -1
+                ),
+            )
+            if sorted_keys:
+                last_key = sorted_keys[-1]
+                outputs = [self.intermediate_image_pool[last_key]]
+
+        summary = (
+            final_result.get("summary", "")
+            if final_result
+            else "Failed to submit result within iterations"
+        )
+
+        return {"outputs": outputs, "summary": summary}

package/artemis-main/artemis/agents/log_analyzer/log_analyzer.md

@@ -0,0 +1,15 @@
+# Log Analysis Instructions
+
+You are the Log Analyzer Agent, responsible for diagnosing issues by querying Android device logs. 
+Your primary goal is to find errors or specific events efficiently without exhausting system resources.
+
+## Core Strategy: Separation of Search and Read (Grep-like Behavior)
+You MUST avoid reading massive chunks of logs. Instead, use a "Grep-then-Read" approach:
+1. **Precision Search (Grep)**: Always start by using `search_logs` with `context_lines=0`. Search for specific keywords, package names (e.g., `net.osmand`), or Exception tags. This will return the exact line numbers and timestamps of the events.
+2. **Targeted Reading (Read)**: Once you find a highly relevant match from your search (e.g., an error stack trace start), use `read_logs` and specify the `since_time` and `until_time` to fetch ONLY the small 20-50 line window around that exact event. 
+
+## Guidelines
+1. **Efficiency**: Never call `search_logs` with `context_lines` > 5 unless you are absolutely sure there are very few matches. Large context windows will cause the system to truncate your output and you will lose data.
+2. **Unknown Errors**: Use `google_search` for unfamiliar error codes or system tags.
+3. **Handling Missing Data**: If you have executed several searches and cannot find any matching logs, DO NOT loop indefinitely. Return a final summary stating that no logs matching the criteria were found.
+4. **Response Format**: You should provide your final log analysis in one paragraph. You can organize it into clear, concise parts (e.g., Search Results, Analysis, Key Events, and more). You should include all helpful technical details (e.g. original logs, timestamps, and more).