android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/artemis/tools/index.py

@@ -0,0 +1,47 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from langchain_core.tools import BaseTool
+
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.tools.tool_wrapper import ToolWrapper
+from third_party.mobile_use.tools.index import (
+    get_tools_from_wrappers as build_tools_from_wrappers,
+)
+
+
+def get_tools_from_wrappers(
+    ctx: "ArtemisContext",
+    wrappers: list[ToolWrapper],
+) -> list[BaseTool]:
+    """Instantiate the available tools of ``wrappers``, traced into the data engine."""
+    return build_tools_from_wrappers(
+        ctx, wrappers, wrap_tool=lambda t: trace_langchain_tool(t, ctx)
+    )
+
+
+def get_tool_by_name(name: str, tools: list[BaseTool]) -> BaseTool | None:
+    """Get a tool by name, stripping any prefixes like 'default_api:'."""
+    normalized_name = name
+    if ":" in normalized_name:
+        normalized_name = normalized_name.split(":")[-1]
+
+    for t in tools:
+        t_name = t.name
+        if ":" in t_name:
+            t_name = t_name.split(":")[-1]
+        if t_name == normalized_name:
+            return t
+    return None

package/artemis-main/artemis/tools/log_tool.py

@@ -0,0 +1,108 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal Log Analyzer tool for inspecting and analyzing Android system logs."""
+
+from typing import Any
+
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field
+
+from artemis.core.tool_failure import ToolFailure
+from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.graph.state import State
+from artemis.tools.base import ArtemisTool
+from artemis.tools.tool_wrapper import ToolWrapper
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+# pylint: disable=too-few-public-methods
+class AnalyzeLogsArgs(BaseModel):
+    """Arguments schema for invoking the Log Analyzer subagent."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    specific_query: str = Field(
+        ...,
+        description=(
+            "Instructions for the log analyzer, specifying what to search for or analyze."
+        ),
+    )
+
+
+class AnalyzeLogsTool(ArtemisTool):
+    """Universal tool for analyzing Android logs using a dedicated hybrid log analyzer agent."""
+
+    def __init__(self):
+        super().__init__(
+            name="analyze_logs",
+            description=(
+                "[DIAGNOSTIC] Analyzes Android logs using a dedicated hybrid log analyzer agent. "
+                "Use this when you need to investigate failures, check for specific error "
+                "logs, or understand system events."
+            ),
+            args_schema=AnalyzeLogsArgs,
+            category="custom",
+        )
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,  # pylint: disable=unused-argument
+        ctx: ArtemisContext | None = None,
+        specific_query: str | None = None,
+        state: State | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        query = (
+            specific_query
+            if specific_query is not None
+            else (
+                kwargs.get("specific_query")
+                or kwargs.get("SpecificQuery")
+                or kwargs.get("query")
+                or kwargs.get("Query")
+                or ""
+            )
+        )
+        st = state if state is not None else kwargs.get("state")
+        logger.info(f"analyze_logs tool called with query: '{query}'")
+        if ctx is None:
+            return ToolFailure("Error: ArtemisContext is required for LogAnalyzer.")
+        analyst = LogAnalyzerNode(ctx)
+        result = await analyst.run(query, st)
+        return result
+
+
+# Universal tool instance & aliases
+analyze_logs = AnalyzeLogsTool()
+AnalyzeLogs = AnalyzeLogsTool
+LogTool = AnalyzeLogsTool
+
+
+def get_analyze_logs_tool(ctx: ArtemisContext) -> BaseTool:
+    """Exports analyze_logs as a LangChain BaseTool."""
+    return trace_langchain_tool(analyze_logs.to_langchain_tool(ctx), ctx)
+
+
+analyze_logs_wrapper = ToolWrapper(
+    tool_fn_getter=get_analyze_logs_tool,
+    on_success_fn=lambda output: f"Log Analyzer replied:\n{output}",
+    on_failure_fn=lambda error: f"Log Analyzer failed: {error}",
+)

package/artemis-main/artemis/tools/mobile/launch_app.py

@@ -0,0 +1,132 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field
+
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.graph.state import State
+from artemis.tools.base import ArtemisTool, ToolCategory
+from artemis.tools.tool_wrapper import ToolWrapper
+from artemis.utils.cython_compat import CyFunctionDetector
+from third_party.mobile_use.tools.mobile.launch_app import LAUNCH_APP_DOCSTRING, find_package
+from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries
+
+
+class LaunchAppArgs(BaseModel):
+    """Arguments schema for launching an application."""
+
+    model_config = {"ignored_types": (CyFunctionDetector,)}
+    app_name: str = Field(
+        ...,
+        description="The natural language name of the application to launch.",
+    )
+
+
+class LaunchAppTool(ArtemisTool):
+    """Universal tool for finding and launching an application on the device."""
+
+    def __init__(self, category: ToolCategory = "action"):
+        super().__init__(
+            name="launch_app",
+            description=LAUNCH_APP_DOCSTRING,
+            args_schema=LaunchAppArgs,
+            category=category,
+        )
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,
+        ctx: ArtemisContext | None = None,
+        app_name: str | None = None,
+        tool_call_id: str | None = None,
+        state: State | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        app = (
+            app_name
+            if app_name is not None
+            else (kwargs.get("app_name") or kwargs.get("AppName") or "")
+        )
+        tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
+        st = state if state is not None else kwargs.get("state")
+
+        success = False
+        error_msg = None
+
+        try:
+            if not app:
+                raise ValueError("app_name parameter is required.")
+
+            if ctx is not None:
+                package_name = await find_package(ctx=ctx, app_name=app)
+                if not package_name:
+                    success = False
+                    outcome = f"Failed to launch app '{app}': Package not found."
+                    error_msg = "Package not found."
+                else:
+                    success, error_msg = await launch_app_with_retries(
+                        ctx=ctx, app_package=package_name
+                    )
+                    outcome = (
+                        f"Launched app '{app}' ({package_name}); foreground confirmed."
+                        if success
+                        else f"Failed to launch app '{app}': {error_msg}"
+                    )
+            elif driver is not None and hasattr(driver, "launch_app"):
+                success = await driver.launch_app(app)
+                outcome = f"Launched app '{app}'." if success else f"Failed to launch app '{app}'."
+                error_msg = None if success else "Launch failed."
+            else:
+                success = False
+                outcome = "Error during launch app: No driver or context provided."
+                error_msg = "No driver or context provided."
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            success = False
+            outcome = f"Error during launch app: {e}"
+            error_msg = str(e)
+
+        if st is not None:
+            additional_kwargs = {} if success else ({"error": error_msg} if error_msg else {})
+            return ToolMessage(
+                tool_call_id=tcid or "",
+                content=outcome,
+                additional_kwargs=additional_kwargs,
+                status="success" if success else "error",
+            )
+
+        return outcome
+
+
+# Universal tool instance & aliases
+launch_app = LaunchAppTool()
+LaunchApp = LaunchAppTool
+
+
+def get_launch_app_tool(ctx: ArtemisContext) -> BaseTool:
+    """Exports launch_app as a LangChain BaseTool."""
+    return trace_langchain_tool(launch_app.to_langchain_tool(ctx), ctx)
+
+
+launch_app_wrapper = ToolWrapper(
+    tool_fn_getter=get_launch_app_tool,
+    on_success_fn=lambda *args, **kwargs: "Launch dispatched",
+    on_failure_fn=lambda *args, **kwargs: "Failure",
+)

package/artemis-main/artemis/tools/mobile/log_utils.py

@@ -0,0 +1,138 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime, timedelta
+import time
+from artemis.context import ArtemisContext
+from third_party.mobile_use.controllers.platform_specific_commands_controller import get_adb_device
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def parse_log_time(time_str: str) -> datetime:
+    """Parses a log timestamp string into a datetime object."""
+    now = datetime.now()
+    current_year = now.year
+
+    def _try_parse(year: int) -> datetime | None:
+        for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"):
+            try:
+                return datetime.strptime(f"{year}-{time_str}", fmt)
+            except ValueError:
+                pass
+        return None
+
+    dt = _try_parse(current_year)
+    if dt is None:
+        raise ValueError(f"Unsupported time format: {time_str}")
+
+    # If parsed time is in the future (allowing 1 hour grace for clock drift),
+    # we assume it belongs to the previous year.
+    if dt > now + timedelta(hours=1):
+        prev_dt = _try_parse(current_year - 1)
+        if prev_dt is not None:
+            dt = prev_dt
+
+    return dt
+
+
+def resolve_time(
+    time_input: str | float | None, session_start: float | None
+) -> tuple[str | None, datetime | None]:
+    """Resolves relative or absolute time input into formatted string and datetime."""
+    if time_input is None:
+        return None, None
+
+    if isinstance(time_input, (int, float)) or (
+        isinstance(time_input, str) and time_input.replace(".", "", 1).isdigit()
+    ):
+        # It's relative seconds
+        if session_start is None:
+            raise ValueError("Cannot resolve relative time without session start time.")
+        rel_sec = float(time_input)
+        abs_ts = session_start + rel_sec
+
+        t_struct = time.localtime(abs_ts)
+        t_str = time.strftime("%m-%d %H:%M:%S", t_struct)
+        ms = int((abs_ts - int(abs_ts)) * 1000)
+        formatted_str = f"{t_str}.{ms:03d}"
+
+        dt = datetime.fromtimestamp(abs_ts)
+        return formatted_str, dt
+
+    # Assume it's absolute time string "MM-DD HH:MM:SS.ms"
+    try:
+        dt = parse_log_time(str(time_input))
+    except ValueError as e:
+        raise ValueError(f"Unsupported time format: {time_input}") from e
+    return str(time_input), dt
+
+
+# pylint: disable=too-many-locals
+def fetch_and_filter_logs(
+    ctx: ArtemisContext,
+    lines: int = 1000,
+    since_time: str | None = None,
+    until_time: str | None = None,
+) -> str:
+    """Fetches logs from ADB device and filters them by time if needed."""
+    device = get_adb_device(ctx)
+
+    session_start = ctx.data_engine.session_start_time if ctx.data_engine else None
+
+    since_str, since_dt = resolve_time(since_time, session_start)
+    _, until_dt = resolve_time(until_time, session_start)
+
+    if since_str:
+        try:
+            logger.info(f"Attempting to fetch logs since {since_str}...")
+            logs = device.shell(f'logcat -v threadtime -t "{since_str}"')
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.warning(
+                f"Failed to fetch logs since time directly: {e}. Falling back"
+                " to reading lines and filtering."
+            )
+            logs = device.shell(f"logcat -v threadtime -t {lines}")
+    else:
+        logger.info(f"Reading last {lines} lines of logs...")
+        logs = device.shell(f"logcat -v threadtime -t {lines}")
+
+    if since_dt or until_dt:
+        filtered_lines = []
+        for line in logs.splitlines():
+            if len(line) >= 18:
+                log_time_str = line[:18]
+                try:
+                    log_dt = parse_log_time(log_time_str)
+
+                    if since_dt and log_dt < since_dt:
+                        continue
+                    if until_dt and log_dt > until_dt:
+                        continue
+
+                    filtered_lines.append(line)
+                except ValueError:
+                    pass
+        # Cap to the requested line limit so timestamp fetches don't overflow context
+        if 0 < lines < len(filtered_lines):
+            filtered_lines = filtered_lines[-lines:]
+        logs = "\n".join(filtered_lines)
+    else:
+        # Ensure fallback capping even when no datetime parsing occurred
+        log_lines = logs.splitlines()
+        if 0 < lines < len(log_lines):
+            logs = "\n".join(log_lines[-lines:])
+
+    return logs

package/artemis-main/artemis/tools/mobile/ocr.py

@@ -0,0 +1,202 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import json
+from pathlib import Path
+from typing import Any
+
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel
+
+from artemis.context import ArtemisContext
+from artemis.data_engine.trace import trace_langchain_tool
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.graph.state import State
+from artemis.tools.base import ArtemisTool, ToolCategory
+from artemis.tools.tool_wrapper import ToolWrapper
+from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
+from third_party.mobile_use.utils.logger import get_logger
+
+
+class OcrArgs(BaseModel):
+    """Arguments schema for OCR recognition."""
+
+
+logger = get_logger(__name__)
+
+OCR_DOCSTRING = (
+    "[EXPLORER] Performs OCR on the current screen and returns the detected"
+    " text along with their coordinates."
+)
+
+
+# pylint: disable=too-many-branches,too-many-locals,too-many-statements
+async def run_ocr_core(
+    ctx: ArtemisContext | None = None,
+    state: State | None = None,
+    driver: BaseDeviceDriver | None = None,
+) -> str:
+    """Universal OCR core function: reads screenshot from state or driver and returns results."""
+    screenshot_b64 = None
+    screenshot_path = getattr(state, "latest_screenshot", None) if state else None
+
+    if screenshot_path:
+        if not Path(screenshot_path).exists():
+            logger.error(f"Screenshot file does not exist: {screenshot_path}")
+            raise FileNotFoundError(f"Screenshot file does not exist: {screenshot_path}")
+
+        try:
+            with open(screenshot_path, "rb") as f:
+                screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
+            logger.info(f"OCR loaded screenshot from file: {screenshot_path}")
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.error(f"Failed to read screenshot from {screenshot_path}: {e}")
+            raise e
+    elif driver is not None and hasattr(driver, "get_screen_data"):
+        try:
+            screen_data = await driver.get_screen_data()
+            if hasattr(screen_data, "screenshot_base64") and screen_data.screenshot_base64:
+                screenshot_b64 = screen_data.screenshot_base64
+            elif hasattr(screen_data, "screenshot_bytes") and screen_data.screenshot_bytes:
+                screenshot_b64 = base64.b64encode(screen_data.screenshot_bytes).decode("utf-8")
+            elif hasattr(screen_data, "base64") and screen_data.base64:
+                screenshot_b64 = screen_data.base64
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logger.error(f"Failed to capture screenshot from driver: {e}")
+            raise e
+    else:
+        logger.error("No screenshot path found in state.latest_screenshot and no driver available")
+        raise ValueError(
+            "No screenshot path found in state.latest_screenshot and no driver available"
+        )
+
+    if not screenshot_b64:
+        raise ValueError("Failed to obtain screenshot for OCR.")
+
+    width = 1080
+    height = 2400
+    try:
+        if ctx and hasattr(ctx, "device") and ctx.device:
+            w = getattr(ctx.device, "device_width", 1080)
+            h = getattr(ctx.device, "device_height", 2400)
+            if isinstance(w, int):
+                width = w
+            if isinstance(h, int):
+                height = h
+        elif driver and hasattr(driver, "screen_size") and driver.screen_size:
+            width, height = driver.screen_size
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logger.warning(f"Failed to get device resolution for OCR normalization: {e}")
+
+    results = await perform_ocr(screenshot_b64)
+    if results:
+        normalized_results = []
+        for item in results:
+            text = item.get("text")
+            vertices = item.get("position") or []
+
+            # Calculate center point
+            x_coords = [v.get("x", 0) for v in vertices]
+            y_coords = [v.get("y", 0) for v in vertices]
+
+            if x_coords and y_coords:
+                min_x, max_x = min(x_coords), max(x_coords)
+                min_y, max_y = min(y_coords), max(y_coords)
+
+                center_x = (min_x + max_x) / 2
+                center_y = (min_y + max_y) / 2
+
+                # Normalize to 0-1000
+                norm_x = int(max(0, min(1000, center_x * 1000 / width)))
+                norm_y = int(max(0, min(1000, center_y * 1000 / height)))
+
+                normalized_results.append({"text": text, "coordinates": [norm_x, norm_y]})
+            else:
+                normalized_results.append({"text": text, "coordinates": None})
+
+        return json.dumps(normalized_results, ensure_ascii=False, indent=2)
+    return "No text detected on the screen."
+
+
+class OcrRecognitionTool(ArtemisTool):
+    """Universal tool for performing OCR on the current screen."""
+
+    def __init__(self, category: ToolCategory = "explorer"):
+        super().__init__(
+            name="ocr_recognition",
+            description=OCR_DOCSTRING,
+            args_schema=OcrArgs,
+            category=category,
+        )
+
+    def is_available(self, ctx: Any = None) -> bool:
+        return is_ocr_configured()
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    async def execute(
+        self,
+        driver: BaseDeviceDriver | None = None,
+        ctx: ArtemisContext | None = None,
+        tool_call_id: str | None = None,
+        state: State | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
+        st = state if state is not None else kwargs.get("state")
+
+        has_failed = False
+        output = ""
+        error_message = None
+
+        try:
+            output = await run_ocr_core(ctx=ctx, state=st, driver=driver)
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            has_failed = True
+            error_message = f"Failed to perform OCR: {str(e)}"
+
+        if st is not None:
+            return ToolMessage(
+                tool_call_id=tcid or "",
+                content=ocr_recognition_wrapper.on_failure_fn(error_message)
+                if has_failed
+                else ocr_recognition_wrapper.on_success_fn(output),
+                additional_kwargs={"error": error_message} if has_failed else {},
+                status="error" if has_failed else "success",
+            )
+
+        if has_failed:
+            return ocr_recognition_wrapper.on_failure_fn(error_message)
+        return ocr_recognition_wrapper.on_success_fn(output)
+
+
+# Universal tool instance & aliases
+ocr_recognition = OcrRecognitionTool()
+OcrRecognition = OcrRecognitionTool
+OCRRecognition = OcrRecognitionTool
+OcrTool = OcrRecognitionTool
+
+
+def get_ocr_tool(ctx: ArtemisContext) -> BaseTool:
+    """Exports ocr_recognition as a LangChain BaseTool."""
+    return trace_langchain_tool(ocr_recognition.to_langchain_tool(ctx), ctx)
+
+
+ocr_recognition_wrapper = ToolWrapper(
+    tool_fn_getter=get_ocr_tool,
+    on_success_fn=lambda output: f"OCR Recognition successful. Results:\n{output}",
+    on_failure_fn=lambda error: f"OCR Recognition failed: {error}",
+    is_available_fn=lambda ctx: is_ocr_configured(),
+)