android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/toolchain/resolver.py

@@ -0,0 +1,185 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Multi-tier cross-platform binary resolution and discovery engine."""
+
+import logging
+import os
+from pathlib import Path
+import shutil
+
+from artemis.platform import OSType, platform
+from artemis.toolchain.descriptors import TOOLS, ToolDescriptor
+
+logger = logging.getLogger(__name__)
+
+
+class ToolchainResolver:
+    """Discovers, validates, and resolves platform-specific binaries with graceful fallbacks."""
+
+    def __init__(self):
+        self._cache: dict[str, str | None] = {}
+
+    def clear_cache(self, tool_name: str | None = None) -> None:
+        """Invalidate cached tool paths to force fresh discovery."""
+        if tool_name:
+            self._cache.pop(tool_name.lower(), None)
+        else:
+            self._cache.clear()
+
+    def resolve(self, tool_name: str, force_refresh: bool = False) -> str | None:
+        """Resolve absolute path to a tool across all discovery tiers."""
+        key = tool_name.lower()
+        if not force_refresh and key in self._cache:
+            cached_val = self._cache[key]
+            if cached_val is not None and Path(cached_val).exists():
+                return cached_val
+            # Cache miss or binary removed: proceed to re-discover
+
+        desc = TOOLS.get(key)
+        if desc is None:
+            # Generic binary lookup
+            res = shutil.which(tool_name)
+            if res and Path(res).exists():
+                self._cache[key] = str(Path(res).resolve())
+                return self._cache[key]
+            self._cache.pop(key, None)
+            return None
+
+        binary_name = (
+            desc.win_binary_name if platform.os_type == OSType.WINDOWS else desc.binary_name
+        )
+
+        # Tier 1: Explicit environment variable override (e.g. ARTEMIS_ADB_PATH)
+        env_var = f"ARTEMIS_{desc.name.upper()}_PATH"
+        env_override = os.getenv(env_var)
+        if env_override and Path(env_override).exists():
+            resolved = str(Path(env_override).resolve())
+            self._cache[key] = resolved
+            return resolved
+
+        # Tier 2: SDK Environment Variables (for Android tools)
+        if desc.sdk_relative_path:
+            for sdk_env in ("ANDROID_HOME", "ANDROID_SDK_ROOT"):
+                sdk_dir = os.getenv(sdk_env)
+                if sdk_dir:
+                    candidate = Path(sdk_dir) / desc.sdk_relative_path
+                    if platform.os_type == OSType.WINDOWS:
+                        candidate = candidate.with_suffix(".exe")
+                    if candidate.exists():
+                        resolved = str(candidate.resolve())
+                        self._cache[key] = resolved
+                        return resolved
+
+        # Tier 3: Common OS default installation paths
+        common_candidates = self._get_os_common_paths(desc)
+        for cand in common_candidates:
+            if cand.exists():
+                resolved = str(cand.resolve())
+                self._cache[key] = resolved
+                return resolved
+
+        # Tier 4: System PATH
+        which_path = shutil.which(binary_name) or shutil.which(desc.name)
+        if which_path:
+            resolved = str(Path(which_path).resolve())
+            self._cache[key] = resolved
+            return resolved
+
+        # Tier 5: Python library embedded binary fallback
+        if desc.embedded_fallback:
+            try:
+                embedded = desc.embedded_fallback()
+                if embedded and Path(embedded).exists():
+                    resolved = str(Path(embedded).resolve())
+                    self._cache[key] = resolved
+                    return resolved
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                # Probing an optional third-party embedded binary; any failure
+                # simply means "tool not found" and the caller handles None.
+                logger.debug(
+                    "Embedded binary fallback for %s skipped: %s", desc.name, exc, exc_info=True
+                )
+
+        # Do not permanently cache negative lookups so dynamic installs are detected immediately
+        self._cache.pop(key, None)
+        return None
+
+    def _get_os_common_paths(self, desc: ToolDescriptor) -> list[Path]:
+        candidates: list[Path] = []
+        home = Path.home()
+
+        if platform.os_type == OSType.WINDOWS:
+            local_appdata = os.getenv("LOCALAPPDATA", "")
+            program_files = os.getenv("ProgramFiles", "C:\\Program Files")
+            program_data = os.getenv("ProgramData", "C:\\ProgramData")
+
+            if desc.name == "adb":
+                if local_appdata:
+                    candidates.append(
+                        Path(local_appdata) / "Android" / "Sdk" / "platform-tools" / "adb.exe"
+                    )
+                candidates.append(Path(program_files) / "Android" / "platform-tools" / "adb.exe")
+            elif desc.name == "scrcpy":
+                candidates.append(Path(program_data) / "chocolatey" / "bin" / "scrcpy.exe")
+                if local_appdata:
+                    candidates.append(
+                        Path(local_appdata) / "Microsoft" / "WinGet" / "Links" / "scrcpy.exe"
+                    )
+            elif desc.name == "ffmpeg":
+                candidates.append(Path(program_data) / "chocolatey" / "bin" / "ffmpeg.exe")
+                if local_appdata:
+                    candidates.append(
+                        Path(local_appdata) / "Microsoft" / "WinGet" / "Links" / "ffmpeg.exe"
+                    )
+
+        elif platform.os_type == OSType.MACOS:
+            if desc.name == "adb":
+                candidates.append(home / "Library" / "Android" / "sdk" / "platform-tools" / "adb")
+                candidates.append(Path("/opt/homebrew/bin/adb"))
+                candidates.append(Path("/usr/local/bin/adb"))
+            elif desc.name in ("ffmpeg", "scrcpy"):
+                candidates.append(Path(f"/opt/homebrew/bin/{desc.name}"))
+                candidates.append(Path(f"/usr/local/bin/{desc.name}"))
+
+        elif platform.os_type == OSType.LINUX:
+            if desc.name == "adb":
+                candidates.append(home / "Android" / "Sdk" / "platform-tools" / "adb")
+                candidates.append(Path("/usr/lib/android-sdk/platform-tools/adb"))
+                candidates.append(Path("/usr/bin/adb"))
+            elif desc.name in ("ffmpeg", "scrcpy"):
+                candidates.append(Path(f"/usr/bin/{desc.name}"))
+                candidates.append(Path(f"/usr/local/bin/{desc.name}"))
+
+        return candidates
+
+    def find_adb(self) -> str:
+        """Find ADB executable or fallback to default name."""
+        return self.resolve("adb") or "adb"
+
+    def find_ffmpeg(self) -> str:
+        """Find FFmpeg executable or fallback to default name."""
+        return self.resolve("ffmpeg") or "ffmpeg"
+
+    def find_scrcpy(self) -> str:
+        """Find scrcpy executable or fallback to default name."""
+        return self.resolve("scrcpy") or "scrcpy"
+
+    def is_installed(self, tool_name: str) -> bool:
+        """Check whether a tool is found on this system."""
+        return self.resolve(tool_name) is not None
+
+
+# Global toolchain resolver singleton
+toolchain: ToolchainResolver = ToolchainResolver()

package/artemis-main/artemis/tools/base.py

@@ -0,0 +1,232 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal Tool Protocol for ARTEMIS.
+
+Enables tools to be authored once using standard Pydantic schemas and async handlers,
+then automatically exported to LangChain Tools (Pro Graph), Google GenAI FunctionDeclarations
+(FlashRunner), and MCP Tools (FastMCP) with zero code duplication.
+"""
+
+import asyncio
+from collections.abc import Callable
+import concurrent.futures
+import inspect
+from typing import Annotated, Any, Literal
+
+from google.genai import types as genai_types
+from langchain_core.tools import BaseTool, StructuredTool
+from langgraph.prebuilt import InjectedState
+from pydantic import BaseModel
+
+from artemis.core.tool_declaration import ToolDeclaration
+from artemis.drivers.base import BaseDeviceDriver
+from artemis.drivers.factory import get_driver
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+ToolCategory = Literal["action", "perception", "system", "memory", "custom"]
+
+
+class ArtemisTool:
+    """Unified tool wrapper encapsulating schema, execution logic, and multi-protocol export."""
+
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        args_schema: type[BaseModel],
+        handler: Callable[..., Any] | None = None,
+        category: ToolCategory = "action",
+    ):
+        self.name = name
+        self.description = description
+        self.args_schema = args_schema
+        self.handler = handler
+        self.category = category
+
+    def is_available(self, ctx: Any = None) -> bool:
+        """Determines if this tool is currently available and should be exposed to agents."""
+        return True
+
+    async def execute(self, driver: BaseDeviceDriver, ctx: Any, **kwargs: Any) -> Any:
+        """Executes the tool handler with injected driver and context."""
+        # Inspect handler signature to only pass supported arguments
+        if self.handler is not None:
+            sig = inspect.signature(self.handler)
+            call_kwargs = {}
+            for param_name in sig.parameters:
+                if param_name == "driver":
+                    call_kwargs["driver"] = driver
+                elif param_name in ("ctx", "context"):
+                    call_kwargs[param_name] = ctx
+                elif param_name in kwargs:
+                    call_kwargs[param_name] = kwargs[param_name]
+
+            if inspect.iscoroutinefunction(self.handler):
+                return await self.handler(**call_kwargs)
+            return self.handler(**call_kwargs)
+        raise NotImplementedError("Subclasses must implement execute() or provide a handler.")
+
+    async def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        """Allows ArtemisTool instances to be invoked directly as callables."""
+        driver = kwargs.pop("driver", None)
+        ctx = kwargs.pop("ctx", None) or kwargs.pop("context", None)
+        return await self.execute(driver=driver, ctx=ctx, *args, **kwargs)
+
+    def to_langchain_tool(self, ctx: Any, name: str | None = None) -> BaseTool:
+        """Exports this tool to a LangChain StructuredTool."""
+
+        def _get_context_driver() -> BaseDeviceDriver | None:
+            if (
+                ctx is not None
+                and hasattr(ctx, "device")
+                and ctx.device
+                and getattr(ctx.device, "device_id", None)
+            ):
+                try:
+                    return get_driver(ctx)
+                except Exception as e:  # pylint: disable=broad-exception-caught
+                    logger.debug(f"Failed to get driver: {e}")
+                    return None
+            return getattr(ctx, "_active_driver", None) if ctx is not None else None
+
+        def _func(state: Annotated[Any, InjectedState] = None, **kwargs):
+            driver = _get_context_driver()
+            if state is not None:
+                kwargs["state"] = state
+            try:
+                loop = asyncio.get_running_loop()
+            except RuntimeError:
+                loop = None
+
+            if loop and loop.is_running():
+                with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+                    return pool.submit(
+                        asyncio.run, self.execute(driver=driver, ctx=ctx, **kwargs)
+                    ).result()
+            return asyncio.run(self.execute(driver=driver, ctx=ctx, **kwargs))
+
+        async def _coroutine(state: Annotated[Any, InjectedState] = None, **kwargs):
+            driver = _get_context_driver()
+            if state is not None:
+                kwargs["state"] = state
+            return await self.execute(driver=driver, ctx=ctx, **kwargs)
+
+        return StructuredTool.from_function(
+            func=_func,
+            coroutine=_coroutine,
+            name=name or self.name,
+            description=self.description,
+            args_schema=self.args_schema,
+        )
+
+    def _flat_schema(self) -> tuple[dict[str, dict[str, Any]], list[str]]:
+        if not self.args_schema:
+            return {}, []
+        return _flatten_json_schema_properties(self.args_schema.model_json_schema())
+
+    def to_tool_declaration(self) -> ToolDeclaration:
+        """Build the Flash declaration from the args_schema used by LangChain."""
+        properties, required = self._flat_schema()
+        return ToolDeclaration(
+            name=self.name,
+            description=self.description,
+            parameters={"type": "object", "properties": properties, "required": required},
+        )
+
+    def to_genai_declaration(self) -> genai_types.FunctionDeclaration:
+        """Exports this tool to a Google GenAI FunctionDeclaration schema."""
+        flat_properties, required = self._flat_schema()
+        properties = {}
+        for prop_name, prop in flat_properties.items():
+            genai_type = _GENAI_TYPES.get(prop["type"], genai_types.Type.STRING)
+            items_schema = None
+            if genai_type == genai_types.Type.ARRAY:
+                item_type = (prop.get("items") or {}).get("type", "string")
+                items_schema = genai_types.Schema(
+                    type=_GENAI_TYPES.get(item_type, genai_types.Type.STRING)
+                )
+            properties[prop_name] = genai_types.Schema(
+                type=genai_type,
+                description=prop.get("description", ""),
+                items=items_schema,
+                enum=[str(v) for v in prop["enum"]] if prop.get("enum") else None,
+            )
+
+        return genai_types.FunctionDeclaration(
+            name=self.name,
+            description=self.description,
+            parameters=genai_types.Schema(
+                type=genai_types.Type.OBJECT,
+                properties=properties,
+                required=required,
+            ),
+        )
+
+
+_GENAI_TYPES = {
+    "string": genai_types.Type.STRING,
+    "integer": genai_types.Type.INTEGER,
+    "number": genai_types.Type.NUMBER,
+    "boolean": genai_types.Type.BOOLEAN,
+    "array": genai_types.Type.ARRAY,
+    "object": genai_types.Type.OBJECT,
+}
+
+
+def _flatten_property(prop_data: dict[str, Any]) -> dict[str, Any]:
+    """One Pydantic JSON-schema property as a flat single-type entry.
+
+    ``anyOf`` unions (``X | None``) collapse to their first non-null member;
+    ``items`` (array element type) and ``enum`` (``Literal`` choices) are
+    carried over; unknown shapes fall back to ``string``.
+    """
+    p_type = prop_data.get("type")
+    items = prop_data.get("items")
+    enum = prop_data.get("enum")
+    if not p_type and "anyOf" in prop_data:
+        for sub_schema in prop_data["anyOf"]:
+            sub_type = sub_schema.get("type")
+            if sub_type and sub_type != "null":
+                p_type = sub_type
+                items = items or sub_schema.get("items")
+                enum = enum or sub_schema.get("enum")
+                break
+    if not p_type:
+        p_type = "string"
+    entry: dict[str, Any] = {"type": p_type}
+    if p_type == "array":
+        entry["items"] = {"type": (items or {}).get("type") or "string"}
+    if enum:
+        entry["enum"] = list(enum)
+    return entry
+
+
+def _flatten_json_schema_properties(
+    schema: dict[str, Any],
+) -> tuple[dict[str, dict[str, Any]], list[str]]:
+    """Flattens a Pydantic model JSON schema into plain JSON-schema properties
+    plus the required-field list. Descriptions are copied verbatim; non-null
+    defaults are kept under ``default``."""
+    properties: dict[str, dict[str, Any]] = {}
+    for prop_name, prop_data in (schema.get("properties") or {}).items():
+        entry = _flatten_property(prop_data)
+        entry["description"] = prop_data.get("description", "")
+        if prop_data.get("default") is not None:
+            entry["default"] = prop_data["default"]
+        properties[prop_name] = entry
+    return properties, list(schema.get("required") or [])