android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/utils/credentials_validator.py

@@ -0,0 +1,201 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Validation utility to verify LLM and Vision API keys against live provider endpoints."""
+
+import httpx
+
+from artemis.llm.google.provider import is_google_family_provider
+
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+# Minimal 1x1 PNG transparent pixel base64 for Vision API testing
+_TINY_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+
+
+def _extract_error_message(resp: httpx.Response) -> str:
+    """Extract a human-friendly error message from a provider HTTP response."""
+    try:
+        data = resp.json()
+        if isinstance(data, dict):
+            # Google style: {"error": {"message": "..."}}
+            err_obj = data.get("error")
+            if isinstance(err_obj, dict):
+                msg = err_obj.get("message")
+                if msg:
+                    return str(msg)
+            elif isinstance(err_obj, str):
+                return err_obj
+
+            # OpenAI / Anthropic / OpenRouter style
+            if "message" in data:
+                return str(data["message"])
+            if "detail" in data:
+                return str(data["detail"])
+    except ValueError:
+        # Response body is not valid JSON; fall through to the text fallback.
+        pass
+
+    # Fallback to response text or HTTP status
+    text = resp.text.strip()
+    if text and len(text) < 200:
+        return text
+    return f"HTTP {resp.status_code}: {resp.reason_phrase}"
+
+
+async def validate_api_key(
+    provider: str,
+    api_key: str,
+    base_url: str | None = None,
+    timeout: float = 12.0,
+) -> tuple[bool, str]:
+    """Tests if the provided API key is valid and usable with the corresponding provider.
+
+    Args:
+        provider: Provider identifier (e.g. 'google', 'gemini', 'ocr', 'openai', 'anthropic', etc.)
+        api_key: Secret API key string.
+        base_url: Optional custom endpoint URL.
+        timeout: Request timeout in seconds.
+
+    Returns:
+        tuple[bool, str]: (is_valid, descriptive_message)
+    """
+    clean_provider = provider.strip().lower()
+    clean_key = api_key.strip()
+
+    if not clean_key:
+        return False, "API key cannot be empty."
+
+    # Placeholders / default template values should be rejected
+    if clean_key in (
+        "API_KEY",
+        "your_gemini_api_key_here",
+        "your_google_cloud_vision_api_key_here",
+        "your_openai_api_key_here",
+        "your_anthropic_api_key_here",
+    ):
+        return False, "Please enter a real API key instead of the placeholder."
+
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            if is_google_family_provider(clean_provider):
+                # Test Google Gemini via generativelanguage API
+                # The key travels in a header, never in the URL: httpx logs request
+                # URLs at INFO level and those logs end up in files.
+                url = "https://generativelanguage.googleapis.com/v1beta/models?pageSize=1"
+                resp = await client.get(url, headers={"x-goog-api-key": clean_key})
+                if resp.status_code == 200:
+                    return True, "Google Gemini API key verified successfully."
+                err_msg = _extract_error_message(resp)
+                return False, f"Gemini API verification failed ({resp.status_code}): {err_msg}"
+
+            elif clean_provider in ("ocr", "vision", "google_vision"):
+                # Test Google Cloud Vision OCR API
+                url = "https://vision.googleapis.com/v1/images:annotate"
+                payload = {
+                    "requests": [
+                        {
+                            "image": {"content": _TINY_PNG_B64},
+                            "features": [{"type": "TEXT_DETECTION"}],
+                        }
+                    ]
+                }
+                resp = await client.post(
+                    url,
+                    json=payload,
+                    headers={"Content-Type": "application/json", "x-goog-api-key": clean_key},
+                )
+                if resp.status_code == 200:
+                    return True, "Google Cloud Vision OCR API key verified successfully."
+                err_msg = _extract_error_message(resp)
+                return False, f"Vision OCR API verification failed ({resp.status_code}): {err_msg}"
+
+            elif clean_provider == "openai":
+                endpoint = (
+                    f"{base_url.rstrip('/')}/models"
+                    if base_url
+                    else "https://api.openai.com/v1/models"
+                )
+                resp = await client.get(endpoint, headers={"Authorization": f"Bearer {clean_key}"})
+                if resp.status_code == 200:
+                    return True, "OpenAI API key verified successfully."
+                err_msg = _extract_error_message(resp)
+                return False, f"OpenAI API verification failed ({resp.status_code}): {err_msg}"
+
+            elif clean_provider in ("anthropic", "claude"):
+                url = "https://api.anthropic.com/v1/models"
+                resp = await client.get(
+                    url,
+                    headers={
+                        "x-api-key": clean_key,
+                        "anthropic-version": "2023-06-01",
+                    },
+                )
+                if resp.status_code == 200:
+                    return True, "Anthropic Claude API key verified successfully."
+                err_msg = _extract_error_message(resp)
+                return False, f"Anthropic API verification failed ({resp.status_code}): {err_msg}"
+
+            elif clean_provider == "openrouter":
+                url = "https://openrouter.ai/api/v1/auth/key"
+                resp = await client.get(url, headers={"Authorization": f"Bearer {clean_key}"})
+                if resp.status_code == 200:
+                    return True, "OpenRouter API key verified successfully."
+                err_msg = _extract_error_message(resp)
+                return False, f"OpenRouter API verification failed ({resp.status_code}): {err_msg}"
+
+            elif clean_provider in ("xai", "grok"):
+                url = "https://api.x.ai/v1/models"
+                resp = await client.get(url, headers={"Authorization": f"Bearer {clean_key}"})
+                if resp.status_code == 200:
+                    return True, "xAI API key verified successfully."
+                err_msg = _extract_error_message(resp)
+                return False, f"xAI API verification failed ({resp.status_code}): {err_msg}"
+
+            elif clean_provider in ("ollama", "vllm", "custom"):
+                if base_url:
+                    endpoint = f"{base_url.rstrip('/')}/models"
+                    headers = (
+                        {"Authorization": f"Bearer {clean_key}"} if clean_key != "EMPTY" else {}
+                    )
+                    resp = await client.get(endpoint, headers=headers)
+                    if resp.status_code == 200:
+                        return True, f"Custom endpoint ({base_url}) verified successfully."
+                    err_msg = _extract_error_message(resp)
+                    return (
+                        False,
+                        f"Custom endpoint verification failed ({resp.status_code}): {err_msg}",
+                    )
+                return True, "Custom endpoint configured."
+
+            else:
+                # Unknown provider - fallback to basic length check
+                if len(clean_key) >= 8:
+                    return True, f"Credentials for {provider} configured."
+                return False, f"Invalid API key format for {provider}."
+
+    except httpx.TimeoutException:
+        logger.warning(f"Timeout verifying API key for {provider}")
+        return (
+            False,
+            f"Connection to {provider.capitalize()} API timed out. Please check network/proxy settings.",
+        )
+    except httpx.RequestError as exc:
+        logger.warning(f"Network error verifying API key for {provider}: {exc}")
+        return False, f"Network error connecting to {provider.capitalize()} API: {exc}"
+    except Exception as exc:
+        logger.error(f"Unexpected error validating API key for {provider}: {exc}")
+        return False, f"Validation error: {exc}"

package/artemis-main/artemis/utils/cv_canvas.py

@@ -0,0 +1,189 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from pathlib import Path
+import cv2
+
+
+class ImageCanvas:
+    def __init__(self, img_id: str, intermediate_dir: str):
+        self.intermediate_dir = intermediate_dir
+
+        pool_path = Path(intermediate_dir) / "intermediate_transforms.json"
+        if not pool_path.exists():
+            raise ValueError(f"Image pool registry file not found at {pool_path}")
+
+        with open(pool_path) as f:
+            pool = json.load(f)
+
+        if img_id not in pool:
+            raise ValueError(f"Image ID '{img_id}' not found in the intermediate image pool.")
+
+        entry = pool[img_id]
+        img_path = entry["path"]
+
+        self.img = cv2.imread(img_path)
+        if self.img is None:
+            raise ValueError(f"Failed to read image at {img_path}")
+        self.width, self.height = self._get_resolution()
+        self.transform = entry["transform"].copy()
+        self.annotations = {}
+
+    def _get_resolution(self) -> tuple[int, int]:
+        height, width = self.img.shape[:2]
+        return width, height
+
+    def normalized_to_pixel_coords(self, x: float, y: float) -> tuple[int, int]:
+        """Convert normalized (0-1000) coordinates to pixel coordinates."""
+        px = int(round(x * self.width / 1000.0))
+        py = int(round(y * self.height / 1000.0))
+        px = max(0, min(self.width - 1, px))
+        py = max(0, min(self.height - 1, py))
+        return px, py
+
+    def draw_dot(self, x: int, y: int, label_idx: int, radius: int = 10) -> "ImageCanvas":
+        cx, cy = x, y
+        # Draw nested white and red circles
+        cv2.circle(self.img, (cx, cy), 2 * radius, (255, 255, 255), -1)
+        cv2.circle(self.img, (cx, cy), radius, (0, 0, 255), -1)
+
+        # Draws text "V{label_idx}" using cv2.putText with PIL-matching offset: (x + 2*radius + 4, y - 22)
+        tx = int(x + 2 * radius + 4)
+        ty = int(y - 22)
+        text = f"V{label_idx}"
+
+        cv2.putText(
+            self.img,
+            text,
+            (tx, ty),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            1.5,
+            (255, 255, 255),
+            10,
+            cv2.LINE_AA,
+        )
+        cv2.putText(
+            self.img,
+            text,
+            (tx, ty),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            1.5,
+            (0, 0, 255),
+            3,
+            cv2.LINE_AA,
+        )
+
+        self.annotations[text] = [cx, cy]
+        return self
+
+    def save(self, final: bool) -> None:
+        pool_json_path = Path(self.intermediate_dir) / "intermediate_transforms.json"
+        if pool_json_path.exists():
+            with open(pool_json_path) as f:
+                pool = json.load(f)
+        else:
+            pool = {}
+
+        # Auto-generate next sequential image_id (e.g. "img_1")
+        img_nums = []
+        for k in pool.keys():
+            if k.startswith("img_"):
+                try:
+                    img_nums.append(int(k[4:]))
+                except ValueError:
+                    pass
+        next_num = max(img_nums) + 1 if img_nums else 1
+        image_id = f"img_{next_num}"
+
+        out_dir = Path(self.intermediate_dir)
+        out_dir.mkdir(parents=True, exist_ok=True)
+        out_path = out_dir / f"{image_id}.jpg"
+
+        success = cv2.imwrite(str(out_path), self.img)
+        if not success:
+            raise ValueError(f"Failed to save image to {out_path}")
+
+        pool[image_id] = {
+            "image_id": image_id,
+            "path": str(out_path.resolve()),
+            "transform": self.transform.copy(),
+            "is_output": final,
+            "annotations": self.annotations.copy(),
+        }
+
+        with open(pool_json_path, "w") as f:
+            json.dump(pool, f, indent=2)
+
+    def crop(self, x: int, y: int, w: int, h: int):
+        if w <= 0 or h <= 0:
+            raise ValueError("Width and height must be strictly positive.")
+
+        # Calculate bounding boxes
+        orig_h, orig_w = self.img.shape[:2]
+        x_end = min(x + w, orig_w)
+        y_end = min(y + h, orig_h)
+        x = max(0, x)
+        y = max(0, y)
+
+        self.img = self.img[y:y_end, x:x_end]
+        self.width, self.height = self._get_resolution()
+
+        # Adjust offsets by considering the current scale factor
+        self.transform["offset_x"] += x / self.transform["scale_x"]
+        self.transform["offset_y"] += y / self.transform["scale_y"]
+
+        # Transform coordinates inside self.annotations
+        new_annotations = {}
+        new_w = x_end - x
+        new_h = y_end - y
+        for label, coord in self.annotations.items():
+            px, py = coord
+            new_px = px - x
+            new_py = py - y
+            if 0 <= new_px < new_w and 0 <= new_py < new_h:
+                new_annotations[label] = [int(new_px), int(new_py)]
+        self.annotations = new_annotations
+
+        return self
+
+    def resize_by_factor(self, factor_x: float, factor_y: float = None):
+        if factor_x <= 0:
+            raise ValueError("Scale factor must be strictly positive.")
+        if factor_y is None:
+            factor_y = factor_x
+
+        height, width = self.img.shape[:2]
+        new_width = int(width * factor_x)
+        new_height = int(height * factor_y)
+
+        if new_width == 0 or new_height == 0:
+            raise ValueError("Resulting dimensions must be greater than 0.")
+
+        self.img = cv2.resize(self.img, (new_width, new_height))
+        self.width, self.height = self._get_resolution()
+        self.transform["scale_x"] *= factor_x
+        self.transform["scale_y"] *= factor_y
+
+        # Transform coordinates inside self.annotations
+        new_annotations = {}
+        for label, coord in self.annotations.items():
+            px, py = coord
+            new_px = int(px * factor_x)
+            new_py = int(py * factor_y)
+            if 0 <= new_px < new_width and 0 <= new_py < new_height:
+                new_annotations[label] = [new_px, new_py]
+        self.annotations = new_annotations
+
+        return self

package/artemis-main/artemis/utils/cython_compat.py

@@ -0,0 +1,45 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pydantic support for Cython builds.
+
+With ``USE_CYTHON=1``, methods defined in a class body compile to
+``cyfunction`` objects, which pydantic would take for fields. Models add
+:class:`CyFunctionDetector` to ``model_config["ignored_types"]`` to prevent it.
+
+The models in ``third_party/`` are not compiled but still need the setting:
+compiled Artemis subclasses inherit their ``model_config``, for example
+``artemis.config.llm.LLMConfig`` from ``LLMConfigBase``.
+"""
+
+
+class _CyFunctionDetectorMeta(type):
+    """Metaclass for detecting cython compiled functions."""
+
+    def __instancecheck__(cls, instance):
+        name = type(instance).__name__
+        return (
+            name
+            in (
+                "cyfunction",
+                "cython_function_or_method",
+                "builtin_function_or_method",
+            )
+            or "cyfunction" in name.lower()
+        )
+
+
+# pylint: disable=too-few-public-methods
+class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
+    """Detector for cython compiled functions."""

package/artemis-main/artemis/utils/element_hit_test.py

@@ -0,0 +1,80 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Element hit testing: which indexed element covers a pixel point.
+
+Used by the Explorer to deduplicate candidates against the screen index.
+Recorded coordinate actions use the model's ``target_description``.
+"""
+
+from typing import Any
+
+from artemis.utils.visualization import parse_bounds
+
+
+def _element_bounds(element: dict[str, Any]) -> tuple[int, int, int, int] | None:
+    """Returns (left, top, right, bottom) pixel bounds of an indexed element.
+
+    Accepts both list bounds (``[l, t, r, b]``, the shape produced by
+    ``format_minimal_list_with_elements``) and Android bounds strings
+    (``"[l,t][r,b]"``). Returns None for missing or malformed bounds.
+    """
+    bounds = element.get("bounds")
+    if isinstance(bounds, str):
+        return parse_bounds(bounds)
+    if isinstance(bounds, (list, tuple)) and len(bounds) == 4:
+        try:
+            left, top, right, bottom = (int(v) for v in bounds)
+        except (TypeError, ValueError):
+            return None
+        return left, top, right, bottom
+    return None
+
+
+def find_element_at_point(
+    elements: list[dict[str, Any]] | None, x: int, y: int
+) -> tuple[dict[str, Any] | None, str]:
+    """Finds the smallest indexed element whose bounds contain pixel point (x, y).
+
+    XML-derived elements are preferred over OCR-derived ones (OCR boxes carry
+    text but weaker structure); within each group the smallest covering area
+    wins. Returns ``(element, source)`` where source is ``"hit_test"``,
+    ``"ocr"``, or ``"none"`` (with element None). Malformed element data is
+    skipped; the function never raises on it.
+    """
+    best_xml: tuple[int, dict[str, Any]] | None = None
+    best_ocr: tuple[int, dict[str, Any]] | None = None
+
+    for element in elements or []:
+        if not isinstance(element, dict):
+            continue
+        rect = _element_bounds(element)
+        if rect is None:
+            continue
+        left, top, right, bottom = rect
+        if not (left <= x <= right and top <= y <= bottom):
+            continue
+        area = max(1, (right - left)) * max(1, (bottom - top))
+        if element.get("is_ocr"):
+            if best_ocr is None or area < best_ocr[0]:
+                best_ocr = (area, element)
+        else:
+            if best_xml is None or area < best_xml[0]:
+                best_xml = (area, element)
+
+    if best_xml is not None:
+        return best_xml[1], "hit_test"
+    if best_ocr is not None:
+        return best_ocr[1], "ocr"
+    return None, "none"