android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/core/diagnostics/engine.py

@@ -0,0 +1,438 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""System Readiness & Diagnostic Orchestration Engine."""
+
+import asyncio
+import os
+import subprocess
+import time
+from typing import Any
+
+from artemis.core.diagnostics.adb_server_connection import adb_server_connection
+from artemis.core.diagnostics.probes.adb_probe import AdbDeviceProbe
+from artemis.core.diagnostics.probes.base import BaseProbe
+from artemis.core.diagnostics.probes.credentials_probe import (
+    LLMCredentialsProbe,
+    VisionOCRProbe,
+)
+from artemis.core.diagnostics.probes.runtime_probe import (
+    PythonRuntimeProbe,
+    SystemConfigProbe,
+)
+from artemis.core.diagnostics.probes.toolchain_probe import ToolchainProbe
+from artemis.core.diagnostics.schema import (
+    DeviceInfo,
+    ProbeAction,
+    ProbeCategory,
+    ProbeResult,
+    ProbeStatus,
+    SystemReadinessReport,
+)
+from artemis.toolchain import toolchain
+from artemis.platform import platform
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ReadinessEngine:
+    """Central orchestration engine executing modular readiness probes."""
+
+    _REPORT_CACHE_TTL_SECONDS = 2.0
+
+    #: Deadline for one probe inside a report. The ADB probe shells out to
+    #: ``adb`` without a deadline of its own, and a wedged ADB server would
+    #: otherwise hang every surface that runs the report (``artemis doctor``,
+    #: the console wizard, ``mobile_diagnose``). A probe that overruns is
+    #: reported as a FAIL with the recovery steps instead.
+    PROBE_TIMEOUT_SECONDS = 30.0
+
+    def __init__(self):
+        self._probes: dict[str, BaseProbe] = {}
+        self._python_probe = PythonRuntimeProbe()
+        self._config_probe = SystemConfigProbe()
+        self._toolchain_probe = ToolchainProbe()
+        self._credentials_probe = LLMCredentialsProbe()
+        self._ocr_probe = VisionOCRProbe()
+        self._adb_probe = AdbDeviceProbe()
+        self._report_cache: SystemReadinessReport | None = None
+        self._report_cache_time = 0.0
+        self._report_cache_generation = -1
+        self._cache_generation = 0
+        self._report_lock = asyncio.Lock()
+
+        # Register default core probes in logical lifecycle order
+        self.register_probe(self._python_probe)
+        self.register_probe(self._config_probe)
+        self.register_probe(self._toolchain_probe)
+        self.register_probe(self._credentials_probe)
+        self.register_probe(self._ocr_probe)
+        self.register_probe(self._adb_probe)
+
+    def register_probe(self, probe: BaseProbe) -> None:
+        """Register a new diagnostic probe."""
+        self._probes[probe.probe_id] = probe
+
+    def unregister_probe(self, probe_id: str) -> None:
+        """Remove a diagnostic probe."""
+        self._probes.pop(probe_id, None)
+
+    def set_probe_target_serial(self, serial: str | None) -> None:
+        """Set the diagnostics probes' preferred device serial.
+
+        This is a probe/report preference only (which device the readiness
+        report highlights and verifies first). Task routing never reads it
+        back: execution targets come from explicit request serials or the
+        device pool. Pass None to clear the preference, e.g. after switching
+        ADB server endpoints.
+        """
+        self.invalidate_cache()
+        if hasattr(self._adb_probe, "set_target_serial"):
+            self._adb_probe.set_target_serial(serial)
+
+    async def run_probe(self, probe_id: str) -> ProbeResult | None:
+        """Execute a single specific probe by ID."""
+        probe = self._probes.get(probe_id)
+        if not probe:
+            return None
+        return await probe.probe()
+
+    async def run_device_submission_probe(self, target_serial: str | None = None) -> ProbeResult:
+        """Run the bounded device gate used by task submission."""
+        return await self._adb_probe.probe_submission_readiness(target_serial=target_serial)
+
+    def invalidate_cache(self) -> None:
+        """Invalidate the UI readiness snapshot after an explicit configuration change."""
+        self._cache_generation += 1
+        self._report_cache = None
+        self._report_cache_time = 0.0
+        self._report_cache_generation = -1
+
+    def _cached_report(self, max_age_seconds: float) -> SystemReadinessReport | None:
+        if self._report_cache is None:
+            return None
+        if self._report_cache_generation != self._cache_generation:
+            return None
+        if time.monotonic() - self._report_cache_time > max_age_seconds:
+            return None
+        return self._report_cache.model_copy(deep=True)
+
+    async def run_all(
+        self,
+        categories: list[ProbeCategory] | None = None,
+        *,
+        force_refresh: bool = False,
+    ) -> SystemReadinessReport:
+        """Return a coalesced readiness snapshot.
+
+        The dashboard polls frequently, so allowing every HTTP request to launch a
+        complete toolchain and ADB scan creates a request storm. Full reports are
+        cached briefly and all concurrent refreshes share one execution. The task
+        submission gate remains independent and always performs its own bounded
+        device check.
+        """
+        cacheable = categories is None
+        request_started = time.monotonic()
+        if cacheable and not force_refresh:
+            cached = self._cached_report(self._REPORT_CACHE_TTL_SECONDS)
+            if cached is not None:
+                return cached
+
+        async with self._report_lock:
+            # A refresh that completed while this caller waited satisfies even a
+            # forced request that began before it, coalescing concurrent clicks.
+            if cacheable and self._report_cache_time >= request_started:
+                cached = self._cached_report(float("inf"))
+                if cached is not None:
+                    return cached
+            if cacheable and not force_refresh:
+                cached = self._cached_report(self._REPORT_CACHE_TTL_SECONDS)
+                if cached is not None:
+                    return cached
+
+            if force_refresh:
+                toolchain.clear_cache()
+                self._adb_probe.invalidate_enrichment_cache()
+
+            build_generation = self._cache_generation
+            report = await self._build_report(categories)
+            # If a device/configuration change happened during the scan, return
+            # this result only to its original caller and never publish it as the
+            # shared snapshot for later requests.
+            if cacheable and build_generation == self._cache_generation:
+                self._report_cache = report.model_copy(deep=True)
+                self._report_cache_time = time.monotonic()
+                self._report_cache_generation = build_generation
+            return report
+
+    async def _build_report(
+        self, categories: list[ProbeCategory] | None = None
+    ) -> SystemReadinessReport:
+        """Execute the underlying probes and compile an uncached report."""
+        target_probes = [
+            probe
+            for probe in self._probes.values()
+            if categories is None or probe.category in categories
+        ]
+
+        # Concurrently execute probes. A probe that raises must not take the
+        # whole report down with it: it becomes a structured FAIL so the
+        # remaining probes still reach the user and the verdict stays honest.
+        outcomes = await asyncio.gather(
+            *[
+                asyncio.wait_for(probe.probe(), timeout=self.PROBE_TIMEOUT_SECONDS)
+                for probe in target_probes
+            ],
+            return_exceptions=True,
+        )
+        results: list[ProbeResult] = [
+            outcome
+            if isinstance(outcome, ProbeResult)
+            else self._crashed_probe_result(probe, outcome)
+            for probe, outcome in zip(target_probes, outcomes, strict=True)
+        ]
+
+        blockers = [r for r in results if r.is_blocker]
+        passed_blockers = [r for r in blockers if r.status == ProbeStatus.PASS]
+        overall_ready = len(blockers) > 0 and len(blockers) == len(passed_blockers)
+
+        # Extract active device info from ADB probe metadata if available
+        active_device: DeviceInfo | None = None
+        adb_result = next((r for r in results if r.id == "android_adb"), None)
+        if adb_result and adb_result.metadata.get("active_device"):
+            try:
+                active_device = DeviceInfo(**adb_result.metadata["active_device"])
+            except (TypeError, ValueError) as exc:
+                # Includes pydantic ValidationError (a ValueError subclass).
+                logger.warning(
+                    f"Malformed active_device metadata from ADB probe; readiness"
+                    f" report will omit the active device: {exc}"
+                )
+
+        return SystemReadinessReport(
+            overall_ready=overall_ready,
+            blocker_count=len(blockers),
+            passed_blocker_count=len(passed_blockers),
+            probes=results,
+            active_device=active_device,
+            os_type=platform.os_type.value,
+            timestamp=time.time(),
+        )
+
+    @staticmethod
+    def _crashed_probe_result(probe: BaseProbe, exc: object) -> ProbeResult:
+        """Turn an exception escaping ``probe.probe()`` into a FAIL result.
+
+        Cancellation and other non-``Exception`` errors are re-raised: they are
+        not a diagnosis of the host, they are the event loop shutting us down.
+        """
+        if isinstance(exc, BaseException) and not isinstance(exc, Exception):
+            raise exc
+        if isinstance(exc, TimeoutError):
+            logger.warning(
+                f"[ReadinessEngine] Probe '{probe.probe_id}' did not finish within"
+                f" {ReadinessEngine.PROBE_TIMEOUT_SECONDS:.0f}s; reporting it as FAIL."
+            )
+            return ProbeResult(
+                id=probe.probe_id,
+                category=probe.category,
+                title=probe.probe_id.replace("_", " ").title(),
+                status=ProbeStatus.FAIL,
+                is_blocker=probe.is_blocker,
+                summary="Probe timed out",
+                description=(
+                    f"The '{probe.probe_id}' check did not finish within"
+                    f" {ReadinessEngine.PROBE_TIMEOUT_SECONDS:.0f}s. A hung ADB server is the"
+                    " usual cause: restart it and run the diagnosis again."
+                ),
+                actions=[
+                    ProbeAction(
+                        action_type="command",
+                        label="Restart ADB",
+                        payload="adb kill-server && adb start-server",
+                    ),
+                ],
+                metadata={
+                    "exception_type": "TimeoutError",
+                    "timeout_seconds": ReadinessEngine.PROBE_TIMEOUT_SECONDS,
+                },
+            )
+        logger.warning(
+            f"[ReadinessEngine] Probe '{probe.probe_id}' crashed; reporting it as FAIL: "
+            f"{type(exc).__name__}: {exc}"
+        )
+        return ProbeResult(
+            id=probe.probe_id,
+            category=probe.category,
+            title=probe.probe_id.replace("_", " ").title(),
+            status=ProbeStatus.FAIL,
+            is_blocker=probe.is_blocker,
+            summary="Probe crashed",
+            description=(
+                f"The '{probe.probe_id}' check raised {type(exc).__name__}: {exc}. "
+                "This is a diagnostics bug or a host permission problem, not a device fault."
+            ),
+            metadata={"exception_type": type(exc).__name__, "exception": str(exc)},
+        )
+
+    async def heal_adb_keys(self, force: bool = False) -> dict[str, Any]:
+        """Auto-heal corrupted or 0-byte ADB authentication keys."""
+        from artemis.core.diagnostics.adb_keys import heal_adb_keys
+
+        logger.info("[ReadinessEngine] Auto-healing ADB authentication keys...")
+        return await asyncio.to_thread(heal_adb_keys, None, force)
+
+    async def restart_adb_server(self) -> dict[str, Any]:
+        """Execute adb kill-server && adb start-server to recover connectivity, auto-healing corrupted keys if needed."""
+        from artemis.core.diagnostics.adb_keys import heal_adb_keys, inspect_adb_keys
+
+        endpoint = adb_server_connection.current_endpoint()
+        if not endpoint.is_local_default:
+            return {
+                "success": True,
+                "skipped": True,
+                "message": (
+                    "Remote ADB is active. Artemis refreshed device discovery without stopping "
+                    "the active ADB server endpoint."
+                ),
+                "endpoint": endpoint.to_dict(),
+            }
+
+        adb_path = toolchain.resolve("adb") or "adb"
+
+        def _restart_sync():
+            # If keys are corrupted, heal them first
+            key_status = inspect_adb_keys()
+            if key_status.is_corrupted:
+                logger.warning(
+                    f"[ReadinessEngine] Corrupted ADB keys detected ({key_status.error_reason}). Auto-healing..."
+                )
+                return heal_adb_keys(adb_path=adb_path)
+
+            try:
+                clean_env = os.environ.copy()
+                clean_env.pop("ADB_SERVER_SOCKET", None)
+                subprocess.run(
+                    [adb_path, "kill-server"],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL,
+                    timeout=5,
+                    env=clean_env,
+                )
+                res = subprocess.run(
+                    [adb_path, "start-server"],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL,
+                    timeout=5,
+                    env=clean_env,
+                )
+                success = res.returncode == 0
+                return {
+                    "success": success,
+                    "message": "ADB server restarted successfully"
+                    if success
+                    else "Failed to restart ADB",
+                    "output": "",
+                }
+            except Exception as exc:
+                return {
+                    "success": False,
+                    "message": f"Error executing ADB restart: {exc}",
+                }
+
+        logger.info("[ReadinessEngine] Restarting ADB server...")
+        return await asyncio.to_thread(_restart_sync)
+
+    async def launch_emulator(self, avd_name: str) -> dict[str, Any]:
+        """Launch an Android emulator by AVD name in the background and track its lifecycle."""
+        if not adb_server_connection.current_endpoint().is_local_default:
+            return {
+                "avd_name": avd_name,
+                "status": "failed",
+                "pid": None,
+                "serial": None,
+                "error": "Switch to local ADB before launching a local emulator.",
+                "stage_message": "Local ADB is not active",
+                "progress_percent": 0,
+                "started_at": None,
+                "elapsed_seconds": 0,
+                "logs": [],
+                "can_retry": True,
+            }
+        from artemis.core.diagnostics.emulator_manager import emulator_manager
+
+        state = await emulator_manager.launch(avd_name)
+        return state.model_dump()
+
+    def get_emulator_status(self) -> dict[str, Any]:
+        """Query current status of background emulator launch."""
+        from artemis.core.diagnostics.emulator_manager import emulator_manager
+
+        return emulator_manager.get_status().model_dump()
+
+    async def stop_emulator(self) -> dict[str, Any]:
+        """Stop current running emulator."""
+        from artemis.core.diagnostics.emulator_manager import emulator_manager
+
+        return await emulator_manager.stop()
+
+    def dismiss_emulator(self) -> dict[str, Any]:
+        """Dismiss current emulator launch state."""
+        from artemis.core.diagnostics.emulator_manager import emulator_manager
+
+        return emulator_manager.dismiss()
+
+    async def connect_wireless_adb(self, host: str, port: int = 5555) -> dict[str, Any]:
+        """Connect to an Android device over Wi-Fi via adb connect."""
+        if not adb_server_connection.current_endpoint().is_local_default:
+            return {
+                "success": False,
+                "message": "Switch to local ADB before connecting a Wireless ADB device.",
+            }
+        clean_host = host.strip()
+        if not clean_host:
+            return {"success": False, "message": "Host IP address cannot be empty"}
+        target = f"{clean_host}:{port}"
+        adb_path = toolchain.resolve("adb") or "adb"
+
+        def _connect_sync():
+            try:
+                res = subprocess.run(
+                    [adb_path, "connect", target],
+                    capture_output=True,
+                    text=True,
+                    timeout=8,
+                )
+                output = (res.stdout + "\n" + res.stderr).strip()
+                success = "connected to" in output.lower() and "failed" not in output.lower()
+                return {
+                    "success": success,
+                    "target": target,
+                    "output": output,
+                    "message": f"Connected to {target}" if success else output,
+                }
+            except Exception as exc:
+                return {
+                    "success": False,
+                    "target": target,
+                    "message": f"Connection error: {exc}",
+                }
+
+        logger.info(f"[ReadinessEngine] Connecting to wireless ADB target '{target}'...")
+        return await asyncio.to_thread(_connect_sync)
+
+
+# Global singleton instance
+readiness_engine = ReadinessEngine()

package/artemis-main/artemis/core/diagnostics/hierarchy_parity.py

@@ -0,0 +1,216 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Real-device parity check between the Accessibility Helper and UIAutomator2.
+
+The unit tests prove the two backends produce the same element *shapes*; only
+a real screen can prove they describe the same *content*. ``compare_backends``
+dumps the current screen with both, then checks what the agent depends on:
+
+* every labelled UIAutomator element (text or content-desc) has a helper
+  element with the same label at the same place (bounds IoU >= 0.5), and vice
+  versa, so neither backend invents or loses what is on screen;
+* the helper emits no negative and no off-screen bounds (its bounds are
+  clipped to the display, like UIAutomator's);
+* dump latency of both, for the record.
+
+``artemis helper parity`` is the CLI front end.
+"""
+
+from __future__ import annotations
+
+from statistics import median
+import time
+from typing import Any
+
+from third_party.mobile_use.clients.ui_automator_client import parse_hierarchy_xml_to_elements
+
+MIN_RECALL = 0.9
+MIN_PRECISION = 0.8
+
+
+def _label(element: dict[str, Any]) -> str:
+    text = str(element.get("text") or "").strip()
+    if text:
+        return text
+    return str(element.get("content-desc") or "").strip()
+
+
+def _iou(a: dict[str, int], b: dict[str, int]) -> float:
+    left = max(a["left"], b["left"])
+    top = max(a["top"], b["top"])
+    right = min(a["right"], b["right"])
+    bottom = min(a["bottom"], b["bottom"])
+    if right <= left or bottom <= top:
+        return 0.0
+    inter = (right - left) * (bottom - top)
+    area_a = max(1, (a["right"] - a["left"]) * (a["bottom"] - a["top"]))
+    area_b = max(1, (b["right"] - b["left"]) * (b["bottom"] - b["top"]))
+    return inter / (area_a + area_b - inter)
+
+
+def analyze_xml(xml: str, width: int, height: int) -> dict[str, Any]:
+    """Counts and labelled elements of one dump, plus the bounds problems the helper must not have."""
+    elements = parse_hierarchy_xml_to_elements(xml)
+    labelled: list[tuple[str, dict[str, int]]] = []
+    negative = offscreen = 0
+    package = None
+    for element in elements:
+        bounds = element.get("parsed_bounds")
+        if package is None and element.get("package"):
+            package = element["package"]
+        if not isinstance(bounds, dict):
+            continue
+        if min(bounds.values()) < 0:
+            negative += 1
+        if (
+            bounds["left"] >= width
+            or bounds["top"] >= height
+            or bounds["right"] > width
+            or bounds["bottom"] > height
+        ):
+            offscreen += 1
+        label = _label(element)
+        if label:
+            labelled.append((label, bounds))
+    return {
+        "nodes": len(elements),
+        "labelled": len(labelled),
+        "labels": labelled,
+        "negative_bounds": negative,
+        "offscreen_bounds": offscreen,
+        "package": package,
+    }
+
+
+def match_labels(
+    source: list[tuple[str, dict[str, int]]], target: list[tuple[str, dict[str, int]]]
+) -> tuple[int, list[str]]:
+    """How many labelled ``source`` elements have a same-label, same-place element in ``target``."""
+    used = [False] * len(target)
+    found = 0
+    missing: list[str] = []
+    for label, bounds in source:
+        hit = False
+        for i, (other_label, other_bounds) in enumerate(target):
+            if used[i] or other_label != label:
+                continue
+            if _iou(bounds, other_bounds) >= 0.5:
+                used[i] = True
+                hit = True
+                break
+        if hit:
+            found += 1
+        else:
+            missing.append(f"{label!r} at {bounds}")
+    return found, missing
+
+
+def compare_dumps(helper_xml: str, uiautomator_xml: str, width: int, height: int) -> dict[str, Any]:
+    """Pure comparison of two dumps of the same screen."""
+    helper = analyze_xml(helper_xml, width, height)
+    u2 = analyze_xml(uiautomator_xml, width, height)
+    u2_found, missing_in_helper = match_labels(u2["labels"], helper["labels"])
+    helper_found, extra_in_helper = match_labels(helper["labels"], u2["labels"])
+    recall = u2_found / u2["labelled"] if u2["labelled"] else 1.0
+    precision = helper_found / helper["labelled"] if helper["labelled"] else 1.0
+    problems: list[str] = []
+    if helper["negative_bounds"]:
+        problems.append(
+            f"helper emitted {helper['negative_bounds']} element(s) with negative bounds"
+        )
+    if helper["offscreen_bounds"]:
+        problems.append(
+            f"helper emitted {helper['offscreen_bounds']} element(s) outside the screen"
+        )
+    if recall < MIN_RECALL:
+        problems.append(
+            f"only {recall:.0%} of UIAutomator's labelled elements appear in the helper dump "
+            f"(need {MIN_RECALL:.0%})"
+        )
+    if precision < MIN_PRECISION:
+        problems.append(
+            f"only {precision:.0%} of the helper's labelled elements appear in the UIAutomator dump "
+            f"(need {MIN_PRECISION:.0%}); the helper is showing things that are not on screen"
+        )
+    for side in (helper, u2):
+        side.pop("labels", None)
+    return {
+        "ok": not problems,
+        "problems": problems,
+        "helper": helper,
+        "uiautomator": u2,
+        "match": {
+            "uiautomator_labelled": u2["labelled"],
+            "uiautomator_in_helper": u2_found,
+            "recall": recall,
+            "helper_labelled": helper["labelled"],
+            "helper_in_uiautomator": helper_found,
+            "precision": precision,
+            "missing_in_helper": missing_in_helper,
+            "extra_in_helper": extra_in_helper,
+        },
+    }
+
+
+def compare_backends(serial: str, rounds: int = 3) -> dict[str, Any]:
+    """Dump the current screen of ``serial`` with both backends and compare (device required)."""
+    from artemis.clients.accessibility_client import AccessibilityClient
+    from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient
+
+    helper = AccessibilityClient(serial)
+    u2 = UIAutomatorClient(serial)
+
+    def timed(dump) -> tuple[str, float]:
+        times: list[float] = []
+        xml = ""
+        for _ in range(rounds):
+            started = time.perf_counter()
+            xml = dump()
+            times.append((time.perf_counter() - started) * 1000.0)
+        return xml, median(times)
+
+    # The two cannot run at once: UIAutomator2's UiAutomation connection unbinds
+    # the helper. Dump with the helper first, then with UIAutomator2, then stop
+    # its server so the helper is back for whoever comes next.
+    helper_version = None
+    try:
+        helper.connect()
+        helper_version = helper.session.version_name if helper.session else None
+        info = helper.get_hierarchy_json()
+        width, height = int(info.get("width") or 0), int(info.get("height") or 0)
+        if width <= 0 or height <= 0:
+            screen = helper.get_screen_data()
+            width, height = screen.width, screen.height
+        helper_xml, helper_ms = timed(helper.get_hierarchy)
+    finally:
+        helper.disconnect()
+
+    try:
+        u2.connect()
+        u2_xml, u2_ms = timed(u2.get_hierarchy)
+    finally:
+        u2.disconnect(stop_server=True)
+
+    report = compare_dumps(helper_xml, u2_xml, width, height)
+    report["serial"] = serial
+    report["screen"] = {"width": width, "height": height}
+    report["helper"]["median_ms"] = helper_ms
+    report["helper"]["version"] = helper_version
+    report["uiautomator"]["median_ms"] = u2_ms
+    report["xml"] = {"helper": helper_xml, "uiautomator": u2_xml}
+    return report
+
+
+__all__ = ["analyze_xml", "compare_backends", "compare_dumps", "match_labels"]