android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/mcp_server/utils/device_utils.py

@@ -0,0 +1,180 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Device discovery, ADB resolution, and emulator management for MCP workflows."""
+
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+
+def resolve_adb_path() -> str:
+    """Resolves the absolute path to the adb binary across platforms."""
+    # 1. Check if adb is in PATH
+    which_adb = shutil.which("adb")
+    if which_adb:
+        return which_adb
+
+    # 2. Check ANDROID_HOME / ANDROID_SDK_ROOT
+    for env_var in ("ANDROID_HOME", "ANDROID_SDK_ROOT"):
+        sdk_root = os.getenv(env_var)
+        if sdk_root:
+            candidate = os.path.join(
+                sdk_root, "platform-tools", "adb.exe" if sys.platform == "win32" else "adb"
+            )
+            if os.path.exists(candidate):
+                return candidate
+
+    # 3. Check standard OS-specific SDK locations
+    candidates = []
+    if sys.platform == "win32":
+        local_appdata = os.getenv("LOCALAPPDATA", "")
+        if local_appdata:
+            candidates.append(
+                os.path.join(local_appdata, "Android", "Sdk", "platform-tools", "adb.exe")
+            )
+    elif sys.platform == "darwin":
+        candidates.extend(
+            [
+                os.path.expanduser("~/Library/Android/sdk/platform-tools/adb"),
+                "/opt/homebrew/bin/adb",
+                "/usr/local/bin/adb",
+            ]
+        )
+    else:  # Linux
+        candidates.extend(
+            [
+                "/usr/bin/adb",
+                os.path.expanduser("~/Android/Sdk/platform-tools/adb"),
+                "/usr/local/bin/adb",
+            ]
+        )
+
+    for cand in candidates:
+        if os.path.exists(cand):
+            return cand
+
+    return "adb"
+
+
+def resolve_emulator_path() -> str | None:
+    """Resolves the path to the Android emulator executable."""
+    which_emulator = shutil.which("emulator")
+    if which_emulator:
+        return which_emulator
+
+    candidates = [
+        os.path.expanduser("~/Android/Sdk/emulator/emulator"),
+        os.path.expanduser("~/Library/Android/sdk/emulator/emulator"),
+    ]
+    if sys.platform == "win32":
+        local_appdata = os.getenv("LOCALAPPDATA", "")
+        if local_appdata:
+            candidates.append(
+                os.path.join(local_appdata, "Android", "Sdk", "emulator", "emulator.exe")
+            )
+
+    for cand in candidates:
+        if os.path.exists(cand):
+            return cand
+    return None
+
+
+def get_connected_devices(adb_path: str | None = None) -> list[str]:
+    """Returns a list of connected and authorized Android device serials."""
+    adb = adb_path or resolve_adb_path()
+    try:
+        res = subprocess.run(
+            [adb, "devices"],
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        devices: list[str] = []
+        for line in res.stdout.splitlines():
+            line_str = line.strip()
+            if line_str and not line_str.startswith("List of devices"):
+                parts = line_str.split()
+                if len(parts) >= 2 and parts[1] == "device":
+                    devices.append(parts[0])
+        return devices
+    except Exception:
+        return []
+
+
+def is_emulator_running(adb_path: str | None = None) -> bool:
+    """Checks if an Android emulator (e.g. emulator-5554) is running and accessible."""
+    adb = adb_path or resolve_adb_path()
+    devices = get_connected_devices(adb)
+    return any(d.startswith("emulator-") for d in devices)
+
+
+def ensure_emulator(
+    avd_name: str = "AndroidWorldAvd",
+    adb_path: str | None = None,
+    emulator_path: str | None = None,
+    timeout_seconds: int = 90,
+) -> bool:
+    """Ensures an emulator is running. If not, boots the specified AVD in the background."""
+    adb = adb_path or resolve_adb_path()
+    if is_emulator_running(adb):
+        return True
+
+    emu_exe = emulator_path or resolve_emulator_path()
+    if not emu_exe or not os.path.exists(emu_exe):
+        return False
+
+    try:
+        command = [emu_exe, "-avd", avd_name, "-no-snapshot", "-grpc", "8554"]
+        if sys.platform == "win32":
+            subprocess.Popen(
+                command,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                creationflags=(subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS),
+            )
+        else:
+            subprocess.Popen(
+                command,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                start_new_session=True,
+            )
+    except Exception:
+        return False
+
+    start_time = time.time()
+    while time.time() - start_time < timeout_seconds:
+        try:
+            res = subprocess.run(
+                [adb, "-s", "emulator-5554", "shell", "getprop", "sys.boot_completed"],
+                stdin=subprocess.DEVNULL,
+                capture_output=True,
+                text=True,
+                timeout=3,
+            )
+            if "1" in res.stdout:
+                time.sleep(2)  # Stabilization
+                return True
+        except (subprocess.SubprocessError, OSError):
+            # adb not ready yet while the emulator boots: keep polling.
+            pass
+        time.sleep(2)
+
+    return False

package/artemis-main/mcp_server/utils/env_utils.py

@@ -0,0 +1,65 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Environment and runtime discovery utilities for MCP."""
+
+import os
+import subprocess
+import sys
+from typing import Any
+
+
+def get_project_root() -> str:
+    """Returns the absolute path to the project root repository."""
+    # Deliberate lazy import: this helper must keep working when the artemis
+    # package (or its config bootstrap) is unavailable or broken, falling back
+    # to filesystem-relative resolution below.
+    try:
+        from artemis.config.paths import ROOT_DIR
+
+        return str(ROOT_DIR)
+    except Exception:
+        # mcp_server/utils/env_utils.py -> root is 3 levels up
+        return os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def resolve_python_executable(project_root: str | None = None) -> str:
+    """Resolves the preferred Python executable (.venv or current sys.executable)."""
+    root = project_root or get_project_root()
+
+    if sys.platform == "win32":
+        venv_python = os.path.join(root, ".venv", "Scripts", "python.exe")
+    else:
+        venv_python = os.path.join(root, ".venv", "bin", "python")
+
+    if os.path.exists(venv_python):
+        return venv_python
+
+    return sys.executable
+
+
+def get_detached_process_kwargs() -> dict[str, Any]:
+    """Returns kwargs for subprocess.Popen to run a fully detached background process cross-platform."""
+    kwargs: dict[str, Any] = {
+        "stdin": subprocess.DEVNULL,
+        "stdout": subprocess.DEVNULL,
+        "stderr": subprocess.DEVNULL,
+    }
+    if sys.platform == "win32":
+        kwargs["creationflags"] = (
+            subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS  # type: ignore[attr-defined]
+        )
+    else:
+        kwargs["start_new_session"] = True
+    return kwargs

package/artemis-main/packages/artemis-accessibility-helper/app/build.gradle.kts

@@ -0,0 +1,35 @@
+plugins {
+    id("com.android.application")
+}
+
+android {
+    namespace = "com.artemis.helper"
+    compileSdk = 35
+
+    defaultConfig {
+        applicationId = "com.artemis.helper"
+        minSdk = 24
+        targetSdk = 35
+        versionCode = 6
+        versionName = "1.2.0"
+    }
+
+    buildTypes {
+        release {
+            isMinifyEnabled = false
+            proguardFiles(
+                getDefaultProguardFile("proguard-android-optimize.txt"),
+                "proguard-rules.pro"
+            )
+        }
+    }
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_1_8
+        targetCompatibility = JavaVersion.VERSION_1_8
+    }
+}
+
+dependencies {
+    // Pure standard Android SDK APIs (android.accessibilityservice, android.view.accessibility, org.json)
+    // Zero external dependencies to maximize stability, guarantee compatibility, and keep APK under 30KB.
+}

package/artemis-main/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.artemis.helper">
+
+    <uses-sdk
+        android:minSdkVersion="24"
+        android:targetSdkVersion="35" />
+
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
+    <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
+
+    <application
+        android:allowBackup="false"
+        android:label="@string/app_name"
+        android:supportsRtl="true">
+
+        <service
+            android:name=".ArtemisAccessibilityService"
+            android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
+            android:exported="true"
+            android:foregroundServiceType="specialUse">
+            <intent-filter>
+                <action android:name="android.accessibilityservice.AccessibilityService" />
+            </intent-filter>
+            <meta-data
+                android:name="android.accessibilityservice"
+                android:resource="@xml/accessibility_service_config" />
+            <property
+                android:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE"
+                android:value="Artemis mobile automation test bridge" />
+        </service>
+
+        <!-- Session token drop: only a sender holding WRITE_SECURE_SETTINGS (the adb
+             shell user, the system) can deliver this broadcast, so no ordinary app on
+             the device can set a token of its own and then talk to the loopback server. -->
+        <receiver
+            android:name=".TokenReceiver"
+            android:exported="true"
+            android:permission="android.permission.WRITE_SECURE_SETTINGS">
+            <intent-filter>
+                <action android:name="com.artemis.helper.SET_TOKEN" />
+            </intent-filter>
+        </receiver>
+
+    </application>
+
+</manifest>

package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java

@@ -0,0 +1,274 @@
+package com.artemis.helper;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * In-memory snapshot of an Accessibility UI element node.
+ * Decoupled from live Android AccessibilityNodeInfo objects to prevent
+ * Binder proxy memory leaks, avoid stale node crashes, and ensure
+ * instant thread-safe serialization to both XML and JSON.
+ *
+ * {@code bounds} are the node's <em>visible</em> bounds: the raw
+ * {@code getBoundsInScreen} rectangle intersected with the display, the
+ * node's window and every scrollable ancestor, exactly like UIAutomator's
+ * {@code getVisibleBoundsInScreen}. They are therefore never negative and
+ * never extend past the screen.
+ */
+public final class A11yNode {
+
+    public int index = 0;
+    public String text = "";
+    public String resourceId = "";
+    public String className = "";
+    public String packageName = "";
+    public String contentDesc = "";
+
+    public boolean checkable = false;
+    public boolean checked = false;
+    public boolean clickable = false;
+    public boolean enabled = true;
+    public boolean focusable = false;
+    public boolean focused = false;
+    public boolean scrollable = false;
+    public boolean longClickable = false;
+    public boolean password = false;
+    public boolean selected = false;
+    public boolean visibleToUser = true;
+
+    // Window metadata for multi-window awareness
+    public int windowId = -1;
+    public String windowType = "";
+    public int windowLayer = 0;
+    public boolean windowActive = false;
+    public boolean windowFocused = false;
+
+    // Advanced semantics for Agent perception & verification
+    public boolean editable = false;
+    public boolean isHeading = false;
+    public boolean screenReaderFocusable = false;
+    public String stateDescription = "";
+    public String errorText = "";
+    public String paneTitle = "";
+    public String tooltip = "";
+
+    public int left = 0;
+    public int top = 0;
+    public int right = 0;
+    public int bottom = 0;
+
+    public int drawingOrder = 0;
+    public String hint = "";
+
+    public final List<A11yNode> children = new ArrayList<>(4);
+
+    public A11yNode() {}
+
+    public int getWidth() {
+        return right - left;
+    }
+
+    public int getHeight() {
+        return bottom - top;
+    }
+
+    public String getBoundsString() {
+        return "[" + left + "," + top + "][" + right + "," + bottom + "]";
+    }
+
+    /**
+     * Serializes this node and all of its descendants into standard UIAutomator XML.
+     */
+    public void writeXml(StringBuilder sb) {
+        sb.append("<node");
+        XmlUtils.appendIntAttribute(sb, "index", index);
+        XmlUtils.appendAttribute(sb, "text", text);
+        XmlUtils.appendAttribute(sb, "resource-id", resourceId);
+        XmlUtils.appendAttribute(sb, "class", className);
+        XmlUtils.appendAttribute(sb, "package", packageName);
+        XmlUtils.appendAttribute(sb, "content-desc", contentDesc);
+        XmlUtils.appendBooleanAttribute(sb, "checkable", checkable);
+        XmlUtils.appendBooleanAttribute(sb, "checked", checked);
+        XmlUtils.appendBooleanAttribute(sb, "clickable", clickable);
+        XmlUtils.appendBooleanAttribute(sb, "enabled", enabled);
+        XmlUtils.appendBooleanAttribute(sb, "focusable", focusable);
+        XmlUtils.appendBooleanAttribute(sb, "focused", focused);
+        XmlUtils.appendBooleanAttribute(sb, "scrollable", scrollable);
+        XmlUtils.appendBooleanAttribute(sb, "long-clickable", longClickable);
+        XmlUtils.appendBooleanAttribute(sb, "password", password);
+        XmlUtils.appendBooleanAttribute(sb, "selected", selected);
+        XmlUtils.appendBooleanAttribute(sb, "visible-to-user", visibleToUser);
+        XmlUtils.appendAttribute(sb, "bounds", getBoundsString());
+        XmlUtils.appendIntAttribute(sb, "drawing-order", drawingOrder);
+        if (!hint.isEmpty()) {
+            XmlUtils.appendAttribute(sb, "hint", hint);
+        }
+
+        // Window metadata (emitted on window root nodes)
+        if (windowId >= 0) {
+            XmlUtils.appendIntAttribute(sb, "window-id", windowId);
+            if (!windowType.isEmpty()) {
+                XmlUtils.appendAttribute(sb, "window-type", windowType);
+            }
+            XmlUtils.appendIntAttribute(sb, "window-layer", windowLayer);
+            if (windowActive) {
+                XmlUtils.appendBooleanAttribute(sb, "window-active", true);
+            }
+            if (windowFocused) {
+                XmlUtils.appendBooleanAttribute(sb, "window-focused", true);
+            }
+        }
+
+        // Extended semantic attributes
+        if (editable) {
+            XmlUtils.appendBooleanAttribute(sb, "editable", true);
+        }
+        if (isHeading) {
+            XmlUtils.appendBooleanAttribute(sb, "heading", true);
+        }
+        if (screenReaderFocusable) {
+            XmlUtils.appendBooleanAttribute(sb, "screen-reader-focusable", true);
+        }
+        if (!stateDescription.isEmpty()) {
+            XmlUtils.appendAttribute(sb, "state-description", stateDescription);
+        }
+        if (!errorText.isEmpty()) {
+            XmlUtils.appendAttribute(sb, "error", errorText);
+        }
+        if (!paneTitle.isEmpty()) {
+            XmlUtils.appendAttribute(sb, "pane-title", paneTitle);
+        }
+        if (!tooltip.isEmpty()) {
+            XmlUtils.appendAttribute(sb, "tooltip", tooltip);
+        }
+
+        if (children.isEmpty()) {
+            sb.append(" />");
+        } else {
+            sb.append('>');
+            for (int i = 0; i < children.size(); i++) {
+                children.get(i).writeXml(sb);
+            }
+            sb.append("</node>");
+        }
+    }
+
+    /**
+     * Serializes this node into a hierarchical JSON tree object.
+     */
+    public JSONObject toTreeJson() {
+        JSONObject obj = toBaseJson();
+        try {
+            if (!children.isEmpty()) {
+                JSONArray childrenArr = new JSONArray();
+                for (int i = 0; i < children.size(); i++) {
+                    childrenArr.put(children.get(i).toTreeJson());
+                }
+                obj.put("children", childrenArr);
+            }
+        } catch (Throwable ignored) {}
+        return obj;
+    }
+
+    /**
+     * Serializes this node into an element JSON object for the flat list.
+     * Does NOT include child references.
+     */
+    public JSONObject toFlatElementJson() {
+        return toBaseJson();
+    }
+
+    private JSONObject toBaseJson() {
+        JSONObject obj = new JSONObject();
+        try {
+            obj.put("class", className);
+            obj.put("package", packageName);
+            obj.put("resource-id", resourceId);
+            obj.put("text", text);
+            obj.put("content-desc", contentDesc);
+            obj.put("bounds", getBoundsString());
+
+            JSONObject bounds = new JSONObject();
+            bounds.put("left", left);
+            bounds.put("top", top);
+            bounds.put("right", right);
+            bounds.put("bottom", bottom);
+            obj.put("parsed_bounds", bounds);
+
+            obj.put("clickable", clickable);
+            obj.put("scrollable", scrollable);
+            obj.put("checkable", checkable);
+            obj.put("checked", checked);
+            obj.put("enabled", enabled);
+            obj.put("focusable", focusable);
+            obj.put("focused", focused);
+            obj.put("selected", selected);
+            obj.put("password", password);
+            obj.put("long-clickable", longClickable);
+            obj.put("visible-to-user", visibleToUser);
+            obj.put("drawing-order", drawingOrder);
+            if (!hint.isEmpty()) {
+                obj.put("hint", hint);
+            }
+
+            // Window metadata
+            if (windowId >= 0) {
+                obj.put("window_id", windowId);
+                obj.put("window_type", windowType);
+                obj.put("window_layer", windowLayer);
+                if (windowActive) obj.put("window_active", true);
+                if (windowFocused) obj.put("window_focused", true);
+            }
+
+            // Extended semantics
+            if (editable) {
+                obj.put("editable", true);
+            }
+            if (isHeading) {
+                obj.put("is_heading", true);
+            }
+            if (screenReaderFocusable) {
+                obj.put("screen_reader_focusable", true);
+            }
+            if (!stateDescription.isEmpty()) {
+                obj.put("state_description", stateDescription);
+            }
+            if (!errorText.isEmpty()) {
+                obj.put("error", errorText);
+            }
+            if (!paneTitle.isEmpty()) {
+                obj.put("pane_title", paneTitle);
+            }
+            if (!tooltip.isEmpty()) {
+                obj.put("tooltip", tooltip);
+            }
+        } catch (Throwable ignored) {}
+        return obj;
+    }
+
+    /**
+     * Determines whether this node contains useful information or interactivity
+     * and should be included in the flat elements array for agent perception.
+     */
+    public boolean isInformativeOrInteractive() {
+        boolean hasContent = !text.isEmpty() || !contentDesc.isEmpty() || !resourceId.isEmpty()
+                || !stateDescription.isEmpty() || !errorText.isEmpty() || !paneTitle.isEmpty();
+        boolean isInteractive = clickable || scrollable || checkable || focusable || longClickable || editable;
+        return hasContent || isInteractive || isHeading;
+    }
+
+    /**
+     * Recursively collects all informative or interactive nodes into a flat list.
+     */
+    public void collectFlatElements(List<JSONObject> flatList) {
+        if (isInformativeOrInteractive()) {
+            flatList.add(toFlatElementJson());
+        }
+        for (int i = 0; i < children.size(); i++) {
+            children.get(i).collectFlatElements(flatList);
+        }
+    }
+}