android-midscene-automation 0.2.3 → 1.1.0

package/artemis-main/mcp_server/base.py

@@ -0,0 +1,49 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared FastMCP instance definition for the mobile automation MCP server."""
+
+import logging
+
+from mcp.server.fastmcp import FastMCP
+
+logger = logging.getLogger(__name__)
+
+try:
+    from mcp.server.fastmcp.server import Settings as FastMCPSettings
+
+    FastMCPSettings.model_rebuild()
+except Exception as exc:
+    # Version-compat shim: older/newer FastMCP releases may not expose
+    # Settings or need the rebuild; the server works without it.
+    logger.debug("FastMCP Settings.model_rebuild skipped: %s", exc, exc_info=True)
+
+# Define the shared FastMCP instance for external IDE and agent clients.
+mcp = FastMCP(
+    "artemis",
+    instructions=(
+        "ARTEMIS is an autonomous mobile AI agent and Android UI automation engine. "
+        "Use mobile_run_task to launch autonomous UI workflows on connected Android devices or emulators, "
+        "mobile_manage_task to check status or steer execution, "
+        "mobile_get_device_state to inspect real-time device screen/hierarchy, "
+        "mobile_inspect_trace to inspect detailed execution steps and visual action overlays, "
+        "and mobile_diagnose whenever a tool errors, no device is found, or the user says "
+        "ARTEMIS is not working: it checks the environment and returns ordered fix steps."
+    ),
+)
+
+# Tool registration is intentionally NOT done here: mcp_server/__init__.py
+# imports mcp_server.tools (which imports this module), so importing this
+# module from anywhere already registers every tool. Importing tools here as
+# well would create a circular import (base -> tools -> base).

package/artemis-main/mcp_server/notifiers/__init__.py

@@ -0,0 +1,74 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal MCP Multi-Environment Notification System."""
+
+from typing import Any, Dict, Optional
+
+from mcp_server.notifiers.agentapi import AgentApiNotifier
+from mcp_server.notifiers.base import BaseNotifier
+from mcp_server.notifiers.composite import CompositeNotifier
+from mcp_server.notifiers.desktop import DesktopNotifier
+from mcp_server.notifiers.file import FileNotifier
+from mcp_server.notifiers.script import ScriptNotifier
+from mcp_server.notifiers.webhook import WebhookNotifier
+
+_default_notifier = CompositeNotifier()
+
+
+def get_default_notifier() -> CompositeNotifier:
+    """Returns the global default composite notifier instance."""
+    return _default_notifier
+
+
+def notify(
+    conversation_id: str,
+    message: str,
+    title: str | None = None,
+    event_type: str = "completed",
+    payload: dict[str, Any] | None = None,
+) -> bool:
+    """Dispatches a notification across all available channels (AgentAPI, Webhook, Desktop, Script, File)."""
+    try:
+        return _default_notifier.notify(
+            conversation_id=conversation_id,
+            message=message,
+            title=title,
+            event_type=event_type,
+            payload=payload,
+        )
+    except Exception as e:
+        import logging
+
+        logging.getLogger("mcp_server.notifiers").debug(f"Global notify error: {e}")
+        return False
+
+
+def notify_jetski(conversation_id: str, message: str) -> bool:
+    """Backward-compatible alias for notify()."""
+    return notify(conversation_id=conversation_id, message=message, event_type="completed")
+
+
+__all__ = [
+    "BaseNotifier",
+    "AgentApiNotifier",
+    "WebhookNotifier",
+    "DesktopNotifier",
+    "ScriptNotifier",
+    "FileNotifier",
+    "CompositeNotifier",
+    "get_default_notifier",
+    "notify",
+    "notify_jetski",
+]

package/artemis-main/mcp_server/notifiers/agentapi.py

@@ -0,0 +1,289 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""AgentAPI Notifier for Jetski / Antigravity environments."""
+
+import json
+import logging
+import os
+import shutil
+import subprocess
+from typing import Any
+
+from mcp_server.notifiers.base import BaseNotifier
+
+logger = logging.getLogger("mcp_server.notifiers.agentapi")
+
+
+class AgentApiNotifier(BaseNotifier):
+    """Notifier implementation for environments supporting the agentapi CLI (e.g. Jetski / Antigravity)."""
+
+    def __init__(self):
+        super().__init__()
+        try:
+            addr = os.environ.get("ANTIGRAVITY_LS_ADDRESS")
+            token = os.environ.get("ANTIGRAVITY_CSRF_TOKEN")
+            if addr and token:
+                self._save_shared_env(addr, token)
+        except Exception as exc:
+            # Environment recovery is opportunistic; notifier construction
+            # must never fail because of it.
+            logger.debug("[EnvRecovery] Skipped saving shared env: %s", exc, exc_info=True)
+
+    @property
+    def name(self) -> str:
+        return "agentapi"
+
+    def _find_agentapi_path(self) -> str | None:
+        """Locates the agentapi executable."""
+        try:
+            which_path = shutil.which("agentapi")
+            if which_path:
+                return which_path
+
+            candidates = [
+                os.path.expanduser("~/.gemini/jetski/bin/agentapi"),
+                os.path.expanduser("~/.artemis/bin/agentapi"),
+                os.path.expanduser("~/bin/agentapi"),
+                "/usr/local/bin/agentapi",
+            ]
+            for cand in candidates:
+                if os.path.exists(cand) and os.access(cand, os.X_OK):
+                    return cand
+        except OSError:
+            # Filesystem probe failed: report the binary as absent.
+            pass
+        return None
+
+    def is_available(self) -> bool:
+        """Returns True if agentapi binary exists or LS address is detected."""
+        try:
+            if self._find_agentapi_path():
+                return True
+            if "ANTIGRAVITY_LS_ADDRESS" in os.environ:
+                return True
+        except OSError:
+            # Filesystem probe failed: report the notifier as unavailable.
+            pass
+        return False
+
+    def _save_shared_env(self, addr: str, token: str) -> None:
+        """Persists recovered environment to shared files for faster subsequent access."""
+        shared_candidates = [
+            os.path.expanduser("~/.gemini/jetski/.jetski_env"),
+            os.path.expanduser("~/.artemis/.artemis_env"),
+        ]
+        try:
+            parent_dir = os.path.dirname(
+                os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            )
+            shared_candidates.append(os.path.join(parent_dir, ".jetski_env"))
+        except OSError:
+            # Repo-relative candidate is optional; the home-dir ones remain.
+            pass
+
+        data = {
+            "ANTIGRAVITY_LS_ADDRESS": addr,
+            "ANTIGRAVITY_CSRF_TOKEN": token,
+        }
+        for shared_file in shared_candidates:
+            try:
+                os.makedirs(os.path.dirname(shared_file), exist_ok=True)
+                with open(shared_file, "w", encoding="utf-8") as f:
+                    json.dump(data, f, indent=2)
+                logger.debug(f"[EnvRecovery] Saved session to {shared_file}.")
+            except Exception as e:
+                logger.debug(f"[EnvRecovery] Error writing shared env file {shared_file}: {e}")
+
+    def _get_candidate_envs(self, force_proc_scan: bool = False) -> list[tuple[str, str]]:
+        """Returns unique candidate (address, token) pairs ordered by priority and recency."""
+        candidates: list[tuple[str, str]] = []
+        seen: set[tuple[str, str]] = set()
+
+        def _add(addr: str | None, token: str | None) -> None:
+            if addr and token and (addr, token) not in seen:
+                seen.add((addr, token))
+                candidates.append((addr, token))
+
+        # 1. Current environment variables
+        _add(
+            os.environ.get("ANTIGRAVITY_LS_ADDRESS"),
+            os.environ.get("ANTIGRAVITY_CSRF_TOKEN"),
+        )
+
+        # 2. Shared env files (if not forcing fresh process scan)
+        if not force_proc_scan:
+            shared_candidates = [
+                os.path.expanduser("~/.gemini/jetski/.jetski_env"),
+                os.path.expanduser("~/.artemis/.artemis_env"),
+            ]
+            try:
+                parent_dir = os.path.dirname(
+                    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+                )
+                shared_candidates.append(os.path.join(parent_dir, ".jetski_env"))
+            except OSError:
+                # Repo-relative candidate is optional; the home-dir ones remain.
+                pass
+
+            for shared_file in shared_candidates:
+                if os.path.exists(shared_file):
+                    try:
+                        with open(shared_file, encoding="utf-8") as f:
+                            env_data = json.load(f)
+                        _add(
+                            env_data.get("ANTIGRAVITY_LS_ADDRESS"),
+                            env_data.get("ANTIGRAVITY_CSRF_TOKEN"),
+                        )
+                    except Exception as e:
+                        logger.debug(
+                            f"[EnvRecovery] Error reading shared env file {shared_file}: {e}"
+                        )
+
+        # 3. Scan process table for active agent sessions, sorted by newest creation time first
+        proc_matches: list[tuple[float, str, str]] = []
+        try:
+            import psutil
+
+            for proc in psutil.process_iter(["create_time", "environ"]):
+                try:
+                    env = proc.info.get("environ")
+                    if env and "ANTIGRAVITY_LS_ADDRESS" in env and "ANTIGRAVITY_CSRF_TOKEN" in env:
+                        ctime = proc.info.get("create_time", 0.0) or 0.0
+                        proc_matches.append(
+                            (
+                                ctime,
+                                env["ANTIGRAVITY_LS_ADDRESS"],
+                                env["ANTIGRAVITY_CSRF_TOKEN"],
+                            )
+                        )
+                except psutil.Error:
+                    # This process vanished or denies access: scan the rest.
+                    continue
+        except ImportError:
+            import glob
+
+            for env_path in glob.glob("/proc/[0-9]*/environ"):
+                try:
+                    mtime = os.path.getmtime(env_path)
+                    with open(env_path, "rb") as f:
+                        content = f.read()
+                    env_vars = content.split(b"\x00")
+                    local_addr = None
+                    local_token = None
+                    for var in env_vars:
+                        if var.startswith(b"ANTIGRAVITY_LS_ADDRESS="):
+                            local_addr = var.split(b"=", 1)[1].decode("utf-8", errors="ignore")
+                        elif var.startswith(b"ANTIGRAVITY_CSRF_TOKEN="):
+                            local_token = var.split(b"=", 1)[1].decode("utf-8", errors="ignore")
+                    if local_addr and local_token:
+                        proc_matches.append((mtime, local_addr, local_token))
+                except OSError:
+                    # /proc entries vanish or deny access as processes exit.
+                    pass
+
+        # Sort matches descending by creation timestamp so active/newest sessions come first
+        proc_matches.sort(key=lambda x: x[0], reverse=True)
+        for _, addr, token in proc_matches:
+            _add(addr, token)
+
+        return candidates
+
+    def _load_shared_env(self, force_proc_scan: bool = False) -> None:
+        """Recovers and loads the newest candidate ANTIGRAVITY_LS_ADDRESS and ANTIGRAVITY_CSRF_TOKEN."""
+        candidates = self._get_candidate_envs(force_proc_scan=force_proc_scan)
+        if candidates:
+            addr, token = candidates[0]
+            os.environ["ANTIGRAVITY_LS_ADDRESS"] = addr
+            os.environ["ANTIGRAVITY_CSRF_TOKEN"] = token
+            logger.debug(f"[EnvRecovery] Loaded session candidate: {addr}")
+
+    def notify(
+        self,
+        conversation_id: str,
+        message: str,
+        title: str | None = None,
+        event_type: str = "completed",
+        payload: dict[str, Any] | None = None,
+    ) -> bool:
+        """Dispatches a notification via the agentapi send-message command."""
+        try:
+            if not conversation_id or conversation_id.lower() in ("default", "none", ""):
+                return False
+
+            agentapi_path = self._find_agentapi_path()
+            if not agentapi_path:
+                return False
+
+            cmd = [agentapi_path, "send-message"]
+            if title:
+                cmd.append(f"--title={title}")
+            cmd.extend([conversation_id, message])
+
+            candidates = self._get_candidate_envs(force_proc_scan=False)
+            if not candidates:
+                candidates = self._get_candidate_envs(force_proc_scan=True)
+
+            for attempt, (addr, token) in enumerate(candidates):
+                env = os.environ.copy()
+                env["ANTIGRAVITY_LS_ADDRESS"] = addr
+                env["ANTIGRAVITY_CSRF_TOKEN"] = token
+                try:
+                    subprocess.run(
+                        cmd,
+                        capture_output=True,
+                        text=True,
+                        check=True,
+                        timeout=10,
+                        env=env,
+                    )
+                    os.environ["ANTIGRAVITY_LS_ADDRESS"] = addr
+                    os.environ["ANTIGRAVITY_CSRF_TOKEN"] = token
+                    self._save_shared_env(addr, token)
+                    logger.info(f"AgentAPI notification sent successfully via {addr}.")
+                    return True
+                except Exception as err:
+                    logger.debug(f"Candidate session {addr} failed (attempt {attempt + 1}): {err}")
+
+            # If all initial candidates failed, force a fresh process scan in case a new session started
+            fresh_candidates = self._get_candidate_envs(force_proc_scan=True)
+            for addr, token in fresh_candidates:
+                if (addr, token) in candidates:
+                    continue
+                env = os.environ.copy()
+                env["ANTIGRAVITY_LS_ADDRESS"] = addr
+                env["ANTIGRAVITY_CSRF_TOKEN"] = token
+                try:
+                    subprocess.run(
+                        cmd,
+                        capture_output=True,
+                        text=True,
+                        check=True,
+                        timeout=10,
+                        env=env,
+                    )
+                    os.environ["ANTIGRAVITY_LS_ADDRESS"] = addr
+                    os.environ["ANTIGRAVITY_CSRF_TOKEN"] = token
+                    self._save_shared_env(addr, token)
+                    logger.info(
+                        f"AgentAPI notification sent successfully on fresh scan via {addr}."
+                    )
+                    return True
+                except Exception as err:
+                    logger.debug(f"Fresh candidate session {addr} failed: {err}")
+
+            return False
+        except Exception as e:
+            logger.debug(f"AgentApiNotifier encountered unhandled error: {e}")
+            return False

package/artemis-main/mcp_server/notifiers/base.py

@@ -0,0 +1,45 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Base interface for MCP notification adapters."""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseNotifier(ABC):
+    """Abstract base class for environment notification providers."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Name of the notifier adapter."""
+        pass
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Checks if the notifier is configured and available in the current runtime environment."""
+        pass
+
+    @abstractmethod
+    def notify(
+        self,
+        conversation_id: str,
+        message: str,
+        title: str | None = None,
+        event_type: str = "completed",
+        payload: dict[str, Any] | None = None,
+    ) -> bool:
+        """Sends a notification to the target conversation / client."""
+        pass

package/artemis-main/mcp_server/notifiers/composite.py

@@ -0,0 +1,87 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Composite Notifier coordinating multi-channel notifications."""
+
+import logging
+from typing import Any
+
+from mcp_server.notifiers.agentapi import AgentApiNotifier
+from mcp_server.notifiers.base import BaseNotifier
+from mcp_server.notifiers.desktop import DesktopNotifier
+from mcp_server.notifiers.file import FileNotifier
+from mcp_server.notifiers.script import ScriptNotifier
+from mcp_server.notifiers.webhook import WebhookNotifier
+
+logger = logging.getLogger("mcp_server.notifiers.composite")
+
+
+class CompositeNotifier(BaseNotifier):
+    """Aggregates multiple notifiers and broadcasts events across all available channels."""
+
+    def __init__(self, notifiers: list[BaseNotifier] | None = None):
+        if notifiers is not None:
+            self._notifiers = notifiers
+        else:
+            self._notifiers = [
+                FileNotifier(),
+                AgentApiNotifier(),
+                WebhookNotifier(),
+                ScriptNotifier(),
+                DesktopNotifier(),
+            ]
+
+    @property
+    def name(self) -> str:
+        return "composite"
+
+    def is_available(self) -> bool:
+        for n in self._notifiers:
+            try:
+                if n.is_available():
+                    return True
+            except Exception:
+                continue
+        return False
+
+    def register_notifier(self, notifier: BaseNotifier) -> None:
+        """Registers an additional custom notifier."""
+        self._notifiers.append(notifier)
+
+    def notify(
+        self,
+        conversation_id: str,
+        message: str,
+        title: str | None = None,
+        event_type: str = "completed",
+        payload: dict[str, Any] | None = None,
+    ) -> bool:
+        any_success = False
+        for notifier in self._notifiers:
+            try:
+                if not notifier.is_available():
+                    continue
+                success = notifier.notify(
+                    conversation_id=conversation_id,
+                    message=message,
+                    title=title,
+                    event_type=event_type,
+                    payload=payload,
+                )
+                if success:
+                    any_success = True
+            except Exception as e:
+                logger.debug(f"Notifier '{notifier.name}' encountered error: {e}")
+
+        return any_success

package/artemis-main/mcp_server/notifiers/desktop.py

@@ -0,0 +1,108 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Desktop Toast / OS notification adapter."""
+
+import logging
+import os
+import shutil
+import subprocess
+import sys
+from typing import Any
+
+from mcp_server.notifiers.base import BaseNotifier
+
+logger = logging.getLogger("mcp_server.notifiers.desktop")
+
+
+class DesktopNotifier(BaseNotifier):
+    """Notifier that displays native desktop notifications across macOS, Linux, and Windows."""
+
+    @property
+    def name(self) -> str:
+        return "desktop"
+
+    def is_available(self) -> bool:
+        try:
+            val = os.getenv("ARTEMIS_DESKTOP_NOTIFY", "").lower()
+            if val in ("0", "false", "no", "off"):
+                return False
+            if os.getenv("CI", "").lower() in ("1", "true", "yes") and val not in (
+                "1",
+                "true",
+                "yes",
+            ):
+                return False
+            if sys.platform == "linux" and not shutil.which("notify-send"):
+                return False
+            if sys.platform == "darwin" and not shutil.which("osascript"):
+                return False
+            return True
+        except Exception:
+            return False
+
+    def notify(
+        self,
+        conversation_id: str,
+        message: str,
+        title: str | None = None,
+        event_type: str = "completed",
+        payload: dict[str, Any] | None = None,
+    ) -> bool:
+        if not self.is_available():
+            return False
+
+        header = title or f"☕ Artemis Task {event_type.capitalize()}"
+        clean_body = message.split("\n\n")[0][:120]
+
+        try:
+            if sys.platform == "linux":
+                if shutil.which("notify-send"):
+                    subprocess.run(
+                        ["notify-send", header, clean_body],
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL,
+                        timeout=3,
+                    )
+                    return True
+            elif sys.platform == "darwin":
+                if shutil.which("osascript"):
+                    script = f'display notification "{clean_body}" with title "{header}"'
+                    subprocess.run(
+                        ["osascript", "-e", script],
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL,
+                        timeout=3,
+                    )
+                    return True
+            elif sys.platform == "win32":
+                ps_cmd = (
+                    f"[Windows.UI.Notifications.ToastNotificationManager, Windows.UI.Notifications, ContentType = WindowsRuntime] > $null; "
+                    f"$template = [Windows.UI.Notifications.ToastNotificationManager]::GetTemplateContent([Windows.UI.Notifications.ToastTemplateType]::ToastText02); "
+                    f'$textNodes = $template.GetElementsByTagName("text"); '
+                    f'$textNodes.Item(0).AppendChild($template.CreateTextNode("{header}")) > $null; '
+                    f'$textNodes.Item(1).AppendChild($template.CreateTextNode("{clean_body}")) > $null; '
+                    f"$toast = [Windows.UI.Notifications.ToastNotification]::new($template); "
+                    f'[Windows.UI.Notifications.ToastNotificationManager]::CreateToastNotifier("Artemis").Show($toast);'
+                )
+                subprocess.run(
+                    ["powershell", "-NoProfile", "-Command", ps_cmd],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL,
+                    timeout=5,
+                )
+                return True
+        except Exception as e:
+            logger.debug(f"Desktop notification failed: {e}")
+        return False