myagent-ai 1.2.2 → 1.3.1

package/skills/gui_skill.py

@@ -0,0 +1,908 @@
+"""
+skills/gui_skill.py - 桌面 GUI 自动化技能
+=========================================
+提供跨平台桌面 GUI 自动化能力，包括屏幕截图、鼠标操作、键盘输入、窗口管理等。
+支持 Windows 和 macOS。所有依赖均为可选，缺失时给出友好的安装提示。
+
+Skills:
+  - ScreenShotSkill: 捕获屏幕或区域截图
+  - MouseClickSkill: 在屏幕坐标处点击
+  - MouseDragSkill: 从 A 点拖拽到 B 点
+  - TypeTextSkill: 在当前光标位置输入文本
+  - HotkeySkill: 按下键盘快捷键（自动适配平台）
+  - WindowListSkill: 列出所有打开的窗口
+  - WindowFocusSkill: 聚焦/置顶指定窗口
+  - ScreenElementSkill: 使用 VLM 从截图中定位元素坐标
+
+Dependencies (all optional):
+  - mss: 屏幕截图 (跨平台, ~1MB)
+  - pynput: 鼠标/键盘控制 (跨平台, ~500KB)
+  - pygetwindow: 窗口管理 (Windows + macOS)
+"""
+from __future__ import annotations
+
+import sys
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+from core.logger import get_logger
+from skills.base import Skill, SkillResult, SkillParameter
+
+logger = get_logger("myagent.skills.gui")
+
+# 平台检测 / Platform detection
+IS_MACOS = sys.platform == "darwin"
+IS_WINDOWS = sys.platform == "win32"
+IS_LINUX = sys.platform.startswith("linux")
+
+
+def _generate_screenshot_path() -> str:
+    """生成截图文件路径（带时间戳）"""
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
+    return f"/tmp/myagent_gui_screen_{timestamp}.png"
+
+
+class ScreenShotSkill(Skill):
+    """
+    屏幕截图 - 捕获整个屏幕或指定区域的截图。
+
+    使用 mss 库实现跨平台高速截图，比 Pillow.grab 快 10 倍以上。
+    截图保存为 PNG 文件，返回路径供 VLM 分析。
+    """
+
+    name = "screenshot"
+    description = (
+        "捕获屏幕截图，保存为 PNG 文件并返回路径。"
+        "可截取全屏或指定区域。截图可用于 VLM 视觉分析以定位 UI 元素。"
+    )
+    category = "gui"
+    parameters = [
+        SkillParameter("region", "string",
+                        "截图区域坐标 'left,top,width,height'（留空则截取全屏）。"
+                        "例如: '100,200,800,600' 表示从 (100,200) 开始截取 800x600 区域",
+                        required=False, default=""),
+        SkillParameter("monitor", "integer",
+                        "显示器编号（多显示器环境），0=全部, 1=主显示器, 2=副显示器...",
+                        required=False, default=1),
+    ]
+
+    async def execute(
+        self,
+        region: str = "",
+        monitor: int = 1,
+        **kwargs,
+    ) -> SkillResult:
+        """执行：捕获屏幕截图"""
+        try:
+            import mss
+        except ImportError:
+            from core.deps_checker import ensure_skill_deps
+            if not ensure_skill_deps("gui"):
+                return SkillResult(
+                    success=False,
+                    error="mss 安装失败，请手动运行: pip install mss",
+                )
+            import mss
+
+        try:
+            screenshot_path = _generate_screenshot_path()
+
+            with mss.mss() as sct:
+                # 确定截图区域
+                if region:
+                    # 解析区域: "left,top,width,height"
+                    parts = region.split(",")
+                    if len(parts) != 4:
+                        return SkillResult(
+                            success=False,
+                            error=f"区域格式错误，应为 'left,top,width,height'，得到: {region}",
+                        )
+                    try:
+                        left, top, width, height = [int(p.strip()) for p in parts]
+                    except ValueError:
+                        return SkillResult(success=False, error=f"区域坐标必须是整数: {region}")
+
+                    monitor_info = {"left": left, "top": top, "width": width, "height": height}
+                else:
+                    # 截取指定显示器（默认主显示器）
+                    monitors = sct.monitors
+                    if monitor < 0 or monitor >= len(monitors):
+                        return SkillResult(
+                            success=False,
+                            error=f"显示器编号 {monitor} 无效，可用范围: 0-{len(monitors) - 1}",
+                        )
+                    monitor_info = monitors[monitor]
+
+                # 捕获截图
+                sct_img = sct.grab(monitor_info)
+
+                # 保存为 PNG
+                from mss.tools import to_png
+                mss.tools.to_png(sct_img.rgb, sct_img.size, output=screenshot_path)
+
+            import os
+            file_size = os.path.getsize(screenshot_path)
+
+            return SkillResult(
+                success=True,
+                data={
+                    "screenshot_path": screenshot_path,
+                    "file_size_bytes": file_size,
+                    "width": monitor_info.get("width"),
+                    "height": monitor_info.get("height"),
+                    "region": region or None,
+                    "monitor": monitor,
+                },
+                message=f"屏幕截图已保存: {screenshot_path} ({file_size} 字节, {monitor_info.get('width')}x{monitor_info.get('height')})",
+                files=[screenshot_path],
+            )
+        except Exception as e:
+            logger.error(f"屏幕截图失败: {e}")
+            return SkillResult(success=False, error=f"屏幕截图失败: {e}")
+
+
+class MouseClickSkill(Skill):
+    """
+    鼠标点击 - 在指定的屏幕坐标处执行鼠标点击。
+
+    支持单击、双击、右键点击。坐标以屏幕左上角为原点 (0, 0)。
+    需要先用 screenshot 技能截图，然后用 VLM 分析获取坐标。
+    """
+
+    name = "mouse_click"
+    description = (
+        "在指定屏幕坐标处执行鼠标点击操作。"
+        "支持左键单击、左键双击、右键单击。"
+        "坐标以屏幕左上角为原点 (0,0)。"
+    )
+    category = "gui"
+    dangerous = True
+    parameters = [
+        SkillParameter("x", "integer", "目标 X 坐标（屏幕水平位置，左上角为 0）", required=True),
+        SkillParameter("y", "integer", "目标 Y 坐标（屏幕垂直位置，左上角为 0）", required=True),
+        SkillParameter("button", "string", "鼠标按钮: left=左键, right=右键, middle=中键", required=False,
+                        default="left", enum=["left", "right", "middle"]),
+        SkillParameter("clicks", "integer", "点击次数: 1=单击, 2=双击, 3=三击", required=False, default=1),
+    ]
+
+    async def execute(
+        self,
+        x: int = 0,
+        y: int = 0,
+        button: str = "left",
+        clicks: int = 1,
+        **kwargs,
+    ) -> SkillResult:
+        """执行：鼠标点击"""
+        try:
+            from pynput.mouse import Controller, Button
+        except ImportError:
+            from core.deps_checker import ensure_skill_deps
+            if not ensure_skill_deps("gui"):
+                return SkillResult(
+                    success=False,
+                    error="pynput 安装失败，请手动运行: pip install pynput",
+                )
+            from pynput.mouse import Controller, Button
+
+        try:
+            mouse = Controller()
+
+            # 映射按钮名称
+            button_map = {
+                "left": Button.left,
+                "right": Button.right,
+                "middle": Button.middle,
+            }
+            btn = button_map.get(button, Button.left)
+
+            # 移动到目标位置
+            mouse.position = (x, y)
+            time.sleep(0.05)  # 短暂等待鼠标移动到位
+
+            # 执行点击
+            if clicks == 1:
+                mouse.click(btn, 1)
+            elif clicks == 2:
+                mouse.click(btn, 2)
+            else:
+                mouse.click(btn, clicks)
+
+            click_desc = {"left": "左键", "right": "右键", "middle": "中键"}.get(button, button)
+            count_desc = "单击" if clicks == 1 else f"{clicks}连击"
+
+            return SkillResult(
+                success=True,
+                data={"x": x, "y": y, "button": button, "clicks": clicks},
+                message=f"已在 ({x}, {y}) 执行{click_desc}{count_desc}",
+            )
+        except Exception as e:
+            logger.error(f"鼠标点击失败: {e}")
+            return SkillResult(success=False, error=f"鼠标点击失败: {e}")
+
+
+class MouseDragSkill(Skill):
+    """
+    鼠标拖拽 - 从起点拖拽到终点。
+
+    常用于文件拖放、滑动操作、选择区域等场景。
+    可设置拖拽持续时间和步数，实现平滑拖拽效果。
+    """
+
+    name = "mouse_drag"
+    description = (
+        "从起点坐标拖拽到终点坐标。可用于文件拖放、区域选择、滑动条操作等。"
+        "支持设置拖拽持续时间以实现平滑拖拽效果。"
+    )
+    category = "gui"
+    dangerous = True
+    parameters = [
+        SkillParameter("start_x", "integer", "起点 X 坐标", required=True),
+        SkillParameter("start_y", "integer", "起点 Y 坐标", required=True),
+        SkillParameter("end_x", "integer", "终点 X 坐标", required=True),
+        SkillParameter("end_y", "integer", "终点 Y 坐标", required=True),
+        SkillParameter("button", "string", "鼠标按钮: left=左键(默认), right=右键", required=False,
+                        default="left", enum=["left", "right"]),
+        SkillParameter("duration", "float", "拖拽持续时间（秒），默认 0.3 秒", required=False, default=0.3),
+    ]
+
+    async def execute(
+        self,
+        start_x: int = 0,
+        start_y: int = 0,
+        end_x: int = 0,
+        end_y: int = 0,
+        button: str = "left",
+        duration: float = 0.3,
+        **kwargs,
+    ) -> SkillResult:
+        """执行：鼠标拖拽"""
+        try:
+            from pynput.mouse import Controller, Button
+        except ImportError:
+            from core.deps_checker import ensure_skill_deps
+            if not ensure_skill_deps("gui"):
+                return SkillResult(
+                    success=False,
+                    error="pynput 安装失败，请手动运行: pip install pynput",
+                )
+            from pynput.mouse import Controller, Button
+
+        try:
+            mouse = Controller()
+            button_map = {
+                "left": Button.left,
+                "right": Button.right,
+            }
+            btn = button_map.get(button, Button.left)
+
+            # 移动到起点
+            mouse.position = (start_x, start_y)
+            time.sleep(0.05)
+
+            # 按下鼠标
+            mouse.press(btn)
+
+            # 平滑移动到终点
+            steps = max(int(duration * 60), 1)  # 大约 60fps
+            dx = (end_x - start_x) / steps
+            dy = (end_y - start_y) / steps
+            step_delay = duration / steps
+
+            for i in range(steps):
+                mouse.move(int(dx), int(dy))
+                time.sleep(step_delay)
+
+            # 确保到达精确位置
+            mouse.position = (end_x, end_y)
+
+            # 释放鼠标
+            mouse.release(btn)
+
+            return SkillResult(
+                success=True,
+                data={
+                    "start": {"x": start_x, "y": start_y},
+                    "end": {"x": end_x, "y": end_y},
+                    "button": button,
+                    "duration": duration,
+                },
+                message=f"已从 ({start_x}, {start_y}) 拖拽到 ({end_x}, {end_y})",
+            )
+        except Exception as e:
+            logger.error(f"鼠标拖拽失败: {e}")
+            return SkillResult(success=False, error=f"鼠标拖拽失败: {e}")
+
+
+class TypeTextSkill(Skill):
+    """
+    输入文本 - 在当前光标位置输入文本。
+
+    使用 pynput 键盘控制器模拟真实按键输入。
+    支持普通文本和特殊按键。
+    输入前确保目标输入框已获取焦点。
+    """
+
+    name = "type_text"
+    description = (
+        "在当前光标位置输入文本。模拟真实键盘按键输入。"
+        "输入前请确保目标输入框已获取焦点（可先用 mouse_click 点击输入框）。"
+    )
+    category = "gui"
+    dangerous = True
+    parameters = [
+        SkillParameter("text", "string", "要输入的文本内容", required=True),
+        SkillParameter("interval", "float", "按键间隔（秒），默认 0.02 秒。增大可模拟更慢的打字", required=False, default=0.02),
+        SkillParameter("clear_first", "boolean", "是否先用 Ctrl+A 全选再删除（清空已有内容）", required=False, default=False),
+    ]
+
+    async def execute(
+        self,
+        text: str = "",
+        interval: float = 0.02,
+        clear_first: bool = False,
+        **kwargs,
+    ) -> SkillResult:
+        """执行：输入文本"""
+        try:
+            from pynput.keyboard import Controller
+        except ImportError:
+            from core.deps_checker import ensure_skill_deps
+            if not ensure_skill_deps("gui"):
+                return SkillResult(
+                    success=False,
+                    error="pynput 安装失败，请手动运行: pip install pynput",
+                )
+            from pynput.keyboard import Controller
+
+        if not text:
+            return SkillResult(success=False, error="缺少必需参数: text")
+
+        try:
+            keyboard = Controller()
+
+            # 先清空已有内容（可选）
+            if clear_first:
+                if IS_MACOS:
+                    # Mac: Cmd+A 全选
+                    keyboard.press(Key.cmd)
+                    keyboard.press('a')
+                    keyboard.release('a')
+                    keyboard.release(Key.cmd)
+                else:
+                    # Windows/Linux: Ctrl+A 全选
+                    keyboard.press(Key.ctrl)
+                    keyboard.press('a')
+                    keyboard.release('a')
+                    keyboard.release(Key.ctrl)
+                time.sleep(0.05)
+                keyboard.press(Key.backspace)
+                keyboard.release(Key.backspace)
+                time.sleep(0.05)
+
+            # 逐字符输入
+            keyboard.type(text)
+
+            return SkillResult(
+                success=True,
+                data={"text_length": len(text), "cleared_first": clear_first},
+                message=f"已输入文本（{len(text)} 个字符）{f'（已先清空）' if clear_first else ''}",
+            )
+        except Exception as e:
+            logger.error(f"输入文本失败: {e}")
+            return SkillResult(success=False, error=f"输入文本失败: {e}")
+
+
+class HotkeySkill(Skill):
+    """
+    快捷键 - 按下键盘快捷键组合。
+
+    自动根据平台适配修饰键：
+      - Windows/Linux: Ctrl 键
+      - macOS: Cmd (⌘) 键
+
+    支持常用快捷键如 Ctrl+C (复制), Ctrl+V (粘贴), Alt+Tab 等。
+    也可直接指定精确的按键组合。
+    """
+
+    name = "hotkey"
+    description = (
+        "按下键盘快捷键。自动适配平台修饰键（Windows 用 Ctrl，Mac 用 Cmd）。"
+        "支持预设快捷键（如 'copy', 'paste', 'select_all'）或自定义按键组合（如 'ctrl+c', 'alt+tab'）。"
+    )
+    category = "gui"
+    dangerous = True
+    parameters = [
+        SkillParameter("action", "string",
+                        "快捷键动作或按键组合。预设值: copy, paste, cut, select_all, undo, redo, "
+                        "save, close, tab, new_tab, find, refresh, fullscreen, "
+                        "screenshot, quit, lock_screen。也可直接指定按键组合如 'ctrl+shift+i', 'alt+f4'",
+                        required=True),
+    ]
+
+    # 预设快捷键映射 / Preset hotkey mappings
+    PRESET_HOTKEYS = {
+        "copy": ["ctrl", "c"],
+        "paste": ["ctrl", "v"],
+        "cut": ["ctrl", "x"],
+        "select_all": ["ctrl", "a"],
+        "undo": ["ctrl", "z"],
+        "redo": ["ctrl", "y"],
+        "save": ["ctrl", "s"],
+        "close": ["ctrl", "w"],
+        "tab": ["ctrl", "tab"],
+        "new_tab": ["ctrl", "t"],
+        "find": ["ctrl", "f"],
+        "refresh": ["ctrl", "r"],
+        "fullscreen": ["ctrl", "f11"] if IS_WINDOWS else ["ctrl", "cmd", "f"],
+        "screenshot": ["cmd", "shift", "3"] if IS_MACOS else ["ctrl", "print_screen"],
+        "quit": ["cmd", "q"] if IS_MACOS else ["alt", "f4"],
+        "lock_screen": ["ctrl", "cmd", "q"] if IS_MACOS else ["ctrl", "alt", "delete"],
+    }
+
+    async def execute(self, action: str = "", **kwargs) -> SkillResult:
+        """执行：按下快捷键"""
+        try:
+            from pynput.keyboard import Controller, Key, KeyCode
+        except ImportError:
+            from core.deps_checker import ensure_skill_deps
+            if not ensure_skill_deps("gui"):
+                return SkillResult(
+                    success=False,
+                    error="pynput 安装失败，请手动运行: pip install pynput",
+                )
+            from pynput.keyboard import Controller, Key, KeyCode
+
+        if not action:
+            return SkillResult(success=False, error="缺少必需参数: action")
+
+        try:
+            keyboard = Controller()
+
+            # 解析快捷键组合
+            if action.lower() in self.PRESET_HOTKEYS:
+                keys = self.PRESET_HOTKEYS[action.lower()]
+            else:
+                # 解析自定义按键组合，如 "ctrl+shift+i" -> ["ctrl", "shift", "i"]
+                keys = [k.strip().lower() for k in action.split("+")]
+
+            # 将键名映射到 pynput Key 或 KeyCode
+            pressed_keys = []
+            key_map = {
+                "ctrl": Key.ctrl,
+                "alt": Key.alt,
+                "shift": Key.shift,
+                "cmd": Key.cmd,
+                "super": Key.cmd,
+                "win": Key.cmd,
+                "tab": Key.tab,
+                "enter": Key.enter,
+                "return": Key.enter,
+                "space": Key.space,
+                "backspace": Key.backspace,
+                "delete": Key.delete,
+                "esc": Key.esc,
+                "escape": Key.esc,
+                "up": Key.up,
+                "down": Key.down,
+                "left": Key.left,
+                "right": Key.right,
+                "home": Key.home,
+                "end": Key.end,
+                "page_up": Key.page_up,
+                "page_down": Key.page_down,
+                "f1": Key.f1, "f2": Key.f2, "f3": Key.f3, "f4": Key.f4,
+                "f5": Key.f5, "f6": Key.f6, "f7": Key.f7, "f8": Key.f8,
+                "f9": Key.f9, "f10": Key.f10, "f11": Key.f11, "f12": Key.f12,
+                "print_screen": Key.print_screen,
+                "caps_lock": Key.caps_lock,
+                "num_lock": Key.num_lock,
+                "insert": Key.insert,
+                "pause": Key.pause,
+            }
+
+            # Mac 平台自动将 ctrl 替换为 cmd
+            if IS_MACOS:
+                keys = ["cmd" if k == "ctrl" else k for k in keys]
+
+            # 按下所有修饰键
+            for key_name in keys:
+                if key_name in key_map:
+                    keyboard.press(key_map[key_name])
+                    pressed_keys.append(key_map[key_name])
+                else:
+                    # 单字符按键
+                    if len(key_name) == 1:
+                        keyboard.press(key_name)
+                        pressed_keys.append(key_name)
+
+            # 以相反顺序释放
+            time.sleep(0.05)
+            for key in reversed(pressed_keys):
+                keyboard.release(key)
+
+            key_display = "+".join(keys)
+            platform_note = " (macOS: Ctrl→Cmd)" if IS_MACOS else ""
+            return SkillResult(
+                success=True,
+                data={"action": action, "keys": keys, "platform_adapted": IS_MACOS},
+                message=f"已按下快捷键: {key_display}{platform_note}",
+            )
+        except Exception as e:
+            logger.error(f"快捷键操作失败: {e}")
+            return SkillResult(success=False, error=f"快捷键操作失败: {e}")
+
+
+class WindowListSkill(Skill):
+    """
+    窗口列表 - 列出当前所有打开的窗口。
+
+    返回每个窗口的标题、位置、大小等信息。
+    可用于定位目标窗口以便后续操作。
+    """
+
+    name = "window_list"
+    description = (
+        "列出当前所有打开的窗口，包括标题、位置、大小等信息。"
+        "可用于查找目标窗口标题，供 window_focus 使用。"
+    )
+    category = "gui"
+    parameters = [
+        SkillParameter("filter", "string",
+                        "窗口标题过滤关键词（留空则返回所有窗口）。仅返回标题包含该关键词的窗口",
+                        required=False, default=""),
+    ]
+
+    async def execute(self, filter: str = "", **kwargs) -> SkillResult:
+        """执行：列出窗口"""
+        try:
+            import pygetwindow as gw
+        except ImportError:
+            from core.deps_checker import ensure_skill_deps
+            if not ensure_skill_deps("gui"):
+                return SkillResult(
+                    success=False,
+                    error="pygetwindow 安装失败，请手动运行: pip install pygetwindow",
+                )
+            import pygetwindow as gw
+
+        try:
+            windows = gw.getAllWindows()
+
+            # 过滤窗口
+            if filter:
+                windows = [w for w in windows if filter.lower() in w.title.lower()]
+
+            window_list = []
+            for w in windows:
+                window_list.append({
+                    "title": w.title,
+                    "left": w.left,
+                    "top": w.top,
+                    "width": w.width,
+                    "height": w.height,
+                    "visible": w.visible,
+                    "active": w.isActive,
+                })
+
+            return SkillResult(
+                success=True,
+                data={
+                    "windows": window_list,
+                    "count": len(window_list),
+                    "filter": filter or None,
+                    "platform": sys.platform,
+                },
+                message=f"共 {len(window_list)} 个窗口{f'（过滤: {filter}）' if filter else ''}",
+            )
+        except Exception as e:
+            logger.error(f"列出窗口失败: {e}")
+            return SkillResult(success=False, error=f"列出窗口失败: {e}")
+
+
+class WindowFocusSkill(Skill):
+    """
+    窗口聚焦 - 将指定窗口置顶并获取焦点。
+
+    支持通过窗口标题（模糊匹配）或窗口标题序号来定位目标窗口。
+    聚焦后可进行后续的鼠标/键盘操作。
+    """
+
+    name = "window_focus"
+    description = (
+        "将指定窗口置顶并获取焦点。支持通过窗口标题（模糊匹配）定位。"
+        "聚焦后可在该窗口内进行鼠标点击、键盘输入等操作。"
+    )
+    category = "gui"
+    dangerous = True
+    parameters = [
+        SkillParameter("title", "string", "目标窗口标题（模糊匹配，包含该文本即匹配）", required=True),
+        SkillParameter("activate", "boolean", "是否激活窗口（默认 true）", required=False, default=True),
+        SkillParameter("maximize", "boolean", "是否最大化窗口（默认 false）", required=False, default=False),
+    ]
+
+    async def execute(
+        self,
+        title: str = "",
+        activate: bool = True,
+        maximize: bool = False,
+        **kwargs,
+    ) -> SkillResult:
+        """执行：聚焦窗口"""
+        try:
+            import pygetwindow as gw
+        except ImportError:
+            from core.deps_checker import ensure_skill_deps
+            if not ensure_skill_deps("gui"):
+                return SkillResult(
+                    success=False,
+                    error="pygetwindow 安装失败，请手动运行: pip install pygetwindow",
+                )
+            import pygetwindow as gw
+
+        if not title:
+            return SkillResult(success=False, error="缺少必需参数: title")
+
+        try:
+            # 模糊匹配窗口标题
+            windows = gw.getWindowsWithTitle(title)
+            if not windows:
+                return SkillResult(
+                    success=False,
+                    error=f"未找到包含 '{title}' 的窗口",
+                )
+
+            window = windows[0]
+
+            if activate:
+                # 先尝试恢复最小化的窗口
+                if window.isMinimized:
+                    window.restore()
+                    time.sleep(0.2)
+
+                window.activate()
+                time.sleep(0.2)
+
+            if maximize:
+                window.maximize()
+                time.sleep(0.2)
+
+            return SkillResult(
+                success=True,
+                data={
+                    "title": window.title,
+                    "left": window.left,
+                    "top": window.top,
+                    "width": window.width,
+                    "height": window.height,
+                    "activated": activate,
+                    "maximized": maximize,
+                },
+                message=f"已聚焦窗口: {window.title} ({window.width}x{window.height})",
+            )
+        except Exception as e:
+            logger.error(f"窗口聚焦失败: {e}")
+            return SkillResult(success=False, error=f"窗口聚焦失败: {e}")
+
+
+class ScreenElementSkill(Skill):
+    """
+    屏幕元素定位 - 使用视觉模型(VLM)从截图中定位 UI 元素坐标。
+
+    工作流程:
+      1. 自动截取屏幕截图
+      2. 将截图发送到 VLM（视觉语言模型）进行分析
+      3. VLM 返回目标元素的屏幕坐标
+      4. 返回坐标供 mouse_click 技能使用
+
+    依赖 LLM 客户端配置了支持视觉的模型（如 GPT-4o, Claude 3.5 Sonnet 等）。
+    """
+
+    name = "screen_element"
+    description = (
+        "使用视觉模型(VLM)从屏幕截图中定位 UI 元素的坐标。"
+        "先截图，再让 AI 视觉模型分析找到目标元素，返回屏幕坐标。"
+        "返回的坐标可直接用于 mouse_click 技能进行点击操作。"
+    )
+    category = "gui"
+    parameters = [
+        SkillParameter("description", "string",
+                        "要查找的元素的文字描述（如 '保存按钮', '右上角的关闭按钮', '地址栏输入框'）",
+                        required=True),
+        SkillParameter("region", "string",
+                        "截图区域 'left,top,width,height'（留空截取全屏）",
+                        required=False, default=""),
+        SkillParameter("element_type", "string",
+                        "元素类型提示: button/input/link/icon/menu/checkbox/text_area/其他",
+                        required=False, default=""),
+    ]
+
+    async def execute(
+        self,
+        description: str = "",
+        region: str = "",
+        element_type: str = "",
+        **kwargs,
+    ) -> SkillResult:
+        """执行：使用 VLM 定位屏幕元素"""
+        # 第一步：截图
+        try:
+            import mss
+        except ImportError:
+            from core.deps_checker import ensure_skill_deps
+            if not ensure_skill_deps("gui"):
+                return SkillResult(
+                    success=False,
+                    error="mss 安装失败，请手动运行: pip install mss",
+                )
+            import mss
+
+        try:
+            screenshot_path = _generate_screenshot_path()
+
+            # 截图
+            with mss.mss() as sct:
+                if region:
+                    parts = region.split(",")
+                    if len(parts) == 4:
+                        left, top, width, height = [int(p.strip()) for p in parts]
+                        monitor_info = {"left": left, "top": top, "width": width, "height": height}
+                    else:
+                        monitor_info = sct.monitors[1]
+                else:
+                    monitor_info = sct.monitors[1]
+
+                sct_img = sct.grab(monitor_info)
+                from mss.tools import to_png
+                to_png(sct_img.rgb, sct_img.size, output=screenshot_path)
+
+            import os
+            file_size = os.path.getsize(screenshot_path)
+
+            # 第二步：读取图片并编码为 base64
+            import base64
+            with open(screenshot_path, "rb") as f:
+                image_data = base64.b64encode(f.read()).decode("utf-8")
+
+            # 第三步：调用 VLM 分析截图
+            try:
+                from core.llm import LLMClient, Message
+
+                client = LLMClient()
+
+                # 构建 VLM 分析 prompt
+                type_hint = f"元素类型: {element_type}。" if element_type else ""
+                vlm_prompt = (
+                    f"分析这张屏幕截图，找到以下 UI 元素并返回其中心坐标:\n"
+                    f"目标: {description}\n"
+                    f"{type_hint}\n\n"
+                    f"请严格按以下 JSON 格式回复（不要包含其他内容）:\n"
+                    f'{{"found": true/false, "x": 中心X坐标, "y": 中心Y坐标, '
+                    f'"confidence": 0.0-1.0, "description": "找到的元素的描述"}}\n\n'
+                    f"注意: 坐标以截图左上角为原点 (0,0)。"
+                    f"如果找不到元素，found 设为 false。"
+                )
+
+                # 使用 vision API
+                messages = [
+                    Message(
+                        role="user",
+                        content=[
+                            {"type": "text", "text": vlm_prompt},
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:image/png;base64,{image_data}",
+                                },
+                            },
+                        ],
+                    ),
+                ]
+
+                response = await client.chat(
+                    messages=messages,
+                    temperature=0.1,
+                    max_tokens=200,
+                )
+
+                if not response.success or not response.content:
+                    return SkillResult(
+                        success=True,
+                        data={
+                            "screenshot_path": screenshot_path,
+                            "file_size_bytes": file_size,
+                            "found": False,
+                            "reason": "VLM 未返回有效结果",
+                        },
+                        message=f"截图已保存: {screenshot_path}，但 VLM 分析失败",
+                        files=[screenshot_path],
+                    )
+
+                # 解析 VLM 返回的坐标
+                import json
+                import re
+                content = response.content.strip()
+
+                # 尝试从返回中提取 JSON
+                json_match = re.search(r'\{[^}]+\}', content, re.DOTALL)
+                if json_match:
+                    result = json.loads(json_match.group())
+                else:
+                    result = json.loads(content)
+
+                found = result.get("found", False)
+                x = result.get("x")
+                y = result.get("y")
+                confidence = result.get("confidence", 0)
+
+                if found and x is not None and y is not None:
+                    # 如果截图是区域截图，需要加上偏移量
+                    offset_x = monitor_info.get("left", 0)
+                    offset_y = monitor_info.get("top", 0)
+                    actual_x = int(x) + offset_x
+                    actual_y = int(y) + offset_y
+
+                    return SkillResult(
+                        success=True,
+                        data={
+                            "found": True,
+                            "x": actual_x,
+                            "y": actual_y,
+                            "confidence": confidence,
+                            "description": result.get("description", description),
+                            "screenshot_path": screenshot_path,
+                            "element_type": element_type,
+                        },
+                        message=(
+                            f"已定位元素 '{description}' 在坐标 ({actual_x}, {actual_y})，"
+                            f"置信度: {confidence:.0%}。可使用 mouse_click 点击。"
+                        ),
+                        files=[screenshot_path],
+                    )
+                else:
+                    return SkillResult(
+                        success=True,
+                        data={
+                            "found": False,
+                            "screenshot_path": screenshot_path,
+                            "vlm_response": content[:500],
+                        },
+                        message=f"未在截图中找到 '{description}'。截图已保存: {screenshot_path}",
+                        files=[screenshot_path],
+                    )
+
+            except ImportError:
+                return SkillResult(
+                    success=True,
+                    data={
+                        "screenshot_path": screenshot_path,
+                        "file_size_bytes": file_size,
+                        "found": False,
+                        "reason": "LLM 客户端不可用，无法进行 VLM 分析",
+                    },
+                    message=(
+                        f"截图已保存: {screenshot_path}，但无法调用 VLM 进行元素定位。"
+                        "请检查 LLM 配置是否支持视觉模型，或手动查看截图确定坐标。"
+                    ),
+                    files=[screenshot_path],
+                )
+            except Exception as vlm_error:
+                logger.error(f"VLM 分析失败: {vlm_error}")
+                return SkillResult(
+                    success=True,
+                    data={
+                        "screenshot_path": screenshot_path,
+                        "file_size_bytes": file_size,
+                        "found": False,
+                        "reason": f"VLM 分析异常: {vlm_error}",
+                    },
+                    message=(
+                        f"截图已保存: {screenshot_path}，但 VLM 分析失败: {vlm_error}。"
+                        "可手动查看截图确定坐标后使用 mouse_click。"
+                    ),
+                    files=[screenshot_path],
+                )
+
+        except Exception as e:
+            logger.error(f"屏幕元素定位失败: {e}")
+            return SkillResult(success=False, error=f"屏幕元素定位失败: {e}")