@ynhcj/xiaoyi-channel 1.1.5 → 1.1.7

package/dist/src/channel.js

@@ -50,7 +50,7 @@ export const xyPlugin = {
     },
     outbound: xyOutbound,
     onboarding: xyOnboardingAdapter,
-    agentTools: [locationTool, noteTool, searchNoteTool, modifyNoteTool, calendarTool, searchCalendarTool, searchPhotoGalleryTool, uploadPhotoTool], // searchContactTool 已暂时禁用
+    agentTools: [locationTool, noteTool, searchNoteTool, modifyNoteTool, calendarTool, searchCalendarTool, searchPhotoGalleryTool, uploadPhotoTool], // searchContactTool, xiaoyiGuiTool 已暂时禁用
     messaging: {
         normalizeTarget: (raw) => {
             const trimmed = raw.trim();

package/dist/src/tools/search-photo-gallery-tool.js

@@ -12,7 +12,18 @@ import { logger } from "../utils/logger.js";
 export const searchPhotoGalleryTool = {
     name: "search_photo_gallery",
     label: "Search Photo Gallery",
-    description: "插件功能描述：搜索用户手机图库中的照片，如果用户说从手机图库中或者从相册中查询xx图片时调用此工具。根据图像描述语料检索匹配的照片,返回照片在手机本地的 mediaUri。注意:返回的 mediaUri 是本地路径,无法直接下载或访问。如果需要下载、查看、使用或展示照片,请使用 upload_photo 工具将 mediaUri 转换为可访问的公网 URL。操作超时时间为60秒,请勿重复调用此工具,如果超时或失败,最多重试一次。注意事项：只有当用户明确表达从手机相册搜索或者从图库搜索时才执行此工具，如果用户仅表达要搜索xxx图片，并没有说明搜索数据源，则不要贸然调用此插件，可以优先尝试websearch或者询问用户是否要从手机图库中搜索。",
+    description: `插件功能描述：搜索用户手机图库中的照片
+  工具使用约束：如果用户说从手机图库中或者从相册中查询xx图片时调用此工具。
+  工具输入输出简介：
+  a. 根据图像描述语料检索匹配的照片,返回照片在手机本地的 mediaUri以及thumbnailUri。
+  b. 返回的 mediaUri以及thumbnailUri 是本地路径,无法直接下载或访问。如果需要下载、查看、使用或展示照片,请使用 upload_photo 工具将 mediaUri或者thumbnailUri 转换为可访问的公网 URL。
+  c. mediaUri代表手机相册中的图片原图路径，图片大小比较大，清晰度比较高
+  d. thumbnailUri代表手机相册中的图片缩略图路径，图片大小比较小，清晰度适中，建议在upload_photo 工具的入参中优先使用此路径，不容易引起上传超时等问题
+  
+  注意事项：
+  a. 只有当用户明确表达从手机相册搜索或者从图库搜索时才执行此工具，如果用户仅表达要搜索xxx图片，并没有说明搜索数据源，则不要贸然调用此插件，可以优先尝试websearch或者询问用户是否要从手机图库中搜索。
+  b. 操作超时时间为60秒,请勿重复调用此工具,如果超时或失败,最多重试一次。
+  `,
     parameters: {
         type: "object",
         properties: {
@@ -52,15 +63,15 @@ export const searchPhotoGalleryTool = {
         logger.log(`[SEARCH_PHOTO_GALLERY_TOOL] ✅ WebSocket manager obtained`);
         // Search for photos
         logger.log(`[SEARCH_PHOTO_GALLERY_TOOL] 📸 Searching for photos...`);
-        const mediaUris = await searchPhotos(wsManager, config, sessionId, taskId, messageId, params.query);
-        if (!mediaUris || mediaUris.length === 0) {
+        const items = await searchPhotos(wsManager, config, sessionId, taskId, messageId, params.query);
+        if (!items || items.length === 0) {
             logger.warn(`[SEARCH_PHOTO_GALLERY_TOOL] ⚠️ No photos found for query: ${params.query}`);
             return {
                 content: [
                     {
                         type: "text",
                         text: JSON.stringify({
-                            mediaUris: [],
+                            items: [],
                             count: 0,
                             message: "未找到匹配的照片"
                         }),
@@ -68,16 +79,16 @@ export const searchPhotoGalleryTool = {
                 ],
             };
         }
-        logger.log(`[SEARCH_PHOTO_GALLERY_TOOL] ✅ Found ${mediaUris.length} photos`);
-        logger.log(`[SEARCH_PHOTO_GALLERY_TOOL]   - mediaUris:`, JSON.stringify(mediaUris));
+        logger.log(`[SEARCH_PHOTO_GALLERY_TOOL] ✅ Found ${items.length} photos`);
+        logger.log(`[SEARCH_PHOTO_GALLERY_TOOL]   - items:`, JSON.stringify(items));
         return {
             content: [
                 {
                     type: "text",
                     text: JSON.stringify({
-                        mediaUris,
-                        count: mediaUris.length,
-                        message: `找到 ${mediaUris.length} 张照片。注意：这些是本地 URI，无法直接访问。如需下载或查看，请使用 upload_photo 工具。`
+                        items,
+                        count: items.length,
+                        message: `找到 ${items.length} 张照片。注意：mediaUri 和 thumbnailUri 是本地路径，无法直接访问。如需下载或查看，请使用 upload_photo 工具。`
                     }),
                 },
             ],
@@ -86,7 +97,7 @@ export const searchPhotoGalleryTool = {
 };
 /**
  * Search for photos using query description
- * Returns array of mediaUri strings
+ * Returns array of photo items with complete information
  */
 async function searchPhotos(wsManager, config, sessionId, taskId, messageId, query) {
     logger.log(`[SEARCH_PHOTO_GALLERY_TOOL] 📦 Building SearchPhotoVideo command...`);
@@ -140,10 +151,8 @@ async function searchPhotos(wsManager, config, sessionId, taskId, messageId, que
                     logger.log(`[SEARCH_PHOTO_GALLERY_TOOL] ✅ Photo search completed successfully`);
                     const result = event.outputs.result;
                     const items = result?.items || [];
-                    // Extract mediaUri from each item
-                    const mediaUris = items.map((item) => item.mediaUri).filter(Boolean);
-                    logger.log(`[SEARCH_PHOTO_GALLERY_TOOL] 📊 Extracted ${mediaUris.length} mediaUris`);
-                    resolve(mediaUris);
+                    logger.log(`[SEARCH_PHOTO_GALLERY_TOOL] 📊 Found ${items.length} photo items`);
+                    resolve(items);
                 }
                 else {
                     logger.error(`[SEARCH_PHOTO_GALLERY_TOOL] ❌ Photo search failed`);

package/dist/src/tools/upload-photo-tool.js

@@ -13,14 +13,24 @@ import { logger } from "../utils/logger.js";
 export const uploadPhotoTool = {
     name: "upload_photo",
     label: "Upload Photo",
-    description: "将手机本地照片回传并获取可公网访问的 URL。使用前必须先调用 search_photo_gallery 工具获取照片的 mediaUri，必须与search_photo_gallery中的mediaUri完全保持一致，不要修改。参数说明：mediaUris 是照片在手机本地的 URI 数组或 JSON 字符串数组（从 search_photo_gallery 工具获取）。限制：每次最多支持传入 5 条 mediaUri。操作超时时间为60秒,请勿重复调用此工具,如果超时或失败,最多重试一次。注意事项：此工具返回的图片链接为用户公网可访问的链接，如果需要后续操作需要下载到本地，如果需要返回给用户查看则直接以图片markdown的形式返回给用户",
+    description: `工具能力描述：将手机本地文件回传并获取可公网访问的 URL。
+  
+  前置工具调用：此工具使用前必须先调用 search_photo_gallery 工具获取照片的 mediaUri或者thumbnailUri
+  工具参数说明：
+  a. 入参中的mediaUris中的mediaUri必须与search_photo_gallery结果中对应的mediaUri或者thumbnailUri完全保持一致，不要自行修改，必须是file:://开头的路径。
+  b. 优先使用search_photo_gallery结果中的thumbnailUri作为入参，thumbnailUri是缩略图，清晰度与文件大小都非常合适展示给用户，如果thumbnailUri不存在或者用户要求使用原图，则使用search_photo_gallery结果中对应的mediaUri
+  c. mediaUris 是照片在手机本地的 URI 数组（从 search_photo_gallery 工具响应中获取）。限制：每次最多支持传入 3 条 mediaUri。
+  
+  注意事项：
+  a. 操作超时时间为60秒,请勿重复调用此工具,如果超时或失败,最多重试一次。
+  b. 此工具返回的图片链接为用户公网可访问的链接，如果需要后续操作需要下载到本地，如果需要返回给用户查看则直接以图片markdown的形式返回给用户`,
     parameters: {
         type: "object",
         properties: {
             mediaUris: {
                 // 不指定 type，允许传入数组或 JSON 字符串
                 // 具体的类型验证和转换在 execute 函数内部进行
-                description: "照片在手机本地的 URI 数组（或 JSON 字符串形式的数组），必须先通过 search_photo_gallery 工具获取。每次最多支持 5 条 URI。支持传入数组 [\"uri1\", \"uri2\"] 或 JSON 字符串 '[\"uri1\", \"uri2\"]'。",
+                description: "照片在手机本地的 URI 数组，必须先通过 search_photo_gallery 工具获取。每次最多支持 3 条 URI。",
             },
         },
         required: ["mediaUris"],

package/dist/src/tools/xiaoyi-gui-tool.d.ts

@@ -0,0 +1,6 @@
+/**
+ * XiaoYi GUI tool - executes phone app interactions through GUI agent.
+ * Simulates user interactions on phone screen (click, swipe, input, navigation, etc.)
+ * to complete tasks that cannot be done through internet APIs.
+ */
+export declare const xiaoyiGuiTool: any;

package/dist/src/tools/xiaoyi-gui-tool.js

@@ -0,0 +1,151 @@
+// XiaoYi GUI tool implementation - simulates phone screen interactions
+import { getXYWebSocketManager } from "../client.js";
+import { sendCommand } from "../formatter.js";
+import { getLatestSessionContext } from "./session-manager.js";
+import { logger } from "../utils/logger.js";
+/**
+ * XiaoYi GUI tool - executes phone app interactions through GUI agent.
+ * Simulates user interactions on phone screen (click, swipe, input, navigation, etc.)
+ * to complete tasks that cannot be done through internet APIs.
+ */
+export const xiaoyiGuiTool = {
+    name: "xiaoyi_gui_agent",
+    label: "XiaoYi GUI Agent",
+    description: `通过模拟人在手机屏幕上的交互行为（点击、滑动、输入、页面导航等），自动完成手机APP中的各类任务。
+
+该工具操作方式类似真实用户在手机上的操作，因此可以完成许多无法通过互联网API实现的任务，例如：
+- 任务需要真实操作手机APP界面
+- 数据仅存在于APP内部
+- 无法通过互联网API获取数据
+- 需要完成用户行为（签到、关注、购买等）
+- 需要在APP中发布或发送内容
+- 需要修改APP或手机设置
+
+理论上，所有可以通过人在手机上操作完成的任务，该Agent都可以尝试执行。
+
+注意事项：
+- 操作超时时间为5分钟（300秒）
+- 该工具执行时间较长，请勿重复调用
+- 如果超时或失败，最多重试一次`,
+    parameters: {
+        type: "object",
+        properties: {
+            query: {
+                type: "string",
+                description: "操作手机的指令以及期望返回的结果。例如：'打开微信，查看最新一条消息的内容' 或 '在设置中开启蓝牙'",
+            },
+        },
+        required: ["query"],
+    },
+    async execute(toolCallId, params) {
+        logger.log(`[XIAOYI_GUI_TOOL] 🚀 Starting execution`);
+        logger.log(`[XIAOYI_GUI_TOOL]   - toolCallId: ${toolCallId}`);
+        logger.log(`[XIAOYI_GUI_TOOL]   - query: ${params.query}`);
+        logger.log(`[XIAOYI_GUI_TOOL]   - timestamp: ${new Date().toISOString()}`);
+        // Validate parameters
+        if (!params.query || typeof params.query !== "string") {
+            logger.error(`[XIAOYI_GUI_TOOL] ❌ FAILED: Invalid query parameter`);
+            throw new Error("Missing or invalid required parameter: query must be a non-empty string");
+        }
+        // Get session context
+        logger.log(`[XIAOYI_GUI_TOOL] 🔍 Attempting to get session context...`);
+        const sessionContext = getLatestSessionContext();
+        if (!sessionContext) {
+            logger.error(`[XIAOYI_GUI_TOOL] ❌ FAILED: No active session found!`);
+            logger.error(`[XIAOYI_GUI_TOOL]   - toolCallId: ${toolCallId}`);
+            throw new Error("No active XY session found. XiaoYi GUI tool can only be used during an active conversation.");
+        }
+        logger.log(`[XIAOYI_GUI_TOOL] ✅ Session context found`);
+        logger.log(`[XIAOYI_GUI_TOOL]   - sessionId: ${sessionContext.sessionId}`);
+        logger.log(`[XIAOYI_GUI_TOOL]   - taskId (interactionId): ${sessionContext.taskId}`);
+        logger.log(`[XIAOYI_GUI_TOOL]   - messageId: ${sessionContext.messageId}`);
+        logger.log(`[XIAOYI_GUI_TOOL]   - agentId: ${sessionContext.agentId}`);
+        const { config, sessionId, taskId, messageId } = sessionContext;
+        // Get WebSocket manager
+        logger.log(`[XIAOYI_GUI_TOOL] 🔌 Getting WebSocket manager...`);
+        const wsManager = getXYWebSocketManager(config);
+        logger.log(`[XIAOYI_GUI_TOOL] ✅ WebSocket manager obtained`);
+        // Build InvokeJarvisGUIAgentRequest command
+        logger.log(`[XIAOYI_GUI_TOOL] 📦 Building InvokeJarvisGUIAgentRequest command...`);
+        const command = {
+            header: {
+                namespace: "ClawAgent",
+                name: "InvokeJarvisGUIAgentRequest",
+            },
+            payload: {
+                query: params.query,
+                sessionId: sessionId,
+                interactionId: taskId, // taskId corresponds to interactionId
+            },
+        };
+        logger.log(`[XIAOYI_GUI_TOOL] 📋 Command details:`, JSON.stringify(command, null, 2));
+        // Send command and wait for response (5 minute timeout)
+        logger.log(`[XIAOYI_GUI_TOOL] ⏳ Setting up promise to wait for GUI agent response...`);
+        logger.log(`[XIAOYI_GUI_TOOL]   - Timeout: 300 seconds (5 minutes)`);
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                logger.error(`[XIAOYI_GUI_TOOL] ⏰ Timeout: No response received within 300 seconds (5 minutes)`);
+                wsManager.off("gui-agent-response", handler);
+                reject(new Error("XiaoYi GUI Agent 操作超时（5分钟）"));
+            }, 300000); // 5 minutes timeout
+            // Listen for GUI agent response events
+            const handler = (event) => {
+                logger.log(`[XIAOYI_GUI_TOOL] 📨 Received event:`, JSON.stringify(event));
+                // Check if this is the InvokeJarvisGUIAgentResponse we're waiting for
+                if (event.header?.namespace === "ClawAgent" &&
+                    event.header?.name === "InvokeJarvisGUIAgentResponse") {
+                    logger.log(`[XIAOYI_GUI_TOOL] 🎯 InvokeJarvisGUIAgentResponse event received`);
+                    logger.log(`[XIAOYI_GUI_TOOL]   - isFinal: ${event.payload?.isFinal}`);
+                    // According to the spec, we only get one response (isFinal: true)
+                    if (event.payload?.isFinal === true) {
+                        clearTimeout(timeout);
+                        wsManager.off("gui-agent-response", handler);
+                        const streamContent = event.payload?.streamInfo?.streamContent;
+                        if (streamContent) {
+                            logger.log(`[XIAOYI_GUI_TOOL] ✅ GUI Agent operation completed successfully`);
+                            logger.log(`[XIAOYI_GUI_TOOL]   - streamContent: ${streamContent}`);
+                            resolve({
+                                content: [
+                                    {
+                                        type: "text",
+                                        text: streamContent,
+                                    }
+                                ]
+                            });
+                        }
+                        else {
+                            logger.error(`[XIAOYI_GUI_TOOL] ❌ Response missing streamContent`);
+                            logger.error(`[XIAOYI_GUI_TOOL]   - payload:`, JSON.stringify(event.payload));
+                            reject(new Error("XiaoYi GUI Agent 响应格式错误：缺少 streamContent"));
+                        }
+                    }
+                    else if (event.payload?.isFinal === false) {
+                        // According to spec, we shouldn't get intermediate responses, but log if we do
+                        logger.log(`[XIAOYI_GUI_TOOL] 📝 Intermediate response received (isFinal: false), waiting for final...`);
+                    }
+                }
+            };
+            // Register event handler
+            // Note: The WebSocket manager needs to emit 'gui-agent-response' when receiving this type of response
+            logger.log(`[XIAOYI_GUI_TOOL] 📡 Registering gui-agent-response handler on WebSocket manager`);
+            wsManager.on("gui-agent-response", handler);
+            // Send the command
+            logger.log(`[XIAOYI_GUI_TOOL] 📤 Sending InvokeJarvisGUIAgentRequest command...`);
+            sendCommand({
+                config,
+                sessionId,
+                taskId,
+                messageId,
+                command,
+            }).then(() => {
+                logger.log(`[XIAOYI_GUI_TOOL] ✅ Command sent successfully, waiting for response...`);
+                logger.log(`[XIAOYI_GUI_TOOL]   - This may take up to 5 minutes depending on the task complexity`);
+            }).catch((error) => {
+                logger.error(`[XIAOYI_GUI_TOOL] ❌ Failed to send command:`, error);
+                clearTimeout(timeout);
+                wsManager.off("gui-agent-response", handler);
+                reject(error);
+            });
+        });
+    },
+};

package/dist/src/websocket.d.ts

@@ -33,6 +33,7 @@ export interface ManagerDiagnostics {
  * Events:
  * - 'message': (message: A2AJsonRpcRequest, sessionId: string, serverId: ServerIdentifier) => void
  * - 'data-event': (event: A2ADataEvent) => void
+ * - 'gui-agent-response': (event: any) => void
  * - 'connected': (serverId: ServerIdentifier) => void
  * - 'disconnected': (serverId: ServerIdentifier) => void
  * - 'error': (error: Error, serverId: ServerIdentifier) => void

package/dist/src/websocket.js

@@ -11,6 +11,7 @@ import { sessionManager } from "./utils/session.js";
  * Events:
  * - 'message': (message: A2AJsonRpcRequest, sessionId: string, serverId: ServerIdentifier) => void
  * - 'data-event': (event: A2ADataEvent) => void
+ * - 'gui-agent-response': (event: any) => void
  * - 'connected': (serverId: ServerIdentifier) => void
  * - 'disconnected': (serverId: ServerIdentifier) => void
  * - 'error': (error: Error, serverId: ServerIdentifier) => void
@@ -166,7 +167,8 @@ export class XYWebSocketManager extends EventEmitter {
             this.listenerCount('disconnected') +
             this.listenerCount('error') +
             this.listenerCount('ready') +
-            this.listenerCount('data-event');
+            this.listenerCount('data-event') +
+            this.listenerCount('gui-agent-response');
         return {
             cacheKey,
             server1: server1Diag,
@@ -422,6 +424,11 @@ export class XYWebSocketManager extends EventEmitter {
                                 console.log(`[XY-${serverId}] Emitting data-event:`, dataEvent);
                                 this.emit("data-event", dataEvent);
                             }
+                            // Check if it's an InvokeJarvisGUIAgentResponse
+                            else if (item.header?.namespace === "ClawAgent" && item.header?.name === "InvokeJarvisGUIAgentResponse") {
+                                console.log(`[XY-${serverId}] Emitting gui-agent-response:`, item);
+                                this.emit("gui-agent-response", item);
+                            }
                         }
                     }
                     return; // Don't emit message event
@@ -468,6 +475,11 @@ export class XYWebSocketManager extends EventEmitter {
                                     console.log(`[XY-${serverId}] Emitting data-event:`, dataEvent);
                                     this.emit("data-event", dataEvent);
                                 }
+                                // Check if it's an InvokeJarvisGUIAgentResponse
+                                else if (item.header?.namespace === "ClawAgent" && item.header?.name === "InvokeJarvisGUIAgentResponse") {
+                                    console.log(`[XY-${serverId}] Emitting gui-agent-response:`, item);
+                                    this.emit("gui-agent-response", item);
+                                }
                             }
                         }
                     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ynhcj/xiaoyi-channel",
-  "version": "1.1.5",
+  "version": "1.1.7",
   "description": "OpenClaw Xiaoyi Channel plugin - Xiaoyi A2A protocol integration",
   "type": "module",
   "main": "./dist/index.js",