@ynhcj/xiaoyi-channel 0.0.44-beta → 0.0.44-next
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +0 -2
- package/dist/index.js +42 -2
- package/dist/src/bot.d.ts +1 -0
- package/dist/src/bot.js +59 -61
- package/dist/src/channel.js +30 -4
- package/dist/src/client.js +0 -9
- package/dist/src/cspl/call-api.d.ts +3 -0
- package/dist/src/cspl/call-api.js +86 -0
- package/dist/src/cspl/config.d.ts +19 -0
- package/dist/src/cspl/config.js +50 -0
- package/dist/src/cspl/constants.d.ts +43 -0
- package/dist/src/cspl/constants.js +22 -0
- package/dist/src/cspl/utils.d.ts +10 -0
- package/dist/src/cspl/utils.js +57 -0
- package/dist/src/file-upload.d.ts +5 -0
- package/dist/src/file-upload.js +88 -6
- package/dist/src/formatter.d.ts +2 -0
- package/dist/src/formatter.js +13 -30
- package/dist/src/heartbeat.js +0 -4
- package/dist/src/monitor.js +8 -10
- package/dist/src/onboarding.d.ts +3 -4
- package/dist/src/onboarding.js +2 -2
- package/dist/src/outbound.d.ts +2 -1
- package/dist/src/outbound.js +1 -19
- package/dist/src/parser.d.ts +6 -0
- package/dist/src/parser.js +16 -0
- package/dist/src/push.js +0 -21
- package/dist/src/reply-dispatcher.d.ts +4 -0
- package/dist/src/reply-dispatcher.js +46 -8
- package/dist/src/steer-injector.d.ts +16 -0
- package/dist/src/steer-injector.js +74 -0
- package/dist/src/thread-bindings.d.ts +54 -0
- package/dist/src/thread-bindings.js +214 -0
- package/dist/src/tools/calendar-tool.js +2 -37
- package/dist/src/tools/call-phone-tool.js +3 -60
- package/dist/src/tools/create-alarm-tool.js +8 -109
- package/dist/src/tools/delete-alarm-tool.js +5 -69
- package/dist/src/tools/device-tool-map.d.ts +4 -0
- package/dist/src/tools/device-tool-map.js +24 -0
- package/dist/src/tools/image-reading-tool.d.ts +5 -0
- package/dist/src/tools/image-reading-tool.js +328 -0
- package/dist/src/tools/location-tool.js +6 -40
- package/dist/src/tools/modify-alarm-tool.js +8 -114
- package/dist/src/tools/modify-note-tool.js +3 -41
- package/dist/src/tools/note-tool.js +4 -16
- package/dist/src/tools/search-alarm-tool.js +12 -118
- package/dist/src/tools/search-calendar-tool.js +4 -81
- package/dist/src/tools/search-contact-tool.js +2 -55
- package/dist/src/tools/search-file-tool.js +4 -61
- package/dist/src/tools/search-message-tool.js +2 -59
- package/dist/src/tools/search-note-tool.js +4 -22
- package/dist/src/tools/search-photo-gallery-tool.js +8 -57
- package/dist/src/tools/send-command-to-car-tool.d.ts +5 -0
- package/dist/src/tools/send-command-to-car-tool.js +85 -0
- package/dist/src/tools/send-file-to-user-tool.js +1 -39
- package/dist/src/tools/send-message-tool.js +5 -56
- package/dist/src/tools/session-manager.d.ts +1 -0
- package/dist/src/tools/session-manager.js +0 -45
- package/dist/src/tools/timestamp-to-utc8-tool.d.ts +12 -0
- package/dist/src/tools/timestamp-to-utc8-tool.js +104 -0
- package/dist/src/tools/upload-file-tool.js +0 -49
- package/dist/src/tools/upload-photo-tool.js +0 -42
- package/dist/src/tools/view-push-result-tool.js +0 -11
- package/dist/src/tools/xiaoyi-collection-tool.js +4 -82
- package/dist/src/tools/xiaoyi-gui-tool.js +0 -34
- package/dist/src/utils/runtime-manager.d.ts +7 -0
- package/dist/src/utils/runtime-manager.js +42 -0
- package/dist/src/websocket.js +33 -13
- package/package.json +3 -4
- package/dist/src/tools/search-photo-tool.d.ts +0 -9
- package/dist/src/tools/search-photo-tool.js +0 -270
- package/dist/src/utils/session.d.ts +0 -34
- package/dist/src/utils/session.js +0 -50
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { getXYWebSocketManager } from "../client.js";
|
|
2
2
|
import { sendCommand } from "../formatter.js";
|
|
3
3
|
import { getCurrentSessionContext } from "./session-manager.js";
|
|
4
|
-
import { logger } from "../utils/logger.js";
|
|
5
4
|
/**
|
|
6
5
|
* XY delete alarm tool - deletes existing alarms on user's device.
|
|
7
6
|
* Requires entityId(s) from search_alarm or create_alarm tool as prerequisite.
|
|
@@ -27,7 +26,9 @@ export const deleteAlarmTool = {
|
|
|
27
26
|
注意事项:
|
|
28
27
|
1. 操作超时时间为60秒,请勿重复调用此工具,如果超时或失败,最多重试一次。
|
|
29
28
|
2. 删除操作不可撤销,请确认 entityId 正确后再删除。
|
|
30
|
-
3. items 参数支持数组或 JSON
|
|
29
|
+
3. items 参数支持数组或 JSON 字符串格式,代码会自动解析。
|
|
30
|
+
|
|
31
|
+
回复约束:如果工具返回没有授权或者其他报错,只需要完整描述没有授权或者其他报错内容即可,不需要主动给用户提供解决方案,例如告诉用户如何授权,如何解决报错等都是不需要的,请严格遵守。`,
|
|
31
32
|
parameters: {
|
|
32
33
|
type: "object",
|
|
33
34
|
properties: {
|
|
@@ -40,85 +41,57 @@ export const deleteAlarmTool = {
|
|
|
40
41
|
required: ["items"],
|
|
41
42
|
},
|
|
42
43
|
async execute(toolCallId, params) {
|
|
43
|
-
logger.log(`[DELETE_ALARM_TOOL] 🚀 Starting execution`);
|
|
44
|
-
logger.log(`[DELETE_ALARM_TOOL] - toolCallId: ${toolCallId}`);
|
|
45
|
-
logger.log(`[DELETE_ALARM_TOOL] - params (raw):`, JSON.stringify(params));
|
|
46
|
-
logger.log(`[DELETE_ALARM_TOOL] - params.items type:`, typeof params.items);
|
|
47
|
-
logger.log(`[DELETE_ALARM_TOOL] - timestamp: ${new Date().toISOString()}`);
|
|
48
44
|
// ===== 参数规范化:兼容数组和 JSON 字符串 =====
|
|
49
45
|
let items = null;
|
|
50
46
|
if (!params.items) {
|
|
51
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Missing parameter: items`);
|
|
52
47
|
throw new Error("Missing required parameter: items");
|
|
53
48
|
}
|
|
54
49
|
// 情况1: 已经是数组
|
|
55
50
|
if (Array.isArray(params.items)) {
|
|
56
|
-
logger.log(`[DELETE_ALARM_TOOL] ✅ items is already an array`);
|
|
57
51
|
items = params.items;
|
|
58
52
|
}
|
|
59
53
|
// 情况2: 是字符串,尝试解析为 JSON 数组
|
|
60
54
|
else if (typeof params.items === 'string') {
|
|
61
|
-
logger.log(`[DELETE_ALARM_TOOL] 🔄 items is a string, attempting to parse as JSON...`);
|
|
62
55
|
try {
|
|
63
56
|
const parsed = JSON.parse(params.items);
|
|
64
57
|
if (Array.isArray(parsed)) {
|
|
65
|
-
logger.log(`[DELETE_ALARM_TOOL] ✅ Successfully parsed JSON string to array`);
|
|
66
58
|
items = parsed;
|
|
67
59
|
}
|
|
68
60
|
else {
|
|
69
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Parsed JSON is not an array:`, typeof parsed);
|
|
70
61
|
throw new Error("items must be an array or a JSON string representing an array");
|
|
71
62
|
}
|
|
72
63
|
}
|
|
73
64
|
catch (parseError) {
|
|
74
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Failed to parse items as JSON:`, parseError);
|
|
75
65
|
throw new Error(`items must be a valid JSON array string. Parse error: ${parseError instanceof Error ? parseError.message : String(parseError)}`);
|
|
76
66
|
}
|
|
77
67
|
}
|
|
78
68
|
// 情况3: 其他类型,报错
|
|
79
69
|
else {
|
|
80
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Invalid items type:`, typeof params.items);
|
|
81
70
|
throw new Error(`items must be an array or a JSON string, got ${typeof params.items}`);
|
|
82
71
|
}
|
|
83
72
|
// 验证数组非空
|
|
84
73
|
if (!items || items.length === 0) {
|
|
85
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ items array is empty`);
|
|
86
74
|
throw new Error("items array cannot be empty");
|
|
87
75
|
}
|
|
88
|
-
logger.log(`[DELETE_ALARM_TOOL] ✅ Normalized items:`, JSON.stringify(items));
|
|
89
76
|
// 验证每个 item 是否有有效的 entityId
|
|
90
77
|
for (let i = 0; i < items.length; i++) {
|
|
91
78
|
const item = items[i];
|
|
92
79
|
if (!item || typeof item !== 'object') {
|
|
93
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Item at index ${i} is not an object`);
|
|
94
80
|
throw new Error(`items[${i}] must be an object with entityId property`);
|
|
95
81
|
}
|
|
96
82
|
if (!item.entityId || typeof item.entityId !== 'string') {
|
|
97
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Item at index ${i} missing or invalid entityId`);
|
|
98
83
|
throw new Error(`items[${i}] must have a valid entityId string property`);
|
|
99
84
|
}
|
|
100
85
|
}
|
|
101
|
-
logger.log(`[DELETE_ALARM_TOOL] - items count: ${items.length}`);
|
|
102
|
-
logger.log(`[DELETE_ALARM_TOOL] - entityIds:`, items.map(item => item.entityId).join(", "));
|
|
103
86
|
// Get session context
|
|
104
|
-
logger.log(`[DELETE_ALARM_TOOL] 🔍 Attempting to get session context...`);
|
|
105
87
|
const sessionContext = getCurrentSessionContext();
|
|
106
88
|
if (!sessionContext) {
|
|
107
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ FAILED: No active session found!`);
|
|
108
|
-
logger.error(`[DELETE_ALARM_TOOL] - toolCallId: ${toolCallId}`);
|
|
109
89
|
throw new Error("No active XY session found. Delete alarm tool can only be used during an active conversation.");
|
|
110
90
|
}
|
|
111
|
-
logger.log(`[DELETE_ALARM_TOOL] ✅ Session context found`);
|
|
112
|
-
logger.log(`[DELETE_ALARM_TOOL] - sessionId: ${sessionContext.sessionId}`);
|
|
113
|
-
logger.log(`[DELETE_ALARM_TOOL] - taskId: ${sessionContext.taskId}`);
|
|
114
|
-
logger.log(`[DELETE_ALARM_TOOL] - messageId: ${sessionContext.messageId}`);
|
|
115
91
|
const { config, sessionId, taskId, messageId } = sessionContext;
|
|
116
92
|
// Get WebSocket manager
|
|
117
|
-
logger.log(`[DELETE_ALARM_TOOL] 🔌 Getting WebSocket manager...`);
|
|
118
93
|
const wsManager = getXYWebSocketManager(config);
|
|
119
|
-
logger.log(`[DELETE_ALARM_TOOL] ✅ WebSocket manager obtained`);
|
|
120
94
|
// Build DeleteAlarm command
|
|
121
|
-
logger.log(`[DELETE_ALARM_TOOL] 📦 Building DeleteAlarm command...`);
|
|
122
95
|
const command = {
|
|
123
96
|
header: {
|
|
124
97
|
namespace: "Common",
|
|
@@ -153,70 +126,35 @@ export const deleteAlarmTool = {
|
|
|
153
126
|
},
|
|
154
127
|
};
|
|
155
128
|
// Send command and wait for response (60 second timeout)
|
|
156
|
-
logger.log(`[DELETE_ALARM_TOOL] ⏳ Setting up promise to wait for alarm deletion response...`);
|
|
157
|
-
logger.log(`[DELETE_ALARM_TOOL] - Timeout: 60 seconds`);
|
|
158
129
|
return new Promise((resolve, reject) => {
|
|
159
130
|
const timeout = setTimeout(() => {
|
|
160
|
-
logger.error(`[DELETE_ALARM_TOOL] ⏰ Timeout: No response received within 60 seconds`);
|
|
161
131
|
wsManager.off("data-event", handler);
|
|
162
132
|
reject(new Error("删除闹钟超时(60秒)"));
|
|
163
133
|
}, 60000);
|
|
164
134
|
// Listen for data events from WebSocket
|
|
165
135
|
const handler = (event) => {
|
|
166
|
-
logger.log(`[DELETE_ALARM_TOOL] 📨 Received data event:`, JSON.stringify(event));
|
|
167
136
|
if (event.intentName === "DeleteAlarm") {
|
|
168
|
-
logger.log(`[DELETE_ALARM_TOOL] 🎯 DeleteAlarm event received`);
|
|
169
|
-
logger.log(`[DELETE_ALARM_TOOL] - status: ${event.status}`);
|
|
170
137
|
clearTimeout(timeout);
|
|
171
138
|
wsManager.off("data-event", handler);
|
|
172
139
|
if (event.status === "success" && event.outputs) {
|
|
173
|
-
|
|
174
|
-
logger.log(`[DELETE_ALARM_TOOL] - outputs:`, JSON.stringify(event.outputs));
|
|
175
|
-
// Check for error code in outputs
|
|
176
|
-
const code = event.outputs.code !== undefined ? event.outputs.code : null;
|
|
177
|
-
if (code !== null && code !== 0) {
|
|
178
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Device returned error`);
|
|
179
|
-
logger.error(`[DELETE_ALARM_TOOL] - code: ${code}`);
|
|
180
|
-
const errorMsg = event.outputs.errorMsg || event.outputs.errMsg || "未知错误";
|
|
181
|
-
logger.error(`[DELETE_ALARM_TOOL] - errorMsg: ${errorMsg}`);
|
|
182
|
-
reject(new Error(`删除闹钟失败: ${errorMsg} (错误代码: ${code})`));
|
|
183
|
-
return;
|
|
184
|
-
}
|
|
185
|
-
// Extract result with safe navigation
|
|
186
|
-
const result = event.outputs.result || {};
|
|
187
|
-
logger.log(`[DELETE_ALARM_TOOL] 📋 Deletion result:`, JSON.stringify(result));
|
|
188
|
-
// Build response with safe navigation
|
|
189
|
-
const response = {
|
|
190
|
-
success: true,
|
|
191
|
-
entityName: result.entityName || "Alarm",
|
|
192
|
-
message: result.message || "Alarm deleted successfully",
|
|
193
|
-
deletedCount: items.length,
|
|
194
|
-
};
|
|
195
|
-
// Add entityIds from request for reference
|
|
196
|
-
response.deletedIds = items.map(item => item.entityId);
|
|
197
|
-
logger.log(`[DELETE_ALARM_TOOL] 🎉 Successfully deleted ${items.length} alarm(s)`);
|
|
140
|
+
// 成功,直接返回完整的 event.outputs JSON 字符串
|
|
198
141
|
resolve({
|
|
199
142
|
content: [
|
|
200
143
|
{
|
|
201
144
|
type: "text",
|
|
202
|
-
text: JSON.stringify(
|
|
145
|
+
text: JSON.stringify(event.outputs),
|
|
203
146
|
},
|
|
204
147
|
],
|
|
205
148
|
});
|
|
206
149
|
}
|
|
207
150
|
else {
|
|
208
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Alarm deletion failed`);
|
|
209
|
-
logger.error(`[DELETE_ALARM_TOOL] - status: ${event.status}`);
|
|
210
|
-
logger.error(`[DELETE_ALARM_TOOL] - outputs:`, JSON.stringify(event.outputs || {}));
|
|
211
151
|
reject(new Error(`删除闹钟失败: ${event.status}`));
|
|
212
152
|
}
|
|
213
153
|
}
|
|
214
154
|
};
|
|
215
155
|
// Register event handler
|
|
216
|
-
logger.log(`[DELETE_ALARM_TOOL] 📡 Registering data-event handler on WebSocket manager`);
|
|
217
156
|
wsManager.on("data-event", handler);
|
|
218
157
|
// Send the command
|
|
219
|
-
logger.log(`[DELETE_ALARM_TOOL] 📤 Sending DeleteAlarm command...`);
|
|
220
158
|
sendCommand({
|
|
221
159
|
config,
|
|
222
160
|
sessionId,
|
|
@@ -225,10 +163,8 @@ export const deleteAlarmTool = {
|
|
|
225
163
|
command,
|
|
226
164
|
})
|
|
227
165
|
.then(() => {
|
|
228
|
-
logger.log(`[DELETE_ALARM_TOOL] ✅ Command sent successfully, waiting for response...`);
|
|
229
166
|
})
|
|
230
167
|
.catch((error) => {
|
|
231
|
-
logger.error(`[DELETE_ALARM_TOOL] ❌ Failed to send command:`, error);
|
|
232
168
|
clearTimeout(timeout);
|
|
233
169
|
wsManager.off("data-event", handler);
|
|
234
170
|
reject(error);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Device type to tool name mapping.
|
|
2
|
+
// Only tools listed under a device type are available for that device.
|
|
3
|
+
// Tools NOT listed here are available to ALL devices (no restriction).
|
|
4
|
+
/** Known device type enum. */
|
|
5
|
+
export const DEVICE_TYPES = ["car", "pc", "phone"];
|
|
6
|
+
/**
|
|
7
|
+
* Map: deviceType → tool names allowed for that device.
|
|
8
|
+
* undefined / empty deviceType → all tools available.
|
|
9
|
+
* Unrecognized deviceType → all tools available.
|
|
10
|
+
* Tool not listed in any device entry → available to all devices.
|
|
11
|
+
*/
|
|
12
|
+
const DEVICE_TOOL_ALLOWLIST = {
|
|
13
|
+
car: ["send_command_to_car"],
|
|
14
|
+
pc: ["location"],
|
|
15
|
+
phone: ["location"],
|
|
16
|
+
};
|
|
17
|
+
export function filterToolsByDevice(tools, deviceType) {
|
|
18
|
+
if (!deviceType)
|
|
19
|
+
return tools;
|
|
20
|
+
const allowedTools = DEVICE_TOOL_ALLOWLIST[deviceType];
|
|
21
|
+
if (!allowedTools)
|
|
22
|
+
return tools; // unrecognized device → no filtering
|
|
23
|
+
return tools.filter((tool) => allowedTools.includes(tool.name));
|
|
24
|
+
}
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
// Image Reading tool implementation
|
|
2
|
+
import { XYFileUploadService } from "../file-upload.js";
|
|
3
|
+
import { getCurrentSessionContext } from "./session-manager.js";
|
|
4
|
+
import fetch from "node-fetch";
|
|
5
|
+
import fs from "fs/promises";
|
|
6
|
+
import path from "path";
|
|
7
|
+
import { v4 as uuidv4 } from "uuid";
|
|
8
|
+
/**
|
|
9
|
+
* Check if value is a remote URL
|
|
10
|
+
*/
|
|
11
|
+
function isRemoteUrl(value) {
|
|
12
|
+
try {
|
|
13
|
+
const url = new URL(value);
|
|
14
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Check if value is a local file path
|
|
22
|
+
*/
|
|
23
|
+
async function isLocalFile(value) {
|
|
24
|
+
try {
|
|
25
|
+
const stats = await fs.stat(value);
|
|
26
|
+
return stats.isFile();
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Download remote file to local temp directory
|
|
34
|
+
*/
|
|
35
|
+
async function downloadRemoteFile(url) {
|
|
36
|
+
try {
|
|
37
|
+
const response = await fetch(url);
|
|
38
|
+
if (!response.ok) {
|
|
39
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
40
|
+
}
|
|
41
|
+
// Get filename from URL or use default
|
|
42
|
+
let filename = url.split("/").pop() || "downloaded_image";
|
|
43
|
+
filename = filename.split("?")[0];
|
|
44
|
+
// Ensure temp directory exists
|
|
45
|
+
const tempDir = "/tmp/xy_channel";
|
|
46
|
+
await fs.mkdir(tempDir, { recursive: true });
|
|
47
|
+
// Generate unique filename to avoid conflicts
|
|
48
|
+
const timestamp = Date.now();
|
|
49
|
+
const ext = path.extname(filename) || ".jpg";
|
|
50
|
+
const baseName = path.basename(filename, ext);
|
|
51
|
+
const uniqueFilename = `${baseName}_${timestamp}${ext}`;
|
|
52
|
+
const localPath = path.join(tempDir, uniqueFilename);
|
|
53
|
+
// Save file to local temp directory
|
|
54
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
55
|
+
const buffer = Buffer.from(arrayBuffer);
|
|
56
|
+
await fs.writeFile(localPath, buffer);
|
|
57
|
+
return localPath;
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
throw new Error(`Failed to download remote file: ${error instanceof Error ? error.message : String(error)}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Process image input: validate and convert local file to OBS URL, keep remote URL unchanged
|
|
65
|
+
*/
|
|
66
|
+
async function processImageInput(imageInput, uploadService) {
|
|
67
|
+
// Check if it's a remote URL
|
|
68
|
+
if (isRemoteUrl(imageInput)) {
|
|
69
|
+
const localPath = await downloadRemoteFile(imageInput);
|
|
70
|
+
const imageUrl = await uploadService.uploadFileAndGetUrl(localPath, "TEMPORARY_MATERIAL_DOC");
|
|
71
|
+
if (!imageUrl) {
|
|
72
|
+
throw new Error("图片上传失败:无法获取图片访问地址");
|
|
73
|
+
}
|
|
74
|
+
return { imageUrl, localPath };
|
|
75
|
+
}
|
|
76
|
+
// Check if it's a local file
|
|
77
|
+
const isLocal = await isLocalFile(imageInput);
|
|
78
|
+
if (isLocal) {
|
|
79
|
+
const imageUrl = await uploadService.uploadFileAndGetUrl(imageInput, "TEMPORARY_MATERIAL_DOC");
|
|
80
|
+
if (!imageUrl) {
|
|
81
|
+
throw new Error("图片上传失败:无法获取图片访问地址");
|
|
82
|
+
}
|
|
83
|
+
return { imageUrl };
|
|
84
|
+
}
|
|
85
|
+
throw new Error(`Invalid image input: must be a remote URL or local file path, got: ${imageInput}`);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Call image understanding API with streaming response
|
|
89
|
+
*/
|
|
90
|
+
async function callImageUnderstandingAPI(imageUrl, text, apiKey, uid, fileUploadUrl) {
|
|
91
|
+
const apiUrl = `${fileUploadUrl}/celia-claw/v1/sse-api/skill/execute`;
|
|
92
|
+
const traceId = uuidv4();
|
|
93
|
+
const headers = {
|
|
94
|
+
"Content-Type": "application/json",
|
|
95
|
+
"Accept": "text/event-stream",
|
|
96
|
+
"x-hag-trace-id": traceId,
|
|
97
|
+
"x-api-key": apiKey,
|
|
98
|
+
"x-request-from": "openclaw",
|
|
99
|
+
"x-uid": uid,
|
|
100
|
+
"x-skill-id": "image_comprehension",
|
|
101
|
+
"x-prd-pkg-name": "com.huawei.hag",
|
|
102
|
+
};
|
|
103
|
+
const payload = {
|
|
104
|
+
version: "1.0",
|
|
105
|
+
session: {
|
|
106
|
+
isNew: false,
|
|
107
|
+
sessionId: "wangyu202410241921",
|
|
108
|
+
interactionId: 0,
|
|
109
|
+
},
|
|
110
|
+
endpoint: {
|
|
111
|
+
device: {
|
|
112
|
+
sid: "3df83a4a8124d7600f66206f96ea1e7e4e21c593adc4246bd20d450d8404cbf3",
|
|
113
|
+
deviceId: "3f35019f-ba4c-4ed5-80c0-6ddcef741200",
|
|
114
|
+
prdVer: "99.0.64.303",
|
|
115
|
+
phoneType: "WLZ-AL10",
|
|
116
|
+
sysVer: "HarmonyOS_2.0.0",
|
|
117
|
+
deviceType: 0,
|
|
118
|
+
timezone: "GMT+08:00",
|
|
119
|
+
},
|
|
120
|
+
locale: "zh-CN",
|
|
121
|
+
sysLocale: "zh",
|
|
122
|
+
countryCode: "CN",
|
|
123
|
+
},
|
|
124
|
+
utterance: { type: "text", original: text },
|
|
125
|
+
actions: [
|
|
126
|
+
{
|
|
127
|
+
actionSn: uuidv4(),
|
|
128
|
+
actionExecutorTask: {
|
|
129
|
+
pluginId: "aeac4e92c32949c1b7fc02de262615e6",
|
|
130
|
+
agentState: "OnShelf",
|
|
131
|
+
actionName: "imageUnderStandStream",
|
|
132
|
+
content: { imageUrl, text },
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
],
|
|
136
|
+
};
|
|
137
|
+
try {
|
|
138
|
+
const response = await fetch(apiUrl, {
|
|
139
|
+
method: "POST",
|
|
140
|
+
headers,
|
|
141
|
+
body: JSON.stringify(payload),
|
|
142
|
+
// @ts-ignore - node-fetch supports this
|
|
143
|
+
timeout: 120000, // 2 minutes timeout
|
|
144
|
+
});
|
|
145
|
+
if (!response.ok) {
|
|
146
|
+
const errorText = await response.text();
|
|
147
|
+
throw new Error(`API request failed: ${response.status} ${response.statusText}`);
|
|
148
|
+
}
|
|
149
|
+
// Process SSE stream
|
|
150
|
+
let lastCaption = "";
|
|
151
|
+
let lineCount = 0;
|
|
152
|
+
let buffer = "";
|
|
153
|
+
// Read the response body as a stream
|
|
154
|
+
if (!response.body) {
|
|
155
|
+
throw new Error("Response body is null");
|
|
156
|
+
}
|
|
157
|
+
for await (const chunk of response.body) {
|
|
158
|
+
if (!chunk)
|
|
159
|
+
continue;
|
|
160
|
+
buffer += chunk.toString();
|
|
161
|
+
const lines = buffer.split("\n");
|
|
162
|
+
buffer = lines.pop() || "";
|
|
163
|
+
for (const line of lines) {
|
|
164
|
+
lineCount++;
|
|
165
|
+
const trimmedLine = line.replace(/\r$/, "");
|
|
166
|
+
if (!trimmedLine)
|
|
167
|
+
continue;
|
|
168
|
+
if (trimmedLine.startsWith("data:")) {
|
|
169
|
+
const dataContent = trimmedLine.substring(5).trim();
|
|
170
|
+
if (dataContent && dataContent !== "[DONE]") {
|
|
171
|
+
try {
|
|
172
|
+
const dataJson = JSON.parse(dataContent);
|
|
173
|
+
// Extract streamContent from abilityInfos
|
|
174
|
+
if (dataJson.abilityInfos && Array.isArray(dataJson.abilityInfos)) {
|
|
175
|
+
for (const info of dataJson.abilityInfos) {
|
|
176
|
+
if (info.actionExecutorResult?.reply?.streamInfo) {
|
|
177
|
+
const streamContent = info.actionExecutorResult.reply.streamInfo.streamContent;
|
|
178
|
+
if (streamContent) {
|
|
179
|
+
lastCaption = streamContent;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
catch (parseError) {
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
if (!lastCaption) {
|
|
192
|
+
throw new Error("No caption received from image understanding API");
|
|
193
|
+
}
|
|
194
|
+
return lastCaption;
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
throw error;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* XY Image Reading tool - performs image understanding using local or remote image URLs.
|
|
202
|
+
* Supports both local file paths and remote URLs.
|
|
203
|
+
*/
|
|
204
|
+
export const imageReadingTool = {
|
|
205
|
+
name: "image_reading",
|
|
206
|
+
label: "Image Reading",
|
|
207
|
+
description: `
|
|
208
|
+
工具使用场景:
|
|
209
|
+
【必须调用此工具的情况】
|
|
210
|
+
1. 用户消息中包含 mediaPath 字段且不为空(表示用户发送了图片)
|
|
211
|
+
2. 用户希望理解图片内容,询问图片是什么,例如:
|
|
212
|
+
- "这是什么?"
|
|
213
|
+
- "图片里有什么?"
|
|
214
|
+
- "帮我看看这张图"
|
|
215
|
+
- "描述一下这张图片"
|
|
216
|
+
- "分析一下这张照片"
|
|
217
|
+
- "这个图片是什么意思"
|
|
218
|
+
- "识别一下图片内容"
|
|
219
|
+
- 或任何关于图片内容的理解、识别、分析类询问
|
|
220
|
+
|
|
221
|
+
当同时满足以上两个条件时,必须优先调用此工具进行图像理解。
|
|
222
|
+
|
|
223
|
+
工具能力描述:对图片进行理解和分析,返回图片的描述内容。
|
|
224
|
+
|
|
225
|
+
工具参数说明:
|
|
226
|
+
a. localUrl:本地图片文件路径(可选,通常从用户消息的 mediaPath 字段获取)
|
|
227
|
+
b. remoteUrl:公网图片地址(可选)
|
|
228
|
+
c. prompt:对图片的提示问题,默认为"描述这张图片内容",可根据用户的具体问题自定义
|
|
229
|
+
d. localUrl 与 remoteUrl 任意一个不为空即可,优先使用 localUrl
|
|
230
|
+
|
|
231
|
+
注意事项:
|
|
232
|
+
a. 支持常见图片格式(jpg, png, gif等)
|
|
233
|
+
b. 远程图片会先下载到本地再处理
|
|
234
|
+
c. 操作超时时间为2分钟(120秒)
|
|
235
|
+
d. 返回图像理解的文本描述内容`,
|
|
236
|
+
parameters: {
|
|
237
|
+
type: "object",
|
|
238
|
+
properties: {
|
|
239
|
+
localUrl: {
|
|
240
|
+
type: "string",
|
|
241
|
+
description: "本地图片文件路径",
|
|
242
|
+
},
|
|
243
|
+
remoteUrl: {
|
|
244
|
+
type: "string",
|
|
245
|
+
description: "公网图片地址(HTTP/HTTPS URL)",
|
|
246
|
+
},
|
|
247
|
+
prompt: {
|
|
248
|
+
type: "string",
|
|
249
|
+
description: "对图片的提示问题,默认为'描述这张图片内容'",
|
|
250
|
+
},
|
|
251
|
+
},
|
|
252
|
+
},
|
|
253
|
+
async execute(toolCallId, params) {
|
|
254
|
+
// Validate that at least one parameter is provided
|
|
255
|
+
if (!params.localUrl && !params.remoteUrl) {
|
|
256
|
+
throw new Error("At least one of localUrl or remoteUrl must be provided");
|
|
257
|
+
}
|
|
258
|
+
// Get prompt (default to "描述这张图片内容")
|
|
259
|
+
const prompt = params.prompt || "描述这张图片内容";
|
|
260
|
+
// Get session context
|
|
261
|
+
const sessionContext = getCurrentSessionContext();
|
|
262
|
+
if (!sessionContext) {
|
|
263
|
+
throw new Error("No active XY session found. Image reading tool can only be used during an active conversation.");
|
|
264
|
+
}
|
|
265
|
+
const { config } = sessionContext;
|
|
266
|
+
// Create upload service
|
|
267
|
+
const uploadService = new XYFileUploadService(config.fileUploadUrl, config.apiKey, config.uid);
|
|
268
|
+
let processedImage = null;
|
|
269
|
+
let downloadedFile = null;
|
|
270
|
+
try {
|
|
271
|
+
// Process image input (prefer localUrl over remoteUrl)
|
|
272
|
+
const imageInput = params.localUrl || params.remoteUrl;
|
|
273
|
+
processedImage = await processImageInput(imageInput, uploadService);
|
|
274
|
+
// Track downloaded file for cleanup
|
|
275
|
+
if (processedImage.localPath) {
|
|
276
|
+
downloadedFile = processedImage.localPath;
|
|
277
|
+
}
|
|
278
|
+
// Call image understanding API
|
|
279
|
+
const caption = await callImageUnderstandingAPI(processedImage.imageUrl, prompt, config.apiKey, config.uid, config.fileUploadUrl);
|
|
280
|
+
// Clean up downloaded file if any
|
|
281
|
+
if (downloadedFile) {
|
|
282
|
+
try {
|
|
283
|
+
await fs.unlink(downloadedFile);
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
return {
|
|
289
|
+
content: [
|
|
290
|
+
{
|
|
291
|
+
type: "text",
|
|
292
|
+
text: JSON.stringify({
|
|
293
|
+
caption,
|
|
294
|
+
prompt,
|
|
295
|
+
imageSource: params.localUrl ? "local" : "remote",
|
|
296
|
+
success: true,
|
|
297
|
+
}),
|
|
298
|
+
},
|
|
299
|
+
],
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
catch (error) {
|
|
303
|
+
// Clean up downloaded file on error
|
|
304
|
+
if (downloadedFile) {
|
|
305
|
+
try {
|
|
306
|
+
await fs.unlink(downloadedFile);
|
|
307
|
+
}
|
|
308
|
+
catch (cleanupError) {
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
const errorMessage = error instanceof Error ? error.message : "图片分析失败";
|
|
312
|
+
// Return error result instead of throwing
|
|
313
|
+
return {
|
|
314
|
+
content: [
|
|
315
|
+
{
|
|
316
|
+
type: "text",
|
|
317
|
+
text: JSON.stringify({
|
|
318
|
+
error: errorMessage,
|
|
319
|
+
prompt,
|
|
320
|
+
imageSource: params.localUrl ? "local" : "remote",
|
|
321
|
+
success: false,
|
|
322
|
+
}),
|
|
323
|
+
},
|
|
324
|
+
],
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
},
|
|
328
|
+
};
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { getXYWebSocketManager } from "../client.js";
|
|
2
2
|
import { sendCommand } from "../formatter.js";
|
|
3
3
|
import { getCurrentSessionContext } from "./session-manager.js";
|
|
4
|
-
import { logger } from "../utils/logger.js";
|
|
5
4
|
/**
|
|
6
5
|
* XY location tool - gets user's current location.
|
|
7
6
|
* Returns WGS84 coordinates (latitude, longitude).
|
|
@@ -9,38 +8,22 @@ import { logger } from "../utils/logger.js";
|
|
|
9
8
|
export const locationTool = {
|
|
10
9
|
name: "get_user_location",
|
|
11
10
|
label: "Get User Location",
|
|
12
|
-
description: "获取用户当前位置(经纬度坐标,WGS84坐标系)。需要用户设备授权位置访问权限。注意:操作超时时间为60
|
|
11
|
+
description: "获取用户当前位置(经纬度坐标,WGS84坐标系)。需要用户设备授权位置访问权限。注意:操作超时时间为60秒,请勿重复调用此工具,如果超时或失败,最多重试一次。回复约束:如果工具返回没有授权或者其他报错,只需要完整描述没有授权或者其他报错内容即可,不需要主动给用户提供解决方案,例如告诉用户如何授权,如何解决报错等都是不需要的,请严格遵守。",
|
|
13
12
|
parameters: {
|
|
14
13
|
type: "object",
|
|
15
14
|
properties: {},
|
|
16
15
|
required: [],
|
|
17
16
|
},
|
|
18
17
|
async execute(toolCallId, params) {
|
|
19
|
-
logger.log(`[LOCATION_TOOL] 🚀 Starting execution`);
|
|
20
|
-
logger.log(`[LOCATION_TOOL] - toolCallId: ${toolCallId}`);
|
|
21
|
-
logger.log(`[LOCATION_TOOL] - params:`, JSON.stringify(params));
|
|
22
|
-
logger.log(`[LOCATION_TOOL] - timestamp: ${new Date().toISOString()}`);
|
|
23
18
|
// Get session context
|
|
24
|
-
logger.log(`[LOCATION_TOOL] 🔍 Attempting to get session context...`);
|
|
25
19
|
const sessionContext = getCurrentSessionContext();
|
|
26
20
|
if (!sessionContext) {
|
|
27
|
-
logger.error(`[LOCATION_TOOL] ❌ FAILED: No active session found!`);
|
|
28
|
-
logger.error(`[LOCATION_TOOL] - toolCallId: ${toolCallId}`);
|
|
29
|
-
logger.error(`[LOCATION_TOOL] - This suggests the session was not registered or already cleaned up`);
|
|
30
21
|
throw new Error("No active XY session found. Location tool can only be used during an active conversation.");
|
|
31
22
|
}
|
|
32
|
-
logger.log(`[LOCATION_TOOL] ✅ Session context found`);
|
|
33
|
-
logger.log(`[LOCATION_TOOL] - sessionId: ${sessionContext.sessionId}`);
|
|
34
|
-
logger.log(`[LOCATION_TOOL] - taskId: ${sessionContext.taskId}`);
|
|
35
|
-
logger.log(`[LOCATION_TOOL] - messageId: ${sessionContext.messageId}`);
|
|
36
|
-
logger.log(`[LOCATION_TOOL] - agentId: ${sessionContext.agentId}`);
|
|
37
23
|
const { config, sessionId, taskId, messageId } = sessionContext;
|
|
38
24
|
// Get WebSocket manager
|
|
39
|
-
logger.log(`[LOCATION_TOOL] 🔌 Getting WebSocket manager...`);
|
|
40
25
|
const wsManager = getXYWebSocketManager(config);
|
|
41
|
-
logger.log(`[LOCATION_TOOL] ✅ WebSocket manager obtained`);
|
|
42
26
|
// Build GetCurrentLocation command
|
|
43
|
-
logger.log(`[LOCATION_TOOL] 📦 Building GetCurrentLocation command...`);
|
|
44
27
|
const command = {
|
|
45
28
|
header: {
|
|
46
29
|
namespace: "Common",
|
|
@@ -55,7 +38,9 @@ export const locationTool = {
|
|
|
55
38
|
dimension: "",
|
|
56
39
|
executeMode: "background",
|
|
57
40
|
intentName: "GetCurrentLocation",
|
|
58
|
-
intentParam: {
|
|
41
|
+
intentParam: {
|
|
42
|
+
isNeedGeoAddress: true,
|
|
43
|
+
},
|
|
59
44
|
needUnlock: true,
|
|
60
45
|
permissionId: [],
|
|
61
46
|
timeOut: 5,
|
|
@@ -70,53 +55,36 @@ export const locationTool = {
|
|
|
70
55
|
},
|
|
71
56
|
};
|
|
72
57
|
// Send command and wait for response (60 second timeout)
|
|
73
|
-
logger.log(`[LOCATION_TOOL] ⏳ Setting up promise to wait for location response...`);
|
|
74
|
-
logger.log(`[LOCATION_TOOL] - Timeout: 60 seconds`);
|
|
75
58
|
return new Promise((resolve, reject) => {
|
|
76
59
|
const timeout = setTimeout(() => {
|
|
77
|
-
logger.error(`[LOCATION_TOOL] ⏰ Timeout: No response received within 60 seconds`);
|
|
78
60
|
wsManager.off("data-event", handler);
|
|
79
61
|
reject(new Error("获取位置超时(60秒)"));
|
|
80
62
|
}, 60000);
|
|
81
63
|
// Listen for data events from WebSocket
|
|
82
64
|
const handler = (event) => {
|
|
83
|
-
logger.log(`[LOCATION_TOOL] 📨 Received data event:`, JSON.stringify(event));
|
|
84
65
|
if (event.intentName === "GetCurrentLocation") {
|
|
85
|
-
logger.log(`[LOCATION_TOOL] 🎯 GetCurrentLocation event received`);
|
|
86
|
-
logger.log(`[LOCATION_TOOL] - status: ${event.status}`);
|
|
87
66
|
clearTimeout(timeout);
|
|
88
67
|
wsManager.off("data-event", handler);
|
|
89
68
|
if (event.status === "success" && event.outputs) {
|
|
90
|
-
|
|
91
|
-
logger.log(`[LOCATION_TOOL] ✅ Location retrieved successfully`);
|
|
92
|
-
logger.log(`[LOCATION_TOOL] - latitude: ${latitude}`);
|
|
93
|
-
logger.log(`[LOCATION_TOOL] - longitude: ${longitude}`);
|
|
69
|
+
// 成功,直接返回完整的 event.outputs JSON 字符串
|
|
94
70
|
resolve({
|
|
95
71
|
content: [
|
|
96
72
|
{
|
|
97
73
|
type: "text",
|
|
98
|
-
text: JSON.stringify(
|
|
99
|
-
latitude,
|
|
100
|
-
longitude,
|
|
101
|
-
coordinateSystem: "WGS84",
|
|
102
|
-
})
|
|
74
|
+
text: JSON.stringify(event.outputs),
|
|
103
75
|
}
|
|
104
76
|
]
|
|
105
77
|
});
|
|
106
78
|
}
|
|
107
79
|
else {
|
|
108
|
-
logger.error(`[LOCATION_TOOL] ❌ Location retrieval failed`);
|
|
109
|
-
logger.error(`[LOCATION_TOOL] - status: ${event.status}`);
|
|
110
80
|
reject(new Error(`获取位置失败: ${event.status}`));
|
|
111
81
|
}
|
|
112
82
|
}
|
|
113
83
|
};
|
|
114
84
|
// Register event handler
|
|
115
85
|
// Note: The WebSocket manager needs to emit 'data-event' when receiving data events
|
|
116
|
-
logger.log(`[LOCATION_TOOL] 📡 Registering data-event handler on WebSocket manager`);
|
|
117
86
|
wsManager.on("data-event", handler);
|
|
118
87
|
// Send the command
|
|
119
|
-
logger.log(`[LOCATION_TOOL] 📤 Sending GetCurrentLocation command...`);
|
|
120
88
|
sendCommand({
|
|
121
89
|
config,
|
|
122
90
|
sessionId,
|
|
@@ -124,9 +92,7 @@ export const locationTool = {
|
|
|
124
92
|
messageId,
|
|
125
93
|
command,
|
|
126
94
|
}).then(() => {
|
|
127
|
-
logger.log(`[LOCATION_TOOL] ✅ Command sent successfully, waiting for response...`);
|
|
128
95
|
}).catch((error) => {
|
|
129
|
-
logger.error(`[LOCATION_TOOL] ❌ Failed to send command:`, error);
|
|
130
96
|
clearTimeout(timeout);
|
|
131
97
|
wsManager.off("data-event", handler);
|
|
132
98
|
reject(error);
|