@greatlhd/ailo-desktop 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/copy-static.mjs +11 -0
  2. package/dist/browser_control.js +767 -0
  3. package/dist/browser_snapshot.js +174 -0
  4. package/dist/cli.js +36 -0
  5. package/dist/code_executor.js +95 -0
  6. package/dist/config_server.js +658 -0
  7. package/dist/connection_util.js +14 -0
  8. package/dist/constants.js +2 -0
  9. package/dist/desktop_state_store.js +57 -0
  10. package/dist/desktop_types.js +1 -0
  11. package/dist/desktop_verifier.js +40 -0
  12. package/dist/dingtalk-handler.js +173 -0
  13. package/dist/dingtalk-types.js +1 -0
  14. package/dist/email_handler.js +501 -0
  15. package/dist/exec_tool.js +90 -0
  16. package/dist/feishu-handler.js +620 -0
  17. package/dist/feishu-types.js +8 -0
  18. package/dist/feishu-utils.js +162 -0
  19. package/dist/fs_tools.js +398 -0
  20. package/dist/index.js +433 -0
  21. package/dist/mcp/config-manager.js +64 -0
  22. package/dist/mcp/index.js +3 -0
  23. package/dist/mcp/rpc.js +109 -0
  24. package/dist/mcp/session.js +140 -0
  25. package/dist/mcp_manager.js +253 -0
  26. package/dist/mouse_keyboard.js +516 -0
  27. package/dist/qq-handler.js +153 -0
  28. package/dist/qq-types.js +15 -0
  29. package/dist/qq-ws.js +178 -0
  30. package/dist/screenshot.js +271 -0
  31. package/dist/skills_hub.js +212 -0
  32. package/dist/skills_manager.js +103 -0
  33. package/dist/static/AGENTS.md +25 -0
  34. package/dist/static/app.css +539 -0
  35. package/dist/static/app.html +292 -0
  36. package/dist/static/app.js +380 -0
  37. package/dist/static/chat.html +994 -0
  38. package/dist/time_tool.js +22 -0
  39. package/dist/utils.js +15 -0
  40. package/package.json +38 -0
  41. package/src/browser_control.ts +739 -0
  42. package/src/browser_snapshot.ts +196 -0
  43. package/src/cli.ts +44 -0
  44. package/src/code_executor.ts +101 -0
  45. package/src/config_server.ts +723 -0
  46. package/src/connection_util.ts +23 -0
  47. package/src/constants.ts +2 -0
  48. package/src/desktop_state_store.ts +64 -0
  49. package/src/desktop_types.ts +44 -0
  50. package/src/desktop_verifier.ts +45 -0
  51. package/src/dingtalk-types.ts +26 -0
  52. package/src/exec_tool.ts +93 -0
  53. package/src/feishu-handler.ts +722 -0
  54. package/src/feishu-types.ts +66 -0
  55. package/src/feishu-utils.ts +174 -0
  56. package/src/fs_tools.ts +411 -0
  57. package/src/index.ts +474 -0
  58. package/src/mcp/config-manager.ts +85 -0
  59. package/src/mcp/index.ts +7 -0
  60. package/src/mcp/rpc.ts +131 -0
  61. package/src/mcp/session.ts +182 -0
  62. package/src/mcp_manager.ts +273 -0
  63. package/src/mouse_keyboard.ts +526 -0
  64. package/src/qq-types.ts +49 -0
  65. package/src/qq-ws.ts +223 -0
  66. package/src/screenshot.ts +297 -0
  67. package/src/static/app.css +539 -0
  68. package/src/static/app.html +292 -0
  69. package/src/static/app.js +380 -0
  70. package/src/static/chat.html +994 -0
  71. package/src/time_tool.ts +24 -0
  72. package/src/utils.ts +22 -0
  73. package/tsconfig.json +13 -0
@@ -0,0 +1,516 @@
1
+ import { spawnSync } from "child_process";
2
+ import * as os from "os";
3
+ import { verifyDesktopAction } from "./desktop_verifier.js";
4
+ import { captureDesktopObservation } from "./screenshot.js";
5
+ function ok(data) {
6
+ return [{ type: "text", text: JSON.stringify({ ok: true, ...data }, null, 2) }];
7
+ }
8
+ function fail(error) {
9
+ return [{ type: "text", text: JSON.stringify({ ok: false, error }, null, 2) }];
10
+ }
11
+ function runPowerShell(script) {
12
+ const r = spawnSync("powershell", ["-Command", script], { encoding: "utf-8", timeout: 10000 });
13
+ return (r.stdout ?? "").trim();
14
+ }
15
+ function runShell(cmd) {
16
+ const r = spawnSync("/bin/sh", ["-c", cmd], { encoding: "utf-8", timeout: 10000 });
17
+ return (r.stdout ?? "").trim();
18
+ }
19
+ function getPrimaryScreenSize() {
20
+ const platform = os.platform();
21
+ if (platform === "win32") {
22
+ const ps = `Add-Type -AssemblyName System.Windows.Forms; $s=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; Write-Output "$($s.Width)x$($s.Height)"`;
23
+ const out = runPowerShell(ps);
24
+ const parts = out.split("x").map(Number);
25
+ const w = parts[0] ?? 1920;
26
+ const h = parts[1] ?? 1080;
27
+ return { width: w, height: h };
28
+ }
29
+ else if (platform === "darwin") {
30
+ const out = runShell("system_profiler SPDisplaysDataType | grep Resolution");
31
+ const m = out.match(/(\d+)\s*x\s*(\d+)/);
32
+ if (m)
33
+ return { width: Number(m[1]), height: Number(m[2]) };
34
+ }
35
+ else {
36
+ const out = runShell("xdpyinfo | grep dimensions");
37
+ const m = out.match(/(\d+)x(\d+)/);
38
+ if (m)
39
+ return { width: Number(m[1]), height: Number(m[2]) };
40
+ }
41
+ return { width: 1920, height: 1080 };
42
+ }
43
+ // ---------------------------------------------------------------------------
44
+ // 平台操作实现
45
+ // ---------------------------------------------------------------------------
46
+ function mouseMove(x, y) {
47
+ const platform = os.platform();
48
+ if (platform === "win32") {
49
+ const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x},${y})`;
50
+ runPowerShell(ps);
51
+ }
52
+ else if (platform === "darwin") {
53
+ spawnSync("cliclick", ["m:" + x + "," + y]);
54
+ }
55
+ else {
56
+ spawnSync("xdotool", ["mousemove", String(x), String(y)]);
57
+ }
58
+ }
59
+ function mouseClick(x, y, button = "left") {
60
+ const platform = os.platform();
61
+ if (platform === "win32") {
62
+ const btnCode = button === "right" ? 2 : 1;
63
+ const downFlag = btnCode === 2 ? "0x0008" : "0x0002";
64
+ const upFlag = btnCode === 2 ? "0x0010" : "0x0004";
65
+ const ps = [
66
+ "Add-Type -AssemblyName System.Windows.Forms",
67
+ `[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x},${y})`,
68
+ '$sig = @"',
69
+ '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, int dwExtraInfo);',
70
+ '"@',
71
+ "$m = Add-Type -MemberDefinition $sig -Name WinMouse -Namespace Win32 -PassThru",
72
+ `$m::mouse_event(${downFlag},0,0,0,0)`,
73
+ `$m::mouse_event(${upFlag},0,0,0,0)`,
74
+ ].join("; ");
75
+ runPowerShell(ps);
76
+ }
77
+ else if (platform === "darwin") {
78
+ const clickCmd = button === "right" ? "rc" : "c";
79
+ spawnSync("cliclick", [clickCmd + ":" + x + "," + y]);
80
+ }
81
+ else {
82
+ mouseMove(x, y);
83
+ const btn = button === "right" ? "3" : button === "middle" ? "2" : "1";
84
+ spawnSync("xdotool", ["click", "--button", btn]);
85
+ }
86
+ }
87
+ function mouseDoubleClick(x, y) {
88
+ const platform = os.platform();
89
+ if (platform === "win32") {
90
+ const ps = [
91
+ "Add-Type -AssemblyName System.Windows.Forms",
92
+ `[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x},${y})`,
93
+ '$sig = @"',
94
+ '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, int dwExtraInfo);',
95
+ '"@',
96
+ "$m = Add-Type -MemberDefinition $sig -Name WinMouse2 -Namespace Win32 -PassThru",
97
+ "$m::mouse_event(0x0002,0,0,0,0); $m::mouse_event(0x0004,0,0,0,0)",
98
+ "Start-Sleep -Milliseconds 50",
99
+ "$m::mouse_event(0x0002,0,0,0,0); $m::mouse_event(0x0004,0,0,0,0)",
100
+ ].join("; ");
101
+ runPowerShell(ps);
102
+ }
103
+ else if (platform === "darwin") {
104
+ spawnSync("cliclick", ["dc:" + x + "," + y]);
105
+ }
106
+ else {
107
+ mouseMove(x, y);
108
+ spawnSync("xdotool", ["click", "--repeat", "2", "--delay", "50", "1"]);
109
+ }
110
+ }
111
+ function mouseDrag(sx, sy, ex, ey) {
112
+ const platform = os.platform();
113
+ if (platform === "win32") {
114
+ const ps = [
115
+ "Add-Type -AssemblyName System.Windows.Forms",
116
+ `[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${sx},${sy})`,
117
+ '$sig = @"',
118
+ '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, int dwExtraInfo);',
119
+ '"@',
120
+ "$m = Add-Type -MemberDefinition $sig -Name WinMouse3 -Namespace Win32 -PassThru",
121
+ "$m::mouse_event(0x0002,0,0,0,0)",
122
+ "Start-Sleep -Milliseconds 50",
123
+ `[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${ex},${ey})`,
124
+ "Start-Sleep -Milliseconds 50",
125
+ "$m::mouse_event(0x0004,0,0,0,0)",
126
+ ].join("; ");
127
+ runPowerShell(ps);
128
+ }
129
+ else if (platform === "darwin") {
130
+ spawnSync("cliclick", ["dd:" + sx + "," + sy, "dm:" + ex + "," + ey, "du:" + ex + "," + ey]);
131
+ }
132
+ else {
133
+ spawnSync("xdotool", ["mousemove", String(sx), String(sy), "mousedown", "1", "mousemove", String(ex), String(ey), "mouseup", "1"]);
134
+ }
135
+ }
136
+ function keyboardType(text) {
137
+ const platform = os.platform();
138
+ if (platform === "win32") {
139
+ // 转义单引号(PowerShell 单引号字符串)
140
+ // 再转义 SendKeys 特殊字符:{} + ^ % ~
141
+ const escaped = text
142
+ .replace(/'/g, "''")
143
+ .replace(/[{}+^%~]/g, (c) => `{${c}}`);
144
+ const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${escaped}')`;
145
+ runPowerShell(ps);
146
+ }
147
+ else if (platform === "darwin") {
148
+ spawnSync("cliclick", ["t:" + text]);
149
+ }
150
+ else {
151
+ spawnSync("xdotool", ["type", "--clearmodifiers", text]);
152
+ }
153
+ }
154
+ function keyboardHotkey(keys) {
155
+ const platform = os.platform();
156
+ const keyNames = keys.split(/\s+/);
157
+ if (platform === "win32") {
158
+ let sendKeysStr = "";
159
+ for (const k of keyNames) {
160
+ const lower = k.toLowerCase();
161
+ const map = {
162
+ ctrl: "^", control: "^", alt: "%", shift: "+",
163
+ enter: "{ENTER}", return: "{ENTER}", tab: "{TAB}",
164
+ escape: "{ESC}", esc: "{ESC}", space: " ",
165
+ backspace: "{BACKSPACE}", delete: "{DELETE}",
166
+ up: "{UP}", down: "{DOWN}", left: "{LEFT}", right: "{RIGHT}",
167
+ home: "{HOME}", end: "{END}", pageup: "{PGUP}", pagedown: "{PGDN}",
168
+ f1: "{F1}", f2: "{F2}", f3: "{F3}", f4: "{F4}",
169
+ f5: "{F5}", f6: "{F6}", f7: "{F7}", f8: "{F8}",
170
+ f9: "{F9}", f10: "{F10}", f11: "{F11}", f12: "{F12}",
171
+ };
172
+ sendKeysStr += map[lower] ?? k;
173
+ }
174
+ const ps = `Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${sendKeysStr}')`;
175
+ runPowerShell(ps);
176
+ }
177
+ else if (platform === "darwin") {
178
+ const macKeys = keyNames.map(k => {
179
+ const map = { ctrl: "cmd", control: "cmd", alt: "alt", shift: "shift", meta: "cmd", cmd: "cmd" };
180
+ return map[k.toLowerCase()] ?? k;
181
+ });
182
+ spawnSync("cliclick", ["kp:" + macKeys.join("+")]);
183
+ }
184
+ else {
185
+ spawnSync("xdotool", ["key", keyNames.join("+")]);
186
+ }
187
+ }
188
+ function mouseScroll(direction, amount) {
189
+ if (direction !== "up" && direction !== "down") {
190
+ throw new Error(`direction 必须是 "up" 或 "down",收到: "${direction}"`);
191
+ }
192
+ const platform = os.platform();
193
+ if (platform === "win32") {
194
+ const delta = direction === "up" ? (amount * 120) : -(amount * 120);
195
+ const ps = [
196
+ '$sig = @"',
197
+ '[DllImport("user32.dll")] public static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, int dwExtraInfo);',
198
+ '"@',
199
+ "$m = Add-Type -MemberDefinition $sig -Name WinMouse4 -Namespace Win32 -PassThru",
200
+ `$m::mouse_event(0x0800,0,0,${delta},0)`,
201
+ ].join("; ");
202
+ runPowerShell(ps);
203
+ }
204
+ else if (platform === "darwin") {
205
+ const scrollAmount = direction === "up" ? amount : -amount;
206
+ spawnSync("cliclick", ["w:" + scrollAmount]);
207
+ }
208
+ else {
209
+ const btn = direction === "up" ? "4" : "5";
210
+ for (let i = 0; i < amount; i++) {
211
+ spawnSync("xdotool", ["click", btn]);
212
+ }
213
+ }
214
+ }
215
+ function normalizeAction(action) {
216
+ return String(action ?? "").trim().toLowerCase();
217
+ }
218
+ function buildActionResult(result, verdict, afterObservation) {
219
+ const payload = {
220
+ ok: result.accepted && result.executed,
221
+ action_result: {
222
+ accepted: result.accepted,
223
+ executed: result.executed,
224
+ action: result.action,
225
+ timestamp: result.timestamp,
226
+ observation_id: result.observationId,
227
+ error: result.error,
228
+ details: result.details,
229
+ },
230
+ };
231
+ if (verdict)
232
+ payload.verdict = verdict;
233
+ if (afterObservation) {
234
+ payload.after_observation = {
235
+ observation_id: afterObservation.id,
236
+ scope: afterObservation.scope,
237
+ coordinate_space: afterObservation.coordinateSpace,
238
+ image_width: afterObservation.imageWidth,
239
+ image_height: afterObservation.imageHeight,
240
+ image_path: afterObservation.image.path,
241
+ };
242
+ }
243
+ const parts = [{ type: "text", text: JSON.stringify(payload, null, 2) }];
244
+ if (afterObservation) {
245
+ parts.push({
246
+ type: "image",
247
+ media: {
248
+ type: "image",
249
+ path: afterObservation.image.path,
250
+ mime: afterObservation.image.mime,
251
+ name: afterObservation.image.name,
252
+ },
253
+ });
254
+ }
255
+ return parts;
256
+ }
257
+ function rejectAction(action, error) {
258
+ return buildActionResult({
259
+ accepted: false,
260
+ executed: false,
261
+ action,
262
+ timestamp: Date.now(),
263
+ error,
264
+ }, { status: "failure", reason: error });
265
+ }
266
+ function resolveObservation(args, deps) {
267
+ const stateStore = deps.stateStore;
268
+ if (!stateStore)
269
+ return null;
270
+ const observationId = typeof args.observation_id === "string" ? args.observation_id.trim() : "";
271
+ if (!observationId)
272
+ return null;
273
+ const observation = stateStore.getObservation(observationId);
274
+ if (!observation)
275
+ throw new Error(`observation_id 无效或已过期: ${observationId}`);
276
+ if (stateStore.isExpired(observation))
277
+ throw new Error(`observation 已过期,请重新 screenshot: ${observationId}`);
278
+ return observation;
279
+ }
280
+ function requireObservation(action, args, deps) {
281
+ const observation = resolveObservation(args, deps);
282
+ if (!observation)
283
+ throw new Error(`${action} 需要 observation_id,请先调用 screenshot 获取 observation`);
284
+ return observation;
285
+ }
286
+ function resolvePointFromObservation(observation, args, xKey, yKey, normXKey, normYKey) {
287
+ const { bounds } = observation.scope;
288
+ const rawX = args[xKey] !== undefined
289
+ ? bounds.x + Number(args[xKey])
290
+ : args[normXKey] !== undefined
291
+ ? bounds.x + bounds.width * Number(args[normXKey]) / 1000
292
+ : NaN;
293
+ const rawY = args[yKey] !== undefined
294
+ ? bounds.y + Number(args[yKey])
295
+ : args[normYKey] !== undefined
296
+ ? bounds.y + bounds.height * Number(args[normYKey]) / 1000
297
+ : NaN;
298
+ if (isNaN(rawX) || isNaN(rawY)) {
299
+ throw new Error(`需要提供 ${xKey}/${yKey}(基于observation的局部像素坐标)或 ${normXKey}/${normYKey}(归一化坐标0-1000)`);
300
+ }
301
+ // clamp 到 bounds 范围内,防止超出屏幕坐标
302
+ return [
303
+ Math.round(Math.max(bounds.x, Math.min(bounds.x + bounds.width, rawX))),
304
+ Math.round(Math.max(bounds.y, Math.min(bounds.y + bounds.height, rawY))),
305
+ ];
306
+ }
307
+ async function captureVerificationObservation(observation, deps) {
308
+ const captureOpts = observation.scope.kind === "screen" && observation.scope.screenIndex !== undefined
309
+ ? { screen: observation.scope.screenIndex }
310
+ : false;
311
+ const result = await captureDesktopObservation(captureOpts);
312
+ if (!result.observation)
313
+ return null;
314
+ deps.stateStore?.saveObservation(result.observation);
315
+ return result.observation;
316
+ }
317
+ export async function mouseKeyboard(args, deps = {}) {
318
+ const action = normalizeAction(args.action);
319
+ if (!action)
320
+ return fail("action 参数必填");
321
+ const verifyAfterAction = args.verify_after_action !== false && !!(args.verify_after_action ?? args.screenshot_after);
322
+ const verificationDelayMs = Math.max(0, Number(args.verification_delay_ms ?? 150));
323
+ try {
324
+ switch (action) {
325
+ case "get_screen_size": {
326
+ const size = getPrimaryScreenSize();
327
+ return ok(size);
328
+ }
329
+ case "click": {
330
+ const observation = requireObservation(action, args, deps);
331
+ const [x, y] = resolvePointFromObservation(observation, args, "x", "y", "norm_x", "norm_y");
332
+ mouseClick(x, y, args.button ?? "left");
333
+ const actionResult = {
334
+ accepted: true,
335
+ executed: true,
336
+ action,
337
+ timestamp: Date.now(),
338
+ observationId: observation.id,
339
+ details: { x, y, button: args.button ?? "left" },
340
+ };
341
+ deps.stateStore?.setLastAction(actionResult);
342
+ if (!verifyAfterAction)
343
+ return buildActionResult(actionResult);
344
+ if (verificationDelayMs > 0)
345
+ await new Promise((resolve) => setTimeout(resolve, verificationDelayMs));
346
+ const afterObservation = await captureVerificationObservation(observation, deps);
347
+ const verdict = verifyDesktopAction({ beforeObservation: observation, afterObservation, actionResult });
348
+ deps.stateStore?.setLastVerdict(verdict);
349
+ return buildActionResult(actionResult, verdict, afterObservation ?? undefined);
350
+ }
351
+ case "double_click": {
352
+ const observation = requireObservation(action, args, deps);
353
+ const [x, y] = resolvePointFromObservation(observation, args, "x", "y", "norm_x", "norm_y");
354
+ mouseDoubleClick(x, y);
355
+ const actionResult = {
356
+ accepted: true,
357
+ executed: true,
358
+ action,
359
+ timestamp: Date.now(),
360
+ observationId: observation.id,
361
+ details: { x, y },
362
+ };
363
+ deps.stateStore?.setLastAction(actionResult);
364
+ if (!verifyAfterAction)
365
+ return buildActionResult(actionResult);
366
+ if (verificationDelayMs > 0)
367
+ await new Promise((resolve) => setTimeout(resolve, verificationDelayMs));
368
+ const afterObservation = await captureVerificationObservation(observation, deps);
369
+ const verdict = verifyDesktopAction({ beforeObservation: observation, afterObservation, actionResult });
370
+ deps.stateStore?.setLastVerdict(verdict);
371
+ return buildActionResult(actionResult, verdict, afterObservation ?? undefined);
372
+ }
373
+ case "right_click": {
374
+ const observation = requireObservation(action, args, deps);
375
+ const [x, y] = resolvePointFromObservation(observation, args, "x", "y", "norm_x", "norm_y");
376
+ mouseClick(x, y, "right");
377
+ const actionResult = {
378
+ accepted: true,
379
+ executed: true,
380
+ action,
381
+ timestamp: Date.now(),
382
+ observationId: observation.id,
383
+ details: { x, y },
384
+ };
385
+ deps.stateStore?.setLastAction(actionResult);
386
+ if (!verifyAfterAction)
387
+ return buildActionResult(actionResult);
388
+ if (verificationDelayMs > 0)
389
+ await new Promise((resolve) => setTimeout(resolve, verificationDelayMs));
390
+ const afterObservation = await captureVerificationObservation(observation, deps);
391
+ const verdict = verifyDesktopAction({ beforeObservation: observation, afterObservation, actionResult });
392
+ deps.stateStore?.setLastVerdict(verdict);
393
+ return buildActionResult(actionResult, verdict, afterObservation ?? undefined);
394
+ }
395
+ case "move": {
396
+ const observation = requireObservation(action, args, deps);
397
+ const [x, y] = resolvePointFromObservation(observation, args, "x", "y", "norm_x", "norm_y");
398
+ mouseMove(x, y);
399
+ const actionResult = {
400
+ accepted: true,
401
+ executed: true,
402
+ action,
403
+ timestamp: Date.now(),
404
+ observationId: observation.id,
405
+ details: { x, y },
406
+ };
407
+ deps.stateStore?.setLastAction(actionResult);
408
+ return buildActionResult(actionResult);
409
+ }
410
+ case "drag": {
411
+ const observation = requireObservation(action, args, deps);
412
+ const [sx, sy] = resolvePointFromObservation(observation, args, "start_x", "start_y", "start_norm_x", "start_norm_y");
413
+ const [ex, ey] = resolvePointFromObservation(observation, args, "end_x", "end_y", "end_norm_x", "end_norm_y");
414
+ mouseDrag(sx, sy, ex, ey);
415
+ const actionResult = {
416
+ accepted: true,
417
+ executed: true,
418
+ action,
419
+ timestamp: Date.now(),
420
+ observationId: observation.id,
421
+ details: { startX: sx, startY: sy, endX: ex, endY: ey },
422
+ };
423
+ deps.stateStore?.setLastAction(actionResult);
424
+ if (!verifyAfterAction)
425
+ return buildActionResult(actionResult);
426
+ if (verificationDelayMs > 0)
427
+ await new Promise((resolve) => setTimeout(resolve, verificationDelayMs));
428
+ const afterObservation = await captureVerificationObservation(observation, deps);
429
+ const verdict = verifyDesktopAction({ beforeObservation: observation, afterObservation, actionResult });
430
+ deps.stateStore?.setLastVerdict(verdict);
431
+ return buildActionResult(actionResult, verdict, afterObservation ?? undefined);
432
+ }
433
+ case "type": {
434
+ const observation = requireObservation(action, args, deps);
435
+ const text = args.text;
436
+ if (!text)
437
+ return rejectAction(action, "type 操作需要 text 参数");
438
+ keyboardType(text);
439
+ const actionResult = {
440
+ accepted: true,
441
+ executed: true,
442
+ action,
443
+ timestamp: Date.now(),
444
+ observationId: observation.id,
445
+ details: { text },
446
+ };
447
+ deps.stateStore?.setLastAction(actionResult);
448
+ if (!verifyAfterAction)
449
+ return buildActionResult(actionResult);
450
+ if (verificationDelayMs > 0)
451
+ await new Promise((resolve) => setTimeout(resolve, verificationDelayMs));
452
+ const afterObservation = await captureVerificationObservation(observation, deps);
453
+ const verdict = verifyDesktopAction({ beforeObservation: observation, afterObservation, actionResult });
454
+ deps.stateStore?.setLastVerdict(verdict);
455
+ return buildActionResult(actionResult, verdict, afterObservation ?? undefined);
456
+ }
457
+ case "hotkey": {
458
+ const observation = requireObservation(action, args, deps);
459
+ const keys = args.keys;
460
+ if (!keys)
461
+ return rejectAction(action, "hotkey 操作需要 keys 参数");
462
+ keyboardHotkey(keys);
463
+ const actionResult = {
464
+ accepted: true,
465
+ executed: true,
466
+ action,
467
+ timestamp: Date.now(),
468
+ observationId: observation.id,
469
+ details: { keys },
470
+ };
471
+ deps.stateStore?.setLastAction(actionResult);
472
+ if (!verifyAfterAction)
473
+ return buildActionResult(actionResult);
474
+ if (verificationDelayMs > 0)
475
+ await new Promise((resolve) => setTimeout(resolve, verificationDelayMs));
476
+ const afterObservation = await captureVerificationObservation(observation, deps);
477
+ const verdict = verifyDesktopAction({ beforeObservation: observation, afterObservation, actionResult });
478
+ deps.stateStore?.setLastVerdict(verdict);
479
+ return buildActionResult(actionResult, verdict, afterObservation ?? undefined);
480
+ }
481
+ case "scroll": {
482
+ const observation = requireObservation(action, args, deps);
483
+ const direction = args.direction ?? "down";
484
+ const amount = args.amount ?? 3;
485
+ if (args.x !== undefined || args.norm_x !== undefined) {
486
+ const [x, y] = resolvePointFromObservation(observation, args, "x", "y", "norm_x", "norm_y");
487
+ mouseMove(x, y);
488
+ }
489
+ mouseScroll(direction, amount);
490
+ const actionResult = {
491
+ accepted: true,
492
+ executed: true,
493
+ action,
494
+ timestamp: Date.now(),
495
+ observationId: observation.id,
496
+ details: { direction, amount },
497
+ };
498
+ deps.stateStore?.setLastAction(actionResult);
499
+ if (!verifyAfterAction)
500
+ return buildActionResult(actionResult);
501
+ if (verificationDelayMs > 0)
502
+ await new Promise((resolve) => setTimeout(resolve, verificationDelayMs));
503
+ const afterObservation = await captureVerificationObservation(observation, deps);
504
+ const verdict = verifyDesktopAction({ beforeObservation: observation, afterObservation, actionResult });
505
+ deps.stateStore?.setLastVerdict(verdict);
506
+ return buildActionResult(actionResult, verdict, afterObservation ?? undefined);
507
+ }
508
+ default:
509
+ return fail(`未知操作: ${action}`);
510
+ }
511
+ }
512
+ catch (err) {
513
+ const msg = err instanceof Error ? err.message : String(err);
514
+ return rejectAction(action, msg);
515
+ }
516
+ }
@@ -0,0 +1,153 @@
1
+ import { textPart, } from "@lmcl/ailo-endpoint-sdk";
2
+ import { DEFAULT_API_BASE } from "./qq-types.js";
3
+ import { QQGatewayClient } from "./qq-ws.js";
4
+ import { createChannelLogger } from "./utils.js";
5
+ export class QQHandler {
6
+ config;
7
+ ctx = null;
8
+ gateway = null;
9
+ lastMsgIdByChatId = new Map();
10
+ constructor(config) {
11
+ this.config = config;
12
+ }
13
+ get apiBase() {
14
+ return (this.config.apiBase ?? DEFAULT_API_BASE).replace(/\/$/, "");
15
+ }
16
+ _log = createChannelLogger("qq", () => this.ctx);
17
+ acceptMessage(msg) {
18
+ if (!this.ctx)
19
+ return;
20
+ this.ctx.accept(msg).catch((err) => this._log("error", "accept failed", { err: String(err) }));
21
+ }
22
+ buildAcceptMessage(opts) {
23
+ const { chatId, text, chatType, senderId, senderName, msgId } = opts;
24
+ const tags = [
25
+ { kind: "channel", value: "QQ", groupWith: true },
26
+ { kind: "conv_type", value: chatType, groupWith: false },
27
+ { kind: "chat_id", value: chatId, groupWith: true, passToTool: true },
28
+ ];
29
+ if (senderName) {
30
+ tags.push({ kind: "participant", value: senderName, groupWith: false });
31
+ }
32
+ if (senderId) {
33
+ tags.push({ kind: "sender_id", value: senderId, groupWith: false, passToTool: true });
34
+ }
35
+ if (msgId) {
36
+ tags.push({ kind: "msg_id", value: msgId, groupWith: false, passToTool: true });
37
+ }
38
+ return { content: [textPart(text)], contextTags: tags };
39
+ }
40
+ stripAtPrefix(content) {
41
+ return content.replace(/<@!\d+>\s*/g, "").trim();
42
+ }
43
+ async start(ctx) {
44
+ this.ctx = ctx;
45
+ const gateway = new QQGatewayClient(this.config, (event, data) => this.handleDispatch(event, data), (level, msg, d) => this._log(level, msg, d));
46
+ this.gateway = gateway;
47
+ await gateway.connect();
48
+ this._log("info", "QQ Bot Gateway 已连接");
49
+ ctx.reportHealth("connected");
50
+ }
51
+ handleIncomingMessage(msg, opts) {
52
+ if ("author" in msg && msg.author?.bot)
53
+ return;
54
+ const text = opts.stripAt ? this.stripAtPrefix(msg.content ?? "") : (msg.content ?? "").trim();
55
+ if (!text)
56
+ return;
57
+ const chatId = `${opts.prefix}:${opts.idField}`;
58
+ if (msg.id)
59
+ this.lastMsgIdByChatId.set(chatId, msg.id);
60
+ this.acceptMessage(this.buildAcceptMessage({
61
+ chatId,
62
+ text,
63
+ chatType: opts.chatType,
64
+ senderId: msg.author?.user_openid ?? msg.author?.id ?? "",
65
+ senderName: msg.author?.username ?? "",
66
+ msgId: msg.id,
67
+ }));
68
+ }
69
+ handleDispatch(event, data) {
70
+ switch (event) {
71
+ case "AT_MESSAGE_CREATE": {
72
+ const msg = data;
73
+ this.handleIncomingMessage(msg, { prefix: "ch", chatType: "频道", idField: msg.channel_id ?? "", stripAt: true });
74
+ break;
75
+ }
76
+ case "DIRECT_MESSAGE_CREATE": {
77
+ const msg = data;
78
+ this.handleIncomingMessage(msg, { prefix: "dm", chatType: "私聊", idField: msg.guild_id ?? "", stripAt: true });
79
+ break;
80
+ }
81
+ case "C2C_MESSAGE_CREATE": {
82
+ const msg = data;
83
+ const userId = msg.author?.user_openid ?? msg.author?.id ?? "";
84
+ this.handleIncomingMessage(msg, { prefix: "c2c", chatType: "私聊", idField: userId, stripAt: false });
85
+ break;
86
+ }
87
+ case "GROUP_AT_MESSAGE_CREATE": {
88
+ const msg = data;
89
+ this.handleIncomingMessage(msg, { prefix: "grp", chatType: "群聊", idField: msg.group_openid ?? msg.group_id ?? "", stripAt: true });
90
+ break;
91
+ }
92
+ case "PUBLIC_MESSAGES_DELETE":
93
+ this._log("debug", "ignored message delete event");
94
+ break;
95
+ default:
96
+ this._log("debug", `unhandled event: ${event}`);
97
+ }
98
+ }
99
+ async sendText(chatId, text, msgId) {
100
+ if (!text?.trim())
101
+ return;
102
+ if (!this.gateway)
103
+ throw new Error("QQ Gateway 未连接");
104
+ const token = this.gateway.getAccessToken();
105
+ if (!token)
106
+ throw new Error("QQ access token 不可用");
107
+ const [kind, id] = chatId.split(":", 2);
108
+ if (!kind || !id)
109
+ throw new Error(`无效的 chat_id 格式: ${chatId}`);
110
+ const resolvedMsgId = msgId || this.lastMsgIdByChatId.get(chatId);
111
+ let url;
112
+ let body;
113
+ switch (kind) {
114
+ case "ch":
115
+ url = `${this.apiBase}/channels/${id}/messages`;
116
+ body = { content: text, msg_id: resolvedMsgId || undefined };
117
+ break;
118
+ case "dm":
119
+ url = `${this.apiBase}/dms/${id}/messages`;
120
+ body = { content: text, msg_id: resolvedMsgId || undefined };
121
+ break;
122
+ case "c2c":
123
+ url = `${this.apiBase}/v2/users/${id}/messages`;
124
+ body = { content: text, msg_type: 0, msg_id: resolvedMsgId || undefined };
125
+ break;
126
+ case "grp":
127
+ url = `${this.apiBase}/v2/groups/${id}/messages`;
128
+ body = { content: text, msg_type: 0, msg_id: resolvedMsgId || undefined };
129
+ break;
130
+ default:
131
+ throw new Error(`不支持的 chat_id 类型: ${kind}`);
132
+ }
133
+ const res = await fetch(url, {
134
+ method: "POST",
135
+ headers: {
136
+ Authorization: `QQBot ${token}`,
137
+ "Content-Type": "application/json",
138
+ },
139
+ body: JSON.stringify(body),
140
+ });
141
+ if (!res.ok) {
142
+ const detail = await res.text().catch(() => "");
143
+ this._log("error", `发送消息失败: HTTP ${res.status}`, { url, detail });
144
+ throw new Error(`QQ 发送失败: HTTP ${res.status}`);
145
+ }
146
+ this._log("debug", `消息已发送到 ${chatId}`);
147
+ }
148
+ async stop() {
149
+ this.gateway?.close();
150
+ this.gateway = null;
151
+ this.ctx = null;
152
+ }
153
+ }
@@ -0,0 +1,15 @@
1
+ export const OP_DISPATCH = 0;
2
+ export const OP_HEARTBEAT = 1;
3
+ export const OP_IDENTIFY = 2;
4
+ export const OP_RESUME = 6;
5
+ export const OP_RECONNECT = 7;
6
+ export const OP_INVALID_SESSION = 9;
7
+ export const OP_HELLO = 10;
8
+ export const OP_HEARTBEAT_ACK = 11;
9
+ export const INTENT_PUBLIC_GUILD_MESSAGES = 1 << 30;
10
+ export const INTENT_DIRECT_MESSAGE = 1 << 12;
11
+ export const INTENT_GROUP_AND_C2C = 1 << 25;
12
+ export const DEFAULT_API_BASE = "https://api.sgroup.qq.com";
13
+ export const TOKEN_URL = "https://bots.qq.com/app/getAppAccessToken";
14
+ export const RECONNECT_DELAYS = [1, 2, 5, 10, 30, 60];
15
+ export const MAX_RECONNECT_ATTEMPTS = 50;