@aiscene/aiserver 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/callback.d.ts +7 -0
- package/dist/api/callback.d.ts.map +1 -0
- package/dist/api/callback.js +116 -0
- package/dist/api/callback.js.map +1 -0
- package/dist/api/device-api.d.ts +13 -0
- package/dist/api/device-api.d.ts.map +1 -0
- package/dist/api/device-api.js +98 -0
- package/dist/api/device-api.js.map +1 -0
- package/dist/api/task-api.d.ts +9 -0
- package/dist/api/task-api.d.ts.map +1 -0
- package/dist/api/task-api.js +47 -0
- package/dist/api/task-api.js.map +1 -0
- package/dist/config/cli.d.ts +30 -0
- package/dist/config/cli.d.ts.map +1 -0
- package/dist/config/cli.js +129 -0
- package/dist/config/cli.js.map +1 -0
- package/dist/config/index.d.ts +6 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +142 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/schema.d.ts +66 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +2 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/core/event-bus.d.ts +14 -0
- package/dist/core/event-bus.d.ts.map +1 -0
- package/dist/core/event-bus.js +33 -0
- package/dist/core/event-bus.js.map +1 -0
- package/dist/core/logger.d.ts +24 -0
- package/dist/core/logger.d.ts.map +1 -0
- package/dist/core/logger.js +52 -0
- package/dist/core/logger.js.map +1 -0
- package/dist/core/types.d.ts +134 -0
- package/dist/core/types.d.ts.map +1 -0
- package/dist/core/types.js +3 -0
- package/dist/core/types.js.map +1 -0
- package/dist/debug/dump-manager.d.ts +10 -0
- package/dist/debug/dump-manager.d.ts.map +1 -0
- package/dist/debug/dump-manager.js +52 -0
- package/dist/debug/dump-manager.js.map +1 -0
- package/dist/debug/screencast.d.ts +11 -0
- package/dist/debug/screencast.d.ts.map +1 -0
- package/dist/debug/screencast.js +88 -0
- package/dist/debug/screencast.js.map +1 -0
- package/dist/debug/session-manager.d.ts +14 -0
- package/dist/debug/session-manager.d.ts.map +1 -0
- package/dist/debug/session-manager.js +76 -0
- package/dist/debug/session-manager.js.map +1 -0
- package/dist/debug/types.d.ts +76 -0
- package/dist/debug/types.d.ts.map +1 -0
- package/dist/debug/types.js +2 -0
- package/dist/debug/types.js.map +1 -0
- package/dist/debug/web-screencast.d.ts +60 -0
- package/dist/debug/web-screencast.d.ts.map +1 -0
- package/dist/debug/web-screencast.js +146 -0
- package/dist/debug/web-screencast.js.map +1 -0
- package/dist/debug/websocket-server.d.ts +27 -0
- package/dist/debug/websocket-server.d.ts.map +1 -0
- package/dist/debug/websocket-server.js +681 -0
- package/dist/debug/websocket-server.js.map +1 -0
- package/dist/device/detector.d.ts +10 -0
- package/dist/device/detector.d.ts.map +1 -0
- package/dist/device/detector.js +100 -0
- package/dist/device/detector.js.map +1 -0
- package/dist/device/heartbeat.d.ts +26 -0
- package/dist/device/heartbeat.d.ts.map +1 -0
- package/dist/device/heartbeat.js +225 -0
- package/dist/device/heartbeat.js.map +1 -0
- package/dist/device/status-manager.d.ts +15 -0
- package/dist/device/status-manager.d.ts.map +1 -0
- package/dist/device/status-manager.js +58 -0
- package/dist/device/status-manager.js.map +1 -0
- package/dist/device/types.d.ts +30 -0
- package/dist/device/types.d.ts.map +1 -0
- package/dist/device/types.js +2 -0
- package/dist/device/types.js.map +1 -0
- package/dist/executor/action-executor.d.ts +25 -0
- package/dist/executor/action-executor.d.ts.map +1 -0
- package/dist/executor/action-executor.js +261 -0
- package/dist/executor/action-executor.js.map +1 -0
- package/dist/executor/android-executor.d.ts +12 -0
- package/dist/executor/android-executor.d.ts.map +1 -0
- package/dist/executor/android-executor.js +127 -0
- package/dist/executor/android-executor.js.map +1 -0
- package/dist/executor/base.d.ts +20 -0
- package/dist/executor/base.d.ts.map +1 -0
- package/dist/executor/base.js +91 -0
- package/dist/executor/base.js.map +1 -0
- package/dist/executor/cli-executor.d.ts +12 -0
- package/dist/executor/cli-executor.d.ts.map +1 -0
- package/dist/executor/cli-executor.js +94 -0
- package/dist/executor/cli-executor.js.map +1 -0
- package/dist/executor/code-executor.d.ts +13 -0
- package/dist/executor/code-executor.d.ts.map +1 -0
- package/dist/executor/code-executor.js +52 -0
- package/dist/executor/code-executor.js.map +1 -0
- package/dist/executor/code-instrument.d.ts +12 -0
- package/dist/executor/code-instrument.d.ts.map +1 -0
- package/dist/executor/code-instrument.js +116 -0
- package/dist/executor/code-instrument.js.map +1 -0
- package/dist/executor/executor-factory.d.ts +7 -0
- package/dist/executor/executor-factory.d.ts.map +1 -0
- package/dist/executor/executor-factory.js +24 -0
- package/dist/executor/executor-factory.js.map +1 -0
- package/dist/executor/ios-executor.d.ts +10 -0
- package/dist/executor/ios-executor.d.ts.map +1 -0
- package/dist/executor/ios-executor.js +91 -0
- package/dist/executor/ios-executor.js.map +1 -0
- package/dist/executor/types.d.ts +14 -0
- package/dist/executor/types.d.ts.map +1 -0
- package/dist/executor/types.js +2 -0
- package/dist/executor/types.js.map +1 -0
- package/dist/executor/worker-entry.d.ts +2 -0
- package/dist/executor/worker-entry.d.ts.map +1 -0
- package/dist/executor/worker-entry.js +61 -0
- package/dist/executor/worker-entry.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +157 -0
- package/dist/index.js.map +1 -0
- package/dist/node/service.d.ts +21 -0
- package/dist/node/service.d.ts.map +1 -0
- package/dist/node/service.js +178 -0
- package/dist/node/service.js.map +1 -0
- package/dist/node/types.d.ts +45 -0
- package/dist/node/types.d.ts.map +1 -0
- package/dist/node/types.js +3 -0
- package/dist/node/types.js.map +1 -0
- package/dist/storage/database.d.ts +6 -0
- package/dist/storage/database.d.ts.map +1 -0
- package/dist/storage/database.js +154 -0
- package/dist/storage/database.js.map +1 -0
- package/dist/storage/repositories/debug-log-repo.d.ts +29 -0
- package/dist/storage/repositories/debug-log-repo.d.ts.map +1 -0
- package/dist/storage/repositories/debug-log-repo.js +90 -0
- package/dist/storage/repositories/debug-log-repo.js.map +1 -0
- package/dist/storage/repositories/device-repo.d.ts +12 -0
- package/dist/storage/repositories/device-repo.d.ts.map +1 -0
- package/dist/storage/repositories/device-repo.js +87 -0
- package/dist/storage/repositories/device-repo.js.map +1 -0
- package/dist/storage/repositories/execution-log-repo.d.ts +9 -0
- package/dist/storage/repositories/execution-log-repo.d.ts.map +1 -0
- package/dist/storage/repositories/execution-log-repo.js +49 -0
- package/dist/storage/repositories/execution-log-repo.js.map +1 -0
- package/dist/storage/repositories/task-repo.d.ts +13 -0
- package/dist/storage/repositories/task-repo.d.ts.map +1 -0
- package/dist/storage/repositories/task-repo.js +109 -0
- package/dist/storage/repositories/task-repo.js.map +1 -0
- package/dist/task/poller.d.ts +25 -0
- package/dist/task/poller.d.ts.map +1 -0
- package/dist/task/poller.js +153 -0
- package/dist/task/poller.js.map +1 -0
- package/dist/task/queue.d.ts +13 -0
- package/dist/task/queue.d.ts.map +1 -0
- package/dist/task/queue.js +38 -0
- package/dist/task/queue.js.map +1 -0
- package/dist/task/scheduler.d.ts +25 -0
- package/dist/task/scheduler.d.ts.map +1 -0
- package/dist/task/scheduler.js +274 -0
- package/dist/task/scheduler.js.map +1 -0
- package/dist/task/types.d.ts +31 -0
- package/dist/task/types.d.ts.map +1 -0
- package/dist/task/types.js +37 -0
- package/dist/task/types.js.map +1 -0
- package/dist/web/server.d.ts +14 -0
- package/dist/web/server.d.ts.map +1 -0
- package/dist/web/server.js +478 -0
- package/dist/web/server.js.map +1 -0
- package/package.json +49 -0
|
@@ -0,0 +1,681 @@
|
|
|
1
|
+
import { WebSocketServer } from 'ws';
|
|
2
|
+
import { fork } from 'child_process';
|
|
3
|
+
import { URL } from 'url';
|
|
4
|
+
import { createLogger } from '../core/logger.js';
|
|
5
|
+
import { sessionManager } from './session-manager.js';
|
|
6
|
+
import { dumpManager } from './dump-manager.js';
|
|
7
|
+
import { screencastManager } from './screencast.js';
|
|
8
|
+
import { webScreencastManager } from './web-screencast.js';
|
|
9
|
+
import { ExecutorFactory } from '../executor/executor-factory.js';
|
|
10
|
+
const logger = createLogger('DebugWebSocket');
|
|
11
|
+
export class DebugWebSocketServer {
|
|
12
|
+
wss = null;
|
|
13
|
+
config;
|
|
14
|
+
port;
|
|
15
|
+
isStarted = false;
|
|
16
|
+
constructor(port = 8002) {
|
|
17
|
+
this.port = port;
|
|
18
|
+
}
|
|
19
|
+
async start(config) {
|
|
20
|
+
this.config = config;
|
|
21
|
+
if (this.isStarted) {
|
|
22
|
+
logger.info('Debug WebSocket server already running');
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
this.wss = new WebSocketServer({
|
|
26
|
+
host: config.server.host,
|
|
27
|
+
port: this.port,
|
|
28
|
+
perMessageDeflate: false,
|
|
29
|
+
});
|
|
30
|
+
this.wss.on('connection', (ws, req) => {
|
|
31
|
+
let parsedUrl;
|
|
32
|
+
try {
|
|
33
|
+
parsedUrl = new URL(req.url || '/', `ws://${req.headers.host || 'localhost'}`);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
ws.close(1008, 'invalid url');
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const pathname = parsedUrl.pathname;
|
|
40
|
+
if (pathname === '/screencast') {
|
|
41
|
+
const sessionId = parsedUrl.searchParams.get('sessionId');
|
|
42
|
+
if (!sessionId) {
|
|
43
|
+
ws.close(1008, 'missing sessionId');
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
// 先尝试 web screencast,再尝试 android screencast
|
|
47
|
+
const ok = webScreencastManager.addViewer(sessionId, ws) || screencastManager.addViewer(sessionId, ws);
|
|
48
|
+
if (!ok) {
|
|
49
|
+
ws.close(1008, 'screencast session not found');
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
logger.info(`Screencast viewer joined: sessionId=${sessionId}`);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
this.handleControlConnection(ws);
|
|
56
|
+
});
|
|
57
|
+
this.isStarted = true;
|
|
58
|
+
logger.info(`Debug WebSocket server started on ${config.server.host}:${this.port}`);
|
|
59
|
+
logger.info(` Control: ws://${config.server.host}:${this.port}`);
|
|
60
|
+
logger.info(` Screencast: ws://${config.server.host}:${this.port}/screencast?sessionId=<id>`);
|
|
61
|
+
}
|
|
62
|
+
handleControlConnection(ws) {
|
|
63
|
+
logger.info('[Control] New connection');
|
|
64
|
+
ws.on('message', async (data) => {
|
|
65
|
+
try {
|
|
66
|
+
const message = JSON.parse(data.toString());
|
|
67
|
+
await this.handleMessage(ws, message);
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
this.sendError(ws, 'Invalid message format');
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
ws.on('close', () => {
|
|
74
|
+
logger.info('[Control] Connection closed');
|
|
75
|
+
for (const sessionId of sessionManager.findByClient(ws)) {
|
|
76
|
+
this.cleanupSession(sessionId);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
ws.on('error', (error) => {
|
|
80
|
+
logger.error(`[Control] WebSocket error: ${error.message}`);
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
async handleMessage(ws, message) {
|
|
84
|
+
switch (message.type) {
|
|
85
|
+
case 'start_debug':
|
|
86
|
+
await this.handleStartDebug(ws, message);
|
|
87
|
+
break;
|
|
88
|
+
case 'stop_debug':
|
|
89
|
+
await this.handleStopDebug(ws, message);
|
|
90
|
+
break;
|
|
91
|
+
case 'get_logs':
|
|
92
|
+
await this.handleGetLogs(ws, message);
|
|
93
|
+
break;
|
|
94
|
+
case 'execute_action':
|
|
95
|
+
await this.handleExecuteAction(ws, message);
|
|
96
|
+
break;
|
|
97
|
+
case 'execute_ai_act':
|
|
98
|
+
await this.handleExecuteAiAct(ws, message);
|
|
99
|
+
break;
|
|
100
|
+
case 'execute_web_action':
|
|
101
|
+
await this.handleExecuteWebAction(ws, message);
|
|
102
|
+
break;
|
|
103
|
+
case 'start_screencast':
|
|
104
|
+
await this.handleStartScreencast(ws, message);
|
|
105
|
+
break;
|
|
106
|
+
case 'stop_screencast':
|
|
107
|
+
await this.handleStopScreencast(ws, message);
|
|
108
|
+
break;
|
|
109
|
+
case 'close_web_session':
|
|
110
|
+
await this.handleCloseWebSession(ws, message);
|
|
111
|
+
break;
|
|
112
|
+
default:
|
|
113
|
+
this.sendError(ws, `Unknown message type: ${message.type}`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
// ==================== Start Debug (fork worker process) ====================
|
|
117
|
+
async handleStartDebug(ws, request) {
|
|
118
|
+
const sessionId = `debug-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
119
|
+
const session = sessionManager.create(sessionId, ws, request.deviceId);
|
|
120
|
+
this.sendMessage(ws, { type: 'session_created', sessionId }, request.deviceId);
|
|
121
|
+
try {
|
|
122
|
+
sessionManager.updateStatus(sessionId, 'running');
|
|
123
|
+
this.sendMessage(ws, { type: 'debug_started', sessionId }, request.deviceId);
|
|
124
|
+
// Build executor config from debug request
|
|
125
|
+
const execConfig = {
|
|
126
|
+
type: request.platform || 'android',
|
|
127
|
+
testName: 'debugtest',
|
|
128
|
+
url: request.url,
|
|
129
|
+
udid: request.deviceId,
|
|
130
|
+
packageName: request.packageName,
|
|
131
|
+
bundleId: request.bundleId,
|
|
132
|
+
mobileMode: request.mobileMode,
|
|
133
|
+
requirement: request.naturalLanguage,
|
|
134
|
+
executionId: sessionId,
|
|
135
|
+
taskId: sessionId,
|
|
136
|
+
nodeId: this.config.task.nodeId,
|
|
137
|
+
};
|
|
138
|
+
// Fork worker process for long-running debug
|
|
139
|
+
const workerPath = new URL('../executor/worker-entry.js', import.meta.url).pathname;
|
|
140
|
+
const workerEnv = {
|
|
141
|
+
...process.env,
|
|
142
|
+
WORKER_CONFIG: JSON.stringify(execConfig),
|
|
143
|
+
OPENAI_BASE_URL: this.config.ai.baseUrl,
|
|
144
|
+
OPENAI_API_KEY: this.config.ai.apiKey,
|
|
145
|
+
MIDSCENE_MODEL_NAME: this.config.ai.modelName,
|
|
146
|
+
MIDSCENE_USE_QWEN3_VL: this.config.ai.useQwen3Vl ? '1' : '0',
|
|
147
|
+
};
|
|
148
|
+
const childProcess = fork(workerPath, ['--worker'], {
|
|
149
|
+
env: workerEnv,
|
|
150
|
+
stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
|
|
151
|
+
});
|
|
152
|
+
session.process = childProcess;
|
|
153
|
+
childProcess.stdout?.on('data', (data) => {
|
|
154
|
+
const output = data.toString();
|
|
155
|
+
sessionManager.addLog(sessionId, output);
|
|
156
|
+
this.sendMessage(ws, { type: 'log_output', sessionId, content: output }, request.deviceId);
|
|
157
|
+
});
|
|
158
|
+
childProcess.stderr?.on('data', (data) => {
|
|
159
|
+
const errorOutput = data.toString();
|
|
160
|
+
sessionManager.addLog(sessionId, `ERROR: ${errorOutput}`);
|
|
161
|
+
this.sendMessage(ws, { type: 'log_output', sessionId, content: `ERROR: ${errorOutput}`, level: 'error' }, request.deviceId);
|
|
162
|
+
});
|
|
163
|
+
childProcess.on('exit', (code) => {
|
|
164
|
+
const success = code === 0;
|
|
165
|
+
sessionManager.updateStatus(sessionId, success ? 'completed' : 'failed');
|
|
166
|
+
this.sendMessage(ws, { type: 'debug_completed', sessionId, success, exitCode: code }, request.deviceId);
|
|
167
|
+
});
|
|
168
|
+
childProcess.on('message', (message) => {
|
|
169
|
+
this.sendMessage(ws, { type: 'execution_result', sessionId, ...(typeof message === 'object' && message ? message : {}) }, request.deviceId);
|
|
170
|
+
});
|
|
171
|
+
childProcess.on('error', (error) => {
|
|
172
|
+
sessionManager.updateStatus(sessionId, 'failed');
|
|
173
|
+
this.sendMessage(ws, { type: 'debug_error', sessionId, error: error.message }, request.deviceId);
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
sessionManager.updateStatus(sessionId, 'failed');
|
|
178
|
+
this.sendMessage(ws, { type: 'debug_error', sessionId, error: error.message }, request.deviceId);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
// ==================== Execute Action (direct in-process) ====================
|
|
182
|
+
async handleExecuteAction(ws, request) {
|
|
183
|
+
const sessionId = `action-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
184
|
+
const session = sessionManager.create(sessionId, ws, request.deviceId);
|
|
185
|
+
try {
|
|
186
|
+
this.sendMessage(ws, { type: 'action_started', sessionId, actionType: request.actionType }, request.deviceId);
|
|
187
|
+
const executor = ExecutorFactory.create('action', {
|
|
188
|
+
sessionId,
|
|
189
|
+
onLog: (msg) => {
|
|
190
|
+
sessionManager.addLog(sessionId, msg);
|
|
191
|
+
this.sendMessage(ws, { type: 'log_output', sessionId, content: msg }, request.deviceId);
|
|
192
|
+
},
|
|
193
|
+
onDump: (dump) => {
|
|
194
|
+
this.sendMessage(ws, { type: 'action_dump', sessionId, dump }, request.deviceId);
|
|
195
|
+
},
|
|
196
|
+
onExecutionLine: (line) => {
|
|
197
|
+
this.sendMessage(ws, { type: 'execution_line', sessionId, line }, request.deviceId);
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
const config = {
|
|
201
|
+
type: request.platform || 'android',
|
|
202
|
+
testName: `action-${request.actionType}`,
|
|
203
|
+
actionType: request.actionType,
|
|
204
|
+
locatePrompt: request.locatePrompt,
|
|
205
|
+
actionParams: request.actionParams,
|
|
206
|
+
udid: request.deviceId,
|
|
207
|
+
packageName: request.packageName,
|
|
208
|
+
testUrl: request.testUrl,
|
|
209
|
+
modelConfig: request.modelConfig,
|
|
210
|
+
};
|
|
211
|
+
const result = await executor.execute(config);
|
|
212
|
+
sessionManager.updateStatus(sessionId, result.success ? 'completed' : 'failed');
|
|
213
|
+
this.sendMessage(ws, {
|
|
214
|
+
type: 'action_result',
|
|
215
|
+
sessionId,
|
|
216
|
+
success: result.success,
|
|
217
|
+
result: result.result,
|
|
218
|
+
dump: result.dump || '',
|
|
219
|
+
reportHTML: result.reportHTML || '',
|
|
220
|
+
error: result.errorMessage,
|
|
221
|
+
}, request.deviceId);
|
|
222
|
+
}
|
|
223
|
+
catch (error) {
|
|
224
|
+
sessionManager.updateStatus(sessionId, 'failed');
|
|
225
|
+
this.sendMessage(ws, {
|
|
226
|
+
type: 'action_result',
|
|
227
|
+
sessionId,
|
|
228
|
+
success: false,
|
|
229
|
+
error: error.message,
|
|
230
|
+
}, request.deviceId);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
// ==================== Execute AI Act ====================
|
|
234
|
+
async handleExecuteAiAct(ws, request) {
|
|
235
|
+
const sessionId = `aiact-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
236
|
+
const session = sessionManager.create(sessionId, ws, request.deviceId);
|
|
237
|
+
try {
|
|
238
|
+
this.sendMessage(ws, { type: 'ai_act_started', sessionId }, request.deviceId);
|
|
239
|
+
const executor = ExecutorFactory.create('action', {
|
|
240
|
+
sessionId,
|
|
241
|
+
onLog: (msg) => {
|
|
242
|
+
sessionManager.addLog(sessionId, msg);
|
|
243
|
+
this.sendMessage(ws, { type: 'log_output', sessionId, content: msg }, request.deviceId);
|
|
244
|
+
},
|
|
245
|
+
onDump: (dump) => {
|
|
246
|
+
this.sendMessage(ws, { type: 'action_dump', sessionId, dump }, request.deviceId);
|
|
247
|
+
},
|
|
248
|
+
});
|
|
249
|
+
const config = {
|
|
250
|
+
type: request.platform || 'android',
|
|
251
|
+
testName: 'ai-act',
|
|
252
|
+
actionType: 'aiAct',
|
|
253
|
+
naturalLanguage: request.naturalLanguage,
|
|
254
|
+
udid: request.deviceId,
|
|
255
|
+
packageName: request.packageName,
|
|
256
|
+
modelConfig: request.modelConfig,
|
|
257
|
+
actionParams: {
|
|
258
|
+
deepLocate: request.deepLocate,
|
|
259
|
+
deepThink: request.deepThink,
|
|
260
|
+
},
|
|
261
|
+
};
|
|
262
|
+
const result = await executor.execute(config);
|
|
263
|
+
sessionManager.updateStatus(sessionId, result.success ? 'completed' : 'failed');
|
|
264
|
+
this.sendMessage(ws, {
|
|
265
|
+
type: 'ai_act_result',
|
|
266
|
+
sessionId,
|
|
267
|
+
success: result.success,
|
|
268
|
+
result: result.result,
|
|
269
|
+
dump: result.dump || '',
|
|
270
|
+
reportHTML: result.reportHTML || '',
|
|
271
|
+
error: result.errorMessage,
|
|
272
|
+
}, request.deviceId);
|
|
273
|
+
}
|
|
274
|
+
catch (error) {
|
|
275
|
+
sessionManager.updateStatus(sessionId, 'failed');
|
|
276
|
+
this.sendMessage(ws, {
|
|
277
|
+
type: 'ai_act_result',
|
|
278
|
+
sessionId,
|
|
279
|
+
success: false,
|
|
280
|
+
error: error.message,
|
|
281
|
+
}, request.deviceId);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// ==================== Execute Web Action (Playwright + Midscene, with screencast) ====================
|
|
285
|
+
async handleExecuteWebAction(ws, request) {
|
|
286
|
+
const sessionId = request.sessionId || `web-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
287
|
+
let session = sessionManager.get(sessionId);
|
|
288
|
+
const isNewSession = !session;
|
|
289
|
+
if (isNewSession) {
|
|
290
|
+
session = sessionManager.create(sessionId, ws);
|
|
291
|
+
session.platform = 'web';
|
|
292
|
+
}
|
|
293
|
+
try {
|
|
294
|
+
this.sendMessage(ws, { type: 'web_action_started', sessionId, actionType: request.actionType });
|
|
295
|
+
// 懒启动浏览器(每个 session 只启一次)
|
|
296
|
+
if (!session.browser) {
|
|
297
|
+
logger.info(`[WebAction] Launching browser for new session: ${sessionId}`);
|
|
298
|
+
// 动态导入 playwright / @midscene/web
|
|
299
|
+
const playwright = await import('playwright');
|
|
300
|
+
const midscene = await import('@midscene/web/playwright');
|
|
301
|
+
const browser = await playwright.chromium.launch({
|
|
302
|
+
headless: request.headless ?? false,
|
|
303
|
+
});
|
|
304
|
+
const context = await browser.newContext({
|
|
305
|
+
viewport: request.viewport || { width: 1280, height: 800 },
|
|
306
|
+
});
|
|
307
|
+
const page = await context.newPage();
|
|
308
|
+
session.browser = browser;
|
|
309
|
+
session.context = context;
|
|
310
|
+
session.page = page;
|
|
311
|
+
session.webAgent = new midscene.PlaywrightAgent(page, {
|
|
312
|
+
modelConfig: request.modelConfig,
|
|
313
|
+
});
|
|
314
|
+
logger.info(`[WebAction] Browser launched, web session ready: ${sessionId}`);
|
|
315
|
+
// 自动开启投屏
|
|
316
|
+
if (request.screencast) {
|
|
317
|
+
const options = typeof request.screencast === 'object' ? request.screencast : {};
|
|
318
|
+
await webScreencastManager.createSession(sessionId, ws, browser, context, page, options);
|
|
319
|
+
session.screencastEnabled = true;
|
|
320
|
+
const localIp = this.config.server.host;
|
|
321
|
+
this.sendMessage(ws, {
|
|
322
|
+
type: 'screencast_ready',
|
|
323
|
+
sessionId,
|
|
324
|
+
wsUrl: `ws://${localIp}:${this.port}/screencast?sessionId=${encodeURIComponent(sessionId)}`,
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// 取出 agent / page
|
|
329
|
+
const agent = session.webAgent;
|
|
330
|
+
const page = session.page;
|
|
331
|
+
// ===== 周期 dump(与移动端 handleExecuteAction 行为一致) =====
|
|
332
|
+
const sendDump = (dump) => {
|
|
333
|
+
if (dump && dump.trim().length > 0) {
|
|
334
|
+
this.sendMessage(ws, {
|
|
335
|
+
type: 'action_dump',
|
|
336
|
+
sessionId,
|
|
337
|
+
dump,
|
|
338
|
+
platform: 'web',
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
const sendInitialWebDump = async () => {
|
|
343
|
+
try {
|
|
344
|
+
if (agent && typeof agent.dumpDataString === 'function') {
|
|
345
|
+
const initialDump = agent.dumpDataString({ inlineScreenshots: true });
|
|
346
|
+
sendDump(initialDump);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
catch (dumpError) {
|
|
350
|
+
logger.warn(`[WebAction] initial dump failed: ${dumpError.message}`);
|
|
351
|
+
}
|
|
352
|
+
};
|
|
353
|
+
// 如果旧的定时器还在,先清掉
|
|
354
|
+
if (session.dumpIntervalId) {
|
|
355
|
+
clearInterval(session.dumpIntervalId);
|
|
356
|
+
session.dumpIntervalId = undefined;
|
|
357
|
+
}
|
|
358
|
+
const startWebDumpInterval = () => {
|
|
359
|
+
const id = setInterval(() => {
|
|
360
|
+
try {
|
|
361
|
+
if (agent && typeof agent.dumpDataString === 'function') {
|
|
362
|
+
const intervalDump = agent.dumpDataString({ inlineScreenshots: true });
|
|
363
|
+
sendDump(intervalDump);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
catch (dumpError) {
|
|
367
|
+
logger.warn(`[WebAction] interval dump failed: ${dumpError.message}`);
|
|
368
|
+
}
|
|
369
|
+
}, 5000);
|
|
370
|
+
session.dumpIntervalId = id;
|
|
371
|
+
};
|
|
372
|
+
const stopWebDumpInterval = () => {
|
|
373
|
+
if (session.dumpIntervalId) {
|
|
374
|
+
clearInterval(session.dumpIntervalId);
|
|
375
|
+
session.dumpIntervalId = undefined;
|
|
376
|
+
}
|
|
377
|
+
};
|
|
378
|
+
// 立即发一次初始 dump,再启动周期
|
|
379
|
+
await sendInitialWebDump();
|
|
380
|
+
startWebDumpInterval();
|
|
381
|
+
// 按 actionType 分发执行
|
|
382
|
+
let result;
|
|
383
|
+
switch (request.actionType) {
|
|
384
|
+
case 'goto': {
|
|
385
|
+
if (!request.url)
|
|
386
|
+
throw new Error('goto requires url parameter');
|
|
387
|
+
result = await page.goto(request.url, { waitUntil: 'networkidle' });
|
|
388
|
+
break;
|
|
389
|
+
}
|
|
390
|
+
case 'aiAction': {
|
|
391
|
+
const text = request.naturalLanguage || request.locatePrompt || '';
|
|
392
|
+
if (!text)
|
|
393
|
+
throw new Error('aiAction requires naturalLanguage or locatePrompt');
|
|
394
|
+
result = await agent.aiAction(text);
|
|
395
|
+
break;
|
|
396
|
+
}
|
|
397
|
+
case 'aiInput': {
|
|
398
|
+
const value = request.actionParams?.value || '';
|
|
399
|
+
result = await agent.aiInput(value, request.locatePrompt || '');
|
|
400
|
+
break;
|
|
401
|
+
}
|
|
402
|
+
case 'aiTap':
|
|
403
|
+
result = await agent.aiTap(request.locatePrompt || '');
|
|
404
|
+
break;
|
|
405
|
+
case 'aiHover':
|
|
406
|
+
result = await agent.aiHover(request.locatePrompt || '');
|
|
407
|
+
break;
|
|
408
|
+
case 'aiAssert':
|
|
409
|
+
result = await agent.aiAssert(request.locatePrompt || '');
|
|
410
|
+
break;
|
|
411
|
+
case 'aiQuery':
|
|
412
|
+
result = await agent.aiQuery(request.locatePrompt || '');
|
|
413
|
+
break;
|
|
414
|
+
case 'aiWaitFor':
|
|
415
|
+
result = await agent.aiWaitFor(request.locatePrompt || '', request.actionParams);
|
|
416
|
+
break;
|
|
417
|
+
case 'aiScroll':
|
|
418
|
+
result = await agent.aiScroll(request.actionParams || {}, request.locatePrompt);
|
|
419
|
+
break;
|
|
420
|
+
case 'runCode': {
|
|
421
|
+
const code = request.actionParams?.code || '';
|
|
422
|
+
if (!code)
|
|
423
|
+
throw new Error('runCode requires actionParams.code');
|
|
424
|
+
const AsyncFunctionCtor = async function () { }.constructor;
|
|
425
|
+
const ctx = {
|
|
426
|
+
agent,
|
|
427
|
+
page,
|
|
428
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
429
|
+
console: {
|
|
430
|
+
log: (...args) => {
|
|
431
|
+
sessionManager.addLog(sessionId, args.map(String).join(' '));
|
|
432
|
+
this.sendMessage(ws, { type: 'log_output', sessionId, content: args.map(String).join(' ') });
|
|
433
|
+
},
|
|
434
|
+
},
|
|
435
|
+
};
|
|
436
|
+
const fn = new AsyncFunctionCtor(code);
|
|
437
|
+
result = await fn.call(ctx);
|
|
438
|
+
break;
|
|
439
|
+
}
|
|
440
|
+
default:
|
|
441
|
+
throw new Error(`Unknown web action type: ${request.actionType}`);
|
|
442
|
+
}
|
|
443
|
+
// 拉取报告
|
|
444
|
+
let dumpString = '';
|
|
445
|
+
let reportHTML = '';
|
|
446
|
+
try {
|
|
447
|
+
if (typeof agent.dumpDataString === 'function') {
|
|
448
|
+
dumpString = agent.dumpDataString({ inlineScreenshots: true });
|
|
449
|
+
}
|
|
450
|
+
if (typeof agent.reportHTMLString === 'function') {
|
|
451
|
+
reportHTML = agent.reportHTMLString({ inlineScreenshots: true });
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
catch (e) {
|
|
455
|
+
logger.warn(`[WebAction] dump/report extract failed: ${e.message}`);
|
|
456
|
+
}
|
|
457
|
+
sessionManager.updateStatus(sessionId, 'completed');
|
|
458
|
+
stopWebDumpInterval();
|
|
459
|
+
// 发送 web_action_result
|
|
460
|
+
this.sendMessage(ws, {
|
|
461
|
+
type: 'web_action_result',
|
|
462
|
+
sessionId,
|
|
463
|
+
actionType: request.actionType,
|
|
464
|
+
success: true,
|
|
465
|
+
result,
|
|
466
|
+
dump: dumpString || '',
|
|
467
|
+
reportHTML: reportHTML || '',
|
|
468
|
+
});
|
|
469
|
+
// 与移动端对齐:同步发一份 action_result,供 debugLogManager 走报告链路
|
|
470
|
+
this.sendMessage(ws, {
|
|
471
|
+
type: 'action_result',
|
|
472
|
+
sessionId,
|
|
473
|
+
actionType: request.actionType,
|
|
474
|
+
success: true,
|
|
475
|
+
result,
|
|
476
|
+
dump: dumpString || '',
|
|
477
|
+
reportHTML: reportHTML || '',
|
|
478
|
+
platform: 'web',
|
|
479
|
+
});
|
|
480
|
+
}
|
|
481
|
+
catch (error) {
|
|
482
|
+
logger.error(`[WebAction] error: ${error.message}`);
|
|
483
|
+
sessionManager.updateStatus(sessionId, 'failed');
|
|
484
|
+
// 失败时也要停止周期 dump
|
|
485
|
+
if (session.dumpIntervalId) {
|
|
486
|
+
clearInterval(session.dumpIntervalId);
|
|
487
|
+
session.dumpIntervalId = undefined;
|
|
488
|
+
}
|
|
489
|
+
// 失败时也尝试提取 dump / reportHTML
|
|
490
|
+
let dumpString = '';
|
|
491
|
+
let reportHTML = '';
|
|
492
|
+
const agentForReport = session?.webAgent;
|
|
493
|
+
try {
|
|
494
|
+
if (agentForReport && typeof agentForReport.dumpDataString === 'function') {
|
|
495
|
+
dumpString = agentForReport.dumpDataString({ inlineScreenshots: true });
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
catch (dumpErr) {
|
|
499
|
+
logger.warn(`[WebAction] (error case) dump extract failed: ${dumpErr.message}`);
|
|
500
|
+
}
|
|
501
|
+
try {
|
|
502
|
+
if (agentForReport && typeof agentForReport.reportHTMLString === 'function') {
|
|
503
|
+
reportHTML = agentForReport.reportHTMLString({ inlineScreenshots: true });
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
catch (reportErr) {
|
|
507
|
+
logger.warn(`[WebAction] (error case) report extract failed: ${reportErr.message}`);
|
|
508
|
+
}
|
|
509
|
+
this.sendMessage(ws, {
|
|
510
|
+
type: 'web_action_result',
|
|
511
|
+
sessionId,
|
|
512
|
+
actionType: request.actionType,
|
|
513
|
+
success: false,
|
|
514
|
+
error: error.message,
|
|
515
|
+
dump: dumpString || '',
|
|
516
|
+
reportHTML: reportHTML || '',
|
|
517
|
+
agentStatus: session?.webAgent ? 'initialized' : 'not_initialized',
|
|
518
|
+
});
|
|
519
|
+
// 与移动端对齐:同步发一份 action_result
|
|
520
|
+
this.sendMessage(ws, {
|
|
521
|
+
type: 'action_result',
|
|
522
|
+
sessionId,
|
|
523
|
+
actionType: request.actionType,
|
|
524
|
+
success: false,
|
|
525
|
+
error: error.message,
|
|
526
|
+
dump: dumpString || '',
|
|
527
|
+
reportHTML: reportHTML || '',
|
|
528
|
+
agentStatus: session?.webAgent ? 'initialized' : 'not_initialized',
|
|
529
|
+
platform: 'web',
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
// ==================== Start Screencast (for existing web session) ====================
|
|
534
|
+
async handleStartScreencast(ws, request) {
|
|
535
|
+
const session = sessionManager.get(request.sessionId);
|
|
536
|
+
if (!session || !session.browser || !session.context || !session.page) {
|
|
537
|
+
this.sendError(ws, `Web session not found or browser not initialized: ${request.sessionId}`);
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
try {
|
|
541
|
+
await webScreencastManager.createSession(request.sessionId, ws, session.browser, session.context, session.page, request.options);
|
|
542
|
+
session.screencastEnabled = true;
|
|
543
|
+
const localIp = this.config.server.host;
|
|
544
|
+
this.sendMessage(ws, {
|
|
545
|
+
type: 'screencast_ready',
|
|
546
|
+
sessionId: request.sessionId,
|
|
547
|
+
wsUrl: `ws://${localIp}:${this.port}/screencast?sessionId=${encodeURIComponent(request.sessionId)}`,
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
catch (error) {
|
|
551
|
+
this.sendError(ws, `Start screencast failed: ${error.message}`);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
// ==================== Stop Screencast (keep browser open) ====================
|
|
555
|
+
async handleStopScreencast(ws, request) {
|
|
556
|
+
try {
|
|
557
|
+
// 停止周期 dump
|
|
558
|
+
const preSession = sessionManager.get(request.sessionId);
|
|
559
|
+
if (preSession?.dumpIntervalId) {
|
|
560
|
+
clearInterval(preSession.dumpIntervalId);
|
|
561
|
+
preSession.dumpIntervalId = undefined;
|
|
562
|
+
}
|
|
563
|
+
// 仅停止 CDP 投屏帧推送,保留浏览器以便继续动作
|
|
564
|
+
const screencast = webScreencastManager.getSession(request.sessionId);
|
|
565
|
+
if (screencast) {
|
|
566
|
+
await webScreencastManager.stopSession(request.sessionId);
|
|
567
|
+
}
|
|
568
|
+
if (preSession) {
|
|
569
|
+
preSession.screencastEnabled = false;
|
|
570
|
+
}
|
|
571
|
+
this.sendMessage(ws, { type: 'screencast_stopped', sessionId: request.sessionId });
|
|
572
|
+
}
|
|
573
|
+
catch (error) {
|
|
574
|
+
this.sendError(ws, `Stop screencast failed: ${error.message}`);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
// ==================== Close Web Session (close browser + screencast) ====================
|
|
578
|
+
async handleCloseWebSession(ws, request) {
|
|
579
|
+
const session = sessionManager.get(request.sessionId);
|
|
580
|
+
if (!session) {
|
|
581
|
+
this.sendError(ws, `Session not found: ${request.sessionId}`);
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
584
|
+
// 停止周期 dump
|
|
585
|
+
if (session.dumpIntervalId) {
|
|
586
|
+
clearInterval(session.dumpIntervalId);
|
|
587
|
+
session.dumpIntervalId = undefined;
|
|
588
|
+
}
|
|
589
|
+
// 停止 web screencast
|
|
590
|
+
if (webScreencastManager.hasSession(request.sessionId)) {
|
|
591
|
+
await webScreencastManager.stopSession(request.sessionId);
|
|
592
|
+
}
|
|
593
|
+
// 关闭浏览器
|
|
594
|
+
try {
|
|
595
|
+
await session.browser?.close?.();
|
|
596
|
+
}
|
|
597
|
+
catch {
|
|
598
|
+
// ignore
|
|
599
|
+
}
|
|
600
|
+
session.browser = undefined;
|
|
601
|
+
session.context = undefined;
|
|
602
|
+
session.page = undefined;
|
|
603
|
+
session.webAgent = undefined;
|
|
604
|
+
session.screencastEnabled = false;
|
|
605
|
+
sessionManager.updateStatus(request.sessionId, 'stopped');
|
|
606
|
+
this.sendMessage(ws, { type: 'web_session_closed', sessionId: request.sessionId });
|
|
607
|
+
}
|
|
608
|
+
// ==================== Stop Debug ====================
|
|
609
|
+
async handleStopDebug(ws, request) {
|
|
610
|
+
const session = sessionManager.get(request.sessionId);
|
|
611
|
+
if (!session) {
|
|
612
|
+
this.sendError(ws, `Session not found: ${request.sessionId}`);
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
// Stop child process
|
|
616
|
+
if (session.process) {
|
|
617
|
+
session.process.kill('SIGTERM');
|
|
618
|
+
}
|
|
619
|
+
// Stop dump
|
|
620
|
+
dumpManager.stopPeriodicDump(request.sessionId);
|
|
621
|
+
// Destroy agent if exists
|
|
622
|
+
if (session.agent && typeof session.agent.destroy === 'function') {
|
|
623
|
+
try {
|
|
624
|
+
await session.agent.destroy();
|
|
625
|
+
}
|
|
626
|
+
catch (error) {
|
|
627
|
+
logger.warn(`Agent destroy error: ${error.message}`);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
sessionManager.updateStatus(request.sessionId, 'stopped');
|
|
631
|
+
this.sendMessage(ws, { type: 'debug_stopped', sessionId: request.sessionId });
|
|
632
|
+
}
|
|
633
|
+
// ==================== Get Logs ====================
|
|
634
|
+
async handleGetLogs(ws, request) {
|
|
635
|
+
const session = sessionManager.get(request.sessionId);
|
|
636
|
+
if (!session) {
|
|
637
|
+
this.sendError(ws, `Session not found: ${request.sessionId}`);
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
this.sendMessage(ws, { type: 'all_logs', sessionId: request.sessionId, logs: session.logs });
|
|
641
|
+
}
|
|
642
|
+
// ==================== Utilities ====================
|
|
643
|
+
sendMessage(ws, message, deviceId) {
|
|
644
|
+
try {
|
|
645
|
+
const msg = {
|
|
646
|
+
...message,
|
|
647
|
+
deviceId,
|
|
648
|
+
timestamp: new Date().toISOString(),
|
|
649
|
+
};
|
|
650
|
+
if (ws.readyState === ws.OPEN) {
|
|
651
|
+
ws.send(JSON.stringify(msg));
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
catch (error) {
|
|
655
|
+
logger.error(`Send message error: ${error.message}`);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
sendError(ws, error) {
|
|
659
|
+
this.sendMessage(ws, { type: 'error', error });
|
|
660
|
+
}
|
|
661
|
+
cleanupSession(sessionId) {
|
|
662
|
+
dumpManager.stopPeriodicDump(sessionId);
|
|
663
|
+
sessionManager.remove(sessionId);
|
|
664
|
+
}
|
|
665
|
+
// ==================== Public API ====================
|
|
666
|
+
getSessions() {
|
|
667
|
+
return sessionManager.getAll();
|
|
668
|
+
}
|
|
669
|
+
async close() {
|
|
670
|
+
await screencastManager.closeAll();
|
|
671
|
+
await webScreencastManager.closeAll();
|
|
672
|
+
dumpManager.stopAll();
|
|
673
|
+
if (this.wss) {
|
|
674
|
+
this.wss.close();
|
|
675
|
+
this.wss = null;
|
|
676
|
+
this.isStarted = false;
|
|
677
|
+
}
|
|
678
|
+
logger.info('Debug WebSocket server closed');
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
//# sourceMappingURL=websocket-server.js.map
|