@aiscene/aiserver 1.7.1 → 1.7.3

package/dist/debug/websocket-server.js

@@ -2,6 +2,7 @@ import { WebSocketServer } from 'ws';
 import { fork } from 'child_process';
 import { URL } from 'url';
 import os from 'os';
+import { randomUUID } from 'crypto';
 import { uploadReportToServer, uploadCapturePackageFile } from '../api/callback.js';
 import fs from 'fs';
 import path from 'path';
@@ -9,6 +10,7 @@ import { createLogger } from '../core/logger.js';
 import { eventBus } from '../core/event-bus.js';
 import { sessionManager } from './session-manager.js';
 import { dumpManager } from './dump-manager.js';
+import { createDebugReportGroupName } from './report-group-name.js';
 import { screencastManager } from './screencast.js';
 import { webScreencastManager } from './web-screencast.js';
 import { webBrowserPool } from './web-browser-pool.js';
@@ -16,6 +18,8 @@ import { ExecutorFactory } from '../executor/executor-factory.js';
 import { instrumentCode } from '../executor/code-instrument.js';
 import { playwrightSwipe } from '../core/native-swipe.js';
 import { whistleManager } from '../proxy/whistle-manager.js';
+import { executeAndroidLogin, executeWebLogin, } from '../login/android-login-context.js';
+import { webRecordingManager } from '../recorder/web-recording-manager.js';
 const logger = createLogger('DebugWebSocket');
 /** 获取本机可用的非内部 IPv4 地址，供客户端连接投屏 WebSocket */
 function getLocalIP() {
@@ -248,6 +252,10 @@ export class DebugWebSocketServer {
         });
         ws.on('close', () => {
             logger.info('[Control] Connection closed');
+            const recordingClientId = ws.recordingClientId;
+            if (recordingClientId) {
+                void webRecordingManager.cancel(recordingClientId);
+            }
             for (const sessionId of sessionManager.findByClient(ws)) {
                 const s = sessionManager.get(sessionId);
                 // Web 调试会话：客户端断开不销毁浏览器/投屏，仅解绑 client，由 TTL 在 30 分钟无活动时自动回收
@@ -302,10 +310,86 @@ export class DebugWebSocketServer {
             case 'unsubscribe_task_logs':
                 this.handleUnsubscribeTaskLogs(ws, message);
                 break;
+            case 'web_recording_start':
+            case 'web_recording_snapshot':
+            case 'web_recording_select':
+            case 'web_recording_input':
+            case 'web_recording_stop':
+            case 'web_recording_cancel':
+            case 'web_recording_replay':
+                await this.handleWebRecording(ws, message);
+                break;
             default:
                 this.sendError(ws, `Unknown message type: ${message.type}`);
         }
     }
+    async handleWebRecording(ws, request) {
+        const clientId = (ws.recordingClientId ||= randomUUID());
+        try {
+            if (request.type === 'web_recording_start') {
+                if (!request.targetUrl)
+                    throw new Error('targetUrl is required');
+                const script = await webRecordingManager.start(clientId, {
+                    name: request.name,
+                    targetUrl: request.targetUrl,
+                });
+                this.sendMessage(ws, { type: 'recording-started', script });
+                this.sendMessage(ws, {
+                    type: 'recording-snapshot',
+                    ...(await webRecordingManager.snapshot(clientId)),
+                });
+                return;
+            }
+            if (request.type === 'web_recording_snapshot') {
+                this.sendMessage(ws, {
+                    type: 'recording-snapshot',
+                    ...(await webRecordingManager.snapshot(clientId)),
+                });
+                return;
+            }
+            if (request.type === 'web_recording_select') {
+                const result = await webRecordingManager.select(clientId, Number(request.x), Number(request.y));
+                if (result.step) {
+                    this.sendMessage(ws, { type: 'recording-step-added', step: result.step });
+                }
+                this.sendMessage(ws, {
+                    type: 'recording-action-done',
+                    action: 'tap',
+                    passwordField: result.passwordField,
+                });
+                return;
+            }
+            if (request.type === 'web_recording_input') {
+                const step = await webRecordingManager.input(clientId, String(request.text || ''), String(request.recordingValue || request.text || ''), request.secret === true);
+                this.sendMessage(ws, { type: 'recording-step-added', step });
+                this.sendMessage(ws, { type: 'recording-input-done', secret: request.secret === true });
+                return;
+            }
+            if (request.type === 'web_recording_stop') {
+                const script = await webRecordingManager.stop(clientId);
+                this.sendMessage(ws, { type: 'recording-stopped', script });
+                return;
+            }
+            if (request.type === 'web_recording_cancel') {
+                await webRecordingManager.cancel(clientId);
+                this.sendMessage(ws, { type: 'recording-cancelled' });
+                return;
+            }
+            if (request.type === 'web_recording_replay') {
+                await webRecordingManager.replay({
+                    script: request.script,
+                    variables: request.variables,
+                });
+                this.sendMessage(ws, { type: 'recording-replay-completed' });
+            }
+        }
+        catch (error) {
+            this.sendMessage(ws, {
+                type: 'recording-error',
+                message: error.message,
+            });
+        }
+    }
     // ==================== Task Log Subscription ====================
     handleSubscribeTaskLogs(ws, request) {
         this.subscribeTaskLog(ws, request.taskId);
@@ -344,6 +428,12 @@ export class DebugWebSocketServer {
         try {
             sessionManager.updateStatus(sessionId, 'running');
             this.sendMessage(ws, { type: 'debug_started', sessionId }, request.deviceId);
+            await executeAndroidLogin({
+                source: request,
+                deviceId: request.deviceId,
+                platform: request.platform,
+                logPrefix: `[Debug ${sessionId}]`,
+            });
             // 判断 naturalLanguage 是代码还是自然语言
             const isCode = this.isCodeScript(request.naturalLanguage || '');
             logger.info(`[Debug] naturalLanguage mode: ${isCode ? 'runCode' : 'aiAction'}, preview: ${(request.naturalLanguage || '').substring(0, 50)}...`);
@@ -377,6 +467,7 @@ export class DebugWebSocketServer {
     async handleStartDebugWeb(ws, request) {
         // 1) 决定 sessionId（客户端传了就用以复用）
         const sessionId = request.sessionId || `debug-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+        const reportGroupName = createDebugReportGroupName(request);
         // 2) 判断是否复用：session 已存在且浏览器/agent 完整
         let session = sessionManager.get(sessionId);
         const isReuse = !!(session && session.browser && session.webAgent && session.page);
@@ -503,6 +594,7 @@ export class DebugWebSocketServer {
                 // modelConfig 走构造参数（与 handleExecuteWebAction 一致，避免多会话相互覆盖 env）
                 session.webAgent = new midscene.PlaywrightAgent(page, {
                     outputFormat: 'html-and-external-assets',
+                    groupName: reportGroupName,
                     modelConfig: this.applyModelConfig(request.modelConfig),
                 });
                 // 注册并发池 + 启动 TTL 计时
@@ -520,7 +612,12 @@ export class DebugWebSocketServer {
                         wsUrl: `ws://${localIp}:${this.port}/screencast?sessionId=${encodeURIComponent(sessionId)}`,
                     });
                 }
-                // 登录（仅首次启动时执行）
+                await executeWebLogin({
+                    source: request,
+                    page,
+                    logPrefix: `[WebDebug ${sessionId}]`,
+                });
+                // 兼容旧版表单登录配置（仅首次启动时执行）
                 if (request.loginConfig && request.loginConfig.loginUrl
                     && request.loginConfig.loginUsername && request.loginConfig.loginPassword) {
                     const loginConfig = { ...request.loginConfig, mobileMode: request.mobileMode || false };
@@ -529,7 +626,13 @@ export class DebugWebSocketServer {
             }
             else {
                 logger.info(`[WebDebug] Reusing existing session: ${sessionId}`);
-                // 复用时若投屏被前端关掉过、需要重新开启，可在此扩展（暂保持原状）
+                // 浏览器和页面继续复用，但每次请求使用新的 Agent，避免报告内容和名称串到上一次调试。
+                const midscene = await import('@aiscene/web/playwright');
+                session.webAgent = new midscene.PlaywrightAgent(session.page, {
+                    outputFormat: 'html-and-external-assets',
+                    groupName: reportGroupName,
+                    modelConfig: this.applyModelConfig(request.modelConfig),
+                });
             }
             // 4) 复用/首启都要：刷新 TTL
             this.touchWebSession(sessionId);
@@ -658,7 +761,7 @@ export class DebugWebSocketServer {
             session.executor = executor;
         const config = {
             type: (request.platform === 'ios' ? 'ios' : 'android'),
-            testName: 'debug-runCode',
+            testName: createDebugReportGroupName(request),
             actionType: 'runCode',
             actionParams: { code: request.naturalLanguage },
             udid: request.deviceId,
@@ -667,10 +770,11 @@ export class DebugWebSocketServer {
             url: request.url,
             modelConfig: fullModelConfig,
             executionId: sessionId,
+            skipAppRestart: request.skipAppRestart,
             mobileMode: request.mobileMode,
             deviceName: request.deviceName,
             // 环境代理配置
-            proxyPort: request.proxyPort || 8899,
+            proxyPort: request.proxyPort,
             // 公司代理平台账号
             proxyAccount: request.proxyAccount,
             // 添加登录配置
@@ -783,7 +887,7 @@ export class DebugWebSocketServer {
         const fullModelConfig = this.applyModelConfig(request.modelConfig);
         const execConfig = {
             type: (request.platform === 'ios' ? 'ios' : 'android'),
-            testName: 'debugtest',
+            testName: createDebugReportGroupName(request),
             url: request.url,
             udid: request.deviceId,
             packageName: request.packageName,
@@ -795,8 +899,9 @@ export class DebugWebSocketServer {
             taskId: sessionId,
             nodeId: this.config.task.nodeId,
             modelConfig: fullModelConfig,
+            skipAppRestart: request.skipAppRestart,
             // 环境代理配置
-            proxyPort: request.proxyPort || 8899,
+            proxyPort: request.proxyPort,
             // 公司代理平台账号
             proxyAccount: request.proxyAccount,
             // 添加登录配置
@@ -959,6 +1064,12 @@ export class DebugWebSocketServer {
         try {
             sessionManager.updateStatus(sessionId, 'running');
             this.sendMessage(ws, { type: 'action_started', sessionId, actionType: request.actionType }, request.deviceId);
+            await executeAndroidLogin({
+                source: request,
+                deviceId: request.deviceId,
+                platform: request.platform,
+                logPrefix: `[Action ${sessionId}]`,
+            });
             const executor = ExecutorFactory.create('action', {
                 sessionId,
                 onLog: (msg) => {
@@ -976,7 +1087,7 @@ export class DebugWebSocketServer {
             session.executor = executor;
             const config = {
                 type: request.platform || 'android',
-                testName: `action-${request.actionType}`,
+                testName: createDebugReportGroupName(request),
                 actionType: request.actionType,
                 locatePrompt: request.locatePrompt,
                 actionParams: request.actionParams,
@@ -1223,6 +1334,12 @@ export class DebugWebSocketServer {
         try {
             sessionManager.updateStatus(sessionId, 'running');
             this.sendMessage(ws, { type: 'ai_act_started', sessionId }, request.deviceId);
+            await executeAndroidLogin({
+                source: request,
+                deviceId: request.deviceId,
+                platform: request.platform,
+                logPrefix: `[AiAct ${sessionId}]`,
+            });
             const executor = ExecutorFactory.create('action', {
                 sessionId,
                 onLog: (msg) => {
@@ -1237,7 +1354,7 @@ export class DebugWebSocketServer {
             session.executor = executor;
             const config = {
                 type: request.platform || 'android',
-                testName: 'ai-act',
+                testName: createDebugReportGroupName(request),
                 actionType: 'aiAct',
                 naturalLanguage: request.naturalLanguage,
                 udid: request.deviceId,
@@ -1434,6 +1551,7 @@ export class DebugWebSocketServer {
     }
     async handleExecuteWebAction(ws, request) {
         const sessionId = request.sessionId || `web-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+        const reportGroupName = createDebugReportGroupName(request);
         let session = sessionManager.get(sessionId);
         const isNewSession = !session;
         if (isNewSession) {
@@ -1546,6 +1664,7 @@ export class DebugWebSocketServer {
                 // 因为 fork 子进程必须通过 env 透传配置，不影响这里。
                 session.webAgent = new midscene.PlaywrightAgent(page, {
                     outputFormat: 'html-and-external-assets',
+                    groupName: reportGroupName,
                     modelConfig: this.applyModelConfig(request.modelConfig),
                 });
                 // 注册到 web session 并发池，开始计入并发数 + TTL
@@ -1564,12 +1683,26 @@ export class DebugWebSocketServer {
                     });
                 }
             }
+            else {
+                // 允许复用已有浏览器页面，但本次操作必须生成独立报告。
+                const midscene = await import('@aiscene/web/playwright');
+                session.webAgent = new midscene.PlaywrightAgent(session.page, {
+                    outputFormat: 'html-and-external-assets',
+                    groupName: reportGroupName,
+                    modelConfig: this.applyModelConfig(request.modelConfig),
+                });
+            }
             // 取出 agent / page
             const agent = session.webAgent;
             const page = session.page;
             // 刷新 web session 活跃时间，避免 TTL 误杀
             this.touchWebSession(sessionId);
-            // 处理登录
+            await executeWebLogin({
+                source: request,
+                page,
+                logPrefix: `[WebAction ${sessionId}]`,
+            });
+            // 兼容旧版表单登录配置
             if (request.loginConfig && request.loginConfig.loginUrl && request.loginConfig.loginUsername && request.loginConfig.loginPassword) {
                 const loginConfig = {
                     ...request.loginConfig,