android-midscene-automation 0.1.38 → 0.1.40

package/server/appium-recorder/report.ts

@@ -1,5 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
-import { join } from 'node:path';
+import { join, relative } from 'node:path';
+import type { ReplayVideo } from './replay-video';
 import { PNG } from 'pngjs';
 import { appDataPath } from '../paths';
 import { loadConfig } from '../config';
@@ -240,6 +241,7 @@ function stepSection(
 }
 
 function createReplayHtml(input: {
+  video?: { url: string; startedAt: string; segments?: Array<{ url: string; startedAt: string; boundaryAt?: string; scriptName?: string }> };
   script: AppiumRecordedScriptRecord;
   deviceId: string;
   resultText: string;
@@ -253,6 +255,7 @@ function createReplayHtml(input: {
 }) {
   const imageCache = new Map<string, string>();
   const payload = jsonForHtml({
+    video: input.video,
     startedAt: input.startedAt.toISOString(),
     durationMs: input.durationMs,
     frames: input.frames.map((frame) => ({
@@ -325,9 +328,11 @@ function createReplayHtml(input: {
     .viewer { display: grid; grid-template-rows: minmax(0, 1fr) auto auto; min-width: 0; min-height: 0; padding: 16px 20px 12px; background: #f4f6f8; }
     .screen { display: flex; align-items: center; justify-content: center; min-height: 0; overflow: hidden; border-radius: 6px; background: #101827; }
     .screen img { display: block; width: 100%; height: 100%; object-fit: contain; transition: opacity .12s ease; }
+    [hidden] { display: none !important; }
     .empty { color: #a8abb2; }
     .caption { min-height: 40px; padding: 10px 4px 0; overflow: hidden; color: #4b515a; font-size: 13px; text-align: center; text-overflow: ellipsis; white-space: nowrap; }
     .controls { display: grid; grid-template-columns: auto auto auto minmax(120px, 1fr) auto; align-items: center; gap: 8px; min-height: 52px; }
+    .controls:has(#toggle-recording:not([hidden])) { grid-template-columns: auto auto auto auto minmax(80px, 1fr) auto; }
     button { min-height: 34px; padding: 0 14px; border: 1px solid #cfd5dd; border-radius: 4px; color: #30343b; background: #fff; cursor: pointer; }
     button:hover:not(:disabled) { border-color: #409eff; color: #1677ff; }
     button:disabled { cursor: not-allowed; opacity: .45; }
@@ -364,6 +369,9 @@ function createReplayHtml(input: {
       .thumbnail { height: 78px; }
       header { align-items: flex-start; flex-direction: column; }
       .meta { justify-content: flex-start; }
+      .controls, .controls:has(#toggle-recording:not([hidden])) { display: flex; flex-wrap: wrap; }
+      .controls button { max-width: 100%; }
+      #seek { flex: 1 1 120px; width: auto; }
     }
   </style>
 </head>
@@ -406,9 +414,11 @@ function createReplayHtml(input: {
         </div>
       </div>
       <section class="viewer">
-        <div class="screen"><img id="screen" alt="设备截图" hidden /><span id="empty" class="empty">本次回放没有可用截图</span></div>
+        <div class="screen"><video id="recording" controls preload="metadata" playsinline hidden style="width:100%;height:100%;object-fit:contain"></video><img id="screen" alt="设备截图" hidden /><span id="empty" class="empty">本次回放没有可用截图</span></div>
         <div id="caption" class="caption">暂无截图</div>
         <div class="controls">
+          <button id="toggle-recording" hidden>查看节点截图</button>
+          <select id="video-segment" aria-label="视频片段" hidden style="max-width:100%;min-width:0"></select>
           <button id="previous" type="button" title="上一帧">上一帧</button>
           <button id="play" type="button">播放</button>
           <button id="next" type="button" title="下一帧">下一帧</button>
@@ -430,6 +440,21 @@ function createReplayHtml(input: {
     const data = JSON.parse(document.querySelector('#replay-data').textContent || '{}');
     const visualChecks = Array.isArray(data.visualChecks) ? data.visualChecks : [];
     const reportStartedAt = Date.parse(data.startedAt) || 0;
+    const recording = document.querySelector('#recording');
+    const toggleRecording = document.querySelector('#toggle-recording');
+    const videos = data.video ? (data.video.segments || [data.video]) : [];
+    const segmentSelect = document.querySelector('#video-segment');
+    let activeVideo = 0;
+    let videoOffset = Math.max(0, (Date.parse(videos[0]?.startedAt) || reportStartedAt) - reportStartedAt);
+    videos.forEach((video, index) => {
+      const option = document.createElement('option');
+      option.value = String(index); option.textContent = (index + 1) + '. ' + (video.scriptName || data.scriptName || '回放视频');
+      segmentSelect.appendChild(option);
+    });
+    segmentSelect.hidden = videos.length < 2;
+    let videoMode = Boolean(data.video);
+    let pendingVideoTime = 0;
+    if (data.video) { recording.src = data.video.url; toggleRecording.hidden = false; }
     const frames = (Array.isArray(data.frames) ? data.frames : []).map((frame) => ({
       ...frame,
       offsetMs: Math.max(0, (Date.parse(frame.capturedAt) || reportStartedAt) - reportStartedAt),
@@ -487,8 +512,9 @@ function createReplayHtml(input: {
       const frameChanged = nextFrame !== currentFrame;
       currentFrame = nextFrame;
       const frame = frames[currentFrame];
-      elements.empty.hidden = Boolean(frame);
-      elements.screen.hidden = !frame;
+      elements.empty.hidden = videoMode || Boolean(frame?.imageUrl);
+      elements.screen.hidden = videoMode || !frame?.imageUrl;
+      recording.hidden = !videoMode;
       if (frame) {
         if (frameChanged || elements.screen.src !== frame.imageUrl) elements.screen.src = frame.imageUrl;
         elements.title.textContent = frame.nodeLabel || '未命名节点';
@@ -507,7 +533,7 @@ function createReplayHtml(input: {
       }
       elements.previous.disabled = !frames.length || currentFrame <= 0;
       elements.next.disabled = !frames.length || currentFrame >= frames.length - 1;
-      elements.play.disabled = frames.length < 2;
+      elements.play.disabled = !videoMode && frames.length < 2;
       elements.seek.value = String(Math.round(currentMs / durationMs * 1000));
       elements.clock.textContent = formatTime(currentMs) + ' / ' + formatTime(durationMs);
       elements.playhead.style.left = (currentMs / durationMs * 100) + '%';
@@ -516,14 +542,61 @@ function createReplayHtml(input: {
       if (scrollStep) document.querySelector('.frame-item.active')?.scrollIntoView({ block: 'nearest', behavior: 'smooth' });
     };
     const pause = () => {
+      recording.pause();
       cancelAnimationFrame(animationFrame);
       animationFrame = 0;
       elements.play.textContent = '播放';
     };
     const seekTo = (milliseconds, scrollStep = true) => {
       currentMs = Math.max(0, Math.min(durationMs, milliseconds));
+      let index = 0;
+      for (let i = 1; i < videos.length; i++) {
+        if (Date.parse(videos[i].boundaryAt || videos[i].startedAt) - reportStartedAt <= currentMs) index = i;
+      }
+      if (index !== activeVideo) {
+        activeVideo = index; segmentSelect.value = String(index);
+        videoOffset = Math.max(0, Date.parse(videos[index].startedAt) - reportStartedAt);
+        recording.src = videos[index].url;
+      }
+      pendingVideoTime = Math.max(0, (currentMs - videoOffset) / 1000);
+      if (data.video && recording.readyState >= 1) recording.currentTime = Math.min(pendingVideoTime, recording.duration || pendingVideoTime);
       render(scrollStep);
     };
+    recording.addEventListener('loadedmetadata', () => { recording.currentTime = Math.min(pendingVideoTime, recording.duration); });
+    segmentSelect.addEventListener('change', () => {
+      pause();
+      const video = videos[Number(segmentSelect.value)];
+      seekTo(Date.parse(video.boundaryAt || video.startedAt) - reportStartedAt);
+    });
+    recording.addEventListener('ended', () => {
+      if (videoMode && activeVideo + 1 < videos.length) {
+        const next = videos[activeVideo + 1];
+        seekTo(Date.parse(next.boundaryAt || next.startedAt) - reportStartedAt);
+        recording.play().catch(() => undefined);
+      }
+    });
+    recording.addEventListener('timeupdate', () => {
+      if (!videoMode || recording.paused) return;
+      currentMs = recording.currentTime * 1000 + videoOffset;
+      render();
+    });
+    recording.addEventListener('seeked', () => {
+      if (!videoMode) return;
+      currentMs = recording.currentTime * 1000 + videoOffset;
+      render();
+    });
+    recording.addEventListener('play', () => { elements.play.textContent = '暂停'; });
+    recording.addEventListener('pause', () => { elements.play.textContent = '播放'; });
+    recording.addEventListener('error', () => {
+      pause(); videoMode = false; render();
+      toggleRecording.textContent = '视频不可用，请将 MP4 与报告放在一起';
+      toggleRecording.disabled = true;
+    });
+    toggleRecording.addEventListener('click', () => {
+      pause(); videoMode = !videoMode;
+      toggleRecording.textContent = videoMode ? '查看节点截图' : '查看回放视频';
+      seekTo(currentMs, false);
+    });
     const tickPlayback = (now) => {
       currentMs = Math.min(durationMs, now - playbackStartedAt);
       render();
@@ -605,7 +678,12 @@ function createReplayHtml(input: {
       article.append(title);
       const detail = document.createElement('dl');
       const region = check.region || {};
-      const entries = [['结果', check.timedOut ? '观察超时' : check.result === null ? '无法判定' : String(check.result)], ['配置', check.settings],
+      const match = check.templateMatch;
+      const entries = [
+        ...(match ? [['实际匹配结果', match.matched ? '匹配到模板' : '未匹配到模板'],
+          ['预期匹配结果', match.expected === 'present' ? '匹配到模板' : '未匹配到模板'],
+          ['匹配得分 / 严格度', (match.score * 100).toFixed(4) + '% / ' + (match.threshold * 100).toFixed(4) + '%']] : []),
+        [match ? '条件是否成立' : '结果', check.timedOut ? '观察超时（按节点超时配置处理）' : check.result === null ? '无法判定' : match ? (check.result ? '成立 → true 分支' : '不成立 → false 分支') : String(check.result)], ['配置', check.settings],
         ['区域', [region.x, region.y, region.width, region.height].join(', ')],
         ['采样', check.sampleCount + ' 帧'], ['耗时', check.durationMs + 'ms'], ['说明', check.message], ...Object.entries(check.metrics)];
       entries.forEach(([key, value]) => {
@@ -685,6 +763,11 @@ function createReplayHtml(input: {
     elements.previous.addEventListener('click', () => { pause(); seekTo(frames[Math.max(0, currentFrame - 1)]?.offsetMs || 0); });
     elements.next.addEventListener('click', () => { pause(); seekTo(frames[Math.min(frames.length - 1, currentFrame + 1)]?.offsetMs || durationMs); });
     elements.play.addEventListener('click', () => {
+      if (videoMode) {
+        if (!recording.paused) return pause();
+        recording.play().catch(() => { elements.caption.textContent = '视频无法播放，请检查 MP4 文件是否存在或使用其他浏览器'; });
+        return;
+      }
       if (animationFrame) return pause();
       if (currentMs >= durationMs) currentMs = 0;
       playbackStartedAt = performance.now() - currentMs;
@@ -719,6 +802,7 @@ function createReplayHtml(input: {
 }
 
 export async function createAppiumReplayReport(input: {
+  video?: ReplayVideo;
   script: AppiumRecordedScriptRecord;
   deviceId: string;
   success: boolean;
@@ -753,6 +837,14 @@ export async function createAppiumReplayReport(input: {
   const visualCheckByStepId = new Map(visualChecks.map((check) => [check.nodeId, check]));
   const visualFailureCount = visualChecks.filter((check) => check.status === 'failed').length;
   const html = createReplayHtml({
+    video: input.video ? {
+      startedAt: input.video.startedAt,
+      url: relative(outputDir, input.video.filePath).split(/[/\\\\]/).map(encodeURIComponent).join('/'),
+      segments: (input.video.segments || [input.video]).map(segment => ({
+        startedAt: segment.startedAt, boundaryAt: segment.boundaryAt, scriptName: segment.scriptName,
+        url: relative(outputDir, segment.filePath).split(/[/\\\\]/).map(encodeURIComponent).join('/'),
+      })),
+    } : undefined,
     script: input.script,
     deviceId: input.deviceId,
     resultText,

package/server/appium-recorder/routes.ts

@@ -1,8 +1,8 @@
 import { execFile } from 'node:child_process';
+import { sendReplayVideo } from './video-response';
 import { getPresetVariables, savePresetVariables } from './variable-store';
 import type { TestVariable } from '../../src/appium-recorder/variables';
 import { deleteRunHistory, getRunHistory, listRunHistory, saveRunHistory } from './run-history';
-import { recognizeDeviceScreen } from './ai-recognition';
 import { readImage, cropImage } from './image-check';
 import { PNG } from 'pngjs';
 import { adbScreenshotBase64 } from './screenshot';
@@ -200,20 +200,7 @@ export async function handleAppiumRecorderRequest(
     }
 
     if (pathname === '/api/appium-recorder/ai-recognition/test' && req.method === 'POST') {
-      const parsed = await readBody<{ deviceId?: string; prompt?: unknown; timeoutMs?: number }>(req);
-      const deviceId = parsed.deviceId?.trim() || selectedDeviceId;
-      if (!deviceId) throw new Error('请选择设备');
-      assertDeviceAllowed(deviceId);
-      if (isRemoteDeviceId(deviceId)) throw new Error('远程设备暂不支持 AI 识别测试');
-      if (replayingDevices.has(deviceId)) { sendJson(res, { message: '设备正在回放，请稍后测试' }, 409); return true; }
-      const controller = new AbortController();
-      const abort = () => controller.abort();
-      res.once('close', abort);
-      try {
-        sendJson(res, await recognizeDeviceScreen({ deviceId, prompt: parsed.prompt, timeoutMs: parsed.timeoutMs, signal: controller.signal }));
-      } finally {
-        res.off('close', abort);
-      }
+      sendJson(res, { message: 'AI 识别操作已移除，请使用图像判断或原生组件判断' }, 410);
       return true;
     }
 
@@ -345,6 +332,17 @@ export async function handleAppiumRecorderRequest(
       return true;
     }
 
+    const videoMatch = pathname.match(/^\/api\/appium-recorder\/scripts\/([^/]+)\/history\/([^/]+)\/video$/);
+    if (videoMatch && (req.method === 'GET' || req.method === 'HEAD')) {
+      const run = getRunHistory(decodeURIComponent(videoMatch[1]), decodeURIComponent(videoMatch[2]));
+      if (!run?.video) { sendJson(res, { error: '本次运行没有视频' }, 404); return true; }
+      assertDeviceAllowed(run.deviceId);
+      const index = Number(new URL(req.url || '', 'http://localhost').searchParams.get('segment') || 0);
+      const segment = Number.isSafeInteger(index) && index >= 0 ? (run.video.segments || [run.video])[index] : undefined;
+      if (!segment) { sendJson(res, { error: '视频片段不存在' }, 404); return true; }
+      await sendReplayVideo(req, res, segment.filePath);
+      return true;
+    }
     const historyMatch = pathname.match(/^\/api\/appium-recorder\/scripts\/([^/]+)\/history(?:\/([^/]+))?$/);
     if (historyMatch) {
       const scriptId = decodeURIComponent(historyMatch[1]);
@@ -362,12 +360,13 @@ export async function handleAppiumRecorderRequest(
     }
     const replayMatch = pathname.match(/^\/api\/appium-recorder\/scripts\/([^/]+)\/replay$/);
     if (replayMatch && req.method === 'POST') {
-      const parsed = await readBody<{ deviceId?: string; parameters?: TestVariable[] }>(req);
+      const parsed = await readBody<{ deviceId?: string; parameters?: TestVariable[]; recordVideo?: boolean }>(req);
       const script = getAppiumRecordedScript(decodeURIComponent(replayMatch[1]));
       if (!script) throw new Error('Appium 录制脚本不存在');
       const deviceId = parsed.deviceId || selectedDeviceId;
       assertDeviceAllowed(deviceId);
       const streamOutput = req.headers.accept?.includes('application/x-ndjson') === true;
+      if (parsed.recordVideo === true && isRemoteDeviceId(deviceId)) throw new Error('远程设备暂不支持回放录屏，请在连接设备的主机本地运行');
       if (streamOutput) {
         res.statusCode = 200;
         res.setHeader('Content-Type', 'application/x-ndjson; charset=utf-8');
@@ -403,7 +402,7 @@ export async function handleAppiumRecorderRequest(
           deviceId,
           streamOutput ? (line) => sendStreamEvent(res, { type: 'log', line }) : undefined,
           replayAbortController.signal,
-          { parameters: parsed.parameters, globalVariables: getPresetVariables() },
+          { parameters: parsed.parameters, globalVariables: getPresetVariables(), recordVideo: parsed.recordVideo === true },
         );
         try { saveRunHistory(result.history); }
         catch { result.output += '\n历史记录保存失败'; }

package/server/appium-recorder/run-history.ts

@@ -1,4 +1,6 @@
 import { createId, querySql, runSql, sqlJson, sqlString } from '../storage/sqlite';
+import { rmSync } from 'node:fs';
+import { basename } from 'node:path';
 import type { RunDetail, RunSummary } from '../../src/appium-recorder/run-history';
 
 function ensureTable() {
@@ -29,6 +31,12 @@ export function getRunHistory(scriptId: string, id: string): RunDetail | null {
 
 export function deleteRunHistory(scriptId: string, id: string) {
   ensureTable();
-  // 不接受文件路径，也不删除原始报告；只删除这条历史的自包含快照。
+  const video = getRunHistory(scriptId, id)?.video;
+  // 只清理服务端生成并关联到该记录的录像，不接受客户端文件路径。
+  for (const segment of video?.segments || (video ? [video] : [])) {
+    if (basename(segment.filePath) === segment.fileName && /^replay-[\d]+-[\da-f-]+\.mp4$/.test(segment.fileName)) {
+      rmSync(segment.filePath, { force: true });
+    }
+  }
   runSql(`PRAGMA secure_delete=ON; DELETE FROM appium_run_history WHERE script_id=${sqlString(scriptId)} AND id=${sqlString(id)};`);
 }

package/server/appium-recorder/scrcpy-recording-source.ts

@@ -0,0 +1,80 @@
+import { execFile } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { AdbServerClient, type Adb } from '@yume-chan/adb';
+import { AdbServerNodeTcpConnector } from '@yume-chan/adb-server-node-tcp';
+import { AdbScrcpyClient, AdbScrcpyOptions3_3_3 } from '@yume-chan/adb-scrcpy';
+import { ScrcpyVideoCodecId } from '@yume-chan/scrcpy';
+import { getAdbCommand } from '../android-sdk';
+
+export function waitForVideoTask<T>(task: Promise<T>, signal: AbortSignal): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const abort = () => reject(signal.reason);
+    signal.addEventListener('abort', abort, { once: true });
+    task.then(resolve, reject).finally(() => signal.removeEventListener('abort', abort));
+    if (signal.aborted) abort();
+  });
+}
+
+function adbCommand(args: string[], signal?: AbortSignal) {
+  return new Promise<void>((resolve, reject) => {
+    execFile(getAdbCommand(), args, { timeout: 10000, signal, windowsHide: true }, error => error ? reject(error) : resolve());
+  });
+}
+
+export async function openRecordingSource(deviceId: string, signal: AbortSignal) {
+  const require = createRequire(import.meta.url);
+  const bin = join(dirname(require.resolve('@midscene/android-playground/package.json')), 'bin');
+  const version = (await readFile(join(bin, 'scrcpy-server.version'), 'utf8')).trim();
+  if (version !== 'v3.3.3') throw new Error(`内置 scrcpy 服务端版本不兼容：${version}`);
+  const remotePath = `/data/local/tmp/midscene-recording-${randomUUID()}.jar`;
+  let adb: Adb | undefined;
+  let client: Awaited<ReturnType<typeof AdbScrcpyClient.start>> | undefined;
+  let closing = false;
+  let logs = '';
+  let closeTask: Promise<void> | undefined;
+  const close = () => closeTask ??= (async () => {
+    closing = true;
+    await waitForVideoTask(Promise.allSettled([client?.close(), adb?.close()]), AbortSignal.timeout(3000)).catch(() => undefined);
+    await adbCommand(['-s', deviceId, 'shell', 'rm', '-f', remotePath]).catch(() => undefined);
+  })();
+  try {
+    await adbCommand(['start-server'], signal);
+    await adbCommand(['-s', deviceId, 'push', join(bin, 'scrcpy-server'), remotePath], signal);
+    const server = new AdbServerClient(new AdbServerNodeTcpConnector({ host: '127.0.0.1', port: 5037 }));
+    adb = await waitForVideoTask(server.createAdb({ serial: deviceId }).then(connection => {
+      if (closing || signal.aborted) { void connection.close(); throw signal.reason || new Error('录屏已取消'); }
+      return connection;
+    }), signal);
+    // 独立 socket ID 和上传路径，停止录制不会关闭正在使用的设备预览。
+    const options = new AdbScrcpyOptions3_3_3({
+      scid: Math.floor(Math.random() * 0x7fffffff).toString(16).padStart(8, '0'),
+      audio: false, control: false, videoCodec: 'h264', sendFrameMeta: true,
+      maxSize: 1280, maxFps: 15, videoBitRate: 2000000,
+      captureOrientation: '@', videoCodecOptions: 'profile=1,max-bframes=0,i-frame-interval=2',
+    });
+    client = await waitForVideoTask(AdbScrcpyClient.start(adb, remotePath, options).then(value => {
+      if (closing || signal.aborted) { void value.close(); throw signal.reason || new Error('录屏已取消'); }
+      return value;
+    }), signal);
+    // 持续消费服务端输出，避免输出管道阻塞编码进程。
+    const outputReader = client.output.getReader();
+    void (async () => {
+      try {
+        while (true) {
+          const { done, value } = await outputReader.read();
+          if (done) break;
+          logs = `${logs}\n${value}`.slice(-4000);
+        }
+      } finally { outputReader.releaseLock(); }
+    })().catch(() => undefined);
+    const video = await waitForVideoTask(client.videoStream!, signal);
+    if (video.metadata.codec !== ScrcpyVideoCodecId.H264) throw new Error('录屏需要 H.264 视频流');
+    return { stream: video.stream, close, diagnostics: () => logs };
+  } catch (error) {
+    await close();
+    throw error;
+  }
+}

package/server/appium-recorder/video-response.ts

@@ -0,0 +1,36 @@
+import { open } from 'node:fs/promises';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+
+// 浏览器按字节加载 MP4；支持 Range 才能直接跳转到录像中间。
+export async function sendReplayVideo(req: IncomingMessage, res: ServerResponse, path: string) {
+  const file = await open(path, 'r').catch(() => null);
+  if (!file) { res.statusCode = 404; res.end('视频文件不存在'); return; }
+  let streaming = false;
+  try {
+    const { size } = await file.stat();
+    let start = 0, end = size - 1;
+    const range = req.headers.range;
+    if (range) {
+      const match = /^bytes=(\d*)-(\d*)$/.exec(range);
+      if (match && (match[1] || match[2])) {
+        start = match[1] ? Number(match[1]) : Math.max(0, size - Number(match[2]));
+        end = match[1] && match[2] ? Math.min(Number(match[2]), size - 1) : size - 1;
+      } else start = size;
+      if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start > end || start >= size) {
+        res.writeHead(416, { 'Content-Range': `bytes */${size}` }); res.end(); return;
+      }
+      res.statusCode = 206;
+      res.setHeader('Content-Range', `bytes ${start}-${end}/${size}`);
+    }
+    res.setHeader('Content-Type', 'video/mp4');
+    res.setHeader('Accept-Ranges', 'bytes');
+    res.setHeader('Cache-Control', 'private, no-store');
+    res.setHeader('Content-Length', String(Math.max(0, end - start + 1)));
+    if (req.method === 'HEAD' || !size) { res.end(); return; }
+    const stream = file.createReadStream({ start, end });
+    streaming = true;
+    res.once('close', () => stream.destroy());
+    stream.once('error', () => res.destroy());
+    stream.pipe(res);
+  } finally { if (!streaming) await file.close(); }
+}

package/src/appium-recorder/AppiumPage.vue

@@ -3,7 +3,7 @@ import { computed, h, onMounted, onUnmounted, provide, reactive, shallowRef, wat
 import { flowBackgroundKey, normalizeFlowBackground } from './flow-appearance';
 import { DEFAULT_NODE_TIMEOUT_MS } from './node-timeout';
 import { ElForm, ElFormItem, ElInputNumber, ElMessage, ElMessageBox, ElOption, ElSelect } from 'element-plus';
-import { Check, CircleClose, CopyDocument, Delete, Document, Download, Edit, Plus, Refresh, Upload, VideoPlay, View, Clock } from '@element-plus/icons-vue';
+import { ArrowDown, Check, CircleClose, CopyDocument, Delete, Document, Download, Edit, Plus, Refresh, Upload, VideoPlay, View, Clock } from '@element-plus/icons-vue';
 import RunHistoryDialog from './RunHistoryDialog.vue';
 import type { AndroidDevice, AppPreset, DeviceAction } from '../types';
 import DevicePreviewPanel from '../components/device/DevicePreviewPanel.vue';
@@ -43,8 +43,6 @@ import NewScriptDialog from './components/NewScriptDialog.vue';
 import VisualChangeDialog from './components/VisualChangeDialog.vue';
 import ImageCheckDialog from './components/ImageCheckDialog.vue';
 import { createImageCheckConfig, validateImageCheck, type ImageCheckConfig } from './image-check';
-import AiRecognitionTestDialog from './components/AiRecognitionTestDialog.vue';
-import { validateAiRecognitionPrompt } from './ai-recognition';
 import {
   createFlowClipboard,
   pasteFlowClipboard,
@@ -98,6 +96,7 @@ const AUTO_TREE_REFRESH_INTERVAL_MS = 1800;
 
 const readonlyFlowActionGroups: FlowActionGroup[] = [];
 const readonlyFlowSelectedIndexes: number[] = [];
+const recordReplayVideo = shallowRef(false);
 
 const props = defineProps<{
   aiRecognitionModelConfigured?: boolean;
@@ -137,7 +136,6 @@ const newScriptRevision = shallowRef(0);
 const storedWorkbenchTab = window.localStorage.getItem(WORKBENCH_TAB_STORAGE_KEY);
 const activeWorkbenchTab = shallowRef<'recording' | 'scripts' | 'variables'>(storedWorkbenchTab === 'scripts' ? 'scripts' : storedWorkbenchTab === 'variables' ? 'variables' : 'recording');
 const steps = shallowRef<AppiumRecordedStep[]>([]);
-const aiRecognitionTestStep = shallowRef<AppiumRecordedStep | null>(null);
 const flowClipboard = shallowRef<FlowClipboard | null>(null);
 const rawXml = shallowRef('');
 const currentActivity = shallowRef('');
@@ -888,11 +886,6 @@ async function refreshTree() {
 
 async function executeFlowStep(index: number) {
   const step = steps.value[index];
-  if (step?.type === 'aiRecognition') {
-    if (replaying.value || recordingBusy.value) { ElMessage.warning('设备正在执行操作，请稍后测试'); return; }
-    aiRecognitionTestStep.value = { ...step };
-    return;
-  }
   if (!step || (step.type !== 'launchApp' && step.type !== 'clearAppData')) return;
   if (launchingApp.value) return;
   if (!selectedDeviceId.value) {
@@ -1391,21 +1384,8 @@ async function addAction(
     return;
   }
   if (action === 'aiRecognition') {
-    const input = await ElMessageBox.prompt('识别内容', '添加 AI 识别', {
-      inputType: 'textarea',
-      inputPlaceholder: '例如：检查当前画面有没有显示黑屏',
-      inputValidator: (value) => {
-        try { validateAiRecognitionPrompt(value); return true; }
-        catch (error) { return error instanceof Error ? error.message : '识别内容无效'; }
-      },
-      confirmButtonText: '添加', cancelButtonText: '取消',
-    }).catch(() => null);
-    if (!input) return;
-    return insertStep({
-      id: createStepId(), type: 'aiRecognition', label: 'AI 识别',
-      value: validateAiRecognitionPrompt(input.value), timeoutMs: DEFAULT_NODE_TIMEOUT_MS,
-      flow: { nodeKind: 'condition' },
-    }, index, branchTarget);
+    ElMessage.warning('AI 识别操作已移除，请使用图像判断或原生组件判断');
+    return;
   }
   if (action === 'textClick') {
     const step = reactive<AppiumRecordedStep>({
@@ -2076,7 +2056,7 @@ async function replayScript() {
   try {
     await pauseAutoTreeRefresh();
     const result = await replayAppiumScript(
-      { id: selectedScript.value.id, deviceId: activeReplayDeviceId.value },
+      { id: selectedScript.value.id, deviceId: activeReplayDeviceId.value, recordVideo: recordReplayVideo.value },
       (line) => {
         replayOutput.value += `${replayOutput.value ? '\n' : ''}${line}`;
       },
@@ -2167,15 +2147,23 @@ watch(
       </el-select>
       <el-button :icon="Plus" :disabled="newScriptDisabled || newScriptDialogVisible" @click="requestNewScript">新建</el-button>
       <el-button :icon="Check" :loading="saving" @click="saveScript">保存</el-button>
-      <el-button
-        type="primary"
-        :icon="VideoPlay"
-        :loading="replaying"
-        :disabled="!selectedScript"
-        @click="replayScript"
-      >
-        回放
-      </el-button>
+      <el-button-group class="replay-button-group">
+        <el-button
+          type="primary"
+          :icon="VideoPlay"
+          :loading="replaying"
+          :disabled="!selectedScript"
+          @click="replayScript"
+        >
+          回放
+        </el-button>
+        <el-popover trigger="click" placement="bottom" :width="260">
+          <template #reference><el-button type="primary" :icon="ArrowDown" :disabled="replaying" aria-label="回放设置" title="回放设置" class="replay-settings-toggle" /></template>
+          <el-tooltip content="使用内置服务端后台录制，无需安装 scrcpy 或 FFmpeg。含敏感变量的运行不录制；分享 HTML 报告时需同时携带 MP4。" placement="bottom" :show-after="300">
+            <el-checkbox v-model="recordReplayVideo" :disabled="replaying">录制回放视频</el-checkbox>
+          </el-tooltip>
+        </el-popover>
+      </el-button-group>
       <el-button
         v-if="replaying"
         type="danger"
@@ -2383,13 +2371,6 @@ watch(
       @cancel="newScriptDialogVisible = false"
     />
 
-    <AiRecognitionTestDialog
-      v-if="aiRecognitionTestStep"
-      :step="aiRecognitionTestStep"
-      :device-id="selectedDeviceId"
-      :model-configured="Boolean(aiRecognitionModelConfigured)"
-      @close="aiRecognitionTestStep = null"
-    />
 
     <ImageCheckDialog
       v-if="imageCheckDraft?.imageCheck"
@@ -2585,4 +2566,6 @@ watch(
 .appium-workbench__flow :deep(.appium-recorded-steps-panel) { display: flex; flex-direction: column; flex: 1; min-height: 0; }
 .appium-workbench__flow :deep(.appium-flow-toolbar) { flex: none; margin-bottom: 8px; }
 .appium-workbench__flow :deep(.appium-flow-canvas--vue:not(.appium-flow-canvas--dialog)) { flex: 1; height: auto; min-height: 120px; }
+.replay-button-group { display: inline-flex; flex-shrink: 0; }
+.replay-settings-toggle { padding-inline: 8px; }
 </style>

package/src/appium-recorder/RunHistoryDialog.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
-import { computed, onMounted, ref } from 'vue';
-import { Delete, Refresh } from '@element-plus/icons-vue';
+import { computed, onMounted, ref, watch } from 'vue';
+import { Delete, Refresh, VideoPlay } from '@element-plus/icons-vue';
 import { ElMessage, ElMessageBox } from 'element-plus';
 import { historyStatistics, type RunSummary, type RunDetail } from './run-history';
 
@@ -16,6 +16,9 @@ const device = ref('');
 const error = ref('');
 const nodeKey = ref('');
 const logQuery = ref('');
+const videoRun = ref<RunSummary>();
+const videoSegment = ref(0);
+watch(videoRun, () => { videoSegment.value = 0; }, { flush: 'sync' });
 const base = `/api/appium-recorder/scripts/${encodeURIComponent(props.scriptId)}/history`;
 const labels = { passed: '通过', failed: '失败', stopped: '已终止' };
 const versions = computed(() => [...new Set(runs.value.map(run => run.appVersion || '未知'))]);
@@ -49,7 +52,7 @@ async function compare() {
   finally { comparing.value = false; }
 }
 async function remove() {
-  try { await ElMessageBox.confirm(`删除选中的 ${selected.value.length} 条历史及截图、日志快照？原始报告文件不受影响。`, '删除历史结果', { type: 'warning', confirmButtonText: '确定', cancelButtonText: '取消' }); }
+  try { await ElMessageBox.confirm(`删除选中的 ${selected.value.length} 条历史及截图、日志快照和回放视频？原始报告将无法再播放已删除的视频。`, '删除历史结果', { type: 'warning', confirmButtonText: '确定', cancelButtonText: '取消' }); }
   catch { return; }
   loading.value = true;
   try {
@@ -98,6 +101,7 @@ onMounted(load);
           <el-table-column prop="deviceId" label="设备" min-width="140" />
           <el-table-column label="结果" width="85"><template #default="{ row }">{{ labels[row.status as keyof typeof labels] }}</template></el-table-column>
           <el-table-column label="耗时" width="100"><template #default="{ row }">{{ seconds(row.durationMs) }}</template></el-table-column>
+          <el-table-column label="录像" width="70"><template #default="{ row }"><el-tooltip v-if="row.video" content="播放回放视频"><el-button :icon="VideoPlay" aria-label="播放回放视频" @click="videoRun = row" /></el-tooltip></template></el-table-column>
         </el-table>
         <h3>失败节点分布</h3>
         <el-table :data="stats.failures" max-height="220" empty-text="暂无节点失败记录">
@@ -129,6 +133,12 @@ onMounted(load);
         </div>
       </section>
     </div>
+    <el-dialog :model-value="Boolean(videoRun)" title="回放视频" width="min(800px, 90vw)" align-center append-to-body destroy-on-close @close="videoRun = undefined">
+      <el-select v-if="videoRun?.video?.segments?.length" v-model="videoSegment" aria-label="视频片段" style="width:100%;margin-bottom:12px">
+        <el-option v-for="(segment, index) in videoRun.video.segments" :key="segment.fileName" :label="`${index + 1}. ${segment.scriptName || videoRun.scriptName}`" :value="index" />
+      </el-select>
+      <video v-if="videoRun" :src="`${base}/${encodeURIComponent(videoRun.id)}/video?segment=${videoSegment}`" controls preload="metadata" style="width:100%;max-height:70vh" />
+    </el-dialog>
   </el-dialog>
 </template>
 

package/src/appium-recorder/action-descriptions.ts

@@ -26,7 +26,7 @@ export const actionDescriptions: Record<InsertAction, string> = {
   checkboxState: '读取原生 Checkbox 的 checked 状态，按 true/false 分支执行。',
   radioButtonState: '读取原生 RadioButton 的 checked 状态，按 true/false 分支执行。',
   checkedState: '读取 Checkbox、RadioButton 或 Switch 的真实勾选状态，按 true/false 分支执行。不支持仅用图片表示状态的普通 ImageView。',
-  aiRecognition: '用配置的模型判断当前画面，返回 true/false。适合语义、提示含义和复杂画面判断；可直接读取的组件属性优先使用原生判断。需要配置 AI 节点模型。',
+  aiRecognition: '此操作已移除，请使用图像判断或原生组件判断。',
   textClick: '在组件树中按指定文字精确或模糊匹配并点击，进入匹配到文字或未匹配到文字分支。不识别图片或自绘画面中的文字。',
   tapIfExists: '找到目标组件时点击，未找到则跳过并继续后续流程。',
   inputIfExists: '找到目标输入组件时写入文字，未找到则跳过。支持 {{变量名}} 引用。',