android-midscene-automation 0.1.38 → 0.1.39

package/CHANGELOG.md

@@ -1,4 +1,12 @@
 # 更新记录
+## v0.1.39
+- Midscene 相关模块统一升级至 1.12.7，
+- scrcpy 视频帧完整性修复：可靠传输视频包、修正关键帧标记，并在缓冲溢出后等待完整关键帧恢复，降低设备预览花屏风险。
+- 回放按钮右侧下拉设置新增默认关闭的“录制回放视频”，使用内置服务端后台接收视频流并封装 MP4，无需安装 scrcpy 或 FFmpeg；远程设备暂不支持。
+- 图像模板判断改为“预期匹配结果”，匹配严格度以百分比配置，100% 时提示细微像素差异风险；节点显示预期结果，日志和 HTML 报告分别展示实际匹配、预期匹配、得分与条件结论，保留原检测算法及已有阈值。
+- 移除 AI 识别操作、测试入口及 Appium 参数中的 AI 模型配置，保留流程背景色设置；旧脚本中的 AI 节点提示替换，回放前阻止执行，不自动删除或改变分支。
+
+
 ## v0.1.38
 - 移除节点类型切换；新节点及未配置的判断等待超时统一为 3000ms，已有自定义时间和延时、长按等操作时长保持不变。
 - 取消 Activity 不匹配时的脚本编辑锁定，不在脚本绑定页面也可编辑节点；页面差异仅作提示，保留录制操作及回放期间的编辑保护。

package/README.md

@@ -1229,6 +1229,14 @@ Linux: ~/.local/share/android-midscene-automation
 
 # 项目更新记录
 
+## v0.1.39
+- Midscene 相关模块统一升级至 1.12.7，
+- scrcpy 视频帧完整性修复：可靠传输视频包、修正关键帧标记，并在缓冲溢出后等待完整关键帧恢复，降低设备预览花屏风险。
+- 回放按钮右侧下拉设置新增默认关闭的“录制回放视频”，使用内置服务端后台接收视频流并封装 MP4，无需安装 scrcpy 或 FFmpeg；远程设备暂不支持。
+- 图像模板判断改为“预期匹配结果”，匹配严格度以百分比配置，100% 时提示细微像素差异风险；节点显示预期结果，日志和 HTML 报告分别展示实际匹配、预期匹配、得分与条件结论，保留原检测算法及已有阈值。
+- 移除 AI 识别操作、测试入口及 Appium 参数中的 AI 模型配置，保留流程背景色设置；旧脚本中的 AI 节点提示替换，回放前阻止执行，不自动删除或改变分支。
+
+
 ## v0.1.38
 - 移除节点类型切换；新节点及未配置的判断等待超时统一为 3000ms，已有自定义时间和延时、长按等操作时长保持不变。
 - 取消 Activity 不匹配时的脚本编辑锁定，不在脚本绑定页面也可编辑节点；页面差异仅作提示，保留录制操作及回放期间的编辑保护。

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "android-midscene-automation",
-  "version": "0.1.38",
+  "version": "0.1.39",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/oooooooko/android-midscene-automation.git"
@@ -36,14 +36,19 @@
     "test:e2e": "playwright test"
   },
   "dependencies": {
+    "@yume-chan/adb": "2.6.3",
+    "@yume-chan/adb-scrcpy": "2.3.2",
+    "@yume-chan/adb-server-node-tcp": "2.5.2",
+    "@yume-chan/scrcpy": "2.3.0",
+    "mediabunny": "1.58.0",
     "@dagrejs/dagre": "1.1.5",
     "@element-plus/icons-vue": "2.3.2",
-    "@midscene/android": "https://registry.npmjs.org/@midscene/android/-/android-1.12.6.tgz",
-    "@midscene/android-playground": "https://registry.npmjs.org/@midscene/android-playground/-/android-playground-1.12.6.tgz",
-    "@midscene/core": "https://registry.npmjs.org/@midscene/core/-/core-1.12.6.tgz",
-    "@midscene/playground": "https://registry.npmjs.org/@midscene/playground/-/playground-1.12.6.tgz",
-    "@midscene/shared": "https://registry.npmjs.org/@midscene/shared/-/shared-1.12.6.tgz",
-    "@midscene/web": "https://registry.npmjs.org/@midscene/web/-/web-1.12.6.tgz",
+    "@midscene/android": "https://registry.npmjs.org/@midscene/android/-/android-1.12.7.tgz",
+    "@midscene/android-playground": "https://registry.npmjs.org/@midscene/android-playground/-/android-playground-1.12.7.tgz",
+    "@midscene/core": "https://registry.npmjs.org/@midscene/core/-/core-1.12.7.tgz",
+    "@midscene/playground": "https://registry.npmjs.org/@midscene/playground/-/playground-1.12.7.tgz",
+    "@midscene/shared": "https://registry.npmjs.org/@midscene/shared/-/shared-1.12.7.tgz",
+    "@midscene/web": "https://registry.npmjs.org/@midscene/web/-/web-1.12.7.tgz",
     "@techstark/opencv-js": "5.0.0-release.1",
     "@vitejs/plugin-vue": "6.0.7",
     "@vue-flow/core": "1.48.2",

package/server/appium-recorder/appium-runner.ts

@@ -6,13 +6,13 @@ import { captureHistoryFrames, readAppVersion } from './history-capture';
 import { BoundedLoopTraversal } from './bounded-loop';
 import { validateLoopSteps } from '../../src/appium-recorder/bounded-loop';
 import { adbScreenshotBase64 } from './screenshot';
-import { recognizeDeviceScreen } from './ai-recognition';
 import { checkImage } from './image-check';
 import { IMAGE_CHECK_MODES, type ImageCheckResult } from '../../src/appium-recorder/image-check';
 import { formatStageLog } from '../../src/appium-recorder/stage-log';
 import { openGalleryOnDevice } from './open-gallery';
 import { stopAppOnDevice } from './stop-app';
 import { DEFAULT_NODE_TIMEOUT_MS } from '../../src/appium-recorder/node-timeout';
+import { startReplayVideo, type ReplayVideo } from './replay-video';
 import { ConditionTimeoutError, AppiumServiceError } from './condition-timeout';
 import { AppiumRequestTimeoutError, timedAppiumFetch, withElementDeadline } from './request-timeout';
 import { textClickSelector } from '../../src/appium-recorder/text-click';
@@ -353,10 +353,11 @@ async function captureReplayFrame(
 ) {
   const context = replayContext.getStore();
   if (!context || !context.deviceId) return;
+  const capturedAt = new Date().toISOString();
   // 执行记录独立于截图，设备截图失败不能让失败节点从统计中消失。
   context.historyEvents.push({ sequence: context.historyEvents.length + 1, scriptName, nodeId: step.id,
     nodeNumber, nodeLabel: step.label, nodeType: step.type, note: step.note || '',
-    selector: replayFrameSelector(step), phase, status, capturedAt: new Date().toISOString(), imageBase64: '' });
+    selector: replayFrameSelector(step), phase, status, capturedAt, imageBase64: '' });
   const appendFrame = (imageBase64: string) => {
     context.frames.push({
       sequence: context.frames.length + 1,
@@ -370,7 +371,7 @@ async function captureReplayFrame(
       selector: replayFrameSelector(step),
       phase,
       status,
-      capturedAt: new Date().toISOString(),
+      capturedAt,
       imageBase64,
     });
   };
@@ -386,6 +387,8 @@ async function captureReplayFrame(
     appendFrame(imageBase64 || await adbScreenshotBase64(context.deviceId));
   } catch (error) {
     context.appiumLog(`[节点 ${nodeNumber}] 截图失败：${errorDetail(error)}`);
+    // 截图失败仍保留时间锚点，报告可定位到该节点的录像。
+    appendFrame('');
   }
 }
 
@@ -1258,13 +1261,7 @@ async function replayFlowSteps(
           }
           continue;
         }
-        // AI 节点使用本次执行前截图，并将同一张图片和识别依据写入结果帧。
-        const beforeFrame = replayContext.getStore()?.frames.at(-1);
-        const recognition = step.type === 'aiRecognition' ? await recognizeDeviceScreen({
-          deviceId, prompt: step.value, timeoutMs: step.timeoutMs,
-          signal: replayContext.getStore()?.signal,
-          imageBase64: beforeFrame?.nodeId === step.id && beforeFrame.phase === 'before' ? beforeFrame.imageBase64 : undefined,
-        }) : undefined;
+        if (step.type === 'aiRecognition') throw new Error('AI 识别操作已移除，请替换此节点后回放');
         let imageResult: ImageCheckResult | undefined;
         if (step.type === 'imageCheck') {
           const resolved = resolveVariableStep(step);
@@ -1273,14 +1270,22 @@ async function replayFlowSteps(
             resolveRegion: () => imageCheckRegion(sessionId, resolved), wait,
             signal: replayContext.getStore()?.signal, saveImages: !variableContext.getStore()?.privacy.enabled });
           const config = resolved.imageCheck;
-          const settings = config ? `${IMAGE_CHECK_MODES[config.mode]}；条件 ${config.expectation}；模板阈值 ${config.threshold}；最小得分差 ${config.minScoreGap}；RGB/亮度容差 ${config.tolerance}；目标色 ${config.color}；像素占比 ${config.ratio}%；观察 ${config.durationMs}ms；间隔 ${config.intervalMs}ms；连续 ${config.consecutive} 帧` : '缺少配置';
+          const settings = config ? [
+            IMAGE_CHECK_MODES[config.mode],
+            config.mode === 'template' ? `预期匹配结果：${config.expectation === 'present' ? '匹配到模板' : '未匹配到模板'}` : '',
+            ['template', 'state'].includes(config.mode) ? `匹配严格度 ${config.threshold * 100}%` : '',
+            config.mode === 'state' ? `最小得分差 ${config.minScoreGap}` : '',
+            ['black', 'color', 'change'].includes(config.mode) ? `RGB/亮度容差 ${config.tolerance}；像素占比 ${config.ratio}%` : '',
+            config.mode === 'color' ? `目标色 ${config.color}` : '',
+            config.mode === 'change' ? `预期：${config.expectation === 'present' ? '画面有变化' : '持续无明显变化'}` : '',
+            config.mode !== 'state' ? `观察 ${config.durationMs}ms；间隔 ${config.intervalMs}ms；连续 ${config.consecutive} 帧` : '',
+          ].filter(Boolean).join('；') : '缺少配置';
           replayContext.getStore()?.imageChecks.push({ ...imageResult, nodeId: step.id, nodeNumber: index + 1, nodeLabel: frameStep.label, scriptName: options.scriptName || '', settings });
           lines.push(`[节点 ${index + 1}] 图像判断：${imageResult.result === null ? '无法判定' : imageResult.result}，${imageResult.message}；${settings}；采样 ${imageResult.sampleCount} 帧，耗时 ${imageResult.durationMs}ms；指标 ${JSON.stringify(imageResult.metrics)}`);
           if (imageResult.result === null) throw new Error(imageResult.message);
           if (imageResult.timedOut) throw new ConditionTimeoutError(imageResult.message);
         }
-        const matched = imageResult ? imageResult.result! : recognition ? recognition.result : await evaluateCondition(sessionId, deviceId, step);
-        if (recognition) lines.push(`[节点 ${index + 1}] AI 识别：${recognition.result}，耗时 ${recognition.durationMs}ms${recognition.reason ? `，依据：${recognition.reason}` : ''}`);
+        const matched = imageResult ? imageResult.result! : await evaluateCondition(sessionId, deviceId, step);
         const branchTargetId = matched ? step.flow?.yesTargetId : step.flow?.noTargetId;
         const next = traversal.branch(step, matched);
         const targetId = branchTargetId || (next === undefined ? '' : steps[next]?.id) || '';
@@ -1292,8 +1297,7 @@ async function replayFlowSteps(
           index + 1,
           options.scriptName || '',
           'after',
-          `判断：${branchLabel}${recognition?.reason ? `；${recognition.reason}` : ''}`,
-          recognition?.imageBase64,
+          `判断：${branchLabel}`,
         );
         index = next;
       } catch (error) {
@@ -1432,7 +1436,7 @@ export async function replayAppiumScript(
   deviceId: string,
   onOutput?: (line: string) => void,
   signal?: AbortSignal,
-  runOptions: { parameters?: TestVariable[]; globalVariables?: TestVariable[]; linkedScripts?: AppiumRecordedScriptRecord[] } = {},
+  runOptions: { recordVideo?: boolean; parameters?: TestVariable[]; globalVariables?: TestVariable[]; linkedScripts?: AppiumRecordedScriptRecord[] } = {},
 ) {
   const targetDeviceId = deviceId || script.deviceId;
   if (!targetDeviceId) throw new Error('未检测到可用设备');
@@ -1440,6 +1444,12 @@ export async function replayAppiumScript(
 
   const scope = new VariableScope(runOptions.globalVariables ?? getPresetVariables(), script.variables || []);
   const scripts = new Map((runOptions.linkedScripts ?? linkedScriptSnapshot(script)).map(item => [item.id, item]));
+  // 旧节点保留图结构供用户替换，但必须在任何设备操作之前阻止回放。
+  for (const candidate of [script, ...scripts.values()]) {
+    if (candidate.steps.some(step => step.type === 'aiRecognition')) {
+      throw new Error(`脚本“${candidate.name}”包含已移除的 AI 识别操作，请改用图像判断或原生组件判断后回放`);
+    }
+  }
   for (const item of validateVariables(runOptions.parameters)) scope.set(item);
   // 提前登记子脚本敏感预设，避免进入子脚本前的原始日志或截图泄露。
   const visited = new Set<string>();
@@ -1492,6 +1502,8 @@ export async function replayAppiumScript(
     let sessionId = '';
     let success = false;
     let stopped = false;
+    let recording: Awaited<ReturnType<typeof startReplayVideo>> | undefined;
+    let video: ReplayVideo | undefined;
     let managedAppium: Awaited<ReturnType<typeof startManagedAppiumServer>> | null = null;
     try {
       validateLoopSteps(script.steps);
@@ -1516,6 +1528,12 @@ export async function replayAppiumScript(
         sessionId = await createSession(targetDeviceId);
       }
       lines.push(`Appium session 已创建：${sessionId}`);
+      if (runOptions.recordVideo) {
+        if (scope.privacy.enabled) throw new Error('本次运行包含敏感变量，为避免泄露已禁止录屏，请关闭录制回放视频');
+        lines.push('正在启动后台录屏...');
+        recording = await startReplayVideo(targetDeviceId, signal);
+        lines.push('后台录屏已启动（MP4 / H.264 / 15fps / 2Mbps，无音频）');
+      }
       if (hasFlowSteps(script.steps)) lines.push('按流程图路径回放...');
       await replayScriptSteps(sessionId, targetDeviceId, script.steps, lines, [script.id], {
         scriptName: script.name,
@@ -1543,6 +1561,19 @@ export async function replayAppiumScript(
         lines.push(`回放终止：${detail}`);
       }
     } finally {
+      if (recording) {
+        try {
+          video = await recording.stop();
+          if (scope.privacy.enabled) {
+            await import('node:fs/promises').then(fs => fs.rm(video!.filePath, { force: true }));
+            video = undefined;
+            lines.push('敏感变量保护：已删除回放视频');
+          } else {
+            lines.push(`回放视频已保存：${video.filePath}`);
+            if (video.warning) lines.push(`录屏警告：${video.warning}`);
+          }
+        } catch (error) { lines.push(`回放视频保存失败：${errorDetail(error)}`); }
+      }
       let managedAppiumStopped = false;
       if (signal?.aborted && managedAppium) {
         await managedAppium.stop();
@@ -1581,6 +1612,7 @@ export async function replayAppiumScript(
         success,
         stopped,
         output: scope.redact(lines.join('\n')),
+        video,
         startedAt,
         completedAt,
         frames: safeFrames,
@@ -1605,9 +1637,11 @@ export async function replayAppiumScript(
       reportId,
       logPath,
       htmlReportPath,
+      videoPath: video?.filePath,
       softFailureCount: context.softFailureCount,
       history: {
         scriptId: script.id, scriptName: script.name, appPackage: script.appPackage,
+        video,
         appVersion, deviceId: targetDeviceId, startedAt: startedAt.toISOString(),
         durationMs: completedAt.getTime() - startedAt.getTime(),
         status: stopped ? 'stopped' as const : success ? 'passed' as const : 'failed' as const,

package/server/appium-recorder/image-check.ts

@@ -128,6 +128,10 @@ export async function checkImage(input: {
       if (!baseline) { baseline = current; observedAt = now(); keep('首帧', current); }
       const measurement = await measure(current, baseline, templates, config);
       result.metrics = { ...measurement.metrics };
+      if (config.mode === 'template') {
+        const score = result.metrics['模板得分']!;
+        result.templateMatch = { matched: score >= config.threshold, expected: config.expectation, score, threshold: config.threshold };
+      }
       input.signal?.throwIfAborted();
       if (config.mode === 'state') {
         if (measurement.matched === null) throw new Error('双模板均未达到阈值或得分过于接近');
@@ -153,6 +157,12 @@ export async function checkImage(input: {
     }
     if (last) keep('判定帧', last);
     result.message = result.timedOut ? '观察超时，未在规定时间内满足条件' : result.result ? '条件成立' : '条件不成立';
+    if (result.templateMatch) {
+      const match = result.templateMatch;
+      const actual = match.matched ? '匹配到模板' : '未匹配到模板';
+      const expected = match.expected === 'present' ? '匹配到模板' : '未匹配到模板';
+      result.message = `实际${actual}：得分 ${(match.score * 100).toFixed(4)}%，${match.matched ? '达到' : '低于'}匹配严格度 ${(match.threshold * 100).toFixed(4)}%；预期${expected}；${result.message}${result.timedOut ? '' : ` → ${result.result} 分支`}`;
+    }
   } catch (error) {
     if (input.signal?.aborted) throw error;
     if (error instanceof AppiumRequestTimeoutError || error instanceof AppiumServiceError) throw error;

package/server/appium-recorder/replay-mp4.ts

@@ -0,0 +1,72 @@
+import { EncodedPacket, EncodedVideoPacketSource, FilePathTarget, Mp4OutputFormat, Output } from 'mediabunny';
+import { h264ParseConfiguration, type ScrcpyMediaStreamPacket } from '@yume-chan/scrcpy';
+
+// 仅封装 H.264，不解码或重新编码；最多保留最后一帧以计算静止画面的持续时间。
+export class ReplayMp4Writer {
+  private source = new EncodedVideoPacketSource('avc');
+  private output: Output;
+  private config?: ReturnType<typeof h264ParseConfiguration>;
+  private configBytes?: Uint8Array;
+  private origin?: bigint;
+  private pending?: { data: Uint8Array; key: boolean; timestamp: number };
+  private first = true;
+  startedAt = 0;
+  private startedMonotonic = 0;
+  private endedMonotonic?: number;
+
+  constructor(path: string) {
+    this.output = new Output({ target: new FilePathTarget(path, { chunkSize: 1024 * 1024 }), format: new Mp4OutputFormat({ fastStart: 'fragmented' }) });
+    this.output.addVideoTrack(this.source);
+  }
+
+  async start() { await this.output.start(); }
+
+  async add(packet: ScrcpyMediaStreamPacket) {
+    if (packet.type === 'configuration') {
+      const config = h264ParseConfiguration(packet.data);
+      if (this.configBytes && !Buffer.from(this.configBytes).equals(Buffer.from(packet.data))) {
+        throw new Error('录屏编码配置发生变化，已保留变化前的视频');
+      }
+      this.config = config;
+      this.configBytes = packet.data.slice();
+      return;
+    }
+    if (!this.config || !this.configBytes) throw new Error('录屏缺少 H.264 配置信息');
+    if (packet.pts === undefined) throw new Error('录屏视频包缺少时间戳');
+    if (typeof packet.keyframe !== 'boolean') throw new Error('录屏视频包缺少关键帧标记');
+    if (this.origin === undefined) {
+      if (!packet.keyframe) return;
+      this.origin = packet.pts;
+      this.startedAt = Date.now();
+      this.startedMonotonic = performance.now();
+    }
+    const timestamp = Number(packet.pts - this.origin) / 1000000;
+    if (!Number.isFinite(timestamp) || timestamp < 0 || (this.pending && timestamp < this.pending.timestamp)) throw new Error('录屏时间戳异常');
+    if (this.pending) await this.flush(Math.max(0.000001, timestamp - this.pending.timestamp));
+    const data = this.first ? Buffer.concat([this.configBytes, packet.data]) : packet.data.slice();
+    this.pending = { data, key: packet.keyframe, timestamp };
+  }
+
+  private async flush(duration: number) {
+    if (!this.pending || !this.config) return;
+    const { data, key, timestamp } = this.pending;
+    const c = this.config;
+    const codec = `avc1.${[c.profileIndex, c.constraintSet, c.levelIndex].map(v => v.toString(16).padStart(2, '0')).join('')}`;
+    await this.source.add(new EncodedPacket(data, key ? 'key' : 'delta', timestamp, duration), this.first ? {
+      decoderConfig: { codec, codedWidth: c.croppedWidth, codedHeight: c.croppedHeight },
+    } : undefined);
+    this.first = false;
+    this.pending = undefined;
+  }
+
+  end() { this.endedMonotonic ??= performance.now(); }
+
+  async finish() {
+    if (!this.startedAt) throw new Error('未收到可录制的关键帧');
+    await this.flush(Math.max(1 / 15, ((this.endedMonotonic ?? performance.now()) - this.startedMonotonic) / 1000 - (this.pending?.timestamp || 0)));
+    this.source.close();
+    await this.output.finalize();
+  }
+
+  async cancel() { await this.output.cancel(); }
+}

package/server/appium-recorder/replay-video.ts

@@ -0,0 +1,99 @@
+import { mkdir, open, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { loadConfig } from '../config';
+import { appDataPath } from '../paths';
+import { openRecordingSource, waitForVideoTask } from './scrcpy-recording-source';
+import { ReplayMp4Writer } from './replay-mp4';
+
+export type ReplayVideo = { filePath: string; fileName: string; startedAt: string; warning?: string };
+
+// 检查 MP4 的索引和媒体块，不将大视频整体读入内存。
+export async function isFinalizedMp4(path: string) {
+  const file = await open(path, 'r');
+  try {
+    const { size } = await file.stat();
+    let offset = 0, media = false, index = false;
+    const header = Buffer.alloc(16);
+    while (offset + 8 <= size) {
+      const { bytesRead } = await file.read(header, 0, 16, offset);
+      const type = header.toString('ascii', 4, 8);
+      let length = header.readUInt32BE(0);
+      if (length === 1) {
+        if (bytesRead < 16) return false;
+        length = Number(header.readBigUInt64BE(8));
+      }
+      if (!length) length = size - offset;
+      if (length < 8 || !Number.isSafeInteger(length) || offset + length > size) return false;
+      if (type === 'mdat' && length > 8) media = true;
+      if (type === 'moov' && length > 8) index = true;
+      offset += length;
+    }
+    return media && index && offset === size;
+  } finally { await file.close(); }
+}
+
+export async function startReplayVideo(deviceId: string, signal?: AbortSignal) {
+  signal?.throwIfAborted();
+  const dir = appDataPath(loadConfig().runtime.reportOutputPath.trim() || 'output');
+  await mkdir(dir, { recursive: true });
+  const fileName = `replay-${Date.now()}-${randomUUID()}.mp4`;
+  const filePath = join(dir, fileName);
+  const startupSignal = AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(30000)]);
+  const source = await openRecordingSource(deviceId, startupSignal);
+  return recordVideoStream(source, filePath, fileName, startupSignal);
+}
+
+// 独立于连接层，便于用真实 H.264 数据验证封装、断流和终止收尾。
+export async function recordVideoStream(source: Awaited<ReturnType<typeof openRecordingSource>>, filePath: string, fileName: string, startupSignal: AbortSignal) {
+  const writer = new ReplayMp4Writer(filePath);
+  const reader = source.stream.getReader();
+  let stopping = false;
+  let warning: string | undefined;
+  let readyResolve!: () => void;
+  let readyReject!: (error: unknown) => void;
+  const ready = new Promise<void>((resolve, reject) => { readyResolve = resolve; readyReject = reject; });
+  void ready.catch(() => undefined);
+  let pump: Promise<void> = Promise.resolve();
+  let stopTask: Promise<ReplayVideo> | undefined;
+  const stop = () => stopTask ??= (async () => {
+    stopping = true;
+    await Promise.allSettled([reader.cancel(), source.close()]);
+    await pump;
+    try {
+      await writer.finish();
+      if (!await isFinalizedMp4(filePath)) throw new Error('录屏未生成完整 MP4');
+      return { filePath, fileName, startedAt: new Date(writer.startedAt).toISOString(), warning };
+    } catch (error) {
+      await writer.cancel().catch(() => undefined);
+      await rm(filePath, { force: true });
+      throw error;
+    }
+  })();
+  try {
+    await writer.start();
+    pump = (async () => {
+      try {
+        while (!stopping) {
+          const { done, value } = await reader.read();
+          if (done) {
+            if (!stopping) throw new Error(`录屏视频流提前结束 ${source.diagnostics()}`);
+            break;
+          }
+          await writer.add(value);
+          if (writer.startedAt) readyResolve();
+        }
+      } catch (error) {
+        warning = `录像可能不完整：${error instanceof Error ? error.message : String(error)}`;
+        readyReject(error);
+        void source.close();
+      } finally { writer.end(); reader.releaseLock(); }
+    })();
+    await waitForVideoTask(ready, startupSignal);
+    return { startedAt: new Date(writer.startedAt).toISOString(), stop };
+  } catch (error) {
+    await stop().catch(() => undefined);
+    await rm(filePath, { force: true });
+    throw error;
+  }
+}

package/server/appium-recorder/report.ts

@@ -1,5 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
-import { join } from 'node:path';
+import { join, relative } from 'node:path';
+import type { ReplayVideo } from './replay-video';
 import { PNG } from 'pngjs';
 import { appDataPath } from '../paths';
 import { loadConfig } from '../config';
@@ -240,6 +241,7 @@ function stepSection(
 }
 
 function createReplayHtml(input: {
+  video?: { url: string; startedAt: string };
   script: AppiumRecordedScriptRecord;
   deviceId: string;
   resultText: string;
@@ -253,6 +255,7 @@ function createReplayHtml(input: {
 }) {
   const imageCache = new Map<string, string>();
   const payload = jsonForHtml({
+    video: input.video,
     startedAt: input.startedAt.toISOString(),
     durationMs: input.durationMs,
     frames: input.frames.map((frame) => ({
@@ -325,9 +328,11 @@ function createReplayHtml(input: {
     .viewer { display: grid; grid-template-rows: minmax(0, 1fr) auto auto; min-width: 0; min-height: 0; padding: 16px 20px 12px; background: #f4f6f8; }
     .screen { display: flex; align-items: center; justify-content: center; min-height: 0; overflow: hidden; border-radius: 6px; background: #101827; }
     .screen img { display: block; width: 100%; height: 100%; object-fit: contain; transition: opacity .12s ease; }
+    [hidden] { display: none !important; }
     .empty { color: #a8abb2; }
     .caption { min-height: 40px; padding: 10px 4px 0; overflow: hidden; color: #4b515a; font-size: 13px; text-align: center; text-overflow: ellipsis; white-space: nowrap; }
     .controls { display: grid; grid-template-columns: auto auto auto minmax(120px, 1fr) auto; align-items: center; gap: 8px; min-height: 52px; }
+    .controls:has(#toggle-recording:not([hidden])) { grid-template-columns: auto auto auto auto minmax(80px, 1fr) auto; }
     button { min-height: 34px; padding: 0 14px; border: 1px solid #cfd5dd; border-radius: 4px; color: #30343b; background: #fff; cursor: pointer; }
     button:hover:not(:disabled) { border-color: #409eff; color: #1677ff; }
     button:disabled { cursor: not-allowed; opacity: .45; }
@@ -364,6 +369,9 @@ function createReplayHtml(input: {
       .thumbnail { height: 78px; }
       header { align-items: flex-start; flex-direction: column; }
       .meta { justify-content: flex-start; }
+      .controls, .controls:has(#toggle-recording:not([hidden])) { display: flex; flex-wrap: wrap; }
+      .controls button { max-width: 100%; }
+      #seek { flex: 1 1 120px; width: auto; }
     }
   </style>
 </head>
@@ -406,9 +414,10 @@ function createReplayHtml(input: {
         </div>
       </div>
       <section class="viewer">
-        <div class="screen"><img id="screen" alt="设备截图" hidden /><span id="empty" class="empty">本次回放没有可用截图</span></div>
+        <div class="screen"><video id="recording" controls preload="metadata" playsinline hidden style="width:100%;height:100%;object-fit:contain"></video><img id="screen" alt="设备截图" hidden /><span id="empty" class="empty">本次回放没有可用截图</span></div>
         <div id="caption" class="caption">暂无截图</div>
         <div class="controls">
+          <button id="toggle-recording" hidden>查看节点截图</button>
           <button id="previous" type="button" title="上一帧">上一帧</button>
           <button id="play" type="button">播放</button>
           <button id="next" type="button" title="下一帧">下一帧</button>
@@ -430,6 +439,12 @@ function createReplayHtml(input: {
     const data = JSON.parse(document.querySelector('#replay-data').textContent || '{}');
     const visualChecks = Array.isArray(data.visualChecks) ? data.visualChecks : [];
     const reportStartedAt = Date.parse(data.startedAt) || 0;
+    const recording = document.querySelector('#recording');
+    const toggleRecording = document.querySelector('#toggle-recording');
+    const videoOffset = Math.max(0, (Date.parse(data.video?.startedAt) || reportStartedAt) - reportStartedAt);
+    let videoMode = Boolean(data.video);
+    let pendingVideoTime = 0;
+    if (data.video) { recording.src = data.video.url; toggleRecording.hidden = false; }
     const frames = (Array.isArray(data.frames) ? data.frames : []).map((frame) => ({
       ...frame,
       offsetMs: Math.max(0, (Date.parse(frame.capturedAt) || reportStartedAt) - reportStartedAt),
@@ -487,8 +502,9 @@ function createReplayHtml(input: {
       const frameChanged = nextFrame !== currentFrame;
       currentFrame = nextFrame;
       const frame = frames[currentFrame];
-      elements.empty.hidden = Boolean(frame);
-      elements.screen.hidden = !frame;
+      elements.empty.hidden = videoMode || Boolean(frame?.imageUrl);
+      elements.screen.hidden = videoMode || !frame?.imageUrl;
+      recording.hidden = !videoMode;
       if (frame) {
         if (frameChanged || elements.screen.src !== frame.imageUrl) elements.screen.src = frame.imageUrl;
         elements.title.textContent = frame.nodeLabel || '未命名节点';
@@ -507,7 +523,7 @@ function createReplayHtml(input: {
       }
       elements.previous.disabled = !frames.length || currentFrame <= 0;
       elements.next.disabled = !frames.length || currentFrame >= frames.length - 1;
-      elements.play.disabled = frames.length < 2;
+      elements.play.disabled = !videoMode && frames.length < 2;
       elements.seek.value = String(Math.round(currentMs / durationMs * 1000));
       elements.clock.textContent = formatTime(currentMs) + ' / ' + formatTime(durationMs);
       elements.playhead.style.left = (currentMs / durationMs * 100) + '%';
@@ -516,14 +532,40 @@ function createReplayHtml(input: {
       if (scrollStep) document.querySelector('.frame-item.active')?.scrollIntoView({ block: 'nearest', behavior: 'smooth' });
     };
     const pause = () => {
+      recording.pause();
       cancelAnimationFrame(animationFrame);
       animationFrame = 0;
       elements.play.textContent = '播放';
     };
     const seekTo = (milliseconds, scrollStep = true) => {
       currentMs = Math.max(0, Math.min(durationMs, milliseconds));
+      pendingVideoTime = Math.max(0, (currentMs - videoOffset) / 1000);
+      if (data.video && recording.readyState >= 1) recording.currentTime = Math.min(pendingVideoTime, recording.duration || pendingVideoTime);
       render(scrollStep);
     };
+    recording.addEventListener('loadedmetadata', () => { recording.currentTime = Math.min(pendingVideoTime, recording.duration); });
+    recording.addEventListener('timeupdate', () => {
+      if (!videoMode || recording.paused) return;
+      currentMs = recording.currentTime * 1000 + videoOffset;
+      render();
+    });
+    recording.addEventListener('seeked', () => {
+      if (!videoMode) return;
+      currentMs = recording.currentTime * 1000 + videoOffset;
+      render();
+    });
+    recording.addEventListener('play', () => { elements.play.textContent = '暂停'; });
+    recording.addEventListener('pause', () => { elements.play.textContent = '播放'; });
+    recording.addEventListener('error', () => {
+      pause(); videoMode = false; render();
+      toggleRecording.textContent = '视频不可用，请将 MP4 与报告放在一起';
+      toggleRecording.disabled = true;
+    });
+    toggleRecording.addEventListener('click', () => {
+      pause(); videoMode = !videoMode;
+      toggleRecording.textContent = videoMode ? '查看节点截图' : '查看回放视频';
+      seekTo(currentMs, false);
+    });
     const tickPlayback = (now) => {
       currentMs = Math.min(durationMs, now - playbackStartedAt);
       render();
@@ -605,7 +647,12 @@ function createReplayHtml(input: {
       article.append(title);
       const detail = document.createElement('dl');
       const region = check.region || {};
-      const entries = [['结果', check.timedOut ? '观察超时' : check.result === null ? '无法判定' : String(check.result)], ['配置', check.settings],
+      const match = check.templateMatch;
+      const entries = [
+        ...(match ? [['实际匹配结果', match.matched ? '匹配到模板' : '未匹配到模板'],
+          ['预期匹配结果', match.expected === 'present' ? '匹配到模板' : '未匹配到模板'],
+          ['匹配得分 / 严格度', (match.score * 100).toFixed(4) + '% / ' + (match.threshold * 100).toFixed(4) + '%']] : []),
+        [match ? '条件是否成立' : '结果', check.timedOut ? '观察超时（按节点超时配置处理）' : check.result === null ? '无法判定' : match ? (check.result ? '成立 → true 分支' : '不成立 → false 分支') : String(check.result)], ['配置', check.settings],
         ['区域', [region.x, region.y, region.width, region.height].join(', ')],
         ['采样', check.sampleCount + ' 帧'], ['耗时', check.durationMs + 'ms'], ['说明', check.message], ...Object.entries(check.metrics)];
       entries.forEach(([key, value]) => {
@@ -685,6 +732,11 @@ function createReplayHtml(input: {
     elements.previous.addEventListener('click', () => { pause(); seekTo(frames[Math.max(0, currentFrame - 1)]?.offsetMs || 0); });
     elements.next.addEventListener('click', () => { pause(); seekTo(frames[Math.min(frames.length - 1, currentFrame + 1)]?.offsetMs || durationMs); });
     elements.play.addEventListener('click', () => {
+      if (videoMode) {
+        if (!recording.paused) return pause();
+        recording.play().catch(() => { elements.caption.textContent = '视频无法播放，请检查 MP4 文件是否存在或使用其他浏览器'; });
+        return;
+      }
       if (animationFrame) return pause();
       if (currentMs >= durationMs) currentMs = 0;
       playbackStartedAt = performance.now() - currentMs;
@@ -719,6 +771,7 @@ function createReplayHtml(input: {
 }
 
 export async function createAppiumReplayReport(input: {
+  video?: ReplayVideo;
   script: AppiumRecordedScriptRecord;
   deviceId: string;
   success: boolean;
@@ -753,6 +806,7 @@ export async function createAppiumReplayReport(input: {
   const visualCheckByStepId = new Map(visualChecks.map((check) => [check.nodeId, check]));
   const visualFailureCount = visualChecks.filter((check) => check.status === 'failed').length;
   const html = createReplayHtml({
+    video: input.video ? { startedAt: input.video.startedAt, url: relative(outputDir, input.video.filePath).split(/[/\\\\]/).map(encodeURIComponent).join('/') } : undefined,
     script: input.script,
     deviceId: input.deviceId,
     resultText,