android-midscene-automation 0.1.37 → 0.1.39

package/CHANGELOG.md

@@ -1,4 +1,17 @@
 # 更新记录
+## v0.1.39
+- Midscene 相关模块统一升级至 1.12.7，
+- scrcpy 视频帧完整性修复：可靠传输视频包、修正关键帧标记，并在缓冲溢出后等待完整关键帧恢复，降低设备预览花屏风险。
+- 回放按钮右侧下拉设置新增默认关闭的“录制回放视频”，使用内置服务端后台接收视频流并封装 MP4，无需安装 scrcpy 或 FFmpeg；远程设备暂不支持。
+- 图像模板判断改为“预期匹配结果”，匹配严格度以百分比配置，100% 时提示细微像素差异风险；节点显示预期结果，日志和 HTML 报告分别展示实际匹配、预期匹配、得分与条件结论，保留原检测算法及已有阈值。
+- 移除 AI 识别操作、测试入口及 Appium 参数中的 AI 模型配置，保留流程背景色设置；旧脚本中的 AI 节点提示替换，回放前阻止执行，不自动删除或改变分支。
+
+
+## v0.1.38
+- 移除节点类型切换；新节点及未配置的判断等待超时统一为 3000ms，已有自定义时间和延时、长按等操作时长保持不变。
+- 取消 Activity 不匹配时的脚本编辑锁定，不在脚本绑定页面也可编辑节点；页面差异仅作提示，保留录制操作及回放期间的编辑保护。
+- 所有分支节点新增“超时后处理”，可预设终止流程、进入左侧或右侧分支，默认终止；覆盖元素等待、文字匹配、勾选控件等待、AI 识别及图像观察超时，日志和 HTML 报告标明超时分支。正常 false、循环次数上限与超时区分处理，设备断开及 Appium 无响应仍终止。
+
 ## v0.1.37
 - 修复元素查找超时未覆盖 Appium 请求的问题，驱动无响应时及时中止并明确报错，不再误判为找不到元素；唯一原生定位优先使用 ID 或 Accessibility ID。
 - 设备预览新增设备信息，展示设备名称、品牌、型号、处理器、Android 版本、物理分辨率及当前分辨率，支持远程设备；问号位于标题右侧，样式与全局变量帮助图标一致。

package/README.md

@@ -1229,6 +1229,19 @@ Linux: ~/.local/share/android-midscene-automation
 
 # 项目更新记录
 
+## v0.1.39
+- Midscene 相关模块统一升级至 1.12.7，
+- scrcpy 视频帧完整性修复：可靠传输视频包、修正关键帧标记，并在缓冲溢出后等待完整关键帧恢复，降低设备预览花屏风险。
+- 回放按钮右侧下拉设置新增默认关闭的“录制回放视频”，使用内置服务端后台接收视频流并封装 MP4，无需安装 scrcpy 或 FFmpeg；远程设备暂不支持。
+- 图像模板判断改为“预期匹配结果”，匹配严格度以百分比配置，100% 时提示细微像素差异风险；节点显示预期结果，日志和 HTML 报告分别展示实际匹配、预期匹配、得分与条件结论，保留原检测算法及已有阈值。
+- 移除 AI 识别操作、测试入口及 Appium 参数中的 AI 模型配置，保留流程背景色设置；旧脚本中的 AI 节点提示替换，回放前阻止执行，不自动删除或改变分支。
+
+
+## v0.1.38
+- 移除节点类型切换；新节点及未配置的判断等待超时统一为 3000ms，已有自定义时间和延时、长按等操作时长保持不变。
+- 取消 Activity 不匹配时的脚本编辑锁定，不在脚本绑定页面也可编辑节点；页面差异仅作提示，保留录制操作及回放期间的编辑保护。
+- 所有分支节点新增“超时后处理”，可预设终止流程、进入左侧或右侧分支，默认终止；覆盖元素等待、文字匹配、勾选控件等待、AI 识别及图像观察超时，日志和 HTML 报告标明超时分支。正常 false、循环次数上限与超时区分处理，设备断开及 Appium 无响应仍终止。
+
 ## v0.1.37
 - 修复元素查找超时未覆盖 Appium 请求的问题，驱动无响应时及时中止并明确报错，不再误判为找不到元素；唯一原生定位优先使用 ID 或 Accessibility ID。
 - 设备预览新增设备信息，展示设备名称、品牌、型号、处理器、Android 版本、物理分辨率及当前分辨率，支持远程设备；问号位于标题右侧，样式与全局变量帮助图标一致。

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "android-midscene-automation",
-  "version": "0.1.37",
+  "version": "0.1.39",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/oooooooko/android-midscene-automation.git"
@@ -36,14 +36,19 @@
     "test:e2e": "playwright test"
   },
   "dependencies": {
+    "@yume-chan/adb": "2.6.3",
+    "@yume-chan/adb-scrcpy": "2.3.2",
+    "@yume-chan/adb-server-node-tcp": "2.5.2",
+    "@yume-chan/scrcpy": "2.3.0",
+    "mediabunny": "1.58.0",
     "@dagrejs/dagre": "1.1.5",
     "@element-plus/icons-vue": "2.3.2",
-    "@midscene/android": "https://registry.npmjs.org/@midscene/android/-/android-1.12.6.tgz",
-    "@midscene/android-playground": "https://registry.npmjs.org/@midscene/android-playground/-/android-playground-1.12.6.tgz",
-    "@midscene/core": "https://registry.npmjs.org/@midscene/core/-/core-1.12.6.tgz",
-    "@midscene/playground": "https://registry.npmjs.org/@midscene/playground/-/playground-1.12.6.tgz",
-    "@midscene/shared": "https://registry.npmjs.org/@midscene/shared/-/shared-1.12.6.tgz",
-    "@midscene/web": "https://registry.npmjs.org/@midscene/web/-/web-1.12.6.tgz",
+    "@midscene/android": "https://registry.npmjs.org/@midscene/android/-/android-1.12.7.tgz",
+    "@midscene/android-playground": "https://registry.npmjs.org/@midscene/android-playground/-/android-playground-1.12.7.tgz",
+    "@midscene/core": "https://registry.npmjs.org/@midscene/core/-/core-1.12.7.tgz",
+    "@midscene/playground": "https://registry.npmjs.org/@midscene/playground/-/playground-1.12.7.tgz",
+    "@midscene/shared": "https://registry.npmjs.org/@midscene/shared/-/shared-1.12.7.tgz",
+    "@midscene/web": "https://registry.npmjs.org/@midscene/web/-/web-1.12.7.tgz",
     "@techstark/opencv-js": "5.0.0-release.1",
     "@vitejs/plugin-vue": "6.0.7",
     "@vue-flow/core": "1.48.2",

package/server/appium-recorder/ai-recognition.ts

@@ -1,4 +1,6 @@
 import OpenAI from 'openai';
+import { DEFAULT_NODE_TIMEOUT_MS } from '../../src/appium-recorder/node-timeout';
+import { ConditionTimeoutError } from './condition-timeout';
 import { loadConfig } from '../config';
 import { adbScreenshotBase64 } from './screenshot';
 import {
@@ -35,7 +37,7 @@ export async function recognizeDeviceScreen(input: {
   if (!['http:', 'https:'].includes(baseURL.protocol)) throw new Error('AI 识别模型 Base URL 必须为 HTTP(S) 地址');
   const startedAt = Date.now();
   const timeoutMs = Number.isFinite(input.timeoutMs) && input.timeoutMs! > 0
-    ? Math.min(300000, Math.max(1000, input.timeoutMs!)) : 60000;
+    ? Math.min(300000, Math.max(1000, input.timeoutMs!)) : DEFAULT_NODE_TIMEOUT_MS;
   const deadline = AbortSignal.timeout(timeoutMs);
   const signal = input.signal ? AbortSignal.any([input.signal, deadline]) : deadline;
   signal.throwIfAborted();
@@ -59,7 +61,7 @@ export async function recognizeDeviceScreen(input: {
     return { ...parseAiRecognitionResult(choice.message.content), durationMs: Date.now() - startedAt, imageBase64 };
   } catch (error) {
     if (input.signal?.aborted) throw input.signal.reason;
-    if (deadline.aborted) throw new Error(`AI 识别超时（${timeoutMs}ms）`);
+    if (deadline.aborted) throw new ConditionTimeoutError(`AI 识别超时（${timeoutMs}ms）`);
     // 不透传模型服务的原始响应，避免 API Key 或敏感请求信息进入报告。
     if (error instanceof OpenAI.APIError) throw new Error(`AI 识别模型请求失败（HTTP ${error.status || '连接异常'}），请检查模型配置及图片输入支持`);
     throw error;

package/server/appium-recorder/appium-runner.ts

@@ -6,12 +6,14 @@ import { captureHistoryFrames, readAppVersion } from './history-capture';
 import { BoundedLoopTraversal } from './bounded-loop';
 import { validateLoopSteps } from '../../src/appium-recorder/bounded-loop';
 import { adbScreenshotBase64 } from './screenshot';
-import { recognizeDeviceScreen } from './ai-recognition';
 import { checkImage } from './image-check';
 import { IMAGE_CHECK_MODES, type ImageCheckResult } from '../../src/appium-recorder/image-check';
 import { formatStageLog } from '../../src/appium-recorder/stage-log';
 import { openGalleryOnDevice } from './open-gallery';
 import { stopAppOnDevice } from './stop-app';
+import { DEFAULT_NODE_TIMEOUT_MS } from '../../src/appium-recorder/node-timeout';
+import { startReplayVideo, type ReplayVideo } from './replay-video';
+import { ConditionTimeoutError, AppiumServiceError } from './condition-timeout';
 import { AppiumRequestTimeoutError, timedAppiumFetch, withElementDeadline } from './request-timeout';
 import { textClickSelector } from '../../src/appium-recorder/text-click';
 import { AsyncLocalStorage } from 'node:async_hooks';
@@ -86,7 +88,7 @@ function isReplayStopped(error?: unknown) {
 function throwIfReplayStopped(error?: unknown) {
   if (isReplayStopped(error)) throw new ReplayStoppedError();
   // 驱动失去响应不是“元素不存在”，不能被备用定位或可选步骤吞掉。
-  if (error instanceof AppiumRequestTimeoutError) throw error;
+  if (error instanceof AppiumRequestTimeoutError || error instanceof AppiumServiceError) throw error;
 }
 
 function errorDetail(error: unknown) {
@@ -209,16 +211,17 @@ function adbSwipe(deviceId: string, step: Required<AppiumRecordedStepRecord>['sw
   });
 }
 
-function adbText(deviceId: string, args: string[]) {
-  return new Promise<string>((resolve) => {
-    execFile(getAdbCommand(), ['-s', deviceId, ...args], { maxBuffer: 4 * 1024 * 1024 }, (_error, stdout, stderr) => {
+function adbText(deviceId: string, args: string[], strict = false) {
+  return new Promise<string>((resolve, reject) => {
+    execFile(getAdbCommand(), ['-s', deviceId, ...args], { maxBuffer: 4 * 1024 * 1024, timeout: 10000, signal: replayContext.getStore()?.signal }, (error, stdout, stderr) => {
+      if (strict && error) { reject(new AppiumServiceError(`读取设备状态失败：${stderr || error.message}`)); return; }
       resolve((stdout || stderr || '').trim());
     });
   });
 }
 
 async function getCurrentActivity(deviceId: string) {
-  const output = await adbText(deviceId, ['shell', 'dumpsys', 'activity', 'activities']);
+  const output = await adbText(deviceId, ['shell', 'dumpsys', 'activity', 'activities'], true);
   const resumedLine = output
     .split(/\r?\n/)
     .find((line) => /(?:topResumedActivity|ResumedActivity|mResumedActivity)/.test(line));
@@ -235,7 +238,7 @@ async function waitForActivity(deviceId: string, step: AppiumRecordedStepRecord)
   const expectedActivity = step.value || '';
   if (!expectedActivity) throw new Error(`${step.label} 缺少目标 Activity`);
   const startedAt = Date.now();
-  const timeoutMs = step.timeoutMs || 10000;
+  const timeoutMs = step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS;
   let currentActivity = '';
   while (Date.now() - startedAt <= timeoutMs) {
     throwIfReplayStopped();
@@ -243,7 +246,7 @@ async function waitForActivity(deviceId: string, step: AppiumRecordedStepRecord)
     if (currentActivity === expectedActivity) return;
     await wait(500);
   }
-  throw new Error(`${step.label} 等待超时，当前 Activity：${currentActivity || '-'}`);
+  throw new ConditionTimeoutError(`${step.label} 等待超时，当前 Activity：${currentActivity || '-'}`);
 }
 
 async function appendSettingsDiagnostics(lines: string[], deviceId: string) {
@@ -300,9 +303,9 @@ async function appiumRequest<T>(
     context?.appiumLog(`[Appium] 异常：${method} ${path}（${errorDetail(error)}）`);
     if (error instanceof AppiumRequestTimeoutError) throw error;
     if (isTimeoutError(error)) {
-      throw new Error(`Appium 请求超时 ${requestUrl}，${errorDetail(error)}`);
+      throw new AppiumServiceError(`Appium 请求超时 ${requestUrl}，${errorDetail(error)}`);
     }
-    throw new Error(`无法连接 Appium 服务 ${requestUrl}，${errorDetail(error)}`);
+    throw new AppiumServiceError(`无法连接 Appium 服务 ${requestUrl}，${errorDetail(error)}`);
   }
 
   context?.appiumLog(`[Appium] 响应：HTTP ${response.status} ${method} ${path}（${Date.now() - startedAt}ms）`);
@@ -325,7 +328,8 @@ async function appiumRequest<T>(
       payload.value?.stacktrace || payload.stacktrace ? `堆栈信息：\n${payload.value?.stacktrace || payload.stacktrace}` : '',
       responseText ? `原始响应：\n${responseText}` : '',
     ].filter(Boolean).join('\n');
-    throw new Error(`Appium ${method} ${path} 失败（HTTP ${response.status}）：\n${detail || '未返回错误详情'}`);
+    const ErrorType = ['no such element', 'stale element reference'].includes(payload.value?.error || payload.error || '') ? Error : AppiumServiceError;
+    throw new ErrorType(`Appium ${method} ${path} 失败（HTTP ${response.status}）：\n${detail || '未返回错误详情'}`);
   }
   return payload;
 }
@@ -349,10 +353,11 @@ async function captureReplayFrame(
 ) {
   const context = replayContext.getStore();
   if (!context || !context.deviceId) return;
+  const capturedAt = new Date().toISOString();
   // 执行记录独立于截图，设备截图失败不能让失败节点从统计中消失。
   context.historyEvents.push({ sequence: context.historyEvents.length + 1, scriptName, nodeId: step.id,
     nodeNumber, nodeLabel: step.label, nodeType: step.type, note: step.note || '',
-    selector: replayFrameSelector(step), phase, status, capturedAt: new Date().toISOString(), imageBase64: '' });
+    selector: replayFrameSelector(step), phase, status, capturedAt, imageBase64: '' });
   const appendFrame = (imageBase64: string) => {
     context.frames.push({
       sequence: context.frames.length + 1,
@@ -366,7 +371,7 @@ async function captureReplayFrame(
       selector: replayFrameSelector(step),
       phase,
       status,
-      capturedAt: new Date().toISOString(),
+      capturedAt,
       imageBase64,
     });
   };
@@ -382,6 +387,8 @@ async function captureReplayFrame(
     appendFrame(imageBase64 || await adbScreenshotBase64(context.deviceId));
   } catch (error) {
     context.appiumLog(`[节点 ${nodeNumber}] 截图失败：${errorDetail(error)}`);
+    // 截图失败仍保留时间锚点，报告可定位到该节点的录像。
+    appendFrame('');
   }
 }
 
@@ -444,7 +451,7 @@ async function findElementBySelector(
   return elementId;
 }
 
-async function findElement(sessionId: string, step: AppiumRecordedStepRecord, timeoutMs = step.timeoutMs ?? 10000) {
+async function findElement(sessionId: string, step: AppiumRecordedStepRecord, timeoutMs = step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS) {
   return withElementDeadline(timeoutMs, () => findElementWithinDeadline(sessionId, step));
 }
 
@@ -513,7 +520,7 @@ async function tapFallback(deviceId: string, step: AppiumRecordedStepRecord) {
 
 async function waitForElement(sessionId: string, step: AppiumRecordedStepRecord) {
   const startedAt = Date.now();
-  const timeoutMs = step.timeoutMs || 10000;
+  const timeoutMs = step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS;
   let lastError: unknown;
   while (Date.now() - startedAt <= timeoutMs) {
     try {
@@ -524,12 +531,12 @@ async function waitForElement(sessionId: string, step: AppiumRecordedStepRecord)
       await wait(500);
     }
   }
-  throw lastError instanceof Error ? lastError : new Error(`${step.label} 等待超时`);
+  throw new ConditionTimeoutError(`${step.label} 等待元素超时（${timeoutMs}ms）：${errorDetail(lastError)}`);
 }
 
 async function findOptionalElement(sessionId: string, step: AppiumRecordedStepRecord) {
   const startedAt = Date.now();
-  const timeoutMs = step.timeoutMs ?? 2000;
+  const timeoutMs = step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS;
   while (Date.now() - startedAt <= timeoutMs) {
     try {
       return await findElement(sessionId, step, Math.max(1, timeoutMs - (Date.now() - startedAt)));
@@ -543,7 +550,7 @@ async function findOptionalElement(sessionId: string, step: AppiumRecordedStepRe
 
 async function waitForElementGone(sessionId: string, step: AppiumRecordedStepRecord) {
   const startedAt = Date.now();
-  const timeoutMs = step.timeoutMs || 10000;
+  const timeoutMs = step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS;
   while (Date.now() - startedAt <= timeoutMs) {
     try {
       await findElement(sessionId, step, Math.max(1, timeoutMs - (Date.now() - startedAt)));
@@ -553,7 +560,7 @@ async function waitForElementGone(sessionId: string, step: AppiumRecordedStepRec
     }
     await wait(500);
   }
-  throw new Error(`${step.label} 等待消失超时`);
+  throw new ConditionTimeoutError(`${step.label} 等待消失超时`);
 }
 
 async function saveScreenshot(deviceId: string) {
@@ -998,7 +1005,7 @@ async function evaluateCondition(sessionId: string, deviceId: string, step: Appi
   }
   if (step.type === 'textClick') {
     const selector = textClickSelector(step);
-    const deadline = Date.now() + Math.max(0, step.timeoutMs ?? 10000);
+    const deadline = Date.now() + Math.max(0, step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS);
     // 查询空结果才是未匹配；通信错误、重复匹配和点击失败走现有异常处理。
     do {
       const payload = await appiumRequest<AppiumValueResponse<Record<string, string>[]>>(`/session/${sessionId}/elements`, {
@@ -1013,7 +1020,10 @@ async function evaluateCondition(sessionId: string, deviceId: string, step: Appi
         await appiumRequest(`/session/${sessionId}/element/${id}/click`, { method: 'POST', body: '{}' });
         return true;
       }
-      if (Date.now() >= deadline) return false;
+      if (Date.now() >= deadline) {
+        if (step.timeoutMs === 0) return false;
+        throw new ConditionTimeoutError(`文字点击等待匹配超时（${step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS}ms）`);
+      }
       await wait(Math.max(0, Math.min(500, deadline - Date.now())));
     } while (true);
   }
@@ -1029,7 +1039,7 @@ async function evaluateCondition(sessionId: string, deviceId: string, step: Appi
       return true;
     }
     if (step.type === 'assertText' || (step.type === 'assertExists' && step.value)) {
-      const elementId = await findElement(sessionId, step);
+      const elementId = await waitForElement(sessionId, step);
       const payload = await appiumRequest<AppiumValueResponse<string>>(`/session/${sessionId}/element/${elementId}/text`);
       const actualText = payload.value || '';
       const expectedText = step.value || '';
@@ -1041,7 +1051,7 @@ async function evaluateCondition(sessionId: string, deviceId: string, step: Appi
     return true;
   } catch (error) {
     throwIfReplayStopped(error);
-    return false;
+    throw error;
   }
 }
 
@@ -1117,7 +1127,7 @@ async function replayLinkedScript(
           type: 'waitActivity',
           label: step.label,
           value: linkedScript.appActivity,
-          timeoutMs: 10000,
+          timeoutMs: DEFAULT_NODE_TIMEOUT_MS,
         });
       } catch (error) {
         throwIfReplayStopped(error);
@@ -1251,13 +1261,7 @@ async function replayFlowSteps(
           }
           continue;
         }
-        // AI 节点使用本次执行前截图，并将同一张图片和识别依据写入结果帧。
-        const beforeFrame = replayContext.getStore()?.frames.at(-1);
-        const recognition = step.type === 'aiRecognition' ? await recognizeDeviceScreen({
-          deviceId, prompt: step.value, timeoutMs: step.timeoutMs,
-          signal: replayContext.getStore()?.signal,
-          imageBase64: beforeFrame?.nodeId === step.id && beforeFrame.phase === 'before' ? beforeFrame.imageBase64 : undefined,
-        }) : undefined;
+        if (step.type === 'aiRecognition') throw new Error('AI 识别操作已移除，请替换此节点后回放');
         let imageResult: ImageCheckResult | undefined;
         if (step.type === 'imageCheck') {
           const resolved = resolveVariableStep(step);
@@ -1266,13 +1270,22 @@ async function replayFlowSteps(
             resolveRegion: () => imageCheckRegion(sessionId, resolved), wait,
             signal: replayContext.getStore()?.signal, saveImages: !variableContext.getStore()?.privacy.enabled });
           const config = resolved.imageCheck;
-          const settings = config ? `${IMAGE_CHECK_MODES[config.mode]}；条件 ${config.expectation}；模板阈值 ${config.threshold}；最小得分差 ${config.minScoreGap}；RGB/亮度容差 ${config.tolerance}；目标色 ${config.color}；像素占比 ${config.ratio}%；观察 ${config.durationMs}ms；间隔 ${config.intervalMs}ms；连续 ${config.consecutive} 帧` : '缺少配置';
+          const settings = config ? [
+            IMAGE_CHECK_MODES[config.mode],
+            config.mode === 'template' ? `预期匹配结果：${config.expectation === 'present' ? '匹配到模板' : '未匹配到模板'}` : '',
+            ['template', 'state'].includes(config.mode) ? `匹配严格度 ${config.threshold * 100}%` : '',
+            config.mode === 'state' ? `最小得分差 ${config.minScoreGap}` : '',
+            ['black', 'color', 'change'].includes(config.mode) ? `RGB/亮度容差 ${config.tolerance}；像素占比 ${config.ratio}%` : '',
+            config.mode === 'color' ? `目标色 ${config.color}` : '',
+            config.mode === 'change' ? `预期：${config.expectation === 'present' ? '画面有变化' : '持续无明显变化'}` : '',
+            config.mode !== 'state' ? `观察 ${config.durationMs}ms；间隔 ${config.intervalMs}ms；连续 ${config.consecutive} 帧` : '',
+          ].filter(Boolean).join('；') : '缺少配置';
           replayContext.getStore()?.imageChecks.push({ ...imageResult, nodeId: step.id, nodeNumber: index + 1, nodeLabel: frameStep.label, scriptName: options.scriptName || '', settings });
           lines.push(`[节点 ${index + 1}] 图像判断：${imageResult.result === null ? '无法判定' : imageResult.result}，${imageResult.message}；${settings}；采样 ${imageResult.sampleCount} 帧，耗时 ${imageResult.durationMs}ms；指标 ${JSON.stringify(imageResult.metrics)}`);
           if (imageResult.result === null) throw new Error(imageResult.message);
+          if (imageResult.timedOut) throw new ConditionTimeoutError(imageResult.message);
         }
-        const matched = imageResult ? imageResult.result! : recognition ? recognition.result : await evaluateCondition(sessionId, deviceId, step);
-        if (recognition) lines.push(`[节点 ${index + 1}] AI 识别：${recognition.result}，耗时 ${recognition.durationMs}ms${recognition.reason ? `，依据：${recognition.reason}` : ''}`);
+        const matched = imageResult ? imageResult.result! : await evaluateCondition(sessionId, deviceId, step);
         const branchTargetId = matched ? step.flow?.yesTargetId : step.flow?.noTargetId;
         const next = traversal.branch(step, matched);
         const targetId = branchTargetId || (next === undefined ? '' : steps[next]?.id) || '';
@@ -1284,8 +1297,7 @@ async function replayFlowSteps(
           index + 1,
           options.scriptName || '',
           'after',
-          `判断：${branchLabel}${recognition?.reason ? `；${recognition.reason}` : ''}`,
-          recognition?.imageBase64,
+          `判断：${branchLabel}`,
         );
         index = next;
       } catch (error) {
@@ -1294,6 +1306,17 @@ async function replayFlowSteps(
           await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'stopped', '已终止');
           throw new ReplayStoppedError();
         }
+        if (error instanceof ConditionTimeoutError && (step.timeoutBranch === 'yes' || step.timeoutBranch === 'no')) {
+          const branch = step.timeoutBranch;
+          const status = `判断超时，按配置进入${branch === 'yes' ? '左' : '右'}侧分支（${flowBranchLabel(step, branch)}）`;
+          lines.push(`[节点 ${index + 1}] ${status}；${error.message}`);
+          await captureReplayFrame(sessionId, frameStep, index + 1, options.scriptName || '', 'after', status);
+          // 循环体必须经过 enter，才能正确记轮次及受最大次数保护。
+          index = step.type === 'loop'
+            ? branch === 'yes' ? traversal.enter(step).next : traversal.exit(step)
+            : traversal.branch(step, branch === 'yes');
+          continue;
+        }
         lines.push(`[节点 ${index + 1}] 失败：${errorDetail(error)}`);
         await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'error', '失败');
         await appendStepDiagnostics(lines, deviceId, step);
@@ -1413,7 +1436,7 @@ export async function replayAppiumScript(
   deviceId: string,
   onOutput?: (line: string) => void,
   signal?: AbortSignal,
-  runOptions: { parameters?: TestVariable[]; globalVariables?: TestVariable[]; linkedScripts?: AppiumRecordedScriptRecord[] } = {},
+  runOptions: { recordVideo?: boolean; parameters?: TestVariable[]; globalVariables?: TestVariable[]; linkedScripts?: AppiumRecordedScriptRecord[] } = {},
 ) {
   const targetDeviceId = deviceId || script.deviceId;
   if (!targetDeviceId) throw new Error('未检测到可用设备');
@@ -1421,6 +1444,12 @@ export async function replayAppiumScript(
 
   const scope = new VariableScope(runOptions.globalVariables ?? getPresetVariables(), script.variables || []);
   const scripts = new Map((runOptions.linkedScripts ?? linkedScriptSnapshot(script)).map(item => [item.id, item]));
+  // 旧节点保留图结构供用户替换，但必须在任何设备操作之前阻止回放。
+  for (const candidate of [script, ...scripts.values()]) {
+    if (candidate.steps.some(step => step.type === 'aiRecognition')) {
+      throw new Error(`脚本“${candidate.name}”包含已移除的 AI 识别操作，请改用图像判断或原生组件判断后回放`);
+    }
+  }
   for (const item of validateVariables(runOptions.parameters)) scope.set(item);
   // 提前登记子脚本敏感预设，避免进入子脚本前的原始日志或截图泄露。
   const visited = new Set<string>();
@@ -1473,6 +1502,8 @@ export async function replayAppiumScript(
     let sessionId = '';
     let success = false;
     let stopped = false;
+    let recording: Awaited<ReturnType<typeof startReplayVideo>> | undefined;
+    let video: ReplayVideo | undefined;
     let managedAppium: Awaited<ReturnType<typeof startManagedAppiumServer>> | null = null;
     try {
       validateLoopSteps(script.steps);
@@ -1497,6 +1528,12 @@ export async function replayAppiumScript(
         sessionId = await createSession(targetDeviceId);
       }
       lines.push(`Appium session 已创建：${sessionId}`);
+      if (runOptions.recordVideo) {
+        if (scope.privacy.enabled) throw new Error('本次运行包含敏感变量，为避免泄露已禁止录屏，请关闭录制回放视频');
+        lines.push('正在启动后台录屏...');
+        recording = await startReplayVideo(targetDeviceId, signal);
+        lines.push('后台录屏已启动（MP4 / H.264 / 15fps / 2Mbps，无音频）');
+      }
       if (hasFlowSteps(script.steps)) lines.push('按流程图路径回放...');
       await replayScriptSteps(sessionId, targetDeviceId, script.steps, lines, [script.id], {
         scriptName: script.name,
@@ -1524,6 +1561,19 @@ export async function replayAppiumScript(
         lines.push(`回放终止：${detail}`);
       }
     } finally {
+      if (recording) {
+        try {
+          video = await recording.stop();
+          if (scope.privacy.enabled) {
+            await import('node:fs/promises').then(fs => fs.rm(video!.filePath, { force: true }));
+            video = undefined;
+            lines.push('敏感变量保护：已删除回放视频');
+          } else {
+            lines.push(`回放视频已保存：${video.filePath}`);
+            if (video.warning) lines.push(`录屏警告：${video.warning}`);
+          }
+        } catch (error) { lines.push(`回放视频保存失败：${errorDetail(error)}`); }
+      }
       let managedAppiumStopped = false;
       if (signal?.aborted && managedAppium) {
         await managedAppium.stop();
@@ -1562,6 +1612,7 @@ export async function replayAppiumScript(
         success,
         stopped,
         output: scope.redact(lines.join('\n')),
+        video,
         startedAt,
         completedAt,
         frames: safeFrames,
@@ -1586,9 +1637,11 @@ export async function replayAppiumScript(
       reportId,
       logPath,
       htmlReportPath,
+      videoPath: video?.filePath,
       softFailureCount: context.softFailureCount,
       history: {
         scriptId: script.id, scriptName: script.name, appPackage: script.appPackage,
+        video,
         appVersion, deviceId: targetDeviceId, startedAt: startedAt.toISOString(),
         durationMs: completedAt.getTime() - startedAt.getTime(),
         status: stopped ? 'stopped' as const : success ? 'passed' as const : 'failed' as const,

package/server/appium-recorder/condition-timeout.ts

@@ -0,0 +1,4 @@
+// 仅业务等待到期使用此错误；设备、通信和配置异常不能按超时分支放行。
+export class ConditionTimeoutError extends Error {}
+
+export class AppiumServiceError extends Error {}

package/server/appium-recorder/image-check.ts

@@ -1,4 +1,6 @@
 import { PNG } from 'pngjs';
+import { AppiumRequestTimeoutError } from './request-timeout';
+import { AppiumServiceError } from './condition-timeout';
 import { validateImageCheck, type ImageCheckConfig, type ImageCheckResult } from '../../src/appium-recorder/image-check';
 import type { AppiumVisualChangeRegion } from '../../src/appium-recorder/types';
 
@@ -126,6 +128,10 @@ export async function checkImage(input: {
       if (!baseline) { baseline = current; observedAt = now(); keep('首帧', current); }
       const measurement = await measure(current, baseline, templates, config);
       result.metrics = { ...measurement.metrics };
+      if (config.mode === 'template') {
+        const score = result.metrics['模板得分']!;
+        result.templateMatch = { matched: score >= config.threshold, expected: config.expectation, score, threshold: config.threshold };
+      }
       input.signal?.throwIfAborted();
       if (config.mode === 'state') {
         if (measurement.matched === null) throw new Error('双模板均未达到阈值或得分过于接近');
@@ -138,20 +144,28 @@ export async function checkImage(input: {
         const changed = maximum >= config.ratio;
         if (changed || (result.sampleCount >= 2 && now() - observedAt >= config.durationMs)) {
           result.result = config.expectation === 'present' ? changed : !changed;
+          result.timedOut = !changed && config.expectation === 'present' && config.durationMs > 0;
           break;
         }
       } else {
         consecutive = measurement.matched ? consecutive + 1 : 0;
         result.metrics['连续满足帧数'] = consecutive;
         if (consecutive >= config.consecutive) { result.result = true; break; }
-        if (now() - observedAt >= config.durationMs) { result.result = false; break; }
+        if (now() - observedAt >= config.durationMs) { result.result = false; result.timedOut = config.durationMs > 0; break; }
       }
       await input.wait(Math.min(config.intervalMs, Math.max(1, config.durationMs - (now() - observedAt))));
     }
     if (last) keep('判定帧', last);
-    result.message = result.result ? '条件成立' : '条件不成立';
+    result.message = result.timedOut ? '观察超时，未在规定时间内满足条件' : result.result ? '条件成立' : '条件不成立';
+    if (result.templateMatch) {
+      const match = result.templateMatch;
+      const actual = match.matched ? '匹配到模板' : '未匹配到模板';
+      const expected = match.expected === 'present' ? '匹配到模板' : '未匹配到模板';
+      result.message = `实际${actual}：得分 ${(match.score * 100).toFixed(4)}%，${match.matched ? '达到' : '低于'}匹配严格度 ${(match.threshold * 100).toFixed(4)}%；预期${expected}；${result.message}${result.timedOut ? '' : ` → ${result.result} 分支`}`;
+    }
   } catch (error) {
     if (input.signal?.aborted) throw error;
+    if (error instanceof AppiumRequestTimeoutError || error instanceof AppiumServiceError) throw error;
     result.message = `无法判定：${error instanceof Error ? error.message : '图像检测异常'}`;
     result.result = null;
   }

package/server/appium-recorder/replay-mp4.ts

@@ -0,0 +1,72 @@
+import { EncodedPacket, EncodedVideoPacketSource, FilePathTarget, Mp4OutputFormat, Output } from 'mediabunny';
+import { h264ParseConfiguration, type ScrcpyMediaStreamPacket } from '@yume-chan/scrcpy';
+
+// 仅封装 H.264，不解码或重新编码；最多保留最后一帧以计算静止画面的持续时间。
+export class ReplayMp4Writer {
+  private source = new EncodedVideoPacketSource('avc');
+  private output: Output;
+  private config?: ReturnType<typeof h264ParseConfiguration>;
+  private configBytes?: Uint8Array;
+  private origin?: bigint;
+  private pending?: { data: Uint8Array; key: boolean; timestamp: number };
+  private first = true;
+  startedAt = 0;
+  private startedMonotonic = 0;
+  private endedMonotonic?: number;
+
+  constructor(path: string) {
+    this.output = new Output({ target: new FilePathTarget(path, { chunkSize: 1024 * 1024 }), format: new Mp4OutputFormat({ fastStart: 'fragmented' }) });
+    this.output.addVideoTrack(this.source);
+  }
+
+  async start() { await this.output.start(); }
+
+  async add(packet: ScrcpyMediaStreamPacket) {
+    if (packet.type === 'configuration') {
+      const config = h264ParseConfiguration(packet.data);
+      if (this.configBytes && !Buffer.from(this.configBytes).equals(Buffer.from(packet.data))) {
+        throw new Error('录屏编码配置发生变化，已保留变化前的视频');
+      }
+      this.config = config;
+      this.configBytes = packet.data.slice();
+      return;
+    }
+    if (!this.config || !this.configBytes) throw new Error('录屏缺少 H.264 配置信息');
+    if (packet.pts === undefined) throw new Error('录屏视频包缺少时间戳');
+    if (typeof packet.keyframe !== 'boolean') throw new Error('录屏视频包缺少关键帧标记');
+    if (this.origin === undefined) {
+      if (!packet.keyframe) return;
+      this.origin = packet.pts;
+      this.startedAt = Date.now();
+      this.startedMonotonic = performance.now();
+    }
+    const timestamp = Number(packet.pts - this.origin) / 1000000;
+    if (!Number.isFinite(timestamp) || timestamp < 0 || (this.pending && timestamp < this.pending.timestamp)) throw new Error('录屏时间戳异常');
+    if (this.pending) await this.flush(Math.max(0.000001, timestamp - this.pending.timestamp));
+    const data = this.first ? Buffer.concat([this.configBytes, packet.data]) : packet.data.slice();
+    this.pending = { data, key: packet.keyframe, timestamp };
+  }
+
+  private async flush(duration: number) {
+    if (!this.pending || !this.config) return;
+    const { data, key, timestamp } = this.pending;
+    const c = this.config;
+    const codec = `avc1.${[c.profileIndex, c.constraintSet, c.levelIndex].map(v => v.toString(16).padStart(2, '0')).join('')}`;
+    await this.source.add(new EncodedPacket(data, key ? 'key' : 'delta', timestamp, duration), this.first ? {
+      decoderConfig: { codec, codedWidth: c.croppedWidth, codedHeight: c.croppedHeight },
+    } : undefined);
+    this.first = false;
+    this.pending = undefined;
+  }
+
+  end() { this.endedMonotonic ??= performance.now(); }
+
+  async finish() {
+    if (!this.startedAt) throw new Error('未收到可录制的关键帧');
+    await this.flush(Math.max(1 / 15, ((this.endedMonotonic ?? performance.now()) - this.startedMonotonic) / 1000 - (this.pending?.timestamp || 0)));
+    this.source.close();
+    await this.output.finalize();
+  }
+
+  async cancel() { await this.output.cancel(); }
+}