android-midscene-automation 0.1.26 → 0.1.28

package/server/appium-recorder/appium-runner.ts

@@ -1,12 +1,20 @@
 import { execFile } from 'node:child_process';
+import { adbScreenshotBase64 } from './screenshot';
+import { recognizeDeviceScreen } from './ai-recognition';
+import { formatStageLog } from '../../src/appium-recorder/stage-log';
+import { textClickSelector } from '../../src/appium-recorder/text-click';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { mkdir, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { getAppiumRecordedScript, type AppiumRecordedScriptRecord, type AppiumRecordedStepRecord } from './repository';
 import { appDataPath } from '../paths';
 import { ensureAndroidSdkAvailable, getAdbCommand } from '../android-sdk';
-import { createAppiumReplayReport, type AppiumReplayFrame } from './report';
+import { createAppiumReplayReport, type AppiumReplayFrame, type AppiumReplayVisualCheck } from './report';
 import { startManagedAppiumServer, usesManagedAppiumServer } from './managed-appium';
+import { compareVisualChangeFrames, type VisualChangeRegion } from './visual-change';
+import { longPressMode, validateLongPress } from '../../src/appium-recorder/long-press';
+import { defaultFlowKind, flowBranchLabel } from '../../src/appium-recorder/flow-labels';
+import { isNativeStateCondition, nativeControlName, nativeControlType, parseNativeBoolean } from '../../src/appium-recorder/native-control-state';
 
 type AppiumSessionResponse = {
   value?: {
@@ -34,11 +42,21 @@ class ReplayStoppedError extends Error {
   }
 }
 
+type PendingVisualChangeCheck = {
+  step: AppiumRecordedStepRecord;
+  nodeNumber: number;
+  scriptName: string;
+};
+
 const replayContext = new AsyncLocalStorage<{
   signal?: AbortSignal;
   appiumLog: (line: string) => void;
   serverUrl: string;
+  deviceId: string;
   frames: AppiumReplayFrame[];
+  visualChecks: AppiumReplayVisualCheck[];
+  pendingVisualChecks: PendingVisualChangeCheck[];
+  softFailureCount: number;
 }>();
 
 const appiumServerUrl = () => replayContext.getStore()?.serverUrl || configuredAppiumServerUrl();
@@ -60,6 +78,11 @@ function errorDetail(error: unknown) {
   return error.message;
 }
 
+function isTimeoutError(error: unknown) {
+  const detail = errorDetail(error).toLowerCase();
+  return detail.includes('timeout') || detail.includes('timed out');
+}
+
 function isStaleElementError(error: unknown) {
   const detail = errorDetail(error);
   return detail.includes('stale element reference') || detail.includes('does not exist in DOM anymore');
@@ -254,6 +277,9 @@ async function appiumRequest<T>(
       throw new ReplayStoppedError();
     }
     context?.appiumLog(`[Appium] 异常：${method} ${path}（${errorDetail(error)}）`);
+    if (isTimeoutError(error)) {
+      throw new Error(`Appium 请求超时 ${requestUrl}，${errorDetail(error)}`);
+    }
     throw new Error(`无法连接 Appium 服务 ${requestUrl}，${errorDetail(error)}`);
   }
 
@@ -284,6 +310,7 @@ async function appiumRequest<T>(
 }
 
 function replayFrameSelector(step: AppiumRecordedStepRecord) {
+  if (step.type === 'textClick') return `${step.flow?.textMatch === 'exact' ? '精准匹配' : '模糊匹配'}：${step.value || ''}`;
   const selector = step.selector;
   if (!selector) return '-';
   if (selector.strategy === 'bounds') return `bounds (${selector.centerX ?? '-'}, ${selector.centerY ?? '-'})`;
@@ -291,15 +318,16 @@ function replayFrameSelector(step: AppiumRecordedStepRecord) {
 }
 
 async function captureReplayFrame(
-  sessionId: string,
+  _sessionId: string,
   step: AppiumRecordedStepRecord,
   nodeNumber: number,
   scriptName: string,
   phase: AppiumReplayFrame['phase'],
   status: string,
+  imageBase64?: string,
 ) {
   const context = replayContext.getStore();
-  if (!context || !sessionId) return;
+  if (!context || !context.deviceId) return;
   const appendFrame = (imageBase64: string) => {
     context.frames.push({
       sequence: context.frames.length + 1,
@@ -308,6 +336,7 @@ async function captureReplayFrame(
       nodeNumber,
       nodeLabel: step.label,
       nodeType: step.type,
+      logContent: step.type === 'log' && phase === 'after' ? formatStageLog(step) : undefined,
       note: step.note || '',
       selector: replayFrameSelector(step),
       phase,
@@ -324,14 +353,8 @@ async function captureReplayFrame(
   }
 
   try {
-    const payload = await appiumRequest<AppiumValueResponse<string>>(
-      `/session/${sessionId}/screenshot`,
-      undefined,
-      { ignoreAbort: true, timeoutMs: 5000 },
-    );
-    const imageBase64 = typeof payload.value === 'string' ? payload.value : '';
-    if (!imageBase64) throw new Error('Appium 未返回截图内容');
-    appendFrame(imageBase64);
+    // Report screenshots use ADB so a slow capture cannot block Appium's command queue.
+    appendFrame(imageBase64 || await adbScreenshotBase64(context.deviceId));
   } catch (error) {
     context.appiumLog(`[节点 ${nodeNumber}] 截图失败：${errorDetail(error)}`);
   }
@@ -443,8 +466,7 @@ async function waitForElement(sessionId: string, step: AppiumRecordedStepRecord)
   let lastError: unknown;
   while (Date.now() - startedAt <= timeoutMs) {
     try {
-      await findElement(sessionId, step);
-      return;
+      return await findElement(sessionId, step);
     } catch (error) {
       throwIfReplayStopped(error);
       lastError = error;
@@ -483,16 +505,183 @@ async function waitForElementGone(sessionId: string, step: AppiumRecordedStepRec
   throw new Error(`${step.label} 等待消失超时`);
 }
 
-async function saveScreenshot(sessionId: string) {
-  const payload = await appiumRequest<AppiumValueResponse<string>>(`/session/${sessionId}/screenshot`);
-  if (!payload.value) throw new Error('Appium 未返回截图数据');
+async function saveScreenshot(deviceId: string) {
   const dir = appDataPath('.midscene-app', 'screenshots');
   await mkdir(dir, { recursive: true });
   const file = join(dir, `appium-${Date.now()}.png`);
-  await writeFile(file, Buffer.from(payload.value, 'base64'));
+  await writeFile(file, Buffer.from(await adbScreenshotBase64(deviceId), 'base64'));
+}
+
+type RunStepMeta = {
+  nodeNumber: number;
+  scriptName: string;
+};
+
+type RunStepResult = string | {
+  message?: string;
+  softFailed?: boolean;
+};
+
+function completedStepStatus(step: AppiumRecordedStepRecord, softFailed: boolean) {
+  if (step.type === 'visualChange') {
+    if (step.visualChange?.role === 'start') return '已记录基准帧';
+    if (step.visualChange?.role === 'end') return '已记录对比帧';
+  }
+  return softFailed ? '视觉变化未达预期' : '成功';
+}
+
+function visualChangeNumber(value: unknown, fallback: number, min: number, max = Number.POSITIVE_INFINITY) {
+  const number = Number(value);
+  if (!Number.isFinite(number)) return fallback;
+  return Math.min(max, Math.max(min, number));
+}
+
+function visualChangeRegion(step: AppiumRecordedStepRecord): VisualChangeRegion {
+  const config = step.visualChange;
+  if (!config?.region) throw new Error(`${step.label} 缺少检测区域`);
+  return {
+    x: Math.max(0, Math.round(Number(config.region.x) || 0)),
+    y: Math.max(0, Math.round(Number(config.region.y) || 0)),
+    width: Math.max(1, Math.round(Number(config.region.width) || 1)),
+    height: Math.max(1, Math.round(Number(config.region.height) || 1)),
+  };
 }
 
-async function runStep(sessionId: string, deviceId: string, step: AppiumRecordedStepRecord) {
+function findVisualFrame(
+  frames: AppiumReplayFrame[],
+  nodeId: string,
+  scriptName: string,
+  preferredPhase: AppiumReplayFrame['phase'],
+) {
+  const matches = frames.filter((frame) => frame.nodeId === nodeId && frame.scriptName === scriptName);
+  if (!matches.length) return undefined;
+  const preferred = preferredPhase === 'before'
+    ? matches.find((frame) => frame.phase === preferredPhase)
+    : [...matches].reverse().find((frame) => frame.phase === preferredPhase);
+  return preferred || (preferredPhase === 'before' ? matches[0] : matches[matches.length - 1]);
+}
+
+const EMPTY_PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=';
+
+function registerVisualChangeStep(
+  step: AppiumRecordedStepRecord,
+  meta: RunStepMeta,
+): RunStepResult {
+  const config = step.visualChange;
+  if (!config) throw new Error(`${step.label} 缺少画面变化检测配置`);
+  if (config.role === 'start') {
+    return {
+      message: `已记录画面变化基准点：${config.pairLabel || step.label}`,
+    };
+  }
+  const context = replayContext.getStore();
+  const endStep = {
+    ...step,
+    visualChange: {
+      ...config,
+      endStepId: config.endStepId || step.id,
+    },
+  };
+  context?.pendingVisualChecks.push({ step: endStep, nodeNumber: meta.nodeNumber, scriptName: meta.scriptName });
+  return {
+    message: `已记录画面变化对比点：${config.pairLabel || step.label}`,
+  };
+}
+
+function appendVisualCheckFailure(lines: string[], check: PendingVisualChangeCheck, message: string) {
+  const context = replayContext.getStore();
+  if (context) {
+    const config = check.step.visualChange;
+    context.softFailureCount += 1;
+    context.visualChecks.push({
+      nodeId: check.step.id,
+      nodeNumber: check.nodeNumber,
+      nodeLabel: check.step.label,
+      status: 'failed',
+      message,
+      region: config?.region ? visualChangeRegion(check.step) : { x: 0, y: 0, width: 1, height: 1 },
+      durationMs: 0,
+      intervalMs: 0,
+      sampleCount: 0,
+      changeRatioThreshold: visualChangeNumber(config?.changeRatioThreshold, 2, 0.01),
+      maxChangeRatio: 0,
+      baselineBase64: EMPTY_PNG_BASE64,
+      comparisonBase64: EMPTY_PNG_BASE64,
+      diffBase64: EMPTY_PNG_BASE64,
+      startNodeId: config?.startStepId,
+      endNodeId: config?.endStepId,
+    });
+  }
+  lines.push(`[节点 ${check.nodeNumber}] 视觉变化检测异常：${message}（继续执行）`);
+}
+
+function processPendingVisualChecks(lines: string[]) {
+  const context = replayContext.getStore();
+  if (!context || !context.pendingVisualChecks.length) return;
+
+  // Visual checks compare captured replay frames after the path finishes, so
+  // they never block or mutate the device state for downstream nodes.
+  context.pendingVisualChecks.forEach((check) => {
+    const config = check.step.visualChange;
+    if (!config?.startStepId || !config.endStepId) {
+      appendVisualCheckFailure(lines, check, `${check.step.label} 缺少开始节点或结束节点`);
+      return;
+    }
+    const baselineFrame = findVisualFrame(context.frames, config.startStepId, check.scriptName, 'after');
+    const comparisonFrame = findVisualFrame(context.frames, config.endStepId, check.scriptName, 'after');
+    if (!baselineFrame || !comparisonFrame) {
+      const missing = [
+        baselineFrame ? '' : '开始节点截图',
+        comparisonFrame ? '' : '结束节点截图',
+      ].filter(Boolean).join('、');
+      appendVisualCheckFailure(lines, check, `${check.step.label} 未捕获到${missing}`);
+      return;
+    }
+    let result: ReturnType<typeof compareVisualChangeFrames>;
+    try {
+      result = compareVisualChangeFrames({
+        baselineScreenshot: Buffer.from(baselineFrame.imageBase64, 'base64'),
+        comparisonScreenshot: Buffer.from(comparisonFrame.imageBase64, 'base64'),
+        region: visualChangeRegion(check.step),
+        changeRatioThreshold: visualChangeNumber(config.changeRatioThreshold, 2, 0.01),
+        pixelmatchThreshold: visualChangeNumber(config.pixelmatchThreshold, 0.1, 0, 1),
+      });
+    } catch (error) {
+      appendVisualCheckFailure(lines, check, `${check.step.label} 对比失败：${errorDetail(error)}`);
+      return;
+    }
+    context.visualChecks.push({
+      nodeId: check.step.id,
+      nodeNumber: check.nodeNumber,
+      nodeLabel: check.step.label,
+      status: result.passed ? 'passed' : 'failed',
+      message: result.message,
+      region: result.region,
+      durationMs: result.durationMs,
+      intervalMs: result.intervalMs,
+      sampleCount: result.sampleCount,
+      changeRatioThreshold: result.changeRatioThreshold,
+      maxChangeRatio: result.maxChangeRatio,
+      baselineBase64: result.baselineBase64,
+      comparisonBase64: result.comparisonBase64,
+      diffBase64: result.diffBase64,
+      startNodeId: config.startStepId,
+      startNodeLabel: `${baselineFrame.nodeNumber}. ${baselineFrame.nodeLabel}`,
+      endNodeId: config.endStepId,
+      endNodeLabel: `${comparisonFrame.nodeNumber}. ${comparisonFrame.nodeLabel}`,
+    });
+    if (!result.passed) context.softFailureCount += 1;
+    lines.push(`[节点 ${check.nodeNumber}] 视觉变化检测${result.passed ? '有变化' : '无明显变化'}：${result.message}${result.passed ? '' : '（继续执行）'}`);
+  });
+  context.pendingVisualChecks = [];
+}
+
+async function runStep(
+  sessionId: string,
+  deviceId: string,
+  step: AppiumRecordedStepRecord,
+  meta: RunStepMeta,
+): Promise<RunStepResult | void> {
   throwIfReplayStopped();
   if (step.type === 'delay') {
     await wait(Math.max(0, step.timeoutMs || 1000));
@@ -528,10 +717,19 @@ async function runStep(sessionId: string, deviceId: string, step: AppiumRecorded
   }
 
   if (step.type === 'screenshot') {
-    await saveScreenshot(sessionId);
+    await saveScreenshot(deviceId);
     return;
   }
 
+  if (step.type === 'visualChange') {
+    return registerVisualChangeStep(step, meta);
+  }
+
+  if (step.type === 'noop') {
+    return;
+  }
+  if (step.type === 'log') return formatStageLog(step);
+
   if (step.type === 'coordinateTap') {
     await tapFallback(deviceId, step);
     return;
@@ -544,13 +742,21 @@ async function runStep(sessionId: string, deviceId: string, step: AppiumRecorded
   }
 
   if (step.type === 'longPress') {
-    if (step.fallback?.strategy !== 'bounds' || !Number.isFinite(step.fallback.centerX) || !Number.isFinite(step.fallback.centerY)) {
-      throw new Error(`${step.label} 缺少长按坐标`);
+    const error = validateLongPress(step);
+    if (error) throw new Error(`${step.label}：${error}`);
+    const duration = step.timeoutMs ?? 800;
+    if (longPressMode(step) === 'element') {
+      const elementId = await findElement(sessionId, step);
+      await appiumRequest(`/session/${sessionId}/execute/sync`, {
+        method: 'POST',
+        body: JSON.stringify({ script: 'mobile: longClickGesture', args: [{ elementId, duration }] }),
+      });
+      return `已长按元素 ${duration}ms`;
     }
-    const x = Number(step.fallback.centerX);
-    const y = Number(step.fallback.centerY);
-    await adbSwipe(deviceId, { startX: x, startY: y, endX: x, endY: y, duration: step.timeoutMs || 800 });
-    return;
+    const x = Number(step.fallback!.centerX);
+    const y = Number(step.fallback!.centerY);
+    await adbSwipe(deviceId, { startX: x, startY: y, endX: x, endY: y, duration });
+    return `已长按坐标 ${x},${y} ${duration}ms`;
   }
 
   if (step.type === 'pinch') {
@@ -681,7 +887,48 @@ async function runStep(sessionId: string, deviceId: string, step: AppiumRecorded
   }
 }
 
+async function readNativeControlState(sessionId: string, step: AppiumRecordedStepRecord) {
+  if (!isNativeStateCondition(step)) throw new Error('不是原生控件状态判断节点');
+  const elementId = await waitForElement(sessionId, step);
+  const attribute = async (name: string) => (
+    await appiumRequest<AppiumValueResponse<unknown>>(`/session/${sessionId}/element/${elementId}/attribute/${name}`)
+  ).value;
+  const className = await attribute('className');
+  if (nativeControlType(className) !== step.type) {
+    throw new Error(`${step.label} 目标不是原生 ${nativeControlName(step.type)}，实际类型：${String(className ?? '未知')}`);
+  }
+  if (parseNativeBoolean(await attribute('checkable')) !== true) {
+    throw new Error(`${step.label} 目标不支持原生勾选状态（checkable 不为 true）`);
+  }
+  const checked = parseNativeBoolean(await attribute('checked'));
+  if (checked === undefined) throw new Error(`${step.label} 无法读取有效的 checked 属性`);
+  return checked;
+}
+
 async function evaluateCondition(sessionId: string, deviceId: string, step: AppiumRecordedStepRecord) {
+  if (step.type === 'textClick') {
+    const selector = textClickSelector(step);
+    const deadline = Date.now() + Math.max(0, step.timeoutMs ?? 10000);
+    // 查询空结果才是未匹配；通信错误、重复匹配和点击失败走现有异常处理。
+    do {
+      const payload = await appiumRequest<AppiumValueResponse<Record<string, string>[]>>(`/session/${sessionId}/elements`, {
+        method: 'POST', body: JSON.stringify(selector),
+      });
+      if (!Array.isArray(payload.value)) throw new Error('文字点击查询返回了无效结果');
+      if (payload.value.length > 1) throw new Error(`匹配到 ${payload.value.length} 个文字元素，请使用更精确的文字`);
+      if (payload.value.length === 1) {
+        const element = payload.value[0]!;
+        const id = element[ELEMENT_KEY] || element.ELEMENT;
+        if (!id) throw new Error('文字点击查询缺少元素 ID');
+        await appiumRequest(`/session/${sessionId}/element/${id}/click`, { method: 'POST', body: '{}' });
+        return true;
+      }
+      if (Date.now() >= deadline) return false;
+      await wait(Math.max(0, Math.min(500, deadline - Date.now())));
+    } while (true);
+  }
+  // 未找到控件、类型错误和属性读取异常不等于未勾选，必须交给回放错误处理。
+  if (isNativeStateCondition(step)) return readNativeControlState(sessionId, step);
   try {
     if (step.type === 'waitActivity') {
       await waitForActivity(deviceId, step);
@@ -710,7 +957,7 @@ async function evaluateCondition(sessionId: string, deviceId: string, step: Appi
 
 function hasFlowSteps(steps: AppiumRecordedStepRecord[]) {
   return steps.some((step) => (
-    step.flow?.nodeKind === 'condition'
+    defaultFlowKind(step) === 'condition'
     || Boolean(step.flow?.yesTargetId)
     || Boolean(step.flow?.noTargetId)
     || Boolean(step.flow?.successTargetId)
@@ -797,6 +1044,20 @@ async function replayLinkedScript(
   lines.push(`连接脚本完成：${linkedScript.name}`);
 }
 
+function appendRunStepResult(lines: string[], nodeNumber: number, result: RunStepResult | void) {
+  if (!result) return false;
+  if (typeof result === 'string') {
+    if (result) lines.push(`[节点 ${nodeNumber}] 结果：${result}`);
+    return false;
+  }
+  if (result.softFailed) {
+    lines.push(`[节点 ${nodeNumber}] 视觉变化未达预期：${result.message || '未检测到明显变化'}（继续执行）`);
+    return true;
+  }
+  if (result.message) lines.push(`[节点 ${nodeNumber}] 结果：${result.message}`);
+  return false;
+}
+
 async function replayLinearSteps(
   sessionId: string,
   deviceId: string,
@@ -814,25 +1075,24 @@ async function replayLinearSteps(
     lines.push(`[节点 ${index + 1}] 开始：${step.label}`);
     await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'before', '执行前');
     try {
+      let softFailed = false;
       if (step.type === 'runScript') {
         await replayLinkedScript(sessionId, deviceId, step, lines, stack);
       } else {
-        const result = await runStep(sessionId, deviceId, step);
-        if (result) lines.push(`[节点 ${index + 1}] 结果：${result}`);
+        const result = await runStep(sessionId, deviceId, step, {
+          nodeNumber: index + 1,
+          scriptName: options.scriptName || '',
+        });
+        softFailed = appendRunStepResult(lines, index + 1, result);
       }
-      lines.push(`[节点 ${index + 1}] 完成：${step.label}`);
-      await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'after', '成功');
+      lines.push(`[节点 ${index + 1}] 完成：${step.label}${softFailed ? '（存在软失败，已继续）' : ''}`);
+      await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'after', completedStepStatus(step, softFailed));
     } catch (error) {
       if (isReplayStopped(error)) {
         lines.push(`[节点 ${index + 1}] 已终止：${step.label}`);
         await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'stopped', '已终止');
         throw new ReplayStoppedError();
       }
-      if (step.optional) {
-        lines.push(`[节点 ${index + 1}] 跳过：${errorDetail(error)}`);
-        await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'error', '已跳过');
-        continue;
-      }
       lines.push(`[节点 ${index + 1}] 失败：${errorDetail(error)}`);
       await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'error', '失败');
       await appendStepDiagnostics(lines, deviceId, step);
@@ -884,23 +1144,33 @@ async function replayFlowSteps(
       continue;
     }
     visitedPath.push(step.label);
-    const nodeKind = step.flow?.nodeKind || (step.type === 'assertExists' || step.type === 'assertText' ? 'assertion' : 'action');
+    const nodeKind = defaultFlowKind(step);
     lines.push(`[节点 ${index + 1}] 开始：${step.label}`);
     await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'before', '执行前');
 
     if (nodeKind === 'condition') {
       try {
-        const matched = await evaluateCondition(sessionId, deviceId, step);
+        // AI 节点使用本次执行前截图，并将同一张图片和识别依据写入结果帧。
+        const beforeFrame = replayContext.getStore()?.frames.at(-1);
+        const recognition = step.type === 'aiRecognition' ? await recognizeDeviceScreen({
+          deviceId, prompt: step.value, timeoutMs: step.timeoutMs,
+          signal: replayContext.getStore()?.signal,
+          imageBase64: beforeFrame?.nodeId === step.id && beforeFrame.phase === 'before' ? beforeFrame.imageBase64 : undefined,
+        }) : undefined;
+        const matched = recognition ? recognition.result : await evaluateCondition(sessionId, deviceId, step);
+        if (recognition) lines.push(`[节点 ${index + 1}] AI 识别：${recognition.result}，耗时 ${recognition.durationMs}ms${recognition.reason ? `，依据：${recognition.reason}` : ''}`);
         const branchTargetId = matched ? step.flow?.yesTargetId : step.flow?.noTargetId;
         const targetId = branchTargetId || step.flow?.successTargetId || '';
-        lines.push(`[节点 ${index + 1}] 判断：${matched ? '是' : '否'}${targetId ? `，进入 ${targetId}` : '，流程结束'}`);
+        const branchLabel = flowBranchLabel(step, matched ? 'yes' : 'no');
+        lines.push(`[节点 ${index + 1}] 判断：${branchLabel}${isNativeStateCondition(step) ? `（checked=${matched}）` : ''}${targetId ? `，进入 ${targetId}` : '，流程结束'}`);
         await captureReplayFrame(
           sessionId,
           step,
           index + 1,
           options.scriptName || '',
           'after',
-          `判断：${matched ? '是' : '否'}`,
+          `判断：${branchLabel}${recognition?.reason ? `；${recognition.reason}` : ''}`,
+          recognition?.imageBase64,
         );
         index = targetId ? idToIndex.get(targetId) : undefined;
       } catch (error) {
@@ -909,20 +1179,27 @@ async function replayFlowSteps(
           await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'stopped', '已终止');
           throw new ReplayStoppedError();
         }
+        lines.push(`[节点 ${index + 1}] 失败：${errorDetail(error)}`);
+        await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'error', '失败');
+        await appendStepDiagnostics(lines, deviceId, step);
         throw error;
       }
       continue;
     }
 
     try {
+      let softFailed = false;
       if (step.type === 'runScript') {
         await replayLinkedScript(sessionId, deviceId, step, lines, stack);
       } else {
-        const result = await runStep(sessionId, deviceId, step);
-        if (result) lines.push(`[节点 ${index + 1}] 结果：${result}`);
+        const result = await runStep(sessionId, deviceId, step, {
+          nodeNumber: index + 1,
+          scriptName: options.scriptName || '',
+        });
+        softFailed = appendRunStepResult(lines, index + 1, result);
       }
-      lines.push(`[节点 ${index + 1}] 完成：${step.label}`);
-      await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'after', '成功');
+      lines.push(`[节点 ${index + 1}] 完成：${step.label}${softFailed ? '（存在软失败，已继续）' : ''}`);
+      await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'after', completedStepStatus(step, softFailed));
       index = nextIndexAfterStep(index, step);
     } catch (error) {
       if (isReplayStopped(error)) {
@@ -930,12 +1207,6 @@ async function replayFlowSteps(
         await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'stopped', '已终止');
         throw new ReplayStoppedError();
       }
-      if (step.optional) {
-        lines.push(`[节点 ${index + 1}] 跳过：${errorDetail(error)}`);
-        await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'error', '已跳过');
-        index = nextIndexAfterStep(index, step);
-        continue;
-      }
       const failureTarget = step.flow?.failureTargetId;
       if (failureTarget && idToIndex.has(failureTarget)) {
         lines.push(`[节点 ${index + 1}] 失败分支：${errorDetail(error)}`);
@@ -997,7 +1268,7 @@ async function replayScriptSteps(
     try {
       await replayLinkedScript(sessionId, deviceId, step, lines, stack);
       lines.push(`[节点 ${nodeNumber}] 完成：${step.label}`);
-      await captureReplayFrame(sessionId, step, nodeNumber, options.scriptName || '', 'after', '成功');
+      await captureReplayFrame(sessionId, step, nodeNumber, options.scriptName || '', 'after', completedStepStatus(step, false));
     } catch (error) {
       if (isReplayStopped(error)) {
         lines.push(`[节点 ${nodeNumber}] 已终止：${step.label}`);
@@ -1033,7 +1304,11 @@ export async function replayAppiumScript(
     signal,
     appiumLog: (line) => lines.push(line),
     serverUrl: configuredAppiumServerUrl(),
+    deviceId: targetDeviceId,
     frames: [] as AppiumReplayFrame[],
+    visualChecks: [] as AppiumReplayVisualCheck[],
+    pendingVisualChecks: [] as PendingVisualChangeCheck[],
+    softFailureCount: 0,
   };
   return replayContext.run(context, async () => {
     lines.push(
@@ -1071,8 +1346,14 @@ export async function replayAppiumScript(
       await replayScriptSteps(sessionId, targetDeviceId, script.steps, lines, [script.id], {
         scriptName: script.name,
       });
-      lines.push('回放完成');
-      success = true;
+      processPendingVisualChecks(lines);
+      if (context.softFailureCount > 0) {
+        lines.push(`回放完成，但存在 ${context.softFailureCount} 个视觉变化未达预期节点`);
+        success = false;
+      } else {
+        lines.push('回放完成');
+        success = true;
+      }
     } catch (error) {
       stopped = isReplayStopped(error);
       const detail = errorDetail(error);
@@ -1126,6 +1407,7 @@ export async function replayAppiumScript(
         startedAt,
         completedAt,
         frames: context.frames,
+        visualChecks: context.visualChecks,
       });
       reportPath = report.filePath;
       reportId = report.id;
@@ -1145,6 +1427,7 @@ export async function replayAppiumScript(
       reportId,
       logPath,
       htmlReportPath,
+      softFailureCount: context.softFailureCount,
     };
   });
 }