android-midscene-automation 0.1.33 → 0.1.35

package/CHANGELOG.md

@@ -1,4 +1,15 @@
 # 更新记录
+## v0.1.35
+- 修复已有图像判断配置无效时导致整个脚本列表加载失败的问题，读取保留原数据，保存和执行时再校验。
+- 图像判断配置显示模板尺寸，提前提示模板超出检测区域，并支持手动一键扩大框选区域；回放尺寸不匹配时展示具体尺寸，避免仅在运行时得到笼统错误。
+- 新增“图像判断”分支节点，支持本地 OpenCV 模板匹配、图片勾选双模板、黑屏、区域颜色和多帧变化检测；支持组件定位、框选区域、模板采集与上传、连续采样及 true/false 分支，证据不足时明确报告无法判定。HTML 报告展示指标和可放大的检测图片，并沿用敏感运行截图保护。
+- 合流后的公共节点上方新增插入入口，支持新增和粘贴操作，所有汇入分支先执行新操作再进入原公共节点。
+- 修复嵌套判断显式合流到外层公共节点时连接被过滤、公共节点提前排列的问题，支持跨层分支合流。
+
+## v0.1.34
+- 修复同一分支内显式跳转被画布误画为顺序连接的问题，失效跳转目标不再错误连接到相邻节点。
+- 修复判断分支默认返回公共后续节点时画布漏连线、节点重叠的问题，嵌套分支显示与回放路径保持一致。
+
 ## v0.1.33
 - 操作菜单移除“断言存在”和“断言文本”，统一保留“判断存在”及其文本匹配配置；旧脚本中的断言节点仍兼容原有执行行为。
 - “点击坐标”节点支持编辑 X、Y 坐标，保存后回放使用新坐标；默认节点名称同步更新，自定义名称保持不变。

package/README.md

@@ -1229,6 +1229,17 @@ Linux: ~/.local/share/android-midscene-automation
 
 # 项目更新记录
 
+## v0.1.35
+- 修复已有图像判断配置无效时导致整个脚本列表加载失败的问题，读取保留原数据，保存和执行时再校验。
+- 图像判断配置显示模板尺寸，提前提示模板超出检测区域，并支持手动一键扩大框选区域；回放尺寸不匹配时展示具体尺寸，避免仅在运行时得到笼统错误。
+- 新增“图像判断”分支节点，支持本地 OpenCV 模板匹配、图片勾选双模板、黑屏、区域颜色和多帧变化检测；支持组件定位、框选区域、模板采集与上传、连续采样及 true/false 分支，证据不足时明确报告无法判定。HTML 报告展示指标和可放大的检测图片，并沿用敏感运行截图保护。
+- 合流后的公共节点上方新增插入入口，支持新增和粘贴操作，所有汇入分支先执行新操作再进入原公共节点。
+- 修复嵌套判断显式合流到外层公共节点时连接被过滤、公共节点提前排列的问题，支持跨层分支合流。
+
+## v0.1.34
+- 修复同一分支内显式跳转被画布误画为顺序连接的问题，失效跳转目标不再错误连接到相邻节点。
+- 修复判断分支默认返回公共后续节点时画布漏连线、节点重叠的问题，嵌套分支显示与回放路径保持一致。
+
 ## v0.1.33
 - 操作菜单移除“断言存在”和“断言文本”，统一保留“判断存在”及其文本匹配配置；旧脚本中的断言节点仍兼容原有执行行为。
 - “点击坐标”节点支持编辑 X、Y 坐标，保存后回放使用新坐标；默认节点名称同步更新，自定义名称保持不变。

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "android-midscene-automation",
-  "version": "0.1.33",
+  "version": "0.1.35",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/oooooooko/android-midscene-automation.git"
@@ -44,6 +44,7 @@
     "@midscene/playground": "https://registry.npmjs.org/@midscene/playground/-/playground-1.12.5.tgz",
     "@midscene/shared": "https://registry.npmjs.org/@midscene/shared/-/shared-1.12.5.tgz",
     "@midscene/web": "https://registry.npmjs.org/@midscene/web/-/web-1.12.5.tgz",
+    "@techstark/opencv-js": "5.0.0-release.1",
     "@vitejs/plugin-vue": "6.0.7",
     "@vue-flow/core": "1.48.2",
     "archiver": "7.0.1",

package/server/appium-recorder/appium-runner.ts

@@ -7,6 +7,8 @@ import { BoundedLoopTraversal } from './bounded-loop';
 import { validateLoopSteps } from '../../src/appium-recorder/bounded-loop';
 import { adbScreenshotBase64 } from './screenshot';
 import { recognizeDeviceScreen } from './ai-recognition';
+import { checkImage } from './image-check';
+import { IMAGE_CHECK_MODES, type ImageCheckResult } from '../../src/appium-recorder/image-check';
 import { formatStageLog } from '../../src/appium-recorder/stage-log';
 import { openGalleryOnDevice } from './open-gallery';
 import { textClickSelector } from '../../src/appium-recorder/text-click';
@@ -66,6 +68,7 @@ const replayContext = new AsyncLocalStorage<{
   frames: AppiumReplayFrame[];
   historyEvents: AppiumReplayFrame[];
   visualChecks: AppiumReplayVisualCheck[];
+  imageChecks: Array<ImageCheckResult & { nodeId: string; nodeNumber: number; nodeLabel: string; scriptName: string; settings: string }>;
   pendingVisualChecks: PendingVisualChangeCheck[];
   visualStartOffsets: Map<string, number>;
   softFailureCount: number;
@@ -470,6 +473,26 @@ async function findElement(sessionId: string, step: AppiumRecordedStepRecord) {
   }
 }
 
+// 图像判断不能使用通用定位的“第一个匹配”及坐标回退，父级和子级均须唯一。
+async function imageCheckRegion(sessionId: string, step: AppiumRecordedStepRecord) {
+  async function unique(selector: AppiumRecordedStepRecord['selector'], parent?: string) {
+    if (!selector || selector.strategy === 'bounds') throw new Error('缺少可唯一定位的组件信息');
+    const path = parent ? `/session/${sessionId}/element/${parent}/elements` : `/session/${sessionId}/elements`;
+    const payload = await appiumRequest<AppiumValueResponse<Array<Record<string, string>>>>(path, {
+      method: 'POST', body: JSON.stringify(toAppiumUsing(selector)),
+    });
+    if (payload.value?.length !== 1) throw new Error(`目标组件匹配 ${payload.value?.length || 0} 个，请限定父级或条目文字`);
+    const id = payload.value[0]?.[ELEMENT_KEY] || payload.value[0]?.ELEMENT;
+    if (!id) throw new Error('目标组件缺少元素标识');
+    return id;
+  }
+  const parent = step.contextSelector ? await unique(step.contextSelector) : undefined;
+  const id = await unique(step.selector, parent);
+  const rect = await appiumRequest<AppiumValueResponse<VisualChangeRegion>>(`/session/${sessionId}/element/${id}/rect`);
+  if (!rect.value) throw new Error('无法读取组件实时区域');
+  return rect.value;
+}
+
 async function tapFallback(deviceId: string, step: AppiumRecordedStepRecord) {
   if (step.fallback?.strategy !== 'bounds' || !Number.isFinite(step.fallback.centerX) || !Number.isFinite(step.fallback.centerY)) {
     throw new Error(`${step.label} 未找到元素，且没有可用坐标兜底`);
@@ -1220,7 +1243,20 @@ async function replayFlowSteps(
           signal: replayContext.getStore()?.signal,
           imageBase64: beforeFrame?.nodeId === step.id && beforeFrame.phase === 'before' ? beforeFrame.imageBase64 : undefined,
         }) : undefined;
-        const matched = recognition ? recognition.result : await evaluateCondition(sessionId, deviceId, step);
+        let imageResult: ImageCheckResult | undefined;
+        if (step.type === 'imageCheck') {
+          const resolved = resolveVariableStep(step);
+          imageResult = await checkImage({ config: resolved.imageCheck!,
+            capture: async () => Buffer.from(await adbScreenshotBase64(deviceId, replayContext.getStore()?.signal), 'base64'),
+            resolveRegion: () => imageCheckRegion(sessionId, resolved), wait,
+            signal: replayContext.getStore()?.signal, saveImages: !variableContext.getStore()?.privacy.enabled });
+          const config = resolved.imageCheck;
+          const settings = config ? `${IMAGE_CHECK_MODES[config.mode]}；条件 ${config.expectation}；模板阈值 ${config.threshold}；最小得分差 ${config.minScoreGap}；RGB/亮度容差 ${config.tolerance}；目标色 ${config.color}；像素占比 ${config.ratio}%；观察 ${config.durationMs}ms；间隔 ${config.intervalMs}ms；连续 ${config.consecutive} 帧` : '缺少配置';
+          replayContext.getStore()?.imageChecks.push({ ...imageResult, nodeId: step.id, nodeNumber: index + 1, nodeLabel: frameStep.label, scriptName: options.scriptName || '', settings });
+          lines.push(`[节点 ${index + 1}] 图像判断：${imageResult.result === null ? '无法判定' : imageResult.result}，${imageResult.message}；${settings}；采样 ${imageResult.sampleCount} 帧，耗时 ${imageResult.durationMs}ms；指标 ${JSON.stringify(imageResult.metrics)}`);
+          if (imageResult.result === null) throw new Error(imageResult.message);
+        }
+        const matched = imageResult ? imageResult.result! : recognition ? recognition.result : await evaluateCondition(sessionId, deviceId, step);
         if (recognition) lines.push(`[节点 ${index + 1}] AI 识别：${recognition.result}，耗时 ${recognition.durationMs}ms${recognition.reason ? `，依据：${recognition.reason}` : ''}`);
         const branchTargetId = matched ? step.flow?.yesTargetId : step.flow?.noTargetId;
         const next = traversal.branch(step, matched);
@@ -1407,6 +1443,7 @@ export async function replayAppiumScript(
     frames: [] as AppiumReplayFrame[],
     historyEvents: [] as AppiumReplayFrame[],
     visualChecks: [] as AppiumReplayVisualCheck[],
+    imageChecks: [] as Array<ImageCheckResult & { nodeId: string; nodeNumber: number; nodeLabel: string; scriptName: string; settings: string }>,
     visualStartOffsets: new Map<string, number>(),
     pendingVisualChecks: [] as PendingVisualChangeCheck[],
     softFailureCount: 0,
@@ -1514,6 +1551,7 @@ export async function replayAppiumScript(
         completedAt,
         frames: safeFrames,
         visualChecks: safeChecks,
+        imageChecks: scope.scrub(context.imageChecks.map(check => scope.privacy.enabled ? { ...check, images: [] } : check)),
       });
       reportPath = report.filePath;
       reportId = report.id;

package/server/appium-recorder/image-check.ts

@@ -0,0 +1,160 @@
+import { PNG } from 'pngjs';
+import { validateImageCheck, type ImageCheckConfig, type ImageCheckResult } from '../../src/appium-recorder/image-check';
+import type { AppiumVisualChangeRegion } from '../../src/appium-recorder/types';
+
+// 延迟初始化，普通回放不加载 WASM；仅缓存运行库，不缓存设备截图或检测结果。
+let runtime: Promise<typeof import('@techstark/opencv-js')> | undefined;
+async function openCv() {
+  runtime ??= import('@techstark/opencv-js').then(async module => {
+    const cv = await module.default;
+    if (!cv.Mat) await new Promise<void>(resolve => { cv.onRuntimeInitialized = resolve; });
+    return cv;
+  });
+  return runtime;
+}
+
+export function readImage(buffer: Buffer) {
+  // 先检查 IHDR 尺寸，再解码，防止超大图片耗尽内存。
+  if (buffer.length < 24 || buffer.length > 24 * 1024 * 1024 || buffer.subarray(0, 8).toString('hex') !== '89504e470d0a1a0a') throw new Error('截图或模板不是有效 PNG');
+  const width = buffer.readUInt32BE(16), height = buffer.readUInt32BE(20);
+  if (!width || !height || width * height > 16000000) throw new Error('图片尺寸过大或无效');
+  return PNG.sync.read(buffer);
+}
+
+export function cropImage(source: PNG, region: AppiumVisualChangeRegion) {
+  const { x, y, width, height } = region;
+  if (![x, y, width, height].every(Number.isInteger) || x < 0 || y < 0 || width < 1 || height < 1
+    || x + width > source.width || y + height > source.height || width * height > 4000000) throw new Error('目标区域无效、越界或超过 400 万像素');
+  const image = new PNG({ width, height });
+  PNG.bitblt(source, image, x, y, width, height, 0, 0);
+  return image;
+}
+
+async function measure(image: PNG, baseline: PNG | undefined, templates: PNG[], config: ImageCheckConfig) {
+  const cv = await openCv();
+  const mat = cv.matFromArray(image.height, image.width, cv.CV_8UC4, image.data);
+  const rgb = new cv.Mat();
+  try {
+    cv.cvtColor(mat, rgb, cv.COLOR_RGBA2RGB);
+    if (config.mode === 'template' || config.mode === 'state') {
+      const scores = templates.map(template => {
+        if (template.width > image.width || template.height > image.height) throw new Error(`模板 ${template.width}×${template.height} 超过实时检测区域 ${image.width}×${image.height}，请扩大检测区域或重新采集对应组件的模板`);
+        const raw = cv.matFromArray(template.height, template.width, cv.CV_8UC4, template.data);
+        const target = new cv.Mat(), output = new cv.Mat(), mask = new cv.Mat();
+        try {
+          cv.cvtColor(raw, target, cv.COLOR_RGBA2RGB);
+          // SQDIFF 不会把零方差的纯色模板错误识别成满分；统一转换为越大越匹配的得分。
+          cv.matchTemplate(rgb, target, output, cv.TM_SQDIFF);
+          const distance = cv.minMaxLoc(output, mask).minVal / (template.width * template.height * 3 * 255 * 255);
+          return Math.max(0, Math.min(1, 1 - Math.sqrt(Math.max(0, distance))));
+        } finally { raw.delete(); target.delete(); output.delete(); mask.delete(); }
+      });
+      const positive = scores[0]!;
+      if (config.mode === 'template') return { matched: config.expectation === 'present' ? positive >= config.threshold : positive < config.threshold, metrics: { '模板得分': positive } };
+      const negative = scores[1]!;
+      const gap = Math.abs(positive - negative);
+      return { matched: Math.max(positive, negative) < config.threshold || gap < config.minScoreGap ? null : positive > negative,
+        metrics: { '选中得分': positive, '未选中得分': negative, '得分差': gap } };
+    }
+    const mask = new cv.Mat();
+    try {
+      if (config.mode === 'black') {
+        const gray = new cv.Mat();
+        try { cv.cvtColor(rgb, gray, cv.COLOR_RGB2GRAY); cv.threshold(gray, mask, config.tolerance, 255, cv.THRESH_BINARY_INV); }
+        finally { gray.delete(); }
+      } else if (config.mode === 'color') {
+        const channels = [1, 3, 5].map(index => parseInt(config.color.slice(index, index + 2), 16));
+        const low = new cv.Mat(rgb.rows, rgb.cols, rgb.type(), new cv.Scalar(...channels.map(v => Math.max(0, v - config.tolerance)), 0));
+        const high = new cv.Mat(rgb.rows, rgb.cols, rgb.type(), new cv.Scalar(...channels.map(v => Math.min(255, v + config.tolerance)), 255));
+        try { cv.inRange(rgb, low, high, mask); } finally { low.delete(); high.delete(); }
+      } else {
+        if (!baseline || baseline.width !== image.width || baseline.height !== image.height) throw new Error('采样期间目标区域尺寸发生变化');
+        const first = cv.matFromArray(baseline.height, baseline.width, cv.CV_8UC4, baseline.data);
+        const diff = new cv.Mat();
+        try {
+          cv.absdiff(mat, first, diff);
+          // 按 RGB 任一通道差异统计，避免不同颜色在灰度化后抵消。
+          const bytes = new Uint8Array(image.width * image.height);
+          for (let i = 0; i < bytes.length; i++) bytes[i] = Math.max(diff.data[i * 4]!, diff.data[i * 4 + 1]!, diff.data[i * 4 + 2]!) > config.tolerance ? 255 : 0;
+          const binary = cv.matFromArray(image.height, image.width, cv.CV_8UC1, bytes);
+          try { binary.copyTo(mask); } finally { binary.delete(); }
+        } finally { first.delete(); diff.delete(); }
+      }
+      const ratio = cv.countNonZero(mask) / (image.width * image.height) * 100;
+      return { matched: ratio >= config.ratio, metrics: { [config.mode === 'black' ? '暗色像素占比 %' : config.mode === 'color' ? '目标颜色占比 %' : '变化比例 %']: ratio } };
+    } finally { mask.delete(); }
+  } finally { mat.delete(); rgb.delete(); }
+}
+
+export async function checkImage(input: {
+  config: ImageCheckConfig;
+  capture: () => Promise<Buffer>;
+  resolveRegion?: () => Promise<AppiumVisualChangeRegion>;
+  signal?: AbortSignal;
+  wait: (ms: number) => Promise<void>;
+  now?: () => number;
+  saveImages?: boolean;
+}): Promise<ImageCheckResult> {
+  const now = input.now || Date.now, started = now();
+  const config = input.config;
+  const result: ImageCheckResult = { result: null, mode: config?.mode, region: config?.region, durationMs: 0, sampleCount: 0, metrics: {}, images: [], message: '' };
+  try {
+    validateImageCheck(config);
+    await openCv();
+    const templates = ['template', 'state'].includes(config.mode)
+      ? [config.template!, ...(config.mode === 'state' ? [config.negativeTemplate!] : [])].map(value => readImage(Buffer.from(value, 'base64'))) : [];
+    for (const template of templates) {
+      for (let i = 3; i < template.data.length; i += 4) if (template.data[i] !== 255) throw new Error('模板含透明像素，请使用设备实际截图采集');
+    }
+    let baseline: PNG | undefined, last: PNG | undefined, consecutive = 0;
+    let maximum = 0, observedAt = now();
+    const keep = (label: string, image: PNG) => {
+      if (input.saveImages !== false) result.images.push({ label, base64: PNG.sync.write(image).toString('base64') });
+    };
+    templates.forEach((image, index) => keep(index ? '未选中模板' : config.mode === 'state' ? '选中模板' : '模板', image));
+    while (true) {
+      input.signal?.throwIfAborted();
+      const region = config.target === 'element' ? await input.resolveRegion?.() : config.region;
+      if (!region) throw new Error('未找到目标元素');
+      const source = readImage(await input.capture());
+      input.signal?.throwIfAborted();
+      if (config.target === 'region' && (source.width !== config.screenWidth || source.height !== config.screenHeight)) throw new Error('屏幕尺寸与采集时不一致，请重新框选区域');
+      const current = cropImage(source, region);
+      result.region = region;
+      result.sampleCount++;
+      last = current;
+      if (!baseline) { baseline = current; observedAt = now(); keep('首帧', current); }
+      const measurement = await measure(current, baseline, templates, config);
+      result.metrics = { ...measurement.metrics };
+      input.signal?.throwIfAborted();
+      if (config.mode === 'state') {
+        if (measurement.matched === null) throw new Error('双模板均未达到阈值或得分过于接近');
+        result.result = measurement.matched;
+        break;
+      }
+      if (config.mode === 'change') {
+        maximum = Math.max(maximum, result.metrics['变化比例 %'] || 0);
+        result.metrics['最大变化比例 %'] = maximum;
+        const changed = maximum >= config.ratio;
+        if (changed || (result.sampleCount >= 2 && now() - observedAt >= config.durationMs)) {
+          result.result = config.expectation === 'present' ? changed : !changed;
+          break;
+        }
+      } else {
+        consecutive = measurement.matched ? consecutive + 1 : 0;
+        result.metrics['连续满足帧数'] = consecutive;
+        if (consecutive >= config.consecutive) { result.result = true; break; }
+        if (now() - observedAt >= config.durationMs) { result.result = false; break; }
+      }
+      await input.wait(Math.min(config.intervalMs, Math.max(1, config.durationMs - (now() - observedAt))));
+    }
+    if (last) keep('判定帧', last);
+    result.message = result.result ? '条件成立' : '条件不成立';
+  } catch (error) {
+    if (input.signal?.aborted) throw error;
+    result.message = `无法判定：${error instanceof Error ? error.message : '图像检测异常'}`;
+    result.result = null;
+  }
+  result.durationMs = now() - started;
+  return result;
+}

package/server/appium-recorder/report.ts

@@ -5,6 +5,9 @@ import { appDataPath } from '../paths';
 import { loadConfig } from '../config';
 import { saveAppiumReplayReport, type AppiumRecordedScriptRecord, type AppiumRecordedStepRecord } from './repository';
 import { flowBranchLabel } from '../../src/appium-recorder/flow-labels';
+import type { ImageCheckResult } from '../../src/appium-recorder/image-check';
+
+type ReportImageCheck = ImageCheckResult & { nodeId: string; nodeNumber: number; nodeLabel: string; scriptName: string; settings: string };
 
 export type AppiumReplayFrame = {
   sequence: number;
@@ -245,6 +248,7 @@ function createReplayHtml(input: {
   completedAt: Date;
   frames: AppiumReplayFrame[];
   visualChecks: AppiumReplayVisualCheck[];
+  imageChecks?: ReportImageCheck[];
   output: string;
 }) {
   const imageCache = new Map<string, string>();
@@ -266,6 +270,9 @@ function createReplayHtml(input: {
       diffBase64: undefined,
     })),
     output: input.output,
+    imageChecks: (input.imageChecks || []).map(check => ({ ...check,
+      images: check.images.map(image => ({ label: image.label, url: htmlImageUrl(image.base64, imageCache) })),
+    })),
   });
   const resultClass = input.resultText === '成功' ? 'success' : input.resultText === '已终止' ? 'stopped' : 'failed';
   return `<!doctype html>
@@ -387,6 +394,7 @@ function createReplayHtml(input: {
         </dl>
       </section>
       <nav id="step-list" class="step-list" aria-label="执行步骤"></nav>
+      <details open><summary>图像判断</summary><div id="image-checks" class="visual-check-list"></div></details>
       <details><summary>视觉变化检测</summary><div id="visual-checks" class="visual-check-list"></div></details>
       <details><summary>完整回放日志</summary><pre id="log"></pre></details>
     </aside>
@@ -587,6 +595,37 @@ function createReplayHtml(input: {
       thumbnail.addEventListener('click', (event) => { event.stopPropagation(); pause(); seekTo(frame.offsetMs); });
       elements['timeline-track'].append(thumbnail);
     });
+    const imageChecksElement = document.querySelector('#image-checks');
+    if (!(data.imageChecks || []).length) imageChecksElement.parentElement.hidden = true;
+    (data.imageChecks || []).forEach(check => {
+      const article = document.createElement('article');
+      article.className = 'visual-check';
+      const title = document.createElement('h3');
+      title.textContent = check.scriptName + ' · ' + check.nodeNumber + '. ' + check.nodeLabel;
+      article.append(title);
+      const detail = document.createElement('dl');
+      const region = check.region || {};
+      const entries = [['结果', check.result === null ? '无法判定' : String(check.result)], ['配置', check.settings],
+        ['区域', [region.x, region.y, region.width, region.height].join(', ')],
+        ['采样', check.sampleCount + ' 帧'], ['耗时', check.durationMs + 'ms'], ['说明', check.message], ...Object.entries(check.metrics)];
+      entries.forEach(([key, value]) => {
+        const dt = document.createElement('dt'), dd = document.createElement('dd');
+        dt.textContent = key; dd.textContent = String(value); detail.append(dt, dd);
+      });
+      article.append(detail);
+      const images = document.createElement('div'); images.className = 'visual-images';
+      check.images.forEach(image => {
+        if (!image.url) return;
+        const figure = document.createElement('div'); figure.className = 'visual-image';
+        const caption = document.createElement('span'); caption.textContent = image.label;
+        const img = document.createElement('img'); img.src = image.url; img.alt = image.label; img.title = '点击放大'; img.tabIndex = 0;
+        const open = () => openImagePreview(image.url, check.nodeLabel + ' · ' + image.label);
+        img.addEventListener('click', open);
+        img.addEventListener('keydown', event => { if (event.key === 'Enter' || event.key === ' ') { event.preventDefault(); open(); } });
+        figure.append(caption, img); images.append(figure);
+      });
+      article.append(images); imageChecksElement.append(article);
+    });
     if (!visualChecks.length) {
       elements['visual-checks'].textContent = '本次没有视觉变化检测节点';
     }
@@ -689,6 +728,7 @@ export async function createAppiumReplayReport(input: {
   completedAt: Date;
   frames?: AppiumReplayFrame[];
   visualChecks?: AppiumReplayVisualCheck[];
+  imageChecks?: ReportImageCheck[];
 }) {
   const configuredOutputPath = loadConfig().runtime.reportOutputPath.trim();
   const outputDir = appDataPath(configuredOutputPath || 'output');
@@ -721,6 +761,7 @@ export async function createAppiumReplayReport(input: {
     completedAt: input.completedAt,
     frames: input.frames || [],
     visualChecks,
+    imageChecks: input.imageChecks,
     output: persistedOutput,
   });
   const markdown = [

package/server/appium-recorder/repository.ts

@@ -10,6 +10,7 @@ import { normalizeLegacyNestedConditionBranches } from '../../src/appium-recorde
 import { validateLoopSteps } from '../../src/appium-recorder/bounded-loop';
 import { validateVariables, validateExtraction, validateReturns, type TestVariable, type VariableExtraction, type ScriptReturn } from '../../src/appium-recorder/variables';
 import { getPresetVariables, savePresetVariables, deleteScriptVariables } from './variable-store';
+import { validateImageCheck, type ImageCheckConfig } from '../../src/appium-recorder/image-check';
 
 export type AppiumRecordedStepRecord = {
   id: string;
@@ -26,6 +27,7 @@ export type AppiumRecordedStepRecord = {
     | 'checkedState'
     | 'radioButtonState'
     | 'aiRecognition'
+    | 'imageCheck'
     | 'textClick'
     | 'key'
     | 'waitActivity'
@@ -89,6 +91,7 @@ export type AppiumRecordedStepRecord = {
   };
   value?: string;
   logPrefix?: string;
+  imageCheck?: ImageCheckConfig;
   extractVariable?: VariableExtraction;
   parameters?: TestVariable[];
   returns?: ScriptReturn[];
@@ -369,6 +372,8 @@ export function saveAppiumRecordedScript(input: {
   const steps = normalizeLegacyNestedConditionBranches(input.steps || []);
   if (input.variables !== undefined) validateVariables(input.variables);
   for (const step of steps) {
+    // 读取旧脚本必须容错，让用户能打开并修正无效配置；仅在写入时校验。
+    if (step.type === 'imageCheck') validateImageCheck(step.imageCheck);
     if (step.type === 'extractVariable') validateExtraction(step.extractVariable);
     if (step.type === 'runScript') { validateVariables(step.parameters); validateReturns(step.returns); }
   }

package/server/appium-recorder/routes.ts

@@ -3,6 +3,10 @@ import { getPresetVariables, savePresetVariables } from './variable-store';
 import type { TestVariable } from '../../src/appium-recorder/variables';
 import { deleteRunHistory, getRunHistory, listRunHistory, saveRunHistory } from './run-history';
 import { recognizeDeviceScreen } from './ai-recognition';
+import { readImage, cropImage } from './image-check';
+import { PNG } from 'pngjs';
+import { adbScreenshotBase64 } from './screenshot';
+import type { AppiumVisualChangeRegion } from '../../src/appium-recorder/types';
 import { readFreshWindowHierarchy } from './tree-dump';
 import type { IncomingMessage, ServerResponse } from 'node:http';
 import {
@@ -180,6 +184,21 @@ export async function handleAppiumRecorderRequest(
       return true;
     }
 
+    if (pathname === '/api/appium-recorder/image-check/capture' && req.method === 'POST') {
+      const parsed = await readBody<{ deviceId: string; region: AppiumVisualChangeRegion; screenWidth: number; screenHeight: number }>(req);
+      const deviceId = parsed.deviceId?.trim() || selectedDeviceId;
+      if (!deviceId) throw new Error('请选择设备');
+      assertDeviceAllowed(deviceId);
+      if (isRemoteDeviceId(deviceId)) throw new Error('远程设备暂不支持图像模板采集');
+      if (replayingDevices.has(deviceId)) { sendJson(res, { message: '设备正在回放，不能采集模板' }, 409); return true; }
+      const source = readImage(Buffer.from(await adbScreenshotBase64(deviceId), 'base64'));
+      if (source.width !== parsed.screenWidth || source.height !== parsed.screenHeight) throw new Error('设备屏幕尺寸已变化，请刷新预览后重新框选');
+      const base64 = PNG.sync.write(cropImage(source, parsed.region)).toString('base64');
+      if (base64.length > 4 * 1024 * 1024) throw new Error('模板区域过大，请缩小到目标图标');
+      sendJson(res, { base64, screenWidth: source.width, screenHeight: source.height });
+      return true;
+    }
+
     if (pathname === '/api/appium-recorder/ai-recognition/test' && req.method === 'POST') {
       const parsed = await readBody<{ deviceId?: string; prompt?: unknown; timeoutMs?: number }>(req);
       const deviceId = parsed.deviceId?.trim() || selectedDeviceId;