android-midscene-automation 0.1.27 → 0.1.28

package/server/appium-recorder/report.ts

@@ -1,8 +1,10 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { PNG } from 'pngjs';
 import { appDataPath } from '../paths';
 import { loadConfig } from '../config';
 import { saveAppiumReplayReport, type AppiumRecordedScriptRecord, type AppiumRecordedStepRecord } from './repository';
+import { flowBranchLabel } from '../../src/appium-recorder/flow-labels';
 
 export type AppiumReplayFrame = {
   sequence: number;
@@ -11,6 +13,7 @@ export type AppiumReplayFrame = {
   nodeNumber: number;
   nodeLabel: string;
   nodeType: string;
+  logContent?: string;
   note: string;
   selector: string;
   phase: 'before' | 'after' | 'error' | 'stopped';
@@ -45,6 +48,9 @@ export type AppiumReplayVisualCheck = {
   diffBase64: string;
 };
 
+const HTML_REPORT_IMAGE_MAX_WIDTH = 540;
+const HTML_REPORT_IMAGE_MAX_HEIGHT = 1200;
+
 function pad(value: number, length = 2) {
   return String(value).padStart(length, '0');
 }
@@ -81,6 +87,54 @@ function jsonForHtml(value: unknown) {
     .replace(/&/g, '\\u0026');
 }
 
+function compressHtmlPngBase64(base64: string) {
+  if (!base64) return base64;
+
+  try {
+    const source = PNG.sync.read(Buffer.from(base64, 'base64'));
+    const scale = Math.min(
+      1,
+      HTML_REPORT_IMAGE_MAX_WIDTH / source.width,
+      HTML_REPORT_IMAGE_MAX_HEIGHT / source.height,
+    );
+
+    if (scale >= 1) return base64;
+
+    const width = Math.max(1, Math.round(source.width * scale));
+    const height = Math.max(1, Math.round(source.height * scale));
+    const target = new PNG({ width, height });
+    const scaleX = source.width / width;
+    const scaleY = source.height / height;
+
+    // HTML 回放只用于预览，降采样能明显降低报告体积，不影响实际检测结果。
+    for (let y = 0; y < height; y += 1) {
+      const sourceY = Math.min(source.height - 1, Math.floor(y * scaleY));
+      for (let x = 0; x < width; x += 1) {
+        const sourceX = Math.min(source.width - 1, Math.floor(x * scaleX));
+        const sourceIndex = (sourceY * source.width + sourceX) << 2;
+        const targetIndex = (y * width + x) << 2;
+        target.data[targetIndex] = source.data[sourceIndex];
+        target.data[targetIndex + 1] = source.data[sourceIndex + 1];
+        target.data[targetIndex + 2] = source.data[sourceIndex + 2];
+        target.data[targetIndex + 3] = source.data[sourceIndex + 3];
+      }
+    }
+
+    return PNG.sync.write(target).toString('base64');
+  } catch {
+    return base64;
+  }
+}
+
+function htmlImageUrl(base64: string, cache: Map<string, string>) {
+  const cached = cache.get(base64);
+  if (cached) return cached;
+
+  const imageUrl = `data:image/png;base64,${compressHtmlPngBase64(base64)}`;
+  cache.set(base64, imageUrl);
+  return imageUrl;
+}
+
 function markdownValue(value: unknown) {
   if (value === undefined || value === null || value === '') return '-';
   return String(value).replace(/\|/g, '\\|').replace(/\r?\n/g, '<br>');
@@ -99,10 +153,12 @@ function stepExecutionLines(outputLines: string[], index: number) {
 
 function executionStatus(lines: string[]) {
   if (!lines.length) return '未执行';
-  if (lines.some((line) => line.includes('失败：') || line.includes('失败分支：'))) return '失败';
-  if (lines.some((line) => line.includes('跳过：'))) return '跳过';
-  if (lines.some((line) => line.includes('判断：'))) return '判断完成';
-  if (lines.some((line) => line.includes('完成：'))) return '成功';
+  // 只识别执行器的状态前缀，用户日志中的“失败”等文字不是执行结果。
+  const messages = lines.map((line) => line.replace(/^\[(?:节点|步骤) \d+\] /, ''));
+  if (messages.some((line) => line.startsWith('失败：') || line.startsWith('失败分支：'))) return '失败';
+  if (messages.some((line) => line.startsWith('跳过：'))) return '跳过';
+  if (messages.some((line) => line.startsWith('判断：'))) return '判断完成';
+  if (messages.some((line) => line.startsWith('完成：'))) return '成功';
   return '中断';
 }
 
@@ -157,9 +213,8 @@ function stepSection(
     `| 上下文 Selector | ${markdownValue(selectorText(step.contextSelector))} |`,
     `| 输入/目标值 | ${markdownValue(step.value)} |`,
     `| 超时时间 | ${markdownValue(step.timeoutMs === undefined ? '-' : `${step.timeoutMs}ms`)} |`,
-    `| 可选步骤 | ${step.optional ? '是' : '否'} |`,
-    `| 是分支 | ${markdownValue(step.flow?.yesTargetId)} |`,
-    `| 否分支 | ${markdownValue(step.flow?.noTargetId)} |`,
+    `| ${flowBranchLabel(step, 'yes')} 分支 | ${markdownValue(step.flow?.yesTargetId)} |`,
+    `| ${flowBranchLabel(step, 'no')} 分支 | ${markdownValue(step.flow?.noTargetId)} |`,
     `| 成功后续节点 | ${markdownValue(step.flow?.successTargetId)} |`,
     `| 执行状态 | **${executionStatus(executionLines)}** |`,
     '',
@@ -192,19 +247,20 @@ function createReplayHtml(input: {
   visualChecks: AppiumReplayVisualCheck[];
   output: string;
 }) {
+  const imageCache = new Map<string, string>();
   const payload = jsonForHtml({
     startedAt: input.startedAt.toISOString(),
     durationMs: input.durationMs,
     frames: input.frames.map((frame) => ({
       ...frame,
-      imageUrl: `data:image/png;base64,${frame.imageBase64}`,
+      imageUrl: htmlImageUrl(frame.imageBase64, imageCache),
       imageBase64: undefined,
     })),
     visualChecks: input.visualChecks.map((check) => ({
       ...check,
-      baselineImageUrl: `data:image/png;base64,${check.baselineBase64}`,
-      comparisonImageUrl: `data:image/png;base64,${check.comparisonBase64}`,
-      diffImageUrl: `data:image/png;base64,${check.diffBase64}`,
+      baselineImageUrl: htmlImageUrl(check.baselineBase64, imageCache),
+      comparisonImageUrl: htmlImageUrl(check.comparisonBase64, imageCache),
+      diffImageUrl: htmlImageUrl(check.diffBase64, imageCache),
       baselineBase64: undefined,
       comparisonBase64: undefined,
       diffBase64: undefined,
@@ -246,6 +302,8 @@ function createReplayHtml(input: {
     .frame-title, .frame-subtitle { display: block; overflow-wrap: anywhere; }
     .frame-title { font-size: 13px; font-weight: 600; }
     .frame-subtitle, .frame-time { margin-top: 3px; color: #7a818b; font-size: 11px; }
+    .frame-log, #node-log { white-space: pre-wrap; overflow-wrap: anywhere; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; }
+    .frame-log { display: block; margin-top: 6px; color: #4b515a; font-size: 12px; line-height: 1.5; }
     .workspace { display: grid; grid-template-rows: 178px minmax(0, 1fr); min-width: 0; min-height: 0; }
     .timeline-panel { min-width: 0; overflow-x: auto; overflow-y: hidden; border-bottom: 1px solid #dfe3e8; background: #fff; }
     .timeline-track { position: relative; height: 178px; min-width: 100%; cursor: pointer; user-select: none; }
@@ -322,6 +380,7 @@ function createReplayHtml(input: {
           <dt>节点</dt><dd id="node">-</dd>
           <dt>阶段</dt><dd id="phase">-</dd>
           <dt>结果</dt><dd id="frame-status">-</dd>
+          <dt id="node-log-label" hidden>日志内容</dt><dd id="node-log" hidden></dd>
           <dt>备注</dt><dd id="note">-</dd>
           <dt>定位</dt><dd id="selector">-</dd>
           <dt>时间</dt><dd id="time">-</dd>
@@ -368,6 +427,24 @@ function createReplayHtml(input: {
       offsetMs: Math.max(0, (Date.parse(frame.capturedAt) || reportStartedAt) - reportStartedAt),
     }));
     const durationMs = Math.max(1000, Number(data.durationMs) || 0, frames.at(-1)?.offsetMs || 0);
+    // 按一次执行配对，而不是按节点 ID 全局去重；连接脚本和重复执行仍保留独立步骤。
+    const steps = [];
+    const frameSteps = [];
+    const pendingSteps = new Map();
+    frames.forEach((frame, index) => {
+      const key = JSON.stringify([frame.scriptName, frame.nodeId, frame.nodeNumber]);
+      const pending = pendingSteps.get(key) || [];
+      let stepIndex = frame.phase === 'before' ? undefined : pending.pop();
+      if (stepIndex === undefined) {
+        stepIndex = steps.length;
+        steps.push({ firstIndex: index, lastIndex: index });
+      } else {
+        steps[stepIndex].lastIndex = index;
+      }
+      if (frame.phase === 'before') pending.push(stepIndex);
+      pendingSteps.set(key, pending);
+      frameSteps[index] = stepIndex;
+    });
     const elements = Object.fromEntries(['screen','empty','previous','play','next','seek','clock','caption','title','script','node','phase','frame-status','note','selector','time','step-list','timeline-track','ruler','playhead','visual-checks','log'].map((id) => [id, document.querySelector('#' + id)]));
     const imagePreview = document.querySelector('#image-preview');
     const imagePreviewImage = document.querySelector('#image-preview-image');
@@ -387,6 +464,7 @@ function createReplayHtml(input: {
     let animationFrame = 0;
     let playbackStartedAt = 0;
     const phaseLabels = { before: '执行前', after: '执行后', error: '失败', stopped: '已终止' };
+    const visualCheckStatusLabel = (status) => status === 'passed' ? '有变化' : '无明显变化';
     const formatTime = (milliseconds) => {
       const seconds = Math.max(0, Math.floor(milliseconds / 1000));
       return String(Math.floor(seconds / 60)).padStart(2, '0') + ':' + String(seconds % 60).padStart(2, '0');
@@ -410,6 +488,10 @@ function createReplayHtml(input: {
         elements.node.textContent = String(frame.nodeNumber || '-');
         elements.phase.textContent = phaseLabels[frame.phase] || frame.phase || '-';
         elements['frame-status'].textContent = frame.status || '-';
+        const logContent = frames[steps[frameSteps[currentFrame]].lastIndex].logContent;
+        document.querySelector('#node-log-label').hidden = !logContent;
+        document.querySelector('#node-log').hidden = !logContent;
+        document.querySelector('#node-log').textContent = logContent || '';
         elements.note.textContent = frame.note || '-';
         elements.selector.textContent = frame.selector || '-';
         elements.time.textContent = new Date(frame.capturedAt).toLocaleString();
@@ -421,7 +503,8 @@ function createReplayHtml(input: {
       elements.seek.value = String(Math.round(currentMs / durationMs * 1000));
       elements.clock.textContent = formatTime(currentMs) + ' / ' + formatTime(durationMs);
       elements.playhead.style.left = (currentMs / durationMs * 100) + '%';
-      document.querySelectorAll('.frame-item, .thumbnail').forEach((item) => item.classList.toggle('active', Number(item.dataset.index) === currentFrame));
+      document.querySelectorAll('.frame-item').forEach((item) => item.classList.toggle('active', Number(item.dataset.step) === frameSteps[currentFrame]));
+      document.querySelectorAll('.thumbnail').forEach((item) => item.classList.toggle('active', Number(item.dataset.index) === currentFrame));
       if (scrollStep) document.querySelector('.frame-item.active')?.scrollIntoView({ block: 'nearest', behavior: 'smooth' });
     };
     const pause = () => {
@@ -453,11 +536,13 @@ function createReplayHtml(input: {
       tick.append(label);
       elements.ruler.append(tick);
     }
-    frames.forEach((frame, index) => {
+    steps.forEach((step, index) => {
+      const frame = frames[step.firstIndex];
+      const result = frames[step.lastIndex];
       const button = document.createElement('button');
       button.type = 'button';
       button.className = 'frame-item';
-      button.dataset.index = String(index);
+      button.dataset.step = String(index);
       const badge = document.createElement('span');
       badge.className = 'frame-index';
       badge.textContent = String(index + 1);
@@ -467,15 +552,28 @@ function createReplayHtml(input: {
       title.textContent = frame.nodeNumber + '. ' + frame.nodeLabel;
       const subtitle = document.createElement('span');
       subtitle.className = 'frame-subtitle';
-      subtitle.textContent = (phaseLabels[frame.phase] || frame.phase) + ' · ' + frame.status;
+      subtitle.textContent = result.phase === 'before' ? '未完成' : result.status;
       content.append(title, subtitle);
+      if (frame.nodeType === 'textClick') {
+        const text = document.createElement('span');
+        text.className = 'frame-log frame-click-text';
+        text.textContent = frame.selector || '未设置点击文字';
+        content.append(text);
+      }
+      if (result.logContent) {
+        const log = document.createElement('span');
+        log.className = 'frame-log';
+        log.textContent = result.logContent;
+        content.append(log);
+      }
       const time = document.createElement('span');
       time.className = 'frame-time';
       time.textContent = formatTime(frame.offsetMs);
       button.append(badge, content, time);
       button.addEventListener('click', () => { pause(); seekTo(frame.offsetMs); });
       elements['step-list'].append(button);
-
+    });
+    frames.forEach((frame, index) => {
       const thumbnail = document.createElement('button');
       thumbnail.type = 'button';
       thumbnail.className = 'thumbnail';

package/server/appium-recorder/repository.ts

@@ -19,6 +19,10 @@ export type AppiumRecordedStepRecord = {
     | 'backIfExists'
     | 'waitFor'
     | 'assertExists'
+    | 'checkboxState'
+    | 'radioButtonState'
+    | 'aiRecognition'
+    | 'textClick'
     | 'key'
     | 'waitActivity'
     | 'delay'
@@ -34,6 +38,7 @@ export type AppiumRecordedStepRecord = {
     | 'pinch'
     | 'runScript'
     | 'noop'
+    | 'log'
     | 'visualChange';
   label: string;
   note?: string;
@@ -74,9 +79,10 @@ export type AppiumRecordedStepRecord = {
     source?: 'node' | 'ancestor' | 'fallback';
   };
   value?: string;
-  optional?: boolean;
+  logPrefix?: string;
   keyCode?: number;
   timeoutMs?: number;
+  longPressMode?: 'element' | 'coordinates';
   flow?: {
     nodeKind?: 'action' | 'condition' | 'assertion';
     yesTargetId?: string;

package/server/appium-recorder/routes.ts

@@ -1,7 +1,6 @@
 import { execFile } from 'node:child_process';
-import fs from 'node:fs/promises';
-import os from 'node:os';
-import path from 'node:path';
+import { recognizeDeviceScreen } from './ai-recognition';
+import { readFreshWindowHierarchy } from './tree-dump';
 import type { IncomingMessage, ServerResponse } from 'node:http';
 import {
   deleteAppiumRecordedScript,
@@ -14,6 +13,7 @@ import {
 import { clearAppDataOnDevice, launchAppOnDevice, replayAppiumScript } from './appium-runner';
 import { isRemoteDeviceId, sendRemoteCommand } from '../remote-agents/registry';
 import { getAdbCommand } from '../android-sdk';
+import { createAppiumScriptExport } from './script-export';
 
 const treeDumpTasks = new Map<string, Promise<string>>();
 const replayingDevices = new Set<string>();
@@ -25,13 +25,6 @@ function sendJson(res: ServerResponse, payload: unknown, statusCode = 200) {
   res.end(JSON.stringify(payload));
 }
 
-function sendJsonDownload(res: ServerResponse, payload: unknown, fileName: string) {
-  res.statusCode = 200;
-  res.setHeader('Content-Type', 'application/json; charset=utf-8');
-  res.setHeader('Content-Disposition', `attachment; filename="appium-script.json"; filename*=UTF-8''${encodeURIComponent(fileName)}`);
-  res.end(JSON.stringify(payload, null, 2));
-}
-
 function sendStreamEvent(res: ServerResponse, payload: unknown) {
   res.write(`${JSON.stringify(payload)}\n`);
 }
@@ -64,17 +57,7 @@ async function dumpWindowHierarchy(deviceId: string) {
   const activeTask = treeDumpTasks.get(deviceId);
   if (activeTask) return activeTask;
 
-  const task = (async () => {
-    const localPath = path.join(os.tmpdir(), `midscene-appium-${deviceId.replace(/[^\w.-]/g, '_')}-${Date.now()}.xml`);
-    const remotePath = '/data/local/tmp/midscene_appium_uidump.xml';
-    await execFileText(getAdbCommand(), ['-s', deviceId, 'shell', 'uiautomator', 'dump', remotePath]);
-    await execFileText(getAdbCommand(), ['-s', deviceId, 'pull', remotePath, localPath]);
-    try {
-      return await fs.readFile(localPath, 'utf8');
-    } finally {
-      await fs.unlink(localPath).catch(() => undefined);
-    }
-  })();
+  const task = readFreshWindowHierarchy(deviceId, getAdbCommand(), execFileText);
   treeDumpTasks.set(deviceId, task);
   try {
     return await task;
@@ -134,6 +117,7 @@ export async function handleAppiumRecorderRequest(
     };
 
     if (pathname === '/api/appium-recorder/tree' && req.method === 'GET') {
+      res.setHeader('Cache-Control', 'no-store');
       const deviceId = requestUrl.searchParams.get('deviceId')?.trim() || selectedDeviceId;
       if (!deviceId) throw new Error('未检测到可用设备');
       assertDeviceAllowed(deviceId);
@@ -192,6 +176,24 @@ export async function handleAppiumRecorderRequest(
       return true;
     }
 
+    if (pathname === '/api/appium-recorder/ai-recognition/test' && req.method === 'POST') {
+      const parsed = await readBody<{ deviceId?: string; prompt?: unknown; timeoutMs?: number }>(req);
+      const deviceId = parsed.deviceId?.trim() || selectedDeviceId;
+      if (!deviceId) throw new Error('请选择设备');
+      assertDeviceAllowed(deviceId);
+      if (isRemoteDeviceId(deviceId)) throw new Error('远程设备暂不支持 AI 识别测试');
+      if (replayingDevices.has(deviceId)) { sendJson(res, { message: '设备正在回放，请稍后测试' }, 409); return true; }
+      const controller = new AbortController();
+      const abort = () => controller.abort();
+      res.once('close', abort);
+      try {
+        sendJson(res, await recognizeDeviceScreen({ deviceId, prompt: parsed.prompt, timeoutMs: parsed.timeoutMs, signal: controller.signal }));
+      } finally {
+        res.off('close', abort);
+      }
+      return true;
+    }
+
     if (pathname === '/api/appium-recorder/launch-app' && req.method === 'POST') {
       const parsed = await readBody<{ deviceId?: string; packageName?: string }>(req);
       const deviceId = parsed.deviceId?.trim() || selectedDeviceId;
@@ -277,17 +279,14 @@ export async function handleAppiumRecorderRequest(
     if (exportMatch && req.method === 'GET') {
       const script = getAppiumRecordedScript(decodeURIComponent(exportMatch[1]));
       if (!script) throw new Error('Appium 录制脚本不存在');
-      sendJsonDownload(res, {
-        schemaVersion: 1,
-        exportedAt: new Date().toISOString(),
-        script: {
-          name: script.name,
-          appPackage: script.appPackage,
-          appActivity: script.appActivity,
-          deviceId: script.deviceId,
-          steps: script.steps,
-        },
-      }, `${script.name}.json`);
+      const exported = await createAppiumScriptExport(script, getAppiumRecordedScript);
+      const fileName = encodeURIComponent(exported.fileName)
+        .replace(/['()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`);
+      res.statusCode = 200;
+      res.setHeader('Content-Type', exported.contentType);
+      res.setHeader('Content-Disposition', `attachment; filename="appium-script.${exported.contentType === 'application/zip' ? 'zip' : 'json'}"; filename*=UTF-8''${fileName}`);
+      res.setHeader('Content-Length', exported.body.length);
+      res.end(exported.body);
       return true;
     }
 

package/server/appium-recorder/screenshot.ts

@@ -0,0 +1,21 @@
+import { execFile } from 'node:child_process';
+import { getAdbCommand } from '../android-sdk';
+
+// 直接读取设备截图，不使用预览缓存，也不进入 Appium 命令队列。
+export function adbScreenshotBase64(deviceId: string, signal?: AbortSignal) {
+  return new Promise<string>((resolve, reject) => {
+    execFile(
+      getAdbCommand(),
+      ['-s', deviceId, 'exec-out', 'screencap', '-p'],
+      { encoding: 'buffer', maxBuffer: 20 * 1024 * 1024, timeout: 8000, signal },
+      (error, stdout, stderr) => {
+        if (error) {
+          reject(new Error(stderr?.toString('utf8').trim() || error.message || 'ADB 截图失败'));
+          return;
+        }
+        if (!stdout.length) { reject(new Error('ADB 未返回截图内容')); return; }
+        resolve(stdout.toString('base64'));
+      },
+    );
+  });
+}

package/server/appium-recorder/script-export.ts

@@ -0,0 +1,67 @@
+import { buffer } from 'node:stream/consumers';
+import archiver from 'archiver';
+import type { AppiumRecordedScriptRecord } from './repository';
+
+function safeExportName(name: string) {
+  const base = Array.from(name.trim().replace(/[<>:"/\\|?*\u0000-\u001f]/g, '_'))
+    .slice(0, 80).join('').replace(/[. ]+$/, '') || 'script';
+  return /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\.|$)/i.test(base) ? `_${base}` : base;
+}
+
+export async function createAppiumScriptExport(
+  root: AppiumRecordedScriptRecord,
+  getScript: (id: string) => AppiumRecordedScriptRecord | null,
+) {
+  const scripts = [root];
+  const visited = new Set([root.id]);
+  // 按脚本 ID 去重，既保留所有分支的依赖，也避免重复引用和循环连接无限展开。
+  for (const script of scripts) {
+    for (const step of script.steps) {
+      if (step.type !== 'runScript') continue;
+      const targetId = step.value || '';
+      if (!targetId) throw new Error(`脚本「${script.name}」的「${step.label}」缺少连接脚本 ID，无法导出`);
+      if (visited.has(targetId)) continue;
+      const target = getScript(targetId);
+      if (!target) throw new Error(`脚本「${script.name}」连接的脚本不存在（${targetId}），无法导出完整脚本包`);
+      visited.add(targetId);
+      scripts.push(target);
+    }
+  }
+
+  const exportedAt = new Date().toISOString();
+  const serialize = (script: AppiumRecordedScriptRecord) => JSON.stringify({
+    schemaVersion: 1,
+    exportedAt,
+    script: {
+      id: script.id,
+      name: script.name,
+      appPackage: script.appPackage,
+      appActivity: script.appActivity,
+      deviceId: script.deviceId,
+      steps: script.steps,
+    },
+  }, null, 2);
+  const baseName = safeExportName(root.name);
+  if (!root.steps.some((step) => step.type === 'runScript')) {
+    return {
+      fileName: `${baseName}.json`,
+      contentType: 'application/json; charset=utf-8',
+      body: Buffer.from(serialize(root)),
+    };
+  }
+
+  const archive = archiver('zip', { zlib: { level: 9 } });
+  const usedNames = new Set<string>();
+  for (const script of scripts) {
+    const name = safeExportName(script.name);
+    let fileName = `${name}.json`;
+    let suffix = 2;
+    // 清理非法字符后可能重名，Windows 解压时还需忽略大小写进行去重。
+    while (usedNames.has(fileName.toLowerCase())) fileName = `${name} (${suffix++}).json`;
+    usedNames.add(fileName.toLowerCase());
+    archive.append(serialize(script), { name: fileName });
+  }
+  // 生成完整压缩包后再发送响应，压缩失败仍可返回普通 JSON 错误。
+  const [body] = await Promise.all([buffer(archive), archive.finalize()]);
+  return { fileName: `${baseName}.zip`, contentType: 'application/zip', body };
+}

package/server/appium-recorder/tree-dump.ts

@@ -0,0 +1,22 @@
+import { randomUUID } from 'node:crypto';
+
+type ExecText = (command: string, args: string[]) => Promise<string>;
+
+export async function readFreshWindowHierarchy(deviceId: string, adb: string, execText: ExecText) {
+  // dump 失败可能仍以 0 退出。每次独立文件，避免误读上一次成功抓取的树。
+  const remotePath = `/data/local/tmp/midscene_uidump_${randomUUID()}.xml`;
+  const run = (args: string[]) => execText(adb, ['-s', deviceId, ...args]);
+  try {
+    const output = await run(['shell', 'uiautomator', 'dump', remotePath]);
+    if (/ERROR:|could not|failed/i.test(output)) throw new Error(output.trim());
+    const xml = await run(['shell', 'cat', remotePath]);
+    if (!xml.includes('<hierarchy') || !xml.includes('</hierarchy>')) {
+      throw new Error('设备未返回有效的组件树');
+    }
+    return xml;
+  } catch (error) {
+    throw new Error(`刷新组件树失败：${error instanceof Error ? error.message : String(error)}`);
+  } finally {
+    await run(['shell', 'rm', '-f', remotePath]).catch(() => undefined);
+  }
+}

package/server/config.ts

@@ -4,8 +4,10 @@ import path from 'node:path';
 import YAML from 'yaml';
 import { appDataPath } from './paths';
 import { loadModelConfigFromDb, saveModelConfigToDb } from './config-store';
+import type { AiRecognitionModel } from '../src/appium-recorder/ai-recognition';
 
 export type AppConfig = {
+  appium: { model: AiRecognitionModel };
   runtime: {
     androidSdkPath: string;
     reportOutputPath: string;
@@ -43,6 +45,7 @@ export class ConfigValidationError extends Error {
 
 function defaultConfig(): AppConfig {
   return {
+    appium: { model: { baseUrl: '', apiKey: '', name: '' } },
     runtime: {
       androidSdkPath: '',
       reportOutputPath: '',
@@ -78,6 +81,13 @@ function normalizeEnv(env: unknown): Record<string, string> {
 function normalizeConfig(config: Partial<AppConfig> | null | undefined): AppConfig {
   const fallback = defaultConfig();
   return {
+    // 旧配置没有 Appium 模型时保留空值，不借用其他模型或改变已有配置。
+    appium: {
+      model: Object.fromEntries(['baseUrl', 'apiKey', 'name'].map((key) => {
+        const value = config?.appium?.model?.[key as keyof AiRecognitionModel];
+        return [key, typeof value === 'string' ? value.trim() : ''];
+      })) as AiRecognitionModel,
+    },
     runtime: {
       androidSdkPath: config?.runtime?.androidSdkPath?.trim() || fallback.runtime.androidSdkPath,
       reportOutputPath: config?.runtime?.reportOutputPath?.trim() || fallback.runtime.reportOutputPath,

package/server/http-api.ts

@@ -1070,7 +1070,7 @@ export function createApiMiddleware() {
     if (req.url === '/api/test-model' && req.method === 'POST') {
       try {
         const parsed = await readBody<{
-          modelKey?: 'midscene' | 'scriptOptimizer';
+          modelKey?: 'midscene' | 'scriptOptimizer' | 'appium';
           model?: {
             provider?: 'custom' | 'codex';
             baseUrl?: string;

package/src/App.vue

@@ -1,4 +1,5 @@
 <script setup lang="ts">
+import { isAiRecognitionModelConfigured } from './appium-recorder/ai-recognition';
 import { computed, onMounted, onUnmounted, reactive, ref, shallowRef, watch } from 'vue';
 import { ElMessage, ElMessageBox } from 'element-plus';
 import {
@@ -99,7 +100,9 @@ const appPresets = ref<AppPreset[]>([]);
 const modelTestStatus = reactive({
   midscene: '',
   scriptOptimizer: '',
+  appium: '',
 });
+const aiRecognitionModelConfigured = shallowRef(false);
 const actionDialog = reactive({
   visible: false,
   title: '',
@@ -149,6 +152,7 @@ const appPresetForm = reactive({
 });
 
 const configForm = reactive<ConfigForm>({
+  appium: { model: { baseUrl: '', apiKey: '', name: '' } },
   runtime: {
     androidSdkPath: '',
     reportOutputPath: '',
@@ -656,6 +660,8 @@ const loadConfig = async () => {
   });
   Object.assign(configForm.midscene.env, payload.midscene.env || {});
   Object.assign(configForm.scriptOptimizer.model, payload.scriptOptimizer.model);
+  Object.assign(configForm.appium.model, payload.appium?.model || { baseUrl: '', apiKey: '', name: '' });
+  aiRecognitionModelConfigured.value = isAiRecognitionModelConfigured(configForm.appium.model);
   if (configForm.midscene.model.provider !== 'codex') {
     rememberCustomMidsceneModel();
   }
@@ -692,7 +698,7 @@ const getMidsceneModelConfigError = () => {
   if (model.provider !== 'codex' && !model.apiKey.trim()) missing.push('API Key');
 
   if (!missing.length) return '';
-  return `Midscene 模型配置不完整：缺少 ${missing.join('、')}。请在“参数配置 > Midscene 模型”中选择“使用 Codex”，或补齐自定义提供方后保存。`;
+  return `Midscene 模型配置不完整：缺少 ${missing.join('、')}。请在“参数配置 > Midscene配置 > Midscene 模型”中选择“使用 Codex”，或补齐自定义提供方后保存。`;
 };
 
 const saveModelConfig = async () => {
@@ -700,7 +706,10 @@ const saveModelConfig = async () => {
   errorMessage.value = '';
   openActionDialog('保存参数配置');
   try {
-    await api.saveConfig(configForm);
+    // 保存的是本次提交快照，不能把请求期间尚未保存的改动标记为已配置。
+    const snapshot = JSON.parse(JSON.stringify(configForm)) as ConfigForm;
+    await api.saveConfig(snapshot);
+    aiRecognitionModelConfigured.value = isAiRecognitionModelConfigured(snapshot.appium.model);
     closeActionDialog();
     ElMessage.success('参数配置已保存');
   } catch (error) {
@@ -776,14 +785,11 @@ const removeAppPreset = async (id: string) => {
   }
 };
 
-const testModel = async (key: 'midscene' | 'scriptOptimizer') => {
+const testModel = async (key: 'midscene' | 'scriptOptimizer' | 'appium') => {
   testingModelKey.value = key;
   errorMessage.value = '';
   modelTestStatus[key] = '';
-  const model =
-    key === 'midscene'
-      ? configForm.midscene.model
-      : configForm.scriptOptimizer.model;
+  const model = configForm[key].model;
 
   try {
     const payload = await api.testModel({ modelKey: key, model });
@@ -1557,6 +1563,7 @@ onUnmounted(() => {
           v-show="activeMenu === 'appium'"
           :active="activeMenu === 'appium'"
           :app-presets="appPresets"
+          :ai-recognition-model-configured="aiRecognitionModelConfigured"
           :device-actions="deviceActions"
           :playground-available="playgroundAvailable"
           :playground-device-id="playgroundDeviceId"

package/src/api.ts

@@ -116,7 +116,7 @@ export function saveConfig(input: ConfigForm) {
 
 // 测试指定模型配置是否可用，通常用于保存前验证 baseUrl/apiKey/model。
 export function testModel(input: {
-  modelKey: 'midscene' | 'scriptOptimizer';
+  modelKey: 'midscene' | 'scriptOptimizer' | 'appium';
   model: {
     provider?: 'custom' | 'codex';
     baseUrl: string;