android-midscene-automation 0.1.26 → 0.1.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/FAQ.md +14 -0
  3. package/README.md +39 -3
  4. package/bin/android-midscene-automation.js +71 -1
  5. package/package.json +7 -2
  6. package/remote-agent/index.ts +2 -11
  7. package/server/appium-recorder/ai-recognition.ts +67 -0
  8. package/server/appium-recorder/appium-runner.ts +335 -52
  9. package/server/appium-recorder/report.ts +297 -16
  10. package/server/appium-recorder/repository.ts +31 -4
  11. package/server/appium-recorder/routes.ts +31 -32
  12. package/server/appium-recorder/screenshot.ts +21 -0
  13. package/server/appium-recorder/script-export.ts +67 -0
  14. package/server/appium-recorder/tree-dump.ts +22 -0
  15. package/server/appium-recorder/visual-change.ts +184 -0
  16. package/server/config.ts +10 -0
  17. package/server/http-api.ts +4 -5
  18. package/src/App.vue +14 -7
  19. package/src/api.ts +1 -1
  20. package/src/appium-recorder/AppiumPage.vue +658 -162
  21. package/src/appium-recorder/ai-recognition.ts +16 -0
  22. package/src/appium-recorder/api.ts +20 -3
  23. package/src/appium-recorder/components/AiRecognitionTestDialog.vue +71 -0
  24. package/src/appium-recorder/components/FlowActionMenu.vue +206 -0
  25. package/src/appium-recorder/components/FlowCanvas.vue +16 -1
  26. package/src/appium-recorder/components/FlowNodeCard.vue +23 -37
  27. package/src/appium-recorder/components/FlowStepEditor.vue +152 -25
  28. package/src/appium-recorder/components/LongPressSettings.vue +58 -0
  29. package/src/appium-recorder/components/NestedConditionBranches.vue +55 -35
  30. package/src/appium-recorder/components/NewScriptDialog.vue +27 -0
  31. package/src/appium-recorder/components/NodeDetail.vue +2 -0
  32. package/src/appium-recorder/components/RecordedSteps.vue +24 -4
  33. package/src/appium-recorder/components/StageLogSettings.vue +17 -0
  34. package/src/appium-recorder/components/TextClickSettings.vue +19 -0
  35. package/src/appium-recorder/components/VisualChangeDialog.vue +151 -0
  36. package/src/appium-recorder/flow-copy.ts +47 -1
  37. package/src/appium-recorder/flow-graph.ts +147 -64
  38. package/src/appium-recorder/flow-labels.ts +35 -6
  39. package/src/appium-recorder/flow-normalize.ts +76 -0
  40. package/src/appium-recorder/flow-remove.ts +69 -0
  41. package/src/appium-recorder/long-press.ts +27 -0
  42. package/src/appium-recorder/native-control-state.ts +26 -0
  43. package/src/appium-recorder/stage-log.ts +21 -0
  44. package/src/appium-recorder/text-click.ts +9 -0
  45. package/src/appium-recorder/tree.ts +3 -0
  46. package/src/appium-recorder/types.ts +34 -2
  47. package/src/appium-recorder/visual-change-pairs.ts +147 -0
  48. package/src/appium-recorder/visual-change.ts +75 -0
  49. package/src/components/config/AppiumConfigPanel.vue +48 -0
  50. package/src/components/config/BaseConfigPanel.vue +90 -0
  51. package/src/components/config/MidsceneConfigPanel.vue +204 -0
  52. package/src/components/device/DevicePreviewPanel.vue +148 -4
  53. package/src/pages/ConfigPage.vue +107 -265
  54. package/src/style.css +43 -4
  55. package/src/types.ts +2 -0
  56. package/src/ui-refresh.css +5 -0
@@ -1,7 +1,6 @@
1
1
  import { execFile } from 'node:child_process';
2
- import fs from 'node:fs/promises';
3
- import os from 'node:os';
4
- import path from 'node:path';
2
+ import { recognizeDeviceScreen } from './ai-recognition';
3
+ import { readFreshWindowHierarchy } from './tree-dump';
5
4
  import type { IncomingMessage, ServerResponse } from 'node:http';
6
5
  import {
7
6
  deleteAppiumRecordedScript,
@@ -14,6 +13,7 @@ import {
14
13
  import { clearAppDataOnDevice, launchAppOnDevice, replayAppiumScript } from './appium-runner';
15
14
  import { isRemoteDeviceId, sendRemoteCommand } from '../remote-agents/registry';
16
15
  import { getAdbCommand } from '../android-sdk';
16
+ import { createAppiumScriptExport } from './script-export';
17
17
 
18
18
  const treeDumpTasks = new Map<string, Promise<string>>();
19
19
  const replayingDevices = new Set<string>();
@@ -25,13 +25,6 @@ function sendJson(res: ServerResponse, payload: unknown, statusCode = 200) {
25
25
  res.end(JSON.stringify(payload));
26
26
  }
27
27
 
28
- function sendJsonDownload(res: ServerResponse, payload: unknown, fileName: string) {
29
- res.statusCode = 200;
30
- res.setHeader('Content-Type', 'application/json; charset=utf-8');
31
- res.setHeader('Content-Disposition', `attachment; filename="appium-script.json"; filename*=UTF-8''${encodeURIComponent(fileName)}`);
32
- res.end(JSON.stringify(payload, null, 2));
33
- }
34
-
35
28
  function sendStreamEvent(res: ServerResponse, payload: unknown) {
36
29
  res.write(`${JSON.stringify(payload)}\n`);
37
30
  }
@@ -64,17 +57,7 @@ async function dumpWindowHierarchy(deviceId: string) {
64
57
  const activeTask = treeDumpTasks.get(deviceId);
65
58
  if (activeTask) return activeTask;
66
59
 
67
- const task = (async () => {
68
- const localPath = path.join(os.tmpdir(), `midscene-appium-${deviceId.replace(/[^\w.-]/g, '_')}-${Date.now()}.xml`);
69
- const remotePath = '/data/local/tmp/midscene_appium_uidump.xml';
70
- await execFileText(getAdbCommand(), ['-s', deviceId, 'shell', 'uiautomator', 'dump', remotePath]);
71
- await execFileText(getAdbCommand(), ['-s', deviceId, 'pull', remotePath, localPath]);
72
- try {
73
- return await fs.readFile(localPath, 'utf8');
74
- } finally {
75
- await fs.unlink(localPath).catch(() => undefined);
76
- }
77
- })();
60
+ const task = readFreshWindowHierarchy(deviceId, getAdbCommand(), execFileText);
78
61
  treeDumpTasks.set(deviceId, task);
79
62
  try {
80
63
  return await task;
@@ -134,6 +117,7 @@ export async function handleAppiumRecorderRequest(
134
117
  };
135
118
 
136
119
  if (pathname === '/api/appium-recorder/tree' && req.method === 'GET') {
120
+ res.setHeader('Cache-Control', 'no-store');
137
121
  const deviceId = requestUrl.searchParams.get('deviceId')?.trim() || selectedDeviceId;
138
122
  if (!deviceId) throw new Error('未检测到可用设备');
139
123
  assertDeviceAllowed(deviceId);
@@ -192,6 +176,24 @@ export async function handleAppiumRecorderRequest(
192
176
  return true;
193
177
  }
194
178
 
179
+ if (pathname === '/api/appium-recorder/ai-recognition/test' && req.method === 'POST') {
180
+ const parsed = await readBody<{ deviceId?: string; prompt?: unknown; timeoutMs?: number }>(req);
181
+ const deviceId = parsed.deviceId?.trim() || selectedDeviceId;
182
+ if (!deviceId) throw new Error('请选择设备');
183
+ assertDeviceAllowed(deviceId);
184
+ if (isRemoteDeviceId(deviceId)) throw new Error('远程设备暂不支持 AI 识别测试');
185
+ if (replayingDevices.has(deviceId)) { sendJson(res, { message: '设备正在回放,请稍后测试' }, 409); return true; }
186
+ const controller = new AbortController();
187
+ const abort = () => controller.abort();
188
+ res.once('close', abort);
189
+ try {
190
+ sendJson(res, await recognizeDeviceScreen({ deviceId, prompt: parsed.prompt, timeoutMs: parsed.timeoutMs, signal: controller.signal }));
191
+ } finally {
192
+ res.off('close', abort);
193
+ }
194
+ return true;
195
+ }
196
+
195
197
  if (pathname === '/api/appium-recorder/launch-app' && req.method === 'POST') {
196
198
  const parsed = await readBody<{ deviceId?: string; packageName?: string }>(req);
197
199
  const deviceId = parsed.deviceId?.trim() || selectedDeviceId;
@@ -277,17 +279,14 @@ export async function handleAppiumRecorderRequest(
277
279
  if (exportMatch && req.method === 'GET') {
278
280
  const script = getAppiumRecordedScript(decodeURIComponent(exportMatch[1]));
279
281
  if (!script) throw new Error('Appium 录制脚本不存在');
280
- sendJsonDownload(res, {
281
- schemaVersion: 1,
282
- exportedAt: new Date().toISOString(),
283
- script: {
284
- name: script.name,
285
- appPackage: script.appPackage,
286
- appActivity: script.appActivity,
287
- deviceId: script.deviceId,
288
- steps: script.steps,
289
- },
290
- }, `${script.name}.json`);
282
+ const exported = await createAppiumScriptExport(script, getAppiumRecordedScript);
283
+ const fileName = encodeURIComponent(exported.fileName)
284
+ .replace(/['()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`);
285
+ res.statusCode = 200;
286
+ res.setHeader('Content-Type', exported.contentType);
287
+ res.setHeader('Content-Disposition', `attachment; filename="appium-script.${exported.contentType === 'application/zip' ? 'zip' : 'json'}"; filename*=UTF-8''${fileName}`);
288
+ res.setHeader('Content-Length', exported.body.length);
289
+ res.end(exported.body);
291
290
  return true;
292
291
  }
293
292
 
@@ -0,0 +1,21 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { getAdbCommand } from '../android-sdk';
3
+
4
+ // 直接读取设备截图,不使用预览缓存,也不进入 Appium 命令队列。
5
+ export function adbScreenshotBase64(deviceId: string, signal?: AbortSignal) {
6
+ return new Promise<string>((resolve, reject) => {
7
+ execFile(
8
+ getAdbCommand(),
9
+ ['-s', deviceId, 'exec-out', 'screencap', '-p'],
10
+ { encoding: 'buffer', maxBuffer: 20 * 1024 * 1024, timeout: 8000, signal },
11
+ (error, stdout, stderr) => {
12
+ if (error) {
13
+ reject(new Error(stderr?.toString('utf8').trim() || error.message || 'ADB 截图失败'));
14
+ return;
15
+ }
16
+ if (!stdout.length) { reject(new Error('ADB 未返回截图内容')); return; }
17
+ resolve(stdout.toString('base64'));
18
+ },
19
+ );
20
+ });
21
+ }
@@ -0,0 +1,67 @@
1
+ import { buffer } from 'node:stream/consumers';
2
+ import archiver from 'archiver';
3
+ import type { AppiumRecordedScriptRecord } from './repository';
4
+
5
+ function safeExportName(name: string) {
6
+ const base = Array.from(name.trim().replace(/[<>:"/\\|?*\u0000-\u001f]/g, '_'))
7
+ .slice(0, 80).join('').replace(/[. ]+$/, '') || 'script';
8
+ return /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\.|$)/i.test(base) ? `_${base}` : base;
9
+ }
10
+
11
+ export async function createAppiumScriptExport(
12
+ root: AppiumRecordedScriptRecord,
13
+ getScript: (id: string) => AppiumRecordedScriptRecord | null,
14
+ ) {
15
+ const scripts = [root];
16
+ const visited = new Set([root.id]);
17
+ // 按脚本 ID 去重,既保留所有分支的依赖,也避免重复引用和循环连接无限展开。
18
+ for (const script of scripts) {
19
+ for (const step of script.steps) {
20
+ if (step.type !== 'runScript') continue;
21
+ const targetId = step.value || '';
22
+ if (!targetId) throw new Error(`脚本「${script.name}」的「${step.label}」缺少连接脚本 ID,无法导出`);
23
+ if (visited.has(targetId)) continue;
24
+ const target = getScript(targetId);
25
+ if (!target) throw new Error(`脚本「${script.name}」连接的脚本不存在(${targetId}),无法导出完整脚本包`);
26
+ visited.add(targetId);
27
+ scripts.push(target);
28
+ }
29
+ }
30
+
31
+ const exportedAt = new Date().toISOString();
32
+ const serialize = (script: AppiumRecordedScriptRecord) => JSON.stringify({
33
+ schemaVersion: 1,
34
+ exportedAt,
35
+ script: {
36
+ id: script.id,
37
+ name: script.name,
38
+ appPackage: script.appPackage,
39
+ appActivity: script.appActivity,
40
+ deviceId: script.deviceId,
41
+ steps: script.steps,
42
+ },
43
+ }, null, 2);
44
+ const baseName = safeExportName(root.name);
45
+ if (!root.steps.some((step) => step.type === 'runScript')) {
46
+ return {
47
+ fileName: `${baseName}.json`,
48
+ contentType: 'application/json; charset=utf-8',
49
+ body: Buffer.from(serialize(root)),
50
+ };
51
+ }
52
+
53
+ const archive = archiver('zip', { zlib: { level: 9 } });
54
+ const usedNames = new Set<string>();
55
+ for (const script of scripts) {
56
+ const name = safeExportName(script.name);
57
+ let fileName = `${name}.json`;
58
+ let suffix = 2;
59
+ // 清理非法字符后可能重名,Windows 解压时还需忽略大小写进行去重。
60
+ while (usedNames.has(fileName.toLowerCase())) fileName = `${name} (${suffix++}).json`;
61
+ usedNames.add(fileName.toLowerCase());
62
+ archive.append(serialize(script), { name: fileName });
63
+ }
64
+ // 生成完整压缩包后再发送响应,压缩失败仍可返回普通 JSON 错误。
65
+ const [body] = await Promise.all([buffer(archive), archive.finalize()]);
66
+ return { fileName: `${baseName}.zip`, contentType: 'application/zip', body };
67
+ }
@@ -0,0 +1,22 @@
1
+ import { randomUUID } from 'node:crypto';
2
+
3
+ type ExecText = (command: string, args: string[]) => Promise<string>;
4
+
5
+ export async function readFreshWindowHierarchy(deviceId: string, adb: string, execText: ExecText) {
6
+ // dump 失败可能仍以 0 退出。每次独立文件,避免误读上一次成功抓取的树。
7
+ const remotePath = `/data/local/tmp/midscene_uidump_${randomUUID()}.xml`;
8
+ const run = (args: string[]) => execText(adb, ['-s', deviceId, ...args]);
9
+ try {
10
+ const output = await run(['shell', 'uiautomator', 'dump', remotePath]);
11
+ if (/ERROR:|could not|failed/i.test(output)) throw new Error(output.trim());
12
+ const xml = await run(['shell', 'cat', remotePath]);
13
+ if (!xml.includes('<hierarchy') || !xml.includes('</hierarchy>')) {
14
+ throw new Error('设备未返回有效的组件树');
15
+ }
16
+ return xml;
17
+ } catch (error) {
18
+ throw new Error(`刷新组件树失败:${error instanceof Error ? error.message : String(error)}`);
19
+ } finally {
20
+ await run(['shell', 'rm', '-f', remotePath]).catch(() => undefined);
21
+ }
22
+ }
@@ -0,0 +1,184 @@
1
+ import { PNG } from 'pngjs';
2
+ import pixelmatch from 'pixelmatch';
3
+
4
+ export type VisualChangeRegion = {
5
+ x: number;
6
+ y: number;
7
+ width: number;
8
+ height: number;
9
+ };
10
+
11
+ export type VisualChangeCheckInput = {
12
+ captureScreenshot: () => Promise<Buffer>;
13
+ wait: (ms: number) => Promise<void>;
14
+ region: VisualChangeRegion;
15
+ durationMs: number;
16
+ intervalMs: number;
17
+ changeRatioThreshold: number;
18
+ pixelmatchThreshold: number;
19
+ };
20
+
21
+ export type VisualChangeFrameCompareInput = {
22
+ baselineScreenshot: Buffer;
23
+ comparisonScreenshot: Buffer;
24
+ region: VisualChangeRegion;
25
+ changeRatioThreshold: number;
26
+ pixelmatchThreshold: number;
27
+ };
28
+
29
+ export type VisualChangeCheckResult = {
30
+ passed: boolean;
31
+ region: VisualChangeRegion;
32
+ durationMs: number;
33
+ intervalMs: number;
34
+ sampleCount: number;
35
+ changeRatioThreshold: number;
36
+ maxChangeRatio: number;
37
+ baselineBase64: string;
38
+ comparisonBase64: string;
39
+ diffBase64: string;
40
+ message: string;
41
+ };
42
+
43
+ function clampInteger(value: number, min: number, max: number) {
44
+ return Math.max(min, Math.min(max, Math.round(value)));
45
+ }
46
+
47
+ function normalizeRegion(region: VisualChangeRegion, imageWidth: number, imageHeight: number) {
48
+ const x = clampInteger(region.x, 0, Math.max(0, imageWidth - 1));
49
+ const y = clampInteger(region.y, 0, Math.max(0, imageHeight - 1));
50
+ const width = clampInteger(region.width, 1, imageWidth - x);
51
+ const height = clampInteger(region.height, 1, imageHeight - y);
52
+ return { x, y, width, height };
53
+ }
54
+
55
+ function readPng(buffer: Buffer) {
56
+ try {
57
+ return PNG.sync.read(buffer);
58
+ } catch (error) {
59
+ throw new Error('截图不是有效的 PNG 数据', { cause: error });
60
+ }
61
+ }
62
+
63
+ function cropPng(source: PNG, region: VisualChangeRegion) {
64
+ const crop = new PNG({ width: region.width, height: region.height });
65
+ for (let row = 0; row < region.height; row += 1) {
66
+ const sourceStart = ((region.y + row) * source.width + region.x) * 4;
67
+ const sourceEnd = sourceStart + region.width * 4;
68
+ const targetStart = row * region.width * 4;
69
+ source.data.copy(crop.data, targetStart, sourceStart, sourceEnd);
70
+ }
71
+ return crop;
72
+ }
73
+
74
+ function pngToBase64(png: PNG) {
75
+ return PNG.sync.write(png).toString('base64');
76
+ }
77
+
78
+ function compareCroppedImages(
79
+ baseline: PNG,
80
+ current: PNG,
81
+ pixelmatchThreshold: number,
82
+ ) {
83
+ const diff = new PNG({ width: baseline.width, height: baseline.height });
84
+ const differentPixels = pixelmatch(
85
+ baseline.data,
86
+ current.data,
87
+ diff.data,
88
+ baseline.width,
89
+ baseline.height,
90
+ {
91
+ threshold: pixelmatchThreshold,
92
+ includeAA: true,
93
+ },
94
+ );
95
+ const totalPixels = baseline.width * baseline.height;
96
+ return {
97
+ diff,
98
+ changedRatio: totalPixels ? (differentPixels / totalPixels) * 100 : 0,
99
+ };
100
+ }
101
+
102
+ // Screenshots can contain tiny encoding or antialiasing differences, so the
103
+ // caller provides a ratio threshold instead of expecting exact image equality.
104
+ export function compareVisualChangeFrames(input: VisualChangeFrameCompareInput): VisualChangeCheckResult {
105
+ const baselineScreenshot = readPng(input.baselineScreenshot);
106
+ const comparisonScreenshot = readPng(input.comparisonScreenshot);
107
+ const region = normalizeRegion(
108
+ input.region,
109
+ Math.min(baselineScreenshot.width, comparisonScreenshot.width),
110
+ Math.min(baselineScreenshot.height, comparisonScreenshot.height),
111
+ );
112
+ const baseline = cropPng(baselineScreenshot, region);
113
+ const comparison = cropPng(comparisonScreenshot, region);
114
+ const diff = compareCroppedImages(baseline, comparison, input.pixelmatchThreshold);
115
+ const passed = diff.changedRatio >= input.changeRatioThreshold;
116
+ return {
117
+ passed,
118
+ region,
119
+ durationMs: 0,
120
+ intervalMs: 0,
121
+ sampleCount: 2,
122
+ changeRatioThreshold: input.changeRatioThreshold,
123
+ maxChangeRatio: diff.changedRatio,
124
+ baselineBase64: pngToBase64(baseline),
125
+ comparisonBase64: pngToBase64(comparison),
126
+ diffBase64: pngToBase64(diff.diff),
127
+ message: passed
128
+ ? `起止节点截图发生变化,变化比例 ${diff.changedRatio.toFixed(2)}%,达到阈值 ${input.changeRatioThreshold}%`
129
+ : `起止节点截图未发生明显变化,变化比例 ${diff.changedRatio.toFixed(2)}%,低于阈值 ${input.changeRatioThreshold}%`,
130
+ };
131
+ }
132
+
133
+ export async function detectVisualChange(input: VisualChangeCheckInput): Promise<VisualChangeCheckResult> {
134
+ const firstScreenshot = readPng(await input.captureScreenshot());
135
+ const region = normalizeRegion(input.region, firstScreenshot.width, firstScreenshot.height);
136
+ const baseline = cropPng(firstScreenshot, region);
137
+ let sampleCount = 0;
138
+ let maxChangeRatio = 0;
139
+ let comparison = baseline;
140
+ let diffImage = new PNG({ width: baseline.width, height: baseline.height });
141
+ const startedAt = Date.now();
142
+ const deadline = startedAt + Math.max(1000, input.durationMs);
143
+ const intervalMs = Math.max(200, input.intervalMs);
144
+
145
+ while (Date.now() < deadline) {
146
+ await input.wait(Math.min(intervalMs, Math.max(0, deadline - Date.now())));
147
+ const currentScreenshot = readPng(await input.captureScreenshot());
148
+ const current = cropPng(currentScreenshot, region);
149
+ const diff = compareCroppedImages(baseline, current, input.pixelmatchThreshold);
150
+ sampleCount += 1;
151
+ comparison = current;
152
+ diffImage = diff.diff;
153
+ maxChangeRatio = Math.max(maxChangeRatio, diff.changedRatio);
154
+ if (diff.changedRatio >= input.changeRatioThreshold) {
155
+ return {
156
+ passed: true,
157
+ region,
158
+ durationMs: input.durationMs,
159
+ intervalMs,
160
+ sampleCount,
161
+ changeRatioThreshold: input.changeRatioThreshold,
162
+ maxChangeRatio,
163
+ baselineBase64: pngToBase64(baseline),
164
+ comparisonBase64: pngToBase64(comparison),
165
+ diffBase64: pngToBase64(diffImage),
166
+ message: `检测到画面变化,最大变化比例 ${maxChangeRatio.toFixed(2)}%,达到阈值 ${input.changeRatioThreshold}%`,
167
+ };
168
+ }
169
+ }
170
+
171
+ return {
172
+ passed: false,
173
+ region,
174
+ durationMs: input.durationMs,
175
+ intervalMs,
176
+ sampleCount,
177
+ changeRatioThreshold: input.changeRatioThreshold,
178
+ maxChangeRatio,
179
+ baselineBase64: pngToBase64(baseline),
180
+ comparisonBase64: pngToBase64(comparison),
181
+ diffBase64: pngToBase64(diffImage),
182
+ message: `${input.durationMs}ms 内目标区域未发生明显变化,最大变化比例 ${maxChangeRatio.toFixed(2)}%,低于阈值 ${input.changeRatioThreshold}%`,
183
+ };
184
+ }
package/server/config.ts CHANGED
@@ -4,8 +4,10 @@ import path from 'node:path';
4
4
  import YAML from 'yaml';
5
5
  import { appDataPath } from './paths';
6
6
  import { loadModelConfigFromDb, saveModelConfigToDb } from './config-store';
7
+ import type { AiRecognitionModel } from '../src/appium-recorder/ai-recognition';
7
8
 
8
9
  export type AppConfig = {
10
+ appium: { model: AiRecognitionModel };
9
11
  runtime: {
10
12
  androidSdkPath: string;
11
13
  reportOutputPath: string;
@@ -43,6 +45,7 @@ export class ConfigValidationError extends Error {
43
45
 
44
46
  function defaultConfig(): AppConfig {
45
47
  return {
48
+ appium: { model: { baseUrl: '', apiKey: '', name: '' } },
46
49
  runtime: {
47
50
  androidSdkPath: '',
48
51
  reportOutputPath: '',
@@ -78,6 +81,13 @@ function normalizeEnv(env: unknown): Record<string, string> {
78
81
  function normalizeConfig(config: Partial<AppConfig> | null | undefined): AppConfig {
79
82
  const fallback = defaultConfig();
80
83
  return {
84
+ // 旧配置没有 Appium 模型时保留空值,不借用其他模型或改变已有配置。
85
+ appium: {
86
+ model: Object.fromEntries(['baseUrl', 'apiKey', 'name'].map((key) => {
87
+ const value = config?.appium?.model?.[key as keyof AiRecognitionModel];
88
+ return [key, typeof value === 'string' ? value.trim() : ''];
89
+ })) as AiRecognitionModel,
90
+ },
81
91
  runtime: {
82
92
  androidSdkPath: config?.runtime?.androidSdkPath?.trim() || fallback.runtime.androidSdkPath,
83
93
  reportOutputPath: config?.runtime?.reportOutputPath?.trim() || fallback.runtime.reportOutputPath,
@@ -106,10 +106,9 @@ const START_ANDROID_PLAYGROUND_WITHOUT_BROWSER = [
106
106
  "const { androidPlaygroundPlatform, ScrcpyServer } = require('@midscene/android-playground');",
107
107
  "const { launchPreparedPlaygroundPlatform } = require('@midscene/playground');",
108
108
  'const STABLE_PREVIEW_OPTIONS = {',
109
- ' maxFps: 15,',
110
- ' maxSize: 1024,',
111
- ' videoBitRate: 3000000,',
112
- " videoCodecOptions: 'i-frame-interval=0',",
109
+ ' maxFps: 10,',
110
+ ' maxSize: 800,',
111
+ ' videoBitRate: 1000000,',
113
112
  '};',
114
113
  'async function main() {',
115
114
  ' const scrcpyServer = new ScrcpyServer();',
@@ -1071,7 +1070,7 @@ export function createApiMiddleware() {
1071
1070
  if (req.url === '/api/test-model' && req.method === 'POST') {
1072
1071
  try {
1073
1072
  const parsed = await readBody<{
1074
- modelKey?: 'midscene' | 'scriptOptimizer';
1073
+ modelKey?: 'midscene' | 'scriptOptimizer' | 'appium';
1075
1074
  model?: {
1076
1075
  provider?: 'custom' | 'codex';
1077
1076
  baseUrl?: string;
package/src/App.vue CHANGED
@@ -1,4 +1,5 @@
1
1
  <script setup lang="ts">
2
+ import { isAiRecognitionModelConfigured } from './appium-recorder/ai-recognition';
2
3
  import { computed, onMounted, onUnmounted, reactive, ref, shallowRef, watch } from 'vue';
3
4
  import { ElMessage, ElMessageBox } from 'element-plus';
4
5
  import {
@@ -99,7 +100,9 @@ const appPresets = ref<AppPreset[]>([]);
99
100
  const modelTestStatus = reactive({
100
101
  midscene: '',
101
102
  scriptOptimizer: '',
103
+ appium: '',
102
104
  });
105
+ const aiRecognitionModelConfigured = shallowRef(false);
103
106
  const actionDialog = reactive({
104
107
  visible: false,
105
108
  title: '',
@@ -149,6 +152,7 @@ const appPresetForm = reactive({
149
152
  });
150
153
 
151
154
  const configForm = reactive<ConfigForm>({
155
+ appium: { model: { baseUrl: '', apiKey: '', name: '' } },
152
156
  runtime: {
153
157
  androidSdkPath: '',
154
158
  reportOutputPath: '',
@@ -656,6 +660,8 @@ const loadConfig = async () => {
656
660
  });
657
661
  Object.assign(configForm.midscene.env, payload.midscene.env || {});
658
662
  Object.assign(configForm.scriptOptimizer.model, payload.scriptOptimizer.model);
663
+ Object.assign(configForm.appium.model, payload.appium?.model || { baseUrl: '', apiKey: '', name: '' });
664
+ aiRecognitionModelConfigured.value = isAiRecognitionModelConfigured(configForm.appium.model);
659
665
  if (configForm.midscene.model.provider !== 'codex') {
660
666
  rememberCustomMidsceneModel();
661
667
  }
@@ -692,7 +698,7 @@ const getMidsceneModelConfigError = () => {
692
698
  if (model.provider !== 'codex' && !model.apiKey.trim()) missing.push('API Key');
693
699
 
694
700
  if (!missing.length) return '';
695
- return `Midscene 模型配置不完整:缺少 ${missing.join('、')}。请在“参数配置 > Midscene 模型”中选择“使用 Codex”,或补齐自定义提供方后保存。`;
701
+ return `Midscene 模型配置不完整:缺少 ${missing.join('、')}。请在“参数配置 > Midscene配置 > Midscene 模型”中选择“使用 Codex”,或补齐自定义提供方后保存。`;
696
702
  };
697
703
 
698
704
  const saveModelConfig = async () => {
@@ -700,7 +706,10 @@ const saveModelConfig = async () => {
700
706
  errorMessage.value = '';
701
707
  openActionDialog('保存参数配置');
702
708
  try {
703
- await api.saveConfig(configForm);
709
+ // 保存的是本次提交快照,不能把请求期间尚未保存的改动标记为已配置。
710
+ const snapshot = JSON.parse(JSON.stringify(configForm)) as ConfigForm;
711
+ await api.saveConfig(snapshot);
712
+ aiRecognitionModelConfigured.value = isAiRecognitionModelConfigured(snapshot.appium.model);
704
713
  closeActionDialog();
705
714
  ElMessage.success('参数配置已保存');
706
715
  } catch (error) {
@@ -776,14 +785,11 @@ const removeAppPreset = async (id: string) => {
776
785
  }
777
786
  };
778
787
 
779
- const testModel = async (key: 'midscene' | 'scriptOptimizer') => {
788
+ const testModel = async (key: 'midscene' | 'scriptOptimizer' | 'appium') => {
780
789
  testingModelKey.value = key;
781
790
  errorMessage.value = '';
782
791
  modelTestStatus[key] = '';
783
- const model =
784
- key === 'midscene'
785
- ? configForm.midscene.model
786
- : configForm.scriptOptimizer.model;
792
+ const model = configForm[key].model;
787
793
 
788
794
  try {
789
795
  const payload = await api.testModel({ modelKey: key, model });
@@ -1557,6 +1563,7 @@ onUnmounted(() => {
1557
1563
  v-show="activeMenu === 'appium'"
1558
1564
  :active="activeMenu === 'appium'"
1559
1565
  :app-presets="appPresets"
1566
+ :ai-recognition-model-configured="aiRecognitionModelConfigured"
1560
1567
  :device-actions="deviceActions"
1561
1568
  :playground-available="playgroundAvailable"
1562
1569
  :playground-device-id="playgroundDeviceId"
package/src/api.ts CHANGED
@@ -116,7 +116,7 @@ export function saveConfig(input: ConfigForm) {
116
116
 
117
117
  // 测试指定模型配置是否可用,通常用于保存前验证 baseUrl/apiKey/model。
118
118
  export function testModel(input: {
119
- modelKey: 'midscene' | 'scriptOptimizer';
119
+ modelKey: 'midscene' | 'scriptOptimizer' | 'appium';
120
120
  model: {
121
121
  provider?: 'custom' | 'codex';
122
122
  baseUrl: string;