android-midscene-automation 0.1.26 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/FAQ.md +14 -0
- package/README.md +39 -3
- package/bin/android-midscene-automation.js +71 -1
- package/package.json +7 -2
- package/remote-agent/index.ts +2 -11
- package/server/appium-recorder/ai-recognition.ts +67 -0
- package/server/appium-recorder/appium-runner.ts +335 -52
- package/server/appium-recorder/report.ts +297 -16
- package/server/appium-recorder/repository.ts +31 -4
- package/server/appium-recorder/routes.ts +31 -32
- package/server/appium-recorder/screenshot.ts +21 -0
- package/server/appium-recorder/script-export.ts +67 -0
- package/server/appium-recorder/tree-dump.ts +22 -0
- package/server/appium-recorder/visual-change.ts +184 -0
- package/server/config.ts +10 -0
- package/server/http-api.ts +4 -5
- package/src/App.vue +14 -7
- package/src/api.ts +1 -1
- package/src/appium-recorder/AppiumPage.vue +658 -162
- package/src/appium-recorder/ai-recognition.ts +16 -0
- package/src/appium-recorder/api.ts +20 -3
- package/src/appium-recorder/components/AiRecognitionTestDialog.vue +71 -0
- package/src/appium-recorder/components/FlowActionMenu.vue +206 -0
- package/src/appium-recorder/components/FlowCanvas.vue +16 -1
- package/src/appium-recorder/components/FlowNodeCard.vue +23 -37
- package/src/appium-recorder/components/FlowStepEditor.vue +152 -25
- package/src/appium-recorder/components/LongPressSettings.vue +58 -0
- package/src/appium-recorder/components/NestedConditionBranches.vue +55 -35
- package/src/appium-recorder/components/NewScriptDialog.vue +27 -0
- package/src/appium-recorder/components/NodeDetail.vue +2 -0
- package/src/appium-recorder/components/RecordedSteps.vue +24 -4
- package/src/appium-recorder/components/StageLogSettings.vue +17 -0
- package/src/appium-recorder/components/TextClickSettings.vue +19 -0
- package/src/appium-recorder/components/VisualChangeDialog.vue +151 -0
- package/src/appium-recorder/flow-copy.ts +47 -1
- package/src/appium-recorder/flow-graph.ts +147 -64
- package/src/appium-recorder/flow-labels.ts +35 -6
- package/src/appium-recorder/flow-normalize.ts +76 -0
- package/src/appium-recorder/flow-remove.ts +69 -0
- package/src/appium-recorder/long-press.ts +27 -0
- package/src/appium-recorder/native-control-state.ts +26 -0
- package/src/appium-recorder/stage-log.ts +21 -0
- package/src/appium-recorder/text-click.ts +9 -0
- package/src/appium-recorder/tree.ts +3 -0
- package/src/appium-recorder/types.ts +34 -2
- package/src/appium-recorder/visual-change-pairs.ts +147 -0
- package/src/appium-recorder/visual-change.ts +75 -0
- package/src/components/config/AppiumConfigPanel.vue +48 -0
- package/src/components/config/BaseConfigPanel.vue +90 -0
- package/src/components/config/MidsceneConfigPanel.vue +204 -0
- package/src/components/device/DevicePreviewPanel.vue +148 -4
- package/src/pages/ConfigPage.vue +107 -265
- package/src/style.css +43 -4
- package/src/types.ts +2 -0
- package/src/ui-refresh.css +5 -0
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { execFile } from 'node:child_process';
|
|
2
|
-
import
|
|
3
|
-
import
|
|
4
|
-
import path from 'node:path';
|
|
2
|
+
import { recognizeDeviceScreen } from './ai-recognition';
|
|
3
|
+
import { readFreshWindowHierarchy } from './tree-dump';
|
|
5
4
|
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
6
5
|
import {
|
|
7
6
|
deleteAppiumRecordedScript,
|
|
@@ -14,6 +13,7 @@ import {
|
|
|
14
13
|
import { clearAppDataOnDevice, launchAppOnDevice, replayAppiumScript } from './appium-runner';
|
|
15
14
|
import { isRemoteDeviceId, sendRemoteCommand } from '../remote-agents/registry';
|
|
16
15
|
import { getAdbCommand } from '../android-sdk';
|
|
16
|
+
import { createAppiumScriptExport } from './script-export';
|
|
17
17
|
|
|
18
18
|
const treeDumpTasks = new Map<string, Promise<string>>();
|
|
19
19
|
const replayingDevices = new Set<string>();
|
|
@@ -25,13 +25,6 @@ function sendJson(res: ServerResponse, payload: unknown, statusCode = 200) {
|
|
|
25
25
|
res.end(JSON.stringify(payload));
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
function sendJsonDownload(res: ServerResponse, payload: unknown, fileName: string) {
|
|
29
|
-
res.statusCode = 200;
|
|
30
|
-
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
|
31
|
-
res.setHeader('Content-Disposition', `attachment; filename="appium-script.json"; filename*=UTF-8''${encodeURIComponent(fileName)}`);
|
|
32
|
-
res.end(JSON.stringify(payload, null, 2));
|
|
33
|
-
}
|
|
34
|
-
|
|
35
28
|
function sendStreamEvent(res: ServerResponse, payload: unknown) {
|
|
36
29
|
res.write(`${JSON.stringify(payload)}\n`);
|
|
37
30
|
}
|
|
@@ -64,17 +57,7 @@ async function dumpWindowHierarchy(deviceId: string) {
|
|
|
64
57
|
const activeTask = treeDumpTasks.get(deviceId);
|
|
65
58
|
if (activeTask) return activeTask;
|
|
66
59
|
|
|
67
|
-
const task = (
|
|
68
|
-
const localPath = path.join(os.tmpdir(), `midscene-appium-${deviceId.replace(/[^\w.-]/g, '_')}-${Date.now()}.xml`);
|
|
69
|
-
const remotePath = '/data/local/tmp/midscene_appium_uidump.xml';
|
|
70
|
-
await execFileText(getAdbCommand(), ['-s', deviceId, 'shell', 'uiautomator', 'dump', remotePath]);
|
|
71
|
-
await execFileText(getAdbCommand(), ['-s', deviceId, 'pull', remotePath, localPath]);
|
|
72
|
-
try {
|
|
73
|
-
return await fs.readFile(localPath, 'utf8');
|
|
74
|
-
} finally {
|
|
75
|
-
await fs.unlink(localPath).catch(() => undefined);
|
|
76
|
-
}
|
|
77
|
-
})();
|
|
60
|
+
const task = readFreshWindowHierarchy(deviceId, getAdbCommand(), execFileText);
|
|
78
61
|
treeDumpTasks.set(deviceId, task);
|
|
79
62
|
try {
|
|
80
63
|
return await task;
|
|
@@ -134,6 +117,7 @@ export async function handleAppiumRecorderRequest(
|
|
|
134
117
|
};
|
|
135
118
|
|
|
136
119
|
if (pathname === '/api/appium-recorder/tree' && req.method === 'GET') {
|
|
120
|
+
res.setHeader('Cache-Control', 'no-store');
|
|
137
121
|
const deviceId = requestUrl.searchParams.get('deviceId')?.trim() || selectedDeviceId;
|
|
138
122
|
if (!deviceId) throw new Error('未检测到可用设备');
|
|
139
123
|
assertDeviceAllowed(deviceId);
|
|
@@ -192,6 +176,24 @@ export async function handleAppiumRecorderRequest(
|
|
|
192
176
|
return true;
|
|
193
177
|
}
|
|
194
178
|
|
|
179
|
+
if (pathname === '/api/appium-recorder/ai-recognition/test' && req.method === 'POST') {
|
|
180
|
+
const parsed = await readBody<{ deviceId?: string; prompt?: unknown; timeoutMs?: number }>(req);
|
|
181
|
+
const deviceId = parsed.deviceId?.trim() || selectedDeviceId;
|
|
182
|
+
if (!deviceId) throw new Error('请选择设备');
|
|
183
|
+
assertDeviceAllowed(deviceId);
|
|
184
|
+
if (isRemoteDeviceId(deviceId)) throw new Error('远程设备暂不支持 AI 识别测试');
|
|
185
|
+
if (replayingDevices.has(deviceId)) { sendJson(res, { message: '设备正在回放,请稍后测试' }, 409); return true; }
|
|
186
|
+
const controller = new AbortController();
|
|
187
|
+
const abort = () => controller.abort();
|
|
188
|
+
res.once('close', abort);
|
|
189
|
+
try {
|
|
190
|
+
sendJson(res, await recognizeDeviceScreen({ deviceId, prompt: parsed.prompt, timeoutMs: parsed.timeoutMs, signal: controller.signal }));
|
|
191
|
+
} finally {
|
|
192
|
+
res.off('close', abort);
|
|
193
|
+
}
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
|
|
195
197
|
if (pathname === '/api/appium-recorder/launch-app' && req.method === 'POST') {
|
|
196
198
|
const parsed = await readBody<{ deviceId?: string; packageName?: string }>(req);
|
|
197
199
|
const deviceId = parsed.deviceId?.trim() || selectedDeviceId;
|
|
@@ -277,17 +279,14 @@ export async function handleAppiumRecorderRequest(
|
|
|
277
279
|
if (exportMatch && req.method === 'GET') {
|
|
278
280
|
const script = getAppiumRecordedScript(decodeURIComponent(exportMatch[1]));
|
|
279
281
|
if (!script) throw new Error('Appium 录制脚本不存在');
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
steps: script.steps,
|
|
289
|
-
},
|
|
290
|
-
}, `${script.name}.json`);
|
|
282
|
+
const exported = await createAppiumScriptExport(script, getAppiumRecordedScript);
|
|
283
|
+
const fileName = encodeURIComponent(exported.fileName)
|
|
284
|
+
.replace(/['()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
285
|
+
res.statusCode = 200;
|
|
286
|
+
res.setHeader('Content-Type', exported.contentType);
|
|
287
|
+
res.setHeader('Content-Disposition', `attachment; filename="appium-script.${exported.contentType === 'application/zip' ? 'zip' : 'json'}"; filename*=UTF-8''${fileName}`);
|
|
288
|
+
res.setHeader('Content-Length', exported.body.length);
|
|
289
|
+
res.end(exported.body);
|
|
291
290
|
return true;
|
|
292
291
|
}
|
|
293
292
|
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { getAdbCommand } from '../android-sdk';
|
|
3
|
+
|
|
4
|
+
// 直接读取设备截图,不使用预览缓存,也不进入 Appium 命令队列。
|
|
5
|
+
export function adbScreenshotBase64(deviceId: string, signal?: AbortSignal) {
|
|
6
|
+
return new Promise<string>((resolve, reject) => {
|
|
7
|
+
execFile(
|
|
8
|
+
getAdbCommand(),
|
|
9
|
+
['-s', deviceId, 'exec-out', 'screencap', '-p'],
|
|
10
|
+
{ encoding: 'buffer', maxBuffer: 20 * 1024 * 1024, timeout: 8000, signal },
|
|
11
|
+
(error, stdout, stderr) => {
|
|
12
|
+
if (error) {
|
|
13
|
+
reject(new Error(stderr?.toString('utf8').trim() || error.message || 'ADB 截图失败'));
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
if (!stdout.length) { reject(new Error('ADB 未返回截图内容')); return; }
|
|
17
|
+
resolve(stdout.toString('base64'));
|
|
18
|
+
},
|
|
19
|
+
);
|
|
20
|
+
});
|
|
21
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { buffer } from 'node:stream/consumers';
|
|
2
|
+
import archiver from 'archiver';
|
|
3
|
+
import type { AppiumRecordedScriptRecord } from './repository';
|
|
4
|
+
|
|
5
|
+
function safeExportName(name: string) {
|
|
6
|
+
const base = Array.from(name.trim().replace(/[<>:"/\\|?*\u0000-\u001f]/g, '_'))
|
|
7
|
+
.slice(0, 80).join('').replace(/[. ]+$/, '') || 'script';
|
|
8
|
+
return /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\.|$)/i.test(base) ? `_${base}` : base;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export async function createAppiumScriptExport(
|
|
12
|
+
root: AppiumRecordedScriptRecord,
|
|
13
|
+
getScript: (id: string) => AppiumRecordedScriptRecord | null,
|
|
14
|
+
) {
|
|
15
|
+
const scripts = [root];
|
|
16
|
+
const visited = new Set([root.id]);
|
|
17
|
+
// 按脚本 ID 去重,既保留所有分支的依赖,也避免重复引用和循环连接无限展开。
|
|
18
|
+
for (const script of scripts) {
|
|
19
|
+
for (const step of script.steps) {
|
|
20
|
+
if (step.type !== 'runScript') continue;
|
|
21
|
+
const targetId = step.value || '';
|
|
22
|
+
if (!targetId) throw new Error(`脚本「${script.name}」的「${step.label}」缺少连接脚本 ID,无法导出`);
|
|
23
|
+
if (visited.has(targetId)) continue;
|
|
24
|
+
const target = getScript(targetId);
|
|
25
|
+
if (!target) throw new Error(`脚本「${script.name}」连接的脚本不存在(${targetId}),无法导出完整脚本包`);
|
|
26
|
+
visited.add(targetId);
|
|
27
|
+
scripts.push(target);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const exportedAt = new Date().toISOString();
|
|
32
|
+
const serialize = (script: AppiumRecordedScriptRecord) => JSON.stringify({
|
|
33
|
+
schemaVersion: 1,
|
|
34
|
+
exportedAt,
|
|
35
|
+
script: {
|
|
36
|
+
id: script.id,
|
|
37
|
+
name: script.name,
|
|
38
|
+
appPackage: script.appPackage,
|
|
39
|
+
appActivity: script.appActivity,
|
|
40
|
+
deviceId: script.deviceId,
|
|
41
|
+
steps: script.steps,
|
|
42
|
+
},
|
|
43
|
+
}, null, 2);
|
|
44
|
+
const baseName = safeExportName(root.name);
|
|
45
|
+
if (!root.steps.some((step) => step.type === 'runScript')) {
|
|
46
|
+
return {
|
|
47
|
+
fileName: `${baseName}.json`,
|
|
48
|
+
contentType: 'application/json; charset=utf-8',
|
|
49
|
+
body: Buffer.from(serialize(root)),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const archive = archiver('zip', { zlib: { level: 9 } });
|
|
54
|
+
const usedNames = new Set<string>();
|
|
55
|
+
for (const script of scripts) {
|
|
56
|
+
const name = safeExportName(script.name);
|
|
57
|
+
let fileName = `${name}.json`;
|
|
58
|
+
let suffix = 2;
|
|
59
|
+
// 清理非法字符后可能重名,Windows 解压时还需忽略大小写进行去重。
|
|
60
|
+
while (usedNames.has(fileName.toLowerCase())) fileName = `${name} (${suffix++}).json`;
|
|
61
|
+
usedNames.add(fileName.toLowerCase());
|
|
62
|
+
archive.append(serialize(script), { name: fileName });
|
|
63
|
+
}
|
|
64
|
+
// 生成完整压缩包后再发送响应,压缩失败仍可返回普通 JSON 错误。
|
|
65
|
+
const [body] = await Promise.all([buffer(archive), archive.finalize()]);
|
|
66
|
+
return { fileName: `${baseName}.zip`, contentType: 'application/zip', body };
|
|
67
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
type ExecText = (command: string, args: string[]) => Promise<string>;
|
|
4
|
+
|
|
5
|
+
export async function readFreshWindowHierarchy(deviceId: string, adb: string, execText: ExecText) {
|
|
6
|
+
// dump 失败可能仍以 0 退出。每次独立文件,避免误读上一次成功抓取的树。
|
|
7
|
+
const remotePath = `/data/local/tmp/midscene_uidump_${randomUUID()}.xml`;
|
|
8
|
+
const run = (args: string[]) => execText(adb, ['-s', deviceId, ...args]);
|
|
9
|
+
try {
|
|
10
|
+
const output = await run(['shell', 'uiautomator', 'dump', remotePath]);
|
|
11
|
+
if (/ERROR:|could not|failed/i.test(output)) throw new Error(output.trim());
|
|
12
|
+
const xml = await run(['shell', 'cat', remotePath]);
|
|
13
|
+
if (!xml.includes('<hierarchy') || !xml.includes('</hierarchy>')) {
|
|
14
|
+
throw new Error('设备未返回有效的组件树');
|
|
15
|
+
}
|
|
16
|
+
return xml;
|
|
17
|
+
} catch (error) {
|
|
18
|
+
throw new Error(`刷新组件树失败:${error instanceof Error ? error.message : String(error)}`);
|
|
19
|
+
} finally {
|
|
20
|
+
await run(['shell', 'rm', '-f', remotePath]).catch(() => undefined);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { PNG } from 'pngjs';
|
|
2
|
+
import pixelmatch from 'pixelmatch';
|
|
3
|
+
|
|
4
|
+
export type VisualChangeRegion = {
|
|
5
|
+
x: number;
|
|
6
|
+
y: number;
|
|
7
|
+
width: number;
|
|
8
|
+
height: number;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
export type VisualChangeCheckInput = {
|
|
12
|
+
captureScreenshot: () => Promise<Buffer>;
|
|
13
|
+
wait: (ms: number) => Promise<void>;
|
|
14
|
+
region: VisualChangeRegion;
|
|
15
|
+
durationMs: number;
|
|
16
|
+
intervalMs: number;
|
|
17
|
+
changeRatioThreshold: number;
|
|
18
|
+
pixelmatchThreshold: number;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export type VisualChangeFrameCompareInput = {
|
|
22
|
+
baselineScreenshot: Buffer;
|
|
23
|
+
comparisonScreenshot: Buffer;
|
|
24
|
+
region: VisualChangeRegion;
|
|
25
|
+
changeRatioThreshold: number;
|
|
26
|
+
pixelmatchThreshold: number;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export type VisualChangeCheckResult = {
|
|
30
|
+
passed: boolean;
|
|
31
|
+
region: VisualChangeRegion;
|
|
32
|
+
durationMs: number;
|
|
33
|
+
intervalMs: number;
|
|
34
|
+
sampleCount: number;
|
|
35
|
+
changeRatioThreshold: number;
|
|
36
|
+
maxChangeRatio: number;
|
|
37
|
+
baselineBase64: string;
|
|
38
|
+
comparisonBase64: string;
|
|
39
|
+
diffBase64: string;
|
|
40
|
+
message: string;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
function clampInteger(value: number, min: number, max: number) {
|
|
44
|
+
return Math.max(min, Math.min(max, Math.round(value)));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function normalizeRegion(region: VisualChangeRegion, imageWidth: number, imageHeight: number) {
|
|
48
|
+
const x = clampInteger(region.x, 0, Math.max(0, imageWidth - 1));
|
|
49
|
+
const y = clampInteger(region.y, 0, Math.max(0, imageHeight - 1));
|
|
50
|
+
const width = clampInteger(region.width, 1, imageWidth - x);
|
|
51
|
+
const height = clampInteger(region.height, 1, imageHeight - y);
|
|
52
|
+
return { x, y, width, height };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function readPng(buffer: Buffer) {
|
|
56
|
+
try {
|
|
57
|
+
return PNG.sync.read(buffer);
|
|
58
|
+
} catch (error) {
|
|
59
|
+
throw new Error('截图不是有效的 PNG 数据', { cause: error });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function cropPng(source: PNG, region: VisualChangeRegion) {
|
|
64
|
+
const crop = new PNG({ width: region.width, height: region.height });
|
|
65
|
+
for (let row = 0; row < region.height; row += 1) {
|
|
66
|
+
const sourceStart = ((region.y + row) * source.width + region.x) * 4;
|
|
67
|
+
const sourceEnd = sourceStart + region.width * 4;
|
|
68
|
+
const targetStart = row * region.width * 4;
|
|
69
|
+
source.data.copy(crop.data, targetStart, sourceStart, sourceEnd);
|
|
70
|
+
}
|
|
71
|
+
return crop;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function pngToBase64(png: PNG) {
|
|
75
|
+
return PNG.sync.write(png).toString('base64');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function compareCroppedImages(
|
|
79
|
+
baseline: PNG,
|
|
80
|
+
current: PNG,
|
|
81
|
+
pixelmatchThreshold: number,
|
|
82
|
+
) {
|
|
83
|
+
const diff = new PNG({ width: baseline.width, height: baseline.height });
|
|
84
|
+
const differentPixels = pixelmatch(
|
|
85
|
+
baseline.data,
|
|
86
|
+
current.data,
|
|
87
|
+
diff.data,
|
|
88
|
+
baseline.width,
|
|
89
|
+
baseline.height,
|
|
90
|
+
{
|
|
91
|
+
threshold: pixelmatchThreshold,
|
|
92
|
+
includeAA: true,
|
|
93
|
+
},
|
|
94
|
+
);
|
|
95
|
+
const totalPixels = baseline.width * baseline.height;
|
|
96
|
+
return {
|
|
97
|
+
diff,
|
|
98
|
+
changedRatio: totalPixels ? (differentPixels / totalPixels) * 100 : 0,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Screenshots can contain tiny encoding or antialiasing differences, so the
|
|
103
|
+
// caller provides a ratio threshold instead of expecting exact image equality.
|
|
104
|
+
export function compareVisualChangeFrames(input: VisualChangeFrameCompareInput): VisualChangeCheckResult {
|
|
105
|
+
const baselineScreenshot = readPng(input.baselineScreenshot);
|
|
106
|
+
const comparisonScreenshot = readPng(input.comparisonScreenshot);
|
|
107
|
+
const region = normalizeRegion(
|
|
108
|
+
input.region,
|
|
109
|
+
Math.min(baselineScreenshot.width, comparisonScreenshot.width),
|
|
110
|
+
Math.min(baselineScreenshot.height, comparisonScreenshot.height),
|
|
111
|
+
);
|
|
112
|
+
const baseline = cropPng(baselineScreenshot, region);
|
|
113
|
+
const comparison = cropPng(comparisonScreenshot, region);
|
|
114
|
+
const diff = compareCroppedImages(baseline, comparison, input.pixelmatchThreshold);
|
|
115
|
+
const passed = diff.changedRatio >= input.changeRatioThreshold;
|
|
116
|
+
return {
|
|
117
|
+
passed,
|
|
118
|
+
region,
|
|
119
|
+
durationMs: 0,
|
|
120
|
+
intervalMs: 0,
|
|
121
|
+
sampleCount: 2,
|
|
122
|
+
changeRatioThreshold: input.changeRatioThreshold,
|
|
123
|
+
maxChangeRatio: diff.changedRatio,
|
|
124
|
+
baselineBase64: pngToBase64(baseline),
|
|
125
|
+
comparisonBase64: pngToBase64(comparison),
|
|
126
|
+
diffBase64: pngToBase64(diff.diff),
|
|
127
|
+
message: passed
|
|
128
|
+
? `起止节点截图发生变化,变化比例 ${diff.changedRatio.toFixed(2)}%,达到阈值 ${input.changeRatioThreshold}%`
|
|
129
|
+
: `起止节点截图未发生明显变化,变化比例 ${diff.changedRatio.toFixed(2)}%,低于阈值 ${input.changeRatioThreshold}%`,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export async function detectVisualChange(input: VisualChangeCheckInput): Promise<VisualChangeCheckResult> {
|
|
134
|
+
const firstScreenshot = readPng(await input.captureScreenshot());
|
|
135
|
+
const region = normalizeRegion(input.region, firstScreenshot.width, firstScreenshot.height);
|
|
136
|
+
const baseline = cropPng(firstScreenshot, region);
|
|
137
|
+
let sampleCount = 0;
|
|
138
|
+
let maxChangeRatio = 0;
|
|
139
|
+
let comparison = baseline;
|
|
140
|
+
let diffImage = new PNG({ width: baseline.width, height: baseline.height });
|
|
141
|
+
const startedAt = Date.now();
|
|
142
|
+
const deadline = startedAt + Math.max(1000, input.durationMs);
|
|
143
|
+
const intervalMs = Math.max(200, input.intervalMs);
|
|
144
|
+
|
|
145
|
+
while (Date.now() < deadline) {
|
|
146
|
+
await input.wait(Math.min(intervalMs, Math.max(0, deadline - Date.now())));
|
|
147
|
+
const currentScreenshot = readPng(await input.captureScreenshot());
|
|
148
|
+
const current = cropPng(currentScreenshot, region);
|
|
149
|
+
const diff = compareCroppedImages(baseline, current, input.pixelmatchThreshold);
|
|
150
|
+
sampleCount += 1;
|
|
151
|
+
comparison = current;
|
|
152
|
+
diffImage = diff.diff;
|
|
153
|
+
maxChangeRatio = Math.max(maxChangeRatio, diff.changedRatio);
|
|
154
|
+
if (diff.changedRatio >= input.changeRatioThreshold) {
|
|
155
|
+
return {
|
|
156
|
+
passed: true,
|
|
157
|
+
region,
|
|
158
|
+
durationMs: input.durationMs,
|
|
159
|
+
intervalMs,
|
|
160
|
+
sampleCount,
|
|
161
|
+
changeRatioThreshold: input.changeRatioThreshold,
|
|
162
|
+
maxChangeRatio,
|
|
163
|
+
baselineBase64: pngToBase64(baseline),
|
|
164
|
+
comparisonBase64: pngToBase64(comparison),
|
|
165
|
+
diffBase64: pngToBase64(diffImage),
|
|
166
|
+
message: `检测到画面变化,最大变化比例 ${maxChangeRatio.toFixed(2)}%,达到阈值 ${input.changeRatioThreshold}%`,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
return {
|
|
172
|
+
passed: false,
|
|
173
|
+
region,
|
|
174
|
+
durationMs: input.durationMs,
|
|
175
|
+
intervalMs,
|
|
176
|
+
sampleCount,
|
|
177
|
+
changeRatioThreshold: input.changeRatioThreshold,
|
|
178
|
+
maxChangeRatio,
|
|
179
|
+
baselineBase64: pngToBase64(baseline),
|
|
180
|
+
comparisonBase64: pngToBase64(comparison),
|
|
181
|
+
diffBase64: pngToBase64(diffImage),
|
|
182
|
+
message: `${input.durationMs}ms 内目标区域未发生明显变化,最大变化比例 ${maxChangeRatio.toFixed(2)}%,低于阈值 ${input.changeRatioThreshold}%`,
|
|
183
|
+
};
|
|
184
|
+
}
|
package/server/config.ts
CHANGED
|
@@ -4,8 +4,10 @@ import path from 'node:path';
|
|
|
4
4
|
import YAML from 'yaml';
|
|
5
5
|
import { appDataPath } from './paths';
|
|
6
6
|
import { loadModelConfigFromDb, saveModelConfigToDb } from './config-store';
|
|
7
|
+
import type { AiRecognitionModel } from '../src/appium-recorder/ai-recognition';
|
|
7
8
|
|
|
8
9
|
export type AppConfig = {
|
|
10
|
+
appium: { model: AiRecognitionModel };
|
|
9
11
|
runtime: {
|
|
10
12
|
androidSdkPath: string;
|
|
11
13
|
reportOutputPath: string;
|
|
@@ -43,6 +45,7 @@ export class ConfigValidationError extends Error {
|
|
|
43
45
|
|
|
44
46
|
function defaultConfig(): AppConfig {
|
|
45
47
|
return {
|
|
48
|
+
appium: { model: { baseUrl: '', apiKey: '', name: '' } },
|
|
46
49
|
runtime: {
|
|
47
50
|
androidSdkPath: '',
|
|
48
51
|
reportOutputPath: '',
|
|
@@ -78,6 +81,13 @@ function normalizeEnv(env: unknown): Record<string, string> {
|
|
|
78
81
|
function normalizeConfig(config: Partial<AppConfig> | null | undefined): AppConfig {
|
|
79
82
|
const fallback = defaultConfig();
|
|
80
83
|
return {
|
|
84
|
+
// 旧配置没有 Appium 模型时保留空值,不借用其他模型或改变已有配置。
|
|
85
|
+
appium: {
|
|
86
|
+
model: Object.fromEntries(['baseUrl', 'apiKey', 'name'].map((key) => {
|
|
87
|
+
const value = config?.appium?.model?.[key as keyof AiRecognitionModel];
|
|
88
|
+
return [key, typeof value === 'string' ? value.trim() : ''];
|
|
89
|
+
})) as AiRecognitionModel,
|
|
90
|
+
},
|
|
81
91
|
runtime: {
|
|
82
92
|
androidSdkPath: config?.runtime?.androidSdkPath?.trim() || fallback.runtime.androidSdkPath,
|
|
83
93
|
reportOutputPath: config?.runtime?.reportOutputPath?.trim() || fallback.runtime.reportOutputPath,
|
package/server/http-api.ts
CHANGED
|
@@ -106,10 +106,9 @@ const START_ANDROID_PLAYGROUND_WITHOUT_BROWSER = [
|
|
|
106
106
|
"const { androidPlaygroundPlatform, ScrcpyServer } = require('@midscene/android-playground');",
|
|
107
107
|
"const { launchPreparedPlaygroundPlatform } = require('@midscene/playground');",
|
|
108
108
|
'const STABLE_PREVIEW_OPTIONS = {',
|
|
109
|
-
' maxFps:
|
|
110
|
-
' maxSize:
|
|
111
|
-
' videoBitRate:
|
|
112
|
-
" videoCodecOptions: 'i-frame-interval=0',",
|
|
109
|
+
' maxFps: 10,',
|
|
110
|
+
' maxSize: 800,',
|
|
111
|
+
' videoBitRate: 1000000,',
|
|
113
112
|
'};',
|
|
114
113
|
'async function main() {',
|
|
115
114
|
' const scrcpyServer = new ScrcpyServer();',
|
|
@@ -1071,7 +1070,7 @@ export function createApiMiddleware() {
|
|
|
1071
1070
|
if (req.url === '/api/test-model' && req.method === 'POST') {
|
|
1072
1071
|
try {
|
|
1073
1072
|
const parsed = await readBody<{
|
|
1074
|
-
modelKey?: 'midscene' | 'scriptOptimizer';
|
|
1073
|
+
modelKey?: 'midscene' | 'scriptOptimizer' | 'appium';
|
|
1075
1074
|
model?: {
|
|
1076
1075
|
provider?: 'custom' | 'codex';
|
|
1077
1076
|
baseUrl?: string;
|
package/src/App.vue
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
<script setup lang="ts">
|
|
2
|
+
import { isAiRecognitionModelConfigured } from './appium-recorder/ai-recognition';
|
|
2
3
|
import { computed, onMounted, onUnmounted, reactive, ref, shallowRef, watch } from 'vue';
|
|
3
4
|
import { ElMessage, ElMessageBox } from 'element-plus';
|
|
4
5
|
import {
|
|
@@ -99,7 +100,9 @@ const appPresets = ref<AppPreset[]>([]);
|
|
|
99
100
|
const modelTestStatus = reactive({
|
|
100
101
|
midscene: '',
|
|
101
102
|
scriptOptimizer: '',
|
|
103
|
+
appium: '',
|
|
102
104
|
});
|
|
105
|
+
const aiRecognitionModelConfigured = shallowRef(false);
|
|
103
106
|
const actionDialog = reactive({
|
|
104
107
|
visible: false,
|
|
105
108
|
title: '',
|
|
@@ -149,6 +152,7 @@ const appPresetForm = reactive({
|
|
|
149
152
|
});
|
|
150
153
|
|
|
151
154
|
const configForm = reactive<ConfigForm>({
|
|
155
|
+
appium: { model: { baseUrl: '', apiKey: '', name: '' } },
|
|
152
156
|
runtime: {
|
|
153
157
|
androidSdkPath: '',
|
|
154
158
|
reportOutputPath: '',
|
|
@@ -656,6 +660,8 @@ const loadConfig = async () => {
|
|
|
656
660
|
});
|
|
657
661
|
Object.assign(configForm.midscene.env, payload.midscene.env || {});
|
|
658
662
|
Object.assign(configForm.scriptOptimizer.model, payload.scriptOptimizer.model);
|
|
663
|
+
Object.assign(configForm.appium.model, payload.appium?.model || { baseUrl: '', apiKey: '', name: '' });
|
|
664
|
+
aiRecognitionModelConfigured.value = isAiRecognitionModelConfigured(configForm.appium.model);
|
|
659
665
|
if (configForm.midscene.model.provider !== 'codex') {
|
|
660
666
|
rememberCustomMidsceneModel();
|
|
661
667
|
}
|
|
@@ -692,7 +698,7 @@ const getMidsceneModelConfigError = () => {
|
|
|
692
698
|
if (model.provider !== 'codex' && !model.apiKey.trim()) missing.push('API Key');
|
|
693
699
|
|
|
694
700
|
if (!missing.length) return '';
|
|
695
|
-
return `Midscene 模型配置不完整:缺少 ${missing.join('、')}。请在“参数配置 > Midscene 模型”中选择“使用 Codex”,或补齐自定义提供方后保存。`;
|
|
701
|
+
return `Midscene 模型配置不完整:缺少 ${missing.join('、')}。请在“参数配置 > Midscene配置 > Midscene 模型”中选择“使用 Codex”,或补齐自定义提供方后保存。`;
|
|
696
702
|
};
|
|
697
703
|
|
|
698
704
|
const saveModelConfig = async () => {
|
|
@@ -700,7 +706,10 @@ const saveModelConfig = async () => {
|
|
|
700
706
|
errorMessage.value = '';
|
|
701
707
|
openActionDialog('保存参数配置');
|
|
702
708
|
try {
|
|
703
|
-
|
|
709
|
+
// 保存的是本次提交快照,不能把请求期间尚未保存的改动标记为已配置。
|
|
710
|
+
const snapshot = JSON.parse(JSON.stringify(configForm)) as ConfigForm;
|
|
711
|
+
await api.saveConfig(snapshot);
|
|
712
|
+
aiRecognitionModelConfigured.value = isAiRecognitionModelConfigured(snapshot.appium.model);
|
|
704
713
|
closeActionDialog();
|
|
705
714
|
ElMessage.success('参数配置已保存');
|
|
706
715
|
} catch (error) {
|
|
@@ -776,14 +785,11 @@ const removeAppPreset = async (id: string) => {
|
|
|
776
785
|
}
|
|
777
786
|
};
|
|
778
787
|
|
|
779
|
-
const testModel = async (key: 'midscene' | 'scriptOptimizer') => {
|
|
788
|
+
const testModel = async (key: 'midscene' | 'scriptOptimizer' | 'appium') => {
|
|
780
789
|
testingModelKey.value = key;
|
|
781
790
|
errorMessage.value = '';
|
|
782
791
|
modelTestStatus[key] = '';
|
|
783
|
-
const model =
|
|
784
|
-
key === 'midscene'
|
|
785
|
-
? configForm.midscene.model
|
|
786
|
-
: configForm.scriptOptimizer.model;
|
|
792
|
+
const model = configForm[key].model;
|
|
787
793
|
|
|
788
794
|
try {
|
|
789
795
|
const payload = await api.testModel({ modelKey: key, model });
|
|
@@ -1557,6 +1563,7 @@ onUnmounted(() => {
|
|
|
1557
1563
|
v-show="activeMenu === 'appium'"
|
|
1558
1564
|
:active="activeMenu === 'appium'"
|
|
1559
1565
|
:app-presets="appPresets"
|
|
1566
|
+
:ai-recognition-model-configured="aiRecognitionModelConfigured"
|
|
1560
1567
|
:device-actions="deviceActions"
|
|
1561
1568
|
:playground-available="playgroundAvailable"
|
|
1562
1569
|
:playground-device-id="playgroundDeviceId"
|
package/src/api.ts
CHANGED
|
@@ -116,7 +116,7 @@ export function saveConfig(input: ConfigForm) {
|
|
|
116
116
|
|
|
117
117
|
// 测试指定模型配置是否可用,通常用于保存前验证 baseUrl/apiKey/model。
|
|
118
118
|
export function testModel(input: {
|
|
119
|
-
modelKey: 'midscene' | 'scriptOptimizer';
|
|
119
|
+
modelKey: 'midscene' | 'scriptOptimizer' | 'appium';
|
|
120
120
|
model: {
|
|
121
121
|
provider?: 'custom' | 'codex';
|
|
122
122
|
baseUrl: string;
|