android-midscene-automation 0.1.41 → 0.1.42

package/CHANGELOG.md

@@ -1,4 +1,15 @@
 # 更新记录
+## v0.1.42
+- 新增 AI 识别详细使用说明，涵盖预设场景、单帧与持续观察、提前结束、分支、去重、超时及开屏广告等配置示例。
+- 预设提示词新增测试场景名称，管理列表同时展示场景和内容；AI 识别按场景名称选择。
+- 修复回放终止或会话创建失败后设备端 UiAutomator2 残留，导致组件树刷新失败的问题。
+- AI 持续观察新增“检测命中后提前结束”模式，边采样去重边识别，命中后停止观察并继续流程或进入“是”分支；到达最长观察时间后完成剩余关键帧判断，未命中走原顺序或“否”分支，支持命中条件校验、取消和逐次模型请求超时。
+- Appium 参数的AI 识别配置，新增持续观察选项，支持 pixelmatch（默认）、OpenCV（SSIM）和不去重选项。
+- AI 识别新增“持续观察”，可设置观察时长与采样间隔，按时间顺序采集多帧后统一判断目标是否出现过或读取内容；观察与模型超时分开计时，支持取消、采样日志与测试画面预览，原单帧识别保持兼容。
+- Midscene 配置新增官方文档中的模型提供方预设。
+- 新增 AI 识别默认顺序执行模式，模型回答写入日志后继续下一节点；勾选“启用分支”及修改分支问题时通过模型验证判断语义，不符合时提示且不保存分支修改。
+- Appium 配置新增默认关闭的“截图与 HTML 报告”开关；开启后才采集节点前后截图并生成 HTML。
+
 ## v0.1.41
 - 修复部分华为设备录屏启动时报编码器错误的问题：使用设备默认编码参数，视频流启动失败时降低分辨率和码率重试，并输出各次尝试的诊断日志。
 

package/README.md

@@ -171,6 +171,8 @@ midscene_run/                  # Midscene 执行报告和运行产物
 
 本文档随 npm 包发布，也可以从 README 的“功能文档”直接打开。常见问题请查看 [常见问题](#常见问题)。
 
+AI 识别的场景选择、预设提示词、持续观察、分支、截图去重及超时配置，请查看 [AI 识别详细使用说明](./AI_RECOGNITION_GUIDE.md)。
+
 ## 使用前准备
 
 ### 1. 连接 Android 设备
@@ -1229,6 +1231,17 @@ Linux: ~/.local/share/android-midscene-automation
 
 # 项目更新记录
 
+## v0.1.42
+- 新增 AI 识别详细使用说明，涵盖预设场景、单帧与持续观察、提前结束、分支、去重、超时及开屏广告等配置示例。
+- 预设提示词新增测试场景名称，管理列表同时展示场景和内容；AI 识别按场景名称选择。
+- 修复回放终止或会话创建失败后设备端 UiAutomator2 残留，导致组件树刷新失败的问题。
+- AI 持续观察新增“检测命中后提前结束”模式，边采样去重边识别，命中后停止观察并继续流程或进入“是”分支；到达最长观察时间后完成剩余关键帧判断，未命中走原顺序或“否”分支，支持命中条件校验、取消和逐次模型请求超时。
+- Appium 参数的AI 识别配置，新增持续观察选项，支持 pixelmatch（默认）、OpenCV（SSIM）和不去重选项。
+- AI 识别新增“持续观察”，可设置观察时长与采样间隔，按时间顺序采集多帧后统一判断目标是否出现过或读取内容；观察与模型超时分开计时，支持取消、采样日志与测试画面预览，原单帧识别保持兼容。
+- Midscene 配置新增官方文档中的模型提供方预设。
+- 新增 AI 识别默认顺序执行模式，模型回答写入日志后继续下一节点；勾选“启用分支”及修改分支问题时通过模型验证判断语义，不符合时提示且不保存分支修改。
+- Appium 配置新增默认关闭的“截图与 HTML 报告”开关；开启后才采集节点前后截图并生成 HTML。
+
 ## v0.1.41
 - 修复部分华为设备录屏启动时报编码器错误的问题：使用设备默认编码参数，视频流启动失败时降低分辨率和码率重试，并输出各次尝试的诊断日志。
 

package/USAGE.md

@@ -4,6 +4,8 @@
 
 本文档随 npm 包发布，也可以从 README 的“功能文档”直接打开。常见问题请查看 [FAQ.md](./FAQ.md)。
 
+AI 识别的场景选择、预设提示词、持续观察、分支、截图去重及超时配置，请查看 [AI 识别详细使用说明](./AI_RECOGNITION_GUIDE.md)。
+
 ## 使用前准备
 
 ### 1. 连接 Android 设备

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "android-midscene-automation",
-  "version": "0.1.41",
+  "version": "0.1.42",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/oooooooko/android-midscene-automation.git"
@@ -23,6 +23,7 @@
     "vite.config.ts",
     "README.md",
     "USAGE.md",
+    "AI_RECOGNITION_GUIDE.md",
     "FAQ.md",
     "CHANGELOG.md"
   ],

package/remote-agent/index.ts

@@ -136,6 +136,7 @@ async function handleCommand(command: RemoteCommand) {
     const script = payload.script as AppiumRecordedScriptRecord | undefined;
     if (!script) throw new Error('远程回放缺少脚本内容');
     return await replayAppiumScript(script, command.deviceId, undefined, undefined, {
+      screenshotReport: payload.screenshotReport === true,
       parameters: payload.parameters as import('../src/appium-recorder/variables').TestVariable[] | undefined,
       globalVariables: payload.globalVariables as import('../src/appium-recorder/variables').TestVariable[] | undefined,
       linkedScripts: payload.linkedScripts as AppiumRecordedScriptRecord[] | undefined,

package/server/appium-auto-save.test.ts

@@ -0,0 +1,60 @@
+// Run: npx tsx --test server/appium-auto-save.test.ts
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { setTimeout as delay } from 'node:timers/promises';
+import { effectScope, reactive } from 'vue';
+import { useAppiumAutoSave } from '../src/config/use-appium-auto-save';
+import type { ConfigForm } from '../src/types';
+
+test('loads quietly, serializes rapid edits, validates colors and surfaces save errors', async () => {
+  const scope = effectScope();
+  const form = reactive({ appium: { model: { baseUrl: '', apiKey: '', name: '' } } } as ConfigForm);
+  const submissions: Omit<ConfigForm['appium'], 'model'>[] = [];
+  const errors: unknown[] = [];
+  let release: (() => void) | undefined;
+  let fail = false;
+  const auto = scope.run(() => useAppiumAutoSave(form, async value => {
+    submissions.push(value);
+    if (fail) throw new Error('offline');
+    if (submissions.length === 1) await new Promise<void>(resolve => { release = resolve; });
+  }, error => errors.push(error)))!;
+  try {
+    form.appium.model.name = 'loaded';
+    auto.initialize();
+    await delay(400);
+    assert.equal(submissions.length, 0, 'loading must never overwrite stored config');
+    form.appium.model.name = 'untested';
+    await delay(400);
+    assert.equal(submissions.length, 0, 'model drafts must never auto-save');
+    form.appium.flowBackgroundColor = '#abc';
+    form.appium.flowBackgroundColor = '#def';
+    await delay(400);
+    assert.equal(submissions.length, 1);
+    assert.equal(submissions[0].flowBackgroundColor, '#def');
+    assert.equal('model' in submissions[0], false);
+    form.appium.flowBackgroundColor = '#123456';
+    await delay(400);
+    assert.equal(submissions.length, 1, 'only one request may be in flight');
+    release!();
+    await delay(20);
+    assert.equal(submissions.length, 2);
+    assert.equal(submissions[1].flowBackgroundColor, '#123456');
+    assert.equal(auto.status.value, '已自动保存');
+    form.appium.flowLineColor = '#';
+    await delay(400);
+    assert.equal(submissions.length, 2, 'partial hex input must not be persisted');
+    form.appium.flowLineColor = '#123';
+    await delay(400);
+    assert.equal(submissions.length, 3);
+    fail = true;
+    form.appium.screenshotReport = true;
+    await delay(400);
+    assert.equal(errors.length, 1);
+    assert.match(auto.status.value, /失败/);
+    fail = false;
+    form.appium.flowBackgroundColor = '#abcdef';
+    await delay(400);
+    assert.equal(auto.status.value, '已自动保存');
+    assert.equal(submissions.at(-1)?.screenshotReport, true);
+  } finally { scope.stop(); }
+});

package/server/appium-config.test.ts

@@ -0,0 +1,118 @@
+// Run: npx tsx --test server/appium-config.test.ts
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createServer } from 'node:http';
+import type { AppConfig } from './config';
+
+test('Appium autosave is isolated from manual config, persists and rejects invalid colors', async () => {
+  const root = await mkdtemp(join(tmpdir(), 'appium-config-check-'));
+  const oldRoot = process.env.ANDROID_MIDSCENE_DATA_ROOT;
+  process.env.ANDROID_MIDSCENE_DATA_ROOT = root;
+  const { createApiMiddleware } = await import('./http-api');
+  const handler = createApiMiddleware();
+  const server = createServer((req, res) => handler(req, res, () => { res.statusCode = 404; res.end(); }));
+  let releaseModel: (() => void) | undefined;
+  let modelStarted: (() => void) | undefined;
+  const modelServer = createServer(async (req, res) => {
+    const chunks = [];
+    for await (const chunk of req) chunks.push(chunk);
+    const body = JSON.parse(Buffer.concat(chunks).toString());
+    res.setHeader('Content-Type', 'application/json');
+    if (body.model === 'fail') {
+      res.statusCode = 401;
+      res.end(JSON.stringify({ error: { message: 'invalid test key' } }));
+      return;
+    }
+    if (body.model === 'slow') {
+      await new Promise<void>(resolve => { releaseModel = resolve; modelStarted?.(); });
+    }
+    res.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: 'OK' } }] }));
+  });
+  try {
+    await new Promise<void>(resolve => modelServer.listen(0, '127.0.0.1', resolve));
+    const modelAddress = modelServer.address();
+    assert.ok(modelAddress && typeof modelAddress === 'object');
+    const model = { baseUrl: `http://127.0.0.1:${modelAddress.port}/v1`, apiKey: 'test', name: 'pass', family: 'gpt-5' };
+    await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+    const address = server.address();
+    assert.ok(address && typeof address === 'object');
+    const base = `http://127.0.0.1:${address.port}`;
+    const post = (path: string, body: unknown) => fetch(base + path, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+    const initial = await (await fetch(base + '/api/config')).json() as AppConfig;
+    const appium = { ...initial.appium, flowLineColor: '#AbC', model: { baseUrl: 'https://example.invalid', apiKey: 'test', name: 'new-model' } };
+    assert.equal((await post('/api/config/appium', { appium, runtime: { androidSdkPath: 'invalid-draft' }, midscene: { model: { name: 'unsaved' } } })).status, 200);
+    let stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal(stored.appium.flowLineColor, '#aabbcc');
+    assert.deepEqual(stored.runtime, initial.runtime);
+    assert.deepEqual(stored.midscene, initial.midscene);
+    const { appium: _appium, ...manual } = structuredClone(initial);
+    manual.midscene.model.name = 'saved-midscene';
+    assert.equal((await post('/api/config', manual)).status, 200);
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.deepEqual(stored.appium.model, initial.appium.model, 'automatic saves must not persist untested model drafts');
+    assert.equal(stored.midscene.model.name, initial.midscene.model.name, 'runtime save must not persist an untested model');
+    assert.equal((await post('/api/config/appium', { appium: { ...appium, flowLineColor: 'invalid' } })).status, 400);
+    assert.equal((await post('/api/config/appium', { appium: null })).status, 400);
+    const { loadModelConfigFromDb } = await import('./config-store');
+    assert.equal(loadModelConfigFromDb()?.appium?.flowLineColor, '#aabbcc');
+
+    for (const key of ['midscene', 'scriptOptimizer', 'appium'] as const) {
+      const before = await (await fetch(base + '/api/config')).json() as AppConfig;
+      const failed = await post('/api/test-model', { modelKey: key, model: { ...model, name: 'fail' }, save: true });
+      assert.equal(failed.status, 500);
+      assert.deepEqual(await (await fetch(base + '/api/config')).json(), before, 'failed test must leave all persisted config unchanged');
+      const passed = await post('/api/test-model', { modelKey: key, model, save: true });
+      assert.equal(passed.status, 200);
+      assert.equal((await passed.json() as { saved: boolean }).saved, true);
+      const after = await (await fetch(base + '/api/config')).json() as AppConfig;
+      assert.equal(after[key].model.name, 'pass');
+      for (const other of ['runtime', 'midscene', 'scriptOptimizer', 'appium'] as const) {
+        if (other !== key) assert.deepEqual(after[other], before[other]);
+      }
+    }
+    // A slow test must not revert an appearance edit saved while it was running.
+    const started = new Promise<void>(resolve => { modelStarted = resolve; });
+    const slowTest = post('/api/test-model', { modelKey: 'appium', model: { ...model, name: 'slow' }, save: true });
+    await started;
+    assert.equal((await post('/api/config/appium', { appium: { flowLineColor: '#123456' } })).status, 200);
+    releaseModel!();
+    assert.equal((await slowTest).status, 200);
+    // A stale automatic-save payload must also preserve the newly tested model.
+    await post('/api/config/appium', { appium: { ...appium, flowLineColor: '#123456' } });
+    await post('/api/config', { runtime: initial.runtime });
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal(stored.appium.model.name, 'slow');
+    assert.equal(stored.midscene.model.name, 'pass');
+    assert.equal(stored.appium.flowLineColor, '#123456');
+    assert.equal(loadModelConfigFromDb()?.appium?.model?.name, 'slow');
+
+    assert.equal(initial.appium.aiPromptPresets?.length, 4, 'older configurations receive default prompt presets');
+    await post('/api/config/appium', { appium: { aiPromptPresets: ['旧提示词内容'] } });
+    const migrated = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.deepEqual(migrated.appium.aiPromptPresets, [{ name: '自定义场景 1', prompt: '旧提示词内容' }]);
+    const customPresets = [{ name: '广告检测', prompt: '当前画面是否出现广告？' }, { name: '标题提取', prompt: '请读取当前页面标题' }];
+    assert.equal((await post('/api/config/appium', { appium: { aiPromptPresets: customPresets } })).status, 200);
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.deepEqual(stored.appium.aiPromptPresets, customPresets);
+    assert.deepEqual(loadModelConfigFromDb()?.appium?.aiPromptPresets, customPresets);
+    for (const invalid of [null, 'prompt', [''], ['x'.repeat(4001)]]) {
+      assert.equal((await post('/api/config/appium', { appium: { aiPromptPresets: invalid } })).status, 400);
+    }
+    assert.deepEqual(loadModelConfigFromDb()?.appium?.aiPromptPresets, customPresets, 'invalid presets must not overwrite the saved list');
+    await post('/api/config/appium', { appium: { aiPromptPresets: [] } });
+    await post('/api/test-model', { modelKey: 'appium', model, save: true });
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.deepEqual(stored.appium.aiPromptPresets, [], 'deleting every preset must not restore defaults on subsequent saves');
+    assert.deepEqual(loadModelConfigFromDb()?.appium?.aiPromptPresets, []);
+  } finally {
+    releaseModel?.();
+    await new Promise<void>(resolve => modelServer.close(() => resolve()));
+    await new Promise<void>(resolve => server.close(() => resolve()));
+    if (oldRoot === undefined) delete process.env.ANDROID_MIDSCENE_DATA_ROOT;
+    else process.env.ANDROID_MIDSCENE_DATA_ROOT = oldRoot;
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/server/appium-recorder/ai-deduplication.test.ts

@@ -0,0 +1,80 @@
+// Run: npx tsx --test server/appium-recorder/ai-deduplication.test.ts
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { PNG } from 'pngjs';
+import { resolveAiDeduplication } from '../../src/appium-recorder/ai-deduplication';
+import { createObservationDeduplicator } from './ai-deduplication';
+import { collectAiObservation } from './ai-observation';
+
+function screenshot(color: number[], width = 96, height = 128, patch = false) {
+  const png = new PNG({ width, height });
+  for (let i = 0; i < png.data.length; i += 4) png.data.set([...color, 255], i);
+  if (patch) png.data.set([255, 0, 0, 255], 0);
+  return PNG.sync.write(png).toString('base64');
+}
+
+const home = screenshot([30, 30, 30]), ad = screenshot([240, 240, 240]);
+
+test('defaults and validation cover both active and inactive algorithms', () => {
+  assert.equal(resolveAiDeduplication().method, 'pixelmatch');
+  for (const value of [null, [], { method: 'fake' }, { pixelmatch: { threshold: NaN } },
+    { pixelmatch: { maxChangedRatio: -1 } }, { opencv: { similarityThreshold: 1.1 } },
+    { method: 'none', opencv: { maxDimension: 127 } }, { opencv: { maxDimension: 512.5 } }]) {
+    assert.throws(() => resolveAiDeduplication(value));
+  }
+});
+
+for (const method of ['pixelmatch', 'opencv', 'none'] as const) {
+  test(`${method}: preserve transient ad and transitions, never share state across observations`, async () => {
+    const config = resolveAiDeduplication({ method });
+    const keep = await createObservationDeduplicator(config);
+    const decisions = [];
+    for (const frame of [home, home, ad, ad, home]) decisions.push(await keep(frame));
+    assert.deepEqual(decisions, method === 'none' ? [true, true, true, true, true] : [true, false, true, false, true]);
+    assert.equal(await keep(screenshot([30, 30, 30], 128, 96)), true, 'rotation must be retained');
+    assert.equal(await (await createObservationDeduplicator(config))(home), true);
+    assert.equal(await keep(screenshot([255, 0, 0])), true);
+    assert.equal(await keep(screenshot([0, 130, 0])), true, 'different colors must not collapse into gray');
+  });
+
+  test(`${method}: full observation window survives deduplication down to one frame`, async () => {
+    const samples: number[] = [];
+    const result = await collectAiObservation({ durationMs: 1000, intervalMs: 500 }, async () => home,
+      new AbortController().signal, frame => { samples.push(frame.index); }, resolveAiDeduplication({ method }));
+    assert.ok(result.observation.durationMs >= 1000);
+    assert.ok(result.observation.sampleCount >= 2);
+    assert.equal(samples.length, result.observation.sampleCount);
+    assert.equal(result.observation.frames.length, method === 'none' ? samples.length : 1);
+    assert.equal(result.observation.deduplicationMethod, method);
+    assert.equal(result.imageBase64, home);
+  });
+}
+
+test('pixelmatch parameters take effect and comparison uses the last retained frame', async () => {
+  const tolerant = await createObservationDeduplicator(resolveAiDeduplication());
+  await tolerant(home);
+  assert.equal(await tolerant(screenshot([30, 30, 30], 96, 128, true)), false);
+  const strict = await createObservationDeduplicator(resolveAiDeduplication({ pixelmatch: { threshold: 0, maxChangedRatio: 0 } }));
+  await strict(home);
+  assert.equal(await strict(screenshot([30, 30, 30], 96, 128, true)), true);
+  const gradual = await createObservationDeduplicator(resolveAiDeduplication({ pixelmatch: { threshold: 0.1, maxChangedRatio: 0 } }));
+  await gradual(screenshot([100, 100, 100]));
+  assert.equal(await gradual(screenshot([110, 110, 110])), false);
+  assert.equal(await gradual(screenshot([130, 130, 130])), true);
+});
+
+test('OpenCV SSIM threshold takes effect on non-identical frames', async () => {
+  const base = screenshot([100, 100, 100]), changed = screenshot([120, 120, 120]);
+  for (const [threshold, expected] of [[0.99, true], [0.95, false]] as const) {
+    const keep = await createObservationDeduplicator(resolveAiDeduplication({ method: 'opencv', opencv: { similarityThreshold: threshold, maxDimension: 128 } }));
+    await keep(base);
+    assert.equal(await keep(changed), expected);
+  }
+});
+
+test('capture errors and cancellation stop observation rather than yielding an absence result', async () => {
+  await assert.rejects(collectAiObservation({ durationMs: 1000, intervalMs: 500 }, async () => { throw new Error('capture failed'); }, new AbortController().signal), /capture failed/);
+  const controller = new AbortController();
+  await assert.rejects(collectAiObservation({ durationMs: 1000, intervalMs: 500 }, async () => home,
+    controller.signal, () => { controller.abort(new Error('cancelled')); }), /cancelled/);
+});

package/server/appium-recorder/ai-deduplication.ts

@@ -0,0 +1,74 @@
+import pixelmatch from 'pixelmatch';
+import type { PNG } from 'pngjs';
+import type { AiDeduplicationConfig } from '../../src/appium-recorder/ai-deduplication';
+import { openCv, readImage } from './image-check';
+
+// 彩色 SSIM：各通道均参与比较，避免灰度化丢失广告颜色变化。
+// 使用 OpenCV 官方 SSIM 的 11×11 Gaussian 窗口和 C1/C2：
+// https://docs.opencv.org/4.12.0/d5/dc4/tutorial_video_input_psnr_ssim.html
+async function structuralSimilarity(first: PNG, second: PNG, maxDimension: number) {
+  const cv = await openCv();
+  const mats: InstanceType<typeof cv.Mat>[] = [];
+  const mat = () => { const value = new cv.Mat(); mats.push(value); return value; };
+  const scale = Math.min(1, maxDimension / Math.max(first.width, first.height));
+  const size = new cv.Size(Math.max(1, Math.round(first.width * scale)), Math.max(1, Math.round(first.height * scale)));
+  try {
+    const prepare = (image: PNG) => {
+      const raw = cv.matFromArray(image.height, image.width, cv.CV_8UC4, image.data);
+      mats.push(raw);
+      const rgb = mat(), resized = mat(), result = mat();
+      cv.cvtColor(raw, rgb, cv.COLOR_RGBA2RGB);
+      cv.resize(rgb, resized, size, 0, 0, cv.INTER_AREA);
+      resized.convertTo(result, cv.CV_32F);
+      return result;
+    };
+    const a = prepare(first), b = prepare(second);
+    const blur = (input: InstanceType<typeof cv.Mat>) => {
+      const result = mat();
+      cv.GaussianBlur(input, result, new cv.Size(11, 11), 1.5, 1.5, cv.BORDER_REFLECT);
+      return result;
+    };
+    const product = (x: InstanceType<typeof cv.Mat>, y: InstanceType<typeof cv.Mat>) => {
+      const result = mat(); cv.multiply(x, y, result); return result;
+    };
+    const meanAMat = blur(a), meanBMat = blur(b);
+    const squareAMat = blur(product(a, a)), squareBMat = blur(product(b, b));
+    const crossMat = blur(product(a, b));
+    const meanA = meanAMat.data32F, meanB = meanBMat.data32F;
+    const squareA = squareAMat.data32F, squareB = squareBMat.data32F, cross = crossMat.data32F;
+    let sum = 0;
+    for (let i = 0; i < meanA.length; i++) {
+      const x = meanA[i]!, y = meanB[i]!;
+      const varianceA = Math.max(0, squareA[i]! - x * x), varianceB = Math.max(0, squareB[i]! - y * y);
+      const covariance = cross[i]! - x * y;
+      sum += ((2 * x * y + 6.5025) * (2 * covariance + 58.5225))
+        / ((x * x + y * y + 6.5025) * (varianceA + varianceB + 58.5225));
+    }
+    return Math.max(-1, Math.min(1, sum / meanA.length));
+  } finally { for (const value of mats.reverse()) value.delete(); }
+}
+
+// 每次观察独立维护上一张保留帧；不与上一张被丢弃的帧比较，以免缓慢变化被连续吞掉。
+export async function createObservationDeduplicator(config: AiDeduplicationConfig) {
+  if (config.method === 'opencv') await openCv();
+  let previous: PNG | undefined;
+  return async (imageBase64: string): Promise<boolean> => {
+    if (config.method === 'none') return true;
+    const current = readImage(Buffer.from(imageBase64, 'base64'));
+    let duplicate = false;
+    if (previous && previous.width === current.width && previous.height === current.height) {
+      if (previous.data.equals(current.data)) {
+        duplicate = true;
+      } else if (config.method === 'pixelmatch') {
+        const count = pixelmatch(previous.data, current.data, undefined, current.width, current.height,
+          { threshold: config.pixelmatch.threshold, includeAA: true });
+        duplicate = count / (current.width * current.height) * 100 <= config.pixelmatch.maxChangedRatio;
+      } else {
+        const score = await structuralSimilarity(previous, current, config.opencv.maxDimension);
+        duplicate = score >= config.opencv.similarityThreshold;
+      }
+    }
+    if (!duplicate) previous = current;
+    return !duplicate;
+  };
+}

package/server/appium-recorder/ai-observation.test.ts

@@ -0,0 +1,125 @@
+// Run: npx tsx --test server/appium-recorder/ai-observation.test.ts
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { setTimeout as delay } from 'node:timers/promises';
+import { PNG } from 'pngjs';
+import { observeUntilAiMatch } from './ai-observation';
+import { resolveAiDeduplication } from '../../src/appium-recorder/ai-deduplication';
+import { validateAiObservation } from '../../src/appium-recorder/ai-recognition';
+
+function image(value: number) {
+  const png = new PNG({ width: 8, height: 8 });
+  png.data.fill(value);
+  for (let i = 3; i < png.data.length; i += 4) png.data[i] = 255;
+  return PNG.sync.write(png).toString('base64');
+}
+const home = image(30), ad = image(230);
+const config = { durationMs: 2000, intervalMs: 500, mode: 'untilMatch' as const };
+const signal = () => new AbortController().signal;
+
+test('old observation settings remain batch-compatible; new mode validates', () => {
+  assert.deepEqual(validateAiObservation({ durationMs: 1000, intervalMs: 500 }), { durationMs: 1000, intervalMs: 500 });
+  assert.deepEqual(validateAiObservation(config), config);
+  assert.throws(() => validateAiObservation({ ...config, mode: 'invalid' }));
+});
+
+test('first-frame match stops before the observation window and leaves no sampler running', async () => {
+  let captures = 0;
+  const result = await observeUntilAiMatch({ config, signal: signal(), capture: async () => { captures++; return ad; },
+    analyze: async () => ({ result: true, reason: '广告出现' }) });
+  assert.equal(result.result, true);
+  assert.equal(result.observation.completion, 'matched');
+  assert.equal(captures, 1);
+  assert.ok(result.observation.durationMs < config.durationMs);
+  await delay(550);
+  assert.equal(captures, 1);
+});
+
+test('slow inference does not lose a transient frame, batches pending frames and never overlaps model requests', async () => {
+  let captures = 0, active = 0, maxActive = 0, calls = 0;
+  const result = await observeUntilAiMatch({ config, signal: signal(),
+    capture: async () => ++captures === 2 ? ad : home,
+    analyze: async frames => {
+      active++; maxActive = Math.max(maxActive, active); calls++;
+      try {
+        if (calls === 1) { await delay(1150); assert.ok(captures >= 3); return { result: false, reason: '首页' }; }
+        assert.equal(frames.length, 2, 'ad and return-home frames must both survive');
+        assert.notEqual(frames[0].imageDataUrl, frames[1].imageDataUrl);
+        return { result: true, reason: '第二次采样出现广告' };
+      } finally { active--; }
+    },
+  });
+  assert.equal(result.result, true);
+  assert.equal(maxActive, 1);
+  assert.equal(calls, 2);
+  assert.equal(result.observation.frames.length, 3);
+  assert.ok(result.observation.durationMs < 2000);
+});
+
+test('no match observes the full window and drains pending frames after sampling stops', async () => {
+  let calls = 0;
+  const result = await observeUntilAiMatch({ config: { ...config, durationMs: 1000 }, signal: signal(),
+    deduplication: resolveAiDeduplication({ method: 'none' }), capture: async () => home,
+    analyze: async () => { calls++; if (calls === 1) await delay(1150); return { result: false, reason: '没有广告' }; },
+  });
+  assert.equal(result.result, false);
+  assert.equal(result.observation.completion, 'elapsed');
+  assert.equal(result.observation.frames.length, result.observation.sampleCount);
+  assert.equal(calls, 2);
+  assert.ok(result.observation.durationMs >= 1000);
+});
+
+test('deduplication avoids repeating model requests for unchanged frames', async () => {
+  const result = await observeUntilAiMatch({ config: { ...config, durationMs: 1000 }, signal: signal(), capture: async () => home,
+    analyze: async () => ({ result: false, reason: '未命中' }) });
+  assert.equal(result.observation.modelRequestCount, 1);
+  assert.equal(result.observation.frames.length, 1);
+  assert.ok(result.observation.sampleCount >= 2);
+});
+
+test('cancellation stops pending inference and sampling, errors never become false', async () => {
+  const controller = new AbortController();
+  let active = 0;
+  const run = observeUntilAiMatch({ config, signal: controller.signal, capture: async () => home,
+    analyze: async (_frames, signal) => {
+      active++;
+      try { controller.abort(new Error('user cancelled')); await delay(10000, undefined, { signal }); return { result: false, reason: '' }; }
+      finally { active--; }
+    } });
+  await assert.rejects(run, /user cancelled/);
+  assert.equal(active, 0);
+  await assert.rejects(observeUntilAiMatch({ config, signal: signal(), capture: async () => home,
+    analyze: async () => { throw new Error('model failed'); } }), /model failed/);
+  await assert.rejects(observeUntilAiMatch({ config, signal: signal(), capture: async () => home,
+    analyze: async () => ({ result: null, reason: '无法判断' }) }), /无法确定/);
+});
+
+test('screenshot failure aborts in-flight inference', async () => {
+  let captures = 0, active = 0;
+  await assert.rejects(observeUntilAiMatch({ config, signal: signal(),
+    capture: async () => { if (++captures === 2) throw new Error('capture failed'); return home; },
+    analyze: async (_frames, signal) => {
+      active++;
+      try { await delay(10000, undefined, { signal }); return { result: false, reason: '' }; }
+      finally { active--; }
+    },
+  }), /capture failed/);
+  assert.equal(active, 0);
+});
+
+test('a match cancels an in-flight screenshot without reporting a capture failure', async () => {
+  let captures = 0, captureActive = 0;
+  const result = await observeUntilAiMatch({ config, signal: signal(),
+    capture: async signal => {
+      if (++captures === 1) return ad;
+      captureActive++;
+      try { await delay(10000, undefined, { signal }); return home; }
+      finally { captureActive--; }
+    },
+    analyze: async () => { await delay(650); return { result: true, reason: '命中' }; },
+  });
+  assert.equal(result.result, true);
+  assert.equal(captures, 2);
+  assert.equal(captureActive, 0);
+  assert.equal(result.observation.sampleCount, 1);
+});