android-midscene-automation 0.1.41 → 0.1.43

package/server/appium-config.test.ts

@@ -0,0 +1,165 @@
+// Run: npx tsx --test server/appium-config.test.ts
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createServer } from 'node:http';
+import type { AppConfig } from './config';
+
+test('Appium autosave is isolated from manual config, persists and rejects invalid colors', async () => {
+  const root = await mkdtemp(join(tmpdir(), 'appium-config-check-'));
+  const oldRoot = process.env.ANDROID_MIDSCENE_DATA_ROOT;
+  process.env.ANDROID_MIDSCENE_DATA_ROOT = root;
+  const { createApiMiddleware } = await import('./http-api');
+  const handler = createApiMiddleware();
+  const server = createServer((req, res) => handler(req, res, () => { res.statusCode = 404; res.end(); }));
+  let releaseModel: (() => void) | undefined;
+  let modelStarted: (() => void) | undefined;
+  const modelServer = createServer(async (req, res) => {
+    const chunks = [];
+    for await (const chunk of req) chunks.push(chunk);
+    const body = JSON.parse(Buffer.concat(chunks).toString());
+    res.setHeader('Content-Type', 'application/json');
+    if (body.model === 'fail') {
+      res.statusCode = 401;
+      res.end(JSON.stringify({ error: { message: 'invalid test key' } }));
+      return;
+    }
+    if (body.model === 'slow') {
+      await new Promise<void>(resolve => { releaseModel = resolve; modelStarted?.(); });
+    }
+    res.end(JSON.stringify({ choices: [{ finish_reason: 'stop', message: { role: 'assistant', content: 'OK' } }] }));
+  });
+  try {
+    await new Promise<void>(resolve => modelServer.listen(0, '127.0.0.1', resolve));
+    const modelAddress = modelServer.address();
+    assert.ok(modelAddress && typeof modelAddress === 'object');
+    const model = { baseUrl: `http://127.0.0.1:${modelAddress.port}/v1`, apiKey: 'test', name: 'pass', family: 'gpt-5' };
+    await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+    const address = server.address();
+    assert.ok(address && typeof address === 'object');
+    const base = `http://127.0.0.1:${address.port}`;
+    const post = (path: string, body: unknown) => fetch(base + path, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+    const initial = await (await fetch(base + '/api/config')).json() as AppConfig;
+    const appium = { ...initial.appium, flowLineColor: '#AbC', model: { baseUrl: 'https://example.invalid', apiKey: 'test', name: 'new-model' } };
+    assert.equal((await post('/api/config/appium', { appium, runtime: { androidSdkPath: 'invalid-draft' }, midscene: { model: { name: 'unsaved' } } })).status, 200);
+    let stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal(stored.appium.flowLineColor, '#aabbcc');
+    assert.deepEqual(stored.runtime, initial.runtime);
+    assert.deepEqual(stored.midscene, initial.midscene);
+    const { appium: _appium, ...manual } = structuredClone(initial);
+    manual.midscene.model.name = 'saved-midscene';
+    assert.equal((await post('/api/config', manual)).status, 200);
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.deepEqual(stored.appium.model, initial.appium.model, 'automatic saves must not persist untested model drafts');
+    assert.equal(stored.midscene.model.name, initial.midscene.model.name, 'runtime save must not persist an untested model');
+    assert.equal((await post('/api/config/appium', { appium: { ...appium, flowLineColor: 'invalid' } })).status, 400);
+    assert.equal((await post('/api/config/appium', { appium: null })).status, 400);
+    const { loadModelConfigFromDb } = await import('./config-store');
+    assert.equal(loadModelConfigFromDb()?.appium?.flowLineColor, '#aabbcc');
+
+    for (const key of ['midscene', 'scriptOptimizer', 'appium'] as const) {
+      const before = await (await fetch(base + '/api/config')).json() as AppConfig;
+      const failed = await post('/api/test-model', { modelKey: key, model: { ...model, name: 'fail' }, save: true });
+      assert.equal(failed.status, 500);
+      assert.deepEqual(await (await fetch(base + '/api/config')).json(), before, 'failed test must leave all persisted config unchanged');
+      const passed = await post('/api/test-model', { modelKey: key, model, save: true });
+      assert.equal(passed.status, 200);
+      assert.equal((await passed.json() as { saved: boolean }).saved, true);
+      const after = await (await fetch(base + '/api/config')).json() as AppConfig;
+      assert.equal(after[key].model.name, 'pass');
+      for (const other of ['runtime', 'midscene', 'scriptOptimizer', 'appium'] as const) {
+        if (other !== key) assert.deepEqual(after[other], before[other]);
+      }
+    }
+    assert.equal((await post('/api/appium-recorder/prompts/optimize', {
+      kind: 'aiRecognition', prompt: '检查有没有广告', condition: true,
+    })).status, 500, 'prompt optimization requires its own tested model');
+    const unavailableSummaryModel = await post('/api/appium-recorder/report-summary/check', {});
+    assert.equal(unavailableSummaryModel.status, 200);
+    const unavailableSummaryStatus = await unavailableSummaryModel.json() as { available: boolean; message?: string };
+    assert.equal(unavailableSummaryStatus.available, false);
+    assert.equal(unavailableSummaryStatus.message, '模型配置不完整');
+    const promptOptimizerBefore = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal((await post('/api/test-model', { modelKey: 'promptOptimizer', model: { ...model, name: 'fail' }, save: true })).status, 500);
+    assert.deepEqual(await (await fetch(base + '/api/config')).json(), promptOptimizerBefore);
+    assert.equal((await post('/api/test-model', { modelKey: 'promptOptimizer', model, save: true })).status, 200);
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal(stored.appium.promptOptimizer?.model.name, 'pass');
+    const optimizedRecognition = await post('/api/appium-recorder/prompts/optimize', {
+      kind: 'aiRecognition', prompt: '检查有没有广告', condition: true,
+    });
+    assert.equal(optimizedRecognition.status, 200);
+    assert.deepEqual(await optimizedRecognition.json(), { prompt: 'OK' });
+    const optimizedSummary = await post('/api/appium-recorder/prompts/optimize', {
+      kind: 'reportSummary', prompt: '总结出图时间',
+    });
+    assert.equal(optimizedSummary.status, 200);
+    assert.deepEqual(await optimizedSummary.json(), { prompt: 'OK' });
+    // Summary settings auto-save independently and reuse the prompt optimizer model.
+    const summaryBefore = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal(summaryBefore.appium.reportSummary?.enabled, false);
+    const customReportPreset = { name: '支付流程', prompt: '输出支付流程的步骤和结果' };
+    assert.equal((await post('/api/config/appium', { appium: { reportSummary: {
+      enabled: true, prompt: '输出用例表格', customPresets: [customReportPreset], model,
+    } } })).status, 200);
+    let summaryConfig = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal(summaryConfig.appium.reportSummary?.prompt, '输出用例表格');
+    assert.deepEqual(summaryConfig.appium.reportSummary?.customPresets, [customReportPreset]);
+    assert.equal('model' in summaryConfig.appium.reportSummary!, false);
+    assert.equal(summaryConfig.appium.promptOptimizer?.model.name, 'pass');
+    const summaryModelStatus = await post('/api/appium-recorder/report-summary/check', {});
+    assert.equal(summaryModelStatus.status, 200);
+    assert.deepEqual(await summaryModelStatus.json(), { enabled: true, available: true });
+    await post('/api/config/appium', { appium: { reportSummary: { enabled: false, prompt: '恢复测试格式', customPresets: [customReportPreset], model: { name: 'stale' } } } });
+    summaryConfig = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal('model' in summaryConfig.appium.reportSummary!, false);
+    assert.deepEqual(summaryConfig.appium.reportSummary?.customPresets, [customReportPreset]);
+    assert.equal(summaryConfig.appium.promptOptimizer?.model.name, 'pass');
+    assert.deepEqual(summaryConfig.appium.model, summaryBefore.appium.model);
+    assert.equal((await post('/api/config/appium', { appium: { reportSummary: { prompt: '' } } })).status, 400);
+    // A slow test must not revert an appearance edit saved while it was running.
+    const started = new Promise<void>(resolve => { modelStarted = resolve; });
+    const slowTest = post('/api/test-model', { modelKey: 'appium', model: { ...model, name: 'slow' }, save: true });
+    await started;
+    assert.equal((await post('/api/config/appium', { appium: { flowLineColor: '#123456' } })).status, 200);
+    releaseModel!();
+    assert.equal((await slowTest).status, 200);
+    // A stale automatic-save payload must also preserve the newly tested model.
+    await post('/api/config/appium', { appium: { ...appium, flowLineColor: '#123456' } });
+    await post('/api/config', { runtime: initial.runtime });
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.equal(stored.appium.model.name, 'slow');
+    assert.equal(stored.appium.promptOptimizer?.model.name, 'pass');
+    assert.equal(stored.midscene.model.name, 'pass');
+    assert.equal(stored.appium.flowLineColor, '#123456');
+    assert.equal(loadModelConfigFromDb()?.appium?.model?.name, 'slow');
+
+    assert.equal(initial.appium.aiPromptPresets?.length, 4, 'older configurations receive default prompt presets');
+    await post('/api/config/appium', { appium: { aiPromptPresets: ['旧提示词内容'] } });
+    const migrated = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.deepEqual(migrated.appium.aiPromptPresets, [{ name: '自定义场景 1', prompt: '旧提示词内容' }]);
+    const customPresets = [{ name: '广告检测', prompt: '当前画面是否出现广告？' }, { name: '标题提取', prompt: '请读取当前页面标题' }];
+    assert.equal((await post('/api/config/appium', { appium: { aiPromptPresets: customPresets } })).status, 200);
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.deepEqual(stored.appium.aiPromptPresets, customPresets);
+    assert.deepEqual(loadModelConfigFromDb()?.appium?.aiPromptPresets, customPresets);
+    for (const invalid of [null, 'prompt', [''], ['x'.repeat(4001)]]) {
+      assert.equal((await post('/api/config/appium', { appium: { aiPromptPresets: invalid } })).status, 400);
+    }
+    assert.deepEqual(loadModelConfigFromDb()?.appium?.aiPromptPresets, customPresets, 'invalid presets must not overwrite the saved list');
+    await post('/api/config/appium', { appium: { aiPromptPresets: [] } });
+    await post('/api/test-model', { modelKey: 'appium', model, save: true });
+    stored = await (await fetch(base + '/api/config')).json() as AppConfig;
+    assert.deepEqual(stored.appium.aiPromptPresets, [], 'deleting every preset must not restore defaults on subsequent saves');
+    assert.deepEqual(loadModelConfigFromDb()?.appium?.aiPromptPresets, []);
+  } finally {
+    releaseModel?.();
+    await new Promise<void>(resolve => modelServer.close(() => resolve()));
+    await new Promise<void>(resolve => server.close(() => resolve()));
+    if (oldRoot === undefined) delete process.env.ANDROID_MIDSCENE_DATA_ROOT;
+    else process.env.ANDROID_MIDSCENE_DATA_ROOT = oldRoot;
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/server/appium-recorder/ai-deduplication.test.ts

@@ -0,0 +1,80 @@
+// Run: npx tsx --test server/appium-recorder/ai-deduplication.test.ts
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { PNG } from 'pngjs';
+import { resolveAiDeduplication } from '../../src/appium-recorder/ai-deduplication';
+import { createObservationDeduplicator } from './ai-deduplication';
+import { collectAiObservation } from './ai-observation';
+
+function screenshot(color: number[], width = 96, height = 128, patch = false) {
+  const png = new PNG({ width, height });
+  for (let i = 0; i < png.data.length; i += 4) png.data.set([...color, 255], i);
+  if (patch) png.data.set([255, 0, 0, 255], 0);
+  return PNG.sync.write(png).toString('base64');
+}
+
+const home = screenshot([30, 30, 30]), ad = screenshot([240, 240, 240]);
+
+test('defaults and validation cover both active and inactive algorithms', () => {
+  assert.equal(resolveAiDeduplication().method, 'pixelmatch');
+  for (const value of [null, [], { method: 'fake' }, { pixelmatch: { threshold: NaN } },
+    { pixelmatch: { maxChangedRatio: -1 } }, { opencv: { similarityThreshold: 1.1 } },
+    { method: 'none', opencv: { maxDimension: 127 } }, { opencv: { maxDimension: 512.5 } }]) {
+    assert.throws(() => resolveAiDeduplication(value));
+  }
+});
+
+for (const method of ['pixelmatch', 'opencv', 'none'] as const) {
+  test(`${method}: preserve transient ad and transitions, never share state across observations`, async () => {
+    const config = resolveAiDeduplication({ method });
+    const keep = await createObservationDeduplicator(config);
+    const decisions = [];
+    for (const frame of [home, home, ad, ad, home]) decisions.push(await keep(frame));
+    assert.deepEqual(decisions, method === 'none' ? [true, true, true, true, true] : [true, false, true, false, true]);
+    assert.equal(await keep(screenshot([30, 30, 30], 128, 96)), true, 'rotation must be retained');
+    assert.equal(await (await createObservationDeduplicator(config))(home), true);
+    assert.equal(await keep(screenshot([255, 0, 0])), true);
+    assert.equal(await keep(screenshot([0, 130, 0])), true, 'different colors must not collapse into gray');
+  });
+
+  test(`${method}: full observation window survives deduplication down to one frame`, async () => {
+    const samples: number[] = [];
+    const result = await collectAiObservation({ durationMs: 1000, intervalMs: 500 }, async () => home,
+      new AbortController().signal, frame => { samples.push(frame.index); }, resolveAiDeduplication({ method }));
+    assert.ok(result.observation.durationMs >= 1000);
+    assert.ok(result.observation.sampleCount >= 2);
+    assert.equal(samples.length, result.observation.sampleCount);
+    assert.equal(result.observation.frames.length, method === 'none' ? samples.length : 1);
+    assert.equal(result.observation.deduplicationMethod, method);
+    assert.equal(result.imageBase64, home);
+  });
+}
+
+test('pixelmatch parameters take effect and comparison uses the last retained frame', async () => {
+  const tolerant = await createObservationDeduplicator(resolveAiDeduplication());
+  await tolerant(home);
+  assert.equal(await tolerant(screenshot([30, 30, 30], 96, 128, true)), false);
+  const strict = await createObservationDeduplicator(resolveAiDeduplication({ pixelmatch: { threshold: 0, maxChangedRatio: 0 } }));
+  await strict(home);
+  assert.equal(await strict(screenshot([30, 30, 30], 96, 128, true)), true);
+  const gradual = await createObservationDeduplicator(resolveAiDeduplication({ pixelmatch: { threshold: 0.1, maxChangedRatio: 0 } }));
+  await gradual(screenshot([100, 100, 100]));
+  assert.equal(await gradual(screenshot([110, 110, 110])), false);
+  assert.equal(await gradual(screenshot([130, 130, 130])), true);
+});
+
+test('OpenCV SSIM threshold takes effect on non-identical frames', async () => {
+  const base = screenshot([100, 100, 100]), changed = screenshot([120, 120, 120]);
+  for (const [threshold, expected] of [[0.99, true], [0.95, false]] as const) {
+    const keep = await createObservationDeduplicator(resolveAiDeduplication({ method: 'opencv', opencv: { similarityThreshold: threshold, maxDimension: 128 } }));
+    await keep(base);
+    assert.equal(await keep(changed), expected);
+  }
+});
+
+test('capture errors and cancellation stop observation rather than yielding an absence result', async () => {
+  await assert.rejects(collectAiObservation({ durationMs: 1000, intervalMs: 500 }, async () => { throw new Error('capture failed'); }, new AbortController().signal), /capture failed/);
+  const controller = new AbortController();
+  await assert.rejects(collectAiObservation({ durationMs: 1000, intervalMs: 500 }, async () => home,
+    controller.signal, () => { controller.abort(new Error('cancelled')); }), /cancelled/);
+});

package/server/appium-recorder/ai-deduplication.ts

@@ -0,0 +1,74 @@
+import pixelmatch from 'pixelmatch';
+import type { PNG } from 'pngjs';
+import type { AiDeduplicationConfig } from '../../src/appium-recorder/ai-deduplication';
+import { openCv, readImage } from './image-check';
+
+// 彩色 SSIM：各通道均参与比较，避免灰度化丢失广告颜色变化。
+// 使用 OpenCV 官方 SSIM 的 11×11 Gaussian 窗口和 C1/C2：
+// https://docs.opencv.org/4.12.0/d5/dc4/tutorial_video_input_psnr_ssim.html
+async function structuralSimilarity(first: PNG, second: PNG, maxDimension: number) {
+  const cv = await openCv();
+  const mats: InstanceType<typeof cv.Mat>[] = [];
+  const mat = () => { const value = new cv.Mat(); mats.push(value); return value; };
+  const scale = Math.min(1, maxDimension / Math.max(first.width, first.height));
+  const size = new cv.Size(Math.max(1, Math.round(first.width * scale)), Math.max(1, Math.round(first.height * scale)));
+  try {
+    const prepare = (image: PNG) => {
+      const raw = cv.matFromArray(image.height, image.width, cv.CV_8UC4, image.data);
+      mats.push(raw);
+      const rgb = mat(), resized = mat(), result = mat();
+      cv.cvtColor(raw, rgb, cv.COLOR_RGBA2RGB);
+      cv.resize(rgb, resized, size, 0, 0, cv.INTER_AREA);
+      resized.convertTo(result, cv.CV_32F);
+      return result;
+    };
+    const a = prepare(first), b = prepare(second);
+    const blur = (input: InstanceType<typeof cv.Mat>) => {
+      const result = mat();
+      cv.GaussianBlur(input, result, new cv.Size(11, 11), 1.5, 1.5, cv.BORDER_REFLECT);
+      return result;
+    };
+    const product = (x: InstanceType<typeof cv.Mat>, y: InstanceType<typeof cv.Mat>) => {
+      const result = mat(); cv.multiply(x, y, result); return result;
+    };
+    const meanAMat = blur(a), meanBMat = blur(b);
+    const squareAMat = blur(product(a, a)), squareBMat = blur(product(b, b));
+    const crossMat = blur(product(a, b));
+    const meanA = meanAMat.data32F, meanB = meanBMat.data32F;
+    const squareA = squareAMat.data32F, squareB = squareBMat.data32F, cross = crossMat.data32F;
+    let sum = 0;
+    for (let i = 0; i < meanA.length; i++) {
+      const x = meanA[i]!, y = meanB[i]!;
+      const varianceA = Math.max(0, squareA[i]! - x * x), varianceB = Math.max(0, squareB[i]! - y * y);
+      const covariance = cross[i]! - x * y;
+      sum += ((2 * x * y + 6.5025) * (2 * covariance + 58.5225))
+        / ((x * x + y * y + 6.5025) * (varianceA + varianceB + 58.5225));
+    }
+    return Math.max(-1, Math.min(1, sum / meanA.length));
+  } finally { for (const value of mats.reverse()) value.delete(); }
+}
+
+// 每次观察独立维护上一张保留帧；不与上一张被丢弃的帧比较，以免缓慢变化被连续吞掉。
+export async function createObservationDeduplicator(config: AiDeduplicationConfig) {
+  if (config.method === 'opencv') await openCv();
+  let previous: PNG | undefined;
+  return async (imageBase64: string): Promise<boolean> => {
+    if (config.method === 'none') return true;
+    const current = readImage(Buffer.from(imageBase64, 'base64'));
+    let duplicate = false;
+    if (previous && previous.width === current.width && previous.height === current.height) {
+      if (previous.data.equals(current.data)) {
+        duplicate = true;
+      } else if (config.method === 'pixelmatch') {
+        const count = pixelmatch(previous.data, current.data, undefined, current.width, current.height,
+          { threshold: config.pixelmatch.threshold, includeAA: true });
+        duplicate = count / (current.width * current.height) * 100 <= config.pixelmatch.maxChangedRatio;
+      } else {
+        const score = await structuralSimilarity(previous, current, config.opencv.maxDimension);
+        duplicate = score >= config.opencv.similarityThreshold;
+      }
+    }
+    if (!duplicate) previous = current;
+    return !duplicate;
+  };
+}

package/server/appium-recorder/ai-observation.test.ts

@@ -0,0 +1,125 @@
+// Run: npx tsx --test server/appium-recorder/ai-observation.test.ts
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { setTimeout as delay } from 'node:timers/promises';
+import { PNG } from 'pngjs';
+import { observeUntilAiMatch } from './ai-observation';
+import { resolveAiDeduplication } from '../../src/appium-recorder/ai-deduplication';
+import { validateAiObservation } from '../../src/appium-recorder/ai-recognition';
+
+function image(value: number) {
+  const png = new PNG({ width: 8, height: 8 });
+  png.data.fill(value);
+  for (let i = 3; i < png.data.length; i += 4) png.data[i] = 255;
+  return PNG.sync.write(png).toString('base64');
+}
+const home = image(30), ad = image(230);
+const config = { durationMs: 2000, intervalMs: 500, mode: 'untilMatch' as const };
+const signal = () => new AbortController().signal;
+
+test('old observation settings remain batch-compatible; new mode validates', () => {
+  assert.deepEqual(validateAiObservation({ durationMs: 1000, intervalMs: 500 }), { durationMs: 1000, intervalMs: 500 });
+  assert.deepEqual(validateAiObservation(config), config);
+  assert.throws(() => validateAiObservation({ ...config, mode: 'invalid' }));
+});
+
+test('first-frame match stops before the observation window and leaves no sampler running', async () => {
+  let captures = 0;
+  const result = await observeUntilAiMatch({ config, signal: signal(), capture: async () => { captures++; return ad; },
+    analyze: async () => ({ result: true, reason: '广告出现' }) });
+  assert.equal(result.result, true);
+  assert.equal(result.observation.completion, 'matched');
+  assert.equal(captures, 1);
+  assert.ok(result.observation.durationMs < config.durationMs);
+  await delay(550);
+  assert.equal(captures, 1);
+});
+
+test('slow inference does not lose a transient frame, batches pending frames and never overlaps model requests', async () => {
+  let captures = 0, active = 0, maxActive = 0, calls = 0;
+  const result = await observeUntilAiMatch({ config, signal: signal(),
+    capture: async () => ++captures === 2 ? ad : home,
+    analyze: async frames => {
+      active++; maxActive = Math.max(maxActive, active); calls++;
+      try {
+        if (calls === 1) { await delay(1150); assert.ok(captures >= 3); return { result: false, reason: '首页' }; }
+        assert.equal(frames.length, 2, 'ad and return-home frames must both survive');
+        assert.notEqual(frames[0].imageDataUrl, frames[1].imageDataUrl);
+        return { result: true, reason: '第二次采样出现广告' };
+      } finally { active--; }
+    },
+  });
+  assert.equal(result.result, true);
+  assert.equal(maxActive, 1);
+  assert.equal(calls, 2);
+  assert.equal(result.observation.frames.length, 3);
+  assert.ok(result.observation.durationMs < 2000);
+});
+
+test('no match observes the full window and drains pending frames after sampling stops', async () => {
+  let calls = 0;
+  const result = await observeUntilAiMatch({ config: { ...config, durationMs: 1000 }, signal: signal(),
+    deduplication: resolveAiDeduplication({ method: 'none' }), capture: async () => home,
+    analyze: async () => { calls++; if (calls === 1) await delay(1150); return { result: false, reason: '没有广告' }; },
+  });
+  assert.equal(result.result, false);
+  assert.equal(result.observation.completion, 'elapsed');
+  assert.equal(result.observation.frames.length, result.observation.sampleCount);
+  assert.equal(calls, 2);
+  assert.ok(result.observation.durationMs >= 1000);
+});
+
+test('deduplication avoids repeating model requests for unchanged frames', async () => {
+  const result = await observeUntilAiMatch({ config: { ...config, durationMs: 1000 }, signal: signal(), capture: async () => home,
+    analyze: async () => ({ result: false, reason: '未命中' }) });
+  assert.equal(result.observation.modelRequestCount, 1);
+  assert.equal(result.observation.frames.length, 1);
+  assert.ok(result.observation.sampleCount >= 2);
+});
+
+test('cancellation stops pending inference and sampling, errors never become false', async () => {
+  const controller = new AbortController();
+  let active = 0;
+  const run = observeUntilAiMatch({ config, signal: controller.signal, capture: async () => home,
+    analyze: async (_frames, signal) => {
+      active++;
+      try { controller.abort(new Error('user cancelled')); await delay(10000, undefined, { signal }); return { result: false, reason: '' }; }
+      finally { active--; }
+    } });
+  await assert.rejects(run, /user cancelled/);
+  assert.equal(active, 0);
+  await assert.rejects(observeUntilAiMatch({ config, signal: signal(), capture: async () => home,
+    analyze: async () => { throw new Error('model failed'); } }), /model failed/);
+  await assert.rejects(observeUntilAiMatch({ config, signal: signal(), capture: async () => home,
+    analyze: async () => ({ result: null, reason: '无法判断' }) }), /无法确定/);
+});
+
+test('screenshot failure aborts in-flight inference', async () => {
+  let captures = 0, active = 0;
+  await assert.rejects(observeUntilAiMatch({ config, signal: signal(),
+    capture: async () => { if (++captures === 2) throw new Error('capture failed'); return home; },
+    analyze: async (_frames, signal) => {
+      active++;
+      try { await delay(10000, undefined, { signal }); return { result: false, reason: '' }; }
+      finally { active--; }
+    },
+  }), /capture failed/);
+  assert.equal(active, 0);
+});
+
+test('a match cancels an in-flight screenshot without reporting a capture failure', async () => {
+  let captures = 0, captureActive = 0;
+  const result = await observeUntilAiMatch({ config, signal: signal(),
+    capture: async signal => {
+      if (++captures === 1) return ad;
+      captureActive++;
+      try { await delay(10000, undefined, { signal }); return home; }
+      finally { captureActive--; }
+    },
+    analyze: async () => { await delay(650); return { result: true, reason: '命中' }; },
+  });
+  assert.equal(result.result, true);
+  assert.equal(captures, 2);
+  assert.equal(captureActive, 0);
+  assert.equal(result.observation.sampleCount, 1);
+});

package/server/appium-recorder/ai-observation.ts

@@ -0,0 +1,125 @@
+import { setTimeout as delay } from 'node:timers/promises';
+import { constrainBase64ImageToMaxSize, convertBase64ImageToJpeg } from '@midscene/shared/img';
+import { MAX_AI_OBSERVATION_FRAMES, type AiObservationConfig, type AiObservationFrame } from '../../src/appium-recorder/ai-recognition';
+import { resolveAiDeduplication, type AiDeduplicationConfig } from '../../src/appium-recorder/ai-deduplication';
+import { createObservationDeduplicator } from './ai-deduplication';
+
+export type ObservationCapture = { imageBase64: string; elapsedMs: number; index: number };
+
+// 采集不依赖模型响应；按实际采样时刻记录，慢设备串行截图，避免积压 ADB 请求。
+export async function collectAiObservation(
+  config: AiObservationConfig,
+  capture: (signal: AbortSignal) => Promise<string>,
+  signal: AbortSignal,
+  onFrame?: (frame: ObservationCapture) => Promise<void> | void,
+  deduplication: AiDeduplicationConfig = resolveAiDeduplication(),
+  streaming?: { stopSignal: AbortSignal; onRetainedFrame: (frame: AiObservationFrame) => void },
+) {
+  signal.throwIfAborted();
+  const shouldKeep = await createObservationDeduplicator(deduplication);
+  signal.throwIfAborted();
+  const startedAt = performance.now();
+  const frames: AiObservationFrame[] = [];
+  let imageBase64 = '';
+  let bytes = 0;
+  let sampleCount = 0;
+  const activeSignal = streaming ? AbortSignal.any([signal, streaming.stopSignal]) : signal;
+  try {
+    while (true) {
+      activeSignal.throwIfAborted();
+      const captureStarted = performance.now();
+      imageBase64 = await capture(activeSignal);
+      activeSignal.throwIfAborted();
+      const elapsedMs = Math.round(performance.now() - startedAt);
+      sampleCount++;
+      let retainedFrame: AiObservationFrame | undefined;
+      if (await shouldKeep(imageBase64)) {
+        const bounded = await constrainBase64ImageToMaxSize(`data:image/png;base64,${imageBase64}`, { maxSize: 1600 });
+        const imageDataUrl = await convertBase64ImageToJpeg(bounded, 85);
+        bytes += imageDataUrl.length;
+        if (bytes > 24 * 1024 * 1024) throw new Error('观察画面超过 24 MB，请缩短观察时长或增大采样间隔');
+        retainedFrame = { elapsedMs, imageDataUrl };
+        frames.push(retainedFrame);
+      }
+      activeSignal.throwIfAborted();
+      await onFrame?.({ imageBase64, elapsedMs, index: sampleCount });
+      activeSignal.throwIfAborted();
+      if (retainedFrame) streaming?.onRetainedFrame(retainedFrame);
+      const now = performance.now();
+      if (now - startedAt >= config.durationMs) break;
+      if (sampleCount >= MAX_AI_OBSERVATION_FRAMES) throw new Error('观察帧数已达上限，尚未覆盖完整观察时长');
+      const nextCapture = Math.min(startedAt + config.durationMs, captureStarted + config.intervalMs);
+      await delay(Math.max(0, Math.ceil(nextCapture - now)), undefined, { signal: activeSignal });
+    }
+  } catch (error) {
+    signal.throwIfAborted();
+    const stopped = streaming?.stopSignal;
+    if (!stopped?.aborted || (error !== stopped.reason && !(error instanceof Error && error.name === 'AbortError'))) throw error;
+  }
+  if (sampleCount < 2 && !streaming?.stopSignal.aborted) throw new Error('设备截图过慢，观察期间不足 2 帧，请增加观察时长');
+  return { imageBase64, observation: { durationMs: Math.round(performance.now() - startedAt), frames, sampleCount, deduplicationMethod: deduplication.method } };
+}
+
+// 采样和识别并行；一次只发一个请求，期间积累的关键帧在下一次一起判断，避免丢掉短暂画面。
+export async function observeUntilAiMatch(input: {
+  config: AiObservationConfig;
+  capture: (signal: AbortSignal) => Promise<string>;
+  signal: AbortSignal;
+  deduplication?: AiDeduplicationConfig;
+  onFrame?: (frame: ObservationCapture) => Promise<void> | void;
+  onProgress?: (message: string) => void;
+  analyze: (frames: AiObservationFrame[], signal: AbortSignal) => Promise<{ result: boolean | null; reason: string }>;
+}) {
+  const stopCapture = new AbortController(), work = new AbortController();
+  const signal = AbortSignal.any([input.signal, work.signal]);
+  const pending: AiObservationFrame[] = [], submitted: AiObservationFrame[] = [];
+  let wake: (() => void) | undefined;
+  let captureDone = false, failure: unknown, failed = false, modelRequestCount = 0;
+  let match: { result: boolean; reason: string } | undefined;
+  const consumer = (async () => {
+    while (true) {
+      signal.throwIfAborted();
+      if (!pending.length) {
+        if (captureDone) return;
+        await new Promise<void>(resolve => { wake = resolve; });
+        continue;
+      }
+      const frames = pending.splice(0);
+      submitted.push(...frames);
+      modelRequestCount++;
+      input.onProgress?.(`第 ${modelRequestCount} 次检测：提交 ${frames.length} 张新关键帧`);
+      const result = await input.analyze(frames, signal);
+      signal.throwIfAborted();
+      if (typeof result.result !== 'boolean') throw new Error('AI 无法确定是否命中条件，已停止观察');
+      if (result.result) {
+        match = { result: true, reason: result.reason };
+        stopCapture.abort();
+        input.onProgress?.('检测命中，停止观察并结束当前节点');
+        return;
+      }
+      input.onProgress?.(`第 ${modelRequestCount} 次检测未命中，继续观察或检查剩余关键帧`);
+    }
+  })().catch(error => { failure = error; failed = true; work.abort(error); });
+  try {
+    const collected = await collectAiObservation(input.config, input.capture, signal, input.onFrame, input.deduplication, {
+      stopSignal: stopCapture.signal,
+      onRetainedFrame: frame => { pending.push(frame); wake?.(); },
+    });
+    captureDone = true;
+    wake?.();
+    if (!match) input.onProgress?.('已到最长观察时间，等待剩余关键帧识别完成');
+    await consumer;
+    if (failed) throw failure;
+    input.signal.throwIfAborted();
+    return {
+      ...collected,
+      ...(match ?? { result: false, reason: '观察时段内未命中条件，所有保留的关键帧均已判断。' }),
+      observation: { ...collected.observation, frames: submitted, modelRequestCount, completion: match ? 'matched' as const : 'elapsed' as const },
+    };
+  } finally {
+    captureDone = true;
+    work.abort();
+    wake?.();
+    await consumer;
+  }
+}