android-midscene-automation 0.1.38 → 0.1.40

package/src/appium-recorder/api.ts

@@ -1,6 +1,5 @@
 import { APP_BASE } from '../api';
 import type { AppiumRecordedScript, AppiumRecordedStep } from './types';
-import type { AiRecognitionResult } from './ai-recognition';
 import type { TestVariable } from './variables';
 import type { AppiumVisualChangeRegion } from './types';
 
@@ -73,13 +72,6 @@ export async function getAppiumScripts() {
   return readJson<{ scripts: AppiumRecordedScript[] }>(response);
 }
 
-export async function testAiRecognition(input: { deviceId: string; prompt: string; timeoutMs?: number }, signal?: AbortSignal) {
-  const response = await fetch(`${APP_BASE}/api/appium-recorder/ai-recognition/test`, {
-    method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input), signal,
-  });
-  return readJson<AiRecognitionResult & { imageBase64: string }>(response);
-}
-
 export function saveAppiumScript(input: {
   variables?: import('./variables').TestVariable[];
   id?: string;
@@ -115,6 +107,7 @@ export async function downloadAppiumScript(id: string) {
 }
 
 type ReplayResult = {
+  videoPath?: string;
   success: boolean;
   stopped?: boolean;
   output: string;
@@ -131,7 +124,7 @@ type ReplayStreamEvent =
   | { type: 'error'; message: string };
 
 export async function replayAppiumScript(
-  input: { id: string; deviceId?: string; parameters?: import('./variables').TestVariable[] },
+  input: { id: string; deviceId?: string; recordVideo?: boolean; parameters?: import('./variables').TestVariable[] },
   onOutput?: (line: string) => void,
 ) {
   const response = await fetch(`${APP_BASE}/api/appium-recorder/scripts/${encodeURIComponent(input.id)}/replay`, {
@@ -140,7 +133,7 @@ export async function replayAppiumScript(
       'Content-Type': 'application/json',
       Accept: 'application/x-ndjson',
     },
-    body: JSON.stringify({ deviceId: input.deviceId, parameters: input.parameters }),
+    body: JSON.stringify({ deviceId: input.deviceId, parameters: input.parameters, recordVideo: input.recordVideo }),
   });
   if (!response.ok || !response.body) return readJson<ReplayResult>(response);
 

package/src/appium-recorder/components/FlowNodeCard.vue

@@ -2,7 +2,6 @@
 import { computed, nextTick, onBeforeUnmount, onMounted, shallowRef, watch } from 'vue';
 import { Handle, Position } from '@vue-flow/core';
 import { CopyDocument, Delete, Edit, Plus, VideoPlay, View, WarningFilled, Connection, RefreshLeft } from '@element-plus/icons-vue';
-import { AI_MODEL_CONFIG_HINT } from '../ai-recognition';
 import type { AppiumRecordedStep } from '../types';
 import {
   PASTE_COMMAND,
@@ -176,8 +175,8 @@ watch(() => props.data, () => {
         @click="handleStepClick"
       >
         <span class="appium-flow-step-card__content">
-          <el-tooltip v-if="data.missingAiModel" :content="AI_MODEL_CONFIG_HINT" placement="top">
-            <el-icon class="appium-ai-model-warning" color="var(--el-color-danger)" role="img" aria-label="AI 识别模型未配置" tabindex="0">
+          <el-tooltip v-if="data.missingAiModel" content="AI 识别操作已移除，请删除此节点并改用图像判断或原生组件判断" placement="top">
+            <el-icon class="appium-ai-model-warning" color="var(--el-color-danger)" role="img" aria-label="AI 识别操作已移除" tabindex="0">
               <WarningFilled />
             </el-icon>
           </el-tooltip>

package/src/appium-recorder/components/FlowStepEditor.vue

@@ -197,12 +197,6 @@ function patchSelector(patch: Partial<AppiumSelector>) {
       <LoopSettings v-if="step.type === 'loop'" :step="step" :disabled="disabled" @update="patchStep" />
       <BreakLoopSettings v-if="step.type === 'breakLoop'" :step="step" :steps="steps" :disabled="disabled" @update="patchStep" />
       <TextClickSettings v-if="step.type === 'textClick'" :step="step" :disabled="disabled" @update="patchStep" />
-      <el-form-item v-if="step.type === 'aiRecognition'" label="识别内容">
-        <el-input
-          :model-value="step.value || ''" type="textarea" :rows="3" maxlength="4000"
-          :disabled="disabled" @update:model-value="patchStep({ value: String($event) })"
-        />
-      </el-form-item>
       <el-form-item
         v-if="step.type === 'input' || step.type === 'inputIfExists' || step.type === 'assertText'"
         label="文本内容"

package/src/appium-recorder/components/ImageCheckDialog.vue

@@ -18,7 +18,7 @@ const slots = computed(() => props.config.mode === 'state'
   ? [{ key: 'template' as const, label: '选中模板' }, { key: 'negativeTemplate' as const, label: '未选中模板' }]
   : props.config.mode === 'template' ? [{ key: 'template' as const, label: '模板' }] : []);
 const numericFields = computed(() => [
-  ...(['template', 'state'].includes(props.config.mode) ? [{ key: 'threshold' as const, label: '匹配得分阈值', min: 0.01, max: 1, step: 0.01 }] : []),
+  ...(['template', 'state'].includes(props.config.mode) ? [{ key: 'threshold' as const, label: '匹配严格度 (%)', min: 1, max: 100, step: 1 }] : []),
   ...(props.config.mode === 'state' ? [{ key: 'minScoreGap' as const, label: '两种状态最小得分差', min: 0.001, max: 1, step: 0.01 }] : []),
   ...(['black', 'color', 'change'].includes(props.config.mode) ? [
     { key: 'tolerance' as const, label: props.config.mode === 'black' ? '暗色亮度阈值 (0–255)' : 'RGB 通道容差 (0–255)', min: 0, max: 255, step: 1 },
@@ -110,10 +110,10 @@ async function upload(event: Event) {
       <el-alert v-if="regionIssue" :title="regionIssue" type="warning" show-icon :closable="false" class="image-check-region-warning">
         <el-button v-if="expandedRegion" size="small" @click="patch({ region: expandedRegion })">扩大检测区域</el-button>
       </el-alert>
-      <el-form-item v-if="config.mode === 'template' || config.mode === 'change'" label="判断条件">
+      <el-form-item v-if="config.mode === 'template' || config.mode === 'change'" :label="config.mode === 'template' ? '预期匹配结果' : '判断条件'">
         <el-radio-group :model-value="config.expectation" @update:model-value="patch({ expectation: $event as 'present' | 'absent' })">
-          <el-radio-button value="present">{{ config.mode === 'template' ? '图片存在' : '画面有变化' }}</el-radio-button>
-          <el-radio-button value="absent">{{ config.mode === 'template' ? '图片不存在' : '持续无明显变化' }}</el-radio-button>
+          <el-radio-button value="present">{{ config.mode === 'template' ? '匹配到模板' : '画面有变化' }}</el-radio-button>
+          <el-radio-button value="absent">{{ config.mode === 'template' ? '未匹配到模板' : '持续无明显变化' }}</el-radio-button>
         </el-radio-group>
       </el-form-item>
       <el-form-item v-if="config.mode === 'color'" label="目标颜色 #RRGGBB">
@@ -123,7 +123,22 @@ async function upload(event: Event) {
       <div class="image-check-grid">
         <template v-for="field in numericFields" :key="field.key">
           <el-form-item :label="field.label">
-            <el-input-number :model-value="config[field.key]" :min="field.min" :max="field.max" :step="field.step" controls-position="right" @update:model-value="$event !== undefined && patch({ [field.key]: $event })" />
+            <template v-if="field.key === 'threshold'" #label>
+              <span class="image-check-threshold-label">{{ field.label }}
+                <el-tooltip effect="dark" placement="top" :show-after="200">
+                  <template #content>
+                    <div class="image-check-threshold-help">
+                      当前画面与参考模板的匹配得分达到此值，才算匹配成功，得分不是正确概率。
+                      数值越高越严格，可能漏掉有细微变化的目标；越低越宽松，可能误认相似图案。
+                      例如设为 98%，需要匹配得分达到 98%。设为 100% 时，极细微的像素差异也会判为不匹配。图片状态判断还会结合两种模板的最小得分差，得分太接近时无法判定。
+                    </div>
+                  </template>
+                  <el-button class="image-check-threshold-button" :icon="QuestionFilled" text aria-label="匹配严格度说明" @click.prevent />
+                </el-tooltip>
+              </span>
+            </template>
+            <el-input-number :model-value="field.key === 'threshold' ? Number((config.threshold * 100).toFixed(8)) : config[field.key]" :min="field.min" :max="field.max" :step="field.step" controls-position="right" @update:model-value="$event !== undefined && patch({ [field.key]: field.key === 'threshold' ? $event / 100 : $event })" />
+            <el-alert v-if="field.key === 'threshold' && config.threshold === 1" title="100% 要求完全匹配，极细微的像素差异也会判为不匹配。" type="warning" :closable="false" show-icon class="image-check-strict-warning" />
           </el-form-item>
           <BranchTimeoutSettings v-if="field.key === 'durationMs'" style="grid-column: 1 / -1" :step="{ id: '', label: '', type: 'imageCheck', timeoutBranch }" :disabled="busy" @update="emit('timeoutBranch', $event)" />
         </template>
@@ -141,9 +156,17 @@ async function upload(event: Event) {
 .image-check-dialog .el-input-number { width: 100%; }
 .image-check-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 0 12px; }
 .image-check-help { margin-left: 8px; color: #606266; }
+.image-check-grid .el-form-item__label { display: flex; align-items: center; min-height: 20px; }
+.image-check-threshold-label { display: inline-flex; align-items: center; gap: 4px; }
+.image-check-dialog .el-button.image-check-threshold-button { flex: 0 0 20px; width: 20px; height: 20px; min-height: 20px; padding: 0; font-size: 14px; }
+.image-check-threshold-help { max-width: min(300px, calc(100vw - 40px)); font-size: 12px; line-height: 1.7; overflow-wrap: anywhere; }
+.image-check-strict-warning { margin-top: 6px; }
 .image-check-template { display: grid; gap: 6px; margin-bottom: 16px; min-width: 0; }
 .image-check-size { color: #606266; font-size: 12px; }
 .image-check-region-warning { margin-bottom: 16px; }
 .image-check-template .el-image, .image-check-empty { height: 96px; width: 100%; background: #eff5f3; border: 1px solid #d5dfdb; border-radius: 4px; }
 .image-check-empty { display: grid; place-items: center; color: #606266; }
+@media (max-width: 480px) {
+  .image-check-grid { grid-template-columns: minmax(0, 1fr); }
+}
 </style>

package/src/appium-recorder/components/NestedConditionBranches.vue

@@ -119,7 +119,6 @@ const actionGroups: Array<{ title: string; actions: Array<{ type: InsertAction;
     actions: [
       { type: 'delay', label: '添加延时' },
       { type: 'popupCondition', label: '判断存在' },
-      { type: 'aiRecognition', label: 'AI 识别' },
       { type: 'imageCheck', label: '图像判断' },
       { type: 'tapIfExists', label: '存在则点击' },
       { type: 'inputIfExists', label: '存在则输入' },

package/src/appium-recorder/components/RecordedSteps.vue

@@ -89,7 +89,6 @@ const insertActionGroups: FlowActionGroup[] = [
     actions: [
       { type: 'delay', label: '添加延时' },
       { type: 'popupCondition', label: '判断存在' },
-      { type: 'aiRecognition', label: 'AI 识别' },
       { type: 'imageCheck', label: '图像判断' },
       { type: 'tapIfExists', label: '存在则点击' },
       { type: 'inputIfExists', label: '存在则输入' },

package/src/appium-recorder/flow-graph.ts

@@ -460,8 +460,8 @@ export function buildFlowGraph(
           && item.step.type !== 'clearAppData'
           && item.step.visualChange?.role !== 'end',
         canEditInput: ['input', 'inputIfExists', 'imageCheck', 'runScript'].includes(item.step.type),
-        canExecute: item.step.type === 'launchApp' || item.step.type === 'clearAppData' || item.step.type === 'aiRecognition',
-        missingAiModel: item.step.type === 'aiRecognition' && !options.aiRecognitionModelConfigured,
+        canExecute: item.step.type === 'launchApp' || item.step.type === 'clearAppData',
+        missingAiModel: item.step.type === 'aiRecognition',
       },
     });
   };

package/src/appium-recorder/flow-labels.ts

@@ -47,7 +47,7 @@ export function flowTypeLabel(step: AppiumRecordedStep) {
     checkboxState: 'Checkbox 状态',
     checkedState: '判断勾选',
     radioButtonState: 'RadioButton 状态',
-    aiRecognition: 'AI 识别',
+    aiRecognition: 'AI 识别（已移除）',
     imageCheck: '图像判断',
     textClick: '文字点击',
     assertText: '断言文本',
@@ -81,7 +81,7 @@ export function flowStepMeta(step: AppiumRecordedStep) {
   if (step.type === 'loop') return `最多 ${step.loop?.maxIterations ?? '?'} 次 · ${step.loop?.exitWhen === 'exists' ? '元素出现时退出' : step.loop?.exitWhen === 'notExists' ? '元素消失时退出' : '固定次数'}${step.loop?.exitWhen !== 'never' ? ` ${step.selector?.value || ''}` : ''}`;
   if (step.type === 'breakLoop') return step.breakLoopTargetId ? '退出指定循环，继续循环结束后的流程' : '退出当前循环，继续循环结束后的流程';
   if (step.type === 'log') return `${step.logPrefix ?? DEFAULT_LOG_PREFIX}:${step.value || ''}`;
-  if (step.type === 'aiRecognition') return step.value || '未填写识别内容';
+  if (step.type === 'aiRecognition') return '此操作已移除，请替换为图像判断或原生组件判断';
   if (step.type === 'longPress') {
     const target = longPressMode(step) === 'element'
       ? `元素 ${step.selector?.strategy || ''} ${step.selector?.value || ''}`

package/src/appium-recorder/image-check.ts

@@ -21,6 +21,7 @@ export type ImageCheckConfig = {
   consecutive: number;
 };
 export type ImageCheckResult = {
+  templateMatch?: { matched: boolean; expected: 'present' | 'absent'; score: number; threshold: number };
   timedOut?: boolean;
   result: boolean | null;
   mode: ImageCheckConfig['mode'];
@@ -115,5 +116,6 @@ export function validateImageCheck(config: ImageCheckConfig | undefined) {
 
 export function imageCheckSummary(config?: ImageCheckConfig) {
   if (!config) return '未配置图像判断';
-  return `${IMAGE_CHECK_MODES[config.mode]} · ${config.target === 'element' ? '组件区域' : '框选区域'} · ${config.durationMs}ms`;
+  const expectation = config.mode === 'template' ? ` · ${config.expectation === 'present' ? '匹配到模板' : '未匹配到模板'}` : '';
+  return `${IMAGE_CHECK_MODES[config.mode]}${expectation} · ${config.target === 'element' ? '组件区域' : '框选区域'} · ${config.durationMs}ms`;
 }

package/src/appium-recorder/run-history.ts

@@ -5,7 +5,10 @@ export type HistoryNode = {
   durationMs: number | null;
 };
 
+export type RunVideo = { filePath: string; fileName: string; startedAt: string; warning?: string; scriptId?: string; scriptName?: string; boundaryAt?: string; segments?: RunVideo[] };
+
 export type RunSummary = {
+  video?: RunVideo;
   id: string;
   scriptId: string;
   scriptName: string;

package/src/components/config/AppiumConfigPanel.vue

@@ -1,7 +1,6 @@
 <script setup lang="ts">
 import type { ConfigForm } from '../../types';
 import FlowAppearanceSettings from './FlowAppearanceSettings.vue';
-import { DEFAULT_FLOW_BACKGROUND, isHexColor } from '../../appium-recorder/flow-appearance';
 
 defineProps<{
   configForm: ConfigForm;
@@ -13,39 +12,5 @@ defineEmits<{ testModel: []; saveModelConfig: [] }>();
 </script>
 
 <template>
-  <el-card shadow="never" class="config-module-card config-appium-card">
-    <template #header>
-      <div class="panel-header">
-        <span>节点模型配置</span>
-        <el-button type="primary" :loading="isSavingModelConfig" :disabled="!isHexColor(configForm.appium.flowBackgroundColor ?? DEFAULT_FLOW_BACKGROUND)" @click="$emit('saveModelConfig')">
-          保存模型配置
-        </el-button>
-      </div>
-    </template>
-    <section class="appium-model-form">
-      <div class="panel-header panel-header--sub">
-        <span>AI 识别模型</span>
-        <el-button :loading="testingModelKey === 'appium'" @click="$emit('testModel')">测试模型</el-button>
-      </div>
-      <el-form label-position="top">
-        <el-form-item label="Base URL">
-          <el-input v-model="configForm.appium.model.baseUrl" />
-        </el-form-item>
-        <el-form-item label="API Key">
-          <el-input v-model="configForm.appium.model.apiKey" show-password />
-        </el-form-item>
-        <el-form-item label="Model Name">
-          <el-input v-model="configForm.appium.model.name" placeholder="支持图片输入的模型" />
-        </el-form-item>
-        <el-form-item v-if="testStatus" label="测试结果">
-          <el-input :model-value="testStatus" readonly />
-        </el-form-item>
-      </el-form>
-    </section>
-  </el-card>
   <FlowAppearanceSettings v-model="configForm.appium.flowBackgroundColor" :saving="isSavingModelConfig" @save="$emit('saveModelConfig')" />
 </template>
-
-<style scoped>
-.appium-model-form { max-width: 640px; }
-</style>

package/src/appium-recorder/components/AiRecognitionTestDialog.vue

@@ -1,71 +0,0 @@
-<script setup lang="ts">
-import { onBeforeUnmount, onMounted, shallowRef } from 'vue';
-import { Refresh } from '@element-plus/icons-vue';
-import { testAiRecognition } from '../api';
-import { AI_MODEL_CONFIG_HINT, validateAiRecognitionPrompt } from '../ai-recognition';
-import type { AppiumRecordedStep } from '../types';
-
-const props = defineProps<{ step: AppiumRecordedStep; deviceId: string; modelConfigured: boolean }>();
-defineEmits<{ close: [] }>();
-const loading = shallowRef(false);
-const error = shallowRef('');
-const result = shallowRef<Awaited<ReturnType<typeof testAiRecognition>> | null>(null);
-let controller: AbortController | undefined;
-
-async function runTest() {
-  if (loading.value) return;
-  error.value = '';
-  result.value = null;
-  controller = new AbortController();
-  loading.value = true;
-  try {
-    if (!props.modelConfigured) throw new Error(AI_MODEL_CONFIG_HINT);
-    if (!props.deviceId) throw new Error('请选择设备');
-    const prompt = validateAiRecognitionPrompt(props.step.value);
-    result.value = await testAiRecognition({ deviceId: props.deviceId, prompt, timeoutMs: props.step.timeoutMs }, controller.signal);
-  } catch (cause) {
-    if (!controller.signal.aborted) error.value = cause instanceof Error ? cause.message : 'AI 识别测试失败';
-  } finally {
-    loading.value = false;
-  }
-}
-
-onMounted(runTest);
-onBeforeUnmount(() => controller?.abort());
-</script>
-
-<template>
-  <el-dialog :model-value="true" title="测试 AI 识别" width="min(560px, calc(100vw - 32px))" append-to-body @close="$emit('close')">
-    <div class="ai-recognition-test" aria-live="polite">
-      <p class="ai-recognition-test__prompt">{{ step.value }}</p>
-      <el-alert v-if="error" type="error" :title="error" :closable="false" show-icon />
-      <p v-if="loading">识别中…</p>
-      <template v-if="result">
-        <div class="ai-recognition-test__result">
-          <strong>{{ String(result.result) }}</strong>
-          <span>{{ result.durationMs }}ms</span>
-        </div>
-        <p>{{ result.reason }}</p>
-        <el-image
-          v-if="result.imageBase64"
-          class="ai-recognition-test__image" fit="contain"
-          :src="`data:image/png;base64,${result.imageBase64}`"
-          :preview-src-list="[`data:image/png;base64,${result.imageBase64}`]"
-          preview-teleported alt="AI 识别使用的设备截图"
-        />
-      </template>
-    </div>
-    <template #footer>
-      <el-button @click="$emit('close')">关闭</el-button>
-      <el-button type="primary" :icon="Refresh" :loading="loading" :disabled="!modelConfigured || !deviceId" @click="runTest">重新测试</el-button>
-    </template>
-  </el-dialog>
-</template>
-
-<style scoped>
-.ai-recognition-test { overflow-wrap: anywhere; }
-.ai-recognition-test__prompt { white-space: pre-wrap; margin-top: 0; }
-.ai-recognition-test__result { display: flex; align-items: center; gap: 16px; }
-.ai-recognition-test__result strong { font-size: 20px; }
-.ai-recognition-test__image { display: block; width: 100%; height: 280px; }
-</style>