android-midscene-automation 0.1.26 → 0.1.28

package/src/appium-recorder/ai-recognition.ts

@@ -0,0 +1,16 @@
+export type AiRecognitionModel = { baseUrl: string; apiKey: string; name: string };
+export type AiRecognitionResult = { result: boolean; reason: string; durationMs: number };
+
+export const AI_MODEL_CONFIG_HINT = '请在参数配置 > Appium配置中填写并保存 AI 识别模型的 Base URL、API Key 和 Model Name';
+
+export function isAiRecognitionModelConfigured(model?: Partial<AiRecognitionModel> | null) {
+  return Boolean(model && [model.baseUrl, model.apiKey, model.name].every(
+    (value) => typeof value === 'string' && value.trim(),
+  ));
+}
+
+export function validateAiRecognitionPrompt(value: unknown): string {
+  if (typeof value !== 'string' || !value.trim()) throw new Error('请输入 AI 识别内容');
+  if (value.trim().length > 4000) throw new Error('AI 识别内容不能超过 4000 字');
+  return value.trim();
+}

package/src/appium-recorder/api.ts

@@ -1,5 +1,6 @@
 import { APP_BASE } from '../api';
 import type { AppiumRecordedScript, AppiumRecordedStep } from './types';
+import type { AiRecognitionResult } from './ai-recognition';
 
 async function readJson<T>(response: Response) {
   const payload = (await response.json().catch(() => ({}))) as T & {
@@ -22,7 +23,7 @@ function postJson<T>(url: string, body?: unknown) {
 }
 
 export async function getAppiumTree(deviceId: string) {
-  const response = await fetch(`${APP_BASE}/api/appium-recorder/tree?deviceId=${encodeURIComponent(deviceId)}`);
+  const response = await fetch(`${APP_BASE}/api/appium-recorder/tree?deviceId=${encodeURIComponent(deviceId)}`, { cache: 'no-store' });
   return readJson<{ deviceId: string; xml: string; activity: string; dumpedAt: string }>(response);
 }
 
@@ -47,6 +48,13 @@ export async function getAppiumScripts() {
   return readJson<{ scripts: AppiumRecordedScript[] }>(response);
 }
 
+export async function testAiRecognition(input: { deviceId: string; prompt: string; timeoutMs?: number }, signal?: AbortSignal) {
+  const response = await fetch(`${APP_BASE}/api/appium-recorder/ai-recognition/test`, {
+    method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input), signal,
+  });
+  return readJson<AiRecognitionResult & { imageBase64: string }>(response);
+}
+
 export function saveAppiumScript(input: {
   id?: string;
   name: string;
@@ -68,8 +76,16 @@ export function importAppiumScript(input: unknown) {
   return postJson<{ script: AppiumRecordedScript }>(`${APP_BASE}/api/appium-recorder/scripts/import`, input);
 }
 
-export function appiumScriptDownloadUrl(id: string) {
-  return `${APP_BASE}/api/appium-recorder/scripts/${encodeURIComponent(id)}/export`;
+export async function downloadAppiumScript(id: string) {
+  const response = await fetch(`${APP_BASE}/api/appium-recorder/scripts/${encodeURIComponent(id)}/export`);
+  if (!response.ok) await readJson(response);
+  const disposition = response.headers.get('Content-Disposition') || '';
+  const encodedName = disposition.match(/filename\*=UTF-8''([^;]+)/i)?.[1];
+  const extension = response.headers.get('Content-Type')?.includes('application/zip') ? 'zip' : 'json';
+  return {
+    fileName: encodedName ? decodeURIComponent(encodedName) : `appium-script.${extension}`,
+    blob: await response.blob(),
+  };
 }
 
 type ReplayResult = {
@@ -80,6 +96,7 @@ type ReplayResult = {
   reportId?: string;
   logPath?: string;
   htmlReportPath?: string;
+  softFailureCount?: number;
 };
 
 type ReplayStreamEvent =

package/src/appium-recorder/components/AiRecognitionTestDialog.vue

@@ -0,0 +1,71 @@
+<script setup lang="ts">
+import { onBeforeUnmount, onMounted, shallowRef } from 'vue';
+import { Refresh } from '@element-plus/icons-vue';
+import { testAiRecognition } from '../api';
+import { AI_MODEL_CONFIG_HINT, validateAiRecognitionPrompt } from '../ai-recognition';
+import type { AppiumRecordedStep } from '../types';
+
+const props = defineProps<{ step: AppiumRecordedStep; deviceId: string; modelConfigured: boolean }>();
+defineEmits<{ close: [] }>();
+const loading = shallowRef(false);
+const error = shallowRef('');
+const result = shallowRef<Awaited<ReturnType<typeof testAiRecognition>> | null>(null);
+let controller: AbortController | undefined;
+
+async function runTest() {
+  if (loading.value) return;
+  error.value = '';
+  result.value = null;
+  controller = new AbortController();
+  loading.value = true;
+  try {
+    if (!props.modelConfigured) throw new Error(AI_MODEL_CONFIG_HINT);
+    if (!props.deviceId) throw new Error('请选择设备');
+    const prompt = validateAiRecognitionPrompt(props.step.value);
+    result.value = await testAiRecognition({ deviceId: props.deviceId, prompt, timeoutMs: props.step.timeoutMs }, controller.signal);
+  } catch (cause) {
+    if (!controller.signal.aborted) error.value = cause instanceof Error ? cause.message : 'AI 识别测试失败';
+  } finally {
+    loading.value = false;
+  }
+}
+
+onMounted(runTest);
+onBeforeUnmount(() => controller?.abort());
+</script>
+
+<template>
+  <el-dialog :model-value="true" title="测试 AI 识别" width="min(560px, calc(100vw - 32px))" append-to-body @close="$emit('close')">
+    <div class="ai-recognition-test" aria-live="polite">
+      <p class="ai-recognition-test__prompt">{{ step.value }}</p>
+      <el-alert v-if="error" type="error" :title="error" :closable="false" show-icon />
+      <p v-if="loading">识别中…</p>
+      <template v-if="result">
+        <div class="ai-recognition-test__result">
+          <strong>{{ String(result.result) }}</strong>
+          <span>{{ result.durationMs }}ms</span>
+        </div>
+        <p>{{ result.reason }}</p>
+        <el-image
+          v-if="result.imageBase64"
+          class="ai-recognition-test__image" fit="contain"
+          :src="`data:image/png;base64,${result.imageBase64}`"
+          :preview-src-list="[`data:image/png;base64,${result.imageBase64}`]"
+          preview-teleported alt="AI 识别使用的设备截图"
+        />
+      </template>
+    </div>
+    <template #footer>
+      <el-button @click="$emit('close')">关闭</el-button>
+      <el-button type="primary" :icon="Refresh" :loading="loading" :disabled="!modelConfigured || !deviceId" @click="runTest">重新测试</el-button>
+    </template>
+  </el-dialog>
+</template>
+
+<style scoped>
+.ai-recognition-test { overflow-wrap: anywhere; }
+.ai-recognition-test__prompt { white-space: pre-wrap; margin-top: 0; }
+.ai-recognition-test__result { display: flex; align-items: center; gap: 16px; }
+.ai-recognition-test__result strong { font-size: 20px; }
+.ai-recognition-test__image { display: block; width: 100%; height: 280px; }
+</style>

package/src/appium-recorder/components/FlowActionMenu.vue

@@ -0,0 +1,206 @@
+<script setup lang="ts">
+import { nextTick, shallowRef } from 'vue';
+import type { MenuInstance, PopoverInstance } from 'element-plus';
+import { PASTE_COMMAND, type FlowActionGroup, type InsertAction } from '../flow-graph';
+
+const props = defineProps<{
+  groups: FlowActionGroup[];
+  disabled?: boolean;
+  clipboardCount?: number;
+  isActionDisabled: (action: InsertAction) => boolean;
+}>();
+
+const emit = defineEmits<{
+  command: [action: InsertAction | typeof PASTE_COMMAND];
+}>();
+
+const popoverRef = shallowRef<PopoverInstance>();
+const triggerRef = shallowRef<HTMLElement>();
+const menuRef = shallowRef<HTMLElement>();
+const menuControlRef = shallowRef<MenuInstance>();
+const menuMounted = shallowRef(false);
+
+async function openMenu() {
+  // 每次打开重新创建面板，避免上次选中的操作被当成当前值而无法重复添加。
+  menuMounted.value = true;
+  await nextTick();
+  menuRef.value?.querySelector<HTMLElement>('[role="menuitem"]')?.focus();
+}
+
+function openGroupOnClick(event: MouseEvent, title: string) {
+  if ((event.target as HTMLElement).closest('.el-sub-menu__title')) {
+    menuControlRef.value?.open(title);
+  }
+}
+
+async function handleMenuKeydown(event: KeyboardEvent) {
+  const target = (event.target as HTMLElement).closest<HTMLElement>('[role="menuitem"]');
+  if (!target || !['Enter', ' ', 'ArrowRight', 'ArrowLeft', 'ArrowUp', 'ArrowDown'].includes(event.key)) return;
+  event.preventDefault();
+  event.stopPropagation();
+  const group = target.dataset.group;
+  if (event.key === 'ArrowUp' || event.key === 'ArrowDown') {
+    const siblings = Array.from(target.parentElement!.querySelectorAll<HTMLElement>(
+      ':scope > [role="menuitem"]:not(.is-disabled)',
+    ));
+    const offset = event.key === 'ArrowUp' ? -1 : 1;
+    siblings[(siblings.indexOf(target) + offset + siblings.length) % siblings.length]?.focus();
+  } else if (event.key === 'ArrowLeft') {
+    const parent = target.parentElement?.closest<HTMLElement>('[data-group]');
+    if (parent?.dataset.group) {
+      menuControlRef.value?.close(parent.dataset.group);
+      parent.focus();
+    }
+  } else if (group) {
+    menuControlRef.value?.open(group);
+    await nextTick();
+    target.querySelector<HTMLElement>('.el-menu-item:not(.is-disabled)')?.focus();
+  } else if (event.key !== 'ArrowRight') {
+    target.click();
+  }
+}
+
+function closeMenu() {
+  popoverRef.value?.hide();
+  triggerRef.value?.querySelector('button')?.focus();
+}
+
+function selectAction(value: string) {
+  if (props.disabled) return;
+  const action = value as InsertAction | typeof PASTE_COMMAND;
+  if (action === PASTE_COMMAND ? !props.clipboardCount : props.isActionDisabled(action)) return;
+  closeMenu();
+  emit('command', action);
+}
+</script>
+
+<template>
+  <span ref="triggerRef" class="appium-action-menu-trigger">
+    <el-popover
+      ref="popoverRef"
+      trigger="click"
+      placement="bottom-start"
+      width="auto"
+      :disabled="disabled"
+      :show-arrow="false"
+      :hide-after="0"
+      popper-class="appium-action-dropdown appium-action-menu"
+      @before-enter="openMenu"
+      @after-leave="menuMounted = false"
+    >
+      <template #reference><slot /></template>
+      <div
+        ref="menuRef"
+        @keydown.capture="handleMenuKeydown"
+        @keydown.esc.stop.prevent="closeMenu"
+        @keydown.tab="closeMenu"
+      >
+        <el-menu
+          v-if="menuMounted"
+          ref="menuControlRef"
+          collapse
+          unique-opened
+          :collapse-transition="false"
+          :show-timeout="100"
+          :hide-timeout="250"
+          :popper-offset="4"
+          @select="selectAction"
+        >
+          <el-sub-menu
+            v-for="group in groups"
+            :key="group.title"
+            :index="group.title"
+            :data-group="group.title"
+            :aria-label="group.title"
+            tabindex="-1"
+            :teleported="false"
+            popper-class="appium-action-submenu"
+            @click="openGroupOnClick($event, group.title)"
+          >
+            <template #title>{{ group.title }}</template>
+            <el-menu-item
+              v-for="action in group.actions"
+              :key="action.type"
+              :index="action.type"
+              :disabled="isActionDisabled(action.type)"
+            >
+              {{ action.label }}
+            </el-menu-item>
+          </el-sub-menu>
+          <el-menu-item v-if="clipboardCount" :index="PASTE_COMMAND" class="appium-action-paste">
+            粘贴 {{ clipboardCount }} 个节点
+          </el-menu-item>
+        </el-menu>
+      </div>
+    </el-popover>
+  </span>
+</template>
+
+<style>
+.appium-action-menu-trigger {
+  display: inline-flex;
+}
+
+.appium-action-menu.el-popover {
+  min-width: 0;
+  max-width: calc(100vw - 24px);
+  padding: 0;
+  overflow: visible;
+}
+
+.appium-action-menu .el-menu {
+  --el-menu-item-height: 34px;
+  --el-menu-sub-item-height: 34px;
+  --el-menu-base-level-padding: 14px;
+  --el-menu-text-color: var(--el-text-color-regular);
+  --el-menu-hover-bg-color: var(--el-color-primary-light-9);
+}
+
+.appium-action-menu .el-menu--collapse {
+  width: 132px;
+  padding: 6px 0;
+  border: 0;
+  border-radius: 6px;
+}
+
+.appium-action-menu .appium-action-paste {
+  font-size: 12px;
+  font-weight: 700;
+}
+
+.appium-action-menu .el-menu--collapse > .el-sub-menu > .el-sub-menu__title .el-sub-menu__icon-arrow {
+  display: inline-flex;
+  position: static;
+  width: 14px;
+  height: 14px;
+  margin: 0 0 0 auto;
+  font-size: 14px;
+  transform: none !important;
+}
+
+.appium-action-menu .el-sub-menu:focus-visible > .el-sub-menu__title,
+.appium-action-menu .el-sub-menu.is-opened > .el-sub-menu__title,
+.appium-action-menu .el-menu-item:focus-visible,
+.appium-action-menu .el-menu-item:not(.is-disabled):hover {
+  color: var(--el-color-primary);
+  background: var(--el-color-primary-light-9);
+}
+
+.appium-action-submenu .el-menu--popup {
+  width: 226px;
+  min-width: 0;
+  max-width: calc(100vw - 24px);
+  max-height: min(360px, calc(100dvh - 48px));
+  overflow-y: auto;
+  overscroll-behavior: contain;
+}
+
+.appium-action-submenu .el-menu-item {
+  height: auto;
+  min-height: 34px;
+  padding: 7px 14px;
+  line-height: 20px;
+  white-space: normal;
+  overflow-wrap: anywhere;
+}
+</style>

package/src/appium-recorder/components/FlowCanvas.vue

@@ -13,6 +13,7 @@ import {
 import type { AppiumRecordedStep } from '../types';
 
 const props = defineProps<{
+  aiRecognitionModelConfigured?: boolean;
   id: string;
   steps: AppiumRecordedStep[];
   expandedStepIndex: number | null;
@@ -55,6 +56,7 @@ let resetViewFrame = 0;
 let pendingResetView = false;
 
 const graph = computed(() => buildFlowGraph(props.steps, {
+  aiRecognitionModelConfigured: props.aiRecognitionModelConfigured,
   expandedStepIndex: props.expandedStepIndex,
   selectedCopyIndexes: props.selectedCopyIndexes,
   copyMode: props.copyMode,
@@ -118,7 +120,19 @@ function handlePaneReady() {
   void nextTick(() => scheduleResetView());
 }
 
-watch(() => props.steps.map((step) => step.id).join('|'), () => {
+function flowStructureKey() {
+  return props.steps.map((step) => [
+    step.id,
+    step.flow?.nodeKind || '',
+    step.flow?.parentConditionId || '',
+    step.flow?.parentBranch || '',
+    step.flow?.yesTargetId || '',
+    step.flow?.noTargetId || '',
+    step.flow?.successTargetId || '',
+  ].join(':')).join('|');
+}
+
+watch(flowStructureKey, () => {
   measuredNodeHeights.value = {};
   void nextTick(scheduleNodeInternalsUpdate);
 });
@@ -168,6 +182,7 @@ function handleInsert(payload: {
       :nodes-draggable="false"
       :nodes-connectable="false"
       :elements-selectable="false"
+      :only-render-visible-elements="false"
       :pan-on-drag="true"
       :zoom-on-scroll="true"
       zoom-activation-key-code="Control"

package/src/appium-recorder/components/FlowNodeCard.vue

@@ -1,7 +1,8 @@
 <script setup lang="ts">
 import { computed, nextTick, onBeforeUnmount, onMounted, shallowRef, watch } from 'vue';
 import { Handle, Position } from '@vue-flow/core';
-import { CopyDocument, Delete, Edit, Plus, VideoPlay, View } from '@element-plus/icons-vue';
+import { CopyDocument, Delete, Edit, Plus, VideoPlay, View, WarningFilled } from '@element-plus/icons-vue';
+import { AI_MODEL_CONFIG_HINT } from '../ai-recognition';
 import type { AppiumRecordedStep } from '../types';
 import {
   PASTE_COMMAND,
@@ -10,6 +11,7 @@ import {
   type InsertAction,
 } from '../flow-graph';
 import FlowStepEditor from './FlowStepEditor.vue';
+import FlowActionMenu from './FlowActionMenu.vue';
 
 const props = defineProps<{
   nodeId: string;
@@ -32,15 +34,9 @@ const emit = defineEmits<{
 const rootRef = shallowRef<HTMLElement | null>(null);
 let resizeObserver: ResizeObserver | null = null;
 
-const branchLabel = computed(() => {
-  if (props.data.kind !== 'branch') return '';
-  return props.data.branch === 'yes' ? '是' : '否';
-});
-
 const insertTitle = computed(() => {
   if (props.data.kind !== 'insert') return '';
-  if (props.data.branch === 'yes') return '在“是”分支插入操作';
-  if (props.data.branch === 'no') return '在“否”分支插入操作';
+  if (props.data.branch) return `在“${props.data.branchLabel || (props.data.branch === 'yes' ? '是' : '否')}”分支插入操作`;
   return props.data.afterIndex < 0 ? '在开始后插入操作' : '插入操作';
 });
 
@@ -92,6 +88,7 @@ watch(() => props.data, () => {
     ref="rootRef"
     class="appium-flow-graph-node nodrag nopan"
     :class="`appium-flow-graph-node--${data.kind}`"
+    :style="data.kind === 'branch' ? { width: `${data.width}px` } : undefined"
     @pointerdown.stop
     @mousedown.stop
     @touchstart.stop
@@ -116,9 +113,10 @@ watch(() => props.data, () => {
       type="button"
       class="appium-flow-branch-pill nodrag nopan"
       :class="`appium-flow-branch-pill--${data.branch}`"
+      style="width: 100%"
       tabindex="-1"
     >
-      {{ branchLabel }}
+      {{ data.label }}
     </button>
 
     <span
@@ -134,11 +132,11 @@ watch(() => props.data, () => {
         aria-hidden="true"
       />
       <el-tooltip v-else content="添加操作" placement="top">
-        <el-dropdown
-          trigger="click"
+        <FlowActionMenu
+          :groups="data.actionGroups"
           :disabled="!data.canOpenInsertMenu"
-          max-height="320px"
-          popper-class="appium-action-dropdown"
+          :clipboard-count="data.clipboardCount"
+          :is-action-disabled="data.isActionDisabled"
           @command="emitInsert"
         >
           <el-button
@@ -148,29 +146,7 @@ watch(() => props.data, () => {
             :disabled="!data.canOpenInsertMenu"
             :aria-label="insertTitle"
           />
-          <template #dropdown>
-            <el-dropdown-menu>
-              <el-dropdown-item
-                v-if="data.clipboardCount"
-                :command="PASTE_COMMAND"
-                divided
-              >
-                粘贴 {{ data.clipboardCount }} 个节点
-              </el-dropdown-item>
-              <template v-for="group in data.actionGroups" :key="group.title">
-                <div class="appium-action-dropdown__group">{{ group.title }}</div>
-                <el-dropdown-item
-                  v-for="action in group.actions"
-                  :key="action.type"
-                  :command="action.type"
-                  :disabled="data.isActionDisabled(action.type)"
-                >
-                  {{ action.label }}
-                </el-dropdown-item>
-              </template>
-            </el-dropdown-menu>
-          </template>
-        </el-dropdown>
+        </FlowActionMenu>
       </el-tooltip>
     </div>
 
@@ -194,6 +170,11 @@ watch(() => props.data, () => {
         @click="handleStepClick"
       >
         <span class="appium-flow-step-card__content">
+          <el-tooltip v-if="data.missingAiModel" :content="AI_MODEL_CONFIG_HINT" placement="top">
+            <el-icon class="appium-ai-model-warning" color="var(--el-color-danger)" role="img" aria-label="AI 识别模型未配置" tabindex="0">
+              <WarningFilled />
+            </el-icon>
+          </el-tooltip>
           <strong :title="data.title">{{ data.title }}</strong>
           <small :title="data.meta">{{ data.meta }}</small>
           <em v-if="data.note" :title="data.note">{{ data.note }}</em>
@@ -215,7 +196,8 @@ watch(() => props.data, () => {
           :icon="VideoPlay"
           :loading="data.launching"
           :disabled="data.disabled"
-          title="立即执行"
+          :title="data.step.type === 'aiRecognition' ? '测试 AI 识别' : '立即执行'"
+          :aria-label="data.step.type === 'aiRecognition' ? '测试 AI 识别' : '立即执行'"
           @click="emit('execute', data.index)"
         />
         <el-button
@@ -262,3 +244,7 @@ watch(() => props.data, () => {
     />
   </div>
 </template>
+
+<style scoped>
+.appium-ai-model-warning { float: left; margin: 2px 6px 0 0; font-size: 16px; }
+</style>