android-midscene-automation 0.1.26 → 0.1.27

package/src/appium-recorder/api.ts

@@ -80,6 +80,7 @@ type ReplayResult = {
   reportId?: string;
   logPath?: string;
   htmlReportPath?: string;
+  softFailureCount?: number;
 };
 
 type ReplayStreamEvent =

package/src/appium-recorder/components/FlowCanvas.vue

@@ -118,7 +118,19 @@ function handlePaneReady() {
   void nextTick(() => scheduleResetView());
 }
 
-watch(() => props.steps.map((step) => step.id).join('|'), () => {
+function flowStructureKey() {
+  return props.steps.map((step) => [
+    step.id,
+    step.flow?.nodeKind || '',
+    step.flow?.parentConditionId || '',
+    step.flow?.parentBranch || '',
+    step.flow?.yesTargetId || '',
+    step.flow?.noTargetId || '',
+    step.flow?.successTargetId || '',
+  ].join(':')).join('|');
+}
+
+watch(flowStructureKey, () => {
   measuredNodeHeights.value = {};
   void nextTick(scheduleNodeInternalsUpdate);
 });
@@ -168,6 +180,7 @@ function handleInsert(payload: {
       :nodes-draggable="false"
       :nodes-connectable="false"
       :elements-selectable="false"
+      :only-render-visible-elements="false"
       :pan-on-drag="true"
       :zoom-on-scroll="true"
       zoom-activation-key-code="Control"

package/src/appium-recorder/components/FlowStepEditor.vue

@@ -1,8 +1,11 @@
 <script setup lang="ts">
+import { computed } from 'vue';
 import type { AppiumRecordedStep, AppiumSelector } from '../types';
+import { normalizeVisualChangeConfig } from '../visual-change';
 
 type FlowKind = 'action' | 'condition' | 'assertion';
 type SwipeGesture = NonNullable<AppiumRecordedStep['swipe']>;
+type VisualChangeConfig = NonNullable<AppiumRecordedStep['visualChange']>;
 
 const props = defineProps<{
   step: AppiumRecordedStep;
@@ -14,9 +17,21 @@ const emit = defineEmits<{
   update: [payload: { index: number; step: AppiumRecordedStep }];
 }>();
 
+const visualChangeConfig = computed(() => normalizeVisualChangeConfig(props.step.visualChange));
+const visualChangeRoleLabel = computed(() => (
+  visualChangeConfig.value.role === 'start'
+    ? `${visualChangeConfig.value.pairLabel || '检测画面变化'} · 开始节点`
+    : visualChangeConfig.value.role === 'end'
+      ? `${visualChangeConfig.value.pairLabel || '检测画面变化'} · 结束节点`
+      : '起止节点截图对比'
+));
+const visualChangeConfigDisabled = computed(() => Boolean(
+  props.disabled || visualChangeConfig.value.role === 'end',
+));
+
 function defaultKind(): FlowKind {
   if (props.step.flow?.nodeKind) return props.step.flow.nodeKind;
-  if (props.step.type === 'assertExists' || props.step.type === 'assertText') return 'assertion';
+  if (props.step.type === 'assertExists' || props.step.type === 'assertText' || props.step.type === 'visualChange') return 'assertion';
   return 'action';
 }
 
@@ -55,6 +70,25 @@ function patchSwipe(patch: Partial<SwipeGesture>) {
   });
 }
 
+function patchVisualChange(patch: Partial<VisualChangeConfig>) {
+  patchStep({
+    visualChange: normalizeVisualChangeConfig({
+      ...normalizeVisualChangeConfig(props.step.visualChange),
+      ...patch,
+    }),
+  });
+}
+
+function patchVisualRegion(key: keyof VisualChangeConfig['region'], value: unknown) {
+  const current = normalizeVisualChangeConfig(props.step.visualChange);
+  patchVisualChange({
+    region: {
+      ...current.region,
+      [key]: Number(value),
+    },
+  });
+}
+
 function timeoutLabel() {
   return props.step.type === 'longPress' ? '长按时间 ms' : '超时时间 ms';
 }
@@ -121,6 +155,91 @@ function patchTimeout(value: unknown) {
           @update:model-value="patchStep({ value: String($event) })"
         />
       </el-form-item>
+      <template v-if="step.type === 'visualChange'">
+        <div class="appium-flow-editor__grid">
+          <el-form-item label="检测标记">
+            <el-input :model-value="visualChangeRoleLabel" disabled />
+          </el-form-item>
+          <el-form-item label="检测目标">
+            <el-select
+              :model-value="visualChangeConfig.mode"
+              :disabled="visualChangeConfigDisabled"
+              @update:model-value="patchVisualChange({ mode: $event as VisualChangeConfig['mode'] })"
+            >
+              <el-option label="当前选中元素" value="selectedElement" />
+              <el-option label="手动区域" value="region" />
+            </el-select>
+          </el-form-item>
+          <el-form-item label="变化阈值 %">
+            <el-input-number
+              :model-value="visualChangeConfig.changeRatioThreshold"
+              :disabled="visualChangeConfigDisabled"
+              :min="0.01"
+              :max="100"
+              :step="0.1"
+              controls-position="right"
+              @update:model-value="patchVisualChange({ changeRatioThreshold: Number($event) })"
+            />
+          </el-form-item>
+        </div>
+        <div class="appium-flow-editor__grid appium-flow-editor__grid--visual">
+          <el-form-item label="X">
+            <el-input-number
+              :model-value="visualChangeConfig.region.x"
+              :disabled="visualChangeConfigDisabled"
+              :min="0"
+              :max="99999"
+              :precision="0"
+              controls-position="right"
+              @update:model-value="patchVisualRegion('x', $event)"
+            />
+          </el-form-item>
+          <el-form-item label="Y">
+            <el-input-number
+              :model-value="visualChangeConfig.region.y"
+              :disabled="visualChangeConfigDisabled"
+              :min="0"
+              :max="99999"
+              :precision="0"
+              controls-position="right"
+              @update:model-value="patchVisualRegion('y', $event)"
+            />
+          </el-form-item>
+          <el-form-item label="宽度">
+            <el-input-number
+              :model-value="visualChangeConfig.region.width"
+              :disabled="visualChangeConfigDisabled"
+              :min="1"
+              :max="99999"
+              :precision="0"
+              controls-position="right"
+              @update:model-value="patchVisualRegion('width', $event)"
+            />
+          </el-form-item>
+          <el-form-item label="高度">
+            <el-input-number
+              :model-value="visualChangeConfig.region.height"
+              :disabled="visualChangeConfigDisabled"
+              :min="1"
+              :max="99999"
+              :precision="0"
+              controls-position="right"
+              @update:model-value="patchVisualRegion('height', $event)"
+            />
+          </el-form-item>
+          <el-form-item label="像素容差">
+            <el-input-number
+              :model-value="visualChangeConfig.pixelmatchThreshold"
+              :disabled="visualChangeConfigDisabled"
+              :min="0"
+              :max="1"
+              :step="0.01"
+              controls-position="right"
+              @update:model-value="patchVisualChange({ pixelmatchThreshold: Number($event) })"
+            />
+          </el-form-item>
+        </div>
+      </template>
       <div v-if="step.type === 'swipe'" class="appium-flow-editor__grid appium-flow-editor__grid--swipe">
         <el-form-item label="起点 X">
           <el-input-number
@@ -222,7 +341,7 @@ function patchTimeout(value: unknown) {
           />
         </el-form-item>
       </div>
-      <el-form-item label="可选步骤">
+      <el-form-item v-if="step.type !== 'visualChange'" label="可选步骤">
         <el-switch
           :model-value="Boolean(step.optional)"
           :disabled="disabled"

package/src/appium-recorder/components/NestedConditionBranches.vue

@@ -32,7 +32,11 @@ type InsertAction =
   | 'assertText'
   | 'waitDisappear'
   | 'waitActivity'
-  | 'runScript';
+  | 'runScript'
+  | 'noop'
+  | 'visualChangeStart'
+  | 'visualChangeEnd'
+  | 'visualChange';
 
 const PASTE_COMMAND = '__paste_flow_nodes__';
 
@@ -87,6 +91,7 @@ const actionGroups: Array<{ title: string; actions: Array<{ type: InsertAction;
       { type: 'keyPower', label: '电源键' },
       { type: 'swipe', label: '滑动' },
       { type: 'pinch', label: '双指缩放' },
+      { type: 'noop', label: '空节点' },
     ],
   },
   {
@@ -103,6 +108,8 @@ const actionGroups: Array<{ title: string; actions: Array<{ type: InsertAction;
       { type: 'assertText', label: '断言文本' },
       { type: 'waitDisappear', label: '等待元素消失' },
       { type: 'waitActivity', label: '等待 Activity' },
+      { type: 'visualChangeStart', label: '检测画面变化开始节点' },
+      { type: 'visualChangeEnd', label: '检测画面变化结束节点' },
     ],
   },
   { title: '流程控制', actions: [{ type: 'runScript', label: '连接脚本' }] },
@@ -110,7 +117,7 @@ const actionGroups: Array<{ title: string; actions: Array<{ type: InsertAction;
 
 function defaultKind(step: AppiumRecordedStep): FlowKind {
   if (step.flow?.nodeKind) return step.flow.nodeKind;
-  if (step.type === 'assertExists' || step.type === 'assertText') return 'assertion';
+  if (step.type === 'assertExists' || step.type === 'assertText' || step.type === 'visualChange') return 'assertion';
   return 'action';
 }
 
@@ -125,7 +132,8 @@ function typeLabel(step: AppiumRecordedStep) {
     waitFor: '等待出现', waitDisappear: '等待消失', assertExists: '断言存在',
     assertText: '断言文本', key: '按键', waitActivity: '等待 Activity', delay: '延时',
     coordinateTap: '坐标点击', swipe: '滑动', launchApp: '启动 APP', clearAppData: '清理 APP 缓存', longPress: '长按',
-    pinch: '双指缩放', runScript: '连接脚本', screenshot: '截图',
+    pinch: '双指缩放', runScript: '连接脚本', screenshot: '截图', visualChange: '检测画面变化',
+    noop: '空节点',
   };
   return labels[step.type] || step.type;
 }
@@ -139,6 +147,14 @@ function stepMeta(step: AppiumRecordedStep) {
     const swipe = step.swipe;
     return swipe ? `[${swipe.startX},${swipe.startY}] -> [${swipe.endX},${swipe.endY}]` : '';
   }
+  if (step.type === 'visualChange') {
+    const config = step.visualChange;
+    if (!config) return '';
+    if (config.role === 'start') return `${config.pairLabel || '检测画面变化'} · 开始 · 阈值 ${config.changeRatioThreshold}%`;
+    if (config.role === 'end') return `${config.pairLabel || '检测画面变化'} · 结束`;
+    return `${config.startStepId && config.endStepId ? '起止节点截图对比' : '请选择起止节点'} · 阈值 ${config.changeRatioThreshold}%`;
+  }
+  if (step.type === 'noop') return '';
   if (defaultKind(step) === 'condition' && step.value) {
     return `${step.flow?.textMatch === 'exact' ? '精准匹配' : '模糊匹配'}：${step.value}`;
   }
@@ -199,6 +215,12 @@ function isActionDisabled(action: InsertAction) {
   return Boolean(props.disabled && !props.allowedLockedActions?.includes(action));
 }
 
+function canCopyStep(step: AppiumRecordedStep) {
+  return step.type !== 'launchApp'
+    && step.type !== 'clearAppData'
+    && step.visualChange?.role !== 'end';
+}
+
 function insert(branch: BranchName, command: string | number | object) {
   if (command === PASTE_COMMAND) emit('paste', props.conditionIndex, branch);
   else emit('insertBranchAction', props.conditionIndex, branch, command as InsertAction);
@@ -275,7 +297,7 @@ function branchSplitPath() {
               </button>
               <span class="appium-flow-node__actions" @click.stop>
                 <el-button
-                  v-if="!copyMode"
+                  v-if="!copyMode && canCopyStep(item.step)"
                   text
                   size="small"
                   :icon="CopyDocument"

package/src/appium-recorder/components/RecordedSteps.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { computed, shallowRef } from 'vue';
+import { computed, nextTick, shallowRef, watch } from 'vue';
 import { ElMessage } from 'element-plus';
 import { Aim, CopyDocument, FullScreen } from '@element-plus/icons-vue';
 import type { AppiumRecordedStep } from '../types';
@@ -35,6 +35,7 @@ const copyMode = shallowRef(false);
 const selectedCopyIndexes = shallowRef<number[]>([]);
 const flowDialogVisible = shallowRef(false);
 const mainResetViewToken = shallowRef(0);
+const dialogResetViewToken = shallowRef(0);
 
 const insertActionGroups: FlowActionGroup[] = [
   {
@@ -56,6 +57,7 @@ const insertActionGroups: FlowActionGroup[] = [
       { type: 'keyPower', label: '电源键' },
       { type: 'swipe', label: '滑动' },
       { type: 'pinch', label: '双指缩放' },
+      { type: 'noop', label: '空节点' },
       { type: 'clearAppData', label: '清理 App 缓存' },
       { type: 'launchApp', label: '启动 App' },
     ],
@@ -74,6 +76,8 @@ const insertActionGroups: FlowActionGroup[] = [
       { type: 'assertText', label: '断言文本' },
       { type: 'waitDisappear', label: '等待元素消失' },
       { type: 'waitActivity', label: '等待 Activity' },
+      { type: 'visualChangeStart', label: '检测画面变化开始节点' },
+      { type: 'visualChangeEnd', label: '检测画面变化结束节点' },
     ],
   },
   {
@@ -169,6 +173,13 @@ function resetMainFlowPosition() {
   mainResetViewToken.value += 1;
 }
 
+watch(flowDialogVisible, (visible) => {
+  if (!visible) return;
+  void nextTick(() => {
+    dialogResetViewToken.value += 1;
+  });
+});
+
 function canOpenInsertMenu() {
   return !props.disabled || Boolean(props.allowedLockedActions?.length);
 }
@@ -308,6 +319,7 @@ function updateStep(index: number, step: AppiumRecordedStep) {
         :remove-disabled="removeDisabled"
         :launching-step-id="launchingStepId"
         :clipboard-count="clipboardCount"
+        :reset-view-token="dialogResetViewToken"
         :start-action-groups="startActionGroups"
         :main-action-groups="mainActionGroups"
         :can-open-insert-menu="canOpenInsertMenu()"

package/src/appium-recorder/components/VisualChangeDialog.vue

@@ -0,0 +1,151 @@
+<script setup lang="ts">
+import { computed } from 'vue';
+import { Aim } from '@element-plus/icons-vue';
+import type { AppiumVisualChangeConfig } from '../types';
+import { normalizeVisualChangeConfig } from '../visual-change';
+
+const props = defineProps<{
+  modelValue: boolean;
+  config: AppiumVisualChangeConfig;
+  hasSelectedElement: boolean;
+  pickingRegion?: boolean;
+}>();
+
+const emit = defineEmits<{
+  'update:modelValue': [value: boolean];
+  'update:config': [value: AppiumVisualChangeConfig];
+  pickRegion: [];
+  confirm: [];
+}>();
+
+const normalizedConfig = computed(() => normalizeVisualChangeConfig(props.config));
+
+function patchConfig(patch: Partial<AppiumVisualChangeConfig>) {
+  emit('update:config', normalizeVisualChangeConfig({
+    ...normalizedConfig.value,
+    ...patch,
+  }));
+}
+
+function patchRegion(key: keyof AppiumVisualChangeConfig['region'], value: unknown) {
+  patchConfig({
+    region: {
+      ...normalizedConfig.value.region,
+      [key]: Number(value),
+    },
+  });
+}
+</script>
+
+<template>
+  <el-dialog
+    :model-value="modelValue"
+    title="检测画面变化"
+    width="480px"
+    top="9vh"
+    class="appium-visual-change-dialog"
+    append-to-body
+    draggable
+    :modal="false"
+    modal-penetrable
+    :close-on-click-modal="false"
+    :close-on-press-escape="false"
+    @update:model-value="emit('update:modelValue', $event)"
+  >
+    <el-form label-position="top" size="small" class="appium-visual-change-form">
+      <el-form-item label="检测目标">
+        <el-radio-group
+          :model-value="normalizedConfig.mode"
+          @update:model-value="patchConfig({ mode: $event as AppiumVisualChangeConfig['mode'] })"
+        >
+          <el-radio-button label="selectedElement" :disabled="!hasSelectedElement">
+            当前选中元素
+          </el-radio-button>
+          <el-radio-button label="region">手动框选区域</el-radio-button>
+        </el-radio-group>
+      </el-form-item>
+
+      <el-alert
+        v-if="normalizedConfig.mode === 'selectedElement' && !hasSelectedElement"
+        type="warning"
+        :closable="false"
+        show-icon
+        title="当前没有可用的选中元素，请先选择组件或改为手动框选区域"
+      />
+
+      <el-form-item v-if="normalizedConfig.mode === 'region'" label="框选区域">
+        <el-button :icon="Aim" :loading="pickingRegion" @click="emit('pickRegion')">
+          {{ pickingRegion ? '请在设备预览拖拽' : '框选/拖动区域' }}
+        </el-button>
+      </el-form-item>
+
+      <div class="appium-visual-change-form__grid">
+        <el-form-item label="X">
+          <el-input-number
+            :model-value="normalizedConfig.region.x"
+            :min="0"
+            :max="99999"
+            :precision="0"
+            controls-position="right"
+            @update:model-value="patchRegion('x', $event)"
+          />
+        </el-form-item>
+        <el-form-item label="Y">
+          <el-input-number
+            :model-value="normalizedConfig.region.y"
+            :min="0"
+            :max="99999"
+            :precision="0"
+            controls-position="right"
+            @update:model-value="patchRegion('y', $event)"
+          />
+        </el-form-item>
+        <el-form-item label="宽度">
+          <el-input-number
+            :model-value="normalizedConfig.region.width"
+            :min="1"
+            :max="99999"
+            :precision="0"
+            controls-position="right"
+            @update:model-value="patchRegion('width', $event)"
+          />
+        </el-form-item>
+        <el-form-item label="高度">
+          <el-input-number
+            :model-value="normalizedConfig.region.height"
+            :min="1"
+            :max="99999"
+            :precision="0"
+            controls-position="right"
+            @update:model-value="patchRegion('height', $event)"
+          />
+        </el-form-item>
+        <el-form-item label="变化阈值 %">
+          <el-input-number
+            :model-value="normalizedConfig.changeRatioThreshold"
+            :min="0.01"
+            :max="100"
+            :step="0.1"
+            controls-position="right"
+            @update:model-value="patchConfig({ changeRatioThreshold: Number($event) })"
+          />
+        </el-form-item>
+        <el-form-item label="像素容差">
+          <el-input-number
+            :model-value="normalizedConfig.pixelmatchThreshold"
+            :min="0"
+            :max="1"
+            :step="0.01"
+            controls-position="right"
+            @update:model-value="patchConfig({ pixelmatchThreshold: Number($event) })"
+          />
+        </el-form-item>
+      </div>
+    </el-form>
+
+    <template #footer>
+      <el-button @click="emit('update:modelValue', false)">取消</el-button>
+      <el-button type="primary" @click="emit('confirm')">添加开始节点</el-button>
+    </template>
+  </el-dialog>
+</template>

package/src/appium-recorder/flow-copy.ts

@@ -58,6 +58,33 @@ function wireContinuation(step: AppiumRecordedStep, targetId: string) {
   if (!step.flow.noTargetId) setFlowTarget(step, 'noTargetId', targetId);
 }
 
+function isVisualChangeStart(step: AppiumRecordedStep) {
+  return step.visualChange?.role === 'start';
+}
+
+function isVisualChangeEnd(step: AppiumRecordedStep) {
+  return step.visualChange?.role === 'end';
+}
+
+function visualChangePairKeys(step: AppiumRecordedStep) {
+  const config = step.visualChange;
+  if (!config) return [];
+  return [
+    config.pairId ? `pair:${config.pairId}` : '',
+    step.id ? `start:${step.id}` : '',
+    config.startStepId ? `start:${config.startStepId}` : '',
+    config.pairLabel ? `label:${config.pairLabel}` : '',
+  ].filter(Boolean);
+}
+
+function hasCopiedVisualChangeStartForEnd(copiedSteps: AppiumRecordedStep[], endStep: AppiumRecordedStep) {
+  const endKeys = new Set(visualChangePairKeys(endStep));
+  return copiedSteps.some((step) => (
+    isVisualChangeStart(step)
+      && visualChangePairKeys(step).some((key) => endKeys.has(key))
+  ));
+}
+
 function branchConditionForTarget(
   steps: AppiumRecordedStep[],
   target: PasteTarget,
@@ -65,10 +92,10 @@ function branchConditionForTarget(
   if (!target.branch) return undefined;
   const anchor = steps[target.afterIndex];
   if (!anchor) return undefined;
+  if (anchor.flow?.nodeKind === 'condition') return anchor;
   if (anchor.flow?.parentConditionId && anchor.flow.parentBranch === target.branch) {
     return steps.find((step) => step.id === anchor.flow?.parentConditionId);
   }
-  if (anchor.flow?.nodeKind === 'condition') return anchor;
   return undefined;
 }
 
@@ -123,6 +150,9 @@ export function createFlowClipboard(steps: AppiumRecordedStep[], indexes: number
   if (copiedSteps.some((step) => step.type === 'launchApp')) {
     throw new Error('启动 APP 节点不能复制');
   }
+  if (copiedSteps.some((step) => isVisualChangeEnd(step) && !hasCopiedVisualChangeStartForEnd(copiedSteps, step))) {
+    throw new Error('检测画面变化结束节点不能单独复制');
+  }
   return { steps: clone(copiedSteps), rootIds };
 }
 
@@ -141,6 +171,11 @@ export function pasteFlowClipboard(
     throw new Error('目标分支不存在');
   }
 
+  const pairMap = new Map<string, string>();
+  clipboard.steps.filter(isVisualChangeStart).forEach((step) => {
+    const pairId = createId();
+    visualChangePairKeys(step).forEach((key) => pairMap.set(key, pairId));
+  });
   const copies = clipboard.steps.map((source) => {
     const step = clone(source);
     step.id = idMap.get(source.id) || createId();
@@ -168,6 +203,17 @@ export function pasteFlowClipboard(
       }
     }
     step.flow = Object.keys(flow).length ? flow : undefined;
+    if (step.visualChange) {
+      const nextPairId = visualChangePairKeys(source)
+        .map((key) => pairMap.get(key))
+        .find(Boolean);
+      step.visualChange = {
+        ...step.visualChange,
+        pairId: nextPairId || step.visualChange.pairId,
+        startStepId: idMap.get(step.visualChange.startStepId || '') || step.visualChange.startStepId,
+        endStepId: idMap.get(step.visualChange.endStepId || '') || step.visualChange.endStepId,
+      };
+    }
     return step;
   });