android-midscene-automation 0.1.26 → 0.1.27

package/src/appium-recorder/flow-graph.ts

@@ -34,7 +34,11 @@ export type InsertAction =
   | 'assertText'
   | 'waitDisappear'
   | 'waitActivity'
-  | 'runScript';
+  | 'runScript'
+  | 'noop'
+  | 'visualChangeStart'
+  | 'visualChangeEnd'
+  | 'visualChange';
 
 export const PASTE_COMMAND = '__paste_flow_nodes__';
 export const FLOW_STEP_NODE_WIDTH = 340;
@@ -50,6 +54,7 @@ const FLOW_STANDARD_LINE_GAP = 24;
 const FLOW_BRANCH_TRUNK_GAP = 96;
 const FLOW_BRANCH_LABEL_GAP = 58;
 const FLOW_BRANCH_MIN_SPREAD = 260;
+const FLOW_BRANCH_SUBTREE_GAP = 80;
 
 export type FlowActionGroup = {
   title: string;
@@ -137,9 +142,16 @@ type Link = {
   visual?: boolean;
 };
 
+type BranchLayout = {
+  yesWidth: number;
+  noWidth: number;
+  centerGap: number;
+  totalWidth: number;
+};
+
 function defaultKind(step: AppiumRecordedStep): FlowKind {
   if (step.flow?.nodeKind) return step.flow.nodeKind;
-  if (step.type === 'assertExists' || step.type === 'assertText') return 'assertion';
+  if (step.type === 'assertExists' || step.type === 'assertText' || step.type === 'visualChange') return 'assertion';
   return 'action';
 }
 
@@ -232,6 +244,21 @@ function estimateStepNodeHeight(label: { title: string; meta: string; note?: str
   return Math.max(FLOW_STEP_NODE_HEIGHT, 34 + textHeight);
 }
 
+function withLaunchAppAction(groups: FlowActionGroup[]) {
+  if (groups.some((group) => group.actions.some((action) => action.type === 'launchApp'))) return groups;
+  const launchAction = { type: 'launchApp' as const, label: '启动 App' };
+  let added = false;
+  const nextGroups = groups.map((group) => {
+    if (group.title !== '设备操作') return group;
+    added = true;
+    return {
+      ...group,
+      actions: [...group.actions, launchAction],
+    };
+  });
+  return added ? nextGroups : [...nextGroups, { title: '设备操作', actions: [launchAction] }];
+}
+
 export function buildFlowGraph(
   steps: AppiumRecordedStep[],
   options: BuildFlowGraphOptions,
@@ -242,9 +269,63 @@ export function buildFlowGraph(
   const layoutLinks: Link[] = [];
   const seenLinks = new Set<string>();
   const allMainItems = mainItems(items);
+  const branchLayouts = new Map<string, BranchLayout>();
+  const measuringBranches = new Set<string>();
+
+  function conditionBranchLayout(condition: AppiumRecordedStep): BranchLayout {
+    const cached = branchLayouts.get(condition.id);
+    if (cached) return cached;
+    if (measuringBranches.has(condition.id)) {
+      return {
+        yesWidth: FLOW_STEP_NODE_WIDTH,
+        noWidth: FLOW_STEP_NODE_WIDTH,
+        centerGap: FLOW_BRANCH_MIN_SPREAD * 2,
+        totalWidth: FLOW_BRANCH_MIN_SPREAD * 2 + FLOW_STEP_NODE_WIDTH,
+      };
+    }
+
+    measuringBranches.add(condition.id);
+    const branchWidth = (branch: FlowBranch) => Math.max(
+      FLOW_STEP_NODE_WIDTH,
+      ...directBranchItems(items, condition.id, branch).map(({ step }) => (
+        defaultKind(step) === 'condition'
+          ? conditionBranchLayout(step).totalWidth
+          : FLOW_STEP_NODE_WIDTH
+      )),
+    );
+    const yesWidth = branchWidth('yes');
+    const noWidth = branchWidth('no');
+    const centerGap = Math.max(
+      FLOW_BRANCH_MIN_SPREAD * 2,
+      yesWidth / 2 + noWidth / 2 + FLOW_BRANCH_SUBTREE_GAP,
+    );
+    const sideExtent = Math.max(
+      centerGap / 2 + yesWidth / 2,
+      centerGap / 2 + noWidth / 2,
+    );
+    const layout = {
+      yesWidth,
+      noWidth,
+      centerGap,
+      totalWidth: Math.max(FLOW_STEP_NODE_WIDTH, sideExtent * 2),
+    };
+    measuringBranches.delete(condition.id);
+    branchLayouts.set(condition.id, layout);
+    return layout;
+  }
+
+  items
+    .filter(({ step }) => defaultKind(step) === 'condition')
+    .forEach(({ step }) => conditionBranchLayout(step));
 
   const addNode = (node: FlowGraphNode) => {
-    nodes.push(node);
+    nodes.push({
+      ...node,
+      dimensions: {
+        width: Number(node.width) || FLOW_STEP_NODE_WIDTH,
+        height: Number(node.height) || FLOW_STEP_NODE_HEIGHT,
+      },
+    } as FlowGraphNode);
   };
   const addVisibleLink = (source: string, target: string, branch?: FlowBranch) => {
     addUniqueLink(links, seenLinks, {
@@ -266,6 +347,11 @@ export function buildFlowGraph(
   ) => {
     const id = insertNodeId(afterIndex, branch, condition?.step.id);
     const isStartInsert = afterIndex < 0 && !branch;
+    const previousStep = afterIndex >= 0 ? steps[afterIndex] : undefined;
+    const canLaunchAfterClearAppData = !isStartInsert
+      && !branch
+      && previousStep?.type === 'clearAppData'
+      && !steps.some((step) => step.type === 'launchApp');
     addNode({
       id,
       type: 'flow-node',
@@ -279,7 +365,11 @@ export function buildFlowGraph(
       connectable: false,
       data: {
         kind: 'insert',
-        actionGroups: isStartInsert ? options.startActionGroups : options.mainActionGroups,
+        actionGroups: isStartInsert
+          ? options.startActionGroups
+          : canLaunchAfterClearAppData
+            ? withLaunchAppAction(options.mainActionGroups)
+            : options.mainActionGroups,
         afterIndex,
         branch,
         conditionIndex: condition?.index,
@@ -328,7 +418,9 @@ export function buildFlowGraph(
           : options.disabled,
         removeDisabled: options.removeDisabled,
         launching: options.launchingStepId === item.step.id,
-        canCopy: item.step.type !== 'launchApp' && item.step.type !== 'clearAppData',
+        canCopy: item.step.type !== 'launchApp'
+          && item.step.type !== 'clearAppData'
+          && item.step.visualChange?.role !== 'end',
         canEditInput: item.step.type === 'input' || item.step.type === 'inputIfExists',
         canExecute: item.step.type === 'launchApp' || item.step.type === 'clearAppData',
       },
@@ -527,7 +619,10 @@ export function buildFlowGraph(
         const yesNode = nodeById.get(branchLinks.find((link) => link.branch === 'yes')?.target || '');
         const noNode = nodeById.get(branchLinks.find((link) => link.branch === 'no')?.target || '');
         const splitX = centerX(splitNode);
-        const spread = FLOW_BRANCH_MIN_SPREAD;
+        const conditionId = splitNode.id.startsWith('split:')
+          ? splitNode.id.slice('split:'.length)
+          : '';
+        const spread = (branchLayouts.get(conditionId)?.centerGap || FLOW_BRANCH_MIN_SPREAD * 2) / 2;
         if (yesNode) setCenterX(yesNode, splitX - spread);
         if (noNode) setCenterX(noNode, splitX + spread);
       });

package/src/appium-recorder/flow-labels.ts

@@ -1,9 +1,10 @@
 import type { FlowKind } from './flow-graph';
 import type { AppiumRecordedStep } from './types';
+import { visualChangeMeta } from './visual-change';
 
 export function defaultFlowKind(step: AppiumRecordedStep): FlowKind {
   if (step.flow?.nodeKind) return step.flow.nodeKind;
-  if (step.type === 'assertExists' || step.type === 'assertText') return 'assertion';
+  if (step.type === 'assertExists' || step.type === 'assertText' || step.type === 'visualChange') return 'assertion';
   return 'action';
 }
 
@@ -35,6 +36,8 @@ export function flowTypeLabel(step: AppiumRecordedStep) {
     longPress: '长按',
     pinch: '双指缩放',
     runScript: '连接脚本',
+    noop: '空节点',
+    visualChange: '检测画面变化',
   };
   return labelMap[step.type] || step.type;
 }
@@ -53,7 +56,9 @@ export function flowStepMeta(step: AppiumRecordedStep) {
   if (step.type === 'launchApp') return step.value || '';
   if (step.type === 'clearAppData') return `${step.value || ''}（清除数据与缓存）`;
   if (step.type === 'runScript') return step.value ? `脚本 ${step.value}` : '';
+  if (step.type === 'noop') return '';
   if (step.type === 'screenshot') return '保存当前截图';
+  if (step.type === 'visualChange') return visualChangeMeta(step);
   if (step.type === 'swipe') {
     const swipe = step.swipe;
     return swipe ? `[${swipe.startX},${swipe.startY}] -> [${swipe.endX},${swipe.endY}]` : '';

package/src/appium-recorder/flow-normalize.ts

@@ -0,0 +1,67 @@
+import type { AppiumRecordedStep } from './types';
+
+type BranchName = 'yes' | 'no';
+
+function isCondition(step: AppiumRecordedStep) {
+  return step.flow?.nodeKind === 'condition';
+}
+
+function isEmptyNestedCondition(step: AppiumRecordedStep) {
+  return isCondition(step)
+    && Boolean(step.flow?.parentConditionId)
+    && Boolean(step.flow?.parentBranch)
+    && !step.flow?.yesTargetId
+    && !step.flow?.noTargetId
+    && !step.flow?.successTargetId;
+}
+
+function sameParentBranch(step: AppiumRecordedStep, parentConditionId: string, parentBranch: BranchName) {
+  return step.flow?.parentConditionId === parentConditionId
+    && step.flow?.parentBranch === parentBranch;
+}
+
+export function normalizeLegacyNestedConditionBranches<T extends AppiumRecordedStep>(steps: T[]) {
+  const nextSteps = [...steps];
+  let changed = false;
+
+  for (let index = 0; index < nextSteps.length; index += 1) {
+    const condition = nextSteps[index];
+    if (!condition || !isEmptyNestedCondition(condition)) continue;
+
+    const parentConditionId = condition.flow?.parentConditionId || '';
+    const parentBranch = condition.flow?.parentBranch;
+    if (!parentConditionId || !parentBranch) continue;
+
+    const childIndexes: number[] = [];
+    for (let childIndex = index + 1; childIndex < nextSteps.length; childIndex += 1) {
+      if (!sameParentBranch(nextSteps[childIndex], parentConditionId, parentBranch)) break;
+      childIndexes.push(childIndex);
+    }
+    if (!childIndexes.length) continue;
+
+    const firstChild = nextSteps[childIndexes[0]];
+    nextSteps[index] = {
+      ...condition,
+      flow: {
+        ...(condition.flow || {}),
+        nodeKind: 'condition',
+        yesTargetId: firstChild.id,
+      },
+    };
+
+    childIndexes.forEach((childIndex) => {
+      const child = nextSteps[childIndex];
+      nextSteps[childIndex] = {
+        ...child,
+        flow: {
+          ...(child.flow || {}),
+          parentConditionId: condition.id,
+          parentBranch: 'yes',
+        },
+      };
+    });
+    changed = true;
+  }
+
+  return changed ? nextSteps : steps;
+}

package/src/appium-recorder/flow-remove.ts

@@ -0,0 +1,67 @@
+import type { AppiumRecordedStep } from './types';
+
+type FlowTargetKey = 'yesTargetId' | 'noTargetId' | 'successTargetId' | 'failureTargetId';
+
+const FLOW_TARGET_KEYS: FlowTargetKey[] = ['yesTargetId', 'noTargetId', 'successTargetId', 'failureTargetId'];
+
+function sameBranch(step: AppiumRecordedStep, removed: AppiumRecordedStep) {
+  return Boolean(
+    removed.flow?.parentConditionId
+    && removed.flow?.parentBranch
+    && step.flow?.parentConditionId === removed.flow.parentConditionId
+    && step.flow?.parentBranch === removed.flow.parentBranch,
+  );
+}
+
+function nextSequentialStep(steps: AppiumRecordedStep[], removed: AppiumRecordedStep, index: number) {
+  if (removed.flow?.parentConditionId && removed.flow.parentBranch) {
+    return steps.slice(index + 1).find((step) => sameBranch(step, removed));
+  }
+  return steps.slice(index + 1).find((step) => !step.flow?.parentConditionId);
+}
+
+function cleanFlow(flow: NonNullable<AppiumRecordedStep['flow']>) {
+  return Object.keys(flow).length ? flow : undefined;
+}
+
+export function removeFlowStep(steps: AppiumRecordedStep[], index: number) {
+  const removed = steps[index];
+  if (!removed) return steps;
+
+  const sequentialStep = nextSequentialStep(steps, removed, index);
+  const replacementId = removed.flow?.successTargetId || sequentialStep?.id || '';
+  const removedParentConditionId = removed.flow?.parentConditionId;
+  const removedParentBranch = removed.flow?.parentBranch;
+
+  return steps
+    .filter((_step, stepIndex) => stepIndex !== index)
+    .map((step) => {
+      if (!step.flow) return step;
+
+      const flow = { ...step.flow };
+      let changed = false;
+
+      FLOW_TARGET_KEYS.forEach((key) => {
+        if (flow[key] !== removed.id) return;
+        if (replacementId) {
+          flow[key] = replacementId;
+        } else {
+          delete flow[key];
+        }
+        changed = true;
+      });
+
+      if (flow.parentConditionId === removed.id) {
+        if (removedParentConditionId && removedParentBranch) {
+          flow.parentConditionId = removedParentConditionId;
+          flow.parentBranch = removedParentBranch;
+        } else {
+          delete flow.parentConditionId;
+          delete flow.parentBranch;
+        }
+        changed = true;
+      }
+
+      return changed ? { ...step, flow: cleanFlow(flow) } : step;
+    });
+}

package/src/appium-recorder/types.ts

@@ -24,6 +24,27 @@ export type AppiumPageSnapshot = {
   selectedNodePath?: string;
 };
 
+export type AppiumVisualChangeRegion = {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+};
+
+export type AppiumVisualChangeConfig = {
+  mode: 'selectedElement' | 'region';
+  region: AppiumVisualChangeRegion;
+  role?: 'start' | 'end';
+  pairId?: string;
+  pairLabel?: string;
+  startStepId?: string;
+  endStepId?: string;
+  durationMs: number;
+  intervalMs: number;
+  changeRatioThreshold: number;
+  pixelmatchThreshold: number;
+};
+
 export type AppiumNode = {
   id: string;
   label: string;
@@ -65,7 +86,9 @@ export type AppiumRecordedStep = {
     | 'assertText'
     | 'longPress'
     | 'pinch'
-    | 'runScript';
+    | 'runScript'
+    | 'noop'
+    | 'visualChange';
   label: string;
   note?: string;
   selector?: AppiumSelector;
@@ -102,6 +125,7 @@ export type AppiumRecordedStep = {
     centerY: number;
     percent: number;
   };
+  visualChange?: AppiumVisualChangeConfig;
   snapshot?: {
     text: string;
     resourceId: string;

package/src/appium-recorder/visual-change-pairs.ts

@@ -0,0 +1,147 @@
+import type { AppiumRecordedStep } from './types';
+
+export type VisualChangeBranchTarget = {
+  stepId: string;
+  branch: 'yes' | 'no';
+};
+
+const FLOW_TARGET_KEYS = ['successTargetId', 'failureTargetId'] as const;
+
+export function visualChangePairNumber(label?: string) {
+  const match = /^检测画面变化(\d+)$/.exec(label || '');
+  return match ? Number(match[1]) : 0;
+}
+
+export function nextVisualChangePairLabel(steps: AppiumRecordedStep[]) {
+  const maxNumber = steps.reduce((max, step) => Math.max(
+    max,
+    visualChangePairNumber(step.visualChange?.pairLabel),
+  ), 0);
+  return `检测画面变化${maxNumber + 1}`;
+}
+
+function rawBranchPathForStep(steps: AppiumRecordedStep[], step?: AppiumRecordedStep) {
+  const path: string[] = [];
+  let current = step;
+  let guard = 0;
+  while (current?.flow?.parentConditionId && current.flow.parentBranch && guard < steps.length) {
+    path.unshift(`${current.flow.parentConditionId}:${current.flow.parentBranch}`);
+    current = steps.find((item) => item.id === current?.flow?.parentConditionId);
+    guard += 1;
+  }
+  return path;
+}
+
+function uniqueBranchPaths(paths: string[][]) {
+  const seen = new Set<string>();
+  return paths.filter((path) => {
+    const key = path.join('|');
+    if (!path.length || seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+function branchTargetPath(
+  steps: AppiumRecordedStep[],
+  condition: AppiumRecordedStep,
+  branch: 'yes' | 'no',
+  seen: Set<string>,
+) {
+  return [...branchPathForStep(steps, condition, new Set(seen)), `${condition.id}:${branch}`];
+}
+
+export function branchPathForStep(
+  steps: AppiumRecordedStep[],
+  step?: AppiumRecordedStep,
+  seen = new Set<string>(),
+) {
+  const rawPath = rawBranchPathForStep(steps, step);
+  if (rawPath.length || !step || seen.has(step.id)) return rawPath;
+  seen.add(step.id);
+
+  const incomingPaths = uniqueBranchPaths(steps.flatMap((source) => {
+    if (!source.flow || source.id === step.id) return [];
+    const paths: string[][] = [];
+    if (source.flow.nodeKind === 'condition') {
+      if (source.flow.yesTargetId === step.id) paths.push(branchTargetPath(steps, source, 'yes', seen));
+      if (source.flow.noTargetId === step.id) paths.push(branchTargetPath(steps, source, 'no', seen));
+    }
+    FLOW_TARGET_KEYS.forEach((key) => {
+      if (source.flow?.[key] === step.id) paths.push(branchPathForStep(steps, source, new Set(seen)));
+    });
+    return paths;
+  }));
+
+  return incomingPaths.length === 1 ? incomingPaths[0] : [];
+}
+
+// Branch paths are stored as an ancestry chain like
+// ["outer:yes", "inner:no"]. A root node before a condition has an empty
+// path, so it can be closed by an end marker in either downstream branch.
+export function branchPathForInsert(
+  steps: AppiumRecordedStep[],
+  index?: number,
+  branchTarget?: VisualChangeBranchTarget,
+) {
+  if (branchTarget) {
+    const condition = steps.find((step) => step.id === branchTarget.stepId);
+    return [...branchPathForStep(steps, condition), `${branchTarget.stepId}:${branchTarget.branch}`];
+  }
+  return typeof index === 'number' && index >= 0
+    ? branchPathForStep(steps, steps[index])
+    : [];
+}
+
+function sameBranchPath(left: string[], right: string[]) {
+  return left.length === right.length && left.every((value, index) => value === right[index]);
+}
+
+function branchPathContains(parent: string[], child: string[]) {
+  return parent.length <= child.length && parent.every((value, index) => value === child[index]);
+}
+
+function visualChangeRole(step: AppiumRecordedStep) {
+  return step.visualChange?.role;
+}
+
+function isEndForStart(endStep: AppiumRecordedStep, startStep: AppiumRecordedStep) {
+  const start = startStep.visualChange;
+  const end = endStep.visualChange;
+  if (!start || end?.role !== 'end') return false;
+  if (start.pairId && end.pairId === start.pairId) return true;
+  if (end.startStepId && end.startStepId === startStep.id) return true;
+  return Boolean(
+    start.pairLabel
+      && end.pairLabel === start.pairLabel
+      && (!start.pairId || !end.pairId || !end.startStepId),
+  );
+}
+
+export function findOpenVisualChangeStart(
+  steps: AppiumRecordedStep[],
+  index?: number,
+  branchTarget?: VisualChangeBranchTarget,
+) {
+  const insertPath = branchPathForInsert(steps, index, branchTarget);
+  const insertIndex = typeof index === 'number' ? index : steps.length - 1;
+  // A start marker may be closed once per concrete branch path. This allows a
+  // root start before a condition to have independent yes/no end markers, while
+  // preventing a yes-branch start from pairing with a no-branch end.
+  const hasEndInInsertBranch = (startStep: AppiumRecordedStep) => steps.some((step) => (
+    isEndForStart(step, startStep)
+    && sameBranchPath(branchPathForStep(steps, step), insertPath)
+  ));
+
+  return steps
+    .map((step, stepIndex) => ({ step, stepIndex, path: branchPathForStep(steps, step) }))
+    .filter(({ step, stepIndex, path }) => (
+      visualChangeRole(step) === 'start'
+      && stepIndex <= insertIndex
+      && branchPathContains(path, insertPath)
+      && !hasEndInInsertBranch(step)
+    ))
+    .sort((left, right) => (
+      right.path.length - left.path.length || right.stepIndex - left.stepIndex
+    ))[0]?.step;
+}

package/src/appium-recorder/visual-change.ts

@@ -0,0 +1,75 @@
+import type { AppiumBounds, AppiumRecordedStep } from './types';
+
+export const VISUAL_CHANGE_DEFAULTS = {
+  durationMs: 5000,
+  intervalMs: 1000,
+  changeRatioThreshold: 2,
+  pixelmatchThreshold: 0.1,
+} as const;
+
+export function createStepId() {
+  return `step_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+
+export function boundsToVisualRegion(bounds: AppiumBounds) {
+  return {
+    x: bounds.left,
+    y: bounds.top,
+    width: Math.max(0, bounds.right - bounds.left),
+    height: Math.max(0, bounds.bottom - bounds.top),
+  };
+}
+
+export function normalizeVisualRegion(
+  region: NonNullable<AppiumRecordedStep['visualChange']>['region'],
+) {
+  return {
+    x: Math.max(0, Math.round(Number(region?.x) || 0)),
+    y: Math.max(0, Math.round(Number(region?.y) || 0)),
+    width: Math.max(1, Math.round(Number(region?.width) || 1)),
+    height: Math.max(1, Math.round(Number(region?.height) || 1)),
+  };
+}
+
+export function normalizeVisualChangeConfig(
+  config?: AppiumRecordedStep['visualChange'],
+): NonNullable<AppiumRecordedStep['visualChange']> {
+  return {
+    mode: config?.mode === 'selectedElement' ? 'selectedElement' : 'region',
+    region: normalizeVisualRegion(config?.region || { x: 0, y: 0, width: 1, height: 1 }),
+    role: config?.role === 'start' ? 'start' : config?.role === 'end' ? 'end' : undefined,
+    pairId: config?.pairId || '',
+    pairLabel: config?.pairLabel || '',
+    startStepId: config?.startStepId || '',
+    endStepId: config?.endStepId || '',
+    durationMs: Math.max(1000, Math.round(Number(config?.durationMs) || VISUAL_CHANGE_DEFAULTS.durationMs)),
+    intervalMs: Math.max(200, Math.round(Number(config?.intervalMs) || VISUAL_CHANGE_DEFAULTS.intervalMs)),
+    changeRatioThreshold: Math.max(0.01, Number(config?.changeRatioThreshold) || VISUAL_CHANGE_DEFAULTS.changeRatioThreshold),
+    pixelmatchThreshold: Math.min(
+      1,
+      Math.max(0, Number(config?.pixelmatchThreshold) || VISUAL_CHANGE_DEFAULTS.pixelmatchThreshold),
+    ),
+  };
+}
+
+export function visualRegionToBounds(
+  id: string,
+  region: NonNullable<AppiumRecordedStep['visualChange']>['region'],
+) {
+  const normalized = normalizeVisualRegion(region);
+  return {
+    id,
+    left: normalized.x,
+    top: normalized.y,
+    right: normalized.x + normalized.width,
+    bottom: normalized.y + normalized.height,
+  };
+}
+
+export function visualChangeMeta(step: AppiumRecordedStep) {
+  const config = normalizeVisualChangeConfig(step.visualChange);
+  if (config.role === 'start') return `${config.pairLabel || '检测画面变化'} · 开始 · 阈值 ${config.changeRatioThreshold}%`;
+  if (config.role === 'end') return `${config.pairLabel || '检测画面变化'} · 结束`;
+  const range = config.startStepId && config.endStepId ? '起止节点截图对比' : '请选择起止节点';
+  return `${range} · 阈值 ${config.changeRatioThreshold}%`;
+}